diff --git a/src/calibre/devices/apple/driver.py b/src/calibre/devices/apple/driver.py index 838edb4e69..8a00957c8a 100644 --- a/src/calibre/devices/apple/driver.py +++ b/src/calibre/devices/apple/driver.py @@ -13,7 +13,8 @@ from calibre.constants import isosx, iswindows from calibre.devices.errors import OpenFeedback, UserFeedback from calibre.devices.usbms.deviceconfig import DeviceConfig from calibre.devices.interface import DevicePlugin -from calibre.ebooks.metadata import authors_to_string, MetaInformation, title_sort +from calibre.ebooks.metadata import (author_to_author_sort, authors_to_string, + MetaInformation, title_sort) from calibre.ebooks.metadata.book.base import Metadata from calibre.utils.config import config_dir, dynamic, prefs from calibre.utils.date import now, parse_date @@ -3478,6 +3479,7 @@ class Book(Metadata): ''' def __init__(self,title,author): Metadata.__init__(self, title, authors=author.split(' & ')) + self.author_sort = author_to_author_sort(author) @property def title_sorter(self): diff --git a/src/calibre/gui2/actions/catalog.py b/src/calibre/gui2/actions/catalog.py index 523184db2c..6c8568e83a 100644 --- a/src/calibre/gui2/actions/catalog.py +++ b/src/calibre/gui2/actions/catalog.py @@ -67,7 +67,7 @@ class GenerateCatalogAction(InterfaceAction): # jobs.results is a list - the first entry is the intended title for the dialog # Subsequent strings are error messages dialog_title = job.result.pop(0) - if re.match('warning:', job.result[0].lower()): + if re.search('warning', job.result[0].lower()): msg = _("Catalog generation complete, with warnings.") warning_dialog(self.gui, dialog_title, msg, det_msg='\n'.join(job.result), show=True) else: diff --git a/src/calibre/gui2/catalog/catalog_epub_mobi.py b/src/calibre/gui2/catalog/catalog_epub_mobi.py index f9708f292e..e0dfffbb35 100644 --- a/src/calibre/gui2/catalog/catalog_epub_mobi.py +++ b/src/calibre/gui2/catalog/catalog_epub_mobi.py @@ -6,17 +6,19 @@ __license__ = 'GPL v3' __copyright__ = '2009, Kovid Goyal ' __docformat__ = 'restructuredtext en' -from copy import copy +import re, sys + from functools import partial from calibre.ebooks.conversion.config import load_defaults -from calibre.gui2 import gprefs, question_dialog +from calibre.gui2 import gprefs, open_url, question_dialog from calibre.utils.icu import sort_key from catalog_epub_mobi_ui import Ui_Form from PyQt4.Qt import (Qt, QAbstractItemView, QCheckBox, QComboBox, QDoubleSpinBox, QIcon, QLineEdit, QObject, QRadioButton, QSize, QSizePolicy, - QTableWidget, QTableWidgetItem, QToolButton, QVBoxLayout, QWidget, + QTableWidget, QTableWidgetItem, QTextEdit, QToolButton, QUrl, + QVBoxLayout, QWidget, SIGNAL) class PluginWidget(QWidget,Ui_Form): @@ -44,6 +46,7 @@ class PluginWidget(QWidget,Ui_Form): LineEditControls = [] RadioButtonControls = [] TableWidgetControls = [] + TextEditControls = [] for item in self.__dict__: if type(self.__dict__[item]) is QCheckBox: @@ -58,6 +61,8 @@ class PluginWidget(QWidget,Ui_Form): RadioButtonControls.append(str(self.__dict__[item].objectName())) elif type(self.__dict__[item]) is QTableWidget: TableWidgetControls.append(str(self.__dict__[item].objectName())) + elif type(self.__dict__[item]) is QTextEdit: + TextEditControls.append(str(self.__dict__[item].objectName())) option_fields = zip(CheckBoxControls, [True for i in CheckBoxControls], @@ -72,20 +77,23 @@ class PluginWidget(QWidget,Ui_Form): # LineEditControls option_fields += zip(['exclude_genre'],['\[.+\]|\+'],['line_edit']) + # TextEditControls + #option_fields += zip(['exclude_genre_results'],['excluded genres will appear here'],['text_edit']) + # SpinBoxControls option_fields += zip(['thumb_width'],[1.00],['spin_box']) # Exclusion rules - option_fields += zip(['exclusion_rules_tw','exclusion_rules_tw'], + option_fields += zip(['exclusion_rules_tw'], [{'ordinal':0, 'enabled':True, 'name':'Catalogs', 'field':'Tags', 'pattern':'Catalog'},], - ['table_widget','table_widget']) + ['table_widget']) # Prefix rules - option_fields += zip(['prefix_rules_tw','prefix_rules_tw','prefix_rules_tw'], + option_fields += zip(['prefix_rules_tw','prefix_rules_tw'], [{'ordinal':0, 'enabled':True, 'name':'Read book', @@ -98,7 +106,7 @@ class PluginWidget(QWidget,Ui_Form): 'field':'Tags', 'pattern':'Wishlist', 'prefix':u'\u00d7'},], - ['table_widget','table_widget','table_widget']) + ['table_widget','table_widget']) self.OPTION_FIELDS = option_fields @@ -110,13 +118,13 @@ class PluginWidget(QWidget,Ui_Form): ''' rule_set = [] for stored_rule in opt_value: - rule = copy(stored_rule) + rule = stored_rule.copy() # Skip disabled and incomplete rules if not rule['enabled']: continue elif not rule['field'] or not rule['pattern']: continue - elif 'prefix' in rule and not rule['prefix']: + elif 'prefix' in rule and rule['prefix'] is None: continue else: if rule['field'] != 'Tags': @@ -130,12 +138,58 @@ class PluginWidget(QWidget,Ui_Form): pr = (rule['name'],rule['field'],rule['pattern'],rule['prefix']) else: pr = (rule['name'],rule['field'],rule['pattern']) + rule_set.append(pr) opt_value = tuple(rule_set) # Strip off the trailing '_tw' opts_dict[c_name[:-3]] = opt_value - def fetchEligibleCustomFields(self): + def exclude_genre_changed(self, regex): + """ Dynamically compute excluded genres. + + Run exclude_genre regex against db.all_tags() to show excluded tags. + PROVISIONAL CODE, NEEDS TESTING + + Args: + regex (QLineEdit.text()): regex to compile, compute + + Output: + self.exclude_genre_results (QLabel): updated to show tags to be excluded as genres + """ + results = _('No genres will be excluded') + if not regex: + self.exclude_genre_results.clear() + self.exclude_genre_results.setText(results) + return + + try: + pattern = re.compile((str(regex))) + except: + results = _("regex error: %s") % sys.exc_info()[1] + else: + excluded_tags = [] + for tag in self.all_tags: + hit = pattern.search(tag) + if hit: + excluded_tags.append(hit.string) + if excluded_tags: + if set(excluded_tags) == set(self.all_tags): + results = _("All genres will be excluded") + else: + results = ', '.join(sorted(excluded_tags)) + finally: + if self.DEBUG: + print(results) + self.exclude_genre_results.clear() + self.exclude_genre_results.setText(results) + + def exclude_genre_reset(self): + for default in self.OPTION_FIELDS: + if default[0] == 'exclude_genre': + self.exclude_genre.setText(default[1]) + break + + def fetch_eligible_custom_fields(self): self.all_custom_fields = self.db.custom_field_keys() custom_fields = {} custom_fields['Tags'] = {'field':'tag', 'datatype':u'text'} @@ -146,28 +200,42 @@ class PluginWidget(QWidget,Ui_Form): 'datatype':field_md['datatype']} self.eligible_custom_fields = custom_fields + def generate_descriptions_changed(self, enabled): + ''' + Toggle Description-related controls + ''' + self.header_note_source_field.setEnabled(enabled) + self.thumb_width.setEnabled(enabled) + self.merge_source_field.setEnabled(enabled) + self.merge_before.setEnabled(enabled) + self.merge_after.setEnabled(enabled) + self.include_hr.setEnabled(enabled) + def initialize(self, name, db): ''' CheckBoxControls (c_type: check_box): ['generate_titles','generate_series','generate_genres', - 'generate_recently_added','generate_descriptions','include_hr'] + 'generate_recently_added','generate_descriptions','include_hr'] ComboBoxControls (c_type: combo_box): ['exclude_source_field','header_note_source_field', - 'merge_source_field'] + 'merge_source_field'] LineEditControls (c_type: line_edit): ['exclude_genre'] RadioButtonControls (c_type: radio_button): - ['merge_before','merge_after'] + ['merge_before','merge_after','generate_new_cover', 'use_existing_cover'] SpinBoxControls (c_type: spin_box): ['thumb_width'] TableWidgetControls (c_type: table_widget): ['exclusion_rules_tw','prefix_rules_tw'] + TextEditControls (c_type: text_edit): + ['exclude_genre_results'] ''' self.name = name self.db = db - self.fetchEligibleCustomFields() + self.all_tags = db.all_tags() + self.fetch_eligible_custom_fields() self.populate_combo_boxes() # Update dialog fields from stored options @@ -200,9 +268,16 @@ class PluginWidget(QWidget,Ui_Form): if opt_value not in prefix_rules: prefix_rules.append(opt_value) - # Add icon to the reset button + # Add icon to the reset button, hook textChanged signal self.reset_exclude_genres_tb.setIcon(QIcon(I('trash.png'))) - self.reset_exclude_genres_tb.clicked.connect(self.reset_exclude_genres) + self.reset_exclude_genres_tb.clicked.connect(self.exclude_genre_reset) + + # Hook textChanged event for exclude_genre QLineEdit + self.exclude_genre.textChanged.connect(self.exclude_genre_changed) + + # Hook Descriptions checkbox for related options, init + self.generate_descriptions.clicked.connect(self.generate_descriptions_changed) + self.generate_descriptions_changed(self.generate_descriptions.isChecked()) # Init self.merge_source_field_name self.merge_source_field_name = '' @@ -226,6 +301,9 @@ class PluginWidget(QWidget,Ui_Form): self.prefix_rules_table = PrefixRules(self.prefix_rules_gb, "prefix_rules_tw",prefix_rules, self.eligible_custom_fields,self.db) + # Initialize excluded genres preview + self.exclude_genre_changed(unicode(getattr(self, 'exclude_genre').text()).strip()) + def options(self): # Save/return the current options # exclude_genre stores literally @@ -275,16 +353,21 @@ class PluginWidget(QWidget,Ui_Form): elif self.merge_after.isChecked(): checked = 'after' include_hr = self.include_hr.isChecked() - opts_dict['merge_comments'] = "%s:%s:%s" % \ + opts_dict['merge_comments_rule'] = "%s:%s:%s" % \ (self.merge_source_field_name, checked, include_hr) opts_dict['header_note_source_field'] = self.header_note_source_field_name + # Fix up exclude_genre regex if blank. Assume blank = no exclusions + if opts_dict['exclude_genre'] == '': + opts_dict['exclude_genre'] = 'a^' + # Append the output profile try: opts_dict['output_profile'] = [load_defaults('page_setup')['output_profile']] except: opts_dict['output_profile'] = ['default'] + if self.DEBUG: print "opts_dict" for opt in sorted(opts_dict.keys(), key=sort_key): @@ -377,11 +460,11 @@ class PluginWidget(QWidget,Ui_Form): self.merge_after.setEnabled(False) self.include_hr.setEnabled(False) - def reset_exclude_genres(self): - for default in self.OPTION_FIELDS: - if default[0] == 'exclude_genre': - self.exclude_genre.setText(default[1]) - break + def show_help(self): + ''' + Display help file + ''' + open_url(QUrl('http://manual.calibre-ebook.com/catalogs.html')) class CheckableTableWidgetItem(QTableWidgetItem): ''' diff --git a/src/calibre/gui2/catalog/catalog_epub_mobi.ui b/src/calibre/gui2/catalog/catalog_epub_mobi.ui index 5891d7a0cc..5e3a195190 100644 --- a/src/calibre/gui2/catalog/catalog_epub_mobi.ui +++ b/src/calibre/gui2/catalog/catalog_epub_mobi.ui @@ -6,7 +6,7 @@ 0 0 - 650 + 658 603 @@ -41,151 +41,74 @@ Included sections - - - - Books by &Genre - - - - - - - Recently &Added - - - - - - - &Descriptions - - - - - - - Books by &Series - - - - - - - Books by &Title - - - true - Books by Author + &Authors false + + + + &Titles + + + + + + + &Series + + + + + + + &Genres + + + + + + + &Recently Added + + + + + + + &Descriptions + + + - + 0 0 - - - 0 - 0 - - - A regular expression describing genres to be excluded from the generated catalog. Genres are derived from the tags applied to your books. -The default pattern \[.+\]|\+ excludes tags of the form [tag], e.g., [Test book], and '+', the default tag for a read book. + The first matching prefix rule applies a prefix to book listings in the generated catalog. - Excluded genres + Prefixes - - - QFormLayout::FieldsStayAtSizeHint - - - - - -1 - - - 0 - - - - - - 175 - 0 - - - - - 200 - 16777215 - - - - Tags to &exclude - - - Qt::AutoText - - - Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter - - - true - - - exclude_genre - - - - - - - - 0 - 0 - - - - - 0 - 0 - - - - - - - - - - - Reset to default - - - ... - - - - + + + @@ -218,22 +141,148 @@ The default pattern \[.+\]|\+ excludes tags of the form [tag], e.g., [Test book] - + 0 0 + + + 0 + 0 + + - The first matching prefix rule applies a prefix to book listings in the generated catalog. + A regular expression describing genres to be excluded from the generated catalog. Genres are derived from the tags applied to your books. +The default pattern \[.+\]|\+ excludes tags of the form [tag], e.g., [Test book], and '+', the default tag for a read book. - Prefixes + Excluded genres - - - + + + + + + 175 + 0 + + + + + 200 + 16777215 + + + + Tags to &exclude (regex): + + + Qt::AutoText + + + Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter + + + true + + + exclude_genre + + + + + + + + 0 + 0 + + + + + 0 + 0 + + + + + + + + + + + Reset to default + + + ... + + + + + + + + 175 + 0 + + + + + 200 + 16777215 + + + + Results of regex: + + + Qt::AutoText + + + Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter + + + true + + + exclude_genre + + + + + + + + 0 + 0 + + + + + 0 + 0 + + + + Tags that will be excluded as genres + + + QFrame::StyledPanel + + + + + + Qt::PlainText + + + true + + @@ -255,142 +304,9 @@ The default pattern \[.+\]|\+ excludes tags of the form [tag], e.g., [Test book] Other options - - - QFormLayout::FieldsStayAtSizeHint - - - - - - - - 175 - 0 - - - - - 200 - 16777215 - - - - &Thumb width - - - Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter - - - merge_source_field - - - - - - - - 0 - 0 - - - - - 137 - 16777215 - - - - Size hint for cover thumbnails included in Descriptions section. - - - inch - - - 2 - - - 1.000000000000000 - - - 2.000000000000000 - - - 0.100000000000000 - - - - - - - Qt::Vertical - - - - - - - &Extra note - - - header_note_source_field - - - - - - - - 0 - 0 - - - - - 0 - 0 - - - - - 16777215 - 16777215 - - - - Custom column source for text to include in Description section. - - - - - - - - - - - - 175 - 0 - - - - - 200 - 16777215 - - - - &Merge with Comments - - - Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter - - - merge_source_field - - - + + + @@ -419,6 +335,9 @@ The default pattern \[.+\]|\+ excludes tags of the form [tag], e.g., [Test book] &Before + + merge_options_bg + @@ -429,6 +348,9 @@ The default pattern \[.+\]|\+ excludes tags of the form [tag], e.g., [Test book] &After + + merge_options_bg + @@ -444,7 +366,196 @@ The default pattern \[.+\]|\+ excludes tags of the form [tag], e.g., [Test book] Separate Comments metadata and additional content with a horizontal rule. - &Separator + Include &Separator + + + + + + + + + + 175 + 0 + + + + + 200 + 16777215 + + + + &Merge with Comments: + + + Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter + + + merge_source_field + + + + + + + + 175 + 20 + + + + Catalog cover: + + + Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter + + + + + + + + + Generate new cover + + + true + + + cover_options_bg + + + + + + + Use existing cover + + + cover_options_bg + + + + + + + + + + + + + + + + E&xtra Description note: + + + Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter + + + header_note_source_field + + + + + + + + + + 0 + 0 + + + + + 0 + 0 + + + + + 16777215 + 16777215 + + + + Custom column source for text to include in Description section. + + + + + + + Qt::Vertical + + + + + + + + 0 + 0 + + + + + 0 + 0 + + + + + 16777215 + 16777215 + + + + &Thumb width: + + + Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter + + + merge_source_field + + + + + + + + 0 + 0 + + + + + 16777215 + 16777215 + + + + Size hint for cover thumbnails included in Descriptions section. + + + inch + + + 2 + + + 1.000000000000000 + + + 2.000000000000000 + + + 0.100000000000000 @@ -457,4 +568,8 @@ The default pattern \[.+\]|\+ excludes tags of the form [tag], e.g., [Test book] + + + + diff --git a/src/calibre/gui2/dialogs/catalog.py b/src/calibre/gui2/dialogs/catalog.py index a8f7ed160f..6ec5dd6d13 100644 --- a/src/calibre/gui2/dialogs/catalog.py +++ b/src/calibre/gui2/dialogs/catalog.py @@ -10,7 +10,7 @@ import os, sys, importlib from calibre.customize.ui import config from calibre.gui2.dialogs.catalog_ui import Ui_Dialog -from calibre.gui2 import dynamic, ResizableDialog +from calibre.gui2 import dynamic, ResizableDialog, info_dialog from calibre.customize.ui import catalog_plugins class Catalog(ResizableDialog, Ui_Dialog): @@ -22,7 +22,6 @@ class Catalog(ResizableDialog, Ui_Dialog): from PyQt4.uic import compileUi ResizableDialog.__init__(self, parent) - self.dbspec, self.ids = dbspec, ids # Display the number of books we've been passed @@ -115,6 +114,7 @@ class Catalog(ResizableDialog, Ui_Dialog): self.format.currentIndexChanged.connect(self.show_plugin_tab) self.buttonBox.button(self.buttonBox.Apply).clicked.connect(self.apply) + self.buttonBox.button(self.buttonBox.Help).clicked.connect(self.help) self.show_plugin_tab(None) geom = dynamic.get('catalog_window_geom', None) @@ -129,6 +129,10 @@ class Catalog(ResizableDialog, Ui_Dialog): if cf in pw.formats: self.tabs.addTab(pw, pw.TITLE) break + if hasattr(self.tabs.widget(1),'show_help'): + self.buttonBox.button(self.buttonBox.Help).setVisible(True) + else: + self.buttonBox.button(self.buttonBox.Help).setVisible(False) def format_changed(self, idx): cf = unicode(self.format.currentText()) @@ -165,6 +169,29 @@ class Catalog(ResizableDialog, Ui_Dialog): self.save_catalog_settings() return ResizableDialog.accept(self) + def help(self): + ''' + To add help functionality for a specific format: + In gui2.catalog.catalog_.py, add the following: + from calibre.gui2 import open_url + from PyQt4.Qt import QUrl + + In the PluginWidget() class, add this method: + def show_help(self): + url = 'file:///' + P('catalog/help_.html') + open_url(QUrl(url)) + + Create the help file at resources/catalog/help_.html + ''' + if self.tabs.count() > 1 and hasattr(self.tabs.widget(1),'show_help'): + try: + self.tabs.widget(1).show_help() + except: + info_dialog(self, _('No help available'), + _('No help available for this output format.'), + show_copy_button=False, + show=True) + def reject(self): dynamic.set('catalog_window_geom', bytearray(self.saveGeometry())) ResizableDialog.reject(self) diff --git a/src/calibre/gui2/dialogs/catalog.ui b/src/calibre/gui2/dialogs/catalog.ui index cf51ac8848..a4366b26c2 100644 --- a/src/calibre/gui2/dialogs/catalog.ui +++ b/src/calibre/gui2/dialogs/catalog.ui @@ -14,7 +14,7 @@ Generate catalog - + :/images/lt.png:/images/lt.png @@ -37,7 +37,7 @@ Qt::Horizontal - QDialogButtonBox::Apply|QDialogButtonBox::Cancel|QDialogButtonBox::Ok + QDialogButtonBox::Apply|QDialogButtonBox::Cancel|QDialogButtonBox::Help|QDialogButtonBox::Ok @@ -54,8 +54,8 @@ 0 0 - 666 - 599 + 650 + 575 diff --git a/src/calibre/library/catalogs/__init__.py b/src/calibre/library/catalogs/__init__.py index 0fcf8266da..a5c98955e8 100644 --- a/src/calibre/library/catalogs/__init__.py +++ b/src/calibre/library/catalogs/__init__.py @@ -17,4 +17,6 @@ FIELDS = ['all', 'title', 'title_sort', 'author_sort', 'authors', 'comments', TEMPLATE_ALLOWED_FIELDS = [ 'author_sort', 'authors', 'id', 'isbn', 'pubdate', 'title_sort', 'publisher', 'series_index', 'series', 'tags', 'timestamp', 'title', 'uuid' ] +class AuthorSortMismatchException(Exception): pass +class EmptyCatalogException(Exception): pass diff --git a/src/calibre/library/catalogs/epub_mobi.py b/src/calibre/library/catalogs/epub_mobi.py index 25385e556c..c92d178f6f 100644 --- a/src/calibre/library/catalogs/epub_mobi.py +++ b/src/calibre/library/catalogs/epub_mobi.py @@ -7,13 +7,14 @@ __license__ = 'GPL v3' __copyright__ = '2012, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import os +import os, shutil, sys, time from collections import namedtuple from calibre import strftime from calibre.customize import CatalogPlugin from calibre.customize.conversion import OptionRecommendation, DummyReporter from calibre.ebooks import calibre_cover +from calibre.library.catalogs import AuthorSortMismatchException, EmptyCatalogException from calibre.ptempfile import PersistentTemporaryFile Option = namedtuple('Option', 'option, default, dest, action, help') @@ -120,9 +121,9 @@ class EPUB_MOBI(CatalogPlugin): help=_("Custom field containing note text to insert in Description header.\n" "Default: '%default'\n" "Applies to: AZW3, ePub, MOBI output formats")), - Option('--merge-comments', + Option('--merge-comments-rule', default='::', - dest='merge_comments', + dest='merge_comments_rule', action = None, help=_("#:[before|after]:[True|False] specifying:\n" " Custom field containing notes to merge with Comments\n" @@ -146,6 +147,13 @@ class EPUB_MOBI(CatalogPlugin): "When multiple rules are defined, the first matching rule will be used.\n" "Default:\n" + '"' + '%default' + '"' + "\n" "Applies to AZW3, ePub, MOBI output formats")), + Option('--use-existing-cover', + default=False, + dest='use_existing_cover', + action = 'store_true', + help=_("Replace existing cover when generating the catalog.\n" + "Default: '%default'\n" + "Applies to: AZW3, ePub, MOBI output formats")), Option('--thumb-width', default='1.0', dest='thumb_width', @@ -182,8 +190,8 @@ class EPUB_MOBI(CatalogPlugin): else: op = "kindle" - opts.descriptionClip = 380 if op.endswith('dx') or 'kindle' not in op else 100 - opts.authorClip = 100 if op.endswith('dx') or 'kindle' not in op else 60 + opts.description_clip = 380 if op.endswith('dx') or 'kindle' not in op else 100 + opts.author_clip = 100 if op.endswith('dx') or 'kindle' not in op else 60 opts.output_profile = op opts.basename = "Catalog" @@ -198,11 +206,12 @@ class EPUB_MOBI(CatalogPlugin): (self.name,self.fmt,'for %s ' % opts.output_profile if opts.output_profile else '', 'CLI' if opts.cli_environment else 'GUI')) - # If exclude_genre is blank, assume user wants all genre tags included + # If exclude_genre is blank, assume user wants all tags as genres if opts.exclude_genre.strip() == '': - opts.exclude_genre = '\[^.\]' - build_log.append(" converting empty exclude_genre to '\[^.\]'") - + #opts.exclude_genre = '\[^.\]' + #build_log.append(" converting empty exclude_genre to '\[^.\]'") + opts.exclude_genre = 'a^' + build_log.append(" converting empty exclude_genre to 'a^'") if opts.connected_device['is_device_connected'] and \ opts.connected_device['kind'] == 'device': if opts.connected_device['serial']: @@ -304,13 +313,13 @@ class EPUB_MOBI(CatalogPlugin): keys.sort() build_log.append(" opts:") for key in keys: - if key in ['catalog_title','authorClip','connected_kindle','descriptionClip', + if key in ['catalog_title','author_clip','connected_kindle','description_clip', 'exclude_book_marker','exclude_genre','exclude_tags', - 'exclusion_rules', - 'header_note_source_field','merge_comments', + 'exclusion_rules', 'fmt', + 'header_note_source_field','merge_comments_rule', 'output_profile','prefix_rules','read_book_marker', 'search_text','sort_by','sort_descriptions_by_author','sync', - 'thumb_width','wishlist_tag']: + 'thumb_width','use_existing_cover','wishlist_tag']: build_log.append(" %s: %s" % (key, repr(opts_dict[key]))) if opts.verbose: @@ -323,26 +332,30 @@ class EPUB_MOBI(CatalogPlugin): if opts.verbose: log.info(" Begin catalog source generation") - catalog.createDirectoryStructure() - catalog.copyResources() - catalog.calculateThumbnailSize() - catalog_source_built = catalog.buildSources() - if opts.verbose: - if catalog_source_built: + try: + catalog_source_built = catalog.build_sources() + if opts.verbose: log.info(" Completed catalog source generation\n") - else: - log.error(" *** Terminated catalog generation, check log for details ***") + except (AuthorSortMismatchException, EmptyCatalogException), e: + log.error(" *** Terminated catalog generation: %s ***" % e) + except: + log.error(" unhandled exception in catalog generator") + raise - if catalog_source_built: + else: recommendations = [] recommendations.append(('remove_fake_margins', False, OptionRecommendation.HIGH)) recommendations.append(('comments', '', OptionRecommendation.HIGH)) - # >>> Use to debug generated catalog code before conversion <<< - if False: - setattr(opts,'debug_pipeline',os.path.expanduser("~/Desktop/Catalog debug")) + """ + >>> Use to debug generated catalog code before pipeline conversion <<< + """ + GENERATE_DEBUG_EPUB = False + if GENERATE_DEBUG_EPUB: + catalog_debug_path = os.path.join(os.path.expanduser('~'),'Desktop','Catalog debug') + setattr(opts,'debug_pipeline',os.path.expanduser(catalog_debug_path)) dp = getattr(opts, 'debug_pipeline', None) if dp is not None: @@ -357,9 +370,9 @@ class EPUB_MOBI(CatalogPlugin): recommendations.append(('book_producer',opts.output_profile, OptionRecommendation.HIGH)) - # If cover exists, use it + # Use existing cover or generate new cover cpath = None - generate_new_cover = False + existing_cover = False try: search_text = 'title:"%s" author:%s' % ( opts.catalog_title.replace('"', '\\"'), 'calibre') @@ -367,19 +380,18 @@ class EPUB_MOBI(CatalogPlugin): if matches: cpath = db.cover(matches[0], index_is_id=True, as_path=True) if cpath and os.path.exists(cpath): - recommendations.append(('cover', cpath, - OptionRecommendation.HIGH)) - log.info("using existing cover") - else: - log.info("no existing cover, generating new cover") - generate_new_cover = True - else: - log.info("no existing cover, generating new cover") - generate_new_cover = True + existing_cover = True except: pass - if generate_new_cover: + if self.opts.use_existing_cover and not existing_cover: + log.warning("no existing catalog cover found") + + if self.opts.use_existing_cover and existing_cover: + recommendations.append(('cover', cpath, OptionRecommendation.HIGH)) + log.info("using existing catalog cover") + else: + log.info("replacing catalog cover") new_cover_path = PersistentTemporaryFile(suffix='.jpg') new_cover = calibre_cover(opts.catalog_title.replace('"', '\\"'), 'calibre') new_cover_path.write(new_cover) @@ -388,7 +400,7 @@ class EPUB_MOBI(CatalogPlugin): # Run ebook-convert from calibre.ebooks.conversion.plumber import Plumber - plumber = Plumber(os.path.join(catalog.catalogPath, + plumber = Plumber(os.path.join(catalog.catalog_path, opts.basename + '.opf'), path_to_output, log, report_progress=notification, abort_after_input_dump=False) plumber.merge_ui_recommendations(recommendations) @@ -399,6 +411,13 @@ class EPUB_MOBI(CatalogPlugin): except: pass + if GENERATE_DEBUG_EPUB: + from calibre.ebooks.tweak import zip_rebuilder + input_path = os.path.join(catalog_debug_path,'input') + shutil.copy(P('catalog/mimetype'),input_path) + shutil.copytree(P('catalog/META-INF'),os.path.join(input_path,'META-INF')) + zip_rebuilder(input_path, os.path.join(catalog_debug_path,'input.epub')) + # returns to gui2.actions.catalog:catalog_generated() return catalog.error diff --git a/src/calibre/library/catalogs/epub_mobi_builder.py b/src/calibre/library/catalogs/epub_mobi_builder.py index 1860dcfe47..1ed85811b9 100644 --- a/src/calibre/library/catalogs/epub_mobi_builder.py +++ b/src/calibre/library/catalogs/epub_mobi_builder.py @@ -9,8 +9,10 @@ from xml.sax.saxutils import escape from calibre import (prepare_string_for_xml, strftime, force_unicode) from calibre.customize.conversion import DummyReporter +from calibre.customize.ui import output_profiles from calibre.ebooks.BeautifulSoup import BeautifulSoup, BeautifulStoneSoup, Tag, NavigableString from calibre.ebooks.chardet import substitute_entites +from calibre.library.catalogs import AuthorSortMismatchException, EmptyCatalogException from calibre.ptempfile import PersistentTemporaryDirectory from calibre.utils.config import config_dir from calibre.utils.date import format_date, is_date_undefined, now as nowf @@ -19,7 +21,6 @@ from calibre.utils.icu import capitalize, collation_order, sort_key from calibre.utils.magick.draw import thumbnail from calibre.utils.zipfile import ZipFile - class CatalogBuilder(object): ''' Generates catalog source files from calibre database @@ -29,15 +30,15 @@ class CatalogBuilder(object): gui2.tools:generate_catalog() or library.cli:command_catalog() called from gui2.convert.gui_conversion:gui_catalog() catalog = Catalog(notification=Reporter()) - catalog.createDirectoryStructure() - catalog.copyResources() - catalog.buildSources() + catalog.build_sources() Options managed in gui2.catalog.catalog_epub_mobi.py ''' + DEBUG = False + # A single number creates 'Last x days' only. # Multiple numbers create 'Last x days', 'x to y days ago' ... - # e.g, [7,15,30,60], [30] + # e.g, [7,15,30,60] or [30] # [] = No date ranges added DATE_RANGE=[30] @@ -46,107 +47,555 @@ class CatalogBuilder(object): # basename output file basename # creator dc:creator in OPF metadata - # descriptionClip limits size of NCX descriptions (Kindle only) - # includeSources Used in processSpecialTags to skip tags like '[SPL]' + # description_clip limits size of NCX descriptions (Kindle only) + # includeSources Used in filter_excluded_genres to skip tags like '[SPL]' # notification Used to check for cancel, report progress # stylesheet CSS stylesheet # title dc:title in OPF metadata, NCX periodical # verbosity level of diagnostic printout - def __init__(self, db, opts, plugin, + """ property decorators for attributes """ + if True: + ''' list of unique authors ''' + @property + def authors(self): + return self.__authors + @authors.setter + def authors(self, val): + self.__authors = val + + ''' dict of bookmarked books ''' + @property + def bookmarked_books(self): + return self.__bookmarked_books + @bookmarked_books.setter + def bookmarked_books(self, val): + self.__bookmarked_books = val + + ''' list of bookmarked books, sorted by date read ''' + @property + def bookmarked_books_by_date_read(self): + return self.__bookmarked_books_by_date_read + @bookmarked_books_by_date_read.setter + def bookmarked_books_by_date_read(self, val): + self.__bookmarked_books_by_date_read = val + + ''' list of books, sorted by author ''' + @property + def books_by_author(self): + return self.__books_by_author + @books_by_author.setter + def books_by_author(self, val): + self.__books_by_author = val + + ''' list of books, grouped by date range (30 days) ''' + @property + def books_by_date_range(self): + return self.__books_by_date_range + @books_by_date_range.setter + def books_by_date_range(self, val): + self.__books_by_date_range = val + + ''' list of books, by date added reverse (most recent first) ''' + @property + def books_by_month(self): + return self.__books_by_month + @books_by_month.setter + def books_by_month(self, val): + self.__books_by_month = val + + ''' list of books in series ''' + @property + def books_by_series(self): + return self.__books_by_series + @books_by_series.setter + def books_by_series(self, val): + self.__books_by_series = val + + ''' list of books, sorted by title ''' + @property + def books_by_title(self): + return self.__books_by_title + @books_by_title.setter + def books_by_title(self, val): + self.__books_by_title = val + + ''' list of books in series, without series prefix ''' + @property + def books_by_title_no_series_prefix(self): + return self.__books_by_title_no_series_prefix + @books_by_title_no_series_prefix.setter + def books_by_title_no_series_prefix(self, val): + self.__books_by_title_no_series_prefix = val + + ''' directory to store cached thumbs ''' + @property + def cache_dir(self): + return self.__cache_dir + + ''' temp dir to store generated catalog ''' + @property + def catalog_path(self): + return self.__catalog_path + + ''' content dir in generated catalog ''' + @property + def content_dir(self): + return self.__content_dir + + ''' track Job progress ''' + @property + def current_step(self): + return self.__current_step + @current_step.setter + def current_step(self, val): + self.__current_step = val + + ''' active database ''' + @property + def db(self): + return self.__db + + ''' cumulative error messages to report at conclusion ''' + @property + def error(self): + return self.__error + @error.setter + def error(self, val): + self.__error = val + + ''' tags to exclude as genres ''' + @property + def excluded_tags(self): + return self.__excluded_tags + + ''' True if generating for Kindle in MOBI format ''' + @property + def generate_for_kindle(self): + return self.__generate_for_kindle + + ''' True if connected Kindle and generating for Kindle ''' + @property + def generate_recently_read(self): + return self.__generate_recently_read + + ''' list of dicts with books by genre ''' + @property + def genres(self): + return self.__genres + @genres.setter + def genres(self, val): + self.__genres = val + + ''' dict of enabled genre tags ''' + @property + def genre_tags_dict(self): + return self.__genre_tags_dict + @genre_tags_dict.setter + def genre_tags_dict(self, val): + self.__genre_tags_dict = val + + ''' Author, Title, Series sections ''' + @property + def html_filelist_1(self): + return self.__html_filelist_1 + @html_filelist_1.setter + def html_filelist_1(self, val): + self.__html_filelist_1 = val + + ''' Date Added, Date Read ''' + @property + def html_filelist_2(self): + return self.__html_filelist_2 + @html_filelist_2.setter + def html_filelist_2(self, val): + self.__html_filelist_2 = val + + ''' additional field to include before/after comments ''' + @property + def merge_comments_rule(self): + return self.__merge_comments_rule + + ''' cumulative HTML for NCX file ''' + @property + def ncx_soup(self): + return self.__ncx_soup + @ncx_soup.setter + def ncx_soup(self, val): + self.__ncx_soup = val + + ''' opts passed from gui2.catalog.catalog_epub_mobi.py ''' + @property + def opts(self): + return self.__opts + + ''' output_profile declares special symbols ''' + @property + def output_profile(self): + return self.__output_profile + + ''' playOrder value for building NCX ''' + @property + def play_order(self): + return self.__play_order + @play_order.setter + def play_order(self, val): + self.__play_order = val + + ''' catalog??? device??? ''' + @property + def plugin(self): + return self.__plugin + + ''' dict of prefix rules ''' + @property + def prefix_rules(self): + return self.__prefix_rules + @prefix_rules.setter + def prefix_rules(self, val): + self.__prefix_rules = val + + ''' used with ProgressReporter() ''' + @property + def progress_int(self): + return self.__progress_int + @progress_int.setter + def progress_int(self, val): + self.__progress_int = val + + ''' used with ProgressReporter() ''' + @property + def progress_string(self): + return self.__progress_string + @progress_string.setter + def progress_string(self, val): + self.__progress_string = val + + ''' Progress Reporter for Jobs ''' + @property + def reporter(self): + return self.__reporter + + ''' stylesheet to include with catalog ''' + @property + def stylesheet(self): + return self.__stylesheet + + ''' device-specific symbol (default empty star) ''' + @property + def SYMBOL_EMPTY_RATING(self): + return self.output_profile.empty_ratings_char + + ''' device-specific symbol (default filled star) ''' + @property + def SYMBOL_FULL_RATING(self): + return self.output_profile.ratings_char + + ''' device-specific symbol for reading progress ''' + @property + def SYMBOL_PROGRESS_READ(self): + psr = '+' + if self.generate_for_kindle: + psr = '▪' + return psr + + ''' device-specific symbol for reading progress ''' + @property + def SYMBOL_PROGRESS_UNREAD(self): + psu = '-' + if self.generate_for_kindle: + psu = '▫' + return psu + + ''' device-specific symbol for reading progress ''' + @property + def SYMBOL_READING(self): + if self.generate_for_kindle: + return self.format_prefix('▷') + else: + return self.format_prefix(' ') + + @property + def thumb_height(self): + return self.__thumb_height + @thumb_height.setter + def thumb_height(self, val): + self.__thumb_height = val + + @property + def thumb_width(self): + return self.__thumb_width + @thumb_width.setter + def thumb_width(self, val): + self.__thumb_width = val + + ''' list of generated thumbs ''' + @property + def thumbs(self): + return self.__thumbs + @thumbs.setter + def thumbs(self, val): + self.__thumbs = val + + ''' full path to thumbs archive ''' + @property + def thumbs_path(self): + return self.__thumbs_path + + ''' used with ProgressReporter() ''' + @property + def total_steps(self): + return self.__total_steps + @total_steps.setter + def total_steps(self, val): + self.__total_steps = val + + ''' switch controlling format of series books in Titles section ''' + @property + def use_series_prefix_in_titles_section(self): + return self.__use_series_prefix_in_titles_section + + def __init__(self, db, _opts, plugin, report_progress=DummyReporter(), - stylesheet="content/stylesheet.css"): - self.__opts = opts - self.__authorClip = opts.authorClip - self.__authors = None - self.__basename = opts.basename - self.__bookmarked_books = None - self.__booksByAuthor = None - self.__booksByDateRead = None - self.__booksByTitle = None - self.__booksByTitle_noSeriesPrefix = None - self.__cache_dir = os.path.join(config_dir, 'caches', 'catalog') - self.__archive_path = os.path.join(self.__cache_dir, "thumbs.zip") - self.__catalogPath = PersistentTemporaryDirectory("_epub_mobi_catalog", prefix='') - self.__contentDir = os.path.join(self.catalogPath, "content") - self.__currentStep = 0.0 - self.__creator = opts.creator + stylesheet="content/stylesheet.css", + init_resources=True): + self.__db = db - self.__defaultPrefix = None - self.__descriptionClip = opts.descriptionClip - self.__error = [] - self.__generateForKindle = True if (self.opts.fmt == 'mobi' and \ - self.opts.output_profile and \ - self.opts.output_profile.startswith("kindle")) else False - self.__generateRecentlyRead = True if self.opts.generate_recently_added \ - and self.opts.connected_kindle \ - and self.generateForKindle \ - else False - self.__genres = None - self.genres = [] - self.__genre_tags_dict = None - self.__htmlFileList_1 = [] - self.__htmlFileList_2 = [] - self.__markerTags = self.getMarkerTags() - self.__ncxSoup = None - self.__output_profile = None - self.__playOrder = 1 + self.__opts = _opts self.__plugin = plugin - self.__prefixRules = [] - self.__progressInt = 0.0 - self.__progressString = '' - f, p, hr = self.opts.merge_comments.split(':') - self.__merge_comments = {'field':f, 'position':p, 'hr':hr} self.__reporter = report_progress self.__stylesheet = stylesheet + self.__cache_dir = os.path.join(config_dir, 'caches', 'catalog') + self.__catalog_path = PersistentTemporaryDirectory("_epub_mobi_catalog", prefix='') + self.__generate_for_kindle = True if (_opts.fmt == 'mobi' and + _opts.output_profile and + _opts.output_profile.startswith("kindle")) else False + + self.__authors = None + self.__bookmarked_books = None + self.__bookmarked_books_by_date_read = None + self.__books_by_author = None + self.__books_by_date_range = None + self.__books_by_month = None + self.__books_by_series = None + self.__books_by_title = None + self.__books_by_title_no_series_prefix = None + self.__content_dir = os.path.join(self.catalog_path, "content") + self.__current_step = 0.0 + self.__error = [] + self.__excluded_tags = self.get_excluded_tags() + self.__generate_recently_read = True if (_opts.generate_recently_added and + _opts.connected_kindle and + self.generate_for_kindle) else False + self.__genres = [] + self.__genre_tags_dict = None + self.__html_filelist_1 = [] + self.__html_filelist_2 = [] + self.__merge_comments_rule = dict(zip(['field','position','hr'],_opts.merge_comments_rule.split(':'))) + self.__ncx_soup = None + self.__output_profile = None + self.__output_profile = self.get_output_profile(_opts) + self.__play_order = 1 + self.__prefix_rules = self.get_prefix_rules() + self.__progress_int = 0.0 + self.__progress_string = '' + self.__thumb_height = 0 + self.__thumb_width = 0 self.__thumbs = None - self.__thumbWidth = 0 - self.__thumbHeight = 0 - self.__title = opts.catalog_title - self.__totalSteps = 6.0 - self.__useSeriesPrefixInTitlesSection = False - self.__verbose = opts.verbose + self.__thumbs_path = os.path.join(self.cache_dir, "thumbs.zip") + self.__total_steps = 6.0 + self.__use_series_prefix_in_titles_section = False - from calibre.customize.ui import output_profiles - for profile in output_profiles(): - if profile.short_name == self.opts.output_profile: - self.__output_profile = profile - break + self.compute_total_steps() + self.calculate_thumbnail_dimensions() + self.confirm_thumbs_archive() + self.load_section_templates() + if init_resources: + self.copy_catalog_resources() - # Process prefix rules - self.processPrefixRules() + """ key() functions """ - # Confirm/create thumbs archive. + def _kf_author_to_author_sort(self, author): + """ Compute author_sort value from author + + Tokenize author string, return capitalized string with last token first + + Args: + author (str): author, e.g. 'John Smith' + + Return: + (str): 'Smith, john' + """ + tokens = author.split() + tokens = tokens[-1:] + tokens[:-1] + if len(tokens) > 1: + tokens[0] += ',' + return ' '.join(tokens).capitalize() + + def _kf_books_by_author_sorter_author(self, book): + """ Generate book sort key with computed author_sort. + + Generate a sort key of computed author_sort, title. Used to look for + author_sort mismatches. + Twiddle included to force series to sort after non-series books. + 'Smith, john Star Wars' + 'Smith, john ~Star Wars 0001.0000' + + Args: + book (dict): book metadata + + Return: + (str): sort key + """ + if not book['series']: + key = '%s %s' % (self._kf_author_to_author_sort(book['author']), + capitalize(book['title_sort'])) + else: + index = book['series_index'] + integer = int(index) + fraction = index-integer + series_index = '%04d%s' % (integer, str('%0.4f' % fraction).lstrip('0')) + key = '%s ~%s %s' % (self._kf_author_to_author_sort(book['author']), + self.generate_sort_title(book['series']), + series_index) + return key + + def _kf_books_by_author_sorter_author_sort(self, book, longest_author_sort=60): + """ Generate book sort key with supplied author_sort. + + Generate a sort key of author_sort, title. + Bang, tilde included to force series to sort after non-series books. + + Args: + book (dict): book metadata + + Return: + (str): sort key + """ + if not book['series']: + fs = '{:<%d}!{!s}' % longest_author_sort + key = fs.format(capitalize(book['author_sort']), + capitalize(book['title_sort'])) + else: + index = book['series_index'] + integer = int(index) + fraction = index-integer + series_index = '%04d%s' % (integer, str('%0.4f' % fraction).lstrip('0')) + fs = '{:<%d}~{!s}{!s}' % longest_author_sort + key = fs.format(capitalize(book['author_sort']), + self.generate_sort_title(book['series']), + series_index) + return key + + """ Methods """ + + def build_sources(self): + """ Generate catalog source files. + + Assemble OPF, HTML and NCX files reflecting catalog options. + Generated source is OEB compliant. + Called from gui2.convert.gui_conversion:gui_catalog() + + Args: + + Exceptions: + AuthorSortMismatchException + EmptyCatalogException + + Results: + error: problems reported during build + + """ + + self.fetch_books_by_title() + self.fetch_books_by_author() + self.fetch_bookmarks() if self.opts.generate_descriptions: - if not os.path.exists(self.__cache_dir): - self.opts.log.info(" creating new thumb cache '%s'" % self.__cache_dir) - os.makedirs(self.__cache_dir) - if not os.path.exists(self.__archive_path): - self.opts.log.info(' creating thumbnail archive, thumb_width: %1.2f"' % - float(self.opts.thumb_width)) - with ZipFile(self.__archive_path, mode='w') as zfw: - zfw.writestr("Catalog Thumbs Archive",'') - else: - try: - with ZipFile(self.__archive_path, mode='r') as zfr: - try: - cached_thumb_width = zfr.read('thumb_width') - except: - cached_thumb_width = "-1" - except: - os.remove(self.__archive_path) - cached_thumb_width = '-1' + self.generate_thumbnails() + self.generate_html_descriptions() + if self.opts.generate_authors: + self.generate_html_by_author() + if self.opts.generate_titles: + self.generate_html_by_title() + if self.opts.generate_series: + self.generate_html_by_series() + if self.opts.generate_genres: + self.generate_html_by_genres() + # If this is the only Section, and there are no genres, bail + if self.opts.section_list == ['Genres'] and not self.genres: + error_msg = _("No genres to catalog.\n") + if not self.opts.cli_environment: + error_msg += _("Check 'Excluded genres' regex in E-book options.\n") + self.opts.log.error(error_msg) + self.error.append(_('No books available to catalog')) + self.error.append(error_msg) + raise EmptyCatalogException, "No genres to catalog" + if self.opts.generate_recently_added: + self.generate_html_by_date_added() + if self.generate_recently_read: + self.generate_html_by_date_read() - if float(cached_thumb_width) != float(self.opts.thumb_width): - self.opts.log.warning(" invalidating cache at '%s'" % self.__archive_path) - self.opts.log.warning(' thumb_width changed: %1.2f" => %1.2f"' % - (float(cached_thumb_width),float(self.opts.thumb_width))) - with ZipFile(self.__archive_path, mode='w') as zfw: - zfw.writestr("Catalog Thumbs Archive",'') - else: - self.opts.log.info(' existing thumb cache at %s, cached_thumb_width: %1.2f"' % - (self.__archive_path, float(cached_thumb_width))) + self.generate_opf() + self.generate_ncx_header() + if self.opts.generate_authors: + self.generate_ncx_by_author(_("Authors")) + if self.opts.generate_titles: + self.generate_ncx_by_title(_("Titles")) + if self.opts.generate_series: + self.generate_ncx_by_series(_("Series")) + if self.opts.generate_genres: + self.generate_ncx_by_genre(_("Genres")) + if self.opts.generate_recently_added: + self.generate_ncx_by_date_added(_("Recently Added")) + if self.generate_recently_read: + self.generate_ncx_by_date_read(_("Recently Read")) + if self.opts.generate_descriptions: + self.generate_ncx_descriptions(_("Descriptions")) + self.write_ncx() + def calculate_thumbnail_dimensions(self): + """ Calculate thumb dimensions based on device DPI. + + Using the specified output profile, calculate thumb_width + in pixels, then set height to width * 1.33. Special-case for + Kindle/MOBI, as rendering off by 2. + *** dead code? *** + + Inputs: + opts.thumb_width (str|float): specified thumb_width + opts.output_profile.dpi (int): device DPI + + Outputs: + thumb_width (float): calculated thumb_width + thumb_height (float): calculated thumb_height + """ + + for x in output_profiles(): + if x.short_name == self.opts.output_profile: + # aspect ratio: 3:4 + self.thumb_width = x.dpi * float(self.opts.thumb_width) + self.thumb_height = self.thumb_width * 1.33 + if 'kindle' in x.short_name and self.opts.fmt == 'mobi': + # Kindle DPI appears to be off by a factor of 2 + self.thumb_width = self.thumb_width/2 + self.thumb_height = self.thumb_height/2 + break + if self.opts.verbose: + self.opts.log(" Thumbnails:") + self.opts.log(" DPI = %d; thumbnail dimensions: %d x %d" % \ + (x.dpi, self.thumb_width, self.thumb_height)) + + def compute_total_steps(self): + """ Calculate number of build steps to generate catalog. + + Calculate total number of build steps based on enabled sections. + + Inputs: + opts.generate_* (bool): enabled sections + + Outputs: + total_steps (int): updated + """ # Tweak build steps based on optional sections: 1 call for HTML, 1 for NCX incremental_jobs = 0 if self.opts.generate_authors: @@ -155,390 +604,110 @@ class CatalogBuilder(object): incremental_jobs += 2 if self.opts.generate_recently_added: incremental_jobs += 2 - if self.generateRecentlyRead: + if self.generate_recently_read: incremental_jobs += 2 if self.opts.generate_series: incremental_jobs += 2 if self.opts.generate_descriptions: # +1 thumbs incremental_jobs += 3 - self.__totalSteps += incremental_jobs + self.total_steps += incremental_jobs - # Load section list templates - templates = {} - execfile(P('catalog/section_list_templates.py'), templates) - for name, template in templates.iteritems(): - if name.startswith('by_') and name.endswith('_template'): - setattr(self, name, force_unicode(template, 'utf-8')) + def confirm_thumbs_archive(self): + """ Validate thumbs archive. - # Accessors - if True: - ''' - @dynamic_property - def xxxx(self): - def fget(self): - return self.__ - def fset(self, val): - self.__ = val - return property(fget=fget, fset=fset) - ''' - @dynamic_property - def authorClip(self): - def fget(self): - return self.__authorClip - def fset(self, val): - self.__authorClip = val - return property(fget=fget, fset=fset) - @dynamic_property - def authors(self): - def fget(self): - return self.__authors - def fset(self, val): - self.__authors = val - return property(fget=fget, fset=fset) - @dynamic_property - def basename(self): - def fget(self): - return self.__basename - def fset(self, val): - self.__basename = val - return property(fget=fget, fset=fset) - @dynamic_property - def bookmarked_books(self): - def fget(self): - return self.__bookmarked_books - def fset(self, val): - self.__bookmarked_books = val - return property(fget=fget, fset=fset) - @dynamic_property - def booksByAuthor(self): - def fget(self): - return self.__booksByAuthor - def fset(self, val): - self.__booksByAuthor = val - return property(fget=fget, fset=fset) - @dynamic_property - def booksByDateRead(self): - def fget(self): - return self.__booksByDateRead - def fset(self, val): - self.__booksByDateRead = val - return property(fget=fget, fset=fset) - @dynamic_property - def booksByTitle(self): - def fget(self): - return self.__booksByTitle - def fset(self, val): - self.__booksByTitle = val - return property(fget=fget, fset=fset) - @dynamic_property - def booksByTitle_noSeriesPrefix(self): - def fget(self): - return self.__booksByTitle_noSeriesPrefix - def fset(self, val): - self.__booksByTitle_noSeriesPrefix = val - return property(fget=fget, fset=fset) - @dynamic_property - def catalogPath(self): - def fget(self): - return self.__catalogPath - def fset(self, val): - self.__catalogPath = val - return property(fget=fget, fset=fset) - @dynamic_property - def contentDir(self): - def fget(self): - return self.__contentDir - def fset(self, val): - self.__contentDir = val - return property(fget=fget, fset=fset) - @dynamic_property - def currentStep(self): - def fget(self): - return self.__currentStep - def fset(self, val): - self.__currentStep = val - return property(fget=fget, fset=fset) - @dynamic_property - def creator(self): - def fget(self): - return self.__creator - def fset(self, val): - self.__creator = val - return property(fget=fget, fset=fset) - @dynamic_property - def db(self): - def fget(self): - return self.__db - return property(fget=fget) - @dynamic_property - def defaultPrefix(self): - def fget(self): - return self.__defaultPrefix - def fset(self, val): - self.__defaultPrefix = val - return property(fget=fget, fset=fset) - @dynamic_property - def descriptionClip(self): - def fget(self): - return self.__descriptionClip - def fset(self, val): - self.__descriptionClip = val - return property(fget=fget, fset=fset) - @dynamic_property - def error(self): - def fget(self): - return self.__error - def fset(self, val): - self.__error = val - return property(fget=fget,fset=fset) - @dynamic_property - def generateForKindle(self): - def fget(self): - return self.__generateForKindle - def fset(self, val): - self.__generateForKindle = val - return property(fget=fget, fset=fset) - @dynamic_property - def generateRecentlyRead(self): - def fget(self): - return self.__generateRecentlyRead - def fset(self, val): - self.__generateRecentlyRead = val - return property(fget=fget, fset=fset) - @dynamic_property - def genres(self): - def fget(self): - return self.__genres - def fset(self, val): - self.__genres = val - return property(fget=fget, fset=fset) - @dynamic_property - def genre_tags_dict(self): - def fget(self): - return self.__genre_tags_dict - def fset(self, val): - self.__genre_tags_dict = val - return property(fget=fget, fset=fset) - @dynamic_property - def htmlFileList_1(self): - def fget(self): - return self.__htmlFileList_1 - def fset(self, val): - self.__htmlFileList_1 = val - return property(fget=fget, fset=fset) - @dynamic_property - def htmlFileList_2(self): - def fget(self): - return self.__htmlFileList_2 - def fset(self, val): - self.__htmlFileList_2 = val - return property(fget=fget, fset=fset) - @dynamic_property - def libraryPath(self): - def fget(self): - return self.__libraryPath - def fset(self, val): - self.__libraryPath = val - return property(fget=fget, fset=fset) - @dynamic_property - def markerTags(self): - def fget(self): - return self.__markerTags - def fset(self, val): - self.__markerTags = val - return property(fget=fget, fset=fset) - @dynamic_property - def ncxSoup(self): - def fget(self): - return self.__ncxSoup - def fset(self, val): - self.__ncxSoup = val - return property(fget=fget, fset=fset) - @dynamic_property - def opts(self): - def fget(self): - return self.__opts - return property(fget=fget) - @dynamic_property - def playOrder(self): - def fget(self): - return self.__playOrder - def fset(self,val): - self.__playOrder = val - return property(fget=fget, fset=fset) - @dynamic_property - def plugin(self): - def fget(self): - return self.__plugin - return property(fget=fget) - @dynamic_property - def prefixRules(self): - def fget(self): - return self.__prefixRules - def fset(self, val): - self.__prefixRules = val - return property(fget=fget, fset=fset) - @dynamic_property - def progressInt(self): - def fget(self): - return self.__progressInt - def fset(self, val): - self.__progressInt = val - return property(fget=fget, fset=fset) - @dynamic_property - def progressString(self): - def fget(self): - return self.__progressString - def fset(self, val): - self.__progressString = val - return property(fget=fget, fset=fset) - @dynamic_property - def reporter(self): - def fget(self): - return self.__reporter - def fset(self, val): - self.__reporter = val - return property(fget=fget, fset=fset) - @dynamic_property - def stylesheet(self): - def fget(self): - return self.__stylesheet - def fset(self, val): - self.__stylesheet = val - return property(fget=fget, fset=fset) - @dynamic_property - def thumbs(self): - def fget(self): - return self.__thumbs - def fset(self, val): - self.__thumbs = val - return property(fget=fget, fset=fset) - def thumbWidth(self): - def fget(self): - return self.__thumbWidth - def fset(self, val): - self.__thumbWidth = val - return property(fget=fget, fset=fset) - def thumbHeight(self): - def fget(self): - return self.__thumbHeight - def fset(self, val): - self.__thumbHeight = val - return property(fget=fget, fset=fset) - @dynamic_property - def title(self): - def fget(self): - return self.__title - def fset(self, val): - self.__title = val - return property(fget=fget, fset=fset) - @dynamic_property - def totalSteps(self): - def fget(self): - return self.__totalSteps - return property(fget=fget) - @dynamic_property - def useSeriesPrefixInTitlesSection(self): - def fget(self): - return self.__useSeriesPrefixInTitlesSection - def fset(self, val): - self.__useSeriesPrefixInTitlesSection = val - return property(fget=fget, fset=fset) - @dynamic_property - def verbose(self): - def fget(self): - return self.__verbose - def fset(self, val): - self.__verbose = val - return property(fget=fget, fset=fset) + Confirm existence of thumbs archive, or create if absent. + Confirm stored thumb_width matches current opts.thumb_width, + or invalidate archive. + generate_thumbnails() writes current thumb_width to archive. - @dynamic_property - def READING_SYMBOL(self): - def fget(self): - return '▷' if self.generateForKindle else \ - '+' - return property(fget=fget) - @dynamic_property - def FULL_RATING_SYMBOL(self): - def fget(self): - return self.__output_profile.ratings_char - return property(fget=fget) - @dynamic_property - def EMPTY_RATING_SYMBOL(self): - def fget(self): - return self.__output_profile.empty_ratings_char - return property(fget=fget) - @dynamic_property + Inputs: + opts.thumb_width (float): requested thumb_width + thumbs_path (file): existing thumbs archive - def READ_PROGRESS_SYMBOL(self): - def fget(self): - return "▪" if self.generateForKindle else '+' - return property(fget=fget) - @dynamic_property - def UNREAD_PROGRESS_SYMBOL(self): - def fget(self): - return "▫" if self.generateForKindle else '-' - return property(fget=fget) - - # Methods - def buildSources(self): - if self.booksByTitle is None: - if not self.fetchBooksByTitle(): - return False - if not self.fetchBooksByAuthor(): - return False - self.fetchBookmarks() + Outputs: + thumbs_path (file): new (non_existent or invalidated), or + validated existing thumbs archive + """ if self.opts.generate_descriptions: - self.generateThumbnails() - self.generateHTMLDescriptions() - if self.opts.generate_authors: - self.generateHTMLByAuthor() - if self.opts.generate_titles: - self.generateHTMLByTitle() - if self.opts.generate_series: - self.generateHTMLBySeries() - if self.opts.generate_genres: - self.generateHTMLByTags() - # If this is the only Section, and there are no genres, bail - if self.opts.section_list == ['Genres'] and not self.genres: - error_msg = _("No enabled genres found to catalog.\n") - if not self.opts.cli_environment: - error_msg += "Check 'Excluded genres'\nin E-book options.\n" - self.opts.log.error(error_msg) - self.error.append(_('No books available to catalog')) - self.error.append(error_msg) - return False - if self.opts.generate_recently_added: - self.generateHTMLByDateAdded() - if self.generateRecentlyRead: - self.generateHTMLByDateRead() + if not os.path.exists(self.cache_dir): + self.opts.log.info(" creating new thumb cache '%s'" % self.cache_dir) + os.makedirs(self.cache_dir) + if not os.path.exists(self.thumbs_path): + self.opts.log.info(' creating thumbnail archive, thumb_width: %1.2f"' % + float(self.opts.thumb_width)) + with ZipFile(self.thumbs_path, mode='w') as zfw: + zfw.writestr("Catalog Thumbs Archive",'') + else: + try: + with ZipFile(self.thumbs_path, mode='r') as zfr: + try: + cached_thumb_width = zfr.read('thumb_width') + except: + cached_thumb_width = "-1" + except: + os.remove(self.thumbs_path) + cached_thumb_width = '-1' - self.generateOPF() - self.generateNCXHeader() - if self.opts.generate_authors: - self.generateNCXByAuthor(_("Authors")) - if self.opts.generate_titles: - self.generateNCXByTitle(_("Titles")) - if self.opts.generate_series: - self.generateNCXBySeries(_("Series")) - if self.opts.generate_genres: - self.generateNCXByGenre(_("Genres")) - if self.opts.generate_recently_added: - self.generateNCXByDateAdded(_("Recently Added")) - if self.generateRecentlyRead: - self.generateNCXByDateRead(_("Recently Read")) - if self.opts.generate_descriptions: - self.generateNCXDescriptions(_("Descriptions")) + if float(cached_thumb_width) != float(self.opts.thumb_width): + self.opts.log.warning(" invalidating cache at '%s'" % self.thumbs_path) + self.opts.log.warning(' thumb_width changed: %1.2f" => %1.2f"' % + (float(cached_thumb_width),float(self.opts.thumb_width))) + with ZipFile(self.thumbs_path, mode='w') as zfw: + zfw.writestr("Catalog Thumbs Archive",'') + else: + self.opts.log.info(' existing thumb cache at %s, cached_thumb_width: %1.2f"' % + (self.thumbs_path, float(cached_thumb_width))) - self.writeNCX() - return True + def convert_html_entities(self, s): + """ Convert string containing HTML entities to its unicode equivalent. - def cleanUp(self): - pass + Convert a string containing HTML entities of the form '&' or '&97;' + to a normalized unicode string. E.g., 'AT&T' converted to 'AT&T'. - def copyResources(self): - '''Move resource files to self.catalogPath''' + Args: + s (str): str containing one or more HTML entities. + + Return: + s (str): converted string + """ + matches = re.findall("&#\d+;", s) + if len(matches) > 0: + hits = set(matches) + for hit in hits: + name = hit[2:-1] + try: + entnum = int(name) + s = s.replace(hit, unichr(entnum)) + except ValueError: + pass + + matches = re.findall("&\w+;", s) + hits = set(matches) + amp = "&" + if amp in hits: + hits.remove(amp) + for hit in hits: + name = hit[1:-1] + if htmlentitydefs.name2codepoint.has_key(name): + s = s.replace(hit, unichr(htmlentitydefs.name2codepoint[name])) + s = s.replace(amp, "&") + return s + + def copy_catalog_resources(self): + """ Copy resources from calibre source to self.catalog_path. + + Copy basic resources - default cover, stylesheet, and masthead (Kindle only) + from calibre resource directory to self.catalog_path, a temporary directory + for constructing the catalog. Files stored to specified destination dirs. + + Inputs: + files_to_copy (files): resource files from calibre resources, which may be overridden locally + + Output: + resource files copied to self.catalog_path/* + """ + self.create_catalog_directory_structure() catalog_resources = P("catalog") files_to_copy = [('','DefaultCover.jpg'), @@ -548,70 +717,236 @@ class CatalogBuilder(object): for file in files_to_copy: if file[0] == '': shutil.copy(os.path.join(catalog_resources,file[1]), - self.catalogPath) + self.catalog_path) else: shutil.copy(os.path.join(catalog_resources,file[1]), - os.path.join(self.catalogPath, file[0])) + os.path.join(self.catalog_path, file[0])) - # Create the custom masthead image overwriting default - # If failure, default mastheadImage.gif should still be in place - if self.generateForKindle: + if self.generate_for_kindle: try: - self.generateMastheadImage(os.path.join(self.catalogPath, + self.generate_masthead_image(os.path.join(self.catalog_path, 'images/mastheadImage.gif')) except: pass - def fetchBooksByAuthor(self): - ''' - Generate a list of titles sorted by author from the database - return = Success - ''' + def create_catalog_directory_structure(self): + """ Create subdirs in catalog output dir. - self.updateProgressFullStep("Sorting database") - self.booksByAuthor = list(self.booksByTitle) - self.booksByAuthor = sorted(self.booksByAuthor, key=self.booksByAuthorSorter_author) + Create /content and /images in self.catalog_path - # Build the unique_authors set from existing data, test for author_sort mismatches - authors = [(record['author'], record['author_sort']) for record in self.booksByAuthor] + Inputs: + catalog_path (path): path to catalog output dir + + Output: + /content, /images created + """ + if not os.path.isdir(self.catalog_path): + os.makedirs(self.catalog_path) + + content_path = self.catalog_path + "/content" + if not os.path.isdir(content_path): + os.makedirs(content_path) + images_path = self.catalog_path + "/images" + if not os.path.isdir(images_path): + os.makedirs(images_path) + + def detect_author_sort_mismatches(self): + """ Detect author_sort mismatches. + + Sort by author, look for inconsistencies in author_sort among + similarly-named authors. Fatal for MOBI generation, a mere + annoyance for EPUB. + + Inputs: + self.books_by_title (list): list of books to catalog + + Output: + self.books_by_author (list): sorted by author + + Exceptions: + AuthorSortMismatchException: author_sort mismatch detected + """ + + self.books_by_author = sorted(list(self.books_by_title), key=self._kf_books_by_author_sorter_author) + authors = [(record['author'], record['author_sort']) for record in self.books_by_author] current_author = authors[0] for (i,author) in enumerate(authors): if author != current_author and i: if author[0] == current_author[0]: if self.opts.fmt == 'mobi': # Exit if building MOBI - error_msg = _( -'''Inconsistent Author Sort values for -Author '{0}': -'{1}' <> '{2}' -Unable to build MOBI catalog.\n -Select all books by '{0}', apply correct Author Sort value in Edit Metadata dialog, then rebuild the catalog.\n''').format(author[0],author[1],current_author[1]) + error_msg = _("

Inconsistent Author Sort values for Author
" + + "'{!s}':

".format(author[0]) + + "

{!s} != {!s}

".format(author[1],current_author[1]) + + "

Unable to build MOBI catalog.
" + + "Select all books by '{!s}', apply correct Author Sort value in Edit Metadata dialog, then rebuild the catalog.\n

".format(author[0])) + self.opts.log.warn('\n*** Metadata error ***') self.opts.log.warn(error_msg) self.error.append('Author Sort mismatch') self.error.append(error_msg) - return False + raise AuthorSortMismatchException, "author_sort mismatch while building MOBI" else: # Warning if building non-MOBI if not self.error: self.error.append('Author Sort mismatch') - error_msg = _( -'''Warning: inconsistent Author Sort values for -Author '{0}': -'{1}' <> '{2}'\n''').format(author[0],author[1],current_author[1]) + error_msg = _("Warning: Inconsistent Author Sort values for Author '{!s}':\n".format(author[0]) + + " {!s} != {!s}\n".format(author[1],current_author[1])) self.opts.log.warn('\n*** Metadata warning ***') self.opts.log.warn(error_msg) self.error.append(error_msg) + continue current_author = author - self.booksByAuthor = sorted(self.booksByAuthor, - key=lambda x: sort_key(self.booksByAuthorSorter_author_sort(x))) + def discover_prefix(self, record): + """ Return a prefix for record. + + Evaluate record against self.prefix_rules. Return assigned prefix + if matched. + + Args: + record (dict): book metadata + + Return: + prefix (str): matched a prefix_rule + None: no match + """ + def _log_prefix_rule_match_info(rule, record): + self.opts.log.info(" %s '%s' by %s (Prefix rule '%s')" % + (rule['prefix'],record['title'], + record['authors'][0], rule['name'])) + + # Compare the record to each rule looking for a match + for rule in self.prefix_rules: + # Literal comparison for Tags field + if rule['field'].lower() == 'tags': + if rule['pattern'].lower() in map(unicode.lower,record['tags']): + if self.opts.verbose: + _log_prefix_rule_match_info(rule, record) + return rule['prefix'] + + # Regex match for custom field + elif rule['field'].startswith('#'): + field_contents = self.db.get_field(record['id'], + rule['field'], + index_is_id=True) + if field_contents == '': + field_contents = None + + if field_contents is not None: + try: + if re.search(rule['pattern'], unicode(field_contents), + re.IGNORECASE) is not None: + if self.opts.verbose: + _log_prefix_rule_match_info(rule, record) + return rule['prefix'] + except: + if self.opts.verbose: + self.opts.log.error("pattern failed to compile: %s" % rule['pattern']) + pass + elif field_contents is None and rule['pattern'] == 'None': + if self.opts.verbose: + _log_prefix_rule_match_info(rule, record) + return rule['prefix'] + + return None + + def establish_equivalencies(self, item_list, key=None): + """ Return icu equivalent sort letter. + + Returns base sort letter for accented characters. Code provided by + chaley, modified to force unaccented base letters for A, O & U when + an accented version would otherwise be returned. + + Args: + item_list (list): list of items, sorted by icu_sort + + Return: + cl_list (list): list of equivalent leading chars, 1:1 correspondence to item_list + """ + + # Hack to force the cataloged leading letter to be + # an unadorned character if the accented version sorts before the unaccented + exceptions = { + u'Ä':u'A', + u'Ö':u'O', + u'Ãœ':u'U' + } + + if key is not None: + sort_field = key + + cl_list = [None] * len(item_list) + last_ordnum = 0 + + for idx, item in enumerate(item_list): + if key: + c = item[sort_field] + else: + c = item + + ordnum, ordlen = collation_order(c) + if last_ordnum != ordnum: + last_c = icu_upper(c[0:ordlen]) + if last_c in exceptions.keys(): + last_c = exceptions[unicode(last_c)] + last_ordnum = ordnum + cl_list[idx] = last_c + + if self.DEBUG and self.opts.verbose: + print(" establish_equivalencies():") + if key: + for idx, item in enumerate(item_list): + print(" %s %s" % (cl_list[idx],item[sort_field])) + else: + print(" %s %s" % (cl_list[0], item)) + + return cl_list + + def fetch_books_by_author(self): + """ Generate a list of books sorted by author. + + Sort the database by author. Report author_sort inconsistencies as warning when + building EPUB or MOBI, error when building MOBI. Collect a list of unique authors + to self.authors. + + Inputs: + self.books_by_title (list): database, sorted by title + + Outputs: + books_by_author: database, sorted by author + authors: list of unique authors + error: author_sort mismatches + + Return: + True: no errors + False: author_sort mismatch detected while building MOBI + """ + + self.update_progress_full_step(_("Sorting database")) + + self.detect_author_sort_mismatches() + + # Sort authors using sort_key to normalize accented letters + # Determine the longest author_sort length before sorting + asl = [i['author_sort'] for i in self.books_by_author] + las = max(asl, key=len) + self.books_by_author = sorted(self.books_by_author, + key=lambda x: sort_key(self._kf_books_by_author_sorter_author_sort(x, len(las)))) + + if self.DEBUG and self.opts.verbose: + tl = [i['title'] for i in self.books_by_author] + lt = max(tl, key=len) + fs = '{:<6}{:<%d} {:<%d} {!s}' % (len(lt),len(las)) + print(fs.format('','Title','Author','Series')) + for i in self.books_by_author: + print(fs.format('', i['title'],i['author_sort'],i['series'])) # Build the unique_authors set from existing data - authors = [(record['author'], capitalize(record['author_sort'])) for record in self.booksByAuthor] + authors = [(record['author'], capitalize(record['author_sort'])) for record in self.books_by_author] # authors[] contains a list of all book authors, with multiple entries for multiple books by author # authors[]: (([0]:friendly [1]:sort)) @@ -642,8 +977,8 @@ Author '{0}': unique_authors.append((current_author[0], icu_title(current_author[1]), books_by_current_author)) - if False and self.verbose: - self.opts.log.info("\nfetchBooksByauthor(): %d unique authors" % len(unique_authors)) + if self.DEBUG and self.opts.verbose: + self.opts.log.info("\nfetch_books_by_author(): %d unique authors" % len(unique_authors)) for author in unique_authors: self.opts.log.info((u" %-50s %-25s %2d" % (author[0][0:45], author[1][0:20], author[2])).encode('utf-8')) @@ -651,60 +986,50 @@ Author '{0}': self.authors = unique_authors return True - def fetchBooksByTitle(self): - self.updateProgressFullStep("Fetching database") + def fetch_books_by_title(self): + """ Populate self.books_by_title from database - self.opts.sort_by = 'title' + Create self.books_by_title from filtered database. + Keys: + authors massaged + author_sort record['author_sort'] or computed + cover massaged record['cover'] + date massaged record['pubdate'] + description massaged record['comments'] + merge_comments + id record['id'] + formats massaged record['formats'] + notes from opts.header_note_source_field + prefix from self.discover_prefix() + publisher massaged record['publisher'] + rating record['rating'] (0 if None) + series record['series'] or None + series_index record['series_index'] or 0.0 + short_description truncated description + tags filtered record['tags'] + timestamp record['timestamp'] + title massaged record['title'] + title_sort computed from record['title'] + uuid record['uuid'] - # Merge opts.exclude_tags with opts.search_text - # Updated to use exact match syntax + Inputs: + data (list): filtered list of book metadata dicts - exclude_tags = [] - for rule in self.opts.exclusion_rules: - if rule[1].lower() == 'tags': - exclude_tags.extend(rule[2].split(',')) + Outputs: + (list) books_by_title - # Remove dups - self.exclude_tags = exclude_tags = list(set(exclude_tags)) + Returns: + True: Successful + False: Empty data, (check filter restrictions) + """ - # Report tag exclusions - if self.opts.verbose and self.exclude_tags: - data = self.db.get_data_as_dict(ids=self.opts.ids) - for record in data: - matched = list(set(record['tags']) & set(exclude_tags)) - if matched : - self.opts.log.info(" - %s by %s (Exclusion rule Tags: '%s')" % - (record['title'], record['authors'][0], str(matched[0]))) - - search_phrase = '' - if exclude_tags: - search_terms = [] - for tag in exclude_tags: - search_terms.append("tag:=%s" % tag) - search_phrase = "not (%s)" % " or ".join(search_terms) - - # If a list of ids are provided, don't use search_text - if self.opts.ids: - self.opts.search_text = search_phrase - else: - if self.opts.search_text: - self.opts.search_text += " " + search_phrase - else: - self.opts.search_text = search_phrase - - # Fetch the database as a dictionary - data = self.plugin.search_sort_db(self.db, self.opts) - data = self.processExclusions(data) - - # Populate this_title{} from data[{},{}] - titles = [] - for record in data: + def _populate_title(record): + ''' populate this_title with massaged metadata ''' this_title = {} this_title['id'] = record['id'] this_title['uuid'] = record['uuid'] - this_title['title'] = self.convertHTMLEntities(record['title']) + this_title['title'] = self.convert_html_entities(record['title']) if record['series']: this_title['series'] = record['series'] this_title['series_index'] = record['series_index'] @@ -712,7 +1037,7 @@ Author '{0}': this_title['series'] = None this_title['series_index'] = 0.0 - this_title['title_sort'] = self.generateSortTitle(this_title['title']) + this_title['title_sort'] = self.generate_sort_title(this_title['title']) if 'authors' in record: # from calibre.ebooks.metadata import authors_to_string # return authors_to_string(self.authors) @@ -726,7 +1051,7 @@ Author '{0}': if 'author_sort' in record and record['author_sort'].strip(): this_title['author_sort'] = record['author_sort'] else: - this_title['author_sort'] = self.author_to_author_sort(this_title['author']) + this_title['author_sort'] = self._kf_author_to_author_sort(this_title['author']) if record['publisher']: this_title['publisher'] = re.sub('&', '&', record['publisher']) @@ -749,7 +1074,7 @@ Author '{0}': if ad_offset >= 0: record['comments'] = record['comments'][:ad_offset] - this_title['description'] = self.markdownComments(record['comments']) + this_title['description'] = self.massage_comments(record['comments']) # Create short description paras = BeautifulSoup(this_title['description']).findAll('p') @@ -758,34 +1083,34 @@ Author '{0}': for token in p.contents: if token.string is not None: tokens.append(token.string) - this_title['short_description'] = self.generateShortDescription(' '.join(tokens), dest="description") + this_title['short_description'] = self.generate_short_description(' '.join(tokens), dest="description") else: this_title['description'] = None this_title['short_description'] = None # Merge with custom field/value - if self.__merge_comments['field']: - this_title['description'] = self.mergeComments(this_title) + if self.merge_comments_rule['field']: + this_title['description'] = self.merge_comments(this_title) if record['cover']: this_title['cover'] = re.sub('&', '&', record['cover']) - this_title['prefix'] = self.discoverPrefix(record) + this_title['prefix'] = self.discover_prefix(record) if record['tags']: - this_title['tags'] = self.processSpecialTags(record['tags'], - this_title, self.opts) + this_title['tags'] = self.filter_excluded_genres(record['tags'], + self.opts.exclude_genre) if record['formats']: formats = [] for format in record['formats']: - formats.append(self.convertHTMLEntities(format)) + formats.append(self.convert_html_entities(format)) this_title['formats'] = formats # Add user notes to be displayed in header # Special case handling for datetime fields and lists if self.opts.header_note_source_field: - field_md = self.__db.metadata_for_field(self.opts.header_note_source_field) - notes = self.__db.get_field(record['id'], + field_md = self.db.metadata_for_field(self.opts.header_note_source_field) + notes = self.db.get_field(record['id'], self.opts.header_note_source_field, index_is_id=True) if notes: @@ -794,39 +1119,74 @@ Author '{0}': notes = ' · '.join(notes) elif field_md['datatype'] == 'datetime': notes = format_date(notes,'dd MMM yyyy') - this_title['notes'] = {'source':field_md['name'], - 'content':notes} + this_title['notes'] = {'source':field_md['name'],'content':notes} + return this_title + + # Entry point + self.update_progress_full_step(_("Fetching database")) + + self.opts.sort_by = 'title' + search_phrase = '' + if self.excluded_tags: + search_terms = [] + for tag in self.excluded_tags: + search_terms.append("tag:=%s" % tag) + search_phrase = "not (%s)" % " or ".join(search_terms) + + # If a list of ids are provided, don't use search_text + if self.opts.ids: + self.opts.search_text = search_phrase + else: + if self.opts.search_text: + self.opts.search_text += " " + search_phrase + else: + self.opts.search_text = search_phrase + + # Fetch the database as a dictionary + data = self.plugin.search_sort_db(self.db, self.opts) + data = self.process_exclusions(data) + + # Populate this_title{} from data[{},{}] + titles = [] + for record in data: + this_title = _populate_title(record) titles.append(this_title) # Re-sort based on title_sort if len(titles): - #self.booksByTitle = sorted(titles, - # key=lambda x:(x['title_sort'].upper(), x['title_sort'].upper())) + self.books_by_title = sorted(titles, key=lambda x: sort_key(x['title_sort'].upper())) - self.booksByTitle = sorted(titles, key=lambda x: sort_key(x['title_sort'].upper())) - - if False and self.verbose: - self.opts.log.info("fetchBooksByTitle(): %d books" % len(self.booksByTitle)) + if self.DEBUG and self.opts.verbose: + self.opts.log.info("fetch_books_by_title(): %d books" % len(self.books_by_title)) self.opts.log.info(" %-40s %-40s" % ('title', 'title_sort')) - for title in self.booksByTitle: + for title in self.books_by_title: self.opts.log.info((u" %-40s %-40s" % (title['title'][0:40], - title['title_sort'][0:40])).decode('mac-roman')) - return True + title['title_sort'][0:40])).encode('utf-8')) else: - error_msg = _("No books found to catalog.\nCheck 'Excluded books' criteria in E-book options.\n") + error_msg = _("No books to catalog.\nCheck 'Excluded books' rules in E-book options.\n") self.opts.log.error('*** ' + error_msg + ' ***') self.error.append(_('No books available to include in catalog')) self.error.append(error_msg) - return False + raise EmptyCatalogException, error_msg - def fetchBookmarks(self): - ''' - Collect bookmarks for catalog entries - This will use the system default save_template specified in + def fetch_bookmarks(self): + """ Interrogate connected Kindle for bookmarks. + + Discover bookmarks associated with books on Kindle downloaded by calibre. + Used in Descriptions to show reading progress, Last Read section showing date + last read. Kindle-specific, for AZW, MOBI, TAN and TXT formats. + Uses the system default save_template specified in Preferences|Add/Save|Sending to device, not a customized one specified in - the Kindle plugin - ''' + the Kindle plugin. + + Inputs: + (): bookmarks from connected Kindle + + Output: + bookmarked_books (dict): dict of Bookmarks + """ + from calibre.devices.usbms.device import Device from calibre.devices.kindle.driver import Bookmark from calibre.ebooks.metadata import MetaInformation @@ -845,7 +1205,7 @@ Author '{0}': def save_template(self): return self._save_template - def resolve_bookmark_paths(storage, path_map): + def _resolve_bookmark_paths(storage, path_map): pop_list = [] book_ext = {} for id in path_map: @@ -881,14 +1241,14 @@ Author '{0}': path_map.pop(id) return path_map, book_ext - if self.generateRecentlyRead: + if self.generate_recently_read: self.opts.log.info(" Collecting Kindle bookmarks matching catalog entries") d = BookmarkDevice(None) d.initialize(self.opts.connected_device['save_template']) bookmarks = {} - for book in self.booksByTitle: + for book in self.books_by_title: if 'formats' in book: path_map = {} id = book['id'] @@ -900,7 +1260,7 @@ Author '{0}': a_path = d.create_upload_path('/', myMeta, 'x.bookmark', create_dirs=False) path_map[id] = dict(path=a_path, fmts=[x.rpartition('.')[2] for x in book['formats']]) - path_map, book_ext = resolve_bookmark_paths(self.opts.connected_device['storage'], path_map) + path_map, book_ext = _resolve_bookmark_paths(self.opts.connected_device['storage'], path_map) if path_map: bookmark_ext = path_map[id].rpartition('.')[2] myBookmark = Bookmark(path_map[id], id, book_ext[id], bookmark_ext) @@ -909,8 +1269,8 @@ Author '{0}': except: book['percent_read'] = 0 dots = int((book['percent_read'] + 5)/10) - dot_string = self.READ_PROGRESS_SYMBOL * dots - empty_dots = self.UNREAD_PROGRESS_SYMBOL * (10 - dots) + dot_string = self.SYMBOL_PROGRESS_READ * dots + empty_dots = self.SYMBOL_PROGRESS_UNREAD * (10 - dots) book['reading_progress'] = '%s%s' % (dot_string,empty_dots) bookmarks[id] = ((myBookmark,book)) @@ -918,176 +1278,215 @@ Author '{0}': else: self.bookmarked_books = {} - def generateHTMLDescriptions(self): - ''' - Write each title to a separate HTML file in contentdir - ''' - self.updateProgressFullStep("'Descriptions'") + def filter_db_tags(self): + """ Remove excluded tags from data set, return normalized genre list. - for (title_num, title) in enumerate(self.booksByTitle): - self.updateProgressMicroStep("Description %d of %d" % \ - (title_num, len(self.booksByTitle)), - float(title_num*100/len(self.booksByTitle))/100) + Filter all db tags, removing excluded tags supplied in opts. + Test for multiple tags resolving to same normalized form. Normalized + tags are flattened to alphanumeric ascii_text. - # Generate the header from user-customizable template - soup = self.generateHTMLDescriptionHeader(title) + Args: + (none) - # Write the book entry to contentdir - outfile = open("%s/book_%d.html" % (self.contentDir, int(title['id'])), 'w') - outfile.write(soup.prettify()) - outfile.close() + Return: + genre_tags_dict (dict): dict of filtered, normalized tags in data set + """ - def generateHTMLByTitle(self): - ''' - Write books by title A-Z to HTML file - ''' - self.updateProgressFullStep("'Titles'") + def _format_tag_list(tags, indent=5, line_break=70, header='Tag list'): + def _next_tag(sorted_tags): + for (i, tag) in enumerate(sorted_tags): + if i < len(tags) - 1: + yield tag + ", " + else: + yield tag - soup = self.generateHTMLEmptyHeader("Books By Alpha Title") - body = soup.find('body') - btc = 0 + ans = '%s%d %s:\n' % (' ' * indent, len(tags), header) + ans += ' ' * (indent + 1) + out_str = '' + sorted_tags = sorted(tags, key=sort_key) + for tag in _next_tag(sorted_tags): + out_str += tag + if len(out_str) >= line_break: + ans += out_str + '\n' + out_str = ' ' * (indent + 1) + return ans + out_str - pTag = Tag(soup, "p") - pTag['class'] = 'title' - ptc = 0 - aTag = Tag(soup,'a') - aTag['id'] = 'section_start' - pTag.insert(ptc, aTag) - ptc += 1 + # Entry point + normalized_tags = [] + friendly_tags = [] + excluded_tags = [] + for tag in self.db.all_tags(): + if tag in self.excluded_tags: + excluded_tags.append(tag) + continue + try: + if re.search(self.opts.exclude_genre, tag): + excluded_tags.append(tag) + continue + except: + self.opts.log.error("\tfilterDbTags(): malformed --exclude-genre regex pattern: %s" % self.opts.exclude_genre) - if not self.__generateForKindle: - # Kindle don't need this because it shows section titles in Periodical format - aTag = Tag(soup, "a") - aTag['id'] = "bytitle" - pTag.insert(ptc,aTag) - ptc += 1 - pTag.insert(ptc,NavigableString(_('Titles'))) + if tag == ' ': + continue - body.insert(btc,pTag) - btc += 1 + normalized_tags.append(self.normalize_tag(tag)) + friendly_tags.append(tag) - divTag = Tag(soup, "div") - dtc = 0 - current_letter = "" + genre_tags_dict = dict(zip(friendly_tags,normalized_tags)) - # Re-sort title list without leading series/series_index - # Incoming title : - if not self.useSeriesPrefixInTitlesSection: - nspt = deepcopy(self.booksByTitle) - nspt = sorted(nspt, key=lambda x: sort_key(x['title_sort'].upper())) - self.booksByTitle_noSeriesPrefix = nspt + # Test for multiple genres resolving to same normalized form + normalized_set = set(normalized_tags) + for normalized in normalized_set: + if normalized_tags.count(normalized) > 1: + self.opts.log.warn(" Warning: multiple tags resolving to genre '%s':" % normalized) + for key in genre_tags_dict: + if genre_tags_dict[key] == normalized: + self.opts.log.warn(" %s" % key) + if self.opts.verbose: + self.opts.log.info('%s' % _format_tag_list(genre_tags_dict, header="enabled genre tags in database")) + self.opts.log.info('%s' % _format_tag_list(excluded_tags, header="excluded genre tags")) - # Establish initial letter equivalencies - sort_equivalents = self.establish_equivalencies(self.booksByTitle, key='title_sort') + return genre_tags_dict - # Loop through the books by title - # Generate one divRunningTag per initial letter for the purposes of - # minimizing widows and orphans on readers that can handle large - # <divs> styled as inline-block - title_list = self.booksByTitle - if not self.useSeriesPrefixInTitlesSection: - title_list = self.booksByTitle_noSeriesPrefix - drtc = 0 - divRunningTag = None - for idx, book in enumerate(title_list): - if self.letter_or_symbol(sort_equivalents[idx]) != current_letter: - # Start a new letter - if drtc and divRunningTag is not None: - divTag.insert(dtc, divRunningTag) - dtc += 1 - divRunningTag = Tag(soup, 'div') - if dtc > 0: - divRunningTag['class'] = "initial_letter" - drtc = 0 - pIndexTag = Tag(soup, "p") - pIndexTag['class'] = "author_title_letter_index" - aTag = Tag(soup, "a") - current_letter = self.letter_or_symbol(sort_equivalents[idx]) - if current_letter == self.SYMBOLS: - aTag['id'] = self.SYMBOLS + "_titles" - pIndexTag.insert(0,aTag) - pIndexTag.insert(1,NavigableString(self.SYMBOLS)) + def filter_excluded_genres(self, tags, regex): + """ Remove excluded tags from a tag list + + Run regex against list of tags, remove matching tags. Return filtered list. + + Args: + tags (list): list of tags + + Return: + tag_list(list): filtered list of tags + """ + + tag_list = [] + + try: + for tag in tags: + tag = self.convert_html_entities(tag) + if re.search(regex, tag): + continue else: - aTag['id'] = self.generateUnicodeName(current_letter) + "_titles" - pIndexTag.insert(0,aTag) - pIndexTag.insert(1,NavigableString(sort_equivalents[idx])) - divRunningTag.insert(dtc,pIndexTag) - drtc += 1 + tag_list.append(tag) + except: + self.opts.log.error("\tfilter_excluded_genres(): malformed --exclude-genre regex pattern: %s" % regex) + return tags - # Add books - pBookTag = Tag(soup, "p") - pBookTag['class'] = "line_item" - ptc = 0 + return tag_list - pBookTag.insert(ptc, self.formatPrefix(book['prefix'],soup)) - ptc += 1 + def format_ncx_text(self, description, dest=None): + """ Massage NCX text for Kindle. - spanTag = Tag(soup, "span") - spanTag['class'] = "entry" - stc = 0 + Convert HTML entities for proper display on Kindle, convert + '&' to '&' (Kindle fails). + Args: + description (str): string, possibly with HTM entities + dest (kwarg): author, title or description - # Link to book - aTag = Tag(soup, "a") - if self.opts.generate_descriptions: - aTag['href'] = "book_%d.html" % (int(float(book['id']))) + Return: + (str): massaged, possibly truncated description + """ + # Kindle TOC descriptions won't render certain characters + # Fix up + massaged = unicode(BeautifulStoneSoup(description, convertEntities=BeautifulStoneSoup.HTML_ENTITIES)) - # Generate the title from the template - args = self.generateFormatArgs(book) - if book['series']: - formatted_title = self.by_titles_series_title_template.format(**args).rstrip() + # Replace '&' with '&' + massaged = re.sub("&","&", massaged) + + if massaged.strip() and dest: + #print traceback.print_stack(limit=3) + return self.generate_short_description(massaged.strip(), dest=dest) + else: + return None + + def format_prefix(self, prefix_char): + """ Generate HTML snippet with prefix character. + + Return a <code> snippet for Kindle, <span> snippet for EPUB. + Optimized to preserve first-column alignment for MOBI, EPUB. + + Args: + prefix_char (str): prefix character or None + + Return: + (str): BeautifulSoup HTML snippet to be inserted into <p> line item entry. + """ + + soup = BeautifulSoup('') + if self.opts.fmt == 'mobi': + codeTag = Tag(soup, "code") + if prefix_char is None: + codeTag.insert(0,NavigableString(' ')) else: - formatted_title = self.by_titles_normal_title_template.format(**args).rstrip() - aTag.insert(0,NavigableString(escape(formatted_title))) - spanTag.insert(stc, aTag) - stc += 1 + codeTag.insert(0,NavigableString(prefix_char)) + return codeTag + else: + spanTag = Tag(soup, "span") + spanTag['class'] = "prefix" + if prefix_char is None: + prefix_char = " " + spanTag.insert(0,NavigableString(prefix_char)) + return spanTag - # Dot - spanTag.insert(stc, NavigableString(" · ")) - stc += 1 + def generate_author_anchor(self, author): + """ Generate legal XHTML anchor. - # Link to author - emTag = Tag(soup, "em") - aTag = Tag(soup, "a") - if self.opts.generate_authors: - aTag['href'] = "%s.html#%s" % ("ByAlphaAuthor", self.generateAuthorAnchor(book['author'])) - aTag.insert(0, NavigableString(book['author'])) - emTag.insert(0,aTag) - spanTag.insert(stc, emTag) - stc += 1 + Convert author to legal XHTML (may contain unicode chars), stripping + non-alphanumeric chars. - pBookTag.insert(ptc, spanTag) - ptc += 1 + Args: + author (str): author name - if divRunningTag is not None: - divRunningTag.insert(drtc, pBookTag) - drtc += 1 + Return: + (str): asciized version of author + """ + return re.sub("\W","", ascii_text(author)) - # Add the last divRunningTag to divTag - if divRunningTag is not None: - divTag.insert(dtc, divRunningTag) - dtc += 1 + def generate_format_args(self, book): + """ Generate the format args for template substitution. - # Add the divTag to the body - body.insert(btc, divTag) - btc += 1 + self.load_section_templates imports string formatting templates of the form + 'by_*_template.py' for use in the various sections. The templates are designed to use + format args, supplied by this method. - # Write the volume to contentdir - outfile_spec = "%s/ByAlphaTitle.html" % (self.contentDir) - outfile = open(outfile_spec, 'w') - outfile.write(soup.prettify()) - outfile.close() - self.htmlFileList_1.append("content/ByAlphaTitle.html") + Args: + book (dict): book metadata - def generateHTMLByAuthor(self): - ''' - Write books by author A-Z - ''' - self.updateProgressFullStep("'Authors'") + Return: + (dict): formatted args for templating + """ + series_index = str(book['series_index']) + if series_index.endswith('.0'): + series_index = series_index[:-2] + args = dict( + title = book['title'], + series = book['series'], + series_index = series_index, + rating = self.generate_rating_string(book), + rating_parens = '(%s)' % self.generate_rating_string(book) if 'rating' in book else '', + pubyear = book['date'].split()[1] if book['date'] else '', + pubyear_parens = "(%s)" % book['date'].split()[1] if book['date'] else '') + return args + + def generate_html_by_author(self): + """ Generate content/ByAlphaAuthor.html. + + Loop through self.books_by_author, generate HTML + with anchors for author and index letters. + + Input: + books_by_author (list): books, sorted by author + + Output: + content/ByAlphaAuthor.html (file) + """ friendly_name = _("Authors") + self.update_progress_full_step("%s HTML" % friendly_name) - soup = self.generateHTMLEmptyHeader(friendly_name) + soup = self.generate_html_empty_header(friendly_name) body = soup.find('body') btc = 0 @@ -1099,7 +1498,7 @@ Author '{0}': divRunningTag = None drtc = 0 - # Loop through booksByAuthor + # Loop through books_by_author # Each author/books group goes in an openingTag div (first) or # a runningTag div (subsequent) book_count = 0 @@ -1107,11 +1506,9 @@ Author '{0}': current_letter = '' current_series = None # Establish initial letter equivalencies - sort_equivalents = self.establish_equivalencies(self.booksByAuthor,key='author_sort') + sort_equivalents = self.establish_equivalencies(self.books_by_author,key='author_sort') - #for book in sorted(self.booksByAuthor, key = self.booksByAuthorSorter_author_sort): - #for book in self.booksByAuthor: - for idx, book in enumerate(self.booksByAuthor): + for idx, book in enumerate(self.books_by_author): book_count += 1 if self.letter_or_symbol(sort_equivalents[idx]) != current_letter : # Start a new letter with Index letter @@ -1140,14 +1537,14 @@ Author '{0}': pIndexTag.insert(0,aTag) pIndexTag.insert(1,NavigableString(self.SYMBOLS)) else: - aTag['id'] = self.generateUnicodeName(current_letter) + '_authors' + aTag['id'] = self.generate_unicode_name(current_letter) + '_authors' pIndexTag.insert(0,aTag) pIndexTag.insert(1,NavigableString(sort_equivalents[idx])) divOpeningTag.insert(dotc,pIndexTag) dotc += 1 if book['author'] != current_author: - # Start a new author + # Start a new authorl current_author = book['author'] author_count += 1 if author_count >= 2: @@ -1172,7 +1569,7 @@ Author '{0}': pAuthorTag = Tag(soup, "p") pAuthorTag['class'] = "author_index" aTag = Tag(soup, "a") - aTag['id'] = "%s" % self.generateAuthorAnchor(current_author) + aTag['id'] = "%s" % self.generate_author_anchor(current_author) aTag.insert(0,NavigableString(current_author)) pAuthorTag.insert(0,aTag) if author_count == 1: @@ -1192,7 +1589,7 @@ Author '{0}': pSeriesTag['class'] = "series_mobi" if self.opts.generate_series: aTag = Tag(soup,'a') - aTag['href'] = "%s.html#%s" % ('BySeries',self.generateSeriesAnchor(book['series'])) + aTag['href'] = "%s.html#%s" % ('BySeries',self.generate_series_anchor(book['series'])) aTag.insert(0, book['series']) pSeriesTag.insert(0, aTag) else: @@ -1212,7 +1609,7 @@ Author '{0}': pBookTag['class'] = "line_item" ptc = 0 - pBookTag.insert(ptc, self.formatPrefix(book['prefix'],soup)) + pBookTag.insert(ptc, self.format_prefix(book['prefix'])) ptc += 1 spanTag = Tag(soup, "span") @@ -1224,7 +1621,7 @@ Author '{0}': aTag['href'] = "book_%d.html" % (int(float(book['id']))) # Generate the title from the template - args = self.generateFormatArgs(book) + args = self.generate_format_args(book) if current_series: #aTag.insert(0,'%s%s' % (escape(book['title'][len(book['series'])+1:]),pubyear)) formatted_title = self.by_authors_series_title_template.format(**args).rstrip() @@ -1246,7 +1643,7 @@ Author '{0}': divRunningTag.insert(drtc,pBookTag) drtc += 1 - # Loop ends here + # loop ends here pTag = Tag(soup, "p") pTag['class'] = 'title' @@ -1256,7 +1653,7 @@ Author '{0}': pTag.insert(ptc, aTag) ptc += 1 - if not self.__generateForKindle: + if not self.generate_for_kindle: # Kindle don't need this because it shows section titles in Periodical format aTag = Tag(soup, "a") anchor_name = friendly_name.lower() @@ -1278,23 +1675,32 @@ Author '{0}': # Add the divTag to the body body.insert(btc, divTag) - # Write the generated file to contentdir - outfile_spec = "%s/ByAlphaAuthor.html" % (self.contentDir) + # Write the generated file to content_dir + outfile_spec = "%s/ByAlphaAuthor.html" % (self.content_dir) outfile = open(outfile_spec, 'w') outfile.write(soup.prettify()) outfile.close() - self.htmlFileList_1.append("content/ByAlphaAuthor.html") + self.html_filelist_1.append("content/ByAlphaAuthor.html") - def generateHTMLByDateAdded(self): - ''' - Write books by reverse chronological order - ''' - self.updateProgressFullStep("'Recently Added'") + def generate_html_by_date_added(self): + """ Generate content/ByDateAdded.html. - def add_books_to_HTML_by_month(this_months_list, dtc): + Loop through self.books_by_title sorted by reverse date, generate HTML. + + Input: + books_by_title (list): books, sorted by title + + Output: + content/ByDateAdded.html (file) + """ + + def _add_books_to_html_by_month(this_months_list, dtc): if len(this_months_list): - - this_months_list = sorted(this_months_list, key=lambda x: sort_key(self.booksByAuthorSorter_author_sort(x))) + # Determine the longest author_sort_length before sorting + asl = [i['author_sort'] for i in this_months_list] + las = max(asl, key=len) + this_months_list = sorted(this_months_list, + key=lambda x: sort_key(self._kf_books_by_author_sorter_author_sort(x, len(las)))) # Create a new month anchor date_string = strftime(u'%B %Y', current_date.timetuple()) @@ -1319,7 +1725,7 @@ Author '{0}': pAuthorTag['class'] = "author_index" aTag = Tag(soup, "a") if self.opts.generate_authors: - aTag['href'] = "%s.html#%s" % ("ByAlphaAuthor", self.generateAuthorAnchor(current_author)) + aTag['href'] = "%s.html#%s" % ("ByAlphaAuthor", self.generate_author_anchor(current_author)) aTag.insert(0,NavigableString(current_author)) pAuthorTag.insert(0,aTag) divTag.insert(dtc,pAuthorTag) @@ -1335,9 +1741,7 @@ Author '{0}': pSeriesTag['class'] = "series_mobi" if self.opts.generate_series: aTag = Tag(soup,'a') - - if self.letter_or_symbol(new_entry['series']) == self.SYMBOLS: - aTag['href'] = "%s.html#%s" % ('BySeries',self.generateSeriesAnchor(new_entry['series'])) + aTag['href'] = "%s.html#%s" % ('BySeries',self.generate_series_anchor(new_entry['series'])) aTag.insert(0, new_entry['series']) pSeriesTag.insert(0, aTag) else: @@ -1352,7 +1756,7 @@ Author '{0}': pBookTag['class'] = "line_item" ptc = 0 - pBookTag.insert(ptc, self.formatPrefix(new_entry['prefix'],soup)) + pBookTag.insert(ptc, self.format_prefix(new_entry['prefix'])) ptc += 1 spanTag = Tag(soup, "span") @@ -1364,7 +1768,7 @@ Author '{0}': aTag['href'] = "book_%d.html" % (int(float(new_entry['id']))) # Generate the title from the template - args = self.generateFormatArgs(new_entry) + args = self.generate_format_args(new_entry) if current_series: formatted_title = self.by_month_added_series_title_template.format(**args).rstrip() else: @@ -1381,7 +1785,7 @@ Author '{0}': dtc += 1 return dtc - def add_books_to_HTML_by_date_range(date_range_list, date_range, dtc): + def _add_books_to_html_by_date_range(date_range_list, date_range, dtc): if len(date_range_list): pIndexTag = Tag(soup, "p") pIndexTag['class'] = "date_index" @@ -1398,7 +1802,7 @@ Author '{0}': pBookTag['class'] = "line_item" ptc = 0 - pBookTag.insert(ptc, self.formatPrefix(new_entry['prefix'],soup)) + pBookTag.insert(ptc, self.format_prefix(new_entry['prefix'])) ptc += 1 spanTag = Tag(soup, "span") @@ -1410,7 +1814,7 @@ Author '{0}': aTag['href'] = "book_%d.html" % (int(float(new_entry['id']))) # Generate the title from the template - args = self.generateFormatArgs(new_entry) + args = self.generate_format_args(new_entry) if new_entry['series']: formatted_title = self.by_recently_added_series_title_template.format(**args).rstrip() else: @@ -1427,7 +1831,7 @@ Author '{0}': emTag = Tag(soup, "em") aTag = Tag(soup, "a") if self.opts.generate_authors: - aTag['href'] = "%s.html#%s" % ("ByAlphaAuthor", self.generateAuthorAnchor(new_entry['author'])) + aTag['href'] = "%s.html#%s" % ("ByAlphaAuthor", self.generate_author_anchor(new_entry['author'])) aTag.insert(0, NavigableString(new_entry['author'])) emTag.insert(0,aTag) spanTag.insert(stc, emTag) @@ -1441,8 +1845,9 @@ Author '{0}': return dtc friendly_name = _("Recently Added") + self.update_progress_full_step("%s HTML" % friendly_name) - soup = self.generateHTMLEmptyHeader(friendly_name) + soup = self.generate_html_empty_header(friendly_name) body = soup.find('body') btc = 0 @@ -1456,7 +1861,7 @@ Author '{0}': pTag.insert(ptc, aTag) ptc += 1 - if not self.__generateForKindle: + if not self.generate_for_kindle: # Kindle don't need this because it shows section titles in Periodical format aTag = Tag(soup, "a") anchor_name = friendly_name.lower() @@ -1473,12 +1878,12 @@ Author '{0}': dtc = 0 # >>> Books by date range <<< - if self.useSeriesPrefixInTitlesSection: - self.booksByDateRange = sorted(self.booksByTitle, + if self.use_series_prefix_in_titles_section: + self.books_by_date_range = sorted(self.books_by_title, key=lambda x:(x['timestamp'], x['timestamp']),reverse=True) else: - nspt = deepcopy(self.booksByTitle) - self.booksByDateRange = sorted(nspt, key=lambda x:(x['timestamp'], x['timestamp']),reverse=True) + nspt = deepcopy(self.books_by_title) + self.books_by_date_range = sorted(nspt, key=lambda x:(x['timestamp'], x['timestamp']),reverse=True) date_range_list = [] today_time = nowf().replace(hour=23, minute=59, second=59) @@ -1489,7 +1894,7 @@ Author '{0}': else: date_range = 'Last %d days' % (self.DATE_RANGE[i]) - for book in self.booksByDateRange: + for book in self.books_by_date_range: book_time = book['timestamp'] delta = today_time-book_time if delta.days <= date_range_limit: @@ -1497,48 +1902,52 @@ Author '{0}': else: break - dtc = add_books_to_HTML_by_date_range(date_range_list, date_range, dtc) + dtc = _add_books_to_html_by_date_range(date_range_list, date_range, dtc) date_range_list = [book] # >>>> Books by month <<<< # Sort titles case-insensitive for by month using series prefix - self.booksByMonth = sorted(self.booksByTitle, + self.books_by_month = sorted(self.books_by_title, key=lambda x:(x['timestamp'], x['timestamp']),reverse=True) # Loop through books by date current_date = datetime.date.fromordinal(1) this_months_list = [] - for book in self.booksByMonth: + for book in self.books_by_month: if book['timestamp'].month != current_date.month or \ book['timestamp'].year != current_date.year: - dtc = add_books_to_HTML_by_month(this_months_list, dtc) + dtc = _add_books_to_html_by_month(this_months_list, dtc) this_months_list = [] current_date = book['timestamp'].date() this_months_list.append(book) # Add the last month's list - add_books_to_HTML_by_month(this_months_list, dtc) + _add_books_to_html_by_month(this_months_list, dtc) # Add the divTag to the body body.insert(btc, divTag) - # Write the generated file to contentdir - outfile_spec = "%s/ByDateAdded.html" % (self.contentDir) + # Write the generated file to content_dir + outfile_spec = "%s/ByDateAdded.html" % (self.content_dir) outfile = open(outfile_spec, 'w') outfile.write(soup.prettify()) outfile.close() - self.htmlFileList_2.append("content/ByDateAdded.html") + self.html_filelist_2.append("content/ByDateAdded.html") - def generateHTMLByDateRead(self): - ''' - Write books by active bookmarks - ''' - friendly_name = _('Recently Read') - self.updateProgressFullStep("'%s'" % friendly_name) - if not self.bookmarked_books: - return + def generate_html_by_date_read(self): + """ Generate content/ByDateRead.html. - def add_books_to_HTML_by_day(todays_list, dtc): + Create self.bookmarked_books_by_date_read from self.bookmarked_books. + Loop through self.bookmarked_books_by_date_read, generate HTML. + + Input: + bookmarked_books_by_date_read (list) + + Output: + content/ByDateRead.html (file) + """ + + def _add_books_to_html_by_day(todays_list, dtc): if len(todays_list): # Create a new day anchor date_string = strftime(u'%A, %B %d', current_date.timetuple()) @@ -1575,7 +1984,7 @@ Author '{0}': emTag = Tag(soup, "em") aTag = Tag(soup, "a") if self.opts.generate_authors: - aTag['href'] = "%s.html#%s" % ("ByAlphaAuthor", self.generateAuthorAnchor(new_entry['author'])) + aTag['href'] = "%s.html#%s" % ("ByAlphaAuthor", self.generate_author_anchor(new_entry['author'])) aTag.insert(0, NavigableString(new_entry['author'])) emTag.insert(0,aTag) pBookTag.insert(ptc, emTag) @@ -1585,7 +1994,7 @@ Author '{0}': dtc += 1 return dtc - def add_books_to_HTML_by_date_range(date_range_list, date_range, dtc): + def _add_books_to_html_by_date_range(date_range_list, date_range, dtc): if len(date_range_list): pIndexTag = Tag(soup, "p") pIndexTag['class'] = "date_index" @@ -1604,8 +2013,8 @@ Author '{0}': # Percent read dots = int((new_entry['percent_read'] + 5)/10) - dot_string = self.READ_PROGRESS_SYMBOL * dots - empty_dots = self.UNREAD_PROGRESS_SYMBOL * (10 - dots) + dot_string = self.SYMBOL_PROGRESS_READ * dots + empty_dots = self.SYMBOL_PROGRESS_UNREAD * (10 - dots) pBookTag.insert(ptc, NavigableString('%s%s' % (dot_string,empty_dots))) ptc += 1 @@ -1624,7 +2033,7 @@ Author '{0}': emTag = Tag(soup, "em") aTag = Tag(soup, "a") if self.opts.generate_authors: - aTag['href'] = "%s.html#%s" % ("ByAlphaAuthor", self.generateAuthorAnchor(new_entry['author'])) + aTag['href'] = "%s.html#%s" % ("ByAlphaAuthor", self.generate_author_anchor(new_entry['author'])) aTag.insert(0, NavigableString(new_entry['author'])) emTag.insert(0,aTag) pBookTag.insert(ptc, emTag) @@ -1634,7 +2043,13 @@ Author '{0}': dtc += 1 return dtc - soup = self.generateHTMLEmptyHeader(friendly_name) + friendly_name = _('Recently Read') + self.update_progress_full_step("%s HTML" % friendly_name) + + if not self.bookmarked_books: + return + + soup = self.generate_html_empty_header(friendly_name) body = soup.find('body') btc = 0 @@ -1667,223 +2082,61 @@ Author '{0}': book[1]['percent_read'] = 0 bookmarked_books.append(book[1]) - self.booksByDateRead = sorted(bookmarked_books, + self.bookmarked_books_by_date_read = sorted(bookmarked_books, key=lambda x:(x['bookmark_timestamp'], x['bookmark_timestamp']),reverse=True) # >>>> Recently read by day <<<< current_date = datetime.date.fromordinal(1) todays_list = [] - for book in self.booksByDateRead: + for book in self.bookmarked_books_by_date_read: bookmark_time = datetime.datetime.utcfromtimestamp(book['bookmark_timestamp']) if bookmark_time.day != current_date.day or \ bookmark_time.month != current_date.month or \ bookmark_time.year != current_date.year: - dtc = add_books_to_HTML_by_day(todays_list, dtc) + dtc = _add_books_to_html_by_day(todays_list, dtc) todays_list = [] current_date = datetime.datetime.utcfromtimestamp(book['bookmark_timestamp']).date() todays_list.append(book) # Add the last day's list - add_books_to_HTML_by_day(todays_list, dtc) + _add_books_to_html_by_day(todays_list, dtc) # Add the divTag to the body body.insert(btc, divTag) - # Write the generated file to contentdir - outfile_spec = "%s/ByDateRead.html" % (self.contentDir) + # Write the generated file to content_dir + outfile_spec = "%s/ByDateRead.html" % (self.content_dir) outfile = open(outfile_spec, 'w') outfile.write(soup.prettify()) outfile.close() - self.htmlFileList_2.append("content/ByDateRead.html") + self.html_filelist_2.append("content/ByDateRead.html") - def generateHTMLBySeries(self): - ''' - Generate a list of series - ''' - self.updateProgressFullStep("Fetching series") + def generate_html_by_genres(self): + """ Generate individual HTML files per tag. - self.opts.sort_by = 'series' + Filter out excluded tags. For each tag qualifying as a genre, + create a separate HTML file. Normalize tags to flatten synonymous tags. - # Merge self.exclude_tags with opts.search_text - # Updated to use exact match syntax + Inputs: + db.all_tags() (list): all database tags - search_phrase = 'series:true ' - if self.exclude_tags: - search_terms = [] - for tag in self.exclude_tags: - search_terms.append("tag:=%s" % tag) - search_phrase += "not (%s)" % " or ".join(search_terms) + Output: + (files): HTML file per genre + """ - # If a list of ids are provided, don't use search_text - if self.opts.ids: - self.opts.search_text = search_phrase - else: - if self.opts.search_text: - self.opts.search_text += " " + search_phrase - else: - self.opts.search_text = search_phrase + self.update_progress_full_step(_("Genres HTML")) - # Fetch the database as a dictionary - data = self.plugin.search_sort_db(self.db, self.opts) + self.genre_tags_dict = self.filter_db_tags() - # Remove exclusions - self.booksBySeries = self.processExclusions(data) - - if not self.booksBySeries: - self.opts.generate_series = False - self.opts.log(" no series found in selected books, cancelling series generation") - return - - # Generate series_sort - for book in self.booksBySeries: - book['series_sort'] = self.generateSortTitle(book['series']) - - friendly_name = _("Series") - - soup = self.generateHTMLEmptyHeader(friendly_name) - body = soup.find('body') - - btc = 0 - divTag = Tag(soup, "div") - dtc = 0 - current_letter = "" - current_series = None - - # Establish initial letter equivalencies - sort_equivalents = self.establish_equivalencies(self.booksBySeries, key='series_sort') - - # Loop through booksBySeries - series_count = 0 - for idx, book in enumerate(self.booksBySeries): - # Check for initial letter change - if self.letter_or_symbol(sort_equivalents[idx]) != current_letter : - # Start a new letter with Index letter - current_letter = self.letter_or_symbol(sort_equivalents[idx]) - pIndexTag = Tag(soup, "p") - pIndexTag['class'] = "series_letter_index" - aTag = Tag(soup, "a") - if current_letter == self.SYMBOLS: - aTag['id'] = self.SYMBOLS + "_series" - pIndexTag.insert(0,aTag) - pIndexTag.insert(1,NavigableString(self.SYMBOLS)) - else: - aTag['id'] = self.generateUnicodeName(current_letter) + "_series" - pIndexTag.insert(0,aTag) - pIndexTag.insert(1,NavigableString(sort_equivalents[idx])) - divTag.insert(dtc,pIndexTag) - dtc += 1 - # Check for series change - if book['series'] != current_series: - # Start a new series - series_count += 1 - current_series = book['series'] - pSeriesTag = Tag(soup,'p') - pSeriesTag['class'] = "series" - if self.opts.fmt == 'mobi': - pSeriesTag['class'] = "series_mobi" - aTag = Tag(soup, 'a') - aTag['id'] = self.generateSeriesAnchor(book['series']) - pSeriesTag.insert(0,aTag) - pSeriesTag.insert(1,NavigableString('%s' % book['series'])) - divTag.insert(dtc,pSeriesTag) - dtc += 1 - - # Add books - pBookTag = Tag(soup, "p") - pBookTag['class'] = "line_item" - ptc = 0 - - book['prefix'] = self.discoverPrefix(book) - pBookTag.insert(ptc, self.formatPrefix(book['prefix'],soup)) - ptc += 1 - - spanTag = Tag(soup, "span") - spanTag['class'] = "entry" - stc = 0 - - aTag = Tag(soup, "a") - if self.opts.generate_descriptions: - aTag['href'] = "book_%d.html" % (int(float(book['id']))) - # Use series, series index if avail else just title - #aTag.insert(0,'%d. %s · %s' % (book['series_index'],escape(book['title']), ' & '.join(book['authors']))) - - if is_date_undefined(book['pubdate']): - book['date'] = None - else: - book['date'] = strftime(u'%B %Y', book['pubdate'].timetuple()) - - args = self.generateFormatArgs(book) - formatted_title = self.by_series_title_template.format(**args).rstrip() - aTag.insert(0,NavigableString(escape(formatted_title))) - - spanTag.insert(stc, aTag) - stc += 1 - - # · - spanTag.insert(stc, NavigableString(' · ')) - stc += 1 - - # Link to author - aTag = Tag(soup, "a") - if self.opts.generate_authors: - aTag['href'] = "%s.html#%s" % ("ByAlphaAuthor", - self.generateAuthorAnchor(escape(' & '.join(book['authors'])))) - aTag.insert(0, NavigableString(' & '.join(book['authors']))) - spanTag.insert(stc, aTag) - stc += 1 - - pBookTag.insert(ptc, spanTag) - ptc += 1 - - divTag.insert(dtc, pBookTag) - dtc += 1 - - pTag = Tag(soup, "p") - pTag['class'] = 'title' - ptc = 0 - aTag = Tag(soup,'a') - aTag['id'] = 'section_start' - pTag.insert(ptc, aTag) - ptc += 1 - - if not self.__generateForKindle: - # Insert the <h2> tag with book_count at the head - aTag = Tag(soup, "a") - anchor_name = friendly_name.lower() - aTag['id'] = anchor_name.replace(" ","") - pTag.insert(0,aTag) - pTag.insert(1,NavigableString('%s' % friendly_name)) - body.insert(btc,pTag) - btc += 1 - - # Add the divTag to the body - body.insert(btc, divTag) - - # Write the generated file to contentdir - outfile_spec = "%s/BySeries.html" % (self.contentDir) - outfile = open(outfile_spec, 'w') - outfile.write(soup.prettify()) - outfile.close() - self.htmlFileList_1.append("content/BySeries.html") - - def generateHTMLByTags(self): - ''' - Generate individual HTML files for each tag, e.g. Fiction, Nonfiction ... - Note that special tags - have already been filtered from books[] - There may be synonomous tags - ''' - self.updateProgressFullStep("'Genres'") - - self.genre_tags_dict = self.filterDbTags(self.db.all_tags()) # Extract books matching filtered_tags genre_list = [] for friendly_tag in sorted(self.genre_tags_dict, key=sort_key): - #print("\ngenerateHTMLByTags(): looking for books with friendly_tag '%s'" % friendly_tag) + #print("\ngenerate_html_by_genres(): looking for books with friendly_tag '%s'" % friendly_tag) # tag_list => { normalized_genre_tag : [{book},{},{}], # normalized_genre_tag : [{book},{},{}] } tag_list = {} - for book in self.booksByAuthor: + for book in self.books_by_author: # Scan each book for tag matching friendly_tag if 'tags' in book and friendly_tag in book['tags']: this_book = {} @@ -1918,11 +2171,11 @@ Author '{0}': if self.opts.verbose: if len(genre_list): self.opts.log.info(" Genre summary: %d active genre tags used in generating catalog with %d titles" % - (len(genre_list), len(self.booksByTitle))) + (len(genre_list), len(self.books_by_title))) for genre in genre_list: for key in genre: - self.opts.log.info(" %s: %d %s" % (self.getFriendlyGenreTag(key), + self.opts.log.info(" %s: %d %s" % (self.get_friendly_genre_tag(key), len(genre[key]), 'titles' if len(genre[key]) > 1 else 'title')) @@ -1956,9 +2209,9 @@ Author '{0}': books_by_current_author += 1 # Write the genre book list as an article - titles_spanned = self.generateHTMLByGenre(genre, True if index==0 else False, + titles_spanned = self.generate_html_by_genre(genre, True if index==0 else False, genre_tag_set[genre], - "%s/Genre_%s.html" % (self.contentDir, + "%s/Genre_%s.html" % (self.content_dir, genre)) tag_file = "content/Genre_%s.html" % genre @@ -1970,1428 +2223,31 @@ Author '{0}': self.genres = master_genre_list - def generateThumbnails(self): - ''' - Generate a thumbnail per cover. If a current thumbnail exists, skip - If a cover doesn't exist, use default - Return list of active thumbs - ''' - self.updateProgressFullStep("'Thumbnails'") - thumbs = ['thumbnail_default.jpg'] - image_dir = "%s/images" % self.catalogPath - for (i,title) in enumerate(self.booksByTitle): - # Update status - self.updateProgressMicroStep("Thumbnail %d of %d" % \ - (i,len(self.booksByTitle)), - i/float(len(self.booksByTitle))) + def generate_html_by_genre(self, genre, section_head, books, outfile): + """ Generate individual genre HTML file. - thumb_file = 'thumbnail_%d.jpg' % int(title['id']) - thumb_generated = True - valid_cover = True - try: - self.generateThumbnail(title, image_dir, thumb_file) - thumbs.append("thumbnail_%d.jpg" % int(title['id'])) - except: - if 'cover' in title and os.path.exists(title['cover']): - valid_cover = False - self.opts.log.warn(" *** Invalid cover file for '%s'***" % - (title['title'])) - if not self.error: - self.error.append('Invalid cover files') - self.error.append("Warning: invalid cover file for '%s', default cover substituted.\n" % (title['title'])) + Generate an individual genre HTML file. Called from generate_html_by_genres() - thumb_generated = False + Args: + genre (str): genre name + section_head (bool): True if starting section + books (dict): list of books in genre + outfile (str): full pathname to output file - if not thumb_generated: - self.opts.log.warn(" using default cover for '%s' (%d)" % (title['title'], title['id'])) - # Confirm thumb exists, default is current - default_thumb_fp = os.path.join(image_dir,"thumbnail_default.jpg") - cover = os.path.join(self.catalogPath, "DefaultCover.png") - title['cover'] = cover + Results: + (file): Genre HTML file written - if not os.path.exists(cover): - shutil.copyfile(I('book.png'), cover) + Returns: + titles_spanned (list): [(first_author, first_book), (last_author, last_book)] + """ - if os.path.isfile(default_thumb_fp): - # Check to see if default cover is newer than thumbnail - # os.path.getmtime() = modified time - # os.path.ctime() = creation time - cover_timestamp = os.path.getmtime(cover) - thumb_timestamp = os.path.getmtime(default_thumb_fp) - if thumb_timestamp < cover_timestamp: - if False and self.verbose: - self.opts.log.warn("updating thumbnail_default for %s" % title['title']) - self.generateThumbnail(title, image_dir, - "thumbnail_default.jpg" if valid_cover else thumb_file) - else: - if False and self.verbose: - self.opts.log.warn(" generating new thumbnail_default.jpg") - self.generateThumbnail(title, image_dir, - "thumbnail_default.jpg" if valid_cover else thumb_file) - # Clear the book's cover property - title['cover'] = None - - - # Write thumb_width to the file, validating cache contents - # Allows detection of aborted catalog builds - with ZipFile(self.__archive_path, mode='a') as zfw: - zfw.writestr('thumb_width', self.opts.thumb_width) - - self.thumbs = thumbs - - def generateOPF(self): - - self.updateProgressFullStep("Generating OPF") - - header = ''' - <?xml version="1.0" encoding="UTF-8"?> - <package xmlns="http://www.idpf.org/2007/opf" version="2.0" unique-identifier="calibre_id"> - <metadata xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:opf="http://www.idpf.org/2007/opf" xmlns:calibre="http://calibre.kovidgoyal.net/2009/metadata" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"> - <dc:language>en-US</dc:language> - <meta name="calibre:publication_type" content="periodical:default"/> - </metadata> - <manifest></manifest> - <spine toc="ncx"></spine> - <guide></guide> - </package> - ''' - # Add the supplied metadata tags - soup = BeautifulStoneSoup(header, selfClosingTags=['item','itemref', 'reference']) - metadata = soup.find('metadata') - mtc = 0 - - titleTag = Tag(soup, "dc:title") - titleTag.insert(0,self.title) - metadata.insert(mtc, titleTag) - mtc += 1 - - creatorTag = Tag(soup, "dc:creator") - creatorTag.insert(0, self.creator) - metadata.insert(mtc, creatorTag) - mtc += 1 - - # Create the OPF tags - manifest = soup.find('manifest') - mtc = 0 - spine = soup.find('spine') - stc = 0 - guide = soup.find('guide') - - itemTag = Tag(soup, "item") - itemTag['id'] = "ncx" - itemTag['href'] = '%s.ncx' % self.basename - itemTag['media-type'] = "application/x-dtbncx+xml" - manifest.insert(mtc, itemTag) - mtc += 1 - - itemTag = Tag(soup, "item") - itemTag['id'] = 'stylesheet' - itemTag['href'] = self.stylesheet - itemTag['media-type'] = 'text/css' - manifest.insert(mtc, itemTag) - mtc += 1 - - itemTag = Tag(soup, "item") - itemTag['id'] = 'mastheadimage-image' - itemTag['href'] = "images/mastheadImage.gif" - itemTag['media-type'] = 'image/gif' - manifest.insert(mtc, itemTag) - mtc += 1 - - # Write the thumbnail images, descriptions to the manifest - sort_descriptions_by = [] - if self.opts.generate_descriptions: - for thumb in self.thumbs: - itemTag = Tag(soup, "item") - itemTag['href'] = "images/%s" % (thumb) - end = thumb.find('.jpg') - itemTag['id'] = "%s-image" % thumb[:end] - itemTag['media-type'] = 'image/jpeg' - manifest.insert(mtc, itemTag) - mtc += 1 - - # HTML files - add descriptions to manifest and spine - sort_descriptions_by = self.booksByAuthor if self.opts.sort_descriptions_by_author \ - else self.booksByTitle - # Add html_files to manifest and spine - - for file in self.htmlFileList_1: - # By Author, By Title, By Series, - itemTag = Tag(soup, "item") - start = file.find('/') + 1 - end = file.find('.') - itemTag['href'] = file - itemTag['id'] = file[start:end].lower() - itemTag['media-type'] = "application/xhtml+xml" - manifest.insert(mtc, itemTag) - mtc += 1 - - # spine - itemrefTag = Tag(soup, "itemref") - itemrefTag['idref'] = file[start:end].lower() - spine.insert(stc, itemrefTag) - stc += 1 - - # Add genre files to manifest and spine - for genre in self.genres: - if False: self.opts.log.info("adding %s to manifest and spine" % genre['tag']) - itemTag = Tag(soup, "item") - start = genre['file'].find('/') + 1 - end = genre['file'].find('.') - itemTag['href'] = genre['file'] - itemTag['id'] = genre['file'][start:end].lower() - itemTag['media-type'] = "application/xhtml+xml" - manifest.insert(mtc, itemTag) - mtc += 1 - - # spine - itemrefTag = Tag(soup, "itemref") - itemrefTag['idref'] = genre['file'][start:end].lower() - spine.insert(stc, itemrefTag) - stc += 1 - - for file in self.htmlFileList_2: - # By Date Added, By Date Read - itemTag = Tag(soup, "item") - start = file.find('/') + 1 - end = file.find('.') - itemTag['href'] = file - itemTag['id'] = file[start:end].lower() - itemTag['media-type'] = "application/xhtml+xml" - manifest.insert(mtc, itemTag) - mtc += 1 - - # spine - itemrefTag = Tag(soup, "itemref") - itemrefTag['idref'] = file[start:end].lower() - spine.insert(stc, itemrefTag) - stc += 1 - - for book in sort_descriptions_by: - # manifest - itemTag = Tag(soup, "item") - itemTag['href'] = "content/book_%d.html" % int(book['id']) - itemTag['id'] = "book%d" % int(book['id']) - itemTag['media-type'] = "application/xhtml+xml" - manifest.insert(mtc, itemTag) - mtc += 1 - - # spine - itemrefTag = Tag(soup, "itemref") - itemrefTag['idref'] = "book%d" % int(book['id']) - spine.insert(stc, itemrefTag) - stc += 1 - - # Guide - referenceTag = Tag(soup, "reference") - referenceTag['type'] = 'masthead' - referenceTag['title'] = 'mastheadimage-image' - referenceTag['href'] = 'images/mastheadImage.gif' - guide.insert(0,referenceTag) - - # Write the OPF file - outfile = open("%s/%s.opf" % (self.catalogPath, self.basename), 'w') - outfile.write(soup.prettify()) - - def generateNCXHeader(self): - - self.updateProgressFullStep("NCX header") - - header = ''' - <?xml version="1.0" encoding="utf-8"?> - <ncx xmlns="http://www.daisy.org/z3986/2005/ncx/" xmlns:calibre="http://calibre.kovidgoyal.net/2009/metadata" version="2005-1" xml:lang="en"> - </ncx> - ''' - soup = BeautifulStoneSoup(header, selfClosingTags=['content','calibre:meta-img']) - - ncx = soup.find('ncx') - navMapTag = Tag(soup, 'navMap') - navPointTag = Tag(soup, 'navPoint') - navPointTag['class'] = "periodical" - navPointTag['id'] = "title" - navPointTag['playOrder'] = self.playOrder - self.playOrder += 1 - navLabelTag = Tag(soup, 'navLabel') - textTag = Tag(soup, 'text') - textTag.insert(0, NavigableString(self.title)) - navLabelTag.insert(0, textTag) - navPointTag.insert(0, navLabelTag) - - if self.opts.generate_authors: - contentTag = Tag(soup, 'content') - contentTag['src'] = "content/ByAlphaAuthor.html" - navPointTag.insert(1, contentTag) - elif self.opts.generate_titles: - contentTag = Tag(soup, 'content') - contentTag['src'] = "content/ByAlphaTitle.html" - navPointTag.insert(1, contentTag) - elif self.opts.generate_series: - contentTag = Tag(soup, 'content') - contentTag['src'] = "content/BySeries.html" - navPointTag.insert(1, contentTag) - elif self.opts.generate_genres: - contentTag = Tag(soup, 'content') - #contentTag['src'] = "content/ByGenres.html" - contentTag['src'] = "%s" % self.genres[0]['file'] - navPointTag.insert(1, contentTag) - elif self.opts.generate_recently_added: - contentTag = Tag(soup, 'content') - contentTag['src'] = "content/ByDateAdded.html" - navPointTag.insert(1, contentTag) - else: - # Descriptions only - sort_descriptions_by = self.booksByAuthor if self.opts.sort_descriptions_by_author \ - else self.booksByTitle - contentTag = Tag(soup, 'content') - contentTag['src'] = "content/book_%d.html" % int(sort_descriptions_by[0]['id']) - navPointTag.insert(1, contentTag) - - cmiTag = Tag(soup, '%s' % 'calibre:meta-img') - cmiTag['id'] = "mastheadImage" - cmiTag['src'] = "images/mastheadImage.gif" - navPointTag.insert(2,cmiTag) - navMapTag.insert(0,navPointTag) - - ncx.insert(0,navMapTag) - self.ncxSoup = soup - - def generateNCXDescriptions(self, tocTitle): - - self.updateProgressFullStep("NCX 'Descriptions'") - - # --- Construct the 'Books by Title' section --- - ncx_soup = self.ncxSoup - body = ncx_soup.find("navPoint") - btc = len(body.contents) - - # Add the section navPoint - navPointTag = Tag(ncx_soup, 'navPoint') - navPointTag['class'] = "section" - navPointTag['id'] = "bytitle-ID" - navPointTag['playOrder'] = self.playOrder - self.playOrder += 1 - navLabelTag = Tag(ncx_soup, 'navLabel') - textTag = Tag(ncx_soup, 'text') - textTag.insert(0, NavigableString(tocTitle)) - navLabelTag.insert(0, textTag) - nptc = 0 - navPointTag.insert(nptc, navLabelTag) - nptc += 1 - contentTag = Tag(ncx_soup,"content") - contentTag['src'] = "content/book_%d.html" % int(self.booksByTitle[0]['id']) - navPointTag.insert(nptc, contentTag) - nptc += 1 - - # Loop over the titles - sort_descriptions_by = self.booksByAuthor if self.opts.sort_descriptions_by_author \ - else self.booksByTitle - - for book in sort_descriptions_by: - navPointVolumeTag = Tag(ncx_soup, 'navPoint') - navPointVolumeTag['class'] = "article" - navPointVolumeTag['id'] = "book%dID" % int(book['id']) - navPointVolumeTag['playOrder'] = self.playOrder - self.playOrder += 1 - navLabelTag = Tag(ncx_soup, "navLabel") - textTag = Tag(ncx_soup, "text") - if book['series']: - series_index = str(book['series_index']) - if series_index.endswith('.0'): - series_index = series_index[:-2] - if self.generateForKindle: - # Don't include Author for Kindle - textTag.insert(0, NavigableString(self.formatNCXText('%s (%s [%s])' % - (book['title'], book['series'], series_index), dest='title'))) - else: - # Include Author for non-Kindle - textTag.insert(0, NavigableString(self.formatNCXText('%s (%s [%s]) · %s ' % - (book['title'], book['series'], series_index, book['author']), dest='title'))) - else: - if self.generateForKindle: - # Don't include Author for Kindle - title_str = self.formatNCXText('%s' % (book['title']), dest='title') - if self.opts.connected_kindle and book['id'] in self.bookmarked_books: - ''' - dots = int((book['percent_read'] + 5)/10) - dot_string = '+' * dots - empty_dots = '-' * (10 - dots) - title_str += ' %s%s' % (dot_string,empty_dots) - ''' - title_str += '*' - textTag.insert(0, NavigableString(title_str)) - else: - # Include Author for non-Kindle - textTag.insert(0, NavigableString(self.formatNCXText('%s · %s' % \ - (book['title'], book['author']), dest='title'))) - navLabelTag.insert(0,textTag) - navPointVolumeTag.insert(0,navLabelTag) - - contentTag = Tag(ncx_soup, "content") - contentTag['src'] = "content/book_%d.html#book%d" % (int(book['id']), int(book['id'])) - navPointVolumeTag.insert(1, contentTag) - - if self.generateForKindle: - # Add the author tag - cmTag = Tag(ncx_soup, '%s' % 'calibre:meta') - cmTag['name'] = "author" - - if book['date']: - navStr = '%s | %s' % (self.formatNCXText(book['author'], dest='author'), - book['date'].split()[1]) - else: - navStr = '%s' % (self.formatNCXText(book['author'], dest='author')) - - if 'tags' in book and len(book['tags']): - navStr = self.formatNCXText(navStr + ' | ' + ' · '.join(sorted(book['tags'])), dest='author') - cmTag.insert(0, NavigableString(navStr)) - navPointVolumeTag.insert(2, cmTag) - - # Add the description tag - if book['short_description']: - cmTag = Tag(ncx_soup, '%s' % 'calibre:meta') - cmTag['name'] = "description" - cmTag.insert(0, NavigableString(self.formatNCXText(book['short_description'], dest='description'))) - navPointVolumeTag.insert(3, cmTag) - - # Add this volume to the section tag - navPointTag.insert(nptc, navPointVolumeTag) - nptc += 1 - - # Add this section to the body - body.insert(btc, navPointTag) - btc += 1 - - self.ncxSoup = ncx_soup - - def generateNCXBySeries(self, tocTitle): - self.updateProgressFullStep("NCX 'Series'") - - def add_to_series_by_letter(current_series_list): - current_series_list = " • ".join(current_series_list) - current_series_list = self.formatNCXText(current_series_list, dest="description") - series_by_letter.append(current_series_list) - - soup = self.ncxSoup - output = "BySeries" - body = soup.find("navPoint") - btc = len(body.contents) - - # --- Construct the 'Books By Series' section --- - navPointTag = Tag(soup, 'navPoint') - navPointTag['class'] = "section" - navPointTag['id'] = "byseries-ID" - navPointTag['playOrder'] = self.playOrder - self.playOrder += 1 - navLabelTag = Tag(soup, 'navLabel') - textTag = Tag(soup, 'text') - textTag.insert(0, NavigableString(tocTitle)) - navLabelTag.insert(0, textTag) - nptc = 0 - navPointTag.insert(nptc, navLabelTag) - nptc += 1 - contentTag = Tag(soup,"content") - contentTag['src'] = "content/%s.html#section_start" % (output) - navPointTag.insert(nptc, contentTag) - nptc += 1 - - series_by_letter = [] - # Establish initial letter equivalencies - sort_equivalents = self.establish_equivalencies(self.booksBySeries, key='series_sort') - - # Loop over the series titles, find start of each letter, add description_preview_count books - # Special switch for using different title list - - title_list = self.booksBySeries - - # Prime the pump - current_letter = self.letter_or_symbol(sort_equivalents[0]) - - title_letters = [current_letter] - current_series_list = [] - current_series = "" - for idx, book in enumerate(title_list): - sort_title = self.generateSortTitle(book['series']) - self.establish_equivalencies([sort_title])[0] - if self.letter_or_symbol(sort_equivalents[idx]) != current_letter: - - # Save the old list - add_to_series_by_letter(current_series_list) - - # Start the new list - current_letter = self.letter_or_symbol(sort_equivalents[idx]) - title_letters.append(current_letter) - current_series = book['series'] - current_series_list = [book['series']] - else: - if len(current_series_list) < self.descriptionClip and \ - book['series'] != current_series : - current_series = book['series'] - current_series_list.append(book['series']) - - # Add the last book list - add_to_series_by_letter(current_series_list) - - # Add *article* entries for each populated series title letter - for (i,books) in enumerate(series_by_letter): - navPointByLetterTag = Tag(soup, 'navPoint') - navPointByLetterTag['class'] = "article" - navPointByLetterTag['id'] = "%sSeries-ID" % (title_letters[i].upper()) - navPointTag['playOrder'] = self.playOrder - self.playOrder += 1 - navLabelTag = Tag(soup, 'navLabel') - textTag = Tag(soup, 'text') - textTag.insert(0, NavigableString(_(u"Series beginning with %s") % \ - (title_letters[i] if len(title_letters[i])>1 else "'" + title_letters[i] + "'"))) - navLabelTag.insert(0, textTag) - navPointByLetterTag.insert(0,navLabelTag) - contentTag = Tag(soup, 'content') - #contentTag['src'] = "content/%s.html#%s_series" % (output, title_letters[i]) - if title_letters[i] == self.SYMBOLS: - contentTag['src'] = "content/%s.html#%s_series" % (output, self.SYMBOLS) - else: - contentTag['src'] = "content/%s.html#%s_series" % (output, self.generateUnicodeName(title_letters[i])) - - navPointByLetterTag.insert(1,contentTag) - - if self.generateForKindle: - cmTag = Tag(soup, '%s' % 'calibre:meta') - cmTag['name'] = "description" - cmTag.insert(0, NavigableString(self.formatNCXText(books, dest='description'))) - navPointByLetterTag.insert(2, cmTag) - - navPointTag.insert(nptc, navPointByLetterTag) - nptc += 1 - - # Add this section to the body - body.insert(btc, navPointTag) - btc += 1 - - self.ncxSoup = soup - - def generateNCXByTitle(self, tocTitle): - self.updateProgressFullStep("NCX 'Titles'") - - def add_to_books_by_letter(current_book_list): - current_book_list = " • ".join(current_book_list) - current_book_list = self.formatNCXText(current_book_list, dest="description") - books_by_letter.append(current_book_list) - - soup = self.ncxSoup - output = "ByAlphaTitle" - body = soup.find("navPoint") - btc = len(body.contents) - - # --- Construct the 'Books By Title' section --- - navPointTag = Tag(soup, 'navPoint') - navPointTag['class'] = "section" - navPointTag['id'] = "byalphatitle-ID" - navPointTag['playOrder'] = self.playOrder - self.playOrder += 1 - navLabelTag = Tag(soup, 'navLabel') - textTag = Tag(soup, 'text') - textTag.insert(0, NavigableString(tocTitle)) - navLabelTag.insert(0, textTag) - nptc = 0 - navPointTag.insert(nptc, navLabelTag) - nptc += 1 - contentTag = Tag(soup,"content") - contentTag['src'] = "content/%s.html#section_start" % (output) - navPointTag.insert(nptc, contentTag) - nptc += 1 - - books_by_letter = [] - - # Establish initial letter equivalencies - sort_equivalents = self.establish_equivalencies(self.booksByTitle, key='title_sort') - - # Loop over the titles, find start of each letter, add description_preview_count books - # Special switch for using different title list - if self.useSeriesPrefixInTitlesSection: - title_list = self.booksByTitle - else: - title_list = self.booksByTitle_noSeriesPrefix - - # Prime the list - current_letter = self.letter_or_symbol(sort_equivalents[0]) - title_letters = [current_letter] - current_book_list = [] - current_book = "" - for idx, book in enumerate(title_list): - #if self.letter_or_symbol(book['title_sort'][0]) != current_letter: - if self.letter_or_symbol(sort_equivalents[idx]) != current_letter: - - # Save the old list - add_to_books_by_letter(current_book_list) - - # Start the new list - #current_letter = self.letter_or_symbol(book['title_sort'][0]) - current_letter = self.letter_or_symbol(sort_equivalents[idx]) - title_letters.append(current_letter) - current_book = book['title'] - current_book_list = [book['title']] - else: - if len(current_book_list) < self.descriptionClip and \ - book['title'] != current_book : - current_book = book['title'] - current_book_list.append(book['title']) - - # Add the last book list - add_to_books_by_letter(current_book_list) - - # Add *article* entries for each populated title letter - for (i,books) in enumerate(books_by_letter): - navPointByLetterTag = Tag(soup, 'navPoint') - navPointByLetterTag['class'] = "article" - navPointByLetterTag['id'] = "%sTitles-ID" % (title_letters[i].upper()) - navPointTag['playOrder'] = self.playOrder - self.playOrder += 1 - navLabelTag = Tag(soup, 'navLabel') - textTag = Tag(soup, 'text') - textTag.insert(0, NavigableString(_(u"Titles beginning with %s") % \ - (title_letters[i] if len(title_letters[i])>1 else "'" + title_letters[i] + "'"))) - navLabelTag.insert(0, textTag) - navPointByLetterTag.insert(0,navLabelTag) - contentTag = Tag(soup, 'content') - if title_letters[i] == self.SYMBOLS: - contentTag['src'] = "content/%s.html#%s_titles" % (output, self.SYMBOLS) - else: - contentTag['src'] = "content/%s.html#%s_titles" % (output, self.generateUnicodeName(title_letters[i])) - navPointByLetterTag.insert(1,contentTag) - - if self.generateForKindle: - cmTag = Tag(soup, '%s' % 'calibre:meta') - cmTag['name'] = "description" - cmTag.insert(0, NavigableString(self.formatNCXText(books, dest='description'))) - navPointByLetterTag.insert(2, cmTag) - - navPointTag.insert(nptc, navPointByLetterTag) - nptc += 1 - - # Add this section to the body - body.insert(btc, navPointTag) - btc += 1 - - self.ncxSoup = soup - - def generateNCXByAuthor(self, tocTitle): - self.updateProgressFullStep("NCX 'Authors'") - - def add_to_author_list(current_author_list, current_letter): - current_author_list = " • ".join(current_author_list) - current_author_list = self.formatNCXText(current_author_list, dest="description") - master_author_list.append((current_author_list, current_letter)) - - soup = self.ncxSoup - HTML_file = "content/ByAlphaAuthor.html" - body = soup.find("navPoint") - btc = len(body.contents) - - # --- Construct the 'Books By Author' *section* --- - navPointTag = Tag(soup, 'navPoint') - navPointTag['class'] = "section" - file_ID = "%s" % tocTitle.lower() - file_ID = file_ID.replace(" ","") - navPointTag['id'] = "%s-ID" % file_ID - navPointTag['playOrder'] = self.playOrder - self.playOrder += 1 - navLabelTag = Tag(soup, 'navLabel') - textTag = Tag(soup, 'text') - textTag.insert(0, NavigableString('%s' % tocTitle)) - navLabelTag.insert(0, textTag) - nptc = 0 - navPointTag.insert(nptc, navLabelTag) - nptc += 1 - contentTag = Tag(soup,"content") - contentTag['src'] = "%s#section_start" % HTML_file - navPointTag.insert(nptc, contentTag) - nptc += 1 - - # Create an NCX article entry for each populated author index letter - # Loop over the sorted_authors list, find start of each letter, - # add description_preview_count artists - # self.authors[0]:friendly [1]:author_sort [2]:book_count - # (<friendly name>, author_sort, book_count) - - # Need to extract a list of author_sort, generate sort_equivalents from that - sort_equivalents = self.establish_equivalencies([x[1] for x in self.authors]) - - master_author_list = [] - # Prime the pump - current_letter = self.letter_or_symbol(sort_equivalents[0]) - current_author_list = [] - for idx, author in enumerate(self.authors): - if self.letter_or_symbol(sort_equivalents[idx]) != current_letter: - # Save the old list - add_to_author_list(current_author_list, current_letter) - - # Start the new list - current_letter = self.letter_or_symbol(sort_equivalents[idx]) - current_author_list = [author[0]] - else: - if len(current_author_list) < self.descriptionClip: - current_author_list.append(author[0]) - - # Add the last author list - add_to_author_list(current_author_list, current_letter) - - # Add *article* entries for each populated author initial letter - # master_author_list{}: [0]:author list [1]:Initial letter - for authors_by_letter in master_author_list: - navPointByLetterTag = Tag(soup, 'navPoint') - navPointByLetterTag['class'] = "article" - navPointByLetterTag['id'] = "%sauthors-ID" % (authors_by_letter[1]) - navPointTag['playOrder'] = self.playOrder - self.playOrder += 1 - navLabelTag = Tag(soup, 'navLabel') - textTag = Tag(soup, 'text') - textTag.insert(0, NavigableString(_("Authors beginning with '%s'") % (authors_by_letter[1]))) - navLabelTag.insert(0, textTag) - navPointByLetterTag.insert(0,navLabelTag) - contentTag = Tag(soup, 'content') - if authors_by_letter[1] == self.SYMBOLS: - contentTag['src'] = "%s#%s_authors" % (HTML_file, authors_by_letter[1]) - else: - contentTag['src'] = "%s#%s_authors" % (HTML_file, self.generateUnicodeName(authors_by_letter[1])) - navPointByLetterTag.insert(1,contentTag) - - if self.generateForKindle: - cmTag = Tag(soup, '%s' % 'calibre:meta') - cmTag['name'] = "description" - cmTag.insert(0, NavigableString(authors_by_letter[0])) - navPointByLetterTag.insert(2, cmTag) - - navPointTag.insert(nptc, navPointByLetterTag) - nptc += 1 - - # Add this section to the body - body.insert(btc, navPointTag) - btc += 1 - - self.ncxSoup = soup - - def generateNCXByDateAdded(self, tocTitle): - self.updateProgressFullStep("NCX 'Recently Added'") - - def add_to_master_month_list(current_titles_list): - book_count = len(current_titles_list) - current_titles_list = " • ".join(current_titles_list) - current_titles_list = self.formatNCXText(current_titles_list, dest='description') - master_month_list.append((current_titles_list, current_date, book_count)) - - def add_to_master_date_range_list(current_titles_list): - book_count = len(current_titles_list) - current_titles_list = " • ".join(current_titles_list) - current_titles_list = self.formatNCXText(current_titles_list, dest='description') - master_date_range_list.append((current_titles_list, date_range, book_count)) - - soup = self.ncxSoup - HTML_file = "content/ByDateAdded.html" - body = soup.find("navPoint") - btc = len(body.contents) - - # --- Construct the 'Recently Added' *section* --- - navPointTag = Tag(soup, 'navPoint') - navPointTag['class'] = "section" - file_ID = "%s" % tocTitle.lower() - file_ID = file_ID.replace(" ","") - navPointTag['id'] = "%s-ID" % file_ID - navPointTag['playOrder'] = self.playOrder - self.playOrder += 1 - navLabelTag = Tag(soup, 'navLabel') - textTag = Tag(soup, 'text') - textTag.insert(0, NavigableString('%s' % tocTitle)) - navLabelTag.insert(0, textTag) - nptc = 0 - navPointTag.insert(nptc, navLabelTag) - nptc += 1 - contentTag = Tag(soup,"content") - contentTag['src'] = "%s#section_start" % HTML_file - navPointTag.insert(nptc, contentTag) - nptc += 1 - - # Create an NCX article entry for each date range - current_titles_list = [] - master_date_range_list = [] - today = datetime.datetime.now() - today_time = datetime.datetime(today.year, today.month, today.day) - for (i,date) in enumerate(self.DATE_RANGE): - if i: - date_range = '%d to %d days ago' % (self.DATE_RANGE[i-1], self.DATE_RANGE[i]) - else: - date_range = 'Last %d days' % (self.DATE_RANGE[i]) - date_range_limit = self.DATE_RANGE[i] - for book in self.booksByDateRange: - book_time = datetime.datetime(book['timestamp'].year, book['timestamp'].month, book['timestamp'].day) - if (today_time-book_time).days <= date_range_limit: - #print "generateNCXByDateAdded: %s added %d days ago" % (book['title'], (today_time-book_time).days) - current_titles_list.append(book['title']) - else: - break - if current_titles_list: - add_to_master_date_range_list(current_titles_list) - current_titles_list = [book['title']] - - # Add *article* entries for each populated date range - # master_date_range_list{}: [0]:titles list [1]:datestr - for books_by_date_range in master_date_range_list: - navPointByDateRangeTag = Tag(soup, 'navPoint') - navPointByDateRangeTag['class'] = "article" - navPointByDateRangeTag['id'] = "%s-ID" % books_by_date_range[1].replace(' ','') - navPointTag['playOrder'] = self.playOrder - self.playOrder += 1 - navLabelTag = Tag(soup, 'navLabel') - textTag = Tag(soup, 'text') - textTag.insert(0, NavigableString(books_by_date_range[1])) - navLabelTag.insert(0, textTag) - navPointByDateRangeTag.insert(0,navLabelTag) - contentTag = Tag(soup, 'content') - contentTag['src'] = "%s#bda_%s" % (HTML_file, - books_by_date_range[1].replace(' ','')) - - navPointByDateRangeTag.insert(1,contentTag) - - if self.generateForKindle: - cmTag = Tag(soup, '%s' % 'calibre:meta') - cmTag['name'] = "description" - cmTag.insert(0, NavigableString(books_by_date_range[0])) - navPointByDateRangeTag.insert(2, cmTag) - - cmTag = Tag(soup, '%s' % 'calibre:meta') - cmTag['name'] = "author" - navStr = '%d titles' % books_by_date_range[2] if books_by_date_range[2] > 1 else \ - '%d title' % books_by_date_range[2] - cmTag.insert(0, NavigableString(navStr)) - navPointByDateRangeTag.insert(3, cmTag) - - navPointTag.insert(nptc, navPointByDateRangeTag) - nptc += 1 - - - - # Create an NCX article entry for each populated month - # Loop over the booksByDate list, find start of each month, - # add description_preview_count titles - # master_month_list(list,date,count) - current_titles_list = [] - master_month_list = [] - current_date = self.booksByMonth[0]['timestamp'] - - for book in self.booksByMonth: - if book['timestamp'].month != current_date.month or \ - book['timestamp'].year != current_date.year: - # Save the old lists - add_to_master_month_list(current_titles_list) - - # Start the new list - current_date = book['timestamp'].date() - current_titles_list = [book['title']] - else: - current_titles_list.append(book['title']) - - # Add the last month list - add_to_master_month_list(current_titles_list) - - # Add *article* entries for each populated month - # master_months_list{}: [0]:titles list [1]:date - for books_by_month in master_month_list: - datestr = strftime(u'%B %Y', books_by_month[1].timetuple()) - navPointByMonthTag = Tag(soup, 'navPoint') - navPointByMonthTag['class'] = "article" - navPointByMonthTag['id'] = "bda_%s-%s-ID" % (books_by_month[1].year,books_by_month[1].month ) - navPointTag['playOrder'] = self.playOrder - self.playOrder += 1 - navLabelTag = Tag(soup, 'navLabel') - textTag = Tag(soup, 'text') - textTag.insert(0, NavigableString(datestr)) - navLabelTag.insert(0, textTag) - navPointByMonthTag.insert(0,navLabelTag) - contentTag = Tag(soup, 'content') - contentTag['src'] = "%s#bda_%s-%s" % (HTML_file, - books_by_month[1].year,books_by_month[1].month) - - navPointByMonthTag.insert(1,contentTag) - - if self.generateForKindle: - cmTag = Tag(soup, '%s' % 'calibre:meta') - cmTag['name'] = "description" - cmTag.insert(0, NavigableString(books_by_month[0])) - navPointByMonthTag.insert(2, cmTag) - - cmTag = Tag(soup, '%s' % 'calibre:meta') - cmTag['name'] = "author" - navStr = '%d titles' % books_by_month[2] if books_by_month[2] > 1 else \ - '%d title' % books_by_month[2] - cmTag.insert(0, NavigableString(navStr)) - navPointByMonthTag.insert(3, cmTag) - - navPointTag.insert(nptc, navPointByMonthTag) - nptc += 1 - - # Add this section to the body - body.insert(btc, navPointTag) - btc += 1 - self.ncxSoup = soup - - def generateNCXByDateRead(self, tocTitle): - self.updateProgressFullStep("NCX 'Recently Read'") - if not self.booksByDateRead: - return - - def add_to_master_day_list(current_titles_list): - book_count = len(current_titles_list) - current_titles_list = " • ".join(current_titles_list) - current_titles_list = self.formatNCXText(current_titles_list, dest='description') - master_day_list.append((current_titles_list, current_date, book_count)) - - def add_to_master_date_range_list(current_titles_list): - book_count = len(current_titles_list) - current_titles_list = " • ".join(current_titles_list) - current_titles_list = self.formatNCXText(current_titles_list, dest='description') - master_date_range_list.append((current_titles_list, date_range, book_count)) - - soup = self.ncxSoup - HTML_file = "content/ByDateRead.html" - body = soup.find("navPoint") - btc = len(body.contents) - - # --- Construct the 'Recently Read' *section* --- - navPointTag = Tag(soup, 'navPoint') - navPointTag['class'] = "section" - file_ID = "%s" % tocTitle.lower() - file_ID = file_ID.replace(" ","") - navPointTag['id'] = "%s-ID" % file_ID - navPointTag['playOrder'] = self.playOrder - self.playOrder += 1 - navLabelTag = Tag(soup, 'navLabel') - textTag = Tag(soup, 'text') - textTag.insert(0, NavigableString('%s' % tocTitle)) - navLabelTag.insert(0, textTag) - nptc = 0 - navPointTag.insert(nptc, navLabelTag) - nptc += 1 - contentTag = Tag(soup,"content") - contentTag['src'] = "%s#section_start" % HTML_file - navPointTag.insert(nptc, contentTag) - nptc += 1 - - # Create an NCX article entry for each date range - current_titles_list = [] - master_date_range_list = [] - today = datetime.datetime.now() - today_time = datetime.datetime(today.year, today.month, today.day) - for (i,date) in enumerate(self.DATE_RANGE): - if i: - date_range = '%d to %d days ago' % (self.DATE_RANGE[i-1], self.DATE_RANGE[i]) - else: - date_range = 'Last %d days' % (self.DATE_RANGE[i]) - date_range_limit = self.DATE_RANGE[i] - for book in self.booksByDateRead: - bookmark_time = datetime.datetime.utcfromtimestamp(book['bookmark_timestamp']) - if (today_time-bookmark_time).days <= date_range_limit: - #print "generateNCXByDateAdded: %s added %d days ago" % (book['title'], (today_time-book_time).days) - current_titles_list.append(book['title']) - else: - break - if current_titles_list: - add_to_master_date_range_list(current_titles_list) - current_titles_list = [book['title']] - - # Create an NCX article entry for each populated day - # Loop over the booksByDate list, find start of each month, - # add description_preview_count titles - # master_month_list(list,date,count) - current_titles_list = [] - master_day_list = [] - current_date = datetime.datetime.utcfromtimestamp(self.booksByDateRead[0]['bookmark_timestamp']) - - for book in self.booksByDateRead: - bookmark_time = datetime.datetime.utcfromtimestamp(book['bookmark_timestamp']) - if bookmark_time.day != current_date.day or \ - bookmark_time.month != current_date.month or \ - bookmark_time.year != current_date.year: - # Save the old lists - add_to_master_day_list(current_titles_list) - - # Start the new list - current_date = datetime.datetime.utcfromtimestamp(book['bookmark_timestamp']).date() - current_titles_list = [book['title']] - else: - current_titles_list.append(book['title']) - - # Add the last day list - add_to_master_day_list(current_titles_list) - - # Add *article* entries for each populated day - # master_day_list{}: [0]:titles list [1]:date - for books_by_day in master_day_list: - datestr = strftime(u'%A, %B %d', books_by_day[1].timetuple()) - navPointByDayTag = Tag(soup, 'navPoint') - navPointByDayTag['class'] = "article" - navPointByDayTag['id'] = "bdr_%s-%s-%sID" % (books_by_day[1].year, - books_by_day[1].month, - books_by_day[1].day ) - navPointTag['playOrder'] = self.playOrder - self.playOrder += 1 - navLabelTag = Tag(soup, 'navLabel') - textTag = Tag(soup, 'text') - textTag.insert(0, NavigableString(datestr)) - navLabelTag.insert(0, textTag) - navPointByDayTag.insert(0,navLabelTag) - contentTag = Tag(soup, 'content') - contentTag['src'] = "%s#bdr_%s-%s-%s" % (HTML_file, - books_by_day[1].year, - books_by_day[1].month, - books_by_day[1].day) - - navPointByDayTag.insert(1,contentTag) - - if self.generateForKindle: - cmTag = Tag(soup, '%s' % 'calibre:meta') - cmTag['name'] = "description" - cmTag.insert(0, NavigableString(books_by_day[0])) - navPointByDayTag.insert(2, cmTag) - - cmTag = Tag(soup, '%s' % 'calibre:meta') - cmTag['name'] = "author" - navStr = '%d titles' % books_by_day[2] if books_by_day[2] > 1 else \ - '%d title' % books_by_day[2] - cmTag.insert(0, NavigableString(navStr)) - navPointByDayTag.insert(3, cmTag) - - navPointTag.insert(nptc, navPointByDayTag) - nptc += 1 - - # Add this section to the body - body.insert(btc, navPointTag) - btc += 1 - self.ncxSoup = soup - - def generateNCXByGenre(self, tocTitle): - # Create an NCX section for 'By Genre' - # Add each genre as an article - # 'tag', 'file', 'authors' - - self.updateProgressFullStep("NCX 'Genres'") - - if not len(self.genres): - self.opts.log.warn(" No genres found in tags.\n" - " No Genre section added to Catalog") - return - - ncx_soup = self.ncxSoup - body = ncx_soup.find("navPoint") - btc = len(body.contents) - - # --- Construct the 'Books By Genre' *section* --- - navPointTag = Tag(ncx_soup, 'navPoint') - navPointTag['class'] = "section" - file_ID = "%s" % tocTitle.lower() - file_ID = file_ID.replace(" ","") - navPointTag['id'] = "%s-ID" % file_ID - navPointTag['playOrder'] = self.playOrder - self.playOrder += 1 - navLabelTag = Tag(ncx_soup, 'navLabel') - textTag = Tag(ncx_soup, 'text') - # textTag.insert(0, NavigableString('%s (%d)' % (section_title, len(genre_list)))) - textTag.insert(0, NavigableString('%s' % tocTitle)) - navLabelTag.insert(0, textTag) - nptc = 0 - navPointTag.insert(nptc, navLabelTag) - nptc += 1 - contentTag = Tag(ncx_soup,"content") - contentTag['src'] = "content/Genre_%s.html#section_start" % self.genres[0]['tag'] - navPointTag.insert(nptc, contentTag) - nptc += 1 - - for genre in self.genres: - # Add an article for each genre - navPointVolumeTag = Tag(ncx_soup, 'navPoint') - navPointVolumeTag['class'] = "article" - navPointVolumeTag['id'] = "genre-%s-ID" % genre['tag'] - navPointVolumeTag['playOrder'] = self.playOrder - self.playOrder += 1 - navLabelTag = Tag(ncx_soup, "navLabel") - textTag = Tag(ncx_soup, "text") - - # GwR *** Can this be optimized? - normalized_tag = None - for friendly_tag in self.genre_tags_dict: - if self.genre_tags_dict[friendly_tag] == genre['tag']: - normalized_tag = self.genre_tags_dict[friendly_tag] - break - textTag.insert(0, self.formatNCXText(NavigableString(friendly_tag), dest='description')) - navLabelTag.insert(0,textTag) - navPointVolumeTag.insert(0,navLabelTag) - contentTag = Tag(ncx_soup, "content") - contentTag['src'] = "content/Genre_%s.html#Genre_%s" % (normalized_tag, normalized_tag) - navPointVolumeTag.insert(1, contentTag) - - if self.generateForKindle: - # Build the author tag - cmTag = Tag(ncx_soup, '%s' % 'calibre:meta') - cmTag['name'] = "author" - # First - Last author - - if len(genre['titles_spanned']) > 1 : - author_range = "%s - %s" % (genre['titles_spanned'][0][0], genre['titles_spanned'][1][0]) - else : - author_range = "%s" % (genre['titles_spanned'][0][0]) - - cmTag.insert(0, NavigableString(author_range)) - navPointVolumeTag.insert(2, cmTag) - - # Build the description tag - cmTag = Tag(ncx_soup, '%s' % 'calibre:meta') - cmTag['name'] = "description" - - if False: - # Form 1: Titles spanned - if len(genre['titles_spanned']) > 1: - title_range = "%s -\n%s" % (genre['titles_spanned'][0][1], genre['titles_spanned'][1][1]) - else: - title_range = "%s" % (genre['titles_spanned'][0][1]) - cmTag.insert(0, NavigableString(self.formatNCXText(title_range, dest='description'))) - else: - # Form 2: title • title • title ... - titles = [] - for title in genre['books']: - titles.append(title['title']) - titles = sorted(titles, key=lambda x:(self.generateSortTitle(x),self.generateSortTitle(x))) - titles_list = self.generateShortDescription(u" • ".join(titles), dest="description") - cmTag.insert(0, NavigableString(self.formatNCXText(titles_list, dest='description'))) - - navPointVolumeTag.insert(3, cmTag) - - # Add this volume to the section tag - navPointTag.insert(nptc, navPointVolumeTag) - nptc += 1 - - # Add this section to the body - body.insert(btc, navPointTag) - btc += 1 - self.ncxSoup = ncx_soup - - def writeNCX(self): - self.updateProgressFullStep("Saving NCX") - - outfile = open("%s/%s.ncx" % (self.catalogPath, self.basename), 'w') - outfile.write(self.ncxSoup.prettify()) - - - # ======================== Helpers ======================== - def author_to_author_sort(self, author): - tokens = author.split() - tokens = tokens[-1:] + tokens[:-1] - if len(tokens) > 1: - tokens[0] += ',' - return ' '.join(tokens).capitalize() - - def booksByAuthorSorter_author_sort(self, book): - ''' - Sort non-series books before series books - ''' - if not book['series']: - key = '%s ~%s' % (capitalize(book['author_sort']), - capitalize(book['title_sort'])) - else: - index = book['series_index'] - integer = int(index) - fraction = index-integer - series_index = '%04d%s' % (integer, str('%0.4f' % fraction).lstrip('0')) - key = '%s %s %s' % (capitalize(book['author_sort']), - self.generateSortTitle(book['series']), - series_index) - return key - - def booksByAuthorSorter_author(self, book): - ''' - Sort non-series books before series books - ''' - if not book['series']: - key = '%s %s' % (self.author_to_author_sort(book['author']), - capitalize(book['title_sort'])) - else: - index = book['series_index'] - integer = int(index) - fraction = index-integer - series_index = '%04d%s' % (integer, str('%0.4f' % fraction).lstrip('0')) - key = '%s ~%s %s' % (self.author_to_author_sort(book['author']), - self.generateSortTitle(book['series']), - series_index) - return key - - def calculateThumbnailSize(self): - ''' Calculate thumbnail dimensions based on device DPI. Scale Kindle by 50% ''' - from calibre.customize.ui import output_profiles - for x in output_profiles(): - if x.short_name == self.opts.output_profile: - # aspect ratio: 3:4 - self.thumbWidth = x.dpi * float(self.opts.thumb_width) - self.thumbHeight = self.thumbWidth * 1.33 - if 'kindle' in x.short_name and self.opts.fmt == 'mobi': - # Kindle DPI appears to be off by a factor of 2 - self.thumbWidth = self.thumbWidth/2 - self.thumbHeight = self.thumbHeight/2 - break - if True and self.verbose: - self.opts.log(" DPI = %d; thumbnail dimensions: %d x %d" % \ - (x.dpi, self.thumbWidth, self.thumbHeight)) - - def convertHTMLEntities(self, s): - matches = re.findall("&#\d+;", s) - if len(matches) > 0: - hits = set(matches) - for hit in hits: - name = hit[2:-1] - try: - entnum = int(name) - s = s.replace(hit, unichr(entnum)) - except ValueError: - pass - - matches = re.findall("&\w+;", s) - hits = set(matches) - amp = "&" - if amp in hits: - hits.remove(amp) - for hit in hits: - name = hit[1:-1] - if htmlentitydefs.name2codepoint.has_key(name): - s = s.replace(hit, unichr(htmlentitydefs.name2codepoint[name])) - s = s.replace(amp, "&") - return s - - def createDirectoryStructure(self): - catalogPath = self.catalogPath - self.cleanUp() - - if not os.path.isdir(catalogPath): - os.makedirs(catalogPath) - - # Create /content and /images - content_path = catalogPath + "/content" - if not os.path.isdir(content_path): - os.makedirs(content_path) - images_path = catalogPath + "/images" - if not os.path.isdir(images_path): - os.makedirs(images_path) - - def discoverPrefix(self, record): - ''' - Evaluate conditions for including prefixes in various listings - ''' - def log_prefix_rule_match_info(rule, record): - self.opts.log.info(" %s %s by %s (Prefix rule '%s': %s:%s)" % - (rule['prefix'],record['title'], - record['authors'][0], rule['name'], - rule['field'],rule['pattern'])) - - # Compare the record to each rule looking for a match - for rule in self.prefixRules: - # Literal comparison for Tags field - if rule['field'].lower() == 'tags': - if rule['pattern'].lower() in map(unicode.lower,record['tags']): - if self.opts.verbose: - log_prefix_rule_match_info(rule, record) - return rule['prefix'] - - # Regex match for custom field - elif rule['field'].startswith('#'): - field_contents = self.__db.get_field(record['id'], - rule['field'], - index_is_id=True) - if field_contents == '': - field_contents = None - - if field_contents is not None: - try: - if re.search(rule['pattern'], unicode(field_contents), - re.IGNORECASE) is not None: - if self.opts.verbose: - log_prefix_rule_match_info(rule, record) - return rule['prefix'] - except: - # Compiling of pat failed, ignore it - if self.opts.verbose: - self.opts.log.error("pattern failed to compile: %s" % rule['pattern']) - pass - elif field_contents is None and rule['pattern'] == 'None': - if self.opts.verbose: - log_prefix_rule_match_info(rule, record) - return rule['prefix'] - - return None - - def establish_equivalencies(self, item_list, key=None): - # Filter for leading letter equivalencies - - # Hack to force the cataloged leading letter to be - # an unadorned character if the accented version sorts before the unaccented - exceptions = { - u'Ä':u'A', - u'Ö':u'O', - u'Ãœ':u'U' - } - - if key is not None: - sort_field = key - - cl_list = [None] * len(item_list) - last_ordnum = 0 - - for idx, item in enumerate(item_list): - if key: - c = item[sort_field] - else: - c = item - - ordnum, ordlen = collation_order(c) - if last_ordnum != ordnum: - last_c = icu_upper(c[0:ordlen]) - if last_c in exceptions.keys(): - last_c = exceptions[unicode(last_c)] - last_ordnum = ordnum - cl_list[idx] = last_c - - if False: - if key: - for idx, item in enumerate(item_list): - print("%s %s" % (cl_list[idx],item[sort_field])) - else: - print("%s %s" % (cl_list[0], item)) - - return cl_list - - def filterDbTags(self, tags): - # Remove the special marker tags from the database's tag list, - # return sorted list of normalized genre tags - - def format_tag_list(tags, indent=5, line_break=70, header='Tag list'): - def next_tag(sorted_tags): - for (i, tag) in enumerate(sorted_tags): - if i < len(tags) - 1: - yield tag + ", " - else: - yield tag - - ans = '%s%d %s:\n' % (' ' * indent, len(tags), header) - ans += ' ' * (indent + 1) - out_str = '' - sorted_tags = sorted(tags, key=sort_key) - for tag in next_tag(sorted_tags): - out_str += tag - if len(out_str) >= line_break: - ans += out_str + '\n' - out_str = ' ' * (indent + 1) - return ans + out_str - - normalized_tags = [] - friendly_tags = [] - excluded_tags = [] - for tag in tags: - if tag in self.markerTags: - excluded_tags.append(tag) - continue - try: - if re.search(self.opts.exclude_genre, tag): - excluded_tags.append(tag) - continue - except: - self.opts.log.error("\tfilterDbTags(): malformed --exclude-genre regex pattern: %s" % self.opts.exclude_genre) - - if tag == ' ': - continue - - normalized_tags.append(re.sub('\W','',ascii_text(tag)).lower()) - friendly_tags.append(tag) - - genre_tags_dict = dict(zip(friendly_tags,normalized_tags)) - - # Test for multiple genres resolving to same normalized form - normalized_set = set(normalized_tags) - for normalized in normalized_set: - if normalized_tags.count(normalized) > 1: - self.opts.log.warn(" Warning: multiple tags resolving to genre '%s':" % normalized) - for key in genre_tags_dict: - if genre_tags_dict[key] == normalized: - self.opts.log.warn(" %s" % key) - if self.verbose: - self.opts.log.info('%s' % format_tag_list(genre_tags_dict, header="enabled genre tags in database")) - self.opts.log.info('%s' % format_tag_list(excluded_tags, header="excluded genre tags")) - - return genre_tags_dict - - def formatNCXText(self, description, dest=None): - # Kindle TOC descriptions won't render certain characters - # Fix up - massaged = unicode(BeautifulStoneSoup(description, convertEntities=BeautifulStoneSoup.HTML_ENTITIES)) - - # Replace '&' with '&' - massaged = re.sub("&","&", massaged) - - if massaged.strip() and dest: - #print traceback.print_stack(limit=3) - return self.generateShortDescription(massaged.strip(), dest=dest) - else: - return None - - def formatPrefix(self,prefix_char,soup): - # Generate the HTML for the prefix portion of the listing - # Kindle Previewer doesn't properly handle style=color:white - # MOBI does a better job allocating blank space with <code> - if self.opts.fmt == 'mobi': - codeTag = Tag(soup, "code") - if prefix_char is None: - codeTag.insert(0,NavigableString(' ')) - else: - codeTag.insert(0,NavigableString(prefix_char)) - return codeTag - else: - spanTag = Tag(soup, "span") - spanTag['class'] = "prefix" - - # color:white was the original technique used to align columns. - # The new technique is to float the prefix left with CSS. - if prefix_char is None: - if True: - prefix_char = " " - else: - del spanTag['class'] - spanTag['style'] = "color:white" - prefix_char = self.defaultPrefix - spanTag.insert(0,NavigableString(prefix_char)) - return spanTag - - def generateAuthorAnchor(self, author): - # Generate a legal XHTML id/href string - return re.sub("\W","", ascii_text(author)) - - def generateFormatArgs(self, book): - series_index = str(book['series_index']) - if series_index.endswith('.0'): - series_index = series_index[:-2] - args = dict( - title = book['title'], - series = book['series'], - series_index = series_index, - rating = self.generateRatingString(book), - rating_parens = '(%s)' % self.generateRatingString(book) if 'rating' in book else '', - pubyear = book['date'].split()[1] if book['date'] else '', - pubyear_parens = "(%s)" % book['date'].split()[1] if book['date'] else '') - return args - - def generateHTMLByGenre(self, genre, section_head, books, outfile): - # Write an HTML file of this genre's book list - # Return a list with [(first_author, first_book), (last_author, last_book)] - - soup = self.generateHTMLGenreHeader(genre) + soup = self.generate_html_genre_header(genre) body = soup.find('body') btc = 0 divTag = Tag(soup, 'div') dtc = 0 - # Insert section tag if this is the section start - first article only if section_head: aTag = Tag(soup,'a') @@ -3409,7 +2265,7 @@ Author '{0}': btc += 1 titleTag = body.find(attrs={'class':'title'}) - titleTag.insert(0,NavigableString('%s' % escape(self.getFriendlyGenreTag(genre)))) + titleTag.insert(0,NavigableString('%s' % escape(self.get_friendly_genre_tag(genre)))) # Insert the books by author list divTag = body.find(attrs={'class':'authors'}) @@ -3427,7 +2283,7 @@ Author '{0}': pAuthorTag['class'] = "author_index" aTag = Tag(soup, "a") if self.opts.generate_authors: - aTag['href'] = "%s.html#%s" % ("ByAlphaAuthor", self.generateAuthorAnchor(book['author'])) + aTag['href'] = "%s.html#%s" % ("ByAlphaAuthor", self.generate_author_anchor(book['author'])) aTag.insert(0, book['author']) pAuthorTag.insert(0,aTag) divTag.insert(dtc,pAuthorTag) @@ -3443,7 +2299,7 @@ Author '{0}': pSeriesTag['class'] = "series_mobi" if self.opts.generate_series: aTag = Tag(soup,'a') - aTag['href'] = "%s.html#%s" % ('BySeries', self.generateSeriesAnchor(book['series'])) + aTag['href'] = "%s.html#%s" % ('BySeries', self.generate_series_anchor(book['series'])) aTag.insert(0, book['series']) pSeriesTag.insert(0, aTag) else: @@ -3459,7 +2315,7 @@ Author '{0}': pBookTag['class'] = "line_item" ptc = 0 - pBookTag.insert(ptc, self.formatPrefix(book['prefix'],soup)) + pBookTag.insert(ptc, self.format_prefix(book['prefix'])) ptc += 1 spanTag = Tag(soup, "span") @@ -3472,7 +2328,7 @@ Author '{0}': aTag['href'] = "book_%d.html" % (int(float(book['id']))) # Generate the title from the template - args = self.generateFormatArgs(book) + args = self.generate_format_args(book) if current_series: #aTag.insert(0,escape(book['title'][len(book['series'])+1:])) formatted_title = self.by_genres_series_title_template.format(**args).rstrip() @@ -3491,7 +2347,7 @@ Author '{0}': divTag.insert(dtc, pBookTag) dtc += 1 - # Write the generated file to contentdir + # Write the generated file to content_dir outfile = open(outfile, 'w') outfile.write(soup.prettify()) outfile.close() @@ -3503,13 +2359,350 @@ Author '{0}': return titles_spanned - def generateHTMLDescriptionHeader(self, book): - ''' - Generate description header from template - ''' + def generate_html_by_series(self): + """ Generate content/BySeries.html. + + Search database for books in series. + + Input: + database + + Output: + content/BySeries.html (file) + + To do: + self.books_by_series = [i for i in self.books_by_title if i['series']] + """ + friendly_name = _("Series") + self.update_progress_full_step("%s HTML" % friendly_name) + + self.opts.sort_by = 'series' + + # Merge self.excluded_tags with opts.search_text + # Updated to use exact match syntax + + search_phrase = 'series:true ' + if self.excluded_tags: + search_terms = [] + for tag in self.excluded_tags: + search_terms.append("tag:=%s" % tag) + search_phrase += "not (%s)" % " or ".join(search_terms) + + # If a list of ids are provided, don't use search_text + if self.opts.ids: + self.opts.search_text = search_phrase + else: + if self.opts.search_text: + self.opts.search_text += " " + search_phrase + else: + self.opts.search_text = search_phrase + + # Fetch the database as a dictionary + data = self.plugin.search_sort_db(self.db, self.opts) + + # Remove exclusions + self.books_by_series = self.process_exclusions(data, log_exclusion=False) + + if not self.books_by_series: + self.opts.generate_series = False + self.opts.log(" no series found in selected books, cancelling series generation") + return + + # Generate series_sort + for book in self.books_by_series: + book['series_sort'] = self.generate_sort_title(book['series']) + + soup = self.generate_html_empty_header(friendly_name) + body = soup.find('body') + + btc = 0 + divTag = Tag(soup, "div") + dtc = 0 + current_letter = "" + current_series = None + + # Establish initial letter equivalencies + sort_equivalents = self.establish_equivalencies(self.books_by_series, key='series_sort') + + # Loop through books_by_series + series_count = 0 + for idx, book in enumerate(self.books_by_series): + # Check for initial letter change + if self.letter_or_symbol(sort_equivalents[idx]) != current_letter : + # Start a new letter with Index letter + current_letter = self.letter_or_symbol(sort_equivalents[idx]) + pIndexTag = Tag(soup, "p") + pIndexTag['class'] = "series_letter_index" + aTag = Tag(soup, "a") + if current_letter == self.SYMBOLS: + aTag['id'] = self.SYMBOLS + "_series" + pIndexTag.insert(0,aTag) + pIndexTag.insert(1,NavigableString(self.SYMBOLS)) + else: + aTag['id'] = self.generate_unicode_name(current_letter) + "_series" + pIndexTag.insert(0,aTag) + pIndexTag.insert(1,NavigableString(sort_equivalents[idx])) + divTag.insert(dtc,pIndexTag) + dtc += 1 + # Check for series change + if book['series'] != current_series: + # Start a new series + series_count += 1 + current_series = book['series'] + pSeriesTag = Tag(soup,'p') + pSeriesTag['class'] = "series" + if self.opts.fmt == 'mobi': + pSeriesTag['class'] = "series_mobi" + aTag = Tag(soup, 'a') + aTag['id'] = self.generate_series_anchor(book['series']) + pSeriesTag.insert(0,aTag) + pSeriesTag.insert(1,NavigableString('%s' % book['series'])) + divTag.insert(dtc,pSeriesTag) + dtc += 1 + + # Add books + pBookTag = Tag(soup, "p") + pBookTag['class'] = "line_item" + ptc = 0 + + book['prefix'] = self.discover_prefix(book) + pBookTag.insert(ptc, self.format_prefix(book['prefix'])) + ptc += 1 + + spanTag = Tag(soup, "span") + spanTag['class'] = "entry" + stc = 0 + + aTag = Tag(soup, "a") + if self.opts.generate_descriptions: + aTag['href'] = "book_%d.html" % (int(float(book['id']))) + # Use series, series index if avail else just title + #aTag.insert(0,'%d. %s · %s' % (book['series_index'],escape(book['title']), ' & '.join(book['authors']))) + + if is_date_undefined(book['pubdate']): + book['date'] = None + else: + book['date'] = strftime(u'%B %Y', book['pubdate'].timetuple()) + + args = self.generate_format_args(book) + formatted_title = self.by_series_title_template.format(**args).rstrip() + aTag.insert(0,NavigableString(escape(formatted_title))) + + spanTag.insert(stc, aTag) + stc += 1 + + # · + spanTag.insert(stc, NavigableString(' · ')) + stc += 1 + + # Link to author + aTag = Tag(soup, "a") + if self.opts.generate_authors: + aTag['href'] = "%s.html#%s" % ("ByAlphaAuthor", + self.generate_author_anchor(escape(' & '.join(book['authors'])))) + aTag.insert(0, NavigableString(' & '.join(book['authors']))) + spanTag.insert(stc, aTag) + stc += 1 + + pBookTag.insert(ptc, spanTag) + ptc += 1 + + divTag.insert(dtc, pBookTag) + dtc += 1 + + pTag = Tag(soup, "p") + pTag['class'] = 'title' + ptc = 0 + aTag = Tag(soup,'a') + aTag['id'] = 'section_start' + pTag.insert(ptc, aTag) + ptc += 1 + + if not self.generate_for_kindle: + # Insert the <h2> tag with book_count at the head + aTag = Tag(soup, "a") + anchor_name = friendly_name.lower() + aTag['id'] = anchor_name.replace(" ","") + pTag.insert(0,aTag) + pTag.insert(1,NavigableString('%s' % friendly_name)) + body.insert(btc,pTag) + btc += 1 + + # Add the divTag to the body + body.insert(btc, divTag) + + # Write the generated file to content_dir + outfile_spec = "%s/BySeries.html" % (self.content_dir) + outfile = open(outfile_spec, 'w') + outfile.write(soup.prettify()) + outfile.close() + self.html_filelist_1.append("content/BySeries.html") + + def generate_html_by_title(self): + """ Generate content/ByAlphaTitle.html. + + Generate HTML of books sorted by title. + + Input: + books_by_title + + Output: + content/ByAlphaTitle.html (file) + """ + + self.update_progress_full_step(_("Titles HTML")) + + soup = self.generate_html_empty_header("Books By Alpha Title") + body = soup.find('body') + btc = 0 + + pTag = Tag(soup, "p") + pTag['class'] = 'title' + ptc = 0 + aTag = Tag(soup,'a') + aTag['id'] = 'section_start' + pTag.insert(ptc, aTag) + ptc += 1 + + if not self.generate_for_kindle: + # Kindle don't need this because it shows section titles in Periodical format + aTag = Tag(soup, "a") + aTag['id'] = "bytitle" + pTag.insert(ptc,aTag) + ptc += 1 + pTag.insert(ptc,NavigableString(_('Titles'))) + + body.insert(btc,pTag) + btc += 1 + + divTag = Tag(soup, "div") + dtc = 0 + current_letter = "" + + # Re-sort title list without leading series/series_index + # Incoming title <series> <series_index>: <title> + if not self.use_series_prefix_in_titles_section: + nspt = deepcopy(self.books_by_title) + nspt = sorted(nspt, key=lambda x: sort_key(x['title_sort'].upper())) + self.books_by_title_no_series_prefix = nspt + + # Establish initial letter equivalencies + sort_equivalents = self.establish_equivalencies(self.books_by_title, key='title_sort') + + # Loop through the books by title + # Generate one divRunningTag per initial letter for the purposes of + # minimizing widows and orphans on readers that can handle large + # <divs> styled as inline-block + title_list = self.books_by_title + if not self.use_series_prefix_in_titles_section: + title_list = self.books_by_title_no_series_prefix + drtc = 0 + divRunningTag = None + for idx, book in enumerate(title_list): + if self.letter_or_symbol(sort_equivalents[idx]) != current_letter: + # Start a new letter + if drtc and divRunningTag is not None: + divTag.insert(dtc, divRunningTag) + dtc += 1 + divRunningTag = Tag(soup, 'div') + if dtc > 0: + divRunningTag['class'] = "initial_letter" + drtc = 0 + pIndexTag = Tag(soup, "p") + pIndexTag['class'] = "author_title_letter_index" + aTag = Tag(soup, "a") + current_letter = self.letter_or_symbol(sort_equivalents[idx]) + if current_letter == self.SYMBOLS: + aTag['id'] = self.SYMBOLS + "_titles" + pIndexTag.insert(0,aTag) + pIndexTag.insert(1,NavigableString(self.SYMBOLS)) + else: + aTag['id'] = self.generate_unicode_name(current_letter) + "_titles" + pIndexTag.insert(0,aTag) + pIndexTag.insert(1,NavigableString(sort_equivalents[idx])) + divRunningTag.insert(dtc,pIndexTag) + drtc += 1 + + # Add books + pBookTag = Tag(soup, "p") + pBookTag['class'] = "line_item" + ptc = 0 + + pBookTag.insert(ptc, self.format_prefix(book['prefix'])) + ptc += 1 + + spanTag = Tag(soup, "span") + spanTag['class'] = "entry" + stc = 0 + + # Link to book + aTag = Tag(soup, "a") + if self.opts.generate_descriptions: + aTag['href'] = "book_%d.html" % (int(float(book['id']))) + + # Generate the title from the template + args = self.generate_format_args(book) + if book['series']: + formatted_title = self.by_titles_series_title_template.format(**args).rstrip() + else: + formatted_title = self.by_titles_normal_title_template.format(**args).rstrip() + aTag.insert(0,NavigableString(escape(formatted_title))) + spanTag.insert(stc, aTag) + stc += 1 + + # Dot + spanTag.insert(stc, NavigableString(" · ")) + stc += 1 + + # Link to author + emTag = Tag(soup, "em") + aTag = Tag(soup, "a") + if self.opts.generate_authors: + aTag['href'] = "%s.html#%s" % ("ByAlphaAuthor", self.generate_author_anchor(book['author'])) + aTag.insert(0, NavigableString(book['author'])) + emTag.insert(0,aTag) + spanTag.insert(stc, emTag) + stc += 1 + + pBookTag.insert(ptc, spanTag) + ptc += 1 + + if divRunningTag is not None: + divRunningTag.insert(drtc, pBookTag) + drtc += 1 + + # Add the last divRunningTag to divTag + if divRunningTag is not None: + divTag.insert(dtc, divRunningTag) + dtc += 1 + + # Add the divTag to the body + body.insert(btc, divTag) + btc += 1 + + # Write the volume to content_dir + outfile_spec = "%s/ByAlphaTitle.html" % (self.content_dir) + outfile = open(outfile_spec, 'w') + outfile.write(soup.prettify()) + outfile.close() + self.html_filelist_1.append("content/ByAlphaTitle.html") + + def generate_html_description_header(self, book): + """ Generate the HTML Description header from template. + + Create HTML Description from book metadata and template. + Called by generate_html_descriptions() + + Args: + book (dict): book metadata + + Return: + soup (BeautifulSoup): HTML Description for book + """ + from calibre.ebooks.oeb.base import XHTML_NS - def generate_html(): + def _generate_html(): args = dict( author=author, author_prefix=author_prefix, @@ -3554,7 +2747,7 @@ Author '{0}': if book['prefix']: author_prefix = book['prefix'] + ' ' + _("by ") elif self.opts.connected_kindle and book['id'] in self.bookmarked_books: - author_prefix = self.READING_SYMBOL + ' ' + _("by ") + author_prefix = self.SYMBOL_READING + ' ' + _("by ") else: author_prefix = _("by ") @@ -3567,7 +2760,7 @@ Author '{0}': for (i, tag) in enumerate(sorted(book.get('tags', []))): aTag = Tag(_soup,'a') if self.opts.generate_genres: - aTag['href'] = "Genre_%s.html" % re.sub("\W","",ascii_text(tag).lower()) + aTag['href'] = "Genre_%s.html" % self.normalize_tag(tag) aTag.insert(0,escape(NavigableString(tag))) genresTag.insert(gtc, aTag) gtc += 1 @@ -3608,8 +2801,8 @@ Author '{0}': stars = int(book['rating']) / 2 rating = '' if stars: - star_string = self.FULL_RATING_SYMBOL * stars - empty_stars = self.EMPTY_RATING_SYMBOL * (5 - stars) + star_string = self.SYMBOL_FULL_RATING * stars + empty_stars = self.SYMBOL_EMPTY_RATING * (5 - stars) rating = '%s%s <br/>' % (star_string,empty_stars) # Notes @@ -3624,10 +2817,8 @@ Author '{0}': if 'description' in book and book['description'] > '': comments = book['description'] - # >>>> Populate the template <<<< - soup = generate_html() - + soup = _generate_html() # >>>> Post-process the template <<<< body = soup.find('body') @@ -3645,7 +2836,7 @@ Author '{0}': if aTag: if book['series']: if self.opts.generate_series: - aTag['href'] = "%s.html#%s" % ('BySeries',self.generateSeriesAnchor(book['series'])) + aTag['href'] = "%s.html#%s" % ('BySeries',self.generate_series_anchor(book['series'])) else: aTag.extract() @@ -3653,7 +2844,7 @@ Author '{0}': aTag = body.find('a', attrs={'class':'author'}) if self.opts.generate_authors and aTag: aTag['href'] = "%s.html#%s" % ("ByAlphaAuthor", - self.generateAuthorAnchor(book['author'])) + self.generate_author_anchor(book['author'])) if publisher == ' ': publisherTag = body.find('td', attrs={'class':'publisher'}) @@ -3681,11 +2872,48 @@ Author '{0}': newEmptyTag.insert(0,NavigableString(' ')) mt.replaceWith(newEmptyTag) - if False: - print soup.prettify() return soup - def generateHTMLEmptyHeader(self, title): + def generate_html_descriptions(self): + """ Generate Description HTML for each book. + + Loop though books, write Description HTML for each book. + + Inputs: + books_by_title (list) + + Output: + (files): Description HTML for each book + """ + + self.update_progress_full_step(_("Descriptions HTML")) + + for (title_num, title) in enumerate(self.books_by_title): + self.update_progress_micro_step("%s %d of %d" % + (_("Description HTML"), + title_num, len(self.books_by_title)), + float(title_num*100/len(self.books_by_title))/100) + + # Generate the header from user-customizable template + soup = self.generate_html_description_header(title) + + # Write the book entry to content_dir + outfile = open("%s/book_%d.html" % (self.content_dir, int(title['id'])), 'w') + outfile.write(soup.prettify()) + outfile.close() + + def generate_html_empty_header(self, title): + """ Return a boilerplate HTML header. + + Generate an HTML header with document title. + + Args: + title (str): document title + + Return: + soup (BeautifulSoup): HTML header with document title inserted + """ + header = ''' <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd"> <html xmlns="http://www.w3.org/1999/xhtml" xmlns:calibre="http://calibre.kovidgoyal.net/2009/metadata"> @@ -3704,30 +2932,50 @@ Author '{0}': titleTag.insert(0,NavigableString(title)) return soup - def generateHTMLGenreHeader(self, title): - header = ''' - <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd"> - <html xmlns="http://www.w3.org/1999/xhtml" xmlns:calibre="http://calibre.kovidgoyal.net/2009/metadata"> - <head> - <meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /> - <link rel="stylesheet" type="text/css" href="stylesheet.css" media="screen" /> - <title> - - -

-
- - - ''' - # Insert the supplied title - soup = BeautifulSoup(header) - titleTag = soup.find('title') - titleTag.insert(0,escape(NavigableString(title))) + def generate_html_genre_header(self, title): + """ Generate HTML header with initial body content + + Start with a generic HTML header, add

and

+ + Args: + title (str): document title + + Return: + soup (BeautifulSoup): HTML with initial

and

tags + """ + + soup = self.generate_html_empty_header(title) + bodyTag = soup.find('body') + pTag = Tag(soup, 'p') + pTag['class'] = 'title' + bodyTag.insert(0,pTag) + divTag = Tag(soup, 'div') + divTag['class'] = 'authors' + bodyTag.insert(1,divTag) return soup - def generateMastheadImage(self, out_path): + def generate_masthead_image(self, out_path): + """ Generate a Kindle masthead image. + + Generate a Kindle masthead image, used with Kindle periodical format. + + Args: + out_path (str): path to write generated masthead image + + Input: + opts.catalog_title (str): Title to render + masthead_font: User-specified font preference (MOBI output option) + + Output: + out_path (file): masthead image (GIF) + """ + from calibre.ebooks.conversion.config import load_defaults from calibre.utils.fonts import fontconfig + + MI_WIDTH = 600 + MI_HEIGHT = 60 + font_path = default_font = P('fonts/liberation/LiberationSerif-Bold.ttf') recs = load_defaults('mobi_output') masthead_font_family = recs.get('masthead_font', 'Default') @@ -3742,9 +2990,6 @@ Author '{0}': if not font_path or not os.access(font_path, os.R_OK): font_path = default_font - MI_WIDTH = 600 - MI_HEIGHT = 60 - try: from PIL import Image, ImageDraw, ImageFont Image, ImageDraw, ImageFont @@ -3758,42 +3003,1227 @@ Author '{0}': except: self.opts.log.error(" Failed to load user-specifed font '%s'" % font_path) font = ImageFont.truetype(default_font, 48) - text = self.title.encode('utf-8') + text = self.opts.catalog_title.encode('utf-8') width, height = draw.textsize(text, font=font) left = max(int((MI_WIDTH - width)/2.), 0) top = max(int((MI_HEIGHT - height)/2.), 0) draw.text((left, top), text, fill=(0,0,0), font=font) img.save(open(out_path, 'wb'), 'GIF') - def generateRatingString(self, book): + def generate_ncx_header(self): + """ Generate the basic NCX file. + + Generate the initial NCX, which is added to depending on included Sections. + + Inputs: + None + + Updated: + play_order (int) + + Outputs: + ncx_soup (file): NCX foundation + """ + + self.update_progress_full_step(_("NCX header")) + + header = ''' + + + + ''' + soup = BeautifulStoneSoup(header, selfClosingTags=['content','calibre:meta-img']) + + ncx = soup.find('ncx') + navMapTag = Tag(soup, 'navMap') + navPointTag = Tag(soup, 'navPoint') + navPointTag['class'] = "periodical" + navPointTag['id'] = "title" + navPointTag['playOrder'] = self.play_order + self.play_order += 1 + navLabelTag = Tag(soup, 'navLabel') + textTag = Tag(soup, 'text') + textTag.insert(0, NavigableString(self.opts.catalog_title)) + navLabelTag.insert(0, textTag) + navPointTag.insert(0, navLabelTag) + + if self.opts.generate_authors: + contentTag = Tag(soup, 'content') + contentTag['src'] = "content/ByAlphaAuthor.html" + navPointTag.insert(1, contentTag) + elif self.opts.generate_titles: + contentTag = Tag(soup, 'content') + contentTag['src'] = "content/ByAlphaTitle.html" + navPointTag.insert(1, contentTag) + elif self.opts.generate_series: + contentTag = Tag(soup, 'content') + contentTag['src'] = "content/BySeries.html" + navPointTag.insert(1, contentTag) + elif self.opts.generate_genres: + contentTag = Tag(soup, 'content') + #contentTag['src'] = "content/ByGenres.html" + contentTag['src'] = "%s" % self.genres[0]['file'] + navPointTag.insert(1, contentTag) + elif self.opts.generate_recently_added: + contentTag = Tag(soup, 'content') + contentTag['src'] = "content/ByDateAdded.html" + navPointTag.insert(1, contentTag) + else: + # Descriptions only + sort_descriptions_by = self.books_by_author if self.opts.sort_descriptions_by_author \ + else self.books_by_title + contentTag = Tag(soup, 'content') + contentTag['src'] = "content/book_%d.html" % int(sort_descriptions_by[0]['id']) + navPointTag.insert(1, contentTag) + + cmiTag = Tag(soup, '%s' % 'calibre:meta-img') + cmiTag['id'] = "mastheadImage" + cmiTag['src'] = "images/mastheadImage.gif" + navPointTag.insert(2,cmiTag) + navMapTag.insert(0,navPointTag) + + ncx.insert(0,navMapTag) + self.ncx_soup = soup + + def generate_ncx_descriptions(self, tocTitle): + """ Add Descriptions to the basic NCX file. + + Generate the Descriptions NCX content, add to self.ncx_soup. + + Inputs: + books_by_author (list) + + Updated: + play_order (int) + + Outputs: + ncx_soup (file): updated + """ + + self.update_progress_full_step(_("NCX for Descriptions")) + + # --- Construct the 'Books by Title' section --- + ncx_soup = self.ncx_soup + body = ncx_soup.find("navPoint") + btc = len(body.contents) + + # Add the section navPoint + navPointTag = Tag(ncx_soup, 'navPoint') + navPointTag['class'] = "section" + navPointTag['id'] = "bytitle-ID" + navPointTag['playOrder'] = self.play_order + self.play_order += 1 + navLabelTag = Tag(ncx_soup, 'navLabel') + textTag = Tag(ncx_soup, 'text') + textTag.insert(0, NavigableString(tocTitle)) + navLabelTag.insert(0, textTag) + nptc = 0 + navPointTag.insert(nptc, navLabelTag) + nptc += 1 + contentTag = Tag(ncx_soup,"content") + contentTag['src'] = "content/book_%d.html" % int(self.books_by_title[0]['id']) + navPointTag.insert(nptc, contentTag) + nptc += 1 + + # Loop over the titles + sort_descriptions_by = self.books_by_author if self.opts.sort_descriptions_by_author \ + else self.books_by_title + + for book in sort_descriptions_by: + navPointVolumeTag = Tag(ncx_soup, 'navPoint') + navPointVolumeTag['class'] = "article" + navPointVolumeTag['id'] = "book%dID" % int(book['id']) + navPointVolumeTag['playOrder'] = self.play_order + self.play_order += 1 + navLabelTag = Tag(ncx_soup, "navLabel") + textTag = Tag(ncx_soup, "text") + if book['series']: + series_index = str(book['series_index']) + if series_index.endswith('.0'): + series_index = series_index[:-2] + if self.generate_for_kindle: + # Don't include Author for Kindle + textTag.insert(0, NavigableString(self.format_ncx_text('%s (%s [%s])' % + (book['title'], book['series'], series_index), dest='title'))) + else: + # Include Author for non-Kindle + textTag.insert(0, NavigableString(self.format_ncx_text('%s (%s [%s]) · %s ' % + (book['title'], book['series'], series_index, book['author']), dest='title'))) + else: + if self.generate_for_kindle: + # Don't include Author for Kindle + title_str = self.format_ncx_text('%s' % (book['title']), dest='title') + if self.opts.connected_kindle and book['id'] in self.bookmarked_books: + ''' + dots = int((book['percent_read'] + 5)/10) + dot_string = '+' * dots + empty_dots = '-' * (10 - dots) + title_str += ' %s%s' % (dot_string,empty_dots) + ''' + title_str += '*' + textTag.insert(0, NavigableString(title_str)) + else: + # Include Author for non-Kindle + textTag.insert(0, NavigableString(self.format_ncx_text('%s · %s' % \ + (book['title'], book['author']), dest='title'))) + navLabelTag.insert(0,textTag) + navPointVolumeTag.insert(0,navLabelTag) + + contentTag = Tag(ncx_soup, "content") + contentTag['src'] = "content/book_%d.html#book%d" % (int(book['id']), int(book['id'])) + navPointVolumeTag.insert(1, contentTag) + + if self.generate_for_kindle: + # Add the author tag + cmTag = Tag(ncx_soup, '%s' % 'calibre:meta') + cmTag['name'] = "author" + + if book['date']: + navStr = '%s | %s' % (self.format_ncx_text(book['author'], dest='author'), + book['date'].split()[1]) + else: + navStr = '%s' % (self.format_ncx_text(book['author'], dest='author')) + + if 'tags' in book and len(book['tags']): + navStr = self.format_ncx_text(navStr + ' | ' + ' · '.join(sorted(book['tags'])), dest='author') + cmTag.insert(0, NavigableString(navStr)) + navPointVolumeTag.insert(2, cmTag) + + # Add the description tag + if book['short_description']: + cmTag = Tag(ncx_soup, '%s' % 'calibre:meta') + cmTag['name'] = "description" + cmTag.insert(0, NavigableString(self.format_ncx_text(book['short_description'], dest='description'))) + navPointVolumeTag.insert(3, cmTag) + + # Add this volume to the section tag + navPointTag.insert(nptc, navPointVolumeTag) + nptc += 1 + + # Add this section to the body + body.insert(btc, navPointTag) + btc += 1 + + self.ncx_soup = ncx_soup + + def generate_ncx_by_series(self, tocTitle): + """ Add Series to the basic NCX file. + + Generate the Series NCX content, add to self.ncx_soup. + + Inputs: + books_by_series (list) + + Updated: + play_order (int) + + Outputs: + ncx_soup (file): updated + """ + + self.update_progress_full_step(_("NCX for Series")) + + def _add_to_series_by_letter(current_series_list): + current_series_list = " • ".join(current_series_list) + current_series_list = self.format_ncx_text(current_series_list, dest="description") + series_by_letter.append(current_series_list) + + soup = self.ncx_soup + output = "BySeries" + body = soup.find("navPoint") + btc = len(body.contents) + + # --- Construct the 'Books By Series' section --- + navPointTag = Tag(soup, 'navPoint') + navPointTag['class'] = "section" + navPointTag['id'] = "byseries-ID" + navPointTag['playOrder'] = self.play_order + self.play_order += 1 + navLabelTag = Tag(soup, 'navLabel') + textTag = Tag(soup, 'text') + textTag.insert(0, NavigableString(tocTitle)) + navLabelTag.insert(0, textTag) + nptc = 0 + navPointTag.insert(nptc, navLabelTag) + nptc += 1 + contentTag = Tag(soup,"content") + contentTag['src'] = "content/%s.html#section_start" % (output) + navPointTag.insert(nptc, contentTag) + nptc += 1 + + series_by_letter = [] + # Establish initial letter equivalencies + sort_equivalents = self.establish_equivalencies(self.books_by_series, key='series_sort') + + # Loop over the series titles, find start of each letter, add description_preview_count books + # Special switch for using different title list + + title_list = self.books_by_series + + # Prime the pump + current_letter = self.letter_or_symbol(sort_equivalents[0]) + + title_letters = [current_letter] + current_series_list = [] + current_series = "" + for idx, book in enumerate(title_list): + sort_title = self.generate_sort_title(book['series']) + self.establish_equivalencies([sort_title])[0] + if self.letter_or_symbol(sort_equivalents[idx]) != current_letter: + + # Save the old list + _add_to_series_by_letter(current_series_list) + + # Start the new list + current_letter = self.letter_or_symbol(sort_equivalents[idx]) + title_letters.append(current_letter) + current_series = book['series'] + current_series_list = [book['series']] + else: + if len(current_series_list) < self.opts.description_clip and \ + book['series'] != current_series : + current_series = book['series'] + current_series_list.append(book['series']) + + # Add the last book list + _add_to_series_by_letter(current_series_list) + + # Add *article* entries for each populated series title letter + for (i,books) in enumerate(series_by_letter): + navPointByLetterTag = Tag(soup, 'navPoint') + navPointByLetterTag['class'] = "article" + navPointByLetterTag['id'] = "%sSeries-ID" % (title_letters[i].upper()) + navPointTag['playOrder'] = self.play_order + self.play_order += 1 + navLabelTag = Tag(soup, 'navLabel') + textTag = Tag(soup, 'text') + textTag.insert(0, NavigableString(_(u"Series beginning with %s") % \ + (title_letters[i] if len(title_letters[i])>1 else "'" + title_letters[i] + "'"))) + navLabelTag.insert(0, textTag) + navPointByLetterTag.insert(0,navLabelTag) + contentTag = Tag(soup, 'content') + #contentTag['src'] = "content/%s.html#%s_series" % (output, title_letters[i]) + if title_letters[i] == self.SYMBOLS: + contentTag['src'] = "content/%s.html#%s_series" % (output, self.SYMBOLS) + else: + contentTag['src'] = "content/%s.html#%s_series" % (output, self.generate_unicode_name(title_letters[i])) + + navPointByLetterTag.insert(1,contentTag) + + if self.generate_for_kindle: + cmTag = Tag(soup, '%s' % 'calibre:meta') + cmTag['name'] = "description" + cmTag.insert(0, NavigableString(self.format_ncx_text(books, dest='description'))) + navPointByLetterTag.insert(2, cmTag) + + navPointTag.insert(nptc, navPointByLetterTag) + nptc += 1 + + # Add this section to the body + body.insert(btc, navPointTag) + btc += 1 + + self.ncx_soup = soup + + def generate_ncx_by_title(self, tocTitle): + """ Add Titles to the basic NCX file. + + Generate the Titles NCX content, add to self.ncx_soup. + + Inputs: + books_by_title (list) + + Updated: + play_order (int) + + Outputs: + ncx_soup (file): updated + """ + + self.update_progress_full_step(_("NCX for Titles")) + + def _add_to_books_by_letter(current_book_list): + current_book_list = " • ".join(current_book_list) + current_book_list = self.format_ncx_text(current_book_list, dest="description") + books_by_letter.append(current_book_list) + + soup = self.ncx_soup + output = "ByAlphaTitle" + body = soup.find("navPoint") + btc = len(body.contents) + + # --- Construct the 'Books By Title' section --- + navPointTag = Tag(soup, 'navPoint') + navPointTag['class'] = "section" + navPointTag['id'] = "byalphatitle-ID" + navPointTag['playOrder'] = self.play_order + self.play_order += 1 + navLabelTag = Tag(soup, 'navLabel') + textTag = Tag(soup, 'text') + textTag.insert(0, NavigableString(tocTitle)) + navLabelTag.insert(0, textTag) + nptc = 0 + navPointTag.insert(nptc, navLabelTag) + nptc += 1 + contentTag = Tag(soup,"content") + contentTag['src'] = "content/%s.html#section_start" % (output) + navPointTag.insert(nptc, contentTag) + nptc += 1 + + books_by_letter = [] + + # Establish initial letter equivalencies + sort_equivalents = self.establish_equivalencies(self.books_by_title, key='title_sort') + + # Loop over the titles, find start of each letter, add description_preview_count books + # Special switch for using different title list + if self.use_series_prefix_in_titles_section: + title_list = self.books_by_title + else: + title_list = self.books_by_title_no_series_prefix + + # Prime the list + current_letter = self.letter_or_symbol(sort_equivalents[0]) + title_letters = [current_letter] + current_book_list = [] + current_book = "" + for idx, book in enumerate(title_list): + #if self.letter_or_symbol(book['title_sort'][0]) != current_letter: + if self.letter_or_symbol(sort_equivalents[idx]) != current_letter: + + # Save the old list + _add_to_books_by_letter(current_book_list) + + # Start the new list + #current_letter = self.letter_or_symbol(book['title_sort'][0]) + current_letter = self.letter_or_symbol(sort_equivalents[idx]) + title_letters.append(current_letter) + current_book = book['title'] + current_book_list = [book['title']] + else: + if len(current_book_list) < self.opts.description_clip and \ + book['title'] != current_book : + current_book = book['title'] + current_book_list.append(book['title']) + + # Add the last book list + _add_to_books_by_letter(current_book_list) + + # Add *article* entries for each populated title letter + for (i,books) in enumerate(books_by_letter): + navPointByLetterTag = Tag(soup, 'navPoint') + navPointByLetterTag['class'] = "article" + navPointByLetterTag['id'] = "%sTitles-ID" % (title_letters[i].upper()) + navPointTag['playOrder'] = self.play_order + self.play_order += 1 + navLabelTag = Tag(soup, 'navLabel') + textTag = Tag(soup, 'text') + textTag.insert(0, NavigableString(_(u"Titles beginning with %s") % \ + (title_letters[i] if len(title_letters[i])>1 else "'" + title_letters[i] + "'"))) + navLabelTag.insert(0, textTag) + navPointByLetterTag.insert(0,navLabelTag) + contentTag = Tag(soup, 'content') + if title_letters[i] == self.SYMBOLS: + contentTag['src'] = "content/%s.html#%s_titles" % (output, self.SYMBOLS) + else: + contentTag['src'] = "content/%s.html#%s_titles" % (output, self.generate_unicode_name(title_letters[i])) + navPointByLetterTag.insert(1,contentTag) + + if self.generate_for_kindle: + cmTag = Tag(soup, '%s' % 'calibre:meta') + cmTag['name'] = "description" + cmTag.insert(0, NavigableString(self.format_ncx_text(books, dest='description'))) + navPointByLetterTag.insert(2, cmTag) + + navPointTag.insert(nptc, navPointByLetterTag) + nptc += 1 + + # Add this section to the body + body.insert(btc, navPointTag) + btc += 1 + + self.ncx_soup = soup + + def generate_ncx_by_author(self, tocTitle): + """ Add Authors to the basic NCX file. + + Generate the Authors NCX content, add to self.ncx_soup. + + Inputs: + authors (list) + + Updated: + play_order (int) + + Outputs: + ncx_soup (file): updated + """ + + self.update_progress_full_step(_("NCX for Authors")) + + def _add_to_author_list(current_author_list, current_letter): + current_author_list = " • ".join(current_author_list) + current_author_list = self.format_ncx_text(current_author_list, dest="description") + master_author_list.append((current_author_list, current_letter)) + + soup = self.ncx_soup + HTML_file = "content/ByAlphaAuthor.html" + body = soup.find("navPoint") + btc = len(body.contents) + + # --- Construct the 'Books By Author' *section* --- + navPointTag = Tag(soup, 'navPoint') + navPointTag['class'] = "section" + file_ID = "%s" % tocTitle.lower() + file_ID = file_ID.replace(" ","") + navPointTag['id'] = "%s-ID" % file_ID + navPointTag['playOrder'] = self.play_order + self.play_order += 1 + navLabelTag = Tag(soup, 'navLabel') + textTag = Tag(soup, 'text') + textTag.insert(0, NavigableString('%s' % tocTitle)) + navLabelTag.insert(0, textTag) + nptc = 0 + navPointTag.insert(nptc, navLabelTag) + nptc += 1 + contentTag = Tag(soup,"content") + contentTag['src'] = "%s#section_start" % HTML_file + navPointTag.insert(nptc, contentTag) + nptc += 1 + + # Create an NCX article entry for each populated author index letter + # Loop over the sorted_authors list, find start of each letter, + # add description_preview_count artists + # self.authors[0]:friendly [1]:author_sort [2]:book_count + # (, author_sort, book_count) + + # Need to extract a list of author_sort, generate sort_equivalents from that + sort_equivalents = self.establish_equivalencies([x[1] for x in self.authors]) + + master_author_list = [] + # Prime the pump + current_letter = self.letter_or_symbol(sort_equivalents[0]) + current_author_list = [] + for idx, author in enumerate(self.authors): + if self.letter_or_symbol(sort_equivalents[idx]) != current_letter: + # Save the old list + _add_to_author_list(current_author_list, current_letter) + + # Start the new list + current_letter = self.letter_or_symbol(sort_equivalents[idx]) + current_author_list = [author[0]] + else: + if len(current_author_list) < self.opts.description_clip: + current_author_list.append(author[0]) + + # Add the last author list + _add_to_author_list(current_author_list, current_letter) + + # Add *article* entries for each populated author initial letter + # master_author_list{}: [0]:author list [1]:Initial letter + for authors_by_letter in master_author_list: + navPointByLetterTag = Tag(soup, 'navPoint') + navPointByLetterTag['class'] = "article" + navPointByLetterTag['id'] = "%sauthors-ID" % (authors_by_letter[1]) + navPointTag['playOrder'] = self.play_order + self.play_order += 1 + navLabelTag = Tag(soup, 'navLabel') + textTag = Tag(soup, 'text') + textTag.insert(0, NavigableString(_("Authors beginning with '%s'") % (authors_by_letter[1]))) + navLabelTag.insert(0, textTag) + navPointByLetterTag.insert(0,navLabelTag) + contentTag = Tag(soup, 'content') + if authors_by_letter[1] == self.SYMBOLS: + contentTag['src'] = "%s#%s_authors" % (HTML_file, authors_by_letter[1]) + else: + contentTag['src'] = "%s#%s_authors" % (HTML_file, self.generate_unicode_name(authors_by_letter[1])) + navPointByLetterTag.insert(1,contentTag) + + if self.generate_for_kindle: + cmTag = Tag(soup, '%s' % 'calibre:meta') + cmTag['name'] = "description" + cmTag.insert(0, NavigableString(authors_by_letter[0])) + navPointByLetterTag.insert(2, cmTag) + + navPointTag.insert(nptc, navPointByLetterTag) + nptc += 1 + + # Add this section to the body + body.insert(btc, navPointTag) + btc += 1 + + self.ncx_soup = soup + + def generate_ncx_by_date_added(self, tocTitle): + """ Add Recently Added to the basic NCX file. + + Generate the Recently Added NCX content, add to self.ncx_soup. + + Inputs: + books_by_date_range (list) + + Updated: + play_order (int) + + Outputs: + ncx_soup (file): updated + """ + + self.update_progress_full_step(_("NCX for Recently Added")) + + def _add_to_master_month_list(current_titles_list): + book_count = len(current_titles_list) + current_titles_list = " • ".join(current_titles_list) + current_titles_list = self.format_ncx_text(current_titles_list, dest='description') + master_month_list.append((current_titles_list, current_date, book_count)) + + def _add_to_master_date_range_list(current_titles_list): + book_count = len(current_titles_list) + current_titles_list = " • ".join(current_titles_list) + current_titles_list = self.format_ncx_text(current_titles_list, dest='description') + master_date_range_list.append((current_titles_list, date_range, book_count)) + + soup = self.ncx_soup + HTML_file = "content/ByDateAdded.html" + body = soup.find("navPoint") + btc = len(body.contents) + + # --- Construct the 'Recently Added' *section* --- + navPointTag = Tag(soup, 'navPoint') + navPointTag['class'] = "section" + file_ID = "%s" % tocTitle.lower() + file_ID = file_ID.replace(" ","") + navPointTag['id'] = "%s-ID" % file_ID + navPointTag['playOrder'] = self.play_order + self.play_order += 1 + navLabelTag = Tag(soup, 'navLabel') + textTag = Tag(soup, 'text') + textTag.insert(0, NavigableString('%s' % tocTitle)) + navLabelTag.insert(0, textTag) + nptc = 0 + navPointTag.insert(nptc, navLabelTag) + nptc += 1 + contentTag = Tag(soup,"content") + contentTag['src'] = "%s#section_start" % HTML_file + navPointTag.insert(nptc, contentTag) + nptc += 1 + + # Create an NCX article entry for each date range + current_titles_list = [] + master_date_range_list = [] + today = datetime.datetime.now() + today_time = datetime.datetime(today.year, today.month, today.day) + for (i,date) in enumerate(self.DATE_RANGE): + if i: + date_range = '%d to %d days ago' % (self.DATE_RANGE[i-1], self.DATE_RANGE[i]) + else: + date_range = 'Last %d days' % (self.DATE_RANGE[i]) + date_range_limit = self.DATE_RANGE[i] + for book in self.books_by_date_range: + book_time = datetime.datetime(book['timestamp'].year, book['timestamp'].month, book['timestamp'].day) + if (today_time-book_time).days <= date_range_limit: + #print "generate_ncx_by_date_added: %s added %d days ago" % (book['title'], (today_time-book_time).days) + current_titles_list.append(book['title']) + else: + break + if current_titles_list: + _add_to_master_date_range_list(current_titles_list) + current_titles_list = [book['title']] + + # Add *article* entries for each populated date range + # master_date_range_list{}: [0]:titles list [1]:datestr + for books_by_date_range in master_date_range_list: + navPointByDateRangeTag = Tag(soup, 'navPoint') + navPointByDateRangeTag['class'] = "article" + navPointByDateRangeTag['id'] = "%s-ID" % books_by_date_range[1].replace(' ','') + navPointTag['playOrder'] = self.play_order + self.play_order += 1 + navLabelTag = Tag(soup, 'navLabel') + textTag = Tag(soup, 'text') + textTag.insert(0, NavigableString(books_by_date_range[1])) + navLabelTag.insert(0, textTag) + navPointByDateRangeTag.insert(0,navLabelTag) + contentTag = Tag(soup, 'content') + contentTag['src'] = "%s#bda_%s" % (HTML_file, + books_by_date_range[1].replace(' ','')) + + navPointByDateRangeTag.insert(1,contentTag) + + if self.generate_for_kindle: + cmTag = Tag(soup, '%s' % 'calibre:meta') + cmTag['name'] = "description" + cmTag.insert(0, NavigableString(books_by_date_range[0])) + navPointByDateRangeTag.insert(2, cmTag) + + cmTag = Tag(soup, '%s' % 'calibre:meta') + cmTag['name'] = "author" + navStr = '%d titles' % books_by_date_range[2] if books_by_date_range[2] > 1 else \ + '%d title' % books_by_date_range[2] + cmTag.insert(0, NavigableString(navStr)) + navPointByDateRangeTag.insert(3, cmTag) + + navPointTag.insert(nptc, navPointByDateRangeTag) + nptc += 1 + + # Create an NCX article entry for each populated month + # Loop over the booksByDate list, find start of each month, + # add description_preview_count titles + # master_month_list(list,date,count) + current_titles_list = [] + master_month_list = [] + current_date = self.books_by_month[0]['timestamp'] + + for book in self.books_by_month: + if book['timestamp'].month != current_date.month or \ + book['timestamp'].year != current_date.year: + # Save the old lists + _add_to_master_month_list(current_titles_list) + + # Start the new list + current_date = book['timestamp'].date() + current_titles_list = [book['title']] + else: + current_titles_list.append(book['title']) + + # Add the last month list + _add_to_master_month_list(current_titles_list) + + # Add *article* entries for each populated month + # master_months_list{}: [0]:titles list [1]:date + for books_by_month in master_month_list: + datestr = strftime(u'%B %Y', books_by_month[1].timetuple()) + navPointByMonthTag = Tag(soup, 'navPoint') + navPointByMonthTag['class'] = "article" + navPointByMonthTag['id'] = "bda_%s-%s-ID" % (books_by_month[1].year,books_by_month[1].month ) + navPointTag['playOrder'] = self.play_order + self.play_order += 1 + navLabelTag = Tag(soup, 'navLabel') + textTag = Tag(soup, 'text') + textTag.insert(0, NavigableString(datestr)) + navLabelTag.insert(0, textTag) + navPointByMonthTag.insert(0,navLabelTag) + contentTag = Tag(soup, 'content') + contentTag['src'] = "%s#bda_%s-%s" % (HTML_file, + books_by_month[1].year,books_by_month[1].month) + + navPointByMonthTag.insert(1,contentTag) + + if self.generate_for_kindle: + cmTag = Tag(soup, '%s' % 'calibre:meta') + cmTag['name'] = "description" + cmTag.insert(0, NavigableString(books_by_month[0])) + navPointByMonthTag.insert(2, cmTag) + + cmTag = Tag(soup, '%s' % 'calibre:meta') + cmTag['name'] = "author" + navStr = '%d titles' % books_by_month[2] if books_by_month[2] > 1 else \ + '%d title' % books_by_month[2] + cmTag.insert(0, NavigableString(navStr)) + navPointByMonthTag.insert(3, cmTag) + + navPointTag.insert(nptc, navPointByMonthTag) + nptc += 1 + + # Add this section to the body + body.insert(btc, navPointTag) + btc += 1 + self.ncx_soup = soup + + def generate_ncx_by_date_read(self, tocTitle): + """ Add By Date Read to the basic NCX file. + + Generate the By Date Read NCX content (Kindle only), add to self.ncx_soup. + + Inputs: + bookmarked_books_by_date_read (list) + + Updated: + play_order (int) + + Outputs: + ncx_soup (file): updated + """ + + def _add_to_master_day_list(current_titles_list): + book_count = len(current_titles_list) + current_titles_list = " • ".join(current_titles_list) + current_titles_list = self.format_ncx_text(current_titles_list, dest='description') + master_day_list.append((current_titles_list, current_date, book_count)) + + def _add_to_master_date_range_list(current_titles_list): + book_count = len(current_titles_list) + current_titles_list = " • ".join(current_titles_list) + current_titles_list = self.format_ncx_text(current_titles_list, dest='description') + master_date_range_list.append((current_titles_list, date_range, book_count)) + + self.update_progress_full_step(_("NCX for Recently Read")) + + if not self.bookmarked_books_by_date_read: + return + + soup = self.ncx_soup + HTML_file = "content/ByDateRead.html" + body = soup.find("navPoint") + btc = len(body.contents) + + # --- Construct the 'Recently Read' *section* --- + navPointTag = Tag(soup, 'navPoint') + navPointTag['class'] = "section" + file_ID = "%s" % tocTitle.lower() + file_ID = file_ID.replace(" ","") + navPointTag['id'] = "%s-ID" % file_ID + navPointTag['playOrder'] = self.play_order + self.play_order += 1 + navLabelTag = Tag(soup, 'navLabel') + textTag = Tag(soup, 'text') + textTag.insert(0, NavigableString('%s' % tocTitle)) + navLabelTag.insert(0, textTag) + nptc = 0 + navPointTag.insert(nptc, navLabelTag) + nptc += 1 + contentTag = Tag(soup,"content") + contentTag['src'] = "%s#section_start" % HTML_file + navPointTag.insert(nptc, contentTag) + nptc += 1 + + # Create an NCX article entry for each date range + current_titles_list = [] + master_date_range_list = [] + today = datetime.datetime.now() + today_time = datetime.datetime(today.year, today.month, today.day) + for (i,date) in enumerate(self.DATE_RANGE): + if i: + date_range = '%d to %d days ago' % (self.DATE_RANGE[i-1], self.DATE_RANGE[i]) + else: + date_range = 'Last %d days' % (self.DATE_RANGE[i]) + date_range_limit = self.DATE_RANGE[i] + for book in self.bookmarked_books_by_date_read: + bookmark_time = datetime.datetime.utcfromtimestamp(book['bookmark_timestamp']) + if (today_time-bookmark_time).days <= date_range_limit: + #print "generate_ncx_by_date_added: %s added %d days ago" % (book['title'], (today_time-book_time).days) + current_titles_list.append(book['title']) + else: + break + if current_titles_list: + _add_to_master_date_range_list(current_titles_list) + current_titles_list = [book['title']] + + # Create an NCX article entry for each populated day + # Loop over the booksByDate list, find start of each month, + # add description_preview_count titles + # master_month_list(list,date,count) + current_titles_list = [] + master_day_list = [] + current_date = datetime.datetime.utcfromtimestamp(self.bookmarked_books_by_date_read[0]['bookmark_timestamp']) + + for book in self.bookmarked_books_by_date_read: + bookmark_time = datetime.datetime.utcfromtimestamp(book['bookmark_timestamp']) + if bookmark_time.day != current_date.day or \ + bookmark_time.month != current_date.month or \ + bookmark_time.year != current_date.year: + # Save the old lists + _add_to_master_day_list(current_titles_list) + + # Start the new list + current_date = datetime.datetime.utcfromtimestamp(book['bookmark_timestamp']).date() + current_titles_list = [book['title']] + else: + current_titles_list.append(book['title']) + + # Add the last day list + _add_to_master_day_list(current_titles_list) + + # Add *article* entries for each populated day + # master_day_list{}: [0]:titles list [1]:date + for books_by_day in master_day_list: + datestr = strftime(u'%A, %B %d', books_by_day[1].timetuple()) + navPointByDayTag = Tag(soup, 'navPoint') + navPointByDayTag['class'] = "article" + navPointByDayTag['id'] = "bdr_%s-%s-%sID" % (books_by_day[1].year, + books_by_day[1].month, + books_by_day[1].day ) + navPointTag['playOrder'] = self.play_order + self.play_order += 1 + navLabelTag = Tag(soup, 'navLabel') + textTag = Tag(soup, 'text') + textTag.insert(0, NavigableString(datestr)) + navLabelTag.insert(0, textTag) + navPointByDayTag.insert(0,navLabelTag) + contentTag = Tag(soup, 'content') + contentTag['src'] = "%s#bdr_%s-%s-%s" % (HTML_file, + books_by_day[1].year, + books_by_day[1].month, + books_by_day[1].day) + + navPointByDayTag.insert(1,contentTag) + + if self.generate_for_kindle: + cmTag = Tag(soup, '%s' % 'calibre:meta') + cmTag['name'] = "description" + cmTag.insert(0, NavigableString(books_by_day[0])) + navPointByDayTag.insert(2, cmTag) + + cmTag = Tag(soup, '%s' % 'calibre:meta') + cmTag['name'] = "author" + navStr = '%d titles' % books_by_day[2] if books_by_day[2] > 1 else \ + '%d title' % books_by_day[2] + cmTag.insert(0, NavigableString(navStr)) + navPointByDayTag.insert(3, cmTag) + + navPointTag.insert(nptc, navPointByDayTag) + nptc += 1 + + # Add this section to the body + body.insert(btc, navPointTag) + btc += 1 + self.ncx_soup = soup + + def generate_ncx_by_genre(self, tocTitle): + """ Add Genres to the basic NCX file. + + Generate the Genre NCX content, add to self.ncx_soup. + + Inputs: + genres (list) + + Updated: + play_order (int) + + Outputs: + ncx_soup (file): updated + """ + + self.update_progress_full_step(_("NCX for Genres")) + + if not len(self.genres): + self.opts.log.warn(" No genres found in tags.\n" + " No Genre section added to Catalog") + return + + ncx_soup = self.ncx_soup + body = ncx_soup.find("navPoint") + btc = len(body.contents) + + # --- Construct the 'Books By Genre' *section* --- + navPointTag = Tag(ncx_soup, 'navPoint') + navPointTag['class'] = "section" + file_ID = "%s" % tocTitle.lower() + file_ID = file_ID.replace(" ","") + navPointTag['id'] = "%s-ID" % file_ID + navPointTag['playOrder'] = self.play_order + self.play_order += 1 + navLabelTag = Tag(ncx_soup, 'navLabel') + textTag = Tag(ncx_soup, 'text') + # textTag.insert(0, NavigableString('%s (%d)' % (section_title, len(genre_list)))) + textTag.insert(0, NavigableString('%s' % tocTitle)) + navLabelTag.insert(0, textTag) + nptc = 0 + navPointTag.insert(nptc, navLabelTag) + nptc += 1 + contentTag = Tag(ncx_soup,"content") + contentTag['src'] = "content/Genre_%s.html#section_start" % self.genres[0]['tag'] + navPointTag.insert(nptc, contentTag) + nptc += 1 + + for genre in self.genres: + # Add an article for each genre + navPointVolumeTag = Tag(ncx_soup, 'navPoint') + navPointVolumeTag['class'] = "article" + navPointVolumeTag['id'] = "genre-%s-ID" % genre['tag'] + navPointVolumeTag['playOrder'] = self.play_order + self.play_order += 1 + navLabelTag = Tag(ncx_soup, "navLabel") + textTag = Tag(ncx_soup, "text") + + # GwR *** Can this be optimized? + normalized_tag = None + for friendly_tag in self.genre_tags_dict: + if self.genre_tags_dict[friendly_tag] == genre['tag']: + normalized_tag = self.genre_tags_dict[friendly_tag] + break + textTag.insert(0, self.format_ncx_text(NavigableString(friendly_tag), dest='description')) + navLabelTag.insert(0,textTag) + navPointVolumeTag.insert(0,navLabelTag) + contentTag = Tag(ncx_soup, "content") + contentTag['src'] = "content/Genre_%s.html#Genre_%s" % (normalized_tag, normalized_tag) + navPointVolumeTag.insert(1, contentTag) + + if self.generate_for_kindle: + # Build the author tag + cmTag = Tag(ncx_soup, '%s' % 'calibre:meta') + cmTag['name'] = "author" + # First - Last author + + if len(genre['titles_spanned']) > 1 : + author_range = "%s - %s" % (genre['titles_spanned'][0][0], genre['titles_spanned'][1][0]) + else : + author_range = "%s" % (genre['titles_spanned'][0][0]) + + cmTag.insert(0, NavigableString(author_range)) + navPointVolumeTag.insert(2, cmTag) + + # Build the description tag + cmTag = Tag(ncx_soup, '%s' % 'calibre:meta') + cmTag['name'] = "description" + + if False: + # Form 1: Titles spanned + if len(genre['titles_spanned']) > 1: + title_range = "%s -\n%s" % (genre['titles_spanned'][0][1], genre['titles_spanned'][1][1]) + else: + title_range = "%s" % (genre['titles_spanned'][0][1]) + cmTag.insert(0, NavigableString(self.format_ncx_text(title_range, dest='description'))) + else: + # Form 2: title • title • title ... + titles = [] + for title in genre['books']: + titles.append(title['title']) + titles = sorted(titles, key=lambda x:(self.generate_sort_title(x),self.generate_sort_title(x))) + titles_list = self.generate_short_description(u" • ".join(titles), dest="description") + cmTag.insert(0, NavigableString(self.format_ncx_text(titles_list, dest='description'))) + + navPointVolumeTag.insert(3, cmTag) + + # Add this volume to the section tag + navPointTag.insert(nptc, navPointVolumeTag) + nptc += 1 + + # Add this section to the body + body.insert(btc, navPointTag) + btc += 1 + self.ncx_soup = ncx_soup + + def generate_opf(self): + """ Generate the OPF file. + + Start with header template, construct manifest, spine and guide. + + Inputs: + genres (list) + html_filelist_1 (list) + html_filelist_2 (list) + thumbs (list) + + Updated: + play_order (int) + + Outputs: + opts.basename + '.opf' (file): written + """ + + self.update_progress_full_step(_("Generating OPF")) + + header = ''' + + + + en-US + + + + + + + ''' + # Add the supplied metadata tags + soup = BeautifulStoneSoup(header, selfClosingTags=['item','itemref', 'reference']) + metadata = soup.find('metadata') + mtc = 0 + + titleTag = Tag(soup, "dc:title") + titleTag.insert(0,self.opts.catalog_title) + metadata.insert(mtc, titleTag) + mtc += 1 + + creatorTag = Tag(soup, "dc:creator") + creatorTag.insert(0, self.opts.creator) + metadata.insert(mtc, creatorTag) + mtc += 1 + + # Create the OPF tags + manifest = soup.find('manifest') + mtc = 0 + spine = soup.find('spine') + stc = 0 + guide = soup.find('guide') + + itemTag = Tag(soup, "item") + itemTag['id'] = "ncx" + itemTag['href'] = '%s.ncx' % self.opts.basename + itemTag['media-type'] = "application/x-dtbncx+xml" + manifest.insert(mtc, itemTag) + mtc += 1 + + itemTag = Tag(soup, "item") + itemTag['id'] = 'stylesheet' + itemTag['href'] = self.stylesheet + itemTag['media-type'] = 'text/css' + manifest.insert(mtc, itemTag) + mtc += 1 + + itemTag = Tag(soup, "item") + itemTag['id'] = 'mastheadimage-image' + itemTag['href'] = "images/mastheadImage.gif" + itemTag['media-type'] = 'image/gif' + manifest.insert(mtc, itemTag) + mtc += 1 + + # Write the thumbnail images, descriptions to the manifest + sort_descriptions_by = [] + if self.opts.generate_descriptions: + for thumb in self.thumbs: + itemTag = Tag(soup, "item") + itemTag['href'] = "images/%s" % (thumb) + end = thumb.find('.jpg') + itemTag['id'] = "%s-image" % thumb[:end] + itemTag['media-type'] = 'image/jpeg' + manifest.insert(mtc, itemTag) + mtc += 1 + + # HTML files - add descriptions to manifest and spine + sort_descriptions_by = self.books_by_author if self.opts.sort_descriptions_by_author \ + else self.books_by_title + # Add html_files to manifest and spine + + for file in self.html_filelist_1: + # By Author, By Title, By Series, + itemTag = Tag(soup, "item") + start = file.find('/') + 1 + end = file.find('.') + itemTag['href'] = file + itemTag['id'] = file[start:end].lower() + itemTag['media-type'] = "application/xhtml+xml" + manifest.insert(mtc, itemTag) + mtc += 1 + + # spine + itemrefTag = Tag(soup, "itemref") + itemrefTag['idref'] = file[start:end].lower() + spine.insert(stc, itemrefTag) + stc += 1 + + # Add genre files to manifest and spine + for genre in self.genres: + itemTag = Tag(soup, "item") + start = genre['file'].find('/') + 1 + end = genre['file'].find('.') + itemTag['href'] = genre['file'] + itemTag['id'] = genre['file'][start:end].lower() + itemTag['media-type'] = "application/xhtml+xml" + manifest.insert(mtc, itemTag) + mtc += 1 + + # spine + itemrefTag = Tag(soup, "itemref") + itemrefTag['idref'] = genre['file'][start:end].lower() + spine.insert(stc, itemrefTag) + stc += 1 + + for file in self.html_filelist_2: + # By Date Added, By Date Read + itemTag = Tag(soup, "item") + start = file.find('/') + 1 + end = file.find('.') + itemTag['href'] = file + itemTag['id'] = file[start:end].lower() + itemTag['media-type'] = "application/xhtml+xml" + manifest.insert(mtc, itemTag) + mtc += 1 + + # spine + itemrefTag = Tag(soup, "itemref") + itemrefTag['idref'] = file[start:end].lower() + spine.insert(stc, itemrefTag) + stc += 1 + + for book in sort_descriptions_by: + # manifest + itemTag = Tag(soup, "item") + itemTag['href'] = "content/book_%d.html" % int(book['id']) + itemTag['id'] = "book%d" % int(book['id']) + itemTag['media-type'] = "application/xhtml+xml" + manifest.insert(mtc, itemTag) + mtc += 1 + + # spine + itemrefTag = Tag(soup, "itemref") + itemrefTag['idref'] = "book%d" % int(book['id']) + spine.insert(stc, itemrefTag) + stc += 1 + + # Guide + referenceTag = Tag(soup, "reference") + referenceTag['type'] = 'masthead' + referenceTag['title'] = 'mastheadimage-image' + referenceTag['href'] = 'images/mastheadImage.gif' + guide.insert(0,referenceTag) + + # Write the OPF file + outfile = open("%s/%s.opf" % (self.catalog_path, self.opts.basename), 'w') + outfile.write(soup.prettify()) + + def generate_rating_string(self, book): + """ Generate rating string for Descriptions. + + Starting with database rating (0-10), return 5 stars, with 0-5 filled, + balance empty. + + Args: + book (dict): book metadata + + Return: + rating (str): 5 stars, 1-5 solid, balance empty. Empty str for no rating. + """ + rating = '' try: if 'rating' in book: stars = int(book['rating']) / 2 if stars: - star_string = self.FULL_RATING_SYMBOL * stars - empty_stars = self.EMPTY_RATING_SYMBOL * (5 - stars) + star_string = self.SYMBOL_FULL_RATING * stars + empty_stars = self.SYMBOL_EMPTY_RATING * (5 - stars) rating = '%s%s' % (star_string,empty_stars) except: # Rating could be None pass return rating - def generateSeriesAnchor(self, series): + def generate_series_anchor(self, series): + """ Generate legal XHTML anchor for series names. + + Flatten series name to ascii_legal text. + + Args: + series (str): series name + + Return: + (str): asciized version of series name + """ + # Generate a legal XHTML id/href string if self.letter_or_symbol(series) == self.SYMBOLS: return "symbol_%s_series" % re.sub('\W','',series).lower() else: return "%s_series" % re.sub('\W','',ascii_text(series)).lower() - def generateShortDescription(self, description, dest=None): - # Truncate the description, on word boundaries if necessary - # Possible destinations: - # description NCX summary - # title NCX title - # author NCX author + def generate_short_description(self, description, dest=None): + """ Generate a truncated version of the supplied string. - def shortDescription(description, limit): + Given a string and NCX destination, truncate string to length specified + in self.opts. + + Args: + description (str): string to truncate + dest (str): NCX destination + description NCX summary + title NCX title + author NCX author + + Return: + (str): truncated description + """ + + def _short_description(description, limit): short_description = "" words = description.split() for word in words: @@ -3809,25 +4239,34 @@ Author '{0}': # No truncation for titles, let the device deal with it return description elif dest == 'author': - if self.authorClip and len(description) < self.authorClip: + if self.opts.author_clip and len(description) < self.opts.author_clip: return description else: - return shortDescription(description, self.authorClip) + return _short_description(description, self.opts.author_clip) elif dest == 'description': - if self.descriptionClip and len(description) < self.descriptionClip: + if self.opts.description_clip and len(description) < self.opts.description_clip: return description else: - return shortDescription(description, self.descriptionClip) + return _short_description(description, self.opts.description_clip) else: print " returning description with unspecified destination '%s'" % description raise RuntimeError - def generateSortTitle(self, title): - ''' - Generate a string suitable for sorting from the title - Ignore leading stop words - Optionally convert leading numbers to strings - ''' + def generate_sort_title(self, title): + """ Generates a sort string from title. + + Based on trunk title_sort algorithm, but also accommodates series + numbers by padding with leading zeroes to force proper numeric + sorting. Option to sort numbers alphabetically, e.g. '1942' sorts + as 'Nineteen forty two'. + + Args: + title (str): + + Return: + (str): sort string + """ + from calibre.ebooks.metadata import title_sort from calibre.library.catalogs.utils import NumberToText @@ -3870,29 +4309,38 @@ Author '{0}': translated.append(word) return ' '.join(translated) - def generateThumbnail(self, title, image_dir, thumb_file): - ''' - Thumbs are cached with the full cover's crc. If the crc doesn't - match, the cover has been changed since the thumb was cached and needs - to be replaced. - ''' + def generate_thumbnail(self, title, image_dir, thumb_file): + """ Create thumbnail of cover or return previously cached thumb. - def open_archive(mode='r'): + Test thumb archive for currently cached cover. Return cached version, or create + and cache new version. Uses calibre.utils.magick.draw to generate thumbnail from + cover. + + Args: + title (dict): book metadata + image_dir (str): directory to write thumb data to + thumb_file (str): filename to save thumb as + + Output: + (file): thumb written to /images + (archive): current thumb archived under cover crc + """ + + def _open_archive(mode='r'): try: - return ZipFile(self.__archive_path, mode=mode, allowZip64=True) + return ZipFile(self.thumbs_path, mode=mode, allowZip64=True) except: - # Happens on windows if the file is opened by another + # occurs under windows if the file is opened by another # process pass # Generate crc for current cover - #self.opts.log.info(" generateThumbnail():") with open(title['cover'], 'rb') as f: data = f.read() cover_crc = hex(zlib.crc32(data)) # Test cache for uuid - zf = open_archive() + zf = _open_archive() if zf is not None: with zf: try: @@ -3906,10 +4354,9 @@ Author '{0}': f.write(thumb_data) return - - # Save thumb for catalog + # Save thumb for catalog. If invalid data, error returns to generate_thumbnails() thumb_data = thumbnail(data, - width=self.thumbWidth, height=self.thumbHeight)[-1] + width=self.thumb_width, height=self.thumb_height)[-1] with open(os.path.join(image_dir, thumb_file), 'wb') as f: f.write(thumb_data) @@ -3917,44 +4364,234 @@ Author '{0}': if zf is not None: # Ensure that the read succeeded # If we failed to open the zip file for reading, # we dont know if it contained the thumb or not - zf = open_archive('a') + zf = _open_archive('a') if zf is not None: with zf: zf.writestr(title['uuid']+cover_crc, thumb_data) - def generateUnicodeName(self, c): - ''' - Generate an anchor name string - ''' + def generate_thumbnails(self): + """ Generate a thumbnail cover for each book. + + Generate or retrieve a thumbnail for each cover. If nonexistent or faulty + cover data, substitute default cover. Checks for updated default cover. + At completion, writes self.opts.thumb_width to archive. + + Inputs: + books_by_title (list): books to catalog + + Output: + thumbs (list): list of referenced thumbnails + """ + + self.update_progress_full_step(_("Thumbnails")) + thumbs = ['thumbnail_default.jpg'] + image_dir = "%s/images" % self.catalog_path + for (i,title) in enumerate(self.books_by_title): + # Update status + self.update_progress_micro_step("%s %d of %d" % + (_("Thumbnail"), i, len(self.books_by_title)), + i/float(len(self.books_by_title))) + + thumb_file = 'thumbnail_%d.jpg' % int(title['id']) + thumb_generated = True + valid_cover = True + try: + self.generate_thumbnail(title, image_dir, thumb_file) + thumbs.append("thumbnail_%d.jpg" % int(title['id'])) + except: + if 'cover' in title and os.path.exists(title['cover']): + valid_cover = False + self.opts.log.warn(" *** Invalid cover file for '%s'***" % + (title['title'])) + if not self.error: + self.error.append('Invalid cover files') + self.error.append("Warning: invalid cover file for '%s', default cover substituted.\n" % (title['title'])) + + thumb_generated = False + + if not thumb_generated: + self.opts.log.warn(" using default cover for '%s' (%d)" % (title['title'], title['id'])) + # Confirm thumb exists, default is current + default_thumb_fp = os.path.join(image_dir,"thumbnail_default.jpg") + cover = os.path.join(self.catalog_path, "DefaultCover.png") + title['cover'] = cover + + if not os.path.exists(cover): + shutil.copyfile(I('book.png'), cover) + + if os.path.isfile(default_thumb_fp): + # Check to see if default cover is newer than thumbnail + # os.path.getmtime() = modified time + # os.path.ctime() = creation time + cover_timestamp = os.path.getmtime(cover) + thumb_timestamp = os.path.getmtime(default_thumb_fp) + if thumb_timestamp < cover_timestamp: + if self.DEBUG and self.opts.verbose: + self.opts.log.warn("updating thumbnail_default for %s" % title['title']) + self.generate_thumbnail(title, image_dir, + "thumbnail_default.jpg" if valid_cover else thumb_file) + else: + if self.DEBUG and self.opts.verbose: + self.opts.log.warn(" generating new thumbnail_default.jpg") + self.generate_thumbnail(title, image_dir, + "thumbnail_default.jpg" if valid_cover else thumb_file) + # Clear the book's cover property + title['cover'] = None + + + # Write thumb_width to the file, validating cache contents + # Allows detection of aborted catalog builds + with ZipFile(self.thumbs_path, mode='a') as zfw: + zfw.writestr('thumb_width', self.opts.thumb_width) + + self.thumbs = thumbs + + def generate_unicode_name(self, c): + """ Generate a legal XHTML anchor from unicode character. + + Generate a legal XHTML anchor from unicode character. + + Args: + c (unicode): character + + Return: + (str): legal XHTML anchor string of unicode charactar name + """ fullname = unicodedata.name(unicode(c)) terms = fullname.split() return "_".join(terms) - def getFriendlyGenreTag(self, genre): + def get_excluded_tags(self): + """ Get excluded_tags from opts.exclusion_rules. + + Parse opts.exclusion_rules for tags to be excluded, return list. + Log books that will be excluded by excluded_tags. + + Inputs: + opts.excluded_tags (tuples): exclusion rules + + Return: + excluded_tags (list): excluded tags + """ + excluded_tags = [] + for rule in self.opts.exclusion_rules: + if rule[1].lower() == 'tags': + excluded_tags.extend(rule[2].split(',')) + + # Remove dups + excluded_tags = list(set(excluded_tags)) + + # Report excluded books + if self.opts.verbose and excluded_tags: + self.opts.log.info(" Excluded books by Tags:") + data = self.db.get_data_as_dict(ids=self.opts.ids) + for record in data: + matched = list(set(record['tags']) & set(excluded_tags)) + if matched: + for rule in self.opts.exclusion_rules: + if rule[1] == 'Tags' and rule[2] == str(matched[0]): + self.opts.log.info(" - '%s' by %s (Exclusion rule '%s')" % + (record['title'], record['authors'][0], rule[0])) + + return excluded_tags + + def get_friendly_genre_tag(self, genre): + """ Return the first friendly_tag matching genre. + + Scan self.genre_tags_dict[] for first friendly_tag matching genre. + genre_tags_dict[] populated in filter_db_tags(). + + Args: + genre (str): genre to match + + Return: + friendly_tag (str): friendly_tag matching genre + """ # Find the first instance of friendly_tag matching genre for friendly_tag in self.genre_tags_dict: if self.genre_tags_dict[friendly_tag] == genre: return friendly_tag - def getMarkerTags(self): - ''' - Return a list of special marker tags to be excluded from genre list - exclusion_rules = ('name','Tags|#column','[]|pattern') - ''' - markerTags = [] - for rule in self.opts.exclusion_rules: - if rule[1].lower() == 'tags': - markerTags.extend(rule[2].split(',')) - return markerTags + def get_output_profile(self, _opts): + """ Return profile matching opts.output_profile + + Input: + _opts (object): build options object + + Return: + (profile): output profile matching name + """ + for profile in output_profiles(): + if profile.short_name == _opts.output_profile: + return profile + + def get_prefix_rules(self): + """ Convert opts.prefix_rules to dict. + + Convert opts.prefix_rules to dict format. + + Input: + opts.prefix_rules (tuples): (name, field, pattern, prefix) + + Return: + (list): list of prefix_rules dicts + """ + pr = [] + if self.opts.prefix_rules: + try: + for rule in self.opts.prefix_rules: + prefix_rule = {} + prefix_rule['name'] = rule[0] + prefix_rule['field'] = rule[1] + prefix_rule['pattern'] = rule[2] + prefix_rule['prefix'] = rule[3] + pr.append(prefix_rule) + except: + self.opts.log.error("malformed prefix_rules: %s" % repr(self.opts.prefix_rules)) + raise + return pr def letter_or_symbol(self,char): + """ Test asciized char for A-z. + + Convert char to ascii, test for A-z. + + Args: + char (chr): character to test + + Return: + (str): char if A-z, else SYMBOLS + """ if not re.search('[a-zA-Z]', ascii_text(char)): return self.SYMBOLS else: return char - def markdownComments(self, comments): - ''' + def load_section_templates(self): + """ Add section templates to local namespace. + + Load section templates from resource directory. If user has made local copies, + these will be used for individual section generation. + generate_format_args() builds args that populate templates. + Templates referenced in individual section builders, e.g. + generate_html_by_title(). + + Inputs: + (files): section template files from resource dir + + Results: + (strs): section templates added to local namespace + """ + + templates = {} + execfile(P('catalog/section_list_templates.py'), templates) + for name, template in templates.iteritems(): + if name.startswith('by_') and name.endswith('_template'): + setattr(self, name, force_unicode(template, 'utf-8')) + + def massage_comments(self, comments): + """ Massage comments to somewhat consistent format. + Convert random comment text to normalized, xml-legal block of

s 'plain text' returns as

plain text

@@ -3977,7 +4614,13 @@ Author '{0}': Deprecated HTML returns as HTML via BeautifulSoup() - ''' + Args: + comments (str): comments from metadata, possibly HTML + + Return: + result (BeautifulSoup): massaged comments in HTML form + """ + # Hackish - ignoring sentences ending or beginning in numbers to avoid # confusion with decimal points. @@ -4067,19 +4710,29 @@ Author '{0}': return result.renderContents(encoding=None) - def mergeComments(self, record): - ''' - merge ['description'] with custom field contents to be displayed in Descriptions - ''' + def merge_comments(self, record): + """ Merge comments with custom column content. + + Merge comments from book metadata with user-specified custom column + content, optionally before or after. Optionally insert
between + fields. + + Args: + record (dict): book metadata + + Return: + merged (str): comments merged with addendum + """ + merged = '' if record['description']: - addendum = self.__db.get_field(record['id'], - self.__merge_comments['field'], + addendum = self.db.get_field(record['id'], + self.merge_comments_rule['field'], index_is_id=True) if addendum is None: addendum = '' - include_hr = eval(self.__merge_comments['hr']) - if self.__merge_comments['position'] == 'before': + include_hr = eval(self.merge_comments_rule['hr']) + if self.merge_comments_rule['position'] == 'before': merged = addendum if include_hr: merged += '
' @@ -4095,33 +4748,47 @@ Author '{0}': merged += addendum else: # Return the custom field contents - merged = self.__db.get_field(record['id'], - self.__merge_comments['field'], + merged = self.db.get_field(record['id'], + self.merge_comments_rule['field'], index_is_id=True) return merged - def processPrefixRules(self): - if self.opts.prefix_rules: - # Put the prefix rules into an ordered list of dicts - try: - for rule in self.opts.prefix_rules: - prefix_rule = {} - prefix_rule['name'] = rule[0] - prefix_rule['field'] = rule[1] - prefix_rule['pattern'] = rule[2] - prefix_rule['prefix'] = rule[3] - self.prefixRules.append(prefix_rule) - except: - self.opts.log.error("malformed self.opts.prefix_rules: %s" % repr(self.opts.prefix_rules)) - raise - # Use the highest order prefix symbol as default - self.defaultPrefix = self.opts.prefix_rules[0][3] + def normalize_tag(self, tag): + """ Generate an XHTML-legal anchor string from tag. + + Parse tag for non-ascii, convert to unicode name. + + Args: + tags (str): tag name possible containing symbols + + Return: + normalized (str): unicode names substituted for non-ascii chars + """ + + normalized = massaged = re.sub('\s','',ascii_text(tag).lower()) + if re.search('\W',normalized): + normalized = '' + for c in massaged: + if re.search('\W',c): + normalized += self.generate_unicode_name(c) + else: + normalized += c + return normalized + + def process_exclusions(self, data_set, log_exclusion=True): + """ Filter data_set based on exclusion_rules. + + Compare each book in data_set to each exclusion_rule. Remove + books matching exclusion criteria. + + Args: + data_set (list): all candidate books + + Return: + (list): filtered data_set + """ - def processExclusions(self, data_set): - ''' - Remove excluded entries - ''' filtered_data_set = [] exclusion_pairs = [] exclusion_set = [] @@ -4132,21 +4799,23 @@ Author '{0}': exclusion_pairs.append((field,pat)) else: continue - if exclusion_pairs: for record in data_set: for exclusion_pair in exclusion_pairs: field,pat = exclusion_pair - field_contents = self.__db.get_field(record['id'], + field_contents = self.db.get_field(record['id'], field, index_is_id=True) if field_contents: - if re.search(pat, unicode(field_contents), - re.IGNORECASE) is not None: - if self.opts.verbose: + matched = re.search(pat, unicode(field_contents), + re.IGNORECASE) + if matched is not None: + if self.opts.verbose and log_exclusion: field_md = self.db.metadata_for_field(field) - self.opts.log.info(" - %s (Exclusion rule '%s': %s:%s)" % - (record['title'], field_md['name'], field,pat)) + for rule in self.opts.exclusion_rules: + if rule[1] == '#%s' % field_md['label']: + self.opts.log.info(" - '%s' by %s (Exclusion rule '%s')" % + (record['title'], record['authors'][0], rule[0])) exclusion_set.append(record) if record in filtered_data_set: filtered_data_set.remove(record) @@ -4159,36 +4828,65 @@ Author '{0}': else: return data_set - def processSpecialTags(self, tags, this_title, opts): + def update_progress_full_step(self, description): + """ Update calibre's job status UI. - tag_list = [] + Call ProgessReporter() with updates. - try: - for tag in tags: - tag = self.convertHTMLEntities(tag) - if re.search(opts.exclude_genre, tag): - continue - else: - tag_list.append(tag) - except: - self.opts.log.error("\tprocessSpecialTags(): malformed --exclude-genre regex pattern: %s" % opts.exclude_genre) - return tags + Args: + description (str): text describing current step - return tag_list + Result: + (UI): Jobs UI updated + """ - def updateProgressFullStep(self, description): - self.currentStep += 1 - self.progressString = description - self.progressInt = float((self.currentStep-1)/self.totalSteps) - self.reporter(self.progressInt, self.progressString) + self.current_step += 1 + self.progress_string = description + self.progress_int = float((self.current_step-1)/self.total_steps) + if not self.progress_int: + self.progress_int = 0.01 + self.reporter(self.progress_int, self.progress_string) if self.opts.cli_environment: - self.opts.log(u"%3.0f%% %s" % (self.progressInt*100, self.progressString)) + self.opts.log(u"%3.0f%% %s" % (self.progress_int*100, self.progress_string)) - def updateProgressMicroStep(self, description, micro_step_pct): - step_range = 100/self.totalSteps - self.progressString = description - coarse_progress = float((self.currentStep-1)/self.totalSteps) + def update_progress_micro_step(self, description, micro_step_pct): + """ Update calibre's job status UI. + + Called from steps requiring more time: + generate_html_descriptions() + generate_thumbnails() + + Args: + description (str): text describing microstep + micro_step_pct (float): percentage of full step + + Results: + (UI): Jobs UI updated + """ + + step_range = 100/self.total_steps + self.progress_string = description + coarse_progress = float((self.current_step-1)/self.total_steps) fine_progress = float((micro_step_pct*step_range)/100) - self.progressInt = coarse_progress + fine_progress - self.reporter(self.progressInt, self.progressString) + self.progress_int = coarse_progress + fine_progress + self.reporter(self.progress_int, self.progress_string) + + def write_ncx(self): + """ Write accumulated ncx_soup to file. + + Expanded description + + Inputs: + catalog_path (str): path to generated catalog + opts.basename (str): catalog basename + + Output: + (file): basename.NCX written + """ + + self.update_progress_full_step(_("Saving NCX")) + + outfile = open("%s/%s.ncx" % (self.catalog_path, self.opts.basename), 'w') + outfile.write(self.ncx_soup.prettify()) +