diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py index 9150380537..0897199289 100644 --- a/src/calibre/customize/builtins.py +++ b/src/calibre/customize/builtins.py @@ -11,6 +11,8 @@ from calibre.constants import numeric_version from calibre.ebooks.metadata.archive import ArchiveExtract, get_cbz_metadata from calibre.ebooks.html.to_zip import HTML2ZIP +plugins = [] + # To archive plugins {{{ class PML2PMLZ(FileTypePlugin): @@ -118,6 +120,7 @@ class TXT2TXTZ(FileTypePlugin): # No images so just import the TXT file. return path_to_ebook +plugins += [HTML2ZIP, PML2PMLZ, TXT2TXTZ, ArchiveExtract,] # }}} # Metadata reader plugins {{{ @@ -400,6 +403,10 @@ class ZipMetadataReader(MetadataReaderPlugin): def get_metadata(self, stream, ftype): from calibre.ebooks.metadata.zip import get_metadata return get_metadata(stream) + +plugins += [x for x in list(locals().values()) if isinstance(x, type) and \ + x.__name__.endswith('MetadataReader')] + # }}} # Metadata writer plugins {{{ @@ -500,8 +507,12 @@ class TXTZMetadataWriter(MetadataWriterPlugin): from calibre.ebooks.metadata.extz import set_metadata set_metadata(stream, mi) +plugins += [x for x in list(locals().values()) if isinstance(x, type) and \ + x.__name__.endswith('MetadataWriter')] + # }}} +# Conversion plugins {{{ from calibre.ebooks.conversion.plugins.comic_input import ComicInput from calibre.ebooks.conversion.plugins.djvu_input import DJVUInput from calibre.ebooks.conversion.plugins.epub_input import EPUBInput @@ -541,65 +552,6 @@ from calibre.ebooks.conversion.plugins.html_output import HTMLOutput from calibre.ebooks.conversion.plugins.htmlz_output import HTMLZOutput from calibre.ebooks.conversion.plugins.snb_output import SNBOutput -from calibre.customize.profiles import input_profiles, output_profiles - -from calibre.devices.apple.driver import ITUNES -from calibre.devices.hanlin.driver import HANLINV3, HANLINV5, BOOX, SPECTRA -from calibre.devices.blackberry.driver import BLACKBERRY, PLAYBOOK -from calibre.devices.cybook.driver import CYBOOK, ORIZON -from calibre.devices.eb600.driver import (EB600, COOL_ER, SHINEBOOK, - POCKETBOOK360, GER2, ITALICA, ECLICTO, DBOOK, INVESBOOK, - BOOQ, ELONEX, POCKETBOOK301, MENTOR, POCKETBOOK602, - POCKETBOOK701, POCKETBOOK360P, PI2) -from calibre.devices.iliad.driver import ILIAD -from calibre.devices.irexdr.driver import IREXDR1000, IREXDR800 -from calibre.devices.jetbook.driver import JETBOOK, MIBUK, JETBOOK_MINI -from calibre.devices.kindle.driver import (KINDLE, KINDLE2, KINDLE_DX, - KINDLE_FIRE) -from calibre.devices.nook.driver import NOOK, NOOK_COLOR -from calibre.devices.prs505.driver import PRS505 -from calibre.devices.prst1.driver import PRST1 -from calibre.devices.user_defined.driver import USER_DEFINED -from calibre.devices.android.driver import ANDROID, S60, WEBOS -from calibre.devices.nokia.driver import N770, N810, E71X, E52 -from calibre.devices.eslick.driver import ESLICK, EBK52 -from calibre.devices.nuut2.driver import NUUT2 -from calibre.devices.iriver.driver import IRIVER_STORY -from calibre.devices.binatone.driver import README -from calibre.devices.hanvon.driver import (N516, EB511, ALEX, AZBOOKA, THEBOOK, - LIBREAIR, ODYSSEY) -from calibre.devices.edge.driver import EDGE -from calibre.devices.teclast.driver import (TECLAST_K3, NEWSMY, IPAPYRUS, - SOVOS, PICO, SUNSTECH_EB700, ARCHOS7O, STASH, WEXLER) -from calibre.devices.sne.driver import SNE -from calibre.devices.misc import (PALMPRE, AVANT, SWEEX, PDNOVEL, - GEMEI, VELOCITYMICRO, PDNOVEL_KOBO, LUMIREAD, ALURATEK_COLOR, - TREKSTOR, EEEREADER, NEXTBOOK, ADAM, MOOVYBOOK, COBY, EX124G) -from calibre.devices.folder_device.driver import FOLDER_DEVICE_FOR_CONFIG -from calibre.devices.kobo.driver import KOBO -from calibre.devices.bambook.driver import BAMBOOK -from calibre.devices.boeye.driver import BOEYE_BEX, BOEYE_BDX - -from calibre.library.catalog import CSV_XML, EPUB_MOBI, BIBTEX -from calibre.ebooks.epub.fix.unmanifested import Unmanifested -from calibre.ebooks.epub.fix.epubcheck import Epubcheck - -plugins = [HTML2ZIP, PML2PMLZ, TXT2TXTZ, ArchiveExtract, CSV_XML, EPUB_MOBI, BIBTEX, Unmanifested, - Epubcheck, ] - -# New metadata download plugins {{{ -from calibre.ebooks.metadata.sources.google import GoogleBooks -from calibre.ebooks.metadata.sources.amazon import Amazon -from calibre.ebooks.metadata.sources.openlibrary import OpenLibrary -from calibre.ebooks.metadata.sources.isbndb import ISBNDB -from calibre.ebooks.metadata.sources.overdrive import OverDrive -from calibre.ebooks.metadata.sources.douban import Douban -from calibre.ebooks.metadata.sources.ozon import Ozon - -plugins += [GoogleBooks, Amazon, OpenLibrary, ISBNDB, OverDrive, Douban, Ozon] - -# }}} - plugins += [ ComicInput, DJVUInput, @@ -642,6 +594,66 @@ plugins += [ HTMLZOutput, SNBOutput, ] +# }}} + +# Catalog plugins {{{ +from calibre.library.catalogs.csv_xml import CSV_XML +from calibre.library.catalogs.bibtex import BIBTEX +from calibre.library.catalogs.epub_mobi import EPUB_MOBI +plugins += [CSV_XML, BIBTEX, EPUB_MOBI] +# }}} + +# EPUB Fix plugins {{{ +from calibre.ebooks.epub.fix.unmanifested import Unmanifested +from calibre.ebooks.epub.fix.epubcheck import Epubcheck +plugins += [Unmanifested, Epubcheck] +# }}} + +# Profiles {{{ +from calibre.customize.profiles import input_profiles, output_profiles +plugins += input_profiles + output_profiles +# }}} + +# Device driver plugins {{{ +from calibre.devices.apple.driver import ITUNES +from calibre.devices.hanlin.driver import HANLINV3, HANLINV5, BOOX, SPECTRA +from calibre.devices.blackberry.driver import BLACKBERRY, PLAYBOOK +from calibre.devices.cybook.driver import CYBOOK, ORIZON +from calibre.devices.eb600.driver import (EB600, COOL_ER, SHINEBOOK, + POCKETBOOK360, GER2, ITALICA, ECLICTO, DBOOK, INVESBOOK, + BOOQ, ELONEX, POCKETBOOK301, MENTOR, POCKETBOOK602, + POCKETBOOK701, POCKETBOOK360P, PI2) +from calibre.devices.iliad.driver import ILIAD +from calibre.devices.irexdr.driver import IREXDR1000, IREXDR800 +from calibre.devices.jetbook.driver import JETBOOK, MIBUK, JETBOOK_MINI +from calibre.devices.kindle.driver import (KINDLE, KINDLE2, KINDLE_DX, + KINDLE_FIRE) +from calibre.devices.nook.driver import NOOK, NOOK_COLOR +from calibre.devices.prs505.driver import PRS505 +from calibre.devices.prst1.driver import PRST1 +from calibre.devices.user_defined.driver import USER_DEFINED +from calibre.devices.android.driver import ANDROID, S60, WEBOS +from calibre.devices.nokia.driver import N770, N810, E71X, E52 +from calibre.devices.eslick.driver import ESLICK, EBK52 +from calibre.devices.nuut2.driver import NUUT2 +from calibre.devices.iriver.driver import IRIVER_STORY +from calibre.devices.binatone.driver import README +from calibre.devices.hanvon.driver import (N516, EB511, ALEX, AZBOOKA, THEBOOK, + LIBREAIR, ODYSSEY) +from calibre.devices.edge.driver import EDGE +from calibre.devices.teclast.driver import (TECLAST_K3, NEWSMY, IPAPYRUS, + SOVOS, PICO, SUNSTECH_EB700, ARCHOS7O, STASH, WEXLER) +from calibre.devices.sne.driver import SNE +from calibre.devices.misc import (PALMPRE, AVANT, SWEEX, PDNOVEL, + GEMEI, VELOCITYMICRO, PDNOVEL_KOBO, LUMIREAD, ALURATEK_COLOR, + TREKSTOR, EEEREADER, NEXTBOOK, ADAM, MOOVYBOOK, COBY, EX124G) +from calibre.devices.folder_device.driver import FOLDER_DEVICE_FOR_CONFIG +from calibre.devices.kobo.driver import KOBO +from calibre.devices.bambook.driver import BAMBOOK +from calibre.devices.boeye.driver import BOEYE_BEX, BOEYE_BDX + + + # Order here matters. The first matched device is the one used. plugins += [ HANLINV3, @@ -716,11 +728,20 @@ plugins += [ BOEYE_BDX, USER_DEFINED, ] -plugins += [x for x in list(locals().values()) if isinstance(x, type) and \ - x.__name__.endswith('MetadataReader')] -plugins += [x for x in list(locals().values()) if isinstance(x, type) and \ - x.__name__.endswith('MetadataWriter')] -plugins += input_profiles + output_profiles +# }}} + +# New metadata download plugins {{{ +from calibre.ebooks.metadata.sources.google import GoogleBooks +from calibre.ebooks.metadata.sources.amazon import Amazon +from calibre.ebooks.metadata.sources.openlibrary import OpenLibrary +from calibre.ebooks.metadata.sources.isbndb import ISBNDB +from calibre.ebooks.metadata.sources.overdrive import OverDrive +from calibre.ebooks.metadata.sources.douban import Douban +from calibre.ebooks.metadata.sources.ozon import Ozon + +plugins += [GoogleBooks, Amazon, OpenLibrary, ISBNDB, OverDrive, Douban, Ozon] + +# }}} # Interface Actions {{{ @@ -1623,3 +1644,4 @@ plugins += [ ] # }}} + diff --git a/src/calibre/gui2/catalog/catalog_bibtex.py b/src/calibre/gui2/catalog/catalog_bibtex.py index 58dae47a90..65999bbdd3 100644 --- a/src/calibre/gui2/catalog/catalog_bibtex.py +++ b/src/calibre/gui2/catalog/catalog_bibtex.py @@ -31,7 +31,7 @@ class PluginWidget(QWidget, Ui_Form): self.setupUi(self) def initialize(self, name, db): #not working properly to update - from calibre.library.catalog import FIELDS + from calibre.library.catalogs import FIELDS self.all_fields = [x for x in FIELDS if x != 'all'] #add custom columns diff --git a/src/calibre/gui2/catalog/catalog_csv_xml.py b/src/calibre/gui2/catalog/catalog_csv_xml.py index ef322f2db9..96acbdfa9b 100644 --- a/src/calibre/gui2/catalog/catalog_csv_xml.py +++ b/src/calibre/gui2/catalog/catalog_csv_xml.py @@ -21,7 +21,7 @@ class PluginWidget(QWidget, Ui_Form): def __init__(self, parent=None): QWidget.__init__(self, parent) self.setupUi(self) - from calibre.library.catalog import FIELDS + from calibre.library.catalogs import FIELDS self.all_fields = [] for x in FIELDS: if x != 'all': diff --git a/src/calibre/library/catalog.py b/src/calibre/library/catalog.py deleted file mode 100644 index f30e200296..0000000000 --- a/src/calibre/library/catalog.py +++ /dev/null @@ -1,5184 +0,0 @@ -# -*- coding: utf-8 -*- - -__license__ = 'GPL v3' -__copyright__ = '2010, Greg Riker' - -import codecs, datetime, htmlentitydefs, os, re, shutil, zlib -from collections import namedtuple -from copy import deepcopy -from xml.sax.saxutils import escape -from lxml import etree -from types import StringType, UnicodeType - -from calibre import (prints, prepare_string_for_xml, strftime, force_unicode) -from calibre.constants import preferred_encoding, DEBUG -from calibre.customize import CatalogPlugin -from calibre.customize.conversion import OptionRecommendation, DummyReporter -from calibre.ebooks.BeautifulSoup import BeautifulSoup, BeautifulStoneSoup, Tag, NavigableString -from calibre.ebooks.chardet import substitute_entites -from calibre.library.save_to_disk import preprocess_template -from calibre.ptempfile import PersistentTemporaryDirectory -from calibre.utils.bibtex import BibTeX -from calibre.utils.config import config_dir -from calibre.utils.date import format_date, isoformat, is_date_undefined, now as nowf -from calibre.utils.html2text import html2text -from calibre.utils.icu import capitalize -from calibre.utils.logging import default_log as log -from calibre.utils.magick.draw import thumbnail -from calibre.utils.zipfile import ZipFile - -FIELDS = ['all', 'title', 'title_sort', 'author_sort', 'authors', 'comments', - 'cover', 'formats','id', 'isbn', 'ondevice', 'pubdate', 'publisher', - 'rating', 'series_index', 'series', 'size', 'tags', 'timestamp', - 'uuid', 'languages'] - -#Allowed fields for template -TEMPLATE_ALLOWED_FIELDS = [ 'author_sort', 'authors', 'id', 'isbn', 'pubdate', 'title_sort', - 'publisher', 'series_index', 'series', 'tags', 'timestamp', 'title', 'uuid' ] - -class CSV_XML(CatalogPlugin): # {{{ - 'CSV/XML catalog generator' - - Option = namedtuple('Option', 'option, default, dest, action, help') - - name = 'Catalog_CSV_XML' - description = 'CSV/XML catalog generator' - supported_platforms = ['windows', 'osx', 'linux'] - author = 'Greg Riker' - version = (1, 0, 0) - file_types = set(['csv','xml']) - - cli_options = [ - Option('--fields', - default = 'all', - dest = 'fields', - action = None, - help = _('The fields to output when cataloging books in the ' - 'database. Should be a comma-separated list of fields.\n' - 'Available fields: %(fields)s,\n' - 'plus user-created custom fields.\n' - 'Example: %(opt)s=title,authors,tags\n' - "Default: '%%default'\n" - "Applies to: CSV, XML output formats")%dict( - fields=', '.join(FIELDS), opt='--fields')), - - Option('--sort-by', - default = 'id', - dest = 'sort_by', - action = None, - help = _('Output field to sort on.\n' - 'Available fields: author_sort, id, rating, size, timestamp, title_sort\n' - "Default: '%default'\n" - "Applies to: CSV, XML output formats"))] - - def run(self, path_to_output, opts, db, notification=DummyReporter()): - self.fmt = path_to_output.rpartition('.')[2] - self.notification = notification - - if opts.verbose: - opts_dict = vars(opts) - log("%s(): Generating %s" % (self.name,self.fmt.upper())) - if opts.connected_device['is_device_connected']: - log(" connected_device: %s" % opts.connected_device['name']) - if opts_dict['search_text']: - log(" --search='%s'" % opts_dict['search_text']) - - if opts_dict['ids']: - log(" Book count: %d" % len(opts_dict['ids'])) - if opts_dict['search_text']: - log(" (--search ignored when a subset of the database is specified)") - - if opts_dict['fields']: - if opts_dict['fields'] == 'all': - log(" Fields: %s" % ', '.join(FIELDS[1:])) - else: - log(" Fields: %s" % opts_dict['fields']) - - # If a list of ids are provided, don't use search_text - if opts.ids: - opts.search_text = None - - data = self.search_sort_db(db, opts) - - if not len(data): - log.error("\nNo matching database entries for search criteria '%s'" % opts.search_text) - #raise SystemExit(1) - - # Get the requested output fields as a list - fields = self.get_output_fields(db, opts) - - # If connected device, add 'On Device' values to data - if opts.connected_device['is_device_connected'] and 'ondevice' in fields: - for entry in data: - entry['ondevice'] = db.catalog_plugin_on_device_temp_mapping[entry['id']]['ondevice'] - - if self.fmt == 'csv': - outfile = codecs.open(path_to_output, 'w', 'utf8') - - # Write a UTF-8 BOM - outfile.write('\xef\xbb\xbf') - - # Output the field headers - outfile.write(u'%s\n' % u','.join(fields)) - - # Output the entry fields - for entry in data: - outstr = [] - for field in fields: - if field.startswith('#'): - item = db.get_field(entry['id'],field,index_is_id=True) - elif field == 'title_sort': - item = entry['sort'] - else: - item = entry[field] - - if item is None: - outstr.append('""') - continue - elif field == 'formats': - fmt_list = [] - for format in item: - fmt_list.append(format.rpartition('.')[2].lower()) - item = ', '.join(fmt_list) - elif field in ['authors','tags']: - item = ', '.join(item) - elif field == 'isbn': - # Could be 9, 10 or 13 digits - item = u'%s' % re.sub(r'[\D]', '', item) - elif field in ['pubdate', 'timestamp']: - item = isoformat(item) - elif field == 'comments': - item = item.replace(u'\r\n',u' ') - item = item.replace(u'\n',u' ') - - # Convert HTML to markdown text - if type(item) is unicode: - opening_tag = re.search('<(\w+)(\x20|>)',item) - if opening_tag: - closing_tag = re.search('<\/%s>$' % opening_tag.group(1), item) - if closing_tag: - item = html2text(item) - - outstr.append(u'"%s"' % unicode(item).replace('"','""')) - - outfile.write(u','.join(outstr) + u'\n') - outfile.close() - - elif self.fmt == 'xml': - from lxml.builder import E - - root = E.calibredb() - for r in data: - record = E.record() - root.append(record) - - for field in fields: - if field.startswith('#'): - val = db.get_field(r['id'],field,index_is_id=True) - if not isinstance(val, (str, unicode)): - val = unicode(val) - item = getattr(E, field.replace('#','_'))(val) - record.append(item) - - for field in ('id', 'uuid', 'publisher', 'rating', 'size', - 'isbn','ondevice'): - if field in fields: - val = r[field] - if not val: - continue - if not isinstance(val, (str, unicode)): - val = unicode(val) - item = getattr(E, field)(val) - record.append(item) - - if 'title' in fields: - title = E.title(r['title'], sort=r['sort']) - record.append(title) - - if 'authors' in fields: - aus = E.authors(sort=r['author_sort']) - for au in r['authors']: - aus.append(E.author(au)) - record.append(aus) - - for field in ('timestamp', 'pubdate'): - if field in fields: - record.append(getattr(E, field)(r[field].isoformat())) - - if 'tags' in fields and r['tags']: - tags = E.tags() - for tag in r['tags']: - tags.append(E.tag(tag)) - record.append(tags) - - if 'comments' in fields and r['comments']: - record.append(E.comments(r['comments'])) - - if 'series' in fields and r['series']: - record.append(E.series(r['series'], - index=str(r['series_index']))) - - if 'cover' in fields and r['cover']: - record.append(E.cover(r['cover'].replace(os.sep, '/'))) - - if 'formats' in fields and r['formats']: - fmt = E.formats() - for f in r['formats']: - fmt.append(E.format(f.replace(os.sep, '/'))) - record.append(fmt) - - with open(path_to_output, 'w') as f: - f.write(etree.tostring(root, encoding='utf-8', - xml_declaration=True, pretty_print=True)) -# }}} - -class BIBTEX(CatalogPlugin): # {{{ - 'BIBTEX catalog generator' - - Option = namedtuple('Option', 'option, default, dest, action, help') - - name = 'Catalog_BIBTEX' - description = 'BIBTEX catalog generator' - supported_platforms = ['windows', 'osx', 'linux'] - author = 'Sengian' - version = (1, 0, 0) - file_types = set(['bib']) - - cli_options = [ - Option('--fields', - default = 'all', - dest = 'fields', - action = None, - help = _('The fields to output when cataloging books in the ' - 'database. Should be a comma-separated list of fields.\n' - 'Available fields: %(fields)s.\n' - 'plus user-created custom fields.\n' - 'Example: %(opt)s=title,authors,tags\n' - "Default: '%%default'\n" - "Applies to: BIBTEX output format")%dict( - fields=', '.join(FIELDS), opt='--fields')), - - Option('--sort-by', - default = 'id', - dest = 'sort_by', - action = None, - help = _('Output field to sort on.\n' - 'Available fields: author_sort, id, rating, size, timestamp, title.\n' - "Default: '%default'\n" - "Applies to: BIBTEX output format")), - - Option('--create-citation', - default = 'True', - dest = 'impcit', - action = None, - help = _('Create a citation for BibTeX entries.\n' - 'Boolean value: True, False\n' - "Default: '%default'\n" - "Applies to: BIBTEX output format")), - - Option('--add-files-path', - default = 'True', - dest = 'addfiles', - action = None, - help = _('Create a file entry if formats is selected for BibTeX entries.\n' - 'Boolean value: True, False\n' - "Default: '%default'\n" - "Applies to: BIBTEX output format")), - - Option('--citation-template', - default = '{authors}{id}', - dest = 'bib_cit', - action = None, - help = _('The template for citation creation from database fields.\n' - 'Should be a template with {} enclosed fields.\n' - 'Available fields: %s.\n' - "Default: '%%default'\n" - "Applies to: BIBTEX output format")%', '.join(TEMPLATE_ALLOWED_FIELDS)), - - Option('--choose-encoding', - default = 'utf8', - dest = 'bibfile_enc', - action = None, - help = _('BibTeX file encoding output.\n' - 'Available types: utf8, cp1252, ascii.\n' - "Default: '%default'\n" - "Applies to: BIBTEX output format")), - - Option('--choose-encoding-configuration', - default = 'strict', - dest = 'bibfile_enctag', - action = None, - help = _('BibTeX file encoding flag.\n' - 'Available types: strict, replace, ignore, backslashreplace.\n' - "Default: '%default'\n" - "Applies to: BIBTEX output format")), - - Option('--entry-type', - default = 'book', - dest = 'bib_entry', - action = None, - help = _('Entry type for BibTeX catalog.\n' - 'Available types: book, misc, mixed.\n' - "Default: '%default'\n" - "Applies to: BIBTEX output format"))] - - def run(self, path_to_output, opts, db, notification=DummyReporter()): - - def create_bibtex_entry(entry, fields, mode, template_citation, - bibtexdict, db, citation_bibtex=True, calibre_files=True): - - #Bibtex doesn't like UTF-8 but keep unicode until writing - #Define starting chain or if book valid strict and not book return a Fail string - - bibtex_entry = [] - if mode != "misc" and check_entry_book_valid(entry) : - bibtex_entry.append(u'@book{') - elif mode != "book" : - bibtex_entry.append(u'@misc{') - else : - #case strict book - return '' - - if citation_bibtex : - # Citation tag - bibtex_entry.append(make_bibtex_citation(entry, template_citation, - bibtexdict)) - bibtex_entry = [u' '.join(bibtex_entry)] - - for field in fields: - if field.startswith('#'): - item = db.get_field(entry['id'],field,index_is_id=True) - if isinstance(item, (bool, float, int)): - item = repr(item) - elif field == 'title_sort': - item = entry['sort'] - else: - item = entry[field] - - #check if the field should be included (none or empty) - if item is None: - continue - try: - if len(item) == 0 : - continue - except TypeError: - pass - - if field == 'authors' : - bibtex_entry.append(u'author = "%s"' % bibtexdict.bibtex_author_format(item)) - - elif field == 'id' : - bibtex_entry.append(u'calibreid = "%s"' % int(item)) - - elif field == 'rating' : - bibtex_entry.append(u'rating = "%s"' % int(item)) - - elif field == 'size' : - bibtex_entry.append(u'%s = "%s octets"' % (field, int(item))) - - elif field == 'tags' : - #A list to flatten - bibtex_entry.append(u'tags = "%s"' % bibtexdict.utf8ToBibtex(u', '.join(item))) - - elif field == 'comments' : - #\n removal - item = item.replace(u'\r\n',u' ') - item = item.replace(u'\n',u' ') - #html to text - try: - item = html2text(item) - except: - log.warn("Failed to convert comments to text") - bibtex_entry.append(u'note = "%s"' % bibtexdict.utf8ToBibtex(item)) - - elif field == 'isbn' : - # Could be 9, 10 or 13 digits - bibtex_entry.append(u'isbn = "%s"' % re.sub(u'[0-9xX]', u'', item)) - - elif field == 'formats' : - #Add file path if format is selected - formats = [format.rpartition('.')[2].lower() for format in item] - bibtex_entry.append(u'formats = "%s"' % u', '.join(formats)) - if calibre_files: - files = [u':%s:%s' % (format, format.rpartition('.')[2].upper())\ - for format in item] - bibtex_entry.append(u'file = "%s"' % u', '.join(files)) - - elif field == 'series_index' : - bibtex_entry.append(u'volume = "%s"' % int(item)) - - elif field == 'timestamp' : - bibtex_entry.append(u'timestamp = "%s"' % isoformat(item).partition('T')[0]) - - elif field == 'pubdate' : - bibtex_entry.append(u'year = "%s"' % item.year) - bibtex_entry.append(u'month = "%s"' % bibtexdict.utf8ToBibtex(strftime("%b", item))) - - elif field.startswith('#') : - bibtex_entry.append(u'custom_%s = "%s"' % (field[1:], - bibtexdict.utf8ToBibtex(item))) - - else: - # elif field in ['title', 'publisher', 'cover', 'uuid', 'ondevice', - # 'author_sort', 'series', 'title_sort'] : - bibtex_entry.append(u'%s = "%s"' % (field, bibtexdict.utf8ToBibtex(item))) - - bibtex_entry = u',\n '.join(bibtex_entry) - bibtex_entry += u' }\n\n' - - return bibtex_entry - - def check_entry_book_valid(entry): - #Check that the required fields are ok for a book entry - for field in ['title', 'authors', 'publisher'] : - if entry[field] is None or len(entry[field]) == 0 : - return False - if entry['pubdate'] is None : - return False - else : - return True - - def make_bibtex_citation(entry, template_citation, bibtexclass): - - #define a function to replace the template entry by its value - def tpl_replace(objtplname) : - - tpl_field = re.sub(u'[\{\}]', u'', objtplname.group()) - - if tpl_field in TEMPLATE_ALLOWED_FIELDS : - if tpl_field in ['pubdate', 'timestamp'] : - tpl_field = isoformat(entry[tpl_field]).partition('T')[0] - elif tpl_field in ['tags', 'authors'] : - tpl_field =entry[tpl_field][0] - elif tpl_field in ['id', 'series_index'] : - tpl_field = str(entry[tpl_field]) - else : - tpl_field = entry[tpl_field] - return tpl_field - else: - return u'' - - if len(template_citation) >0 : - tpl_citation = bibtexclass.utf8ToBibtex( - bibtexclass.ValidateCitationKey(re.sub(u'\{[^{}]*\}', - tpl_replace, template_citation))) - - if len(tpl_citation) >0 : - return tpl_citation - - if len(entry["isbn"]) > 0 : - template_citation = u'%s' % re.sub(u'[\D]',u'', entry["isbn"]) - - else : - template_citation = u'%s' % str(entry["id"]) - - return bibtexclass.ValidateCitationKey(template_citation) - - self.fmt = path_to_output.rpartition('.')[2] - self.notification = notification - - # Combobox options - bibfile_enc = ['utf8', 'cp1252', 'ascii'] - bibfile_enctag = ['strict', 'replace', 'ignore', 'backslashreplace'] - bib_entry = ['mixed', 'misc', 'book'] - - # Needed beacause CLI return str vs int by widget - try: - bibfile_enc = bibfile_enc[opts.bibfile_enc] - bibfile_enctag = bibfile_enctag[opts.bibfile_enctag] - bib_entry = bib_entry[opts.bib_entry] - except: - if opts.bibfile_enc in bibfile_enc : - bibfile_enc = opts.bibfile_enc - else : - log.warn("Incorrect --choose-encoding flag, revert to default") - bibfile_enc = bibfile_enc[0] - if opts.bibfile_enctag in bibfile_enctag : - bibfile_enctag = opts.bibfile_enctag - else : - log.warn("Incorrect --choose-encoding-configuration flag, revert to default") - bibfile_enctag = bibfile_enctag[0] - if opts.bib_entry in bib_entry : - bib_entry = opts.bib_entry - else : - log.warn("Incorrect --entry-type flag, revert to default") - bib_entry = bib_entry[0] - - if opts.verbose: - opts_dict = vars(opts) - log("%s(): Generating %s" % (self.name,self.fmt)) - if opts.connected_device['is_device_connected']: - log(" connected_device: %s" % opts.connected_device['name']) - if opts_dict['search_text']: - log(" --search='%s'" % opts_dict['search_text']) - - if opts_dict['ids']: - log(" Book count: %d" % len(opts_dict['ids'])) - if opts_dict['search_text']: - log(" (--search ignored when a subset of the database is specified)") - - if opts_dict['fields']: - if opts_dict['fields'] == 'all': - log(" Fields: %s" % ', '.join(FIELDS[1:])) - else: - log(" Fields: %s" % opts_dict['fields']) - - log(" Output file will be encoded in %s with %s flag" % (bibfile_enc, bibfile_enctag)) - - log(" BibTeX entry type is %s with a citation like '%s' flag" % (bib_entry, opts_dict['bib_cit'])) - - # If a list of ids are provided, don't use search_text - if opts.ids: - opts.search_text = None - - data = self.search_sort_db(db, opts) - - if not len(data): - log.error("\nNo matching database entries for search criteria '%s'" % opts.search_text) - - # Get the requested output fields as a list - fields = self.get_output_fields(db, opts) - - if not len(data): - log.error("\nNo matching database entries for search criteria '%s'" % opts.search_text) - - #Initialize BibTeX class - bibtexc = BibTeX() - - #Entries writing after Bibtex formating (or not) - if bibfile_enc != 'ascii' : - bibtexc.ascii_bibtex = False - else : - bibtexc.ascii_bibtex = True - - #Check citation choice and go to default in case of bad CLI - if isinstance(opts.impcit, (StringType, UnicodeType)) : - if opts.impcit == 'False' : - citation_bibtex= False - elif opts.impcit == 'True' : - citation_bibtex= True - else : - log.warn("Incorrect --create-citation, revert to default") - citation_bibtex= True - else : - citation_bibtex= opts.impcit - - #Check add file entry and go to default in case of bad CLI - if isinstance(opts.addfiles, (StringType, UnicodeType)) : - if opts.addfiles == 'False' : - addfiles_bibtex = False - elif opts.addfiles == 'True' : - addfiles_bibtex = True - else : - log.warn("Incorrect --add-files-path, revert to default") - addfiles_bibtex= True - else : - addfiles_bibtex = opts.addfiles - - #Preprocess for error and light correction - template_citation = preprocess_template(opts.bib_cit) - - #Open output and write entries - with codecs.open(path_to_output, 'w', bibfile_enc, bibfile_enctag)\ - as outfile: - #File header - nb_entries = len(data) - - #check in book strict if all is ok else throw a warning into log - if bib_entry == 'book' : - nb_books = len(filter(check_entry_book_valid, data)) - if nb_books < nb_entries : - log.warn("Only %d entries in %d are book compatible" % (nb_books, nb_entries)) - nb_entries = nb_books - - # If connected device, add 'On Device' values to data - if opts.connected_device['is_device_connected'] and 'ondevice' in fields: - for entry in data: - entry['ondevice'] = db.catalog_plugin_on_device_temp_mapping[entry['id']]['ondevice'] - - outfile.write(u'%%%Calibre catalog\n%%%{0} entries in catalog\n\n'.format(nb_entries)) - outfile.write(u'@preamble{"This catalog of %d entries was generated by calibre on %s"}\n\n' - % (nb_entries, nowf().strftime("%A, %d. %B %Y %H:%M").decode(preferred_encoding))) - - for entry in data: - outfile.write(create_bibtex_entry(entry, fields, bib_entry, template_citation, - bibtexc, db, citation_bibtex, addfiles_bibtex)) -# }}} - -class EPUB_MOBI(CatalogPlugin): # {{{ - 'ePub catalog generator' - - Option = namedtuple('Option', 'option, default, dest, action, help') - - name = 'Catalog_EPUB_MOBI' - description = 'EPUB/MOBI catalog generator' - supported_platforms = ['windows', 'osx', 'linux'] - minimum_calibre_version = (0, 7, 40) - author = 'Greg Riker' - version = (1, 0, 0) - file_types = set(['epub','mobi']) - - THUMB_SMALLEST = "1.0" - THUMB_LARGEST = "2.0" - - ''' - # Deprecated, keeping this just in case there are complaints - Option('--numbers-as-text', - default=False, - dest='numbers_as_text', - action = None, - help=_("Sort titles with leading numbers as text, e.g.,\n'2001: A Space Odyssey' sorts as \n'Two Thousand One: A Space Odyssey'.\n" - "Default: '%default'\n" - "Applies to: ePub, MOBI output formats")), - ''' - - cli_options = [Option('--catalog-title', - default = 'My Books', - dest = 'catalog_title', - action = None, - help = _('Title of generated catalog used as title in metadata.\n' - "Default: '%default'\n" - "Applies to: ePub, MOBI output formats")), - Option('--debug-pipeline', - default=None, - dest='debug_pipeline', - action = None, - help=_("Save the output from different stages of the conversion " - "pipeline to the specified " - "directory. Useful if you are unsure at which stage " - "of the conversion process a bug is occurring.\n" - "Default: '%default'\n" - "Applies to: ePub, MOBI output formats")), - Option('--exclude-book-marker', - default=':', - dest='exclude_book_marker', - action = None, - help=_("field:pattern specifying custom field/contents indicating book should be excluded.\n" - "Default: '%default'\n" - "Applies to ePub, MOBI output formats")), - Option('--exclude-genre', - default='\[.+\]', - dest='exclude_genre', - action = None, - help=_("Regex describing tags to exclude as genres.\n" "Default: '%default' excludes bracketed tags, e.g. '[]'\n" - "Applies to: ePub, MOBI output formats")), - Option('--exclude-tags', - default=('~,'+_('Catalog')), - dest='exclude_tags', - action = None, - help=_("Comma-separated list of tag words indicating book should be excluded from output. " - "For example: 'skip' will match 'skip this book' and 'Skip will like this'. " - "Default: '%default'\n" - "Applies to: ePub, MOBI output formats")), - Option('--generate-authors', - default=False, - dest='generate_authors', - action = 'store_true', - help=_("Include 'Authors' section in catalog.\n" - "Default: '%default'\n" - "Applies to: ePub, MOBI output formats")), - Option('--generate-descriptions', - default=False, - dest='generate_descriptions', - action = 'store_true', - help=_("Include 'Descriptions' section in catalog.\n" - "Default: '%default'\n" - "Applies to: ePub, MOBI output formats")), - Option('--generate-genres', - default=False, - dest='generate_genres', - action = 'store_true', - help=_("Include 'Genres' section in catalog.\n" - "Default: '%default'\n" - "Applies to: ePub, MOBI output formats")), - Option('--generate-titles', - default=False, - dest='generate_titles', - action = 'store_true', - help=_("Include 'Titles' section in catalog.\n" - "Default: '%default'\n" - "Applies to: ePub, MOBI output formats")), - Option('--generate-series', - default=False, - dest='generate_series', - action = 'store_true', - help=_("Include 'Series' section in catalog.\n" - "Default: '%default'\n" - "Applies to: ePub, MOBI output formats")), - Option('--generate-recently-added', - default=False, - dest='generate_recently_added', - action = 'store_true', - help=_("Include 'Recently Added' section in catalog.\n" - "Default: '%default'\n" - "Applies to: ePub, MOBI output formats")), - Option('--header-note-source-field', - default='', - dest='header_note_source_field', - action = None, - help=_("Custom field containing note text to insert in Description header.\n" - "Default: '%default'\n" - "Applies to: ePub, MOBI output formats")), - Option('--merge-comments', - default='::', - dest='merge_comments', - action = None, - help=_(":[before|after]:[True|False] specifying:\n" - " Custom field containing notes to merge with Comments\n" - " [before|after] Placement of notes with respect to Comments\n" - " [True|False] - A horizontal rule is inserted between notes and Comments\n" - "Default: '%default'\n" - "Applies to ePub, MOBI output formats")), - Option('--output-profile', - default=None, - dest='output_profile', - action = None, - help=_("Specifies the output profile. In some cases, an output profile is required to optimize the catalog for the device. For example, 'kindle' or 'kindle_dx' creates a structured Table of Contents with Sections and Articles.\n" - "Default: '%default'\n" - "Applies to: ePub, MOBI output formats")), - Option('--read-book-marker', - default='tag:+', - dest='read_book_marker', - action = None, - help=_("field:pattern indicating book has been read.\n" "Default: '%default'\n" - "Applies to ePub, MOBI output formats")), - Option('--thumb-width', - default='1.0', - dest='thumb_width', - action = None, - help=_("Size hint (in inches) for book covers in catalog.\n" - "Range: 1.0 - 2.0\n" - "Default: '%default'\n" - "Applies to ePub, MOBI output formats")), - Option('--wishlist-tag', - default='Wishlist', - dest='wishlist_tag', - action = None, - help=_("Tag indicating book to be displayed as wishlist item.\n" "Default: '%default'\n" - "Applies to: ePub, MOBI output formats")), - ] - - class NumberToText(object): - ''' - Converts numbers to text - 4.56 => four point fifty-six - 456 => four hundred fifty-six - 4:56 => four fifty-six - ''' - ORDINALS = ['zeroth','first','second','third','fourth','fifth','sixth','seventh','eighth','ninth'] - lessThanTwenty = ["","one","two","three","four","five","six","seven","eight","nine", - "ten","eleven","twelve","thirteen","fourteen","fifteen","sixteen","seventeen", - "eighteen","nineteen"] - tens = ["","","twenty","thirty","forty","fifty","sixty","seventy","eighty","ninety"] - hundreds = ["","one","two","three","four","five","six","seven","eight","nine"] - - def __init__(self, number, verbose=False): - self.number = number - self.number_as_float = 0.0 - self.text = '' - self.verbose = verbose - self.log = log - self.numberTranslate() - - def stringFromInt(self, intToTranslate): - # Convert intToTranslate to string - # intToTranslate is a three-digit number - - tensComponentString = "" - hundredsComponent = intToTranslate - (intToTranslate % 100) - tensComponent = intToTranslate % 100 - - # Build the hundreds component - if hundredsComponent: - hundredsComponentString = "%s hundred" % self.hundreds[hundredsComponent/100] - else: - hundredsComponentString = "" - - # Build the tens component - if tensComponent < 20: - tensComponentString = self.lessThanTwenty[tensComponent] - else: - tensPart = "" - onesPart = "" - - # Get the tens part - tensPart = self.tens[tensComponent / 10] - onesPart = self.lessThanTwenty[tensComponent % 10] - - if intToTranslate % 10: - tensComponentString = "%s-%s" % (tensPart, onesPart) - else: - tensComponentString = "%s" % tensPart - - # Concatenate the results - result = '' - if hundredsComponent and not tensComponent: - result = hundredsComponentString - elif not hundredsComponent and tensComponent: - result = tensComponentString - elif hundredsComponent and tensComponent: - result = hundredsComponentString + " " + tensComponentString - else: - prints(" NumberToText.stringFromInt(): empty result translating %d" % intToTranslate) - return result - - def numberTranslate(self): - hundredsNumber = 0 - thousandsNumber = 0 - hundredsString = "" - thousandsString = "" - resultString = "" - self.suffix = '' - - if self.verbose: self.log("numberTranslate(): %s" % self.number) - - # Special case ordinals - if re.search('[st|nd|rd|th]',self.number): - self.number = re.sub(',','',self.number) - ordinal_suffix = re.search('[\D]', self.number) - ordinal_number = re.sub('\D','',re.sub(',','',self.number)) - if self.verbose: self.log("Ordinal: %s" % ordinal_number) - self.number_as_float = ordinal_number - self.suffix = self.number[ordinal_suffix.start():] - if int(ordinal_number) > 9: - # Some typos (e.g., 'twentyth'), acceptable - self.text = '%s' % (EPUB_MOBI.NumberToText(ordinal_number).text) - else: - self.text = '%s' % (self.ORDINALS[int(ordinal_number)]) - - # Test for time - elif re.search(':',self.number): - if self.verbose: self.log("Time: %s" % self.number) - self.number_as_float = re.sub(':','.',self.number) - time_strings = self.number.split(":") - hours = EPUB_MOBI.NumberToText(time_strings[0]).text - minutes = EPUB_MOBI.NumberToText(time_strings[1]).text - self.text = '%s-%s' % (hours.capitalize(), minutes) - - # Test for % - elif re.search('%', self.number): - if self.verbose: self.log("Percent: %s" % self.number) - self.number_as_float = self.number.split('%')[0] - self.text = EPUB_MOBI.NumberToText(self.number.replace('%',' percent')).text - - # Test for decimal - elif re.search('\.',self.number): - if self.verbose: self.log("Decimal: %s" % self.number) - self.number_as_float = self.number - decimal_strings = self.number.split(".") - left = EPUB_MOBI.NumberToText(decimal_strings[0]).text - right = EPUB_MOBI.NumberToText(decimal_strings[1]).text - self.text = '%s point %s' % (left.capitalize(), right) - - # Test for hypenated - elif re.search('-', self.number): - if self.verbose: self.log("Hyphenated: %s" % self.number) - self.number_as_float = self.number.split('-')[0] - strings = self.number.split('-') - if re.search('[0-9]+', strings[0]): - left = EPUB_MOBI.NumberToText(strings[0]).text - right = strings[1] - else: - left = strings[0] - right = EPUB_MOBI.NumberToText(strings[1]).text - self.text = '%s-%s' % (left, right) - - # Test for only commas and numbers - elif re.search(',', self.number) and not re.search('[^0-9,]',self.number): - if self.verbose: self.log("Comma(s): %s" % self.number) - self.number_as_float = re.sub(',','',self.number) - self.text = EPUB_MOBI.NumberToText(self.number_as_float).text - - # Test for hybrid e.g., 'K2, 2nd, 10@10' - elif re.search('[\D]+', self.number): - if self.verbose: self.log("Hybrid: %s" % self.number) - # Split the token into number/text - number_position = re.search('\d',self.number).start() - text_position = re.search('\D',self.number).start() - if number_position < text_position: - number = self.number[:text_position] - text = self.number[text_position:] - self.text = '%s%s' % (EPUB_MOBI.NumberToText(number).text,text) - else: - text = self.number[:number_position] - number = self.number[number_position:] - self.text = '%s%s' % (text, EPUB_MOBI.NumberToText(number).text) - - else: - if self.verbose: self.log("Clean: %s" % self.number) - try: - self.float_as_number = float(self.number) - number = int(self.number) - except: - return - - if number > 10**9: - self.text = "%d out of range" % number - return - - if number == 10**9: - self.text = "one billion" - else : - # Isolate the three-digit number groups - millionsNumber = number/10**6 - thousandsNumber = (number - (millionsNumber * 10**6))/10**3 - hundredsNumber = number - (millionsNumber * 10**6) - (thousandsNumber * 10**3) - if self.verbose: - print "Converting %s %s %s" % (millionsNumber, thousandsNumber, hundredsNumber) - - # Convert hundredsNumber - if hundredsNumber : - hundredsString = self.stringFromInt(hundredsNumber) - - # Convert thousandsNumber - if thousandsNumber: - if number > 1099 and number < 2000: - resultString = '%s %s' % (self.lessThanTwenty[number/100], - self.stringFromInt(number % 100)) - self.text = resultString.strip().capitalize() - return - else: - thousandsString = self.stringFromInt(thousandsNumber) - - # Convert millionsNumber - if millionsNumber: - millionsString = self.stringFromInt(millionsNumber) - - # Concatenate the strings - resultString = '' - if millionsNumber: - resultString += "%s million " % millionsString - - if thousandsNumber: - resultString += "%s thousand " % thousandsString - - if hundredsNumber: - resultString += "%s" % hundredsString - - if not millionsNumber and not thousandsNumber and not hundredsNumber: - resultString = "zero" - - if self.verbose: - self.log(u'resultString: %s' % resultString) - self.text = resultString.strip().capitalize() - - class CatalogBuilder(object): - ''' - Generates catalog source files from calibre database - - Flow of control: - gui2.actions.catalog:generate_catalog() - gui2.tools:generate_catalog() or library.cli:command_catalog() - called from gui2.convert.gui_conversion:gui_catalog() - catalog = Catalog(notification=Reporter()) - catalog.createDirectoryStructure() - catalog.copyResources() - catalog.buildSources() - Options managed in gui2.catalog.catalog_epub_mobi.py - ''' - - # A single number creates 'Last x days' only. - # Multiple numbers create 'Last x days', 'x to y days ago' ... - # e.g, [7,15,30,60], [30] - # [] = No date ranges added - DATE_RANGE=[30] - - # basename output file basename - # creator dc:creator in OPF metadata - # descriptionClip limits size of NCX descriptions (Kindle only) - # includeSources Used in processSpecialTags to skip tags like '[SPL]' - # notification Used to check for cancel, report progress - # stylesheet CSS stylesheet - # title dc:title in OPF metadata, NCX periodical - # verbosity level of diagnostic printout - - def __init__(self, db, opts, plugin, - report_progress=DummyReporter(), - stylesheet="content/stylesheet.css"): - self.__opts = opts - self.__authorClip = opts.authorClip - self.__authors = None - self.__basename = opts.basename - self.__bookmarked_books = None - self.__booksByAuthor = None - self.__booksByDateRead = None - self.__booksByTitle = None - self.__booksByTitle_noSeriesPrefix = None - self.__cache_dir = os.path.join(config_dir, 'caches', 'catalog') - self.__archive_path = os.path.join(self.__cache_dir, "thumbs.zip") - self.__catalogPath = PersistentTemporaryDirectory("_epub_mobi_catalog", prefix='') - self.__contentDir = os.path.join(self.catalogPath, "content") - self.__currentStep = 0.0 - self.__creator = opts.creator - self.__db = db - self.__descriptionClip = opts.descriptionClip - self.__error = [] - self.__generateForKindle = True if (self.opts.fmt == 'mobi' and \ - self.opts.output_profile and \ - self.opts.output_profile.startswith("kindle")) else False - self.__generateRecentlyRead = True if self.opts.generate_recently_added \ - and self.opts.connected_kindle \ - and self.generateForKindle \ - else False - self.__genres = None - self.genres = [] - self.__genre_tags_dict = None - self.__htmlFileList_1 = [] - self.__htmlFileList_2 = [] - self.__markerTags = self.getMarkerTags() - self.__ncxSoup = None - self.__output_profile = None - self.__playOrder = 1 - self.__plugin = plugin - self.__progressInt = 0.0 - self.__progressString = '' - f, _, p = opts.read_book_marker.partition(':') - self.__read_book_marker = {'field':f, 'pattern':p} - f, p, hr = self.opts.merge_comments.split(':') - self.__merge_comments = {'field':f, 'position':p, 'hr':hr} - self.__reporter = report_progress - self.__stylesheet = stylesheet - self.__thumbs = None - self.__thumbWidth = 0 - self.__thumbHeight = 0 - self.__title = opts.catalog_title - self.__totalSteps = 6.0 - self.__useSeriesPrefixInTitlesSection = False - self.__verbose = opts.verbose - - from calibre.customize.ui import output_profiles - for profile in output_profiles(): - if profile.short_name == self.opts.output_profile: - self.__output_profile = profile - break - - # Confirm/create thumbs archive. - if self.opts.generate_descriptions: - if not os.path.exists(self.__cache_dir): - self.opts.log.info(" creating new thumb cache '%s'" % self.__cache_dir) - os.makedirs(self.__cache_dir) - if not os.path.exists(self.__archive_path): - self.opts.log.info(' creating thumbnail archive, thumb_width: %1.2f"' % - float(self.opts.thumb_width)) - with ZipFile(self.__archive_path, mode='w') as zfw: - zfw.writestr("Catalog Thumbs Archive",'') - else: - try: - with ZipFile(self.__archive_path, mode='r') as zfr: - try: - cached_thumb_width = zfr.read('thumb_width') - except: - cached_thumb_width = "-1" - except: - os.remove(self.__archive_path) - cached_thumb_width = '-1' - - if float(cached_thumb_width) != float(self.opts.thumb_width): - self.opts.log.warning(" invalidating cache at '%s'" % self.__archive_path) - self.opts.log.warning(' thumb_width changed: %1.2f" => %1.2f"' % - (float(cached_thumb_width),float(self.opts.thumb_width))) - with ZipFile(self.__archive_path, mode='w') as zfw: - zfw.writestr("Catalog Thumbs Archive",'') - else: - self.opts.log.info(' existing thumb cache at %s, cached_thumb_width: %1.2f"' % - (self.__archive_path, float(cached_thumb_width))) - - # Tweak build steps based on optional sections: 1 call for HTML, 1 for NCX - incremental_jobs = 0 - if self.opts.generate_authors: - incremental_jobs += 2 - if self.opts.generate_titles: - incremental_jobs += 2 - if self.opts.generate_recently_added: - incremental_jobs += 2 - if self.generateRecentlyRead: - incremental_jobs += 2 - if self.opts.generate_series: - incremental_jobs += 2 - if self.opts.generate_descriptions: - # +1 thumbs - incremental_jobs += 3 - self.__totalSteps += incremental_jobs - - # Load section list templates - templates = {} - execfile(P('catalog/section_list_templates.py'), templates) - for name, template in templates.iteritems(): - if name.startswith('by_') and name.endswith('_template'): - setattr(self, name, force_unicode(template, 'utf-8')) - - # Accessors - if True: - ''' - @dynamic_property - def xxxx(self): - def fget(self): - return self.__ - def fset(self, val): - self.__ = val - return property(fget=fget, fset=fset) - ''' - @dynamic_property - def authorClip(self): - def fget(self): - return self.__authorClip - def fset(self, val): - self.__authorClip = val - return property(fget=fget, fset=fset) - @dynamic_property - def authors(self): - def fget(self): - return self.__authors - def fset(self, val): - self.__authors = val - return property(fget=fget, fset=fset) - @dynamic_property - def basename(self): - def fget(self): - return self.__basename - def fset(self, val): - self.__basename = val - return property(fget=fget, fset=fset) - @dynamic_property - def bookmarked_books(self): - def fget(self): - return self.__bookmarked_books - def fset(self, val): - self.__bookmarked_books = val - return property(fget=fget, fset=fset) - @dynamic_property - def booksByAuthor(self): - def fget(self): - return self.__booksByAuthor - def fset(self, val): - self.__booksByAuthor = val - return property(fget=fget, fset=fset) - @dynamic_property - def booksByDateRead(self): - def fget(self): - return self.__booksByDateRead - def fset(self, val): - self.__booksByDateRead = val - return property(fget=fget, fset=fset) - @dynamic_property - def booksByTitle(self): - def fget(self): - return self.__booksByTitle - def fset(self, val): - self.__booksByTitle = val - return property(fget=fget, fset=fset) - @dynamic_property - def booksByTitle_noSeriesPrefix(self): - def fget(self): - return self.__booksByTitle_noSeriesPrefix - def fset(self, val): - self.__booksByTitle_noSeriesPrefix = val - return property(fget=fget, fset=fset) - @dynamic_property - def catalogPath(self): - def fget(self): - return self.__catalogPath - def fset(self, val): - self.__catalogPath = val - return property(fget=fget, fset=fset) - @dynamic_property - def contentDir(self): - def fget(self): - return self.__contentDir - def fset(self, val): - self.__contentDir = val - return property(fget=fget, fset=fset) - @dynamic_property - def currentStep(self): - def fget(self): - return self.__currentStep - def fset(self, val): - self.__currentStep = val - return property(fget=fget, fset=fset) - @dynamic_property - def creator(self): - def fget(self): - return self.__creator - def fset(self, val): - self.__creator = val - return property(fget=fget, fset=fset) - @dynamic_property - def db(self): - def fget(self): - return self.__db - return property(fget=fget) - @dynamic_property - def descriptionClip(self): - def fget(self): - return self.__descriptionClip - def fset(self, val): - self.__descriptionClip = val - return property(fget=fget, fset=fset) - @dynamic_property - def error(self): - def fget(self): - return self.__error - def fset(self, val): - self.__error = val - return property(fget=fget,fset=fset) - @dynamic_property - def generateForKindle(self): - def fget(self): - return self.__generateForKindle - def fset(self, val): - self.__generateForKindle = val - return property(fget=fget, fset=fset) - @dynamic_property - def generateRecentlyRead(self): - def fget(self): - return self.__generateRecentlyRead - def fset(self, val): - self.__generateRecentlyRead = val - return property(fget=fget, fset=fset) - @dynamic_property - def genres(self): - def fget(self): - return self.__genres - def fset(self, val): - self.__genres = val - return property(fget=fget, fset=fset) - @dynamic_property - def genre_tags_dict(self): - def fget(self): - return self.__genre_tags_dict - def fset(self, val): - self.__genre_tags_dict = val - return property(fget=fget, fset=fset) - @dynamic_property - def htmlFileList_1(self): - def fget(self): - return self.__htmlFileList_1 - def fset(self, val): - self.__htmlFileList_1 = val - return property(fget=fget, fset=fset) - @dynamic_property - def htmlFileList_2(self): - def fget(self): - return self.__htmlFileList_2 - def fset(self, val): - self.__htmlFileList_2 = val - return property(fget=fget, fset=fset) - @dynamic_property - def libraryPath(self): - def fget(self): - return self.__libraryPath - def fset(self, val): - self.__libraryPath = val - return property(fget=fget, fset=fset) - @dynamic_property - def markerTags(self): - def fget(self): - return self.__markerTags - def fset(self, val): - self.__markerTags = val - return property(fget=fget, fset=fset) - @dynamic_property - def ncxSoup(self): - def fget(self): - return self.__ncxSoup - def fset(self, val): - self.__ncxSoup = val - return property(fget=fget, fset=fset) - @dynamic_property - def opts(self): - def fget(self): - return self.__opts - return property(fget=fget) - @dynamic_property - def playOrder(self): - def fget(self): - return self.__playOrder - def fset(self,val): - self.__playOrder = val - return property(fget=fget, fset=fset) - @dynamic_property - def plugin(self): - def fget(self): - return self.__plugin - return property(fget=fget) - @dynamic_property - def progressInt(self): - def fget(self): - return self.__progressInt - def fset(self, val): - self.__progressInt = val - return property(fget=fget, fset=fset) - @dynamic_property - def progressString(self): - def fget(self): - return self.__progressString - def fset(self, val): - self.__progressString = val - return property(fget=fget, fset=fset) - @dynamic_property - def reporter(self): - def fget(self): - return self.__reporter - def fset(self, val): - self.__reporter = val - return property(fget=fget, fset=fset) - @dynamic_property - def stylesheet(self): - def fget(self): - return self.__stylesheet - def fset(self, val): - self.__stylesheet = val - return property(fget=fget, fset=fset) - @dynamic_property - def thumbs(self): - def fget(self): - return self.__thumbs - def fset(self, val): - self.__thumbs = val - return property(fget=fget, fset=fset) - def thumbWidth(self): - def fget(self): - return self.__thumbWidth - def fset(self, val): - self.__thumbWidth = val - return property(fget=fget, fset=fset) - def thumbHeight(self): - def fget(self): - return self.__thumbHeight - def fset(self, val): - self.__thumbHeight = val - return property(fget=fget, fset=fset) - @dynamic_property - def title(self): - def fget(self): - return self.__title - def fset(self, val): - self.__title = val - return property(fget=fget, fset=fset) - @dynamic_property - def totalSteps(self): - def fget(self): - return self.__totalSteps - return property(fget=fget) - @dynamic_property - def useSeriesPrefixInTitlesSection(self): - def fget(self): - return self.__useSeriesPrefixInTitlesSection - def fset(self, val): - self.__useSeriesPrefixInTitlesSection = val - return property(fget=fget, fset=fset) - @dynamic_property - def verbose(self): - def fget(self): - return self.__verbose - def fset(self, val): - self.__verbose = val - return property(fget=fget, fset=fset) - - @dynamic_property - def MISSING_SYMBOL(self): - def fget(self): - return self.__output_profile.missing_char - return property(fget=fget) - @dynamic_property - def NOT_READ_SYMBOL(self): - def fget(self): - return '%s' % self.__output_profile.read_char - return property(fget=fget) - @dynamic_property - def READING_SYMBOL(self): - def fget(self): - return '▷' if self.generateForKindle else \ - '+' - return property(fget=fget) - @dynamic_property - def READ_SYMBOL(self): - def fget(self): - return self.__output_profile.read_char - return property(fget=fget) - @dynamic_property - def FULL_RATING_SYMBOL(self): - def fget(self): - return self.__output_profile.ratings_char - return property(fget=fget) - @dynamic_property - def EMPTY_RATING_SYMBOL(self): - def fget(self): - return self.__output_profile.empty_ratings_char - return property(fget=fget) - @dynamic_property - def READ_PROGRESS_SYMBOL(self): - def fget(self): - return "▪" if self.generateForKindle else '+' - return property(fget=fget) - @dynamic_property - def UNREAD_PROGRESS_SYMBOL(self): - def fget(self): - return "▫" if self.generateForKindle else '-' - return property(fget=fget) - - # Methods - def buildSources(self): - if self.booksByTitle is None: - if not self.fetchBooksByTitle(): - return False - if not self.fetchBooksByAuthor(): - return False - self.fetchBookmarks() - if self.opts.generate_descriptions: - self.generateThumbnails() - self.generateHTMLDescriptions() - if self.opts.generate_authors: - self.generateHTMLByAuthor() - if self.opts.generate_titles: - self.generateHTMLByTitle() - if self.opts.generate_series: - self.generateHTMLBySeries() - if self.opts.generate_genres: - self.generateHTMLByTags() - # If this is the only Section, and there are no genres, bail - if self.opts.section_list == ['Genres'] and not self.genres: - error_msg = _("No enabled genres found to catalog.\n") - if not self.opts.cli_environment: - error_msg += "Check 'Excluded genres'\nin E-book options.\n" - self.opts.log.error(error_msg) - self.error.append(_('No books available to catalog')) - self.error.append(error_msg) - return False - if self.opts.generate_recently_added: - self.generateHTMLByDateAdded() - if self.generateRecentlyRead: - self.generateHTMLByDateRead() - - self.generateOPF() - self.generateNCXHeader() - if self.opts.generate_authors: - self.generateNCXByAuthor("Authors") - if self.opts.generate_titles: - self.generateNCXByTitle("Titles") - if self.opts.generate_series: - self.generateNCXBySeries("Series") - if self.opts.generate_genres: - self.generateNCXByGenre("Genres") - if self.opts.generate_recently_added: - self.generateNCXByDateAdded("Recently Added") - if self.generateRecentlyRead: - self.generateNCXByDateRead("Recently Read") - if self.opts.generate_descriptions: - self.generateNCXDescriptions("Descriptions") - - self.writeNCX() - return True - - def cleanUp(self): - pass - - def copyResources(self): - '''Move resource files to self.catalogPath''' - catalog_resources = P("catalog") - - files_to_copy = [('','DefaultCover.jpg'), - ('content','stylesheet.css'), - ('images','mastheadImage.gif')] - - for file in files_to_copy: - if file[0] == '': - shutil.copy(os.path.join(catalog_resources,file[1]), - self.catalogPath) - else: - shutil.copy(os.path.join(catalog_resources,file[1]), - os.path.join(self.catalogPath, file[0])) - - # Create the custom masthead image overwriting default - # If failure, default mastheadImage.gif should still be in place - if self.generateForKindle: - try: - self.generateMastheadImage(os.path.join(self.catalogPath, - 'images/mastheadImage.gif')) - except: - pass - - def fetchBooksByAuthor(self): - ''' - Generate a list of titles sorted by author from the database - return = Success - ''' - - self.updateProgressFullStep("Sorting database") - self.booksByAuthor = list(self.booksByTitle) - - # Test for author_sort mismatches - self.booksByAuthor = sorted(self.booksByAuthor, key=self.booksByAuthorSorter_author) - # Build the unique_authors set from existing data - authors = [(record['author'], record['author_sort']) for record in self.booksByAuthor] - current_author = authors[0] - for (i,author) in enumerate(authors): - if author != current_author and i: - if author[0] == current_author[0]: - if self.opts.fmt == 'mobi': - # Exit if building MOBI - error_msg = _( -'''Inconsistent Author Sort values for -Author '{0}': -'{1}' <> '{2}' -Unable to build MOBI catalog.\n -Select all books by '{0}', apply correct Author Sort value in Edit Metadata dialog, then rebuild the catalog.\n''').format(author[0],author[1],current_author[1]) - self.opts.log.warn('\n*** Metadata error ***') - self.opts.log.warn(error_msg) - - self.error.append('Author Sort mismatch') - self.error.append(error_msg) - return False - else: - # Warning if building non-MOBI - if not self.error: - self.error.append('Author Sort mismatch') - - error_msg = _( -'''Warning: inconsistent Author Sort values for -Author '{0}': -'{1}' <> '{2}'\n''').format(author[0],author[1],current_author[1]) - self.opts.log.warn('\n*** Metadata warning ***') - self.opts.log.warn(error_msg) - self.error.append(error_msg) - - current_author = author - - self.booksByAuthor = sorted(self.booksByAuthor, key=self.booksByAuthorSorter_author_sort) - - # Build the unique_authors set from existing data - authors = [(record['author'], capitalize(record['author_sort'])) for record in self.booksByAuthor] - - # authors[] contains a list of all book authors, with multiple entries for multiple books by author - # authors[]: (([0]:friendly [1]:sort)) - # unique_authors[]: (([0]:friendly [1]:sort [2]:book_count)) - books_by_current_author = 0 - current_author = authors[0] - multiple_authors = False - unique_authors = [] - for (i,author) in enumerate(authors): - if author != current_author: - # Note that current_author and author are tuples: (friendly, sort) - multiple_authors = True - - # New author, save the previous author/sort/count - unique_authors.append((current_author[0], icu_title(current_author[1]), - books_by_current_author)) - current_author = author - books_by_current_author = 1 - elif i==0 and len(authors) == 1: - # Allow for single-book lists - unique_authors.append((current_author[0], icu_title(current_author[1]), - books_by_current_author)) - else: - books_by_current_author += 1 - else: - # Add final author to list or single-author dataset - if (current_author == author and len(authors) > 1) or not multiple_authors: - unique_authors.append((current_author[0], icu_title(current_author[1]), - books_by_current_author)) - - if False and self.verbose: - self.opts.log.info("\nfetchBooksByauthor(): %d unique authors" % len(unique_authors)) - for author in unique_authors: - self.opts.log.info((u" %-50s %-25s %2d" % (author[0][0:45], author[1][0:20], - author[2])).encode('utf-8')) - - self.authors = unique_authors - return True - - def fetchBooksByTitle(self): - self.updateProgressFullStep("Fetching database") - - self.opts.sort_by = 'title' - - # Merge opts.exclude_tags with opts.search_text - # Updated to use exact match syntax - empty_exclude_tags = False if len(self.opts.exclude_tags) else True - search_phrase = '' - if not empty_exclude_tags: - exclude_tags = self.opts.exclude_tags.split(',') - search_terms = [] - for tag in exclude_tags: - search_terms.append("tag:=%s" % tag) - search_phrase = "not (%s)" % " or ".join(search_terms) - # If a list of ids are provided, don't use search_text - if self.opts.ids: - self.opts.search_text = search_phrase - else: - if self.opts.search_text: - self.opts.search_text += " " + search_phrase - else: - self.opts.search_text = search_phrase - - # Fetch the database as a dictionary - data = self.plugin.search_sort_db(self.db, self.opts) - data = self.processExclusions(data) - - # Populate this_title{} from data[{},{}] - titles = [] - for record in data: - this_title = {} - - this_title['id'] = record['id'] - this_title['uuid'] = record['uuid'] - - this_title['title'] = self.convertHTMLEntities(record['title']) - if record['series']: - this_title['series'] = record['series'] - this_title['series_index'] = record['series_index'] - else: - this_title['series'] = None - this_title['series_index'] = 0.0 - - this_title['title_sort'] = self.generateSortTitle(this_title['title']) - if 'authors' in record: - # from calibre.ebooks.metadata import authors_to_string - # return authors_to_string(self.authors) - - this_title['authors'] = record['authors'] - if record['authors']: - this_title['author'] = " & ".join(record['authors']) - else: - this_title['author'] = 'Unknown' - - if 'author_sort' in record and record['author_sort'].strip(): - this_title['author_sort'] = record['author_sort'] - else: - this_title['author_sort'] = self.author_to_author_sort(this_title['author']) - - if record['publisher']: - this_title['publisher'] = re.sub('&', '&', record['publisher']) - - this_title['rating'] = record['rating'] if record['rating'] else 0 - - if is_date_undefined(record['pubdate']): - this_title['date'] = None - else: - this_title['date'] = strftime(u'%B %Y', record['pubdate'].timetuple()) - - this_title['timestamp'] = record['timestamp'] - - if record['comments']: - # Strip annotations - a_offset = record['comments'].find('
') - ad_offset = record['comments'].find('
') - if a_offset >= 0: - record['comments'] = record['comments'][:a_offset] - if ad_offset >= 0: - record['comments'] = record['comments'][:ad_offset] - - this_title['description'] = self.markdownComments(record['comments']) - - # Create short description - paras = BeautifulSoup(this_title['description']).findAll('p') - tokens = [] - for p in paras: - for token in p.contents: - if token.string is not None: - tokens.append(token.string) - this_title['short_description'] = self.generateShortDescription(' '.join(tokens), dest="description") - else: - this_title['description'] = None - this_title['short_description'] = None - - # Merge with custom field/value - if self.__merge_comments['field']: - this_title['description'] = self.mergeComments(this_title) - - if record['cover']: - this_title['cover'] = re.sub('&', '&', record['cover']) - - this_title['read'] = self.discoverReadStatus(record) - - if record['tags']: - this_title['tags'] = self.processSpecialTags(record['tags'], - this_title, self.opts) - if record['formats']: - formats = [] - for format in record['formats']: - formats.append(self.convertHTMLEntities(format)) - this_title['formats'] = formats - - # Add user notes to be displayed in header - # Special case handling for datetime fields and lists - if self.opts.header_note_source_field: - field_md = self.__db.metadata_for_field(self.opts.header_note_source_field) - notes = self.__db.get_field(record['id'], - self.opts.header_note_source_field, - index_is_id=True) - if notes: - if field_md['datatype'] == 'text': - if isinstance(notes,list): - notes = ' · '.join(notes) - elif field_md['datatype'] == 'datetime': - notes = format_date(notes,'dd MMM yyyy') - this_title['notes'] = {'source':field_md['name'], - 'content':notes} - - titles.append(this_title) - - # Re-sort based on title_sort - if len(titles): - self.booksByTitle = sorted(titles, - key=lambda x:(x['title_sort'].upper(), x['title_sort'].upper())) - if False and self.verbose: - self.opts.log.info("fetchBooksByTitle(): %d books" % len(self.booksByTitle)) - self.opts.log.info(" %-40s %-40s" % ('title', 'title_sort')) - for title in self.booksByTitle: - self.opts.log.info((u" %-40s %-40s" % (title['title'][0:40], - title['title_sort'][0:40])).decode('mac-roman')) - return True - else: - error_msg = _("No books found to catalog.\nCheck 'Excluded books' criteria in E-book options.\n") - self.opts.log.error('*** ' + error_msg + ' ***') - self.error.append(_('No books available to include in catalog')) - self.error.append(error_msg) - return False - - def fetchBookmarks(self): - ''' - Collect bookmarks for catalog entries - This will use the system default save_template specified in - Preferences|Add/Save|Sending to device, not a customized one specified in - the Kindle plugin - ''' - from calibre.devices.usbms.device import Device - from calibre.devices.kindle.driver import Bookmark - from calibre.ebooks.metadata import MetaInformation - - MBP_FORMATS = [u'azw', u'mobi', u'prc', u'txt'] - mbp_formats = set(MBP_FORMATS) - PDR_FORMATS = [u'pdf'] - pdr_formats = set(PDR_FORMATS) - TAN_FORMATS = [u'tpz', u'azw1'] - tan_formats = set(TAN_FORMATS) - - class BookmarkDevice(Device): - def initialize(self, save_template): - self._save_template = save_template - self.SUPPORTS_SUB_DIRS = True - def save_template(self): - return self._save_template - - def resolve_bookmark_paths(storage, path_map): - pop_list = [] - book_ext = {} - for id in path_map: - file_fmts = set() - for fmt in path_map[id]['fmts']: - file_fmts.add(fmt) - - bookmark_extension = None - if file_fmts.intersection(mbp_formats): - book_extension = list(file_fmts.intersection(mbp_formats))[0] - bookmark_extension = 'mbp' - elif file_fmts.intersection(tan_formats): - book_extension = list(file_fmts.intersection(tan_formats))[0] - bookmark_extension = 'tan' - elif file_fmts.intersection(pdr_formats): - book_extension = list(file_fmts.intersection(pdr_formats))[0] - bookmark_extension = 'pdr' - - if bookmark_extension: - for vol in storage: - bkmk_path = path_map[id]['path'].replace(os.path.abspath('/'),vol) - bkmk_path = bkmk_path.replace('bookmark',bookmark_extension) - if os.path.exists(bkmk_path): - path_map[id] = bkmk_path - book_ext[id] = book_extension - break - else: - pop_list.append(id) - else: - pop_list.append(id) - # Remove non-existent bookmark templates - for id in pop_list: - path_map.pop(id) - return path_map, book_ext - - if self.generateRecentlyRead: - self.opts.log.info(" Collecting Kindle bookmarks matching catalog entries") - - d = BookmarkDevice(None) - d.initialize(self.opts.connected_device['save_template']) - - bookmarks = {} - for book in self.booksByTitle: - if 'formats' in book: - path_map = {} - id = book['id'] - original_title = book['title'][book['title'].find(':') + 2:] if book['series'] \ - else book['title'] - myMeta = MetaInformation(original_title, - authors=book['authors']) - myMeta.author_sort = book['author_sort'] - a_path = d.create_upload_path('/', myMeta, 'x.bookmark', create_dirs=False) - path_map[id] = dict(path=a_path, fmts=[x.rpartition('.')[2] for x in book['formats']]) - - path_map, book_ext = resolve_bookmark_paths(self.opts.connected_device['storage'], path_map) - if path_map: - bookmark_ext = path_map[id].rpartition('.')[2] - myBookmark = Bookmark(path_map[id], id, book_ext[id], bookmark_ext) - try: - book['percent_read'] = min(float(100*myBookmark.last_read / myBookmark.book_length),100) - except: - book['percent_read'] = 0 - dots = int((book['percent_read'] + 5)/10) - dot_string = self.READ_PROGRESS_SYMBOL * dots - empty_dots = self.UNREAD_PROGRESS_SYMBOL * (10 - dots) - book['reading_progress'] = '%s%s' % (dot_string,empty_dots) - bookmarks[id] = ((myBookmark,book)) - - self.bookmarked_books = bookmarks - else: - self.bookmarked_books = {} - - def generateHTMLDescriptions(self): - ''' - Write each title to a separate HTML file in contentdir - ''' - self.updateProgressFullStep("'Descriptions'") - - for (title_num, title) in enumerate(self.booksByTitle): - self.updateProgressMicroStep("Description %d of %d" % \ - (title_num, len(self.booksByTitle)), - float(title_num*100/len(self.booksByTitle))/100) - - # Generate the header from user-customizable template - soup = self.generateHTMLDescriptionHeader(title) - - # Write the book entry to contentdir - outfile = open("%s/book_%d.html" % (self.contentDir, int(title['id'])), 'w') - outfile.write(soup.prettify()) - outfile.close() - - def generateHTMLByTitle(self): - ''' - Write books by title A-Z to HTML file - ''' - self.updateProgressFullStep("'Titles'") - - soup = self.generateHTMLEmptyHeader("Books By Alpha Title") - body = soup.find('body') - btc = 0 - - # Insert section tag - aTag = Tag(soup,'a') - aTag['name'] = 'section_start' - body.insert(btc, aTag) - btc += 1 - - # Insert the anchor - aTag = Tag(soup, "a") - aTag['name'] = "bytitle" - body.insert(btc, aTag) - btc += 1 - - if not self.__generateForKindle: - # We don't need this because the Kindle shows section titles - #

By Title

- pTag = Tag(soup, "p") - pTag['class'] = 'title' - aTag = Tag(soup, "a") - aTag['name'] = "bytitle" - pTag.insert(0,aTag) - pTag.insert(1,NavigableString('Titles')) - body.insert(btc,pTag) - btc += 1 - - divTag = Tag(soup, "div") - dtc = 0 - current_letter = "" - - # Re-sort title list without leading series/series_index - # Incoming title : - if not self.useSeriesPrefixInTitlesSection: - nspt = deepcopy(self.booksByTitle) - nspt = sorted(nspt, key=lambda x:(x['title_sort'].upper(), x['title_sort'].upper())) - self.booksByTitle_noSeriesPrefix = nspt - - # Loop through the books by title - # Generate one divRunningTag per initial letter for the purposes of - # minimizing widows and orphans on readers that can handle large - # <divs> styled as inline-block - title_list = self.booksByTitle - if not self.useSeriesPrefixInTitlesSection: - title_list = self.booksByTitle_noSeriesPrefix - drtc = 0 - divRunningTag = None - for book in title_list: - if self.letter_or_symbol(book['title_sort'][0]) != current_letter : - # Start a new letter - if drtc and divRunningTag is not None: - divTag.insert(dtc, divRunningTag) - dtc += 1 - divRunningTag = Tag(soup, 'div') - if dtc > 0: - divRunningTag['class'] = "initial_letter" - drtc = 0 - current_letter = self.letter_or_symbol(book['title_sort'][0]) - pIndexTag = Tag(soup, "p") - pIndexTag['class'] = "author_title_letter_index" - aTag = Tag(soup, "a") - aTag['name'] = "%s" % self.letter_or_symbol(book['title_sort'][0]) - pIndexTag.insert(0,aTag) - pIndexTag.insert(1,NavigableString(self.letter_or_symbol(book['title_sort'][0]))) - divRunningTag.insert(dtc,pIndexTag) - drtc += 1 - - # Add books - pBookTag = Tag(soup, "p") - ptc = 0 - - # book with read|reading|unread symbol or wishlist item - if self.opts.wishlist_tag in book.get('tags', []): - pBookTag['class'] = "wishlist_item" - pBookTag.insert(ptc,NavigableString(self.MISSING_SYMBOL)) - ptc += 1 - else: - if book['read']: - # check mark - pBookTag.insert(ptc,NavigableString(self.READ_SYMBOL)) - pBookTag['class'] = "read_book" - ptc += 1 - elif book['id'] in self.bookmarked_books: - pBookTag.insert(ptc,NavigableString(self.READING_SYMBOL)) - pBookTag['class'] = "read_book" - ptc += 1 - else: - # hidden check mark - pBookTag['class'] = "unread_book" - pBookTag.insert(ptc,NavigableString(self.NOT_READ_SYMBOL)) - ptc += 1 - - # Link to book - aTag = Tag(soup, "a") - if self.opts.generate_descriptions: - aTag['href'] = "book_%d.html" % (int(float(book['id']))) - - # Generate the title from the template - args = self.generateFormatArgs(book) - if book['series']: - formatted_title = self.by_titles_series_title_template.format(**args).rstrip() - else: - formatted_title = self.by_titles_normal_title_template.format(**args).rstrip() - aTag.insert(0,NavigableString(escape(formatted_title))) - pBookTag.insert(ptc, aTag) - ptc += 1 - - # Dot - pBookTag.insert(ptc, NavigableString(" · ")) - ptc += 1 - - # Link to author - emTag = Tag(soup, "em") - aTag = Tag(soup, "a") - if self.opts.generate_authors: - aTag['href'] = "%s.html#%s" % ("ByAlphaAuthor", self.generateAuthorAnchor(book['author'])) - aTag.insert(0, NavigableString(book['author'])) - emTag.insert(0,aTag) - pBookTag.insert(ptc, emTag) - ptc += 1 - - if divRunningTag is not None: - divRunningTag.insert(drtc, pBookTag) - drtc += 1 - - # Add the last divRunningTag to divTag - if divRunningTag is not None: - divTag.insert(dtc, divRunningTag) - dtc += 1 - - # Add the divTag to the body - body.insert(btc, divTag) - btc += 1 - - # Write the volume to contentdir - outfile_spec = "%s/ByAlphaTitle.html" % (self.contentDir) - outfile = open(outfile_spec, 'w') - outfile.write(soup.prettify()) - outfile.close() - self.htmlFileList_1.append("content/ByAlphaTitle.html") - - def generateHTMLByAuthor(self): - ''' - Write books by author A-Z - ''' - self.updateProgressFullStep("'Authors'") - - friendly_name = "Authors" - - soup = self.generateHTMLEmptyHeader(friendly_name) - body = soup.find('body') - - btc = 0 - - # Insert section tag - aTag = Tag(soup,'a') - aTag['name'] = 'section_start' - body.insert(btc, aTag) - btc += 1 - - # Insert the anchor - aTag = Tag(soup, "a") - anchor_name = friendly_name.lower() - aTag['name'] = anchor_name.replace(" ","") - body.insert(btc, aTag) - btc += 1 - - divTag = Tag(soup, "div") - dtc = 0 - divOpeningTag = None - dotc = 0 - divRunningTag = None - drtc = 0 - - # Loop through booksByAuthor - # Each author/books group goes in an openingTag div (first) or - # a runningTag div (subsequent) - book_count = 0 - current_author = '' - current_letter = '' - current_series = None - #for book in sorted(self.booksByAuthor, key = self.booksByAuthorSorter_author_sort): - for book in self.booksByAuthor: - - book_count += 1 - if self.letter_or_symbol(book['author_sort'][0].upper()) != current_letter : - # Start a new letter with Index letter - if divOpeningTag is not None: - divTag.insert(dtc, divOpeningTag) - dtc += 1 - dotc = 0 - if divRunningTag is not None: - divTag.insert(dtc, divRunningTag) - dtc += 1 - drtc = 0 - divRunningTag = None - - current_letter = self.letter_or_symbol(book['author_sort'][0].upper()) - author_count = 0 - divOpeningTag = Tag(soup, 'div') - if dtc > 0: - divOpeningTag['class'] = "initial_letter" - dotc = 0 - pIndexTag = Tag(soup, "p") - pIndexTag['class'] = "author_title_letter_index" - aTag = Tag(soup, "a") - aTag['name'] = "%sauthors" % self.letter_or_symbol(current_letter) - pIndexTag.insert(0,aTag) - pIndexTag.insert(1,NavigableString(self.letter_or_symbol(book['author_sort'][0].upper()))) - divOpeningTag.insert(dotc,pIndexTag) - dotc += 1 - - if book['author'] != current_author: - # Start a new author - current_author = book['author'] - author_count += 1 - if author_count >= 2: - # Add divOpeningTag to divTag, kill divOpeningTag - if divOpeningTag: - divTag.insert(dtc, divOpeningTag) - dtc += 1 - divOpeningTag = None - dotc = 0 - - # Create a divRunningTag for the next author - if author_count > 2: - divTag.insert(dtc, divRunningTag) - dtc += 1 - - divRunningTag = Tag(soup, 'div') - divRunningTag['class'] = "author_logical_group" - drtc = 0 - - non_series_books = 0 - current_series = None - pAuthorTag = Tag(soup, "p") - pAuthorTag['class'] = "author_index" - aTag = Tag(soup, "a") - aTag['name'] = "%s" % self.generateAuthorAnchor(current_author) - aTag.insert(0,NavigableString(current_author)) - pAuthorTag.insert(0,aTag) - if author_count == 1: - divOpeningTag.insert(dotc, pAuthorTag) - dotc += 1 - else: - divRunningTag.insert(drtc,pAuthorTag) - drtc += 1 - - # Check for series - if book['series'] and book['series'] != current_series: - # Start a new series - current_series = book['series'] - pSeriesTag = Tag(soup,'p') - pSeriesTag['class'] = "series" - - if self.opts.generate_series: - aTag = Tag(soup,'a') - aTag['href'] = "%s.html#%s_series" % ('BySeries', - re.sub('\W','',book['series']).lower()) - aTag.insert(0, book['series']) - #pSeriesTag.insert(0, NavigableString(self.NOT_READ_SYMBOL)) - pSeriesTag.insert(0, aTag) - else: - #pSeriesTag.insert(0,NavigableString(self.NOT_READ_SYMBOL + '%s' % book['series'])) - pSeriesTag.insert(0,NavigableString('%s' % book['series'])) - - if author_count == 1: - divOpeningTag.insert(dotc, pSeriesTag) - dotc += 1 - elif divRunningTag is not None: - divRunningTag.insert(drtc,pSeriesTag) - drtc += 1 - if current_series and not book['series']: - current_series = None - - # Add books - pBookTag = Tag(soup, "p") - ptc = 0 - - # book with read|reading|unread symbol or wishlist item - if self.opts.wishlist_tag in book.get('tags', []): - pBookTag['class'] = "wishlist_item" - pBookTag.insert(ptc,NavigableString(self.MISSING_SYMBOL)) - ptc += 1 - else: - if book['read']: - # check mark - pBookTag.insert(ptc,NavigableString(self.READ_SYMBOL)) - pBookTag['class'] = "read_book" - ptc += 1 - elif book['id'] in self.bookmarked_books: - pBookTag.insert(ptc,NavigableString(self.READING_SYMBOL)) - pBookTag['class'] = "read_book" - ptc += 1 - else: - # hidden check mark - pBookTag['class'] = "unread_book" - pBookTag.insert(ptc,NavigableString(self.NOT_READ_SYMBOL)) - ptc += 1 - - aTag = Tag(soup, "a") - if self.opts.generate_descriptions: - aTag['href'] = "book_%d.html" % (int(float(book['id']))) - - # Generate the title from the template - args = self.generateFormatArgs(book) - if current_series: - #aTag.insert(0,'%s%s' % (escape(book['title'][len(book['series'])+1:]),pubyear)) - formatted_title = self.by_authors_series_title_template.format(**args).rstrip() - else: - #aTag.insert(0,'%s%s' % (escape(book['title']), pubyear)) - formatted_title = self.by_authors_normal_title_template.format(**args).rstrip() - non_series_books += 1 - aTag.insert(0,NavigableString(escape(formatted_title))) - - pBookTag.insert(ptc, aTag) - ptc += 1 - - if author_count == 1: - divOpeningTag.insert(dotc, pBookTag) - dotc += 1 - elif divRunningTag: - divRunningTag.insert(drtc,pBookTag) - drtc += 1 - - # Loop ends here - - if not self.__generateForKindle: - # Insert the <h2> tag with book_count at the head - #<h2><a name="byalphaauthor" id="byalphaauthor"></a>By Author</h2> - pTag = Tag(soup, "p") - pTag['class'] = 'title' - aTag = Tag(soup, "a") - anchor_name = friendly_name.lower() - aTag['name'] = anchor_name.replace(" ","") - pTag.insert(0,aTag) - #h2Tag.insert(1,NavigableString('%s (%d)' % (friendly_name, book_count))) - pTag.insert(1,NavigableString('%s' % (friendly_name))) - body.insert(btc,pTag) - btc += 1 - - if author_count == 1: - divTag.insert(dtc, divOpeningTag) - dtc += 1 - elif divRunningTag is not None: - divTag.insert(dtc, divRunningTag) - dtc += 1 - - # Add the divTag to the body - body.insert(btc, divTag) - - # Write the generated file to contentdir - outfile_spec = "%s/ByAlphaAuthor.html" % (self.contentDir) - outfile = open(outfile_spec, 'w') - outfile.write(soup.prettify()) - outfile.close() - self.htmlFileList_1.append("content/ByAlphaAuthor.html") - - def generateHTMLByDateAdded(self): - ''' - Write books by reverse chronological order - ''' - self.updateProgressFullStep("'Recently Added'") - - def add_books_to_HTML_by_month(this_months_list, dtc): - if len(this_months_list): - - #this_months_list = sorted(this_months_list, key=self.booksByAuthorSorter_author_sort) - - # Create a new month anchor - date_string = strftime(u'%B %Y', current_date.timetuple()) - pIndexTag = Tag(soup, "p") - pIndexTag['class'] = "date_index" - aTag = Tag(soup, "a") - aTag['name'] = "bda_%s-%s" % (current_date.year, current_date.month) - pIndexTag.insert(0,aTag) - pIndexTag.insert(1,NavigableString(date_string)) - divTag.insert(dtc,pIndexTag) - dtc += 1 - current_author = None - current_series = None - - for new_entry in this_months_list: - if new_entry['author'] != current_author: - # Start a new author - current_author = new_entry['author'] - non_series_books = 0 - current_series = None - pAuthorTag = Tag(soup, "p") - pAuthorTag['class'] = "author_index" - aTag = Tag(soup, "a") - if self.opts.generate_authors: - aTag['name'] = "%s" % self.generateAuthorAnchor(current_author) - aTag.insert(0,NavigableString(current_author)) - pAuthorTag.insert(0,aTag) - divTag.insert(dtc,pAuthorTag) - dtc += 1 - - # Check for series - if new_entry['series'] and new_entry['series'] != current_series: - # Start a new series - current_series = new_entry['series'] - pSeriesTag = Tag(soup,'p') - pSeriesTag['class'] = "series" - if self.opts.generate_series: - aTag = Tag(soup,'a') - aTag['href'] = "%s.html#%s_series" % ('BySeries', - re.sub('\W','',new_entry['series']).lower()) - aTag.insert(0, new_entry['series']) - pSeriesTag.insert(0, aTag) - else: - pSeriesTag.insert(0,NavigableString('%s' % new_entry['series'])) - divTag.insert(dtc,pSeriesTag) - dtc += 1 - if current_series and not new_entry['series']: - current_series = None - - # Add books - pBookTag = Tag(soup, "p") - ptc = 0 - - # book with read|reading|unread symbol or wishlist item - if self.opts.wishlist_tag in new_entry.get('tags', []): - pBookTag['class'] = "wishlist_item" - pBookTag.insert(ptc,NavigableString(self.MISSING_SYMBOL)) - ptc += 1 - else: - if new_entry['read']: - # check mark - pBookTag.insert(ptc,NavigableString(self.READ_SYMBOL)) - pBookTag['class'] = "read_book" - ptc += 1 - elif new_entry['id'] in self.bookmarked_books: - pBookTag.insert(ptc,NavigableString(self.READING_SYMBOL)) - pBookTag['class'] = "read_book" - ptc += 1 - else: - # hidden check mark - pBookTag['class'] = "unread_book" - pBookTag.insert(ptc,NavigableString(self.NOT_READ_SYMBOL)) - ptc += 1 - - aTag = Tag(soup, "a") - if self.opts.generate_descriptions: - aTag['href'] = "book_%d.html" % (int(float(new_entry['id']))) - - # Generate the title from the template - args = self.generateFormatArgs(new_entry) - if current_series: - formatted_title = self.by_month_added_series_title_template.format(**args).rstrip() - else: - formatted_title = self.by_month_added_normal_title_template.format(**args).rstrip() - non_series_books += 1 - aTag.insert(0,NavigableString(escape(formatted_title))) - pBookTag.insert(ptc, aTag) - ptc += 1 - - divTag.insert(dtc, pBookTag) - dtc += 1 - return dtc - - def add_books_to_HTML_by_date_range(date_range_list, date_range, dtc): - if len(date_range_list): - pIndexTag = Tag(soup, "p") - pIndexTag['class'] = "date_index" - aTag = Tag(soup, "a") - aTag['name'] = "bda_%s" % date_range.replace(' ','') - pIndexTag.insert(0,aTag) - pIndexTag.insert(1,NavigableString(date_range)) - divTag.insert(dtc,pIndexTag) - dtc += 1 - - for new_entry in date_range_list: - # Add books - pBookTag = Tag(soup, "p") - ptc = 0 - - # book with read|reading|unread symbol or wishlist item - if self.opts.wishlist_tag in new_entry.get('tags', []): - pBookTag['class'] = "wishlist_item" - pBookTag.insert(ptc,NavigableString(self.MISSING_SYMBOL)) - ptc += 1 - else: - if new_entry['read']: - # check mark - pBookTag.insert(ptc,NavigableString(self.READ_SYMBOL)) - pBookTag['class'] = "read_book" - ptc += 1 - elif new_entry['id'] in self.bookmarked_books: - pBookTag.insert(ptc,NavigableString(self.READING_SYMBOL)) - pBookTag['class'] = "read_book" - ptc += 1 - else: - # hidden check mark - pBookTag['class'] = "unread_book" - pBookTag.insert(ptc,NavigableString(self.NOT_READ_SYMBOL)) - ptc += 1 - - aTag = Tag(soup, "a") - if self.opts.generate_descriptions: - aTag['href'] = "book_%d.html" % (int(float(new_entry['id']))) - - # Generate the title from the template - args = self.generateFormatArgs(new_entry) - if new_entry['series']: - formatted_title = self.by_recently_added_series_title_template.format(**args).rstrip() - else: - formatted_title = self.by_recently_added_normal_title_template.format(**args).rstrip() - aTag.insert(0,NavigableString(escape(formatted_title))) - pBookTag.insert(ptc, aTag) - ptc += 1 - - # Dot - pBookTag.insert(ptc, NavigableString(" · ")) - ptc += 1 - - # Link to author - emTag = Tag(soup, "em") - aTag = Tag(soup, "a") - if self.opts.generate_authors: - aTag['href'] = "%s.html#%s" % ("ByAlphaAuthor", self.generateAuthorAnchor(new_entry['author'])) - aTag.insert(0, NavigableString(new_entry['author'])) - emTag.insert(0,aTag) - pBookTag.insert(ptc, emTag) - ptc += 1 - - divTag.insert(dtc, pBookTag) - dtc += 1 - return dtc - - friendly_name = "Recently Added" - - soup = self.generateHTMLEmptyHeader(friendly_name) - body = soup.find('body') - - btc = 0 - - # Insert section tag - aTag = Tag(soup,'a') - aTag['name'] = 'section_start' - body.insert(btc, aTag) - btc += 1 - - # Insert the anchor - aTag = Tag(soup, "a") - anchor_name = friendly_name.lower() - aTag['name'] = anchor_name.replace(" ","") - body.insert(btc, aTag) - btc += 1 - - if not self.__generateForKindle: - #<h2><a name="byalphaauthor" id="byalphaauthor"></a>By Author</h2> - pTag = Tag(soup, "p") - pTag['class'] = 'title' - aTag = Tag(soup, "a") - anchor_name = friendly_name.lower() - aTag['name'] = anchor_name.replace(" ","") - pTag.insert(0,aTag) - pTag.insert(1,NavigableString('%s' % friendly_name)) - body.insert(btc,pTag) - btc += 1 - - divTag = Tag(soup, "div") - dtc = 0 - - # >>> Books by date range <<< - if self.useSeriesPrefixInTitlesSection: - self.booksByDateRange = sorted(self.booksByTitle, - key=lambda x:(x['timestamp'], x['timestamp']),reverse=True) - else: - nspt = deepcopy(self.booksByTitle) - self.booksByDateRange = sorted(nspt, key=lambda x:(x['timestamp'], x['timestamp']),reverse=True) - - date_range_list = [] - today_time = nowf().replace(hour=23, minute=59, second=59) - for (i, date) in enumerate(self.DATE_RANGE): - date_range_limit = self.DATE_RANGE[i] - if i: - date_range = '%d to %d days ago' % (self.DATE_RANGE[i-1], self.DATE_RANGE[i]) - else: - date_range = 'Last %d days' % (self.DATE_RANGE[i]) - - for book in self.booksByDateRange: - book_time = book['timestamp'] - delta = today_time-book_time - if delta.days <= date_range_limit: - date_range_list.append(book) - else: - break - - dtc = add_books_to_HTML_by_date_range(date_range_list, date_range, dtc) - date_range_list = [book] - - # >>>> Books by month <<<< - # Sort titles case-insensitive for by month using series prefix - self.booksByMonth = sorted(self.booksByTitle, - key=lambda x:(x['timestamp'], x['timestamp']),reverse=True) - - # Loop through books by date - current_date = datetime.date.fromordinal(1) - this_months_list = [] - for book in self.booksByMonth: - if book['timestamp'].month != current_date.month or \ - book['timestamp'].year != current_date.year: - dtc = add_books_to_HTML_by_month(this_months_list, dtc) - this_months_list = [] - current_date = book['timestamp'].date() - this_months_list.append(book) - - # Add the last month's list - add_books_to_HTML_by_month(this_months_list, dtc) - - # Add the divTag to the body - body.insert(btc, divTag) - - # Write the generated file to contentdir - outfile_spec = "%s/ByDateAdded.html" % (self.contentDir) - outfile = open(outfile_spec, 'w') - outfile.write(soup.prettify()) - outfile.close() - self.htmlFileList_2.append("content/ByDateAdded.html") - - def generateHTMLByDateRead(self): - ''' - Write books by active bookmarks - ''' - friendly_name = 'Recently Read' - self.updateProgressFullStep("'%s'" % friendly_name) - if not self.bookmarked_books: - return - - def add_books_to_HTML_by_day(todays_list, dtc): - if len(todays_list): - # Create a new day anchor - date_string = strftime(u'%A, %B %d', current_date.timetuple()) - pIndexTag = Tag(soup, "p") - pIndexTag['class'] = "date_index" - aTag = Tag(soup, "a") - aTag['name'] = "bdr_%s-%s-%s" % (current_date.year, current_date.month, current_date.day) - pIndexTag.insert(0,aTag) - pIndexTag.insert(1,NavigableString(date_string)) - divTag.insert(dtc,pIndexTag) - dtc += 1 - - for new_entry in todays_list: - pBookTag = Tag(soup, "p") - pBookTag['class'] = "date_read" - ptc = 0 - - # Percent read - pBookTag.insert(ptc, NavigableString(new_entry['reading_progress'])) - ptc += 1 - - aTag = Tag(soup, "a") - if self.opts.generate_descriptions: - aTag['href'] = "book_%d.html" % (int(float(new_entry['id']))) - aTag.insert(0,escape(new_entry['title'])) - pBookTag.insert(ptc, aTag) - ptc += 1 - - # Dot - pBookTag.insert(ptc, NavigableString(" · ")) - ptc += 1 - - # Link to author - emTag = Tag(soup, "em") - aTag = Tag(soup, "a") - if self.opts.generate_authors: - aTag['href'] = "%s.html#%s" % ("ByAlphaAuthor", self.generateAuthorAnchor(new_entry['author'])) - aTag.insert(0, NavigableString(new_entry['author'])) - emTag.insert(0,aTag) - pBookTag.insert(ptc, emTag) - ptc += 1 - - divTag.insert(dtc, pBookTag) - dtc += 1 - return dtc - - def add_books_to_HTML_by_date_range(date_range_list, date_range, dtc): - if len(date_range_list): - pIndexTag = Tag(soup, "p") - pIndexTag['class'] = "date_index" - aTag = Tag(soup, "a") - aTag['name'] = "bdr_%s" % date_range.replace(' ','') - pIndexTag.insert(0,aTag) - pIndexTag.insert(1,NavigableString(date_range)) - divTag.insert(dtc,pIndexTag) - dtc += 1 - - for new_entry in date_range_list: - # Add books - pBookTag = Tag(soup, "p") - pBookTag['class'] = "date_read" - ptc = 0 - - # Percent read - dots = int((new_entry['percent_read'] + 5)/10) - dot_string = self.READ_PROGRESS_SYMBOL * dots - empty_dots = self.UNREAD_PROGRESS_SYMBOL * (10 - dots) - pBookTag.insert(ptc, NavigableString('%s%s' % (dot_string,empty_dots))) - ptc += 1 - - aTag = Tag(soup, "a") - if self.opts.generate_descriptions: - aTag['href'] = "book_%d.html" % (int(float(new_entry['id']))) - aTag.insert(0,escape(new_entry['title'])) - pBookTag.insert(ptc, aTag) - ptc += 1 - - # Dot - pBookTag.insert(ptc, NavigableString(" · ")) - ptc += 1 - - # Link to author - emTag = Tag(soup, "em") - aTag = Tag(soup, "a") - if self.opts.generate_authors: - aTag['href'] = "%s.html#%s" % ("ByAlphaAuthor", self.generateAuthorAnchor(new_entry['author'])) - aTag.insert(0, NavigableString(new_entry['author'])) - emTag.insert(0,aTag) - pBookTag.insert(ptc, emTag) - ptc += 1 - - divTag.insert(dtc, pBookTag) - dtc += 1 - return dtc - - soup = self.generateHTMLEmptyHeader(friendly_name) - body = soup.find('body') - - btc = 0 - - # Insert section tag - aTag = Tag(soup,'a') - aTag['name'] = 'section_start' - body.insert(btc, aTag) - btc += 1 - - # Insert the anchor - aTag = Tag(soup, "a") - anchor_name = friendly_name.lower() - aTag['name'] = anchor_name.replace(" ","") - body.insert(btc, aTag) - btc += 1 - - divTag = Tag(soup, "div") - dtc = 0 - - # self.bookmarked_books: (Bookmark, book) - bookmarked_books = [] - for bm_book in self.bookmarked_books: - book = self.bookmarked_books[bm_book] - #print "bm_book: %s" % bm_book - book[1]['bookmark_timestamp'] = book[0].timestamp - try: - book[1]['percent_read'] = min(float(100*book[0].last_read / book[0].book_length),100) - except: - book[1]['percent_read'] = 0 - bookmarked_books.append(book[1]) - - self.booksByDateRead = sorted(bookmarked_books, - key=lambda x:(x['bookmark_timestamp'], x['bookmark_timestamp']),reverse=True) - - # >>>> Recently read by day <<<< - current_date = datetime.date.fromordinal(1) - todays_list = [] - for book in self.booksByDateRead: - bookmark_time = datetime.datetime.utcfromtimestamp(book['bookmark_timestamp']) - if bookmark_time.day != current_date.day or \ - bookmark_time.month != current_date.month or \ - bookmark_time.year != current_date.year: - dtc = add_books_to_HTML_by_day(todays_list, dtc) - todays_list = [] - current_date = datetime.datetime.utcfromtimestamp(book['bookmark_timestamp']).date() - todays_list.append(book) - - # Add the last day's list - add_books_to_HTML_by_day(todays_list, dtc) - - # Add the divTag to the body - body.insert(btc, divTag) - - # Write the generated file to contentdir - outfile_spec = "%s/ByDateRead.html" % (self.contentDir) - outfile = open(outfile_spec, 'w') - outfile.write(soup.prettify()) - outfile.close() - self.htmlFileList_2.append("content/ByDateRead.html") - - def generateHTMLBySeries(self): - ''' - Generate a list of series - ''' - self.updateProgressFullStep("Fetching series") - - self.opts.sort_by = 'series' - - # Merge opts.exclude_tags with opts.search_text - # Updated to use exact match syntax - empty_exclude_tags = False if len(self.opts.exclude_tags) else True - search_phrase = 'series:true ' - if not empty_exclude_tags: - exclude_tags = self.opts.exclude_tags.split(',') - search_terms = [] - for tag in exclude_tags: - search_terms.append("tag:=%s" % tag) - search_phrase += "not (%s)" % " or ".join(search_terms) - - # If a list of ids are provided, don't use search_text - if self.opts.ids: - self.opts.search_text = search_phrase - else: - if self.opts.search_text: - self.opts.search_text += " " + search_phrase - else: - self.opts.search_text = search_phrase - - # Fetch the database as a dictionary - data = self.plugin.search_sort_db(self.db, self.opts) - self.booksBySeries = self.processExclusions(data) - - if not self.booksBySeries: - self.opts.generate_series = False - self.opts.log(" no series found in selected books, cancelling series generation") - return - - friendly_name = "Series" - - soup = self.generateHTMLEmptyHeader(friendly_name) - body = soup.find('body') - - btc = 0 - - # Insert section tag - aTag = Tag(soup,'a') - aTag['name'] = 'section_start' - body.insert(btc, aTag) - btc += 1 - - # Insert the anchor - aTag = Tag(soup, "a") - anchor_name = friendly_name.lower() - aTag['name'] = anchor_name.replace(" ","") - body.insert(btc, aTag) - btc += 1 - - divTag = Tag(soup, "div") - dtc = 0 - current_letter = "" - current_series = None - - # Loop through booksBySeries - series_count = 0 - for book in self.booksBySeries: - # Check for initial letter change - sort_title = self.generateSortTitle(book['series']) - if self.letter_or_symbol(sort_title[0].upper()) != current_letter : - # Start a new letter with Index letter - current_letter = self.letter_or_symbol(sort_title[0].upper()) - pIndexTag = Tag(soup, "p") - pIndexTag['class'] = "series_letter_index" - aTag = Tag(soup, "a") - aTag['name'] = "%s_series" % self.letter_or_symbol(current_letter) - pIndexTag.insert(0,aTag) - pIndexTag.insert(1,NavigableString(self.letter_or_symbol(sort_title[0].upper()))) - divTag.insert(dtc,pIndexTag) - dtc += 1 - - # Check for series change - if book['series'] != current_series: - # Start a new series - series_count += 1 - current_series = book['series'] - pSeriesTag = Tag(soup,'p') - pSeriesTag['class'] = "series" - aTag = Tag(soup, 'a') - aTag['name'] = "%s_series" % re.sub('\W','',book['series']).lower() - pSeriesTag.insert(0,aTag) - pSeriesTag.insert(1,NavigableString('%s' % book['series'])) - divTag.insert(dtc,pSeriesTag) - dtc += 1 - - # Add books - pBookTag = Tag(soup, "p") - ptc = 0 - - book['read'] = self.discoverReadStatus(book) - - # book with read|reading|unread symbol or wishlist item - if self.opts.wishlist_tag in book.get('tags', []): - pBookTag['class'] = "wishlist_item" - pBookTag.insert(ptc,NavigableString(self.MISSING_SYMBOL)) - ptc += 1 - else: - if book.get('read', False): - # check mark - pBookTag.insert(ptc,NavigableString(self.READ_SYMBOL)) - pBookTag['class'] = "read_book" - ptc += 1 - elif book['id'] in self.bookmarked_books: - pBookTag.insert(ptc,NavigableString(self.READING_SYMBOL)) - pBookTag['class'] = "read_book" - ptc += 1 - else: - # hidden check mark - pBookTag['class'] = "unread_book" - pBookTag.insert(ptc,NavigableString(self.NOT_READ_SYMBOL)) - ptc += 1 - - aTag = Tag(soup, "a") - if self.opts.generate_descriptions: - aTag['href'] = "book_%d.html" % (int(float(book['id']))) - # Use series, series index if avail else just title - #aTag.insert(0,'%d. %s · %s' % (book['series_index'],escape(book['title']), ' & '.join(book['authors']))) - - if is_date_undefined(book['pubdate']): - book['date'] = None - else: - book['date'] = strftime(u'%B %Y', book['pubdate'].timetuple()) - - args = self.generateFormatArgs(book) - formatted_title = self.by_series_title_template.format(**args).rstrip() - aTag.insert(0,NavigableString(escape(formatted_title))) - pBookTag.insert(ptc, aTag) - ptc += 1 - - # · - pBookTag.insert(ptc, NavigableString(' · ')) - ptc += 1 - - # Link to author - aTag = Tag(soup, "a") - if self.opts.generate_authors: - aTag['href'] = "%s.html#%s" % ("ByAlphaAuthor", - self.generateAuthorAnchor(escape(' & '.join(book['authors'])))) - aTag.insert(0, NavigableString(' & '.join(book['authors']))) - pBookTag.insert(ptc, aTag) - ptc += 1 - - divTag.insert(dtc, pBookTag) - dtc += 1 - - if not self.__generateForKindle: - # Insert the <h2> tag with book_count at the head - #<h2><a name="byseries" id="byseries"></a>By Series</h2> - pTag = Tag(soup, "p") - pTag['class'] = 'title' - aTag = Tag(soup, "a") - anchor_name = friendly_name.lower() - aTag['name'] = anchor_name.replace(" ","") - pTag.insert(0,aTag) - #h2Tag.insert(1,NavigableString('%s (%d)' % (friendly_name, series_count))) - pTag.insert(1,NavigableString('%s' % friendly_name)) - body.insert(btc,pTag) - btc += 1 - - # Add the divTag to the body - body.insert(btc, divTag) - - # Write the generated file to contentdir - outfile_spec = "%s/BySeries.html" % (self.contentDir) - outfile = open(outfile_spec, 'w') - outfile.write(soup.prettify()) - outfile.close() - self.htmlFileList_1.append("content/BySeries.html") - - def generateHTMLByTags(self): - ''' - Generate individual HTML files for each tag, e.g. Fiction, Nonfiction ... - Note that special tags - have already been filtered from books[] - There may be synonomous tags - ''' - self.updateProgressFullStep("'Genres'") - - self.genre_tags_dict = self.filterDbTags(self.db.all_tags()) - - # Extract books matching filtered_tags - genre_list = [] - for friendly_tag in sorted(self.genre_tags_dict): - #print "\ngenerateHTMLByTags(): looking for books with friendly_tag '%s'" % friendly_tag - # tag_list => { normalized_genre_tag : [{book},{},{}], - # normalized_genre_tag : [{book},{},{}] } - - tag_list = {} - for book in self.booksByAuthor: - # Scan each book for tag matching friendly_tag - if 'tags' in book and friendly_tag in book['tags']: - this_book = {} - this_book['author'] = book['author'] - this_book['title'] = book['title'] - this_book['author_sort'] = capitalize(book['author_sort']) - this_book['read'] = book['read'] - this_book['tags'] = book['tags'] - this_book['id'] = book['id'] - this_book['series'] = book['series'] - this_book['series_index'] = book['series_index'] - this_book['date'] = book['date'] - normalized_tag = self.genre_tags_dict[friendly_tag] - genre_tag_list = [key for genre in genre_list for key in genre] - if normalized_tag in genre_tag_list: - for existing_genre in genre_list: - for key in existing_genre: - new_book = None - if key == normalized_tag: - for book in existing_genre[key]: - if book['title'] == this_book['title']: - new_book = False - break - else: - new_book = True - if new_book: - existing_genre[key].append(this_book) - else: - tag_list[normalized_tag] = [this_book] - genre_list.append(tag_list) - - if self.opts.verbose: - if len(genre_list): - self.opts.log.info(" Genre summary: %d active genre tags used in generating catalog with %d titles" % - (len(genre_list), len(self.booksByTitle))) - - for genre in genre_list: - for key in genre: - self.opts.log.info(" %s: %d %s" % (self.getFriendlyGenreTag(key), - len(genre[key]), - 'titles' if len(genre[key]) > 1 else 'title')) - - - # Write the results - # genre_list = [ {friendly_tag:[{book},{book}]}, {friendly_tag:[{book},{book}]}, ...] - master_genre_list = [] - for genre_tag_set in genre_list: - for (index, genre) in enumerate(genre_tag_set): - #print "genre: %s \t genre_tag_set[genre]: %s" % (genre, genre_tag_set[genre]) - - # Create sorted_authors[0] = friendly, [1] = author_sort for NCX creation - authors = [] - for book in genre_tag_set[genre]: - authors.append((book['author'],book['author_sort'])) - - # authors[] contains a list of all book authors, with multiple entries for multiple books by author - # Create unique_authors with a count of books per author as the third tuple element - books_by_current_author = 1 - current_author = authors[0] - unique_authors = [] - for (i,author) in enumerate(authors): - if author != current_author and i: - unique_authors.append((current_author[0], current_author[1], books_by_current_author)) - current_author = author - books_by_current_author = 1 - elif i==0 and len(authors) == 1: - # Allow for single-book lists - unique_authors.append((current_author[0], current_author[1], books_by_current_author)) - else: - books_by_current_author += 1 - - # Write the genre book list as an article - titles_spanned = self.generateHTMLByGenre(genre, True if index==0 else False, - genre_tag_set[genre], - "%s/Genre_%s.html" % (self.contentDir, - genre)) - - tag_file = "content/Genre_%s.html" % genre - master_genre_list.append({'tag':genre, - 'file':tag_file, - 'authors':unique_authors, - 'books':genre_tag_set[genre], - 'titles_spanned':titles_spanned}) - - self.genres = master_genre_list - - def generateThumbnails(self): - ''' - Generate a thumbnail per cover. If a current thumbnail exists, skip - If a cover doesn't exist, use default - Return list of active thumbs - ''' - self.updateProgressFullStep("'Thumbnails'") - thumbs = ['thumbnail_default.jpg'] - image_dir = "%s/images" % self.catalogPath - for (i,title) in enumerate(self.booksByTitle): - # Update status - self.updateProgressMicroStep("Thumbnail %d of %d" % \ - (i,len(self.booksByTitle)), - i/float(len(self.booksByTitle))) - - thumb_file = 'thumbnail_%d.jpg' % int(title['id']) - thumb_generated = True - valid_cover = True - try: - self.generateThumbnail(title, image_dir, thumb_file) - thumbs.append("thumbnail_%d.jpg" % int(title['id'])) - except: - if 'cover' in title and os.path.exists(title['cover']): - valid_cover = False - self.opts.log.warn(" *** Invalid cover file for '%s'***" % - (title['title'])) - if not self.error: - self.error.append('Invalid cover files') - self.error.append("Warning: invalid cover file for '%s', default cover substituted.\n" % (title['title'])) - - thumb_generated = False - - if not thumb_generated: - self.opts.log.warn(" using default cover for '%s' (%d)" % (title['title'], title['id'])) - # Confirm thumb exists, default is current - default_thumb_fp = os.path.join(image_dir,"thumbnail_default.jpg") - cover = os.path.join(self.catalogPath, "DefaultCover.png") - title['cover'] = cover - - if not os.path.exists(cover): - shutil.copyfile(I('book.png'), cover) - - if os.path.isfile(default_thumb_fp): - # Check to see if default cover is newer than thumbnail - # os.path.getmtime() = modified time - # os.path.ctime() = creation time - cover_timestamp = os.path.getmtime(cover) - thumb_timestamp = os.path.getmtime(default_thumb_fp) - if thumb_timestamp < cover_timestamp: - if False and self.verbose: - self.opts.log.warn("updating thumbnail_default for %s" % title['title']) - self.generateThumbnail(title, image_dir, - "thumbnail_default.jpg" if valid_cover else thumb_file) - else: - if False and self.verbose: - self.opts.log.warn(" generating new thumbnail_default.jpg") - self.generateThumbnail(title, image_dir, - "thumbnail_default.jpg" if valid_cover else thumb_file) - # Clear the book's cover property - title['cover'] = None - - - # Write thumb_width to the file, validating cache contents - # Allows detection of aborted catalog builds - with ZipFile(self.__archive_path, mode='a') as zfw: - zfw.writestr('thumb_width', self.opts.thumb_width) - - self.thumbs = thumbs - - def generateOPF(self): - - self.updateProgressFullStep("Generating OPF") - - header = ''' - <?xml version="1.0" encoding="UTF-8"?> - <package xmlns="http://www.idpf.org/2007/opf" version="2.0" unique-identifier="calibre_id"> - <metadata xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:opf="http://www.idpf.org/2007/opf" xmlns:calibre="http://calibre.kovidgoyal.net/2009/metadata" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"> - <dc:language>en-US</dc:language> - <meta name="calibre:publication_type" content="periodical:default"/> - </metadata> - <manifest></manifest> - <spine toc="ncx"></spine> - <guide></guide> - </package> - ''' - # Add the supplied metadata tags - soup = BeautifulStoneSoup(header, selfClosingTags=['item','itemref', 'reference']) - metadata = soup.find('metadata') - mtc = 0 - - titleTag = Tag(soup, "dc:title") - titleTag.insert(0,self.title) - metadata.insert(mtc, titleTag) - mtc += 1 - - creatorTag = Tag(soup, "dc:creator") - creatorTag.insert(0, self.creator) - metadata.insert(mtc, creatorTag) - mtc += 1 - - # Create the OPF tags - manifest = soup.find('manifest') - mtc = 0 - spine = soup.find('spine') - stc = 0 - guide = soup.find('guide') - - itemTag = Tag(soup, "item") - itemTag['id'] = "ncx" - itemTag['href'] = '%s.ncx' % self.basename - itemTag['media-type'] = "application/x-dtbncx+xml" - manifest.insert(mtc, itemTag) - mtc += 1 - - itemTag = Tag(soup, "item") - itemTag['id'] = 'stylesheet' - itemTag['href'] = self.stylesheet - itemTag['media-type'] = 'text/css' - manifest.insert(mtc, itemTag) - mtc += 1 - - itemTag = Tag(soup, "item") - itemTag['id'] = 'mastheadimage-image' - itemTag['href'] = "images/mastheadImage.gif" - itemTag['media-type'] = 'image/gif' - manifest.insert(mtc, itemTag) - mtc += 1 - - # Write the thumbnail images, descriptions to the manifest - sort_descriptions_by = [] - if self.opts.generate_descriptions: - for thumb in self.thumbs: - itemTag = Tag(soup, "item") - itemTag['href'] = "images/%s" % (thumb) - end = thumb.find('.jpg') - itemTag['id'] = "%s-image" % thumb[:end] - itemTag['media-type'] = 'image/jpeg' - manifest.insert(mtc, itemTag) - mtc += 1 - - # HTML files - add descriptions to manifest and spine - sort_descriptions_by = self.booksByAuthor if self.opts.sort_descriptions_by_author \ - else self.booksByTitle - # Add html_files to manifest and spine - - for file in self.htmlFileList_1: - # By Author, By Title, By Series, - itemTag = Tag(soup, "item") - start = file.find('/') + 1 - end = file.find('.') - itemTag['href'] = file - itemTag['id'] = file[start:end].lower() - itemTag['media-type'] = "application/xhtml+xml" - manifest.insert(mtc, itemTag) - mtc += 1 - - # spine - itemrefTag = Tag(soup, "itemref") - itemrefTag['idref'] = file[start:end].lower() - spine.insert(stc, itemrefTag) - stc += 1 - - # Add genre files to manifest and spine - for genre in self.genres: - if False: self.opts.log.info("adding %s to manifest and spine" % genre['tag']) - itemTag = Tag(soup, "item") - start = genre['file'].find('/') + 1 - end = genre['file'].find('.') - itemTag['href'] = genre['file'] - itemTag['id'] = genre['file'][start:end].lower() - itemTag['media-type'] = "application/xhtml+xml" - manifest.insert(mtc, itemTag) - mtc += 1 - - # spine - itemrefTag = Tag(soup, "itemref") - itemrefTag['idref'] = genre['file'][start:end].lower() - spine.insert(stc, itemrefTag) - stc += 1 - - for file in self.htmlFileList_2: - # By Date Added, By Date Read - itemTag = Tag(soup, "item") - start = file.find('/') + 1 - end = file.find('.') - itemTag['href'] = file - itemTag['id'] = file[start:end].lower() - itemTag['media-type'] = "application/xhtml+xml" - manifest.insert(mtc, itemTag) - mtc += 1 - - # spine - itemrefTag = Tag(soup, "itemref") - itemrefTag['idref'] = file[start:end].lower() - spine.insert(stc, itemrefTag) - stc += 1 - - for book in sort_descriptions_by: - # manifest - itemTag = Tag(soup, "item") - itemTag['href'] = "content/book_%d.html" % int(book['id']) - itemTag['id'] = "book%d" % int(book['id']) - itemTag['media-type'] = "application/xhtml+xml" - manifest.insert(mtc, itemTag) - mtc += 1 - - # spine - itemrefTag = Tag(soup, "itemref") - itemrefTag['idref'] = "book%d" % int(book['id']) - spine.insert(stc, itemrefTag) - stc += 1 - - # Guide - referenceTag = Tag(soup, "reference") - referenceTag['type'] = 'masthead' - referenceTag['title'] = 'mastheadimage-image' - referenceTag['href'] = 'images/mastheadImage.gif' - guide.insert(0,referenceTag) - - # Write the OPF file - outfile = open("%s/%s.opf" % (self.catalogPath, self.basename), 'w') - outfile.write(soup.prettify()) - - def generateNCXHeader(self): - - self.updateProgressFullStep("NCX header") - - header = ''' - <?xml version="1.0" encoding="utf-8"?> - <ncx xmlns="http://www.daisy.org/z3986/2005/ncx/" xmlns:calibre="http://calibre.kovidgoyal.net/2009/metadata" version="2005-1" xml:lang="en"> - </ncx> - ''' - soup = BeautifulStoneSoup(header, selfClosingTags=['content','calibre:meta-img']) - - ncx = soup.find('ncx') - navMapTag = Tag(soup, 'navMap') - navPointTag = Tag(soup, 'navPoint') - navPointTag['class'] = "periodical" - navPointTag['id'] = "title" - navPointTag['playOrder'] = self.playOrder - self.playOrder += 1 - navLabelTag = Tag(soup, 'navLabel') - textTag = Tag(soup, 'text') - textTag.insert(0, NavigableString(self.title)) - navLabelTag.insert(0, textTag) - navPointTag.insert(0, navLabelTag) - - if self.opts.generate_authors: - contentTag = Tag(soup, 'content') - contentTag['src'] = "content/ByAlphaAuthor.html" - navPointTag.insert(1, contentTag) - elif self.opts.generate_titles: - contentTag = Tag(soup, 'content') - contentTag['src'] = "content/ByAlphaTitle.html" - navPointTag.insert(1, contentTag) - elif self.opts.generate_series: - contentTag = Tag(soup, 'content') - contentTag['src'] = "content/BySeries.html" - navPointTag.insert(1, contentTag) - elif self.opts.generate_genres: - contentTag = Tag(soup, 'content') - #contentTag['src'] = "content/ByGenres.html" - contentTag['src'] = "%s" % self.genres[0]['file'] - navPointTag.insert(1, contentTag) - elif self.opts.generate_recently_added: - contentTag = Tag(soup, 'content') - contentTag['src'] = "content/ByDateAdded.html" - navPointTag.insert(1, contentTag) - else: - # Descriptions only - sort_descriptions_by = self.booksByAuthor if self.opts.sort_descriptions_by_author \ - else self.booksByTitle - contentTag = Tag(soup, 'content') - contentTag['src'] = "content/book_%d.html" % int(sort_descriptions_by[0]['id']) - navPointTag.insert(1, contentTag) - - cmiTag = Tag(soup, '%s' % 'calibre:meta-img') - cmiTag['name'] = "mastheadImage" - cmiTag['src'] = "images/mastheadImage.gif" - navPointTag.insert(2,cmiTag) - navMapTag.insert(0,navPointTag) - - ncx.insert(0,navMapTag) - self.ncxSoup = soup - - def generateNCXDescriptions(self, tocTitle): - - self.updateProgressFullStep("NCX 'Descriptions'") - - # --- Construct the 'Books by Title' section --- - ncx_soup = self.ncxSoup - body = ncx_soup.find("navPoint") - btc = len(body.contents) - - # Add the section navPoint - navPointTag = Tag(ncx_soup, 'navPoint') - navPointTag['class'] = "section" - navPointTag['id'] = "bytitle-ID" - navPointTag['playOrder'] = self.playOrder - self.playOrder += 1 - navLabelTag = Tag(ncx_soup, 'navLabel') - textTag = Tag(ncx_soup, 'text') - textTag.insert(0, NavigableString(tocTitle)) - navLabelTag.insert(0, textTag) - nptc = 0 - navPointTag.insert(nptc, navLabelTag) - nptc += 1 - contentTag = Tag(ncx_soup,"content") - contentTag['src'] = "content/book_%d.html" % int(self.booksByTitle[0]['id']) - navPointTag.insert(nptc, contentTag) - nptc += 1 - - # Loop over the titles - sort_descriptions_by = self.booksByAuthor if self.opts.sort_descriptions_by_author \ - else self.booksByTitle - - for book in sort_descriptions_by: - navPointVolumeTag = Tag(ncx_soup, 'navPoint') - navPointVolumeTag['class'] = "article" - navPointVolumeTag['id'] = "book%dID" % int(book['id']) - navPointVolumeTag['playOrder'] = self.playOrder - self.playOrder += 1 - navLabelTag = Tag(ncx_soup, "navLabel") - textTag = Tag(ncx_soup, "text") - if book['series']: - series_index = str(book['series_index']) - if series_index.endswith('.0'): - series_index = series_index[:-2] - if self.generateForKindle: - # Don't include Author for Kindle - textTag.insert(0, NavigableString(self.formatNCXText('%s (%s [%s])' % - (book['title'], book['series'], series_index), dest='title'))) - else: - # Include Author for non-Kindle - textTag.insert(0, NavigableString(self.formatNCXText('%s (%s [%s]) · %s ' % - (book['title'], book['series'], series_index, book['author']), dest='title'))) - else: - if self.generateForKindle: - # Don't include Author for Kindle - title_str = self.formatNCXText('%s' % (book['title']), dest='title') - if self.opts.connected_kindle and book['id'] in self.bookmarked_books: - ''' - dots = int((book['percent_read'] + 5)/10) - dot_string = '+' * dots - empty_dots = '-' * (10 - dots) - title_str += ' %s%s' % (dot_string,empty_dots) - ''' - title_str += '*' - textTag.insert(0, NavigableString(title_str)) - else: - # Include Author for non-Kindle - textTag.insert(0, NavigableString(self.formatNCXText('%s · %s' % \ - (book['title'], book['author']), dest='title'))) - navLabelTag.insert(0,textTag) - navPointVolumeTag.insert(0,navLabelTag) - - contentTag = Tag(ncx_soup, "content") - contentTag['src'] = "content/book_%d.html#book%d" % (int(book['id']), int(book['id'])) - navPointVolumeTag.insert(1, contentTag) - - if self.generateForKindle: - # Add the author tag - cmTag = Tag(ncx_soup, '%s' % 'calibre:meta') - cmTag['name'] = "author" - - if book['date']: - navStr = '%s | %s' % (self.formatNCXText(book['author'], dest='author'), - book['date'].split()[1]) - else: - navStr = '%s' % (self.formatNCXText(book['author'], dest='author')) - - if 'tags' in book and len(book['tags']): - navStr = self.formatNCXText(navStr + ' | ' + ' · '.join(sorted(book['tags'])), dest='author') - cmTag.insert(0, NavigableString(navStr)) - navPointVolumeTag.insert(2, cmTag) - - # Add the description tag - if book['short_description']: - cmTag = Tag(ncx_soup, '%s' % 'calibre:meta') - cmTag['name'] = "description" - cmTag.insert(0, NavigableString(self.formatNCXText(book['short_description'], dest='description'))) - navPointVolumeTag.insert(3, cmTag) - - # Add this volume to the section tag - navPointTag.insert(nptc, navPointVolumeTag) - nptc += 1 - - # Add this section to the body - body.insert(btc, navPointTag) - btc += 1 - - self.ncxSoup = ncx_soup - - def generateNCXBySeries(self, tocTitle): - self.updateProgressFullStep("NCX 'Series'") - - def add_to_series_by_letter(current_series_list): - current_series_list = " • ".join(current_series_list) - current_series_list = self.formatNCXText(current_series_list, dest="description") - series_by_letter.append(current_series_list) - - soup = self.ncxSoup - output = "BySeries" - body = soup.find("navPoint") - btc = len(body.contents) - - # --- Construct the 'Books By Series' section --- - navPointTag = Tag(soup, 'navPoint') - navPointTag['class'] = "section" - navPointTag['id'] = "byseries-ID" - navPointTag['playOrder'] = self.playOrder - self.playOrder += 1 - navLabelTag = Tag(soup, 'navLabel') - textTag = Tag(soup, 'text') - textTag.insert(0, NavigableString(tocTitle)) - navLabelTag.insert(0, textTag) - nptc = 0 - navPointTag.insert(nptc, navLabelTag) - nptc += 1 - contentTag = Tag(soup,"content") - contentTag['src'] = "content/%s.html#section_start" % (output) - navPointTag.insert(nptc, contentTag) - nptc += 1 - - series_by_letter = [] - - # Loop over the series titles, find start of each letter, add description_preview_count books - # Special switch for using different title list - title_list = self.booksBySeries - current_letter = self.letter_or_symbol(self.generateSortTitle(title_list[0]['series'])[0]) - title_letters = [current_letter] - current_series_list = [] - current_series = "" - for book in title_list: - sort_title = self.generateSortTitle(book['series']) - if self.letter_or_symbol(sort_title[0]) != current_letter: - # Save the old list - add_to_series_by_letter(current_series_list) - - # Start the new list - current_letter = self.letter_or_symbol(sort_title[0]) - title_letters.append(current_letter) - current_series = book['series'] - current_series_list = [book['series']] - else: - if len(current_series_list) < self.descriptionClip and \ - book['series'] != current_series : - current_series = book['series'] - current_series_list.append(book['series']) - - # Add the last book list - add_to_series_by_letter(current_series_list) - - # Add *article* entries for each populated series title letter - for (i,books) in enumerate(series_by_letter): - navPointByLetterTag = Tag(soup, 'navPoint') - navPointByLetterTag['class'] = "article" - navPointByLetterTag['id'] = "%sSeries-ID" % (title_letters[i].upper()) - navPointTag['playOrder'] = self.playOrder - self.playOrder += 1 - navLabelTag = Tag(soup, 'navLabel') - textTag = Tag(soup, 'text') - textTag.insert(0, NavigableString(u"Series beginning with %s" % \ - (title_letters[i] if len(title_letters[i])>1 else "'" + title_letters[i] + "'"))) - navLabelTag.insert(0, textTag) - navPointByLetterTag.insert(0,navLabelTag) - contentTag = Tag(soup, 'content') - contentTag['src'] = "content/%s.html#%s_series" % (output, title_letters[i]) - navPointByLetterTag.insert(1,contentTag) - - if self.generateForKindle: - cmTag = Tag(soup, '%s' % 'calibre:meta') - cmTag['name'] = "description" - cmTag.insert(0, NavigableString(self.formatNCXText(books, dest='description'))) - navPointByLetterTag.insert(2, cmTag) - - navPointTag.insert(nptc, navPointByLetterTag) - nptc += 1 - - # Add this section to the body - body.insert(btc, navPointTag) - btc += 1 - - self.ncxSoup = soup - - def generateNCXByTitle(self, tocTitle): - self.updateProgressFullStep("NCX 'Titles'") - - def add_to_books_by_letter(current_book_list): - current_book_list = " • ".join(current_book_list) - current_book_list = self.formatNCXText(current_book_list, dest="description") - books_by_letter.append(current_book_list) - - soup = self.ncxSoup - output = "ByAlphaTitle" - body = soup.find("navPoint") - btc = len(body.contents) - - # --- Construct the 'Books By Title' section --- - navPointTag = Tag(soup, 'navPoint') - navPointTag['class'] = "section" - navPointTag['id'] = "byalphatitle-ID" - navPointTag['playOrder'] = self.playOrder - self.playOrder += 1 - navLabelTag = Tag(soup, 'navLabel') - textTag = Tag(soup, 'text') - textTag.insert(0, NavigableString(tocTitle)) - navLabelTag.insert(0, textTag) - nptc = 0 - navPointTag.insert(nptc, navLabelTag) - nptc += 1 - contentTag = Tag(soup,"content") - contentTag['src'] = "content/%s.html#section_start" % (output) - navPointTag.insert(nptc, contentTag) - nptc += 1 - - books_by_letter = [] - - # Loop over the titles, find start of each letter, add description_preview_count books - # Special switch for using different title list - if self.useSeriesPrefixInTitlesSection: - title_list = self.booksByTitle - else: - title_list = self.booksByTitle_noSeriesPrefix - current_letter = self.letter_or_symbol(title_list[0]['title_sort'][0]) - title_letters = [current_letter] - current_book_list = [] - current_book = "" - for book in title_list: - if self.letter_or_symbol(book['title_sort'][0]) != current_letter: - # Save the old list - add_to_books_by_letter(current_book_list) - - # Start the new list - current_letter = self.letter_or_symbol(book['title_sort'][0]) - title_letters.append(current_letter) - current_book = book['title'] - current_book_list = [book['title']] - else: - if len(current_book_list) < self.descriptionClip and \ - book['title'] != current_book : - current_book = book['title'] - current_book_list.append(book['title']) - - # Add the last book list - add_to_books_by_letter(current_book_list) - - # Add *article* entries for each populated title letter - for (i,books) in enumerate(books_by_letter): - navPointByLetterTag = Tag(soup, 'navPoint') - navPointByLetterTag['class'] = "article" - navPointByLetterTag['id'] = "%sTitles-ID" % (title_letters[i].upper()) - navPointTag['playOrder'] = self.playOrder - self.playOrder += 1 - navLabelTag = Tag(soup, 'navLabel') - textTag = Tag(soup, 'text') - textTag.insert(0, NavigableString(u"Titles beginning with %s" % \ - (title_letters[i] if len(title_letters[i])>1 else "'" + title_letters[i] + "'"))) - navLabelTag.insert(0, textTag) - navPointByLetterTag.insert(0,navLabelTag) - contentTag = Tag(soup, 'content') - contentTag['src'] = "content/%s.html#%s" % (output, title_letters[i]) - navPointByLetterTag.insert(1,contentTag) - - if self.generateForKindle: - cmTag = Tag(soup, '%s' % 'calibre:meta') - cmTag['name'] = "description" - cmTag.insert(0, NavigableString(self.formatNCXText(books, dest='description'))) - navPointByLetterTag.insert(2, cmTag) - - navPointTag.insert(nptc, navPointByLetterTag) - nptc += 1 - - # Add this section to the body - body.insert(btc, navPointTag) - btc += 1 - - self.ncxSoup = soup - - def generateNCXByAuthor(self, tocTitle): - self.updateProgressFullStep("NCX 'Authors'") - - def add_to_author_list(current_author_list, current_letter): - current_author_list = " • ".join(current_author_list) - current_author_list = self.formatNCXText(current_author_list, dest="description") - master_author_list.append((current_author_list, current_letter)) - - soup = self.ncxSoup - HTML_file = "content/ByAlphaAuthor.html" - body = soup.find("navPoint") - btc = len(body.contents) - - # --- Construct the 'Books By Author' *section* --- - navPointTag = Tag(soup, 'navPoint') - navPointTag['class'] = "section" - file_ID = "%s" % tocTitle.lower() - file_ID = file_ID.replace(" ","") - navPointTag['id'] = "%s-ID" % file_ID - navPointTag['playOrder'] = self.playOrder - self.playOrder += 1 - navLabelTag = Tag(soup, 'navLabel') - textTag = Tag(soup, 'text') - textTag.insert(0, NavigableString('%s' % tocTitle)) - navLabelTag.insert(0, textTag) - nptc = 0 - navPointTag.insert(nptc, navLabelTag) - nptc += 1 - contentTag = Tag(soup,"content") - contentTag['src'] = "%s#section_start" % HTML_file - navPointTag.insert(nptc, contentTag) - nptc += 1 - - # Create an NCX article entry for each populated author index letter - # Loop over the sorted_authors list, find start of each letter, - # add description_preview_count artists - # self.authors[0]:friendly [1]:author_sort [2]:book_count - master_author_list = [] - # self.authors[0][1][0] = Initial letter of author_sort[0] - current_letter = self.letter_or_symbol(self.authors[0][1][0]) - current_author_list = [] - for author in self.authors: - if self.letter_or_symbol(author[1][0]) != current_letter: - # Save the old list - add_to_author_list(current_author_list, current_letter) - - # Start the new list - current_letter = self.letter_or_symbol(author[1][0]) - current_author_list = [author[0]] - else: - if len(current_author_list) < self.descriptionClip: - current_author_list.append(author[0]) - - # Add the last author list - add_to_author_list(current_author_list, current_letter) - - # Add *article* entries for each populated author initial letter - # master_author_list{}: [0]:author list [1]:Initial letter - for authors_by_letter in master_author_list: - navPointByLetterTag = Tag(soup, 'navPoint') - navPointByLetterTag['class'] = "article" - navPointByLetterTag['id'] = "%sauthors-ID" % (authors_by_letter[1]) - navPointTag['playOrder'] = self.playOrder - self.playOrder += 1 - navLabelTag = Tag(soup, 'navLabel') - textTag = Tag(soup, 'text') - textTag.insert(0, NavigableString("Authors beginning with '%s'" % (authors_by_letter[1]))) - navLabelTag.insert(0, textTag) - navPointByLetterTag.insert(0,navLabelTag) - contentTag = Tag(soup, 'content') - contentTag['src'] = "%s#%sauthors" % (HTML_file, authors_by_letter[1]) - - navPointByLetterTag.insert(1,contentTag) - - if self.generateForKindle: - cmTag = Tag(soup, '%s' % 'calibre:meta') - cmTag['name'] = "description" - cmTag.insert(0, NavigableString(authors_by_letter[0])) - navPointByLetterTag.insert(2, cmTag) - - navPointTag.insert(nptc, navPointByLetterTag) - nptc += 1 - - # Add this section to the body - body.insert(btc, navPointTag) - btc += 1 - - self.ncxSoup = soup - - def generateNCXByDateAdded(self, tocTitle): - self.updateProgressFullStep("NCX 'Recently Added'") - - def add_to_master_month_list(current_titles_list): - book_count = len(current_titles_list) - current_titles_list = " • ".join(current_titles_list) - current_titles_list = self.formatNCXText(current_titles_list, dest='description') - master_month_list.append((current_titles_list, current_date, book_count)) - - def add_to_master_date_range_list(current_titles_list): - book_count = len(current_titles_list) - current_titles_list = " • ".join(current_titles_list) - current_titles_list = self.formatNCXText(current_titles_list, dest='description') - master_date_range_list.append((current_titles_list, date_range, book_count)) - - soup = self.ncxSoup - HTML_file = "content/ByDateAdded.html" - body = soup.find("navPoint") - btc = len(body.contents) - - # --- Construct the 'Recently Added' *section* --- - navPointTag = Tag(soup, 'navPoint') - navPointTag['class'] = "section" - file_ID = "%s" % tocTitle.lower() - file_ID = file_ID.replace(" ","") - navPointTag['id'] = "%s-ID" % file_ID - navPointTag['playOrder'] = self.playOrder - self.playOrder += 1 - navLabelTag = Tag(soup, 'navLabel') - textTag = Tag(soup, 'text') - textTag.insert(0, NavigableString('%s' % tocTitle)) - navLabelTag.insert(0, textTag) - nptc = 0 - navPointTag.insert(nptc, navLabelTag) - nptc += 1 - contentTag = Tag(soup,"content") - contentTag['src'] = "%s#section_start" % HTML_file - navPointTag.insert(nptc, contentTag) - nptc += 1 - - # Create an NCX article entry for each date range - current_titles_list = [] - master_date_range_list = [] - today = datetime.datetime.now() - today_time = datetime.datetime(today.year, today.month, today.day) - for (i,date) in enumerate(self.DATE_RANGE): - if i: - date_range = '%d to %d days ago' % (self.DATE_RANGE[i-1], self.DATE_RANGE[i]) - else: - date_range = 'Last %d days' % (self.DATE_RANGE[i]) - date_range_limit = self.DATE_RANGE[i] - for book in self.booksByDateRange: - book_time = datetime.datetime(book['timestamp'].year, book['timestamp'].month, book['timestamp'].day) - if (today_time-book_time).days <= date_range_limit: - #print "generateNCXByDateAdded: %s added %d days ago" % (book['title'], (today_time-book_time).days) - current_titles_list.append(book['title']) - else: - break - if current_titles_list: - add_to_master_date_range_list(current_titles_list) - current_titles_list = [book['title']] - - # Add *article* entries for each populated date range - # master_date_range_list{}: [0]:titles list [1]:datestr - for books_by_date_range in master_date_range_list: - navPointByDateRangeTag = Tag(soup, 'navPoint') - navPointByDateRangeTag['class'] = "article" - navPointByDateRangeTag['id'] = "%s-ID" % books_by_date_range[1].replace(' ','') - navPointTag['playOrder'] = self.playOrder - self.playOrder += 1 - navLabelTag = Tag(soup, 'navLabel') - textTag = Tag(soup, 'text') - textTag.insert(0, NavigableString(books_by_date_range[1])) - navLabelTag.insert(0, textTag) - navPointByDateRangeTag.insert(0,navLabelTag) - contentTag = Tag(soup, 'content') - contentTag['src'] = "%s#bda_%s" % (HTML_file, - books_by_date_range[1].replace(' ','')) - - navPointByDateRangeTag.insert(1,contentTag) - - if self.generateForKindle: - cmTag = Tag(soup, '%s' % 'calibre:meta') - cmTag['name'] = "description" - cmTag.insert(0, NavigableString(books_by_date_range[0])) - navPointByDateRangeTag.insert(2, cmTag) - - cmTag = Tag(soup, '%s' % 'calibre:meta') - cmTag['name'] = "author" - navStr = '%d titles' % books_by_date_range[2] if books_by_date_range[2] > 1 else \ - '%d title' % books_by_date_range[2] - cmTag.insert(0, NavigableString(navStr)) - navPointByDateRangeTag.insert(3, cmTag) - - navPointTag.insert(nptc, navPointByDateRangeTag) - nptc += 1 - - - - # Create an NCX article entry for each populated month - # Loop over the booksByDate list, find start of each month, - # add description_preview_count titles - # master_month_list(list,date,count) - current_titles_list = [] - master_month_list = [] - current_date = self.booksByMonth[0]['timestamp'] - - for book in self.booksByMonth: - if book['timestamp'].month != current_date.month or \ - book['timestamp'].year != current_date.year: - # Save the old lists - add_to_master_month_list(current_titles_list) - - # Start the new list - current_date = book['timestamp'].date() - current_titles_list = [book['title']] - else: - current_titles_list.append(book['title']) - - # Add the last month list - add_to_master_month_list(current_titles_list) - - # Add *article* entries for each populated month - # master_months_list{}: [0]:titles list [1]:date - for books_by_month in master_month_list: - datestr = strftime(u'%B %Y', books_by_month[1].timetuple()) - navPointByMonthTag = Tag(soup, 'navPoint') - navPointByMonthTag['class'] = "article" - navPointByMonthTag['id'] = "bda_%s-%s-ID" % (books_by_month[1].year,books_by_month[1].month ) - navPointTag['playOrder'] = self.playOrder - self.playOrder += 1 - navLabelTag = Tag(soup, 'navLabel') - textTag = Tag(soup, 'text') - textTag.insert(0, NavigableString(datestr)) - navLabelTag.insert(0, textTag) - navPointByMonthTag.insert(0,navLabelTag) - contentTag = Tag(soup, 'content') - contentTag['src'] = "%s#bda_%s-%s" % (HTML_file, - books_by_month[1].year,books_by_month[1].month) - - navPointByMonthTag.insert(1,contentTag) - - if self.generateForKindle: - cmTag = Tag(soup, '%s' % 'calibre:meta') - cmTag['name'] = "description" - cmTag.insert(0, NavigableString(books_by_month[0])) - navPointByMonthTag.insert(2, cmTag) - - cmTag = Tag(soup, '%s' % 'calibre:meta') - cmTag['name'] = "author" - navStr = '%d titles' % books_by_month[2] if books_by_month[2] > 1 else \ - '%d title' % books_by_month[2] - cmTag.insert(0, NavigableString(navStr)) - navPointByMonthTag.insert(3, cmTag) - - navPointTag.insert(nptc, navPointByMonthTag) - nptc += 1 - - # Add this section to the body - body.insert(btc, navPointTag) - btc += 1 - self.ncxSoup = soup - - def generateNCXByDateRead(self, tocTitle): - self.updateProgressFullStep("NCX 'Recently Read'") - if not self.booksByDateRead: - return - - def add_to_master_day_list(current_titles_list): - book_count = len(current_titles_list) - current_titles_list = " • ".join(current_titles_list) - current_titles_list = self.formatNCXText(current_titles_list, dest='description') - master_day_list.append((current_titles_list, current_date, book_count)) - - def add_to_master_date_range_list(current_titles_list): - book_count = len(current_titles_list) - current_titles_list = " • ".join(current_titles_list) - current_titles_list = self.formatNCXText(current_titles_list, dest='description') - master_date_range_list.append((current_titles_list, date_range, book_count)) - - soup = self.ncxSoup - HTML_file = "content/ByDateRead.html" - body = soup.find("navPoint") - btc = len(body.contents) - - # --- Construct the 'Recently Read' *section* --- - navPointTag = Tag(soup, 'navPoint') - navPointTag['class'] = "section" - file_ID = "%s" % tocTitle.lower() - file_ID = file_ID.replace(" ","") - navPointTag['id'] = "%s-ID" % file_ID - navPointTag['playOrder'] = self.playOrder - self.playOrder += 1 - navLabelTag = Tag(soup, 'navLabel') - textTag = Tag(soup, 'text') - textTag.insert(0, NavigableString('%s' % tocTitle)) - navLabelTag.insert(0, textTag) - nptc = 0 - navPointTag.insert(nptc, navLabelTag) - nptc += 1 - contentTag = Tag(soup,"content") - contentTag['src'] = "%s#section_start" % HTML_file - navPointTag.insert(nptc, contentTag) - nptc += 1 - - # Create an NCX article entry for each date range - current_titles_list = [] - master_date_range_list = [] - today = datetime.datetime.now() - today_time = datetime.datetime(today.year, today.month, today.day) - for (i,date) in enumerate(self.DATE_RANGE): - if i: - date_range = '%d to %d days ago' % (self.DATE_RANGE[i-1], self.DATE_RANGE[i]) - else: - date_range = 'Last %d days' % (self.DATE_RANGE[i]) - date_range_limit = self.DATE_RANGE[i] - for book in self.booksByDateRead: - bookmark_time = datetime.datetime.utcfromtimestamp(book['bookmark_timestamp']) - if (today_time-bookmark_time).days <= date_range_limit: - #print "generateNCXByDateAdded: %s added %d days ago" % (book['title'], (today_time-book_time).days) - current_titles_list.append(book['title']) - else: - break - if current_titles_list: - add_to_master_date_range_list(current_titles_list) - current_titles_list = [book['title']] - - # Create an NCX article entry for each populated day - # Loop over the booksByDate list, find start of each month, - # add description_preview_count titles - # master_month_list(list,date,count) - current_titles_list = [] - master_day_list = [] - current_date = datetime.datetime.utcfromtimestamp(self.booksByDateRead[0]['bookmark_timestamp']) - - for book in self.booksByDateRead: - bookmark_time = datetime.datetime.utcfromtimestamp(book['bookmark_timestamp']) - if bookmark_time.day != current_date.day or \ - bookmark_time.month != current_date.month or \ - bookmark_time.year != current_date.year: - # Save the old lists - add_to_master_day_list(current_titles_list) - - # Start the new list - current_date = datetime.datetime.utcfromtimestamp(book['bookmark_timestamp']).date() - current_titles_list = [book['title']] - else: - current_titles_list.append(book['title']) - - # Add the last day list - add_to_master_day_list(current_titles_list) - - # Add *article* entries for each populated day - # master_day_list{}: [0]:titles list [1]:date - for books_by_day in master_day_list: - datestr = strftime(u'%A, %B %d', books_by_day[1].timetuple()) - navPointByDayTag = Tag(soup, 'navPoint') - navPointByDayTag['class'] = "article" - navPointByDayTag['id'] = "bdr_%s-%s-%sID" % (books_by_day[1].year, - books_by_day[1].month, - books_by_day[1].day ) - navPointTag['playOrder'] = self.playOrder - self.playOrder += 1 - navLabelTag = Tag(soup, 'navLabel') - textTag = Tag(soup, 'text') - textTag.insert(0, NavigableString(datestr)) - navLabelTag.insert(0, textTag) - navPointByDayTag.insert(0,navLabelTag) - contentTag = Tag(soup, 'content') - contentTag['src'] = "%s#bdr_%s-%s-%s" % (HTML_file, - books_by_day[1].year, - books_by_day[1].month, - books_by_day[1].day) - - navPointByDayTag.insert(1,contentTag) - - if self.generateForKindle: - cmTag = Tag(soup, '%s' % 'calibre:meta') - cmTag['name'] = "description" - cmTag.insert(0, NavigableString(books_by_day[0])) - navPointByDayTag.insert(2, cmTag) - - cmTag = Tag(soup, '%s' % 'calibre:meta') - cmTag['name'] = "author" - navStr = '%d titles' % books_by_day[2] if books_by_day[2] > 1 else \ - '%d title' % books_by_day[2] - cmTag.insert(0, NavigableString(navStr)) - navPointByDayTag.insert(3, cmTag) - - navPointTag.insert(nptc, navPointByDayTag) - nptc += 1 - - # Add this section to the body - body.insert(btc, navPointTag) - btc += 1 - self.ncxSoup = soup - - def generateNCXByGenre(self, tocTitle): - # Create an NCX section for 'By Genre' - # Add each genre as an article - # 'tag', 'file', 'authors' - - self.updateProgressFullStep("NCX 'Genres'") - - if not len(self.genres): - self.opts.log.warn(" No genres found in tags.\n" - " No Genre section added to Catalog") - return - - ncx_soup = self.ncxSoup - body = ncx_soup.find("navPoint") - btc = len(body.contents) - - # --- Construct the 'Books By Genre' *section* --- - navPointTag = Tag(ncx_soup, 'navPoint') - navPointTag['class'] = "section" - file_ID = "%s" % tocTitle.lower() - file_ID = file_ID.replace(" ","") - navPointTag['id'] = "%s-ID" % file_ID - navPointTag['playOrder'] = self.playOrder - self.playOrder += 1 - navLabelTag = Tag(ncx_soup, 'navLabel') - textTag = Tag(ncx_soup, 'text') - # textTag.insert(0, NavigableString('%s (%d)' % (section_title, len(genre_list)))) - textTag.insert(0, NavigableString('%s' % tocTitle)) - navLabelTag.insert(0, textTag) - nptc = 0 - navPointTag.insert(nptc, navLabelTag) - nptc += 1 - contentTag = Tag(ncx_soup,"content") - contentTag['src'] = "content/Genre_%s.html#section_start" % self.genres[0]['tag'] - navPointTag.insert(nptc, contentTag) - nptc += 1 - - for genre in self.genres: - # Add an article for each genre - navPointVolumeTag = Tag(ncx_soup, 'navPoint') - navPointVolumeTag['class'] = "article" - navPointVolumeTag['id'] = "genre-%s-ID" % genre['tag'] - navPointVolumeTag['playOrder'] = self.playOrder - self.playOrder += 1 - navLabelTag = Tag(ncx_soup, "navLabel") - textTag = Tag(ncx_soup, "text") - - # GwR *** Can this be optimized? - normalized_tag = None - for friendly_tag in self.genre_tags_dict: - if self.genre_tags_dict[friendly_tag] == genre['tag']: - normalized_tag = self.genre_tags_dict[friendly_tag] - break - textTag.insert(0, self.formatNCXText(NavigableString(friendly_tag), dest='description')) - navLabelTag.insert(0,textTag) - navPointVolumeTag.insert(0,navLabelTag) - contentTag = Tag(ncx_soup, "content") - contentTag['src'] = "content/Genre_%s.html#Genre_%s" % (normalized_tag, normalized_tag) - navPointVolumeTag.insert(1, contentTag) - - if self.generateForKindle: - # Build the author tag - cmTag = Tag(ncx_soup, '%s' % 'calibre:meta') - cmTag['name'] = "author" - # First - Last author - - if len(genre['titles_spanned']) > 1 : - author_range = "%s - %s" % (genre['titles_spanned'][0][0], genre['titles_spanned'][1][0]) - else : - author_range = "%s" % (genre['titles_spanned'][0][0]) - - cmTag.insert(0, NavigableString(author_range)) - navPointVolumeTag.insert(2, cmTag) - - # Build the description tag - cmTag = Tag(ncx_soup, '%s' % 'calibre:meta') - cmTag['name'] = "description" - - if False: - # Form 1: Titles spanned - if len(genre['titles_spanned']) > 1: - title_range = "%s -\n%s" % (genre['titles_spanned'][0][1], genre['titles_spanned'][1][1]) - else: - title_range = "%s" % (genre['titles_spanned'][0][1]) - cmTag.insert(0, NavigableString(self.formatNCXText(title_range, dest='description'))) - else: - # Form 2: title • title • title ... - titles = [] - for title in genre['books']: - titles.append(title['title']) - titles = sorted(titles, key=lambda x:(self.generateSortTitle(x),self.generateSortTitle(x))) - titles_list = self.generateShortDescription(u" • ".join(titles), dest="description") - cmTag.insert(0, NavigableString(self.formatNCXText(titles_list, dest='description'))) - - navPointVolumeTag.insert(3, cmTag) - - # Add this volume to the section tag - navPointTag.insert(nptc, navPointVolumeTag) - nptc += 1 - - # Add this section to the body - body.insert(btc, navPointTag) - btc += 1 - self.ncxSoup = ncx_soup - - def writeNCX(self): - self.updateProgressFullStep("Saving NCX") - - outfile = open("%s/%s.ncx" % (self.catalogPath, self.basename), 'w') - outfile.write(self.ncxSoup.prettify()) - - - # ======================== Helpers ======================== - def author_to_author_sort(self, author): - tokens = author.split() - tokens = tokens[-1:] + tokens[:-1] - if len(tokens) > 1: - tokens[0] += ',' - return ' '.join(tokens).capitalize() - - def booksByAuthorSorter_author_sort(self, book): - ''' - Sort non-series books before series books - ''' - if not book['series']: - key = '%s %s' % (capitalize(book['author_sort']), - capitalize(book['title_sort'])) - else: - index = book['series_index'] - integer = int(index) - fraction = index-integer - series_index = '%04d%s' % (integer, str('%0.4f' % fraction).lstrip('0')) - key = '%s ~%s %s' % (capitalize(book['author_sort']), - self.generateSortTitle(book['series']), - series_index) - return key - - def booksByAuthorSorter_author(self, book): - ''' - Sort non-series books before series books - ''' - if not book['series']: - key = '%s %s' % (self.author_to_author_sort(book['author']), - capitalize(book['title_sort'])) - else: - index = book['series_index'] - integer = int(index) - fraction = index-integer - series_index = '%04d%s' % (integer, str('%0.4f' % fraction).lstrip('0')) - key = '%s ~%s %s' % (self.author_to_author_sort(book['author']), - self.generateSortTitle(book['series']), - series_index) - return key - - def calculateThumbnailSize(self): - ''' Calculate thumbnail dimensions based on device DPI. Scale Kindle by 50% ''' - from calibre.customize.ui import output_profiles - for x in output_profiles(): - if x.short_name == self.opts.output_profile: - # aspect ratio: 3:4 - self.thumbWidth = x.dpi * float(self.opts.thumb_width) - self.thumbHeight = self.thumbWidth * 1.33 - if 'kindle' in x.short_name and self.opts.fmt == 'mobi': - # Kindle DPI appears to be off by a factor of 2 - self.thumbWidth = self.thumbWidth/2 - self.thumbHeight = self.thumbHeight/2 - break - if True and self.verbose: - self.opts.log(" DPI = %d; thumbnail dimensions: %d x %d" % \ - (x.dpi, self.thumbWidth, self.thumbHeight)) - - def convertHTMLEntities(self, s): - matches = re.findall("&#\d+;", s) - if len(matches) > 0: - hits = set(matches) - for hit in hits: - name = hit[2:-1] - try: - entnum = int(name) - s = s.replace(hit, unichr(entnum)) - except ValueError: - pass - - matches = re.findall("&\w+;", s) - hits = set(matches) - amp = "&" - if amp in hits: - hits.remove(amp) - for hit in hits: - name = hit[1:-1] - if htmlentitydefs.name2codepoint.has_key(name): - s = s.replace(hit, unichr(htmlentitydefs.name2codepoint[name])) - s = s.replace(amp, "&") - return s - - def createDirectoryStructure(self): - catalogPath = self.catalogPath - self.cleanUp() - - if not os.path.isdir(catalogPath): - os.makedirs(catalogPath) - - # Create /content and /images - content_path = catalogPath + "/content" - if not os.path.isdir(content_path): - os.makedirs(content_path) - images_path = catalogPath + "/images" - if not os.path.isdir(images_path): - os.makedirs(images_path) - - def discoverReadStatus(self, record): - ''' - Given a field:pattern spec, discover if this book marked as read - - if field == tag, scan tags for pattern - if custom field, try regex match for pattern - This allows maximum flexibility with fields of type - datatype bool: #field_name:True - datatype text: #field_name:<string> - datatype datetime: #field_name:.* - - ''' - # Legacy handling of special 'read' tag - field = self.__read_book_marker['field'] - pat = self.__read_book_marker['pattern'] - if field == 'tag' and pat in record['tags']: - return True - - field_contents = self.__db.get_field(record['id'], - field, - index_is_id=True) - if field_contents: - try: - if re.search(pat, unicode(field_contents), - re.IGNORECASE) is not None: - return True - except: - # Compiling of pat failed, ignore it - pass - - return False - - def filterDbTags(self, tags): - # Remove the special marker tags from the database's tag list, - # return sorted list of normalized genre tags - - def format_tag_list(tags, indent=5, line_break=70, header='Tag list'): - def next_tag(sorted_tags): - for (i, tag) in enumerate(sorted_tags): - if i < len(tags) - 1: - yield tag + ", " - else: - yield tag - - ans = '%s%d %s:\n' % (' ' * indent, len(tags), header) - ans += ' ' * (indent + 1) - out_str = '' - sorted_tags = sorted(tags) - for tag in next_tag(sorted_tags): - out_str += tag - if len(out_str) >= line_break: - ans += out_str + '\n' - out_str = ' ' * (indent + 1) - return ans + out_str - - normalized_tags = [] - friendly_tags = [] - excluded_tags = [] - for tag in tags: - if tag in self.markerTags: - excluded_tags.append(tag) - continue - if re.search(self.opts.exclude_genre, tag): - excluded_tags.append(tag) - continue - if tag == ' ': - continue - - normalized_tags.append(re.sub('\W','',tag).lower()) - friendly_tags.append(tag) - - genre_tags_dict = dict(zip(friendly_tags,normalized_tags)) - - # Test for multiple genres resolving to same normalized form - normalized_set = set(normalized_tags) - for normalized in normalized_set: - if normalized_tags.count(normalized) > 1: - self.opts.log.warn(" Warning: multiple tags resolving to genre '%s':" % normalized) - for key in genre_tags_dict: - if genre_tags_dict[key] == normalized: - self.opts.log.warn(" %s" % key) - if self.verbose: - self.opts.log.info('%s' % format_tag_list(genre_tags_dict, header="enabled genre tags in database")) - self.opts.log.info('%s' % format_tag_list(excluded_tags, header="excluded genre tags")) - - return genre_tags_dict - - def formatNCXText(self, description, dest=None): - # Kindle TOC descriptions won't render certain characters - # Fix up - massaged = unicode(BeautifulStoneSoup(description, convertEntities=BeautifulStoneSoup.HTML_ENTITIES)) - - # Replace '&' with '&' - massaged = re.sub("&","&", massaged) - - if massaged.strip() and dest: - #print traceback.print_stack(limit=3) - return self.generateShortDescription(massaged.strip(), dest=dest) - else: - return None - - def generateAuthorAnchor(self, author): - # Strip white space to '' - return re.sub("\W","", author) - - def generateFormatArgs(self, book): - series_index = str(book['series_index']) - if series_index.endswith('.0'): - series_index = series_index[:-2] - args = dict( - title = book['title'], - series = book['series'], - series_index = series_index, - rating = self.generateRatingString(book), - rating_parens = '(%s)' % self.generateRatingString(book) if 'rating' in book else '', - pubyear = book['date'].split()[1] if book['date'] else '', - pubyear_parens = "(%s)" % book['date'].split()[1] if book['date'] else '') - return args - - def generateHTMLByGenre(self, genre, section_head, books, outfile): - # Write an HTML file of this genre's book list - # Return a list with [(first_author, first_book), (last_author, last_book)] - - soup = self.generateHTMLGenreHeader(genre) - body = soup.find('body') - - btc = 0 - - # Insert section tag if this is the section start - first article only - if section_head: - aTag = Tag(soup,'a') - aTag['name'] = 'section_start' - body.insert(btc, aTag) - btc += 1 - - # Create an anchor from the tag - aTag = Tag(soup, 'a') - aTag['name'] = "Genre_%s" % genre - body.insert(btc,aTag) - btc += 1 - - titleTag = body.find(attrs={'class':'title'}) - titleTag.insert(0,NavigableString('%s' % escape(self.getFriendlyGenreTag(genre)))) - - # Insert the books by author list - divTag = body.find(attrs={'class':'authors'}) - dtc = 0 - - current_author = '' - current_series = None - for book in books: - if book['author'] != current_author: - # Start a new author with link - current_author = book['author'] - non_series_books = 0 - current_series = None - pAuthorTag = Tag(soup, "p") - pAuthorTag['class'] = "author_index" - aTag = Tag(soup, "a") - if self.opts.generate_authors: - aTag['href'] = "%s.html#%s" % ("ByAlphaAuthor", self.generateAuthorAnchor(book['author'])) - aTag.insert(0, book['author']) - pAuthorTag.insert(0,aTag) - divTag.insert(dtc,pAuthorTag) - dtc += 1 - - # Check for series - if book['series'] and book['series'] != current_series: - # Start a new series - current_series = book['series'] - pSeriesTag = Tag(soup,'p') - pSeriesTag['class'] = "series" - if self.opts.generate_series: - aTag = Tag(soup,'a') - aTag['href'] = "%s.html#%s_series" % ('BySeries', - re.sub('\W','',book['series']).lower()) - aTag.insert(0, book['series']) - pSeriesTag.insert(0, aTag) - else: - pSeriesTag.insert(0,NavigableString('%s' % book['series'])) - divTag.insert(dtc,pSeriesTag) - dtc += 1 - - if current_series and not book['series']: - current_series = None - - # Add books - pBookTag = Tag(soup, "p") - ptc = 0 - - # book with read|reading|unread symbol or wishlist item - if self.opts.wishlist_tag in book.get('tags', []): - pBookTag['class'] = "wishlist_item" - pBookTag.insert(ptc,NavigableString(self.MISSING_SYMBOL)) - ptc += 1 - else: - if book['read']: - # check mark - pBookTag.insert(ptc,NavigableString(self.READ_SYMBOL)) - pBookTag['class'] = "read_book" - ptc += 1 - elif book['id'] in self.bookmarked_books: - pBookTag.insert(ptc,NavigableString(self.READING_SYMBOL)) - pBookTag['class'] = "read_book" - ptc += 1 - else: - # hidden check mark - pBookTag['class'] = "unread_book" - pBookTag.insert(ptc,NavigableString(self.NOT_READ_SYMBOL)) - ptc += 1 - - # Add the book title - aTag = Tag(soup, "a") - if self.opts.generate_descriptions: - aTag['href'] = "book_%d.html" % (int(float(book['id']))) - - # Generate the title from the template - args = self.generateFormatArgs(book) - if current_series: - #aTag.insert(0,escape(book['title'][len(book['series'])+1:])) - formatted_title = self.by_genres_series_title_template.format(**args).rstrip() - else: - #aTag.insert(0,escape(book['title'])) - formatted_title = self.by_genres_normal_title_template.format(**args).rstrip() - non_series_books += 1 - aTag.insert(0,NavigableString(escape(formatted_title))) - - pBookTag.insert(ptc, aTag) - ptc += 1 - - divTag.insert(dtc, pBookTag) - dtc += 1 - - # Write the generated file to contentdir - outfile = open(outfile, 'w') - outfile.write(soup.prettify()) - outfile.close() - - if len(books) > 1: - titles_spanned = [(books[0]['author'],books[0]['title']), (books[-1]['author'],books[-1]['title'])] - else: - titles_spanned = [(books[0]['author'],books[0]['title'])] - - return titles_spanned - - def generateHTMLDescriptionHeader(self, book): - ''' - Generate description header from template - ''' - from calibre.ebooks.oeb.base import XHTML_NS - - def generate_html(): - args = dict( - author=author, - author_prefix=author_prefix, - comments=comments, - css=css, - formats=formats, - genres=genres, - note_content=note_content, - note_source=note_source, - pubdate=pubdate, - publisher=publisher, - pubmonth=pubmonth, - pubyear=pubyear, - rating=rating, - series=series, - series_index=series_index, - thumb=thumb, - title=title, - title_str=title_str, - xmlns=XHTML_NS, - ) - - generated_html = P('catalog/template.xhtml', - data=True).decode('utf-8').format(**args) - generated_html = substitute_entites(generated_html) - return BeautifulSoup(generated_html) - - # Generate the template arguments - css = P('catalog/stylesheet.css', data=True).decode('utf-8') - title_str = title = escape(book['title']) - series = '' - series_index = '' - if book['series']: - series = escape(book['series']) - series_index = str(book['series_index']) - if series_index.endswith('.0'): - series_index = series_index[:-2] - - # Author, author_prefix (read|reading|none symbol or missing symbol) - author = book['author'] - if self.opts.wishlist_tag in book.get('tags', []): - author_prefix = self.MISSING_SYMBOL + " by " - else: - if book['read']: - author_prefix = self.READ_SYMBOL + " by " - elif self.opts.connected_kindle and book['id'] in self.bookmarked_books: - author_prefix = self.READING_SYMBOL + " by " - else: - author_prefix = "by " - - # Genres - genres = '' - if 'tags' in book: - _soup = BeautifulSoup('') - genresTag = Tag(_soup,'p') - gtc = 0 - for (i, tag) in enumerate(sorted(book.get('tags', []))): - aTag = Tag(_soup,'a') - if self.opts.generate_genres: - aTag['href'] = "Genre_%s.html" % re.sub("\W","",tag.lower()) - aTag.insert(0,escape(NavigableString(tag))) - genresTag.insert(gtc, aTag) - gtc += 1 - if i < len(book['tags'])-1: - genresTag.insert(gtc, NavigableString(' · ')) - gtc += 1 - genres = genresTag.renderContents() - - # Formats - formats = [] - if 'formats' in book: - for format in sorted(book['formats']): - formats.append(format.rpartition('.')[2].upper()) - formats = ' · '.join(formats) - - # Date of publication - if book['date']: - pubdate = book['date'] - pubmonth, pubyear = pubdate.split() - else: - pubdate = pubyear = pubmonth = '' - - # Thumb - _soup = BeautifulSoup('<html>',selfClosingTags=['img']) - thumb = Tag(_soup,"img") - if 'cover' in book and book['cover']: - thumb['src'] = "../images/thumbnail_%d.jpg" % int(book['id']) - else: - thumb['src'] = "../images/thumbnail_default.jpg" - thumb['alt'] = "cover thumbnail" - - # Publisher - publisher = ' ' - if 'publisher' in book: - publisher = book['publisher'] - - # Rating - stars = int(book['rating']) / 2 - rating = '' - if stars: - star_string = self.FULL_RATING_SYMBOL * stars - empty_stars = self.EMPTY_RATING_SYMBOL * (5 - stars) - rating = '%s%s <br/>' % (star_string,empty_stars) - - # Notes - note_source = '' - note_content = '' - if 'notes' in book: - note_source = book['notes']['source'] - note_content = book['notes']['content'] - - # Comments - comments = '' - if 'description' in book and book['description'] > '': - comments = book['description'] - - - # >>>> Populate the template <<<< - soup = generate_html() - - - # >>>> Post-process the template <<<< - body = soup.find('body') - btc = 0 - # Insert the title anchor for inbound links - aTag = Tag(soup, "a") - aTag['name'] = "book%d" % int(book['id']) - body.insert(btc, aTag) - btc += 1 - - # Insert the link to the series or remove <a class="series"> - aTag = body.find('a', attrs={'class':'series_id'}) - if aTag: - if book['series']: - if self.opts.generate_series: - aTag['href'] = "%s.html#%s_series" % ('BySeries', - re.sub('\W','',book['series']).lower()) - else: - aTag.extract() - - # Insert the author link - aTag = body.find('a', attrs={'class':'author'}) - if self.opts.generate_authors and aTag: - aTag['href'] = "%s.html#%s" % ("ByAlphaAuthor", - self.generateAuthorAnchor(book['author'])) - - if publisher == ' ': - publisherTag = body.find('td', attrs={'class':'publisher'}) - if publisherTag: - publisherTag.contents[0].replaceWith(' ') - - if not genres: - genresTag = body.find('p',attrs={'class':'genres'}) - if genresTag: - genresTag.extract() - - if not formats: - formatsTag = body.find('p',attrs={'class':'formats'}) - if formatsTag: - formatsTag.extract() - - if note_content == '': - tdTag = body.find('td', attrs={'class':'notes'}) - if tdTag: - tdTag.contents[0].replaceWith(' ') - - emptyTags = body.findAll('td', attrs={'class':'empty'}) - for mt in emptyTags: - newEmptyTag = Tag(BeautifulSoup(),'td') - newEmptyTag.insert(0,NavigableString(' ')) - mt.replaceWith(newEmptyTag) - - if False: - print soup.prettify() - return soup - - def generateHTMLEmptyHeader(self, title): - header = ''' - <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd"> - <html xmlns="http://www.w3.org/1999/xhtml" xmlns:calibre="http://calibre.kovidgoyal.net/2009/metadata"> - <head> - <meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /> - <link rel="stylesheet" type="text/css" href="stylesheet.css" media="screen" /> - <title> - - - - - ''' - # Insert the supplied title - soup = BeautifulSoup(header) - titleTag = soup.find('title') - titleTag.insert(0,NavigableString(title)) - return soup - - def generateHTMLGenreHeader(self, title): - header = ''' - - - - - - - - -

-
- - - ''' - # Insert the supplied title - soup = BeautifulSoup(header) - titleTag = soup.find('title') - titleTag.insert(0,escape(NavigableString(title))) - return soup - - def generateMastheadImage(self, out_path): - from calibre.ebooks.conversion.config import load_defaults - from calibre.utils.fonts import fontconfig - font_path = default_font = P('fonts/liberation/LiberationSerif-Bold.ttf') - recs = load_defaults('mobi_output') - masthead_font_family = recs.get('masthead_font', 'Default') - - if masthead_font_family != 'Default': - masthead_font = fontconfig.files_for_family(masthead_font_family) - # Assume 'normal' always in dict, else use default - # {'normal': (path_to_font, friendly name)} - if 'normal' in masthead_font: - font_path = masthead_font['normal'][0] - - if not font_path or not os.access(font_path, os.R_OK): - font_path = default_font - - MI_WIDTH = 600 - MI_HEIGHT = 60 - - try: - from PIL import Image, ImageDraw, ImageFont - Image, ImageDraw, ImageFont - except ImportError: - import Image, ImageDraw, ImageFont - - img = Image.new('RGB', (MI_WIDTH, MI_HEIGHT), 'white') - draw = ImageDraw.Draw(img) - try: - font = ImageFont.truetype(font_path, 48) - except: - self.opts.log.error(" Failed to load user-specifed font '%s'" % font_path) - font = ImageFont.truetype(default_font, 48) - text = self.title.encode('utf-8') - width, height = draw.textsize(text, font=font) - left = max(int((MI_WIDTH - width)/2.), 0) - top = max(int((MI_HEIGHT - height)/2.), 0) - draw.text((left, top), text, fill=(0,0,0), font=font) - img.save(open(out_path, 'wb'), 'GIF') - - def generateRatingString(self, book): - rating = '' - try: - if 'rating' in book: - stars = int(book['rating']) / 2 - if stars: - star_string = self.FULL_RATING_SYMBOL * stars - empty_stars = self.EMPTY_RATING_SYMBOL * (5 - stars) - rating = '%s%s' % (star_string,empty_stars) - except: - # Rating could be None - pass - return rating - - def generateShortDescription(self, description, dest=None): - # Truncate the description, on word boundaries if necessary - # Possible destinations: - # description NCX summary - # title NCX title - # author NCX author - - def shortDescription(description, limit): - short_description = "" - words = description.split() - for word in words: - short_description += word + " " - if len(short_description) > limit: - short_description += "..." - return short_description - - if not description: - return None - - if dest == 'title': - # No truncation for titles, let the device deal with it - return description - elif dest == 'author': - if self.authorClip and len(description) < self.authorClip: - return description - else: - return shortDescription(description, self.authorClip) - elif dest == 'description': - if self.descriptionClip and len(description) < self.descriptionClip: - return description - else: - return shortDescription(description, self.descriptionClip) - else: - print " returning description with unspecified destination '%s'" % description - raise RuntimeError - - def generateSortTitle(self, title): - ''' - Generate a string suitable for sorting from the title - Ignore leading stop words - Optionally convert leading numbers to strings - ''' - from calibre.ebooks.metadata import title_sort - - # Strip stop words - title_words = title_sort(title).split() - translated = [] - - for (i,word) in enumerate(title_words): - # Leading numbers optionally translated to text equivalent - # Capitalize leading sort word - if i==0: - # *** Keep this code in case we need to restore numbers_as_text *** - if False: - #if self.opts.numbers_as_text and re.match('[0-9]+',word[0]): - translated.append(EPUB_MOBI.NumberToText(word).text.capitalize()) - else: - if re.match('[0-9]+',word[0]): - word = word.replace(',','') - suffix = re.search('[\D]', word) - if suffix: - word = '%10.0f%s' % (float(word[:suffix.start()]),word[suffix.start():]) - else: - word = '%10.0f' % (float(word)) - - # If leading char > 'A', insert symbol as leading forcing lower sort - # '/' sorts below numbers, g - if self.letter_or_symbol(word[0]) != word[0]: - if word[0] > 'A' or (ord('9') < ord(word[0]) < ord('A')) : - translated.append('/') - translated.append(capitalize(word)) - - else: - if re.search('[0-9]+',word[0]): - word = word.replace(',','') - suffix = re.search('[\D]', word) - if suffix: - word = '%10.0f%s' % (float(word[:suffix.start()]),word[suffix.start():]) - else: - word = '%10.0f' % (float(word)) - translated.append(word) - return ' '.join(translated) - - def generateThumbnail(self, title, image_dir, thumb_file): - ''' - Thumbs are cached with the full cover's crc. If the crc doesn't - match, the cover has been changed since the thumb was cached and needs - to be replaced. - ''' - - def open_archive(mode='r'): - try: - return ZipFile(self.__archive_path, mode=mode) - except: - # Happens on windows if the file is opened by another - # process - pass - - # Generate crc for current cover - #self.opts.log.info(" generateThumbnail():") - with open(title['cover'], 'rb') as f: - data = f.read() - cover_crc = hex(zlib.crc32(data)) - - # Test cache for uuid - zf = open_archive() - if zf is not None: - with zf: - try: - zf.getinfo(title['uuid']+cover_crc) - except: - pass - else: - # uuid found in cache with matching crc - thumb_data = zf.read(title['uuid']+cover_crc) - with open(os.path.join(image_dir, thumb_file), 'wb') as f: - f.write(thumb_data) - return - - - # Save thumb for catalog - thumb_data = thumbnail(data, - width=self.thumbWidth, height=self.thumbHeight)[-1] - with open(os.path.join(image_dir, thumb_file), 'wb') as f: - f.write(thumb_data) - - # Save thumb to archive - if zf is not None: # Ensure that the read succeeded - # If we failed to open the zip file for reading, - # we dont know if it contained the thumb or not - zf = open_archive('a') - if zf is not None: - with zf: - zf.writestr(title['uuid']+cover_crc, thumb_data) - - def getFriendlyGenreTag(self, genre): - # Find the first instance of friendly_tag matching genre - for friendly_tag in self.genre_tags_dict: - if self.genre_tags_dict[friendly_tag] == genre: - return friendly_tag - - def getMarkerTags(self): - ''' Return a list of special marker tags to be excluded from genre list ''' - markerTags = [] - markerTags.extend(self.opts.exclude_tags.split(',')) - return markerTags - - def letter_or_symbol(self,char): - if not re.search('[a-zA-Z]',char): - return 'Symbols' - else: - return char - - def markdownComments(self, comments): - ''' - Convert random comment text to normalized, xml-legal block of

s - 'plain text' returns as -

plain text

- - 'plain text with minimal markup' returns as -

plain text with minimal markup

- - '

pre-formatted text

returns untouched - - 'A line of text\n\nFollowed by a line of text' returns as -

A line of text

-

Followed by a line of text

- - 'A line of text.\nA second line of text.\rA third line of text' returns as -

A line of text.
A second line of text.
A third line of text.

- - '...end of a paragraph.Somehow the break was lost...' returns as -

...end of a paragraph.

-

Somehow the break was lost...

- - Deprecated HTML returns as HTML via BeautifulSoup() - - ''' - # Hackish - ignoring sentences ending or beginning in numbers to avoid - # confusion with decimal points. - - # Explode lost CRs to \n\n - for lost_cr in re.finditer('([a-z])([\.\?!])([A-Z])',comments): - comments = comments.replace(lost_cr.group(), - '%s%s\n\n%s' % (lost_cr.group(1), - lost_cr.group(2), - lost_cr.group(3))) - # Extract pre-built elements - annotations, etc. - if not isinstance(comments, unicode): - comments = comments.decode('utf-8', 'replace') - soup = BeautifulSoup(comments) - elems = soup.findAll('div') - for elem in elems: - elem.extract() - - # Reconstruct comments w/o
s - comments = soup.renderContents(None) - - # Convert \n\n to

s - if re.search('\n\n', comments): - soup = BeautifulSoup() - split_ps = comments.split(u'\n\n') - tsc = 0 - for p in split_ps: - pTag = Tag(soup,'p') - pTag.insert(0,p) - soup.insert(tsc,pTag) - tsc += 1 - comments = soup.renderContents(None) - - # Convert solo returns to
- comments = re.sub('[\r\n]','
', comments) - - # Convert two hypens to emdash - comments = re.sub('--','—',comments) - soup = BeautifulSoup(comments) - result = BeautifulSoup() - rtc = 0 - open_pTag = False - - all_tokens = list(soup.contents) - for token in all_tokens: - if type(token) is NavigableString: - if not open_pTag: - pTag = Tag(result,'p') - open_pTag = True - ptc = 0 - pTag.insert(ptc,prepare_string_for_xml(token)) - ptc += 1 - - elif token.name in ['br','b','i','em']: - if not open_pTag: - pTag = Tag(result,'p') - open_pTag = True - ptc = 0 - pTag.insert(ptc, token) - ptc += 1 - - else: - if open_pTag: - result.insert(rtc, pTag) - rtc += 1 - open_pTag = False - ptc = 0 - # Clean up NavigableStrings for xml - sub_tokens = list(token.contents) - for sub_token in sub_tokens: - if type(sub_token) is NavigableString: - sub_token.replaceWith(prepare_string_for_xml(sub_token)) - result.insert(rtc, token) - rtc += 1 - - if open_pTag: - result.insert(rtc, pTag) - rtc += 1 - - paras = result.findAll('p') - for p in paras: - p['class'] = 'description' - - # Add back

elems initially removed - for elem in elems: - result.insert(rtc,elem) - rtc += 1 - - return result.renderContents(encoding=None) - - def mergeComments(self, record): - ''' - merge ['description'] with custom field contents to be displayed in Descriptions - ''' - merged = '' - if record['description']: - addendum = self.__db.get_field(record['id'], - self.__merge_comments['field'], - index_is_id=True) - if addendum is None: - addendum = '' - include_hr = eval(self.__merge_comments['hr']) - if self.__merge_comments['position'] == 'before': - merged = addendum - if include_hr: - merged += '
' - else: - merged += '\n' - merged += record['description'] - else: - merged = record['description'] - if include_hr: - merged += '
' - else: - merged += '\n' - merged += addendum - else: - # Return the custom field contents - merged = self.__db.get_field(record['id'], - self.__merge_comments['field'], - index_is_id=True) - - return merged - - def processExclusions(self, data_set): - ''' - Remove excluded entries - ''' - field, pat = self.opts.exclude_book_marker.split(':') - if pat == '': - return data_set - filtered_data_set = [] - for record in data_set: - field_contents = self.__db.get_field(record['id'], - field, - index_is_id=True) - if field_contents: - if re.search(pat, unicode(field_contents), - re.IGNORECASE) is not None: - continue - filtered_data_set.append(record) - - return filtered_data_set - - def processSpecialTags(self, tags, this_title, opts): - tag_list = [] - for tag in tags: - tag = self.convertHTMLEntities(tag) - if re.search(opts.exclude_genre, tag): - continue - elif self.__read_book_marker['field'] == 'tag' and \ - tag == self.__read_book_marker['pattern']: - # remove 'read' tag - continue - else: - tag_list.append(tag) - return tag_list - - def updateProgressFullStep(self, description): - self.currentStep += 1 - self.progressString = description - self.progressInt = float((self.currentStep-1)/self.totalSteps) - self.reporter(self.progressInt, self.progressString) - if self.opts.cli_environment: - self.opts.log(u"%3.0f%% %s" % (self.progressInt*100, self.progressString)) - - def updateProgressMicroStep(self, description, micro_step_pct): - step_range = 100/self.totalSteps - self.progressString = description - coarse_progress = float((self.currentStep-1)/self.totalSteps) - fine_progress = float((micro_step_pct*step_range)/100) - self.progressInt = coarse_progress + fine_progress - self.reporter(self.progressInt, self.progressString) - - def run(self, path_to_output, opts, db, notification=DummyReporter()): - opts.log = log - opts.fmt = self.fmt = path_to_output.rpartition('.')[2] - - # Add local options - opts.creator = '%s, %s %s, %s' % (strftime('%A'), strftime('%B'), strftime('%d').lstrip('0'), strftime('%Y')) - opts.creator_sort_as = '%s %s' % ('calibre', strftime('%Y-%m-%d')) - opts.connected_kindle = False - - # Finalize output_profile - op = opts.output_profile - if op is None: - op = 'default' - - if opts.connected_device['name'] and 'kindle' in opts.connected_device['name'].lower(): - opts.connected_kindle = True - if opts.connected_device['serial'] and \ - opts.connected_device['serial'][:4] in ['B004','B005']: - op = "kindle_dx" - else: - op = "kindle" - opts.descriptionClip = 380 if op.endswith('dx') or 'kindle' not in op else 100 - opts.authorClip = 100 if op.endswith('dx') or 'kindle' not in op else 60 - opts.output_profile = op - - opts.basename = "Catalog" - opts.cli_environment = not hasattr(opts,'sync') - - # Hard-wired to always sort descriptions by author, with series after non-series - opts.sort_descriptions_by_author = True - - build_log = [] - - build_log.append(u"%s(): Generating %s %sin %s environment" % - (self.name,self.fmt,'for %s ' % opts.output_profile if opts.output_profile else '', - 'CLI' if opts.cli_environment else 'GUI')) - - # If exclude_genre is blank, assume user wants all genre tags included - if opts.exclude_genre.strip() == '': - opts.exclude_genre = '\[^.\]' - build_log.append(" converting empty exclude_genre to '\[^.\]'") - - if opts.connected_device['is_device_connected'] and \ - opts.connected_device['kind'] == 'device': - if opts.connected_device['serial']: - build_log.append(u" connected_device: '%s' #%s%s " % \ - (opts.connected_device['name'], - opts.connected_device['serial'][0:4], - 'x' * (len(opts.connected_device['serial']) - 4))) - for storage in opts.connected_device['storage']: - if storage: - build_log.append(u" mount point: %s" % storage) - else: - build_log.append(u" connected_device: '%s'" % opts.connected_device['name']) - try: - for storage in opts.connected_device['storage']: - if storage: - build_log.append(u" mount point: %s" % storage) - except: - build_log.append(u" (no mount points)") - else: - build_log.append(u" connected_device: '%s'" % opts.connected_device['name']) - - opts_dict = vars(opts) - if opts_dict['ids']: - build_log.append(" book count: %d" % len(opts_dict['ids'])) - - sections_list = [] - if opts.generate_authors: - sections_list.append('Authors') - if opts.generate_titles: - sections_list.append('Titles') - if opts.generate_series: - sections_list.append('Series') - if opts.generate_genres: - sections_list.append('Genres') - if opts.generate_recently_added: - sections_list.append('Recently Added') - if opts.generate_descriptions: - sections_list.append('Descriptions') - - if not sections_list: - if opts.cli_environment: - opts.log.warn('*** No Section switches specified, enabling all Sections ***') - opts.generate_authors = True - opts.generate_titles = True - opts.generate_series = True - opts.generate_genres = True - opts.generate_recently_added = True - opts.generate_descriptions = True - sections_list = ['Authors','Titles','Series','Genres','Recently Added','Descriptions'] - else: - opts.log.warn('\n*** No enabled Sections, terminating catalog generation ***') - return ["No Included Sections","No enabled Sections.\nCheck E-book options tab\n'Included sections'\n"] - if opts.fmt == 'mobi' and sections_list == ['Descriptions']: - warning = _("\n*** Adding 'By Authors' Section required for MOBI output ***") - opts.log.warn(warning) - sections_list.insert(0,'Authors') - opts.generate_authors = True - - opts.log(u" Sections: %s" % ', '.join(sections_list)) - opts.section_list = sections_list - - # Limit thumb_width to 1.0" - 2.0" - try: - if float(opts.thumb_width) < float(self.THUMB_SMALLEST): - log.warning("coercing thumb_width from '%s' to '%s'" % (opts.thumb_width,self.THUMB_SMALLEST)) - opts.thumb_width = self.THUMB_SMALLEST - if float(opts.thumb_width) > float(self.THUMB_LARGEST): - log.warning("coercing thumb_width from '%s' to '%s'" % (opts.thumb_width,self.THUMB_LARGEST)) - opts.thumb_width = self.THUMB_LARGEST - opts.thumb_width = "%.2f" % float(opts.thumb_width) - except: - log.error("coercing thumb_width from '%s' to '%s'" % (opts.thumb_width,self.THUMB_SMALLEST)) - opts.thumb_width = "1.0" - - # Display opts - keys = opts_dict.keys() - keys.sort() - build_log.append(" opts:") - for key in keys: - if key in ['catalog_title','authorClip','connected_kindle','descriptionClip', - 'exclude_book_marker','exclude_genre','exclude_tags', - 'header_note_source_field','merge_comments', - 'output_profile','read_book_marker', - 'search_text','sort_by','sort_descriptions_by_author','sync', - 'thumb_width','wishlist_tag']: - build_log.append(" %s: %s" % (key, repr(opts_dict[key]))) - - if opts.verbose: - log('\n'.join(line for line in build_log)) - - self.opts = opts - - # Launch the Catalog builder - catalog = self.CatalogBuilder(db, opts, self, report_progress=notification) - - if opts.verbose: - log.info(" Begin catalog source generation") - catalog.createDirectoryStructure() - catalog.copyResources() - catalog.calculateThumbnailSize() - catalog_source_built = catalog.buildSources() - - if opts.verbose: - if catalog_source_built: - log.info(" Completed catalog source generation\n") - else: - log.error(" *** Terminated catalog generation, check log for details ***") - - if catalog_source_built: - recommendations = [] - recommendations.append(('remove_fake_margins', False, - OptionRecommendation.HIGH)) - if DEBUG: - recommendations.append(('comments', '\n'.join(line for line in build_log), - OptionRecommendation.HIGH)) - else: - recommendations.append(('comments', '', OptionRecommendation.HIGH)) - - dp = getattr(opts, 'debug_pipeline', None) - if dp is not None: - recommendations.append(('debug_pipeline', dp, - OptionRecommendation.HIGH)) - - if opts.fmt == 'mobi' and opts.output_profile and opts.output_profile.startswith("kindle"): - recommendations.append(('output_profile', opts.output_profile, - OptionRecommendation.HIGH)) - recommendations.append(('no_inline_toc', True, - OptionRecommendation.HIGH)) - recommendations.append(('book_producer',opts.output_profile, - OptionRecommendation.HIGH)) - - # If cover exists, use it - cpath = None - try: - search_text = 'title:"%s" author:%s' % ( - opts.catalog_title.replace('"', '\\"'), 'calibre') - matches = db.search(search_text, return_matches=True) - if matches: - cpath = db.cover(matches[0], index_is_id=True, as_path=True) - if cpath and os.path.exists(cpath): - recommendations.append(('cover', cpath, - OptionRecommendation.HIGH)) - except: - pass - - # Run ebook-convert - from calibre.ebooks.conversion.plumber import Plumber - plumber = Plumber(os.path.join(catalog.catalogPath, - opts.basename + '.opf'), path_to_output, log, report_progress=notification, - abort_after_input_dump=False) - plumber.merge_ui_recommendations(recommendations) - plumber.run() - - try: - os.remove(cpath) - except: - pass - - # returns to gui2.actions.catalog:catalog_generated() - return catalog.error -# }}} diff --git a/src/calibre/library/catalogs/__init__.py b/src/calibre/library/catalogs/__init__.py new file mode 100644 index 0000000000..a53fffb344 --- /dev/null +++ b/src/calibre/library/catalogs/__init__.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +__license__ = 'GPL v3' +__copyright__ = '2012, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + + +FIELDS = ['all', 'title', 'title_sort', 'author_sort', 'authors', 'comments', + 'cover', 'formats','id', 'isbn', 'ondevice', 'pubdate', 'publisher', + 'rating', 'series_index', 'series', 'size', 'tags', 'timestamp', + 'uuid', 'languages'] + +#Allowed fields for template +TEMPLATE_ALLOWED_FIELDS = [ 'author_sort', 'authors', 'id', 'isbn', 'pubdate', 'title_sort', + 'publisher', 'series_index', 'series', 'tags', 'timestamp', 'title', 'uuid' ] + + diff --git a/src/calibre/library/catalogs/bibtex.py b/src/calibre/library/catalogs/bibtex.py new file mode 100644 index 0000000000..1815940931 --- /dev/null +++ b/src/calibre/library/catalogs/bibtex.py @@ -0,0 +1,396 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai + +__license__ = 'GPL v3' +__copyright__ = '2012, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + +import re, codecs +from collections import namedtuple +from types import StringType, UnicodeType + +from calibre import (strftime) +from calibre.customize import CatalogPlugin +from calibre.library.catalogs import FIELDS, TEMPLATE_ALLOWED_FIELDS +from calibre.utils.logging import default_log as log +from calibre.customize.conversion import DummyReporter +from calibre.constants import preferred_encoding + + +class BIBTEX(CatalogPlugin): + 'BIBTEX catalog generator' + + Option = namedtuple('Option', 'option, default, dest, action, help') + + name = 'Catalog_BIBTEX' + description = 'BIBTEX catalog generator' + supported_platforms = ['windows', 'osx', 'linux'] + author = 'Sengian' + version = (1, 0, 0) + file_types = set(['bib']) + + cli_options = [ + Option('--fields', + default = 'all', + dest = 'fields', + action = None, + help = _('The fields to output when cataloging books in the ' + 'database. Should be a comma-separated list of fields.\n' + 'Available fields: %(fields)s.\n' + 'plus user-created custom fields.\n' + 'Example: %(opt)s=title,authors,tags\n' + "Default: '%%default'\n" + "Applies to: BIBTEX output format")%dict( + fields=', '.join(FIELDS), opt='--fields')), + + Option('--sort-by', + default = 'id', + dest = 'sort_by', + action = None, + help = _('Output field to sort on.\n' + 'Available fields: author_sort, id, rating, size, timestamp, title.\n' + "Default: '%default'\n" + "Applies to: BIBTEX output format")), + + Option('--create-citation', + default = 'True', + dest = 'impcit', + action = None, + help = _('Create a citation for BibTeX entries.\n' + 'Boolean value: True, False\n' + "Default: '%default'\n" + "Applies to: BIBTEX output format")), + + Option('--add-files-path', + default = 'True', + dest = 'addfiles', + action = None, + help = _('Create a file entry if formats is selected for BibTeX entries.\n' + 'Boolean value: True, False\n' + "Default: '%default'\n" + "Applies to: BIBTEX output format")), + + Option('--citation-template', + default = '{authors}{id}', + dest = 'bib_cit', + action = None, + help = _('The template for citation creation from database fields.\n' + 'Should be a template with {} enclosed fields.\n' + 'Available fields: %s.\n' + "Default: '%%default'\n" + "Applies to: BIBTEX output format")%', '.join(TEMPLATE_ALLOWED_FIELDS)), + + Option('--choose-encoding', + default = 'utf8', + dest = 'bibfile_enc', + action = None, + help = _('BibTeX file encoding output.\n' + 'Available types: utf8, cp1252, ascii.\n' + "Default: '%default'\n" + "Applies to: BIBTEX output format")), + + Option('--choose-encoding-configuration', + default = 'strict', + dest = 'bibfile_enctag', + action = None, + help = _('BibTeX file encoding flag.\n' + 'Available types: strict, replace, ignore, backslashreplace.\n' + "Default: '%default'\n" + "Applies to: BIBTEX output format")), + + Option('--entry-type', + default = 'book', + dest = 'bib_entry', + action = None, + help = _('Entry type for BibTeX catalog.\n' + 'Available types: book, misc, mixed.\n' + "Default: '%default'\n" + "Applies to: BIBTEX output format"))] + + def run(self, path_to_output, opts, db, notification=DummyReporter()): + from calibre.utils.date import isoformat + from calibre.utils.html2text import html2text + from calibre.utils.bibtex import BibTeX + from calibre.library.save_to_disk import preprocess_template + from calibre.utils.date import now as nowf + + def create_bibtex_entry(entry, fields, mode, template_citation, + bibtexdict, db, citation_bibtex=True, calibre_files=True): + + #Bibtex doesn't like UTF-8 but keep unicode until writing + #Define starting chain or if book valid strict and not book return a Fail string + + bibtex_entry = [] + if mode != "misc" and check_entry_book_valid(entry) : + bibtex_entry.append(u'@book{') + elif mode != "book" : + bibtex_entry.append(u'@misc{') + else : + #case strict book + return '' + + if citation_bibtex : + # Citation tag + bibtex_entry.append(make_bibtex_citation(entry, template_citation, + bibtexdict)) + bibtex_entry = [u' '.join(bibtex_entry)] + + for field in fields: + if field.startswith('#'): + item = db.get_field(entry['id'],field,index_is_id=True) + if isinstance(item, (bool, float, int)): + item = repr(item) + elif field == 'title_sort': + item = entry['sort'] + else: + item = entry[field] + + #check if the field should be included (none or empty) + if item is None: + continue + try: + if len(item) == 0 : + continue + except TypeError: + pass + + if field == 'authors' : + bibtex_entry.append(u'author = "%s"' % bibtexdict.bibtex_author_format(item)) + + elif field == 'id' : + bibtex_entry.append(u'calibreid = "%s"' % int(item)) + + elif field == 'rating' : + bibtex_entry.append(u'rating = "%s"' % int(item)) + + elif field == 'size' : + bibtex_entry.append(u'%s = "%s octets"' % (field, int(item))) + + elif field == 'tags' : + #A list to flatten + bibtex_entry.append(u'tags = "%s"' % bibtexdict.utf8ToBibtex(u', '.join(item))) + + elif field == 'comments' : + #\n removal + item = item.replace(u'\r\n',u' ') + item = item.replace(u'\n',u' ') + #html to text + try: + item = html2text(item) + except: + log.warn("Failed to convert comments to text") + bibtex_entry.append(u'note = "%s"' % bibtexdict.utf8ToBibtex(item)) + + elif field == 'isbn' : + # Could be 9, 10 or 13 digits + bibtex_entry.append(u'isbn = "%s"' % re.sub(u'[0-9xX]', u'', item)) + + elif field == 'formats' : + #Add file path if format is selected + formats = [format.rpartition('.')[2].lower() for format in item] + bibtex_entry.append(u'formats = "%s"' % u', '.join(formats)) + if calibre_files: + files = [u':%s:%s' % (format, format.rpartition('.')[2].upper())\ + for format in item] + bibtex_entry.append(u'file = "%s"' % u', '.join(files)) + + elif field == 'series_index' : + bibtex_entry.append(u'volume = "%s"' % int(item)) + + elif field == 'timestamp' : + bibtex_entry.append(u'timestamp = "%s"' % isoformat(item).partition('T')[0]) + + elif field == 'pubdate' : + bibtex_entry.append(u'year = "%s"' % item.year) + bibtex_entry.append(u'month = "%s"' % bibtexdict.utf8ToBibtex(strftime("%b", item))) + + elif field.startswith('#') and isinstance(item, basestring): + bibtex_entry.append(u'custom_%s = "%s"' % (field[1:], + bibtexdict.utf8ToBibtex(item))) + + elif isinstance(item, basestring): + # elif field in ['title', 'publisher', 'cover', 'uuid', 'ondevice', + # 'author_sort', 'series', 'title_sort'] : + bibtex_entry.append(u'%s = "%s"' % (field, bibtexdict.utf8ToBibtex(item))) + + bibtex_entry = u',\n '.join(bibtex_entry) + bibtex_entry += u' }\n\n' + + return bibtex_entry + + def check_entry_book_valid(entry): + #Check that the required fields are ok for a book entry + for field in ['title', 'authors', 'publisher'] : + if entry[field] is None or len(entry[field]) == 0 : + return False + if entry['pubdate'] is None : + return False + else : + return True + + def make_bibtex_citation(entry, template_citation, bibtexclass): + + #define a function to replace the template entry by its value + def tpl_replace(objtplname) : + + tpl_field = re.sub(u'[\{\}]', u'', objtplname.group()) + + if tpl_field in TEMPLATE_ALLOWED_FIELDS : + if tpl_field in ['pubdate', 'timestamp'] : + tpl_field = isoformat(entry[tpl_field]).partition('T')[0] + elif tpl_field in ['tags', 'authors'] : + tpl_field =entry[tpl_field][0] + elif tpl_field in ['id', 'series_index'] : + tpl_field = str(entry[tpl_field]) + else : + tpl_field = entry[tpl_field] + return tpl_field + else: + return u'' + + if len(template_citation) >0 : + tpl_citation = bibtexclass.utf8ToBibtex( + bibtexclass.ValidateCitationKey(re.sub(u'\{[^{}]*\}', + tpl_replace, template_citation))) + + if len(tpl_citation) >0 : + return tpl_citation + + if len(entry["isbn"]) > 0 : + template_citation = u'%s' % re.sub(u'[\D]',u'', entry["isbn"]) + + else : + template_citation = u'%s' % str(entry["id"]) + + return bibtexclass.ValidateCitationKey(template_citation) + + self.fmt = path_to_output.rpartition('.')[2] + self.notification = notification + + # Combobox options + bibfile_enc = ['utf8', 'cp1252', 'ascii'] + bibfile_enctag = ['strict', 'replace', 'ignore', 'backslashreplace'] + bib_entry = ['mixed', 'misc', 'book'] + + # Needed beacause CLI return str vs int by widget + try: + bibfile_enc = bibfile_enc[opts.bibfile_enc] + bibfile_enctag = bibfile_enctag[opts.bibfile_enctag] + bib_entry = bib_entry[opts.bib_entry] + except: + if opts.bibfile_enc in bibfile_enc : + bibfile_enc = opts.bibfile_enc + else : + log.warn("Incorrect --choose-encoding flag, revert to default") + bibfile_enc = bibfile_enc[0] + if opts.bibfile_enctag in bibfile_enctag : + bibfile_enctag = opts.bibfile_enctag + else : + log.warn("Incorrect --choose-encoding-configuration flag, revert to default") + bibfile_enctag = bibfile_enctag[0] + if opts.bib_entry in bib_entry : + bib_entry = opts.bib_entry + else : + log.warn("Incorrect --entry-type flag, revert to default") + bib_entry = bib_entry[0] + + if opts.verbose: + opts_dict = vars(opts) + log("%s(): Generating %s" % (self.name,self.fmt)) + if opts.connected_device['is_device_connected']: + log(" connected_device: %s" % opts.connected_device['name']) + if opts_dict['search_text']: + log(" --search='%s'" % opts_dict['search_text']) + + if opts_dict['ids']: + log(" Book count: %d" % len(opts_dict['ids'])) + if opts_dict['search_text']: + log(" (--search ignored when a subset of the database is specified)") + + if opts_dict['fields']: + if opts_dict['fields'] == 'all': + log(" Fields: %s" % ', '.join(FIELDS[1:])) + else: + log(" Fields: %s" % opts_dict['fields']) + + log(" Output file will be encoded in %s with %s flag" % (bibfile_enc, bibfile_enctag)) + + log(" BibTeX entry type is %s with a citation like '%s' flag" % (bib_entry, opts_dict['bib_cit'])) + + # If a list of ids are provided, don't use search_text + if opts.ids: + opts.search_text = None + + data = self.search_sort_db(db, opts) + + if not len(data): + log.error("\nNo matching database entries for search criteria '%s'" % opts.search_text) + + # Get the requested output fields as a list + fields = self.get_output_fields(db, opts) + + if not len(data): + log.error("\nNo matching database entries for search criteria '%s'" % opts.search_text) + + #Initialize BibTeX class + bibtexc = BibTeX() + + #Entries writing after Bibtex formating (or not) + if bibfile_enc != 'ascii' : + bibtexc.ascii_bibtex = False + else : + bibtexc.ascii_bibtex = True + + #Check citation choice and go to default in case of bad CLI + if isinstance(opts.impcit, (StringType, UnicodeType)) : + if opts.impcit == 'False' : + citation_bibtex= False + elif opts.impcit == 'True' : + citation_bibtex= True + else : + log.warn("Incorrect --create-citation, revert to default") + citation_bibtex= True + else : + citation_bibtex= opts.impcit + + #Check add file entry and go to default in case of bad CLI + if isinstance(opts.addfiles, (StringType, UnicodeType)) : + if opts.addfiles == 'False' : + addfiles_bibtex = False + elif opts.addfiles == 'True' : + addfiles_bibtex = True + else : + log.warn("Incorrect --add-files-path, revert to default") + addfiles_bibtex= True + else : + addfiles_bibtex = opts.addfiles + + #Preprocess for error and light correction + template_citation = preprocess_template(opts.bib_cit) + + #Open output and write entries + with codecs.open(path_to_output, 'w', bibfile_enc, bibfile_enctag)\ + as outfile: + #File header + nb_entries = len(data) + + #check in book strict if all is ok else throw a warning into log + if bib_entry == 'book' : + nb_books = len(filter(check_entry_book_valid, data)) + if nb_books < nb_entries : + log.warn("Only %d entries in %d are book compatible" % (nb_books, nb_entries)) + nb_entries = nb_books + + # If connected device, add 'On Device' values to data + if opts.connected_device['is_device_connected'] and 'ondevice' in fields: + for entry in data: + entry['ondevice'] = db.catalog_plugin_on_device_temp_mapping[entry['id']]['ondevice'] + + outfile.write(u'%%%Calibre catalog\n%%%{0} entries in catalog\n\n'.format(nb_entries)) + outfile.write(u'@preamble{"This catalog of %d entries was generated by calibre on %s"}\n\n' + % (nb_entries, nowf().strftime("%A, %d. %B %Y %H:%M").decode(preferred_encoding))) + + for entry in data: + outfile.write(create_bibtex_entry(entry, fields, bib_entry, template_citation, + bibtexc, db, citation_bibtex, addfiles_bibtex)) + diff --git a/src/calibre/library/catalogs/csv_xml.py b/src/calibre/library/catalogs/csv_xml.py new file mode 100644 index 0000000000..a09084889b --- /dev/null +++ b/src/calibre/library/catalogs/csv_xml.py @@ -0,0 +1,214 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai + +__license__ = 'GPL v3' +__copyright__ = '2012, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + +import re, codecs, os +from collections import namedtuple + +from calibre.customize import CatalogPlugin +from calibre.library.catalogs import FIELDS +from calibre.utils.logging import default_log as log +from calibre.customize.conversion import DummyReporter + +class CSV_XML(CatalogPlugin): + 'CSV/XML catalog generator' + + Option = namedtuple('Option', 'option, default, dest, action, help') + + name = 'Catalog_CSV_XML' + description = 'CSV/XML catalog generator' + supported_platforms = ['windows', 'osx', 'linux'] + author = 'Greg Riker' + version = (1, 0, 0) + file_types = set(['csv','xml']) + + cli_options = [ + Option('--fields', + default = 'all', + dest = 'fields', + action = None, + help = _('The fields to output when cataloging books in the ' + 'database. Should be a comma-separated list of fields.\n' + 'Available fields: %(fields)s,\n' + 'plus user-created custom fields.\n' + 'Example: %(opt)s=title,authors,tags\n' + "Default: '%%default'\n" + "Applies to: CSV, XML output formats")%dict( + fields=', '.join(FIELDS), opt='--fields')), + + Option('--sort-by', + default = 'id', + dest = 'sort_by', + action = None, + help = _('Output field to sort on.\n' + 'Available fields: author_sort, id, rating, size, timestamp, title_sort\n' + "Default: '%default'\n" + "Applies to: CSV, XML output formats"))] + + def run(self, path_to_output, opts, db, notification=DummyReporter()): + from calibre.utils.date import isoformat + from calibre.utils.html2text import html2text + from lxml import etree + + self.fmt = path_to_output.rpartition('.')[2] + self.notification = notification + + if opts.verbose: + opts_dict = vars(opts) + log("%s(): Generating %s" % (self.name,self.fmt.upper())) + if opts.connected_device['is_device_connected']: + log(" connected_device: %s" % opts.connected_device['name']) + if opts_dict['search_text']: + log(" --search='%s'" % opts_dict['search_text']) + + if opts_dict['ids']: + log(" Book count: %d" % len(opts_dict['ids'])) + if opts_dict['search_text']: + log(" (--search ignored when a subset of the database is specified)") + + if opts_dict['fields']: + if opts_dict['fields'] == 'all': + log(" Fields: %s" % ', '.join(FIELDS[1:])) + else: + log(" Fields: %s" % opts_dict['fields']) + + # If a list of ids are provided, don't use search_text + if opts.ids: + opts.search_text = None + + data = self.search_sort_db(db, opts) + + if not len(data): + log.error("\nNo matching database entries for search criteria '%s'" % opts.search_text) + #raise SystemExit(1) + + # Get the requested output fields as a list + fields = self.get_output_fields(db, opts) + + # If connected device, add 'On Device' values to data + if opts.connected_device['is_device_connected'] and 'ondevice' in fields: + for entry in data: + entry['ondevice'] = db.catalog_plugin_on_device_temp_mapping[entry['id']]['ondevice'] + + if self.fmt == 'csv': + outfile = codecs.open(path_to_output, 'w', 'utf8') + + # Write a UTF-8 BOM + outfile.write('\xef\xbb\xbf') + + # Output the field headers + outfile.write(u'%s\n' % u','.join(fields)) + + # Output the entry fields + for entry in data: + outstr = [] + for field in fields: + if field.startswith('#'): + item = db.get_field(entry['id'],field,index_is_id=True) + elif field == 'title_sort': + item = entry['sort'] + else: + item = entry[field] + + if item is None: + outstr.append('""') + continue + elif field == 'formats': + fmt_list = [] + for format in item: + fmt_list.append(format.rpartition('.')[2].lower()) + item = ', '.join(fmt_list) + elif field in ['authors','tags']: + item = ', '.join(item) + elif field == 'isbn': + # Could be 9, 10 or 13 digits + item = u'%s' % re.sub(r'[\D]', '', item) + elif field in ['pubdate', 'timestamp']: + item = isoformat(item) + elif field == 'comments': + item = item.replace(u'\r\n',u' ') + item = item.replace(u'\n',u' ') + + # Convert HTML to markdown text + if type(item) is unicode: + opening_tag = re.search('<(\w+)(\x20|>)',item) + if opening_tag: + closing_tag = re.search('<\/%s>$' % opening_tag.group(1), item) + if closing_tag: + item = html2text(item) + + outstr.append(u'"%s"' % unicode(item).replace('"','""')) + + outfile.write(u','.join(outstr) + u'\n') + outfile.close() + + elif self.fmt == 'xml': + from lxml.builder import E + + root = E.calibredb() + for r in data: + record = E.record() + root.append(record) + + for field in fields: + if field.startswith('#'): + val = db.get_field(r['id'],field,index_is_id=True) + if not isinstance(val, (str, unicode)): + val = unicode(val) + item = getattr(E, field.replace('#','_'))(val) + record.append(item) + + for field in ('id', 'uuid', 'publisher', 'rating', 'size', + 'isbn','ondevice'): + if field in fields: + val = r[field] + if not val: + continue + if not isinstance(val, (str, unicode)): + val = unicode(val) + item = getattr(E, field)(val) + record.append(item) + + if 'title' in fields: + title = E.title(r['title'], sort=r['sort']) + record.append(title) + + if 'authors' in fields: + aus = E.authors(sort=r['author_sort']) + for au in r['authors']: + aus.append(E.author(au)) + record.append(aus) + + for field in ('timestamp', 'pubdate'): + if field in fields: + record.append(getattr(E, field)(r[field].isoformat())) + + if 'tags' in fields and r['tags']: + tags = E.tags() + for tag in r['tags']: + tags.append(E.tag(tag)) + record.append(tags) + + if 'comments' in fields and r['comments']: + record.append(E.comments(r['comments'])) + + if 'series' in fields and r['series']: + record.append(E.series(r['series'], + index=str(r['series_index']))) + + if 'cover' in fields and r['cover']: + record.append(E.cover(r['cover'].replace(os.sep, '/'))) + + if 'formats' in fields and r['formats']: + fmt = E.formats() + for f in r['formats']: + fmt.append(E.format(f.replace(os.sep, '/'))) + record.append(fmt) + + with open(path_to_output, 'w') as f: + f.write(etree.tostring(root, encoding='utf-8', + xml_declaration=True, pretty_print=True)) + diff --git a/src/calibre/library/catalogs/epub_mobi.py b/src/calibre/library/catalogs/epub_mobi.py new file mode 100644 index 0000000000..97669f6ad8 --- /dev/null +++ b/src/calibre/library/catalogs/epub_mobi.py @@ -0,0 +1,365 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +__license__ = 'GPL v3' +__copyright__ = '2012, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + +import os +from collections import namedtuple + +from calibre import strftime +from calibre.constants import DEBUG +from calibre.customize import CatalogPlugin +from calibre.customize.conversion import OptionRecommendation, DummyReporter +from calibre.utils.logging import default_log as log + +Option = namedtuple('Option', 'option, default, dest, action, help') + +class EPUB_MOBI(CatalogPlugin): + 'ePub catalog generator' + + name = 'Catalog_EPUB_MOBI' + description = 'EPUB/MOBI catalog generator' + supported_platforms = ['windows', 'osx', 'linux'] + minimum_calibre_version = (0, 7, 40) + author = 'Greg Riker' + version = (1, 0, 0) + file_types = set(['epub','mobi']) + + THUMB_SMALLEST = "1.0" + THUMB_LARGEST = "2.0" + + cli_options = [Option('--catalog-title', # {{{ + default = 'My Books', + dest = 'catalog_title', + action = None, + help = _('Title of generated catalog used as title in metadata.\n' + "Default: '%default'\n" + "Applies to: ePub, MOBI output formats")), + Option('--debug-pipeline', + default=None, + dest='debug_pipeline', + action = None, + help=_("Save the output from different stages of the conversion " + "pipeline to the specified " + "directory. Useful if you are unsure at which stage " + "of the conversion process a bug is occurring.\n" + "Default: '%default'\n" + "Applies to: ePub, MOBI output formats")), + Option('--exclude-book-marker', + default=':', + dest='exclude_book_marker', + action = None, + help=_("field:pattern specifying custom field/contents indicating book should be excluded.\n" + "Default: '%default'\n" + "Applies to ePub, MOBI output formats")), + Option('--exclude-genre', + default='\[.+\]', + dest='exclude_genre', + action = None, + help=_("Regex describing tags to exclude as genres.\n" "Default: '%default' excludes bracketed tags, e.g. '[]'\n" + "Applies to: ePub, MOBI output formats")), + Option('--exclude-tags', + default=('~,'+_('Catalog')), + dest='exclude_tags', + action = None, + help=_("Comma-separated list of tag words indicating book should be excluded from output. " + "For example: 'skip' will match 'skip this book' and 'Skip will like this'. " + "Default: '%default'\n" + "Applies to: ePub, MOBI output formats")), + Option('--generate-authors', + default=False, + dest='generate_authors', + action = 'store_true', + help=_("Include 'Authors' section in catalog.\n" + "Default: '%default'\n" + "Applies to: ePub, MOBI output formats")), + Option('--generate-descriptions', + default=False, + dest='generate_descriptions', + action = 'store_true', + help=_("Include 'Descriptions' section in catalog.\n" + "Default: '%default'\n" + "Applies to: ePub, MOBI output formats")), + Option('--generate-genres', + default=False, + dest='generate_genres', + action = 'store_true', + help=_("Include 'Genres' section in catalog.\n" + "Default: '%default'\n" + "Applies to: ePub, MOBI output formats")), + Option('--generate-titles', + default=False, + dest='generate_titles', + action = 'store_true', + help=_("Include 'Titles' section in catalog.\n" + "Default: '%default'\n" + "Applies to: ePub, MOBI output formats")), + Option('--generate-series', + default=False, + dest='generate_series', + action = 'store_true', + help=_("Include 'Series' section in catalog.\n" + "Default: '%default'\n" + "Applies to: ePub, MOBI output formats")), + Option('--generate-recently-added', + default=False, + dest='generate_recently_added', + action = 'store_true', + help=_("Include 'Recently Added' section in catalog.\n" + "Default: '%default'\n" + "Applies to: ePub, MOBI output formats")), + Option('--header-note-source-field', + default='', + dest='header_note_source_field', + action = None, + help=_("Custom field containing note text to insert in Description header.\n" + "Default: '%default'\n" + "Applies to: ePub, MOBI output formats")), + Option('--merge-comments', + default='::', + dest='merge_comments', + action = None, + help=_(":[before|after]:[True|False] specifying:\n" + " Custom field containing notes to merge with Comments\n" + " [before|after] Placement of notes with respect to Comments\n" + " [True|False] - A horizontal rule is inserted between notes and Comments\n" + "Default: '%default'\n" + "Applies to ePub, MOBI output formats")), + Option('--output-profile', + default=None, + dest='output_profile', + action = None, + help=_("Specifies the output profile. In some cases, an output profile is required to optimize the catalog for the device. For example, 'kindle' or 'kindle_dx' creates a structured Table of Contents with Sections and Articles.\n" + "Default: '%default'\n" + "Applies to: ePub, MOBI output formats")), + Option('--read-book-marker', + default='tag:+', + dest='read_book_marker', + action = None, + help=_("field:pattern indicating book has been read.\n" "Default: '%default'\n" + "Applies to ePub, MOBI output formats")), + Option('--thumb-width', + default='1.0', + dest='thumb_width', + action = None, + help=_("Size hint (in inches) for book covers in catalog.\n" + "Range: 1.0 - 2.0\n" + "Default: '%default'\n" + "Applies to ePub, MOBI output formats")), + Option('--wishlist-tag', + default='Wishlist', + dest='wishlist_tag', + action = None, + help=_("Tag indicating book to be displayed as wishlist item.\n" "Default: '%default'\n" + "Applies to: ePub, MOBI output formats")), + ] + # }}} + + def run(self, path_to_output, opts, db, notification=DummyReporter()): + from calibre.library.catalogs.epub_mobi_builder import CatalogBuilder + opts.log = log + opts.fmt = self.fmt = path_to_output.rpartition('.')[2] + + # Add local options + opts.creator = '%s, %s %s, %s' % (strftime('%A'), strftime('%B'), strftime('%d').lstrip('0'), strftime('%Y')) + opts.creator_sort_as = '%s %s' % ('calibre', strftime('%Y-%m-%d')) + opts.connected_kindle = False + + # Finalize output_profile + op = opts.output_profile + if op is None: + op = 'default' + + if opts.connected_device['name'] and 'kindle' in opts.connected_device['name'].lower(): + opts.connected_kindle = True + if opts.connected_device['serial'] and \ + opts.connected_device['serial'][:4] in ['B004','B005']: + op = "kindle_dx" + else: + op = "kindle" + opts.descriptionClip = 380 if op.endswith('dx') or 'kindle' not in op else 100 + opts.authorClip = 100 if op.endswith('dx') or 'kindle' not in op else 60 + opts.output_profile = op + + opts.basename = "Catalog" + opts.cli_environment = not hasattr(opts,'sync') + + # Hard-wired to always sort descriptions by author, with series after non-series + opts.sort_descriptions_by_author = True + + build_log = [] + + build_log.append(u"%s(): Generating %s %sin %s environment" % + (self.name,self.fmt,'for %s ' % opts.output_profile if opts.output_profile else '', + 'CLI' if opts.cli_environment else 'GUI')) + + # If exclude_genre is blank, assume user wants all genre tags included + if opts.exclude_genre.strip() == '': + opts.exclude_genre = '\[^.\]' + build_log.append(" converting empty exclude_genre to '\[^.\]'") + + if opts.connected_device['is_device_connected'] and \ + opts.connected_device['kind'] == 'device': + if opts.connected_device['serial']: + build_log.append(u" connected_device: '%s' #%s%s " % \ + (opts.connected_device['name'], + opts.connected_device['serial'][0:4], + 'x' * (len(opts.connected_device['serial']) - 4))) + for storage in opts.connected_device['storage']: + if storage: + build_log.append(u" mount point: %s" % storage) + else: + build_log.append(u" connected_device: '%s'" % opts.connected_device['name']) + try: + for storage in opts.connected_device['storage']: + if storage: + build_log.append(u" mount point: %s" % storage) + except: + build_log.append(u" (no mount points)") + else: + build_log.append(u" connected_device: '%s'" % opts.connected_device['name']) + + opts_dict = vars(opts) + if opts_dict['ids']: + build_log.append(" book count: %d" % len(opts_dict['ids'])) + + sections_list = [] + if opts.generate_authors: + sections_list.append('Authors') + if opts.generate_titles: + sections_list.append('Titles') + if opts.generate_series: + sections_list.append('Series') + if opts.generate_genres: + sections_list.append('Genres') + if opts.generate_recently_added: + sections_list.append('Recently Added') + if opts.generate_descriptions: + sections_list.append('Descriptions') + + if not sections_list: + if opts.cli_environment: + opts.log.warn('*** No Section switches specified, enabling all Sections ***') + opts.generate_authors = True + opts.generate_titles = True + opts.generate_series = True + opts.generate_genres = True + opts.generate_recently_added = True + opts.generate_descriptions = True + sections_list = ['Authors','Titles','Series','Genres','Recently Added','Descriptions'] + else: + opts.log.warn('\n*** No enabled Sections, terminating catalog generation ***') + return ["No Included Sections","No enabled Sections.\nCheck E-book options tab\n'Included sections'\n"] + if opts.fmt == 'mobi' and sections_list == ['Descriptions']: + warning = _("\n*** Adding 'By Authors' Section required for MOBI output ***") + opts.log.warn(warning) + sections_list.insert(0,'Authors') + opts.generate_authors = True + + opts.log(u" Sections: %s" % ', '.join(sections_list)) + opts.section_list = sections_list + + # Limit thumb_width to 1.0" - 2.0" + try: + if float(opts.thumb_width) < float(self.THUMB_SMALLEST): + log.warning("coercing thumb_width from '%s' to '%s'" % (opts.thumb_width,self.THUMB_SMALLEST)) + opts.thumb_width = self.THUMB_SMALLEST + if float(opts.thumb_width) > float(self.THUMB_LARGEST): + log.warning("coercing thumb_width from '%s' to '%s'" % (opts.thumb_width,self.THUMB_LARGEST)) + opts.thumb_width = self.THUMB_LARGEST + opts.thumb_width = "%.2f" % float(opts.thumb_width) + except: + log.error("coercing thumb_width from '%s' to '%s'" % (opts.thumb_width,self.THUMB_SMALLEST)) + opts.thumb_width = "1.0" + + # Display opts + keys = opts_dict.keys() + keys.sort() + build_log.append(" opts:") + for key in keys: + if key in ['catalog_title','authorClip','connected_kindle','descriptionClip', + 'exclude_book_marker','exclude_genre','exclude_tags', + 'header_note_source_field','merge_comments', + 'output_profile','read_book_marker', + 'search_text','sort_by','sort_descriptions_by_author','sync', + 'thumb_width','wishlist_tag']: + build_log.append(" %s: %s" % (key, repr(opts_dict[key]))) + + if opts.verbose: + log('\n'.join(line for line in build_log)) + + self.opts = opts + + # Launch the Catalog builder + catalog = CatalogBuilder(db, opts, self, report_progress=notification) + + if opts.verbose: + log.info(" Begin catalog source generation") + catalog.createDirectoryStructure() + catalog.copyResources() + catalog.calculateThumbnailSize() + catalog_source_built = catalog.buildSources() + + if opts.verbose: + if catalog_source_built: + log.info(" Completed catalog source generation\n") + else: + log.error(" *** Terminated catalog generation, check log for details ***") + + if catalog_source_built: + recommendations = [] + recommendations.append(('remove_fake_margins', False, + OptionRecommendation.HIGH)) + if DEBUG: + recommendations.append(('comments', '\n'.join(line for line in build_log), + OptionRecommendation.HIGH)) + else: + recommendations.append(('comments', '', OptionRecommendation.HIGH)) + + dp = getattr(opts, 'debug_pipeline', None) + if dp is not None: + recommendations.append(('debug_pipeline', dp, + OptionRecommendation.HIGH)) + + if opts.fmt == 'mobi' and opts.output_profile and opts.output_profile.startswith("kindle"): + recommendations.append(('output_profile', opts.output_profile, + OptionRecommendation.HIGH)) + recommendations.append(('no_inline_toc', True, + OptionRecommendation.HIGH)) + recommendations.append(('book_producer',opts.output_profile, + OptionRecommendation.HIGH)) + + # If cover exists, use it + cpath = None + try: + search_text = 'title:"%s" author:%s' % ( + opts.catalog_title.replace('"', '\\"'), 'calibre') + matches = db.search(search_text, return_matches=True) + if matches: + cpath = db.cover(matches[0], index_is_id=True, as_path=True) + if cpath and os.path.exists(cpath): + recommendations.append(('cover', cpath, + OptionRecommendation.HIGH)) + except: + pass + + # Run ebook-convert + from calibre.ebooks.conversion.plumber import Plumber + plumber = Plumber(os.path.join(catalog.catalogPath, + opts.basename + '.opf'), path_to_output, log, report_progress=notification, + abort_after_input_dump=False) + plumber.merge_ui_recommendations(recommendations) + plumber.run() + + try: + os.remove(cpath) + except: + pass + + # returns to gui2.actions.catalog:catalog_generated() + return catalog.error + diff --git a/src/calibre/library/catalogs/epub_mobi_builder.py b/src/calibre/library/catalogs/epub_mobi_builder.py new file mode 100644 index 0000000000..1383379db2 --- /dev/null +++ b/src/calibre/library/catalogs/epub_mobi_builder.py @@ -0,0 +1,4040 @@ +# -*- coding: utf-8 -*- + +__license__ = 'GPL v3' +__copyright__ = '2010, Greg Riker' + +import datetime, htmlentitydefs, os, re, shutil, zlib +from copy import deepcopy +from xml.sax.saxutils import escape + +from calibre import (prepare_string_for_xml, strftime, force_unicode) +from calibre.customize.conversion import DummyReporter +from calibre.ebooks.BeautifulSoup import BeautifulSoup, BeautifulStoneSoup, Tag, NavigableString +from calibre.ebooks.chardet import substitute_entites +from calibre.ptempfile import PersistentTemporaryDirectory +from calibre.utils.config import config_dir +from calibre.utils.date import format_date, is_date_undefined, now as nowf +from calibre.utils.icu import capitalize +from calibre.utils.magick.draw import thumbnail +from calibre.utils.zipfile import ZipFile + + + +class CatalogBuilder(object): + ''' + Generates catalog source files from calibre database + + Flow of control: + gui2.actions.catalog:generate_catalog() + gui2.tools:generate_catalog() or library.cli:command_catalog() + called from gui2.convert.gui_conversion:gui_catalog() + catalog = Catalog(notification=Reporter()) + catalog.createDirectoryStructure() + catalog.copyResources() + catalog.buildSources() + Options managed in gui2.catalog.catalog_epub_mobi.py + ''' + + # A single number creates 'Last x days' only. + # Multiple numbers create 'Last x days', 'x to y days ago' ... + # e.g, [7,15,30,60], [30] + # [] = No date ranges added + DATE_RANGE=[30] + + # basename output file basename + # creator dc:creator in OPF metadata + # descriptionClip limits size of NCX descriptions (Kindle only) + # includeSources Used in processSpecialTags to skip tags like '[SPL]' + # notification Used to check for cancel, report progress + # stylesheet CSS stylesheet + # title dc:title in OPF metadata, NCX periodical + # verbosity level of diagnostic printout + + def __init__(self, db, opts, plugin, + report_progress=DummyReporter(), + stylesheet="content/stylesheet.css"): + self.__opts = opts + self.__authorClip = opts.authorClip + self.__authors = None + self.__basename = opts.basename + self.__bookmarked_books = None + self.__booksByAuthor = None + self.__booksByDateRead = None + self.__booksByTitle = None + self.__booksByTitle_noSeriesPrefix = None + self.__cache_dir = os.path.join(config_dir, 'caches', 'catalog') + self.__archive_path = os.path.join(self.__cache_dir, "thumbs.zip") + self.__catalogPath = PersistentTemporaryDirectory("_epub_mobi_catalog", prefix='') + self.__contentDir = os.path.join(self.catalogPath, "content") + self.__currentStep = 0.0 + self.__creator = opts.creator + self.__db = db + self.__descriptionClip = opts.descriptionClip + self.__error = [] + self.__generateForKindle = True if (self.opts.fmt == 'mobi' and \ + self.opts.output_profile and \ + self.opts.output_profile.startswith("kindle")) else False + self.__generateRecentlyRead = True if self.opts.generate_recently_added \ + and self.opts.connected_kindle \ + and self.generateForKindle \ + else False + self.__genres = None + self.genres = [] + self.__genre_tags_dict = None + self.__htmlFileList_1 = [] + self.__htmlFileList_2 = [] + self.__markerTags = self.getMarkerTags() + self.__ncxSoup = None + self.__output_profile = None + self.__playOrder = 1 + self.__plugin = plugin + self.__progressInt = 0.0 + self.__progressString = '' + f, _, p = opts.read_book_marker.partition(':') + self.__read_book_marker = {'field':f, 'pattern':p} + f, p, hr = self.opts.merge_comments.split(':') + self.__merge_comments = {'field':f, 'position':p, 'hr':hr} + self.__reporter = report_progress + self.__stylesheet = stylesheet + self.__thumbs = None + self.__thumbWidth = 0 + self.__thumbHeight = 0 + self.__title = opts.catalog_title + self.__totalSteps = 6.0 + self.__useSeriesPrefixInTitlesSection = False + self.__verbose = opts.verbose + + from calibre.customize.ui import output_profiles + for profile in output_profiles(): + if profile.short_name == self.opts.output_profile: + self.__output_profile = profile + break + + # Confirm/create thumbs archive. + if self.opts.generate_descriptions: + if not os.path.exists(self.__cache_dir): + self.opts.log.info(" creating new thumb cache '%s'" % self.__cache_dir) + os.makedirs(self.__cache_dir) + if not os.path.exists(self.__archive_path): + self.opts.log.info(' creating thumbnail archive, thumb_width: %1.2f"' % + float(self.opts.thumb_width)) + with ZipFile(self.__archive_path, mode='w') as zfw: + zfw.writestr("Catalog Thumbs Archive",'') + else: + try: + with ZipFile(self.__archive_path, mode='r') as zfr: + try: + cached_thumb_width = zfr.read('thumb_width') + except: + cached_thumb_width = "-1" + except: + os.remove(self.__archive_path) + cached_thumb_width = '-1' + + if float(cached_thumb_width) != float(self.opts.thumb_width): + self.opts.log.warning(" invalidating cache at '%s'" % self.__archive_path) + self.opts.log.warning(' thumb_width changed: %1.2f" => %1.2f"' % + (float(cached_thumb_width),float(self.opts.thumb_width))) + with ZipFile(self.__archive_path, mode='w') as zfw: + zfw.writestr("Catalog Thumbs Archive",'') + else: + self.opts.log.info(' existing thumb cache at %s, cached_thumb_width: %1.2f"' % + (self.__archive_path, float(cached_thumb_width))) + + # Tweak build steps based on optional sections: 1 call for HTML, 1 for NCX + incremental_jobs = 0 + if self.opts.generate_authors: + incremental_jobs += 2 + if self.opts.generate_titles: + incremental_jobs += 2 + if self.opts.generate_recently_added: + incremental_jobs += 2 + if self.generateRecentlyRead: + incremental_jobs += 2 + if self.opts.generate_series: + incremental_jobs += 2 + if self.opts.generate_descriptions: + # +1 thumbs + incremental_jobs += 3 + self.__totalSteps += incremental_jobs + + # Load section list templates + templates = {} + execfile(P('catalog/section_list_templates.py'), templates) + for name, template in templates.iteritems(): + if name.startswith('by_') and name.endswith('_template'): + setattr(self, name, force_unicode(template, 'utf-8')) + + # Accessors + if True: + ''' + @dynamic_property + def xxxx(self): + def fget(self): + return self.__ + def fset(self, val): + self.__ = val + return property(fget=fget, fset=fset) + ''' + @dynamic_property + def authorClip(self): + def fget(self): + return self.__authorClip + def fset(self, val): + self.__authorClip = val + return property(fget=fget, fset=fset) + @dynamic_property + def authors(self): + def fget(self): + return self.__authors + def fset(self, val): + self.__authors = val + return property(fget=fget, fset=fset) + @dynamic_property + def basename(self): + def fget(self): + return self.__basename + def fset(self, val): + self.__basename = val + return property(fget=fget, fset=fset) + @dynamic_property + def bookmarked_books(self): + def fget(self): + return self.__bookmarked_books + def fset(self, val): + self.__bookmarked_books = val + return property(fget=fget, fset=fset) + @dynamic_property + def booksByAuthor(self): + def fget(self): + return self.__booksByAuthor + def fset(self, val): + self.__booksByAuthor = val + return property(fget=fget, fset=fset) + @dynamic_property + def booksByDateRead(self): + def fget(self): + return self.__booksByDateRead + def fset(self, val): + self.__booksByDateRead = val + return property(fget=fget, fset=fset) + @dynamic_property + def booksByTitle(self): + def fget(self): + return self.__booksByTitle + def fset(self, val): + self.__booksByTitle = val + return property(fget=fget, fset=fset) + @dynamic_property + def booksByTitle_noSeriesPrefix(self): + def fget(self): + return self.__booksByTitle_noSeriesPrefix + def fset(self, val): + self.__booksByTitle_noSeriesPrefix = val + return property(fget=fget, fset=fset) + @dynamic_property + def catalogPath(self): + def fget(self): + return self.__catalogPath + def fset(self, val): + self.__catalogPath = val + return property(fget=fget, fset=fset) + @dynamic_property + def contentDir(self): + def fget(self): + return self.__contentDir + def fset(self, val): + self.__contentDir = val + return property(fget=fget, fset=fset) + @dynamic_property + def currentStep(self): + def fget(self): + return self.__currentStep + def fset(self, val): + self.__currentStep = val + return property(fget=fget, fset=fset) + @dynamic_property + def creator(self): + def fget(self): + return self.__creator + def fset(self, val): + self.__creator = val + return property(fget=fget, fset=fset) + @dynamic_property + def db(self): + def fget(self): + return self.__db + return property(fget=fget) + @dynamic_property + def descriptionClip(self): + def fget(self): + return self.__descriptionClip + def fset(self, val): + self.__descriptionClip = val + return property(fget=fget, fset=fset) + @dynamic_property + def error(self): + def fget(self): + return self.__error + def fset(self, val): + self.__error = val + return property(fget=fget,fset=fset) + @dynamic_property + def generateForKindle(self): + def fget(self): + return self.__generateForKindle + def fset(self, val): + self.__generateForKindle = val + return property(fget=fget, fset=fset) + @dynamic_property + def generateRecentlyRead(self): + def fget(self): + return self.__generateRecentlyRead + def fset(self, val): + self.__generateRecentlyRead = val + return property(fget=fget, fset=fset) + @dynamic_property + def genres(self): + def fget(self): + return self.__genres + def fset(self, val): + self.__genres = val + return property(fget=fget, fset=fset) + @dynamic_property + def genre_tags_dict(self): + def fget(self): + return self.__genre_tags_dict + def fset(self, val): + self.__genre_tags_dict = val + return property(fget=fget, fset=fset) + @dynamic_property + def htmlFileList_1(self): + def fget(self): + return self.__htmlFileList_1 + def fset(self, val): + self.__htmlFileList_1 = val + return property(fget=fget, fset=fset) + @dynamic_property + def htmlFileList_2(self): + def fget(self): + return self.__htmlFileList_2 + def fset(self, val): + self.__htmlFileList_2 = val + return property(fget=fget, fset=fset) + @dynamic_property + def libraryPath(self): + def fget(self): + return self.__libraryPath + def fset(self, val): + self.__libraryPath = val + return property(fget=fget, fset=fset) + @dynamic_property + def markerTags(self): + def fget(self): + return self.__markerTags + def fset(self, val): + self.__markerTags = val + return property(fget=fget, fset=fset) + @dynamic_property + def ncxSoup(self): + def fget(self): + return self.__ncxSoup + def fset(self, val): + self.__ncxSoup = val + return property(fget=fget, fset=fset) + @dynamic_property + def opts(self): + def fget(self): + return self.__opts + return property(fget=fget) + @dynamic_property + def playOrder(self): + def fget(self): + return self.__playOrder + def fset(self,val): + self.__playOrder = val + return property(fget=fget, fset=fset) + @dynamic_property + def plugin(self): + def fget(self): + return self.__plugin + return property(fget=fget) + @dynamic_property + def progressInt(self): + def fget(self): + return self.__progressInt + def fset(self, val): + self.__progressInt = val + return property(fget=fget, fset=fset) + @dynamic_property + def progressString(self): + def fget(self): + return self.__progressString + def fset(self, val): + self.__progressString = val + return property(fget=fget, fset=fset) + @dynamic_property + def reporter(self): + def fget(self): + return self.__reporter + def fset(self, val): + self.__reporter = val + return property(fget=fget, fset=fset) + @dynamic_property + def stylesheet(self): + def fget(self): + return self.__stylesheet + def fset(self, val): + self.__stylesheet = val + return property(fget=fget, fset=fset) + @dynamic_property + def thumbs(self): + def fget(self): + return self.__thumbs + def fset(self, val): + self.__thumbs = val + return property(fget=fget, fset=fset) + def thumbWidth(self): + def fget(self): + return self.__thumbWidth + def fset(self, val): + self.__thumbWidth = val + return property(fget=fget, fset=fset) + def thumbHeight(self): + def fget(self): + return self.__thumbHeight + def fset(self, val): + self.__thumbHeight = val + return property(fget=fget, fset=fset) + @dynamic_property + def title(self): + def fget(self): + return self.__title + def fset(self, val): + self.__title = val + return property(fget=fget, fset=fset) + @dynamic_property + def totalSteps(self): + def fget(self): + return self.__totalSteps + return property(fget=fget) + @dynamic_property + def useSeriesPrefixInTitlesSection(self): + def fget(self): + return self.__useSeriesPrefixInTitlesSection + def fset(self, val): + self.__useSeriesPrefixInTitlesSection = val + return property(fget=fget, fset=fset) + @dynamic_property + def verbose(self): + def fget(self): + return self.__verbose + def fset(self, val): + self.__verbose = val + return property(fget=fget, fset=fset) + + @dynamic_property + def MISSING_SYMBOL(self): + def fget(self): + return self.__output_profile.missing_char + return property(fget=fget) + @dynamic_property + def NOT_READ_SYMBOL(self): + def fget(self): + return '%s' % self.__output_profile.read_char + return property(fget=fget) + @dynamic_property + def READING_SYMBOL(self): + def fget(self): + return '▷' if self.generateForKindle else \ + '+' + return property(fget=fget) + @dynamic_property + def READ_SYMBOL(self): + def fget(self): + return self.__output_profile.read_char + return property(fget=fget) + @dynamic_property + def FULL_RATING_SYMBOL(self): + def fget(self): + return self.__output_profile.ratings_char + return property(fget=fget) + @dynamic_property + def EMPTY_RATING_SYMBOL(self): + def fget(self): + return self.__output_profile.empty_ratings_char + return property(fget=fget) + @dynamic_property + def READ_PROGRESS_SYMBOL(self): + def fget(self): + return "▪" if self.generateForKindle else '+' + return property(fget=fget) + @dynamic_property + def UNREAD_PROGRESS_SYMBOL(self): + def fget(self): + return "▫" if self.generateForKindle else '-' + return property(fget=fget) + + # Methods + def buildSources(self): + if self.booksByTitle is None: + if not self.fetchBooksByTitle(): + return False + if not self.fetchBooksByAuthor(): + return False + self.fetchBookmarks() + if self.opts.generate_descriptions: + self.generateThumbnails() + self.generateHTMLDescriptions() + if self.opts.generate_authors: + self.generateHTMLByAuthor() + if self.opts.generate_titles: + self.generateHTMLByTitle() + if self.opts.generate_series: + self.generateHTMLBySeries() + if self.opts.generate_genres: + self.generateHTMLByTags() + # If this is the only Section, and there are no genres, bail + if self.opts.section_list == ['Genres'] and not self.genres: + error_msg = _("No enabled genres found to catalog.\n") + if not self.opts.cli_environment: + error_msg += "Check 'Excluded genres'\nin E-book options.\n" + self.opts.log.error(error_msg) + self.error.append(_('No books available to catalog')) + self.error.append(error_msg) + return False + if self.opts.generate_recently_added: + self.generateHTMLByDateAdded() + if self.generateRecentlyRead: + self.generateHTMLByDateRead() + + self.generateOPF() + self.generateNCXHeader() + if self.opts.generate_authors: + self.generateNCXByAuthor("Authors") + if self.opts.generate_titles: + self.generateNCXByTitle("Titles") + if self.opts.generate_series: + self.generateNCXBySeries("Series") + if self.opts.generate_genres: + self.generateNCXByGenre("Genres") + if self.opts.generate_recently_added: + self.generateNCXByDateAdded("Recently Added") + if self.generateRecentlyRead: + self.generateNCXByDateRead("Recently Read") + if self.opts.generate_descriptions: + self.generateNCXDescriptions("Descriptions") + + self.writeNCX() + return True + + def cleanUp(self): + pass + + def copyResources(self): + '''Move resource files to self.catalogPath''' + catalog_resources = P("catalog") + + files_to_copy = [('','DefaultCover.jpg'), + ('content','stylesheet.css'), + ('images','mastheadImage.gif')] + + for file in files_to_copy: + if file[0] == '': + shutil.copy(os.path.join(catalog_resources,file[1]), + self.catalogPath) + else: + shutil.copy(os.path.join(catalog_resources,file[1]), + os.path.join(self.catalogPath, file[0])) + + # Create the custom masthead image overwriting default + # If failure, default mastheadImage.gif should still be in place + if self.generateForKindle: + try: + self.generateMastheadImage(os.path.join(self.catalogPath, + 'images/mastheadImage.gif')) + except: + pass + + def fetchBooksByAuthor(self): + ''' + Generate a list of titles sorted by author from the database + return = Success + ''' + + self.updateProgressFullStep("Sorting database") + self.booksByAuthor = list(self.booksByTitle) + + # Test for author_sort mismatches + self.booksByAuthor = sorted(self.booksByAuthor, key=self.booksByAuthorSorter_author) + # Build the unique_authors set from existing data + authors = [(record['author'], record['author_sort']) for record in self.booksByAuthor] + current_author = authors[0] + for (i,author) in enumerate(authors): + if author != current_author and i: + if author[0] == current_author[0]: + if self.opts.fmt == 'mobi': + # Exit if building MOBI + error_msg = _( +'''Inconsistent Author Sort values for +Author '{0}': +'{1}' <> '{2}' +Unable to build MOBI catalog.\n +Select all books by '{0}', apply correct Author Sort value in Edit Metadata dialog, then rebuild the catalog.\n''').format(author[0],author[1],current_author[1]) + self.opts.log.warn('\n*** Metadata error ***') + self.opts.log.warn(error_msg) + + self.error.append('Author Sort mismatch') + self.error.append(error_msg) + return False + else: + # Warning if building non-MOBI + if not self.error: + self.error.append('Author Sort mismatch') + + error_msg = _( +'''Warning: inconsistent Author Sort values for +Author '{0}': +'{1}' <> '{2}'\n''').format(author[0],author[1],current_author[1]) + self.opts.log.warn('\n*** Metadata warning ***') + self.opts.log.warn(error_msg) + self.error.append(error_msg) + + current_author = author + + self.booksByAuthor = sorted(self.booksByAuthor, key=self.booksByAuthorSorter_author_sort) + + # Build the unique_authors set from existing data + authors = [(record['author'], capitalize(record['author_sort'])) for record in self.booksByAuthor] + + # authors[] contains a list of all book authors, with multiple entries for multiple books by author + # authors[]: (([0]:friendly [1]:sort)) + # unique_authors[]: (([0]:friendly [1]:sort [2]:book_count)) + books_by_current_author = 0 + current_author = authors[0] + multiple_authors = False + unique_authors = [] + for (i,author) in enumerate(authors): + if author != current_author: + # Note that current_author and author are tuples: (friendly, sort) + multiple_authors = True + + # New author, save the previous author/sort/count + unique_authors.append((current_author[0], icu_title(current_author[1]), + books_by_current_author)) + current_author = author + books_by_current_author = 1 + elif i==0 and len(authors) == 1: + # Allow for single-book lists + unique_authors.append((current_author[0], icu_title(current_author[1]), + books_by_current_author)) + else: + books_by_current_author += 1 + else: + # Add final author to list or single-author dataset + if (current_author == author and len(authors) > 1) or not multiple_authors: + unique_authors.append((current_author[0], icu_title(current_author[1]), + books_by_current_author)) + + if False and self.verbose: + self.opts.log.info("\nfetchBooksByauthor(): %d unique authors" % len(unique_authors)) + for author in unique_authors: + self.opts.log.info((u" %-50s %-25s %2d" % (author[0][0:45], author[1][0:20], + author[2])).encode('utf-8')) + + self.authors = unique_authors + return True + + def fetchBooksByTitle(self): + self.updateProgressFullStep("Fetching database") + + self.opts.sort_by = 'title' + + # Merge opts.exclude_tags with opts.search_text + # Updated to use exact match syntax + empty_exclude_tags = False if len(self.opts.exclude_tags) else True + search_phrase = '' + if not empty_exclude_tags: + exclude_tags = self.opts.exclude_tags.split(',') + search_terms = [] + for tag in exclude_tags: + search_terms.append("tag:=%s" % tag) + search_phrase = "not (%s)" % " or ".join(search_terms) + # If a list of ids are provided, don't use search_text + if self.opts.ids: + self.opts.search_text = search_phrase + else: + if self.opts.search_text: + self.opts.search_text += " " + search_phrase + else: + self.opts.search_text = search_phrase + + # Fetch the database as a dictionary + data = self.plugin.search_sort_db(self.db, self.opts) + data = self.processExclusions(data) + + # Populate this_title{} from data[{},{}] + titles = [] + for record in data: + this_title = {} + + this_title['id'] = record['id'] + this_title['uuid'] = record['uuid'] + + this_title['title'] = self.convertHTMLEntities(record['title']) + if record['series']: + this_title['series'] = record['series'] + this_title['series_index'] = record['series_index'] + else: + this_title['series'] = None + this_title['series_index'] = 0.0 + + this_title['title_sort'] = self.generateSortTitle(this_title['title']) + if 'authors' in record: + # from calibre.ebooks.metadata import authors_to_string + # return authors_to_string(self.authors) + + this_title['authors'] = record['authors'] + if record['authors']: + this_title['author'] = " & ".join(record['authors']) + else: + this_title['author'] = 'Unknown' + + if 'author_sort' in record and record['author_sort'].strip(): + this_title['author_sort'] = record['author_sort'] + else: + this_title['author_sort'] = self.author_to_author_sort(this_title['author']) + + if record['publisher']: + this_title['publisher'] = re.sub('&', '&', record['publisher']) + + this_title['rating'] = record['rating'] if record['rating'] else 0 + + if is_date_undefined(record['pubdate']): + this_title['date'] = None + else: + this_title['date'] = strftime(u'%B %Y', record['pubdate'].timetuple()) + + this_title['timestamp'] = record['timestamp'] + + if record['comments']: + # Strip annotations + a_offset = record['comments'].find('
') + ad_offset = record['comments'].find('
') + if a_offset >= 0: + record['comments'] = record['comments'][:a_offset] + if ad_offset >= 0: + record['comments'] = record['comments'][:ad_offset] + + this_title['description'] = self.markdownComments(record['comments']) + + # Create short description + paras = BeautifulSoup(this_title['description']).findAll('p') + tokens = [] + for p in paras: + for token in p.contents: + if token.string is not None: + tokens.append(token.string) + this_title['short_description'] = self.generateShortDescription(' '.join(tokens), dest="description") + else: + this_title['description'] = None + this_title['short_description'] = None + + # Merge with custom field/value + if self.__merge_comments['field']: + this_title['description'] = self.mergeComments(this_title) + + if record['cover']: + this_title['cover'] = re.sub('&', '&', record['cover']) + + this_title['read'] = self.discoverReadStatus(record) + + if record['tags']: + this_title['tags'] = self.processSpecialTags(record['tags'], + this_title, self.opts) + if record['formats']: + formats = [] + for format in record['formats']: + formats.append(self.convertHTMLEntities(format)) + this_title['formats'] = formats + + # Add user notes to be displayed in header + # Special case handling for datetime fields and lists + if self.opts.header_note_source_field: + field_md = self.__db.metadata_for_field(self.opts.header_note_source_field) + notes = self.__db.get_field(record['id'], + self.opts.header_note_source_field, + index_is_id=True) + if notes: + if field_md['datatype'] == 'text': + if isinstance(notes,list): + notes = ' · '.join(notes) + elif field_md['datatype'] == 'datetime': + notes = format_date(notes,'dd MMM yyyy') + this_title['notes'] = {'source':field_md['name'], + 'content':notes} + + titles.append(this_title) + + # Re-sort based on title_sort + if len(titles): + self.booksByTitle = sorted(titles, + key=lambda x:(x['title_sort'].upper(), x['title_sort'].upper())) + if False and self.verbose: + self.opts.log.info("fetchBooksByTitle(): %d books" % len(self.booksByTitle)) + self.opts.log.info(" %-40s %-40s" % ('title', 'title_sort')) + for title in self.booksByTitle: + self.opts.log.info((u" %-40s %-40s" % (title['title'][0:40], + title['title_sort'][0:40])).decode('mac-roman')) + return True + else: + error_msg = _("No books found to catalog.\nCheck 'Excluded books' criteria in E-book options.\n") + self.opts.log.error('*** ' + error_msg + ' ***') + self.error.append(_('No books available to include in catalog')) + self.error.append(error_msg) + return False + + def fetchBookmarks(self): + ''' + Collect bookmarks for catalog entries + This will use the system default save_template specified in + Preferences|Add/Save|Sending to device, not a customized one specified in + the Kindle plugin + ''' + from calibre.devices.usbms.device import Device + from calibre.devices.kindle.driver import Bookmark + from calibre.ebooks.metadata import MetaInformation + + MBP_FORMATS = [u'azw', u'mobi', u'prc', u'txt'] + mbp_formats = set(MBP_FORMATS) + PDR_FORMATS = [u'pdf'] + pdr_formats = set(PDR_FORMATS) + TAN_FORMATS = [u'tpz', u'azw1'] + tan_formats = set(TAN_FORMATS) + + class BookmarkDevice(Device): + def initialize(self, save_template): + self._save_template = save_template + self.SUPPORTS_SUB_DIRS = True + def save_template(self): + return self._save_template + + def resolve_bookmark_paths(storage, path_map): + pop_list = [] + book_ext = {} + for id in path_map: + file_fmts = set() + for fmt in path_map[id]['fmts']: + file_fmts.add(fmt) + + bookmark_extension = None + if file_fmts.intersection(mbp_formats): + book_extension = list(file_fmts.intersection(mbp_formats))[0] + bookmark_extension = 'mbp' + elif file_fmts.intersection(tan_formats): + book_extension = list(file_fmts.intersection(tan_formats))[0] + bookmark_extension = 'tan' + elif file_fmts.intersection(pdr_formats): + book_extension = list(file_fmts.intersection(pdr_formats))[0] + bookmark_extension = 'pdr' + + if bookmark_extension: + for vol in storage: + bkmk_path = path_map[id]['path'].replace(os.path.abspath('/'),vol) + bkmk_path = bkmk_path.replace('bookmark',bookmark_extension) + if os.path.exists(bkmk_path): + path_map[id] = bkmk_path + book_ext[id] = book_extension + break + else: + pop_list.append(id) + else: + pop_list.append(id) + # Remove non-existent bookmark templates + for id in pop_list: + path_map.pop(id) + return path_map, book_ext + + if self.generateRecentlyRead: + self.opts.log.info(" Collecting Kindle bookmarks matching catalog entries") + + d = BookmarkDevice(None) + d.initialize(self.opts.connected_device['save_template']) + + bookmarks = {} + for book in self.booksByTitle: + if 'formats' in book: + path_map = {} + id = book['id'] + original_title = book['title'][book['title'].find(':') + 2:] if book['series'] \ + else book['title'] + myMeta = MetaInformation(original_title, + authors=book['authors']) + myMeta.author_sort = book['author_sort'] + a_path = d.create_upload_path('/', myMeta, 'x.bookmark', create_dirs=False) + path_map[id] = dict(path=a_path, fmts=[x.rpartition('.')[2] for x in book['formats']]) + + path_map, book_ext = resolve_bookmark_paths(self.opts.connected_device['storage'], path_map) + if path_map: + bookmark_ext = path_map[id].rpartition('.')[2] + myBookmark = Bookmark(path_map[id], id, book_ext[id], bookmark_ext) + try: + book['percent_read'] = min(float(100*myBookmark.last_read / myBookmark.book_length),100) + except: + book['percent_read'] = 0 + dots = int((book['percent_read'] + 5)/10) + dot_string = self.READ_PROGRESS_SYMBOL * dots + empty_dots = self.UNREAD_PROGRESS_SYMBOL * (10 - dots) + book['reading_progress'] = '%s%s' % (dot_string,empty_dots) + bookmarks[id] = ((myBookmark,book)) + + self.bookmarked_books = bookmarks + else: + self.bookmarked_books = {} + + def generateHTMLDescriptions(self): + ''' + Write each title to a separate HTML file in contentdir + ''' + self.updateProgressFullStep("'Descriptions'") + + for (title_num, title) in enumerate(self.booksByTitle): + self.updateProgressMicroStep("Description %d of %d" % \ + (title_num, len(self.booksByTitle)), + float(title_num*100/len(self.booksByTitle))/100) + + # Generate the header from user-customizable template + soup = self.generateHTMLDescriptionHeader(title) + + # Write the book entry to contentdir + outfile = open("%s/book_%d.html" % (self.contentDir, int(title['id'])), 'w') + outfile.write(soup.prettify()) + outfile.close() + + def generateHTMLByTitle(self): + ''' + Write books by title A-Z to HTML file + ''' + self.updateProgressFullStep("'Titles'") + + soup = self.generateHTMLEmptyHeader("Books By Alpha Title") + body = soup.find('body') + btc = 0 + + # Insert section tag + aTag = Tag(soup,'a') + aTag['name'] = 'section_start' + body.insert(btc, aTag) + btc += 1 + + # Insert the anchor + aTag = Tag(soup, "a") + aTag['name'] = "bytitle" + body.insert(btc, aTag) + btc += 1 + + if not self.__generateForKindle: + # We don't need this because the Kindle shows section titles + #

By Title

+ pTag = Tag(soup, "p") + pTag['class'] = 'title' + aTag = Tag(soup, "a") + aTag['name'] = "bytitle" + pTag.insert(0,aTag) + pTag.insert(1,NavigableString('Titles')) + body.insert(btc,pTag) + btc += 1 + + divTag = Tag(soup, "div") + dtc = 0 + current_letter = "" + + # Re-sort title list without leading series/series_index + # Incoming title : + if not self.useSeriesPrefixInTitlesSection: + nspt = deepcopy(self.booksByTitle) + nspt = sorted(nspt, key=lambda x:(x['title_sort'].upper(), x['title_sort'].upper())) + self.booksByTitle_noSeriesPrefix = nspt + + # Loop through the books by title + # Generate one divRunningTag per initial letter for the purposes of + # minimizing widows and orphans on readers that can handle large + # <divs> styled as inline-block + title_list = self.booksByTitle + if not self.useSeriesPrefixInTitlesSection: + title_list = self.booksByTitle_noSeriesPrefix + drtc = 0 + divRunningTag = None + for book in title_list: + if self.letter_or_symbol(book['title_sort'][0]) != current_letter : + # Start a new letter + if drtc and divRunningTag is not None: + divTag.insert(dtc, divRunningTag) + dtc += 1 + divRunningTag = Tag(soup, 'div') + if dtc > 0: + divRunningTag['class'] = "initial_letter" + drtc = 0 + current_letter = self.letter_or_symbol(book['title_sort'][0]) + pIndexTag = Tag(soup, "p") + pIndexTag['class'] = "author_title_letter_index" + aTag = Tag(soup, "a") + aTag['name'] = "%s" % self.letter_or_symbol(book['title_sort'][0]) + pIndexTag.insert(0,aTag) + pIndexTag.insert(1,NavigableString(self.letter_or_symbol(book['title_sort'][0]))) + divRunningTag.insert(dtc,pIndexTag) + drtc += 1 + + # Add books + pBookTag = Tag(soup, "p") + ptc = 0 + + # book with read|reading|unread symbol or wishlist item + if self.opts.wishlist_tag in book.get('tags', []): + pBookTag['class'] = "wishlist_item" + pBookTag.insert(ptc,NavigableString(self.MISSING_SYMBOL)) + ptc += 1 + else: + if book['read']: + # check mark + pBookTag.insert(ptc,NavigableString(self.READ_SYMBOL)) + pBookTag['class'] = "read_book" + ptc += 1 + elif book['id'] in self.bookmarked_books: + pBookTag.insert(ptc,NavigableString(self.READING_SYMBOL)) + pBookTag['class'] = "read_book" + ptc += 1 + else: + # hidden check mark + pBookTag['class'] = "unread_book" + pBookTag.insert(ptc,NavigableString(self.NOT_READ_SYMBOL)) + ptc += 1 + + # Link to book + aTag = Tag(soup, "a") + if self.opts.generate_descriptions: + aTag['href'] = "book_%d.html" % (int(float(book['id']))) + + # Generate the title from the template + args = self.generateFormatArgs(book) + if book['series']: + formatted_title = self.by_titles_series_title_template.format(**args).rstrip() + else: + formatted_title = self.by_titles_normal_title_template.format(**args).rstrip() + aTag.insert(0,NavigableString(escape(formatted_title))) + pBookTag.insert(ptc, aTag) + ptc += 1 + + # Dot + pBookTag.insert(ptc, NavigableString(" · ")) + ptc += 1 + + # Link to author + emTag = Tag(soup, "em") + aTag = Tag(soup, "a") + if self.opts.generate_authors: + aTag['href'] = "%s.html#%s" % ("ByAlphaAuthor", self.generateAuthorAnchor(book['author'])) + aTag.insert(0, NavigableString(book['author'])) + emTag.insert(0,aTag) + pBookTag.insert(ptc, emTag) + ptc += 1 + + if divRunningTag is not None: + divRunningTag.insert(drtc, pBookTag) + drtc += 1 + + # Add the last divRunningTag to divTag + if divRunningTag is not None: + divTag.insert(dtc, divRunningTag) + dtc += 1 + + # Add the divTag to the body + body.insert(btc, divTag) + btc += 1 + + # Write the volume to contentdir + outfile_spec = "%s/ByAlphaTitle.html" % (self.contentDir) + outfile = open(outfile_spec, 'w') + outfile.write(soup.prettify()) + outfile.close() + self.htmlFileList_1.append("content/ByAlphaTitle.html") + + def generateHTMLByAuthor(self): + ''' + Write books by author A-Z + ''' + self.updateProgressFullStep("'Authors'") + + friendly_name = "Authors" + + soup = self.generateHTMLEmptyHeader(friendly_name) + body = soup.find('body') + + btc = 0 + + # Insert section tag + aTag = Tag(soup,'a') + aTag['name'] = 'section_start' + body.insert(btc, aTag) + btc += 1 + + # Insert the anchor + aTag = Tag(soup, "a") + anchor_name = friendly_name.lower() + aTag['name'] = anchor_name.replace(" ","") + body.insert(btc, aTag) + btc += 1 + + divTag = Tag(soup, "div") + dtc = 0 + divOpeningTag = None + dotc = 0 + divRunningTag = None + drtc = 0 + + # Loop through booksByAuthor + # Each author/books group goes in an openingTag div (first) or + # a runningTag div (subsequent) + book_count = 0 + current_author = '' + current_letter = '' + current_series = None + #for book in sorted(self.booksByAuthor, key = self.booksByAuthorSorter_author_sort): + for book in self.booksByAuthor: + + book_count += 1 + if self.letter_or_symbol(book['author_sort'][0].upper()) != current_letter : + # Start a new letter with Index letter + if divOpeningTag is not None: + divTag.insert(dtc, divOpeningTag) + dtc += 1 + dotc = 0 + if divRunningTag is not None: + divTag.insert(dtc, divRunningTag) + dtc += 1 + drtc = 0 + divRunningTag = None + + current_letter = self.letter_or_symbol(book['author_sort'][0].upper()) + author_count = 0 + divOpeningTag = Tag(soup, 'div') + if dtc > 0: + divOpeningTag['class'] = "initial_letter" + dotc = 0 + pIndexTag = Tag(soup, "p") + pIndexTag['class'] = "author_title_letter_index" + aTag = Tag(soup, "a") + aTag['name'] = "%sauthors" % self.letter_or_symbol(current_letter) + pIndexTag.insert(0,aTag) + pIndexTag.insert(1,NavigableString(self.letter_or_symbol(book['author_sort'][0].upper()))) + divOpeningTag.insert(dotc,pIndexTag) + dotc += 1 + + if book['author'] != current_author: + # Start a new author + current_author = book['author'] + author_count += 1 + if author_count >= 2: + # Add divOpeningTag to divTag, kill divOpeningTag + if divOpeningTag: + divTag.insert(dtc, divOpeningTag) + dtc += 1 + divOpeningTag = None + dotc = 0 + + # Create a divRunningTag for the next author + if author_count > 2: + divTag.insert(dtc, divRunningTag) + dtc += 1 + + divRunningTag = Tag(soup, 'div') + divRunningTag['class'] = "author_logical_group" + drtc = 0 + + non_series_books = 0 + current_series = None + pAuthorTag = Tag(soup, "p") + pAuthorTag['class'] = "author_index" + aTag = Tag(soup, "a") + aTag['name'] = "%s" % self.generateAuthorAnchor(current_author) + aTag.insert(0,NavigableString(current_author)) + pAuthorTag.insert(0,aTag) + if author_count == 1: + divOpeningTag.insert(dotc, pAuthorTag) + dotc += 1 + else: + divRunningTag.insert(drtc,pAuthorTag) + drtc += 1 + + # Check for series + if book['series'] and book['series'] != current_series: + # Start a new series + current_series = book['series'] + pSeriesTag = Tag(soup,'p') + pSeriesTag['class'] = "series" + + if self.opts.generate_series: + aTag = Tag(soup,'a') + aTag['href'] = "%s.html#%s_series" % ('BySeries', + re.sub('\W','',book['series']).lower()) + aTag.insert(0, book['series']) + #pSeriesTag.insert(0, NavigableString(self.NOT_READ_SYMBOL)) + pSeriesTag.insert(0, aTag) + else: + #pSeriesTag.insert(0,NavigableString(self.NOT_READ_SYMBOL + '%s' % book['series'])) + pSeriesTag.insert(0,NavigableString('%s' % book['series'])) + + if author_count == 1: + divOpeningTag.insert(dotc, pSeriesTag) + dotc += 1 + elif divRunningTag is not None: + divRunningTag.insert(drtc,pSeriesTag) + drtc += 1 + if current_series and not book['series']: + current_series = None + + # Add books + pBookTag = Tag(soup, "p") + ptc = 0 + + # book with read|reading|unread symbol or wishlist item + if self.opts.wishlist_tag in book.get('tags', []): + pBookTag['class'] = "wishlist_item" + pBookTag.insert(ptc,NavigableString(self.MISSING_SYMBOL)) + ptc += 1 + else: + if book['read']: + # check mark + pBookTag.insert(ptc,NavigableString(self.READ_SYMBOL)) + pBookTag['class'] = "read_book" + ptc += 1 + elif book['id'] in self.bookmarked_books: + pBookTag.insert(ptc,NavigableString(self.READING_SYMBOL)) + pBookTag['class'] = "read_book" + ptc += 1 + else: + # hidden check mark + pBookTag['class'] = "unread_book" + pBookTag.insert(ptc,NavigableString(self.NOT_READ_SYMBOL)) + ptc += 1 + + aTag = Tag(soup, "a") + if self.opts.generate_descriptions: + aTag['href'] = "book_%d.html" % (int(float(book['id']))) + + # Generate the title from the template + args = self.generateFormatArgs(book) + if current_series: + #aTag.insert(0,'%s%s' % (escape(book['title'][len(book['series'])+1:]),pubyear)) + formatted_title = self.by_authors_series_title_template.format(**args).rstrip() + else: + #aTag.insert(0,'%s%s' % (escape(book['title']), pubyear)) + formatted_title = self.by_authors_normal_title_template.format(**args).rstrip() + non_series_books += 1 + aTag.insert(0,NavigableString(escape(formatted_title))) + + pBookTag.insert(ptc, aTag) + ptc += 1 + + if author_count == 1: + divOpeningTag.insert(dotc, pBookTag) + dotc += 1 + elif divRunningTag: + divRunningTag.insert(drtc,pBookTag) + drtc += 1 + + # Loop ends here + + if not self.__generateForKindle: + # Insert the <h2> tag with book_count at the head + #<h2><a name="byalphaauthor" id="byalphaauthor"></a>By Author</h2> + pTag = Tag(soup, "p") + pTag['class'] = 'title' + aTag = Tag(soup, "a") + anchor_name = friendly_name.lower() + aTag['name'] = anchor_name.replace(" ","") + pTag.insert(0,aTag) + #h2Tag.insert(1,NavigableString('%s (%d)' % (friendly_name, book_count))) + pTag.insert(1,NavigableString('%s' % (friendly_name))) + body.insert(btc,pTag) + btc += 1 + + if author_count == 1: + divTag.insert(dtc, divOpeningTag) + dtc += 1 + elif divRunningTag is not None: + divTag.insert(dtc, divRunningTag) + dtc += 1 + + # Add the divTag to the body + body.insert(btc, divTag) + + # Write the generated file to contentdir + outfile_spec = "%s/ByAlphaAuthor.html" % (self.contentDir) + outfile = open(outfile_spec, 'w') + outfile.write(soup.prettify()) + outfile.close() + self.htmlFileList_1.append("content/ByAlphaAuthor.html") + + def generateHTMLByDateAdded(self): + ''' + Write books by reverse chronological order + ''' + self.updateProgressFullStep("'Recently Added'") + + def add_books_to_HTML_by_month(this_months_list, dtc): + if len(this_months_list): + + #this_months_list = sorted(this_months_list, key=self.booksByAuthorSorter_author_sort) + + # Create a new month anchor + date_string = strftime(u'%B %Y', current_date.timetuple()) + pIndexTag = Tag(soup, "p") + pIndexTag['class'] = "date_index" + aTag = Tag(soup, "a") + aTag['name'] = "bda_%s-%s" % (current_date.year, current_date.month) + pIndexTag.insert(0,aTag) + pIndexTag.insert(1,NavigableString(date_string)) + divTag.insert(dtc,pIndexTag) + dtc += 1 + current_author = None + current_series = None + + for new_entry in this_months_list: + if new_entry['author'] != current_author: + # Start a new author + current_author = new_entry['author'] + non_series_books = 0 + current_series = None + pAuthorTag = Tag(soup, "p") + pAuthorTag['class'] = "author_index" + aTag = Tag(soup, "a") + if self.opts.generate_authors: + aTag['name'] = "%s" % self.generateAuthorAnchor(current_author) + aTag.insert(0,NavigableString(current_author)) + pAuthorTag.insert(0,aTag) + divTag.insert(dtc,pAuthorTag) + dtc += 1 + + # Check for series + if new_entry['series'] and new_entry['series'] != current_series: + # Start a new series + current_series = new_entry['series'] + pSeriesTag = Tag(soup,'p') + pSeriesTag['class'] = "series" + if self.opts.generate_series: + aTag = Tag(soup,'a') + aTag['href'] = "%s.html#%s_series" % ('BySeries', + re.sub('\W','',new_entry['series']).lower()) + aTag.insert(0, new_entry['series']) + pSeriesTag.insert(0, aTag) + else: + pSeriesTag.insert(0,NavigableString('%s' % new_entry['series'])) + divTag.insert(dtc,pSeriesTag) + dtc += 1 + if current_series and not new_entry['series']: + current_series = None + + # Add books + pBookTag = Tag(soup, "p") + ptc = 0 + + # book with read|reading|unread symbol or wishlist item + if self.opts.wishlist_tag in new_entry.get('tags', []): + pBookTag['class'] = "wishlist_item" + pBookTag.insert(ptc,NavigableString(self.MISSING_SYMBOL)) + ptc += 1 + else: + if new_entry['read']: + # check mark + pBookTag.insert(ptc,NavigableString(self.READ_SYMBOL)) + pBookTag['class'] = "read_book" + ptc += 1 + elif new_entry['id'] in self.bookmarked_books: + pBookTag.insert(ptc,NavigableString(self.READING_SYMBOL)) + pBookTag['class'] = "read_book" + ptc += 1 + else: + # hidden check mark + pBookTag['class'] = "unread_book" + pBookTag.insert(ptc,NavigableString(self.NOT_READ_SYMBOL)) + ptc += 1 + + aTag = Tag(soup, "a") + if self.opts.generate_descriptions: + aTag['href'] = "book_%d.html" % (int(float(new_entry['id']))) + + # Generate the title from the template + args = self.generateFormatArgs(new_entry) + if current_series: + formatted_title = self.by_month_added_series_title_template.format(**args).rstrip() + else: + formatted_title = self.by_month_added_normal_title_template.format(**args).rstrip() + non_series_books += 1 + aTag.insert(0,NavigableString(escape(formatted_title))) + pBookTag.insert(ptc, aTag) + ptc += 1 + + divTag.insert(dtc, pBookTag) + dtc += 1 + return dtc + + def add_books_to_HTML_by_date_range(date_range_list, date_range, dtc): + if len(date_range_list): + pIndexTag = Tag(soup, "p") + pIndexTag['class'] = "date_index" + aTag = Tag(soup, "a") + aTag['name'] = "bda_%s" % date_range.replace(' ','') + pIndexTag.insert(0,aTag) + pIndexTag.insert(1,NavigableString(date_range)) + divTag.insert(dtc,pIndexTag) + dtc += 1 + + for new_entry in date_range_list: + # Add books + pBookTag = Tag(soup, "p") + ptc = 0 + + # book with read|reading|unread symbol or wishlist item + if self.opts.wishlist_tag in new_entry.get('tags', []): + pBookTag['class'] = "wishlist_item" + pBookTag.insert(ptc,NavigableString(self.MISSING_SYMBOL)) + ptc += 1 + else: + if new_entry['read']: + # check mark + pBookTag.insert(ptc,NavigableString(self.READ_SYMBOL)) + pBookTag['class'] = "read_book" + ptc += 1 + elif new_entry['id'] in self.bookmarked_books: + pBookTag.insert(ptc,NavigableString(self.READING_SYMBOL)) + pBookTag['class'] = "read_book" + ptc += 1 + else: + # hidden check mark + pBookTag['class'] = "unread_book" + pBookTag.insert(ptc,NavigableString(self.NOT_READ_SYMBOL)) + ptc += 1 + + aTag = Tag(soup, "a") + if self.opts.generate_descriptions: + aTag['href'] = "book_%d.html" % (int(float(new_entry['id']))) + + # Generate the title from the template + args = self.generateFormatArgs(new_entry) + if new_entry['series']: + formatted_title = self.by_recently_added_series_title_template.format(**args).rstrip() + else: + formatted_title = self.by_recently_added_normal_title_template.format(**args).rstrip() + aTag.insert(0,NavigableString(escape(formatted_title))) + pBookTag.insert(ptc, aTag) + ptc += 1 + + # Dot + pBookTag.insert(ptc, NavigableString(" · ")) + ptc += 1 + + # Link to author + emTag = Tag(soup, "em") + aTag = Tag(soup, "a") + if self.opts.generate_authors: + aTag['href'] = "%s.html#%s" % ("ByAlphaAuthor", self.generateAuthorAnchor(new_entry['author'])) + aTag.insert(0, NavigableString(new_entry['author'])) + emTag.insert(0,aTag) + pBookTag.insert(ptc, emTag) + ptc += 1 + + divTag.insert(dtc, pBookTag) + dtc += 1 + return dtc + + friendly_name = "Recently Added" + + soup = self.generateHTMLEmptyHeader(friendly_name) + body = soup.find('body') + + btc = 0 + + # Insert section tag + aTag = Tag(soup,'a') + aTag['name'] = 'section_start' + body.insert(btc, aTag) + btc += 1 + + # Insert the anchor + aTag = Tag(soup, "a") + anchor_name = friendly_name.lower() + aTag['name'] = anchor_name.replace(" ","") + body.insert(btc, aTag) + btc += 1 + + if not self.__generateForKindle: + #<h2><a name="byalphaauthor" id="byalphaauthor"></a>By Author</h2> + pTag = Tag(soup, "p") + pTag['class'] = 'title' + aTag = Tag(soup, "a") + anchor_name = friendly_name.lower() + aTag['name'] = anchor_name.replace(" ","") + pTag.insert(0,aTag) + pTag.insert(1,NavigableString('%s' % friendly_name)) + body.insert(btc,pTag) + btc += 1 + + divTag = Tag(soup, "div") + dtc = 0 + + # >>> Books by date range <<< + if self.useSeriesPrefixInTitlesSection: + self.booksByDateRange = sorted(self.booksByTitle, + key=lambda x:(x['timestamp'], x['timestamp']),reverse=True) + else: + nspt = deepcopy(self.booksByTitle) + self.booksByDateRange = sorted(nspt, key=lambda x:(x['timestamp'], x['timestamp']),reverse=True) + + date_range_list = [] + today_time = nowf().replace(hour=23, minute=59, second=59) + for (i, date) in enumerate(self.DATE_RANGE): + date_range_limit = self.DATE_RANGE[i] + if i: + date_range = '%d to %d days ago' % (self.DATE_RANGE[i-1], self.DATE_RANGE[i]) + else: + date_range = 'Last %d days' % (self.DATE_RANGE[i]) + + for book in self.booksByDateRange: + book_time = book['timestamp'] + delta = today_time-book_time + if delta.days <= date_range_limit: + date_range_list.append(book) + else: + break + + dtc = add_books_to_HTML_by_date_range(date_range_list, date_range, dtc) + date_range_list = [book] + + # >>>> Books by month <<<< + # Sort titles case-insensitive for by month using series prefix + self.booksByMonth = sorted(self.booksByTitle, + key=lambda x:(x['timestamp'], x['timestamp']),reverse=True) + + # Loop through books by date + current_date = datetime.date.fromordinal(1) + this_months_list = [] + for book in self.booksByMonth: + if book['timestamp'].month != current_date.month or \ + book['timestamp'].year != current_date.year: + dtc = add_books_to_HTML_by_month(this_months_list, dtc) + this_months_list = [] + current_date = book['timestamp'].date() + this_months_list.append(book) + + # Add the last month's list + add_books_to_HTML_by_month(this_months_list, dtc) + + # Add the divTag to the body + body.insert(btc, divTag) + + # Write the generated file to contentdir + outfile_spec = "%s/ByDateAdded.html" % (self.contentDir) + outfile = open(outfile_spec, 'w') + outfile.write(soup.prettify()) + outfile.close() + self.htmlFileList_2.append("content/ByDateAdded.html") + + def generateHTMLByDateRead(self): + ''' + Write books by active bookmarks + ''' + friendly_name = 'Recently Read' + self.updateProgressFullStep("'%s'" % friendly_name) + if not self.bookmarked_books: + return + + def add_books_to_HTML_by_day(todays_list, dtc): + if len(todays_list): + # Create a new day anchor + date_string = strftime(u'%A, %B %d', current_date.timetuple()) + pIndexTag = Tag(soup, "p") + pIndexTag['class'] = "date_index" + aTag = Tag(soup, "a") + aTag['name'] = "bdr_%s-%s-%s" % (current_date.year, current_date.month, current_date.day) + pIndexTag.insert(0,aTag) + pIndexTag.insert(1,NavigableString(date_string)) + divTag.insert(dtc,pIndexTag) + dtc += 1 + + for new_entry in todays_list: + pBookTag = Tag(soup, "p") + pBookTag['class'] = "date_read" + ptc = 0 + + # Percent read + pBookTag.insert(ptc, NavigableString(new_entry['reading_progress'])) + ptc += 1 + + aTag = Tag(soup, "a") + if self.opts.generate_descriptions: + aTag['href'] = "book_%d.html" % (int(float(new_entry['id']))) + aTag.insert(0,escape(new_entry['title'])) + pBookTag.insert(ptc, aTag) + ptc += 1 + + # Dot + pBookTag.insert(ptc, NavigableString(" · ")) + ptc += 1 + + # Link to author + emTag = Tag(soup, "em") + aTag = Tag(soup, "a") + if self.opts.generate_authors: + aTag['href'] = "%s.html#%s" % ("ByAlphaAuthor", self.generateAuthorAnchor(new_entry['author'])) + aTag.insert(0, NavigableString(new_entry['author'])) + emTag.insert(0,aTag) + pBookTag.insert(ptc, emTag) + ptc += 1 + + divTag.insert(dtc, pBookTag) + dtc += 1 + return dtc + + def add_books_to_HTML_by_date_range(date_range_list, date_range, dtc): + if len(date_range_list): + pIndexTag = Tag(soup, "p") + pIndexTag['class'] = "date_index" + aTag = Tag(soup, "a") + aTag['name'] = "bdr_%s" % date_range.replace(' ','') + pIndexTag.insert(0,aTag) + pIndexTag.insert(1,NavigableString(date_range)) + divTag.insert(dtc,pIndexTag) + dtc += 1 + + for new_entry in date_range_list: + # Add books + pBookTag = Tag(soup, "p") + pBookTag['class'] = "date_read" + ptc = 0 + + # Percent read + dots = int((new_entry['percent_read'] + 5)/10) + dot_string = self.READ_PROGRESS_SYMBOL * dots + empty_dots = self.UNREAD_PROGRESS_SYMBOL * (10 - dots) + pBookTag.insert(ptc, NavigableString('%s%s' % (dot_string,empty_dots))) + ptc += 1 + + aTag = Tag(soup, "a") + if self.opts.generate_descriptions: + aTag['href'] = "book_%d.html" % (int(float(new_entry['id']))) + aTag.insert(0,escape(new_entry['title'])) + pBookTag.insert(ptc, aTag) + ptc += 1 + + # Dot + pBookTag.insert(ptc, NavigableString(" · ")) + ptc += 1 + + # Link to author + emTag = Tag(soup, "em") + aTag = Tag(soup, "a") + if self.opts.generate_authors: + aTag['href'] = "%s.html#%s" % ("ByAlphaAuthor", self.generateAuthorAnchor(new_entry['author'])) + aTag.insert(0, NavigableString(new_entry['author'])) + emTag.insert(0,aTag) + pBookTag.insert(ptc, emTag) + ptc += 1 + + divTag.insert(dtc, pBookTag) + dtc += 1 + return dtc + + soup = self.generateHTMLEmptyHeader(friendly_name) + body = soup.find('body') + + btc = 0 + + # Insert section tag + aTag = Tag(soup,'a') + aTag['name'] = 'section_start' + body.insert(btc, aTag) + btc += 1 + + # Insert the anchor + aTag = Tag(soup, "a") + anchor_name = friendly_name.lower() + aTag['name'] = anchor_name.replace(" ","") + body.insert(btc, aTag) + btc += 1 + + divTag = Tag(soup, "div") + dtc = 0 + + # self.bookmarked_books: (Bookmark, book) + bookmarked_books = [] + for bm_book in self.bookmarked_books: + book = self.bookmarked_books[bm_book] + #print "bm_book: %s" % bm_book + book[1]['bookmark_timestamp'] = book[0].timestamp + try: + book[1]['percent_read'] = min(float(100*book[0].last_read / book[0].book_length),100) + except: + book[1]['percent_read'] = 0 + bookmarked_books.append(book[1]) + + self.booksByDateRead = sorted(bookmarked_books, + key=lambda x:(x['bookmark_timestamp'], x['bookmark_timestamp']),reverse=True) + + # >>>> Recently read by day <<<< + current_date = datetime.date.fromordinal(1) + todays_list = [] + for book in self.booksByDateRead: + bookmark_time = datetime.datetime.utcfromtimestamp(book['bookmark_timestamp']) + if bookmark_time.day != current_date.day or \ + bookmark_time.month != current_date.month or \ + bookmark_time.year != current_date.year: + dtc = add_books_to_HTML_by_day(todays_list, dtc) + todays_list = [] + current_date = datetime.datetime.utcfromtimestamp(book['bookmark_timestamp']).date() + todays_list.append(book) + + # Add the last day's list + add_books_to_HTML_by_day(todays_list, dtc) + + # Add the divTag to the body + body.insert(btc, divTag) + + # Write the generated file to contentdir + outfile_spec = "%s/ByDateRead.html" % (self.contentDir) + outfile = open(outfile_spec, 'w') + outfile.write(soup.prettify()) + outfile.close() + self.htmlFileList_2.append("content/ByDateRead.html") + + def generateHTMLBySeries(self): + ''' + Generate a list of series + ''' + self.updateProgressFullStep("Fetching series") + + self.opts.sort_by = 'series' + + # Merge opts.exclude_tags with opts.search_text + # Updated to use exact match syntax + empty_exclude_tags = False if len(self.opts.exclude_tags) else True + search_phrase = 'series:true ' + if not empty_exclude_tags: + exclude_tags = self.opts.exclude_tags.split(',') + search_terms = [] + for tag in exclude_tags: + search_terms.append("tag:=%s" % tag) + search_phrase += "not (%s)" % " or ".join(search_terms) + + # If a list of ids are provided, don't use search_text + if self.opts.ids: + self.opts.search_text = search_phrase + else: + if self.opts.search_text: + self.opts.search_text += " " + search_phrase + else: + self.opts.search_text = search_phrase + + # Fetch the database as a dictionary + data = self.plugin.search_sort_db(self.db, self.opts) + self.booksBySeries = self.processExclusions(data) + + if not self.booksBySeries: + self.opts.generate_series = False + self.opts.log(" no series found in selected books, cancelling series generation") + return + + friendly_name = "Series" + + soup = self.generateHTMLEmptyHeader(friendly_name) + body = soup.find('body') + + btc = 0 + + # Insert section tag + aTag = Tag(soup,'a') + aTag['name'] = 'section_start' + body.insert(btc, aTag) + btc += 1 + + # Insert the anchor + aTag = Tag(soup, "a") + anchor_name = friendly_name.lower() + aTag['name'] = anchor_name.replace(" ","") + body.insert(btc, aTag) + btc += 1 + + divTag = Tag(soup, "div") + dtc = 0 + current_letter = "" + current_series = None + + # Loop through booksBySeries + series_count = 0 + for book in self.booksBySeries: + # Check for initial letter change + sort_title = self.generateSortTitle(book['series']) + if self.letter_or_symbol(sort_title[0].upper()) != current_letter : + # Start a new letter with Index letter + current_letter = self.letter_or_symbol(sort_title[0].upper()) + pIndexTag = Tag(soup, "p") + pIndexTag['class'] = "series_letter_index" + aTag = Tag(soup, "a") + aTag['name'] = "%s_series" % self.letter_or_symbol(current_letter) + pIndexTag.insert(0,aTag) + pIndexTag.insert(1,NavigableString(self.letter_or_symbol(sort_title[0].upper()))) + divTag.insert(dtc,pIndexTag) + dtc += 1 + + # Check for series change + if book['series'] != current_series: + # Start a new series + series_count += 1 + current_series = book['series'] + pSeriesTag = Tag(soup,'p') + pSeriesTag['class'] = "series" + aTag = Tag(soup, 'a') + aTag['name'] = "%s_series" % re.sub('\W','',book['series']).lower() + pSeriesTag.insert(0,aTag) + pSeriesTag.insert(1,NavigableString('%s' % book['series'])) + divTag.insert(dtc,pSeriesTag) + dtc += 1 + + # Add books + pBookTag = Tag(soup, "p") + ptc = 0 + + book['read'] = self.discoverReadStatus(book) + + # book with read|reading|unread symbol or wishlist item + if self.opts.wishlist_tag in book.get('tags', []): + pBookTag['class'] = "wishlist_item" + pBookTag.insert(ptc,NavigableString(self.MISSING_SYMBOL)) + ptc += 1 + else: + if book.get('read', False): + # check mark + pBookTag.insert(ptc,NavigableString(self.READ_SYMBOL)) + pBookTag['class'] = "read_book" + ptc += 1 + elif book['id'] in self.bookmarked_books: + pBookTag.insert(ptc,NavigableString(self.READING_SYMBOL)) + pBookTag['class'] = "read_book" + ptc += 1 + else: + # hidden check mark + pBookTag['class'] = "unread_book" + pBookTag.insert(ptc,NavigableString(self.NOT_READ_SYMBOL)) + ptc += 1 + + aTag = Tag(soup, "a") + if self.opts.generate_descriptions: + aTag['href'] = "book_%d.html" % (int(float(book['id']))) + # Use series, series index if avail else just title + #aTag.insert(0,'%d. %s · %s' % (book['series_index'],escape(book['title']), ' & '.join(book['authors']))) + + if is_date_undefined(book['pubdate']): + book['date'] = None + else: + book['date'] = strftime(u'%B %Y', book['pubdate'].timetuple()) + + args = self.generateFormatArgs(book) + formatted_title = self.by_series_title_template.format(**args).rstrip() + aTag.insert(0,NavigableString(escape(formatted_title))) + pBookTag.insert(ptc, aTag) + ptc += 1 + + # · + pBookTag.insert(ptc, NavigableString(' · ')) + ptc += 1 + + # Link to author + aTag = Tag(soup, "a") + if self.opts.generate_authors: + aTag['href'] = "%s.html#%s" % ("ByAlphaAuthor", + self.generateAuthorAnchor(escape(' & '.join(book['authors'])))) + aTag.insert(0, NavigableString(' & '.join(book['authors']))) + pBookTag.insert(ptc, aTag) + ptc += 1 + + divTag.insert(dtc, pBookTag) + dtc += 1 + + if not self.__generateForKindle: + # Insert the <h2> tag with book_count at the head + #<h2><a name="byseries" id="byseries"></a>By Series</h2> + pTag = Tag(soup, "p") + pTag['class'] = 'title' + aTag = Tag(soup, "a") + anchor_name = friendly_name.lower() + aTag['name'] = anchor_name.replace(" ","") + pTag.insert(0,aTag) + #h2Tag.insert(1,NavigableString('%s (%d)' % (friendly_name, series_count))) + pTag.insert(1,NavigableString('%s' % friendly_name)) + body.insert(btc,pTag) + btc += 1 + + # Add the divTag to the body + body.insert(btc, divTag) + + # Write the generated file to contentdir + outfile_spec = "%s/BySeries.html" % (self.contentDir) + outfile = open(outfile_spec, 'w') + outfile.write(soup.prettify()) + outfile.close() + self.htmlFileList_1.append("content/BySeries.html") + + def generateHTMLByTags(self): + ''' + Generate individual HTML files for each tag, e.g. Fiction, Nonfiction ... + Note that special tags - have already been filtered from books[] + There may be synonomous tags + ''' + self.updateProgressFullStep("'Genres'") + + self.genre_tags_dict = self.filterDbTags(self.db.all_tags()) + + # Extract books matching filtered_tags + genre_list = [] + for friendly_tag in sorted(self.genre_tags_dict): + #print "\ngenerateHTMLByTags(): looking for books with friendly_tag '%s'" % friendly_tag + # tag_list => { normalized_genre_tag : [{book},{},{}], + # normalized_genre_tag : [{book},{},{}] } + + tag_list = {} + for book in self.booksByAuthor: + # Scan each book for tag matching friendly_tag + if 'tags' in book and friendly_tag in book['tags']: + this_book = {} + this_book['author'] = book['author'] + this_book['title'] = book['title'] + this_book['author_sort'] = capitalize(book['author_sort']) + this_book['read'] = book['read'] + this_book['tags'] = book['tags'] + this_book['id'] = book['id'] + this_book['series'] = book['series'] + this_book['series_index'] = book['series_index'] + this_book['date'] = book['date'] + normalized_tag = self.genre_tags_dict[friendly_tag] + genre_tag_list = [key for genre in genre_list for key in genre] + if normalized_tag in genre_tag_list: + for existing_genre in genre_list: + for key in existing_genre: + new_book = None + if key == normalized_tag: + for book in existing_genre[key]: + if book['title'] == this_book['title']: + new_book = False + break + else: + new_book = True + if new_book: + existing_genre[key].append(this_book) + else: + tag_list[normalized_tag] = [this_book] + genre_list.append(tag_list) + + if self.opts.verbose: + if len(genre_list): + self.opts.log.info(" Genre summary: %d active genre tags used in generating catalog with %d titles" % + (len(genre_list), len(self.booksByTitle))) + + for genre in genre_list: + for key in genre: + self.opts.log.info(" %s: %d %s" % (self.getFriendlyGenreTag(key), + len(genre[key]), + 'titles' if len(genre[key]) > 1 else 'title')) + + + # Write the results + # genre_list = [ {friendly_tag:[{book},{book}]}, {friendly_tag:[{book},{book}]}, ...] + master_genre_list = [] + for genre_tag_set in genre_list: + for (index, genre) in enumerate(genre_tag_set): + #print "genre: %s \t genre_tag_set[genre]: %s" % (genre, genre_tag_set[genre]) + + # Create sorted_authors[0] = friendly, [1] = author_sort for NCX creation + authors = [] + for book in genre_tag_set[genre]: + authors.append((book['author'],book['author_sort'])) + + # authors[] contains a list of all book authors, with multiple entries for multiple books by author + # Create unique_authors with a count of books per author as the third tuple element + books_by_current_author = 1 + current_author = authors[0] + unique_authors = [] + for (i,author) in enumerate(authors): + if author != current_author and i: + unique_authors.append((current_author[0], current_author[1], books_by_current_author)) + current_author = author + books_by_current_author = 1 + elif i==0 and len(authors) == 1: + # Allow for single-book lists + unique_authors.append((current_author[0], current_author[1], books_by_current_author)) + else: + books_by_current_author += 1 + + # Write the genre book list as an article + titles_spanned = self.generateHTMLByGenre(genre, True if index==0 else False, + genre_tag_set[genre], + "%s/Genre_%s.html" % (self.contentDir, + genre)) + + tag_file = "content/Genre_%s.html" % genre + master_genre_list.append({'tag':genre, + 'file':tag_file, + 'authors':unique_authors, + 'books':genre_tag_set[genre], + 'titles_spanned':titles_spanned}) + + self.genres = master_genre_list + + def generateThumbnails(self): + ''' + Generate a thumbnail per cover. If a current thumbnail exists, skip + If a cover doesn't exist, use default + Return list of active thumbs + ''' + self.updateProgressFullStep("'Thumbnails'") + thumbs = ['thumbnail_default.jpg'] + image_dir = "%s/images" % self.catalogPath + for (i,title) in enumerate(self.booksByTitle): + # Update status + self.updateProgressMicroStep("Thumbnail %d of %d" % \ + (i,len(self.booksByTitle)), + i/float(len(self.booksByTitle))) + + thumb_file = 'thumbnail_%d.jpg' % int(title['id']) + thumb_generated = True + valid_cover = True + try: + self.generateThumbnail(title, image_dir, thumb_file) + thumbs.append("thumbnail_%d.jpg" % int(title['id'])) + except: + if 'cover' in title and os.path.exists(title['cover']): + valid_cover = False + self.opts.log.warn(" *** Invalid cover file for '%s'***" % + (title['title'])) + if not self.error: + self.error.append('Invalid cover files') + self.error.append("Warning: invalid cover file for '%s', default cover substituted.\n" % (title['title'])) + + thumb_generated = False + + if not thumb_generated: + self.opts.log.warn(" using default cover for '%s' (%d)" % (title['title'], title['id'])) + # Confirm thumb exists, default is current + default_thumb_fp = os.path.join(image_dir,"thumbnail_default.jpg") + cover = os.path.join(self.catalogPath, "DefaultCover.png") + title['cover'] = cover + + if not os.path.exists(cover): + shutil.copyfile(I('book.png'), cover) + + if os.path.isfile(default_thumb_fp): + # Check to see if default cover is newer than thumbnail + # os.path.getmtime() = modified time + # os.path.ctime() = creation time + cover_timestamp = os.path.getmtime(cover) + thumb_timestamp = os.path.getmtime(default_thumb_fp) + if thumb_timestamp < cover_timestamp: + if False and self.verbose: + self.opts.log.warn("updating thumbnail_default for %s" % title['title']) + self.generateThumbnail(title, image_dir, + "thumbnail_default.jpg" if valid_cover else thumb_file) + else: + if False and self.verbose: + self.opts.log.warn(" generating new thumbnail_default.jpg") + self.generateThumbnail(title, image_dir, + "thumbnail_default.jpg" if valid_cover else thumb_file) + # Clear the book's cover property + title['cover'] = None + + + # Write thumb_width to the file, validating cache contents + # Allows detection of aborted catalog builds + with ZipFile(self.__archive_path, mode='a') as zfw: + zfw.writestr('thumb_width', self.opts.thumb_width) + + self.thumbs = thumbs + + def generateOPF(self): + + self.updateProgressFullStep("Generating OPF") + + header = ''' + <?xml version="1.0" encoding="UTF-8"?> + <package xmlns="http://www.idpf.org/2007/opf" version="2.0" unique-identifier="calibre_id"> + <metadata xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:opf="http://www.idpf.org/2007/opf" xmlns:calibre="http://calibre.kovidgoyal.net/2009/metadata" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"> + <dc:language>en-US</dc:language> + <meta name="calibre:publication_type" content="periodical:default"/> + </metadata> + <manifest></manifest> + <spine toc="ncx"></spine> + <guide></guide> + </package> + ''' + # Add the supplied metadata tags + soup = BeautifulStoneSoup(header, selfClosingTags=['item','itemref', 'reference']) + metadata = soup.find('metadata') + mtc = 0 + + titleTag = Tag(soup, "dc:title") + titleTag.insert(0,self.title) + metadata.insert(mtc, titleTag) + mtc += 1 + + creatorTag = Tag(soup, "dc:creator") + creatorTag.insert(0, self.creator) + metadata.insert(mtc, creatorTag) + mtc += 1 + + # Create the OPF tags + manifest = soup.find('manifest') + mtc = 0 + spine = soup.find('spine') + stc = 0 + guide = soup.find('guide') + + itemTag = Tag(soup, "item") + itemTag['id'] = "ncx" + itemTag['href'] = '%s.ncx' % self.basename + itemTag['media-type'] = "application/x-dtbncx+xml" + manifest.insert(mtc, itemTag) + mtc += 1 + + itemTag = Tag(soup, "item") + itemTag['id'] = 'stylesheet' + itemTag['href'] = self.stylesheet + itemTag['media-type'] = 'text/css' + manifest.insert(mtc, itemTag) + mtc += 1 + + itemTag = Tag(soup, "item") + itemTag['id'] = 'mastheadimage-image' + itemTag['href'] = "images/mastheadImage.gif" + itemTag['media-type'] = 'image/gif' + manifest.insert(mtc, itemTag) + mtc += 1 + + # Write the thumbnail images, descriptions to the manifest + sort_descriptions_by = [] + if self.opts.generate_descriptions: + for thumb in self.thumbs: + itemTag = Tag(soup, "item") + itemTag['href'] = "images/%s" % (thumb) + end = thumb.find('.jpg') + itemTag['id'] = "%s-image" % thumb[:end] + itemTag['media-type'] = 'image/jpeg' + manifest.insert(mtc, itemTag) + mtc += 1 + + # HTML files - add descriptions to manifest and spine + sort_descriptions_by = self.booksByAuthor if self.opts.sort_descriptions_by_author \ + else self.booksByTitle + # Add html_files to manifest and spine + + for file in self.htmlFileList_1: + # By Author, By Title, By Series, + itemTag = Tag(soup, "item") + start = file.find('/') + 1 + end = file.find('.') + itemTag['href'] = file + itemTag['id'] = file[start:end].lower() + itemTag['media-type'] = "application/xhtml+xml" + manifest.insert(mtc, itemTag) + mtc += 1 + + # spine + itemrefTag = Tag(soup, "itemref") + itemrefTag['idref'] = file[start:end].lower() + spine.insert(stc, itemrefTag) + stc += 1 + + # Add genre files to manifest and spine + for genre in self.genres: + if False: self.opts.log.info("adding %s to manifest and spine" % genre['tag']) + itemTag = Tag(soup, "item") + start = genre['file'].find('/') + 1 + end = genre['file'].find('.') + itemTag['href'] = genre['file'] + itemTag['id'] = genre['file'][start:end].lower() + itemTag['media-type'] = "application/xhtml+xml" + manifest.insert(mtc, itemTag) + mtc += 1 + + # spine + itemrefTag = Tag(soup, "itemref") + itemrefTag['idref'] = genre['file'][start:end].lower() + spine.insert(stc, itemrefTag) + stc += 1 + + for file in self.htmlFileList_2: + # By Date Added, By Date Read + itemTag = Tag(soup, "item") + start = file.find('/') + 1 + end = file.find('.') + itemTag['href'] = file + itemTag['id'] = file[start:end].lower() + itemTag['media-type'] = "application/xhtml+xml" + manifest.insert(mtc, itemTag) + mtc += 1 + + # spine + itemrefTag = Tag(soup, "itemref") + itemrefTag['idref'] = file[start:end].lower() + spine.insert(stc, itemrefTag) + stc += 1 + + for book in sort_descriptions_by: + # manifest + itemTag = Tag(soup, "item") + itemTag['href'] = "content/book_%d.html" % int(book['id']) + itemTag['id'] = "book%d" % int(book['id']) + itemTag['media-type'] = "application/xhtml+xml" + manifest.insert(mtc, itemTag) + mtc += 1 + + # spine + itemrefTag = Tag(soup, "itemref") + itemrefTag['idref'] = "book%d" % int(book['id']) + spine.insert(stc, itemrefTag) + stc += 1 + + # Guide + referenceTag = Tag(soup, "reference") + referenceTag['type'] = 'masthead' + referenceTag['title'] = 'mastheadimage-image' + referenceTag['href'] = 'images/mastheadImage.gif' + guide.insert(0,referenceTag) + + # Write the OPF file + outfile = open("%s/%s.opf" % (self.catalogPath, self.basename), 'w') + outfile.write(soup.prettify()) + + def generateNCXHeader(self): + + self.updateProgressFullStep("NCX header") + + header = ''' + <?xml version="1.0" encoding="utf-8"?> + <ncx xmlns="http://www.daisy.org/z3986/2005/ncx/" xmlns:calibre="http://calibre.kovidgoyal.net/2009/metadata" version="2005-1" xml:lang="en"> + </ncx> + ''' + soup = BeautifulStoneSoup(header, selfClosingTags=['content','calibre:meta-img']) + + ncx = soup.find('ncx') + navMapTag = Tag(soup, 'navMap') + navPointTag = Tag(soup, 'navPoint') + navPointTag['class'] = "periodical" + navPointTag['id'] = "title" + navPointTag['playOrder'] = self.playOrder + self.playOrder += 1 + navLabelTag = Tag(soup, 'navLabel') + textTag = Tag(soup, 'text') + textTag.insert(0, NavigableString(self.title)) + navLabelTag.insert(0, textTag) + navPointTag.insert(0, navLabelTag) + + if self.opts.generate_authors: + contentTag = Tag(soup, 'content') + contentTag['src'] = "content/ByAlphaAuthor.html" + navPointTag.insert(1, contentTag) + elif self.opts.generate_titles: + contentTag = Tag(soup, 'content') + contentTag['src'] = "content/ByAlphaTitle.html" + navPointTag.insert(1, contentTag) + elif self.opts.generate_series: + contentTag = Tag(soup, 'content') + contentTag['src'] = "content/BySeries.html" + navPointTag.insert(1, contentTag) + elif self.opts.generate_genres: + contentTag = Tag(soup, 'content') + #contentTag['src'] = "content/ByGenres.html" + contentTag['src'] = "%s" % self.genres[0]['file'] + navPointTag.insert(1, contentTag) + elif self.opts.generate_recently_added: + contentTag = Tag(soup, 'content') + contentTag['src'] = "content/ByDateAdded.html" + navPointTag.insert(1, contentTag) + else: + # Descriptions only + sort_descriptions_by = self.booksByAuthor if self.opts.sort_descriptions_by_author \ + else self.booksByTitle + contentTag = Tag(soup, 'content') + contentTag['src'] = "content/book_%d.html" % int(sort_descriptions_by[0]['id']) + navPointTag.insert(1, contentTag) + + cmiTag = Tag(soup, '%s' % 'calibre:meta-img') + cmiTag['name'] = "mastheadImage" + cmiTag['src'] = "images/mastheadImage.gif" + navPointTag.insert(2,cmiTag) + navMapTag.insert(0,navPointTag) + + ncx.insert(0,navMapTag) + self.ncxSoup = soup + + def generateNCXDescriptions(self, tocTitle): + + self.updateProgressFullStep("NCX 'Descriptions'") + + # --- Construct the 'Books by Title' section --- + ncx_soup = self.ncxSoup + body = ncx_soup.find("navPoint") + btc = len(body.contents) + + # Add the section navPoint + navPointTag = Tag(ncx_soup, 'navPoint') + navPointTag['class'] = "section" + navPointTag['id'] = "bytitle-ID" + navPointTag['playOrder'] = self.playOrder + self.playOrder += 1 + navLabelTag = Tag(ncx_soup, 'navLabel') + textTag = Tag(ncx_soup, 'text') + textTag.insert(0, NavigableString(tocTitle)) + navLabelTag.insert(0, textTag) + nptc = 0 + navPointTag.insert(nptc, navLabelTag) + nptc += 1 + contentTag = Tag(ncx_soup,"content") + contentTag['src'] = "content/book_%d.html" % int(self.booksByTitle[0]['id']) + navPointTag.insert(nptc, contentTag) + nptc += 1 + + # Loop over the titles + sort_descriptions_by = self.booksByAuthor if self.opts.sort_descriptions_by_author \ + else self.booksByTitle + + for book in sort_descriptions_by: + navPointVolumeTag = Tag(ncx_soup, 'navPoint') + navPointVolumeTag['class'] = "article" + navPointVolumeTag['id'] = "book%dID" % int(book['id']) + navPointVolumeTag['playOrder'] = self.playOrder + self.playOrder += 1 + navLabelTag = Tag(ncx_soup, "navLabel") + textTag = Tag(ncx_soup, "text") + if book['series']: + series_index = str(book['series_index']) + if series_index.endswith('.0'): + series_index = series_index[:-2] + if self.generateForKindle: + # Don't include Author for Kindle + textTag.insert(0, NavigableString(self.formatNCXText('%s (%s [%s])' % + (book['title'], book['series'], series_index), dest='title'))) + else: + # Include Author for non-Kindle + textTag.insert(0, NavigableString(self.formatNCXText('%s (%s [%s]) · %s ' % + (book['title'], book['series'], series_index, book['author']), dest='title'))) + else: + if self.generateForKindle: + # Don't include Author for Kindle + title_str = self.formatNCXText('%s' % (book['title']), dest='title') + if self.opts.connected_kindle and book['id'] in self.bookmarked_books: + ''' + dots = int((book['percent_read'] + 5)/10) + dot_string = '+' * dots + empty_dots = '-' * (10 - dots) + title_str += ' %s%s' % (dot_string,empty_dots) + ''' + title_str += '*' + textTag.insert(0, NavigableString(title_str)) + else: + # Include Author for non-Kindle + textTag.insert(0, NavigableString(self.formatNCXText('%s · %s' % \ + (book['title'], book['author']), dest='title'))) + navLabelTag.insert(0,textTag) + navPointVolumeTag.insert(0,navLabelTag) + + contentTag = Tag(ncx_soup, "content") + contentTag['src'] = "content/book_%d.html#book%d" % (int(book['id']), int(book['id'])) + navPointVolumeTag.insert(1, contentTag) + + if self.generateForKindle: + # Add the author tag + cmTag = Tag(ncx_soup, '%s' % 'calibre:meta') + cmTag['name'] = "author" + + if book['date']: + navStr = '%s | %s' % (self.formatNCXText(book['author'], dest='author'), + book['date'].split()[1]) + else: + navStr = '%s' % (self.formatNCXText(book['author'], dest='author')) + + if 'tags' in book and len(book['tags']): + navStr = self.formatNCXText(navStr + ' | ' + ' · '.join(sorted(book['tags'])), dest='author') + cmTag.insert(0, NavigableString(navStr)) + navPointVolumeTag.insert(2, cmTag) + + # Add the description tag + if book['short_description']: + cmTag = Tag(ncx_soup, '%s' % 'calibre:meta') + cmTag['name'] = "description" + cmTag.insert(0, NavigableString(self.formatNCXText(book['short_description'], dest='description'))) + navPointVolumeTag.insert(3, cmTag) + + # Add this volume to the section tag + navPointTag.insert(nptc, navPointVolumeTag) + nptc += 1 + + # Add this section to the body + body.insert(btc, navPointTag) + btc += 1 + + self.ncxSoup = ncx_soup + + def generateNCXBySeries(self, tocTitle): + self.updateProgressFullStep("NCX 'Series'") + + def add_to_series_by_letter(current_series_list): + current_series_list = " • ".join(current_series_list) + current_series_list = self.formatNCXText(current_series_list, dest="description") + series_by_letter.append(current_series_list) + + soup = self.ncxSoup + output = "BySeries" + body = soup.find("navPoint") + btc = len(body.contents) + + # --- Construct the 'Books By Series' section --- + navPointTag = Tag(soup, 'navPoint') + navPointTag['class'] = "section" + navPointTag['id'] = "byseries-ID" + navPointTag['playOrder'] = self.playOrder + self.playOrder += 1 + navLabelTag = Tag(soup, 'navLabel') + textTag = Tag(soup, 'text') + textTag.insert(0, NavigableString(tocTitle)) + navLabelTag.insert(0, textTag) + nptc = 0 + navPointTag.insert(nptc, navLabelTag) + nptc += 1 + contentTag = Tag(soup,"content") + contentTag['src'] = "content/%s.html#section_start" % (output) + navPointTag.insert(nptc, contentTag) + nptc += 1 + + series_by_letter = [] + + # Loop over the series titles, find start of each letter, add description_preview_count books + # Special switch for using different title list + title_list = self.booksBySeries + current_letter = self.letter_or_symbol(self.generateSortTitle(title_list[0]['series'])[0]) + title_letters = [current_letter] + current_series_list = [] + current_series = "" + for book in title_list: + sort_title = self.generateSortTitle(book['series']) + if self.letter_or_symbol(sort_title[0]) != current_letter: + # Save the old list + add_to_series_by_letter(current_series_list) + + # Start the new list + current_letter = self.letter_or_symbol(sort_title[0]) + title_letters.append(current_letter) + current_series = book['series'] + current_series_list = [book['series']] + else: + if len(current_series_list) < self.descriptionClip and \ + book['series'] != current_series : + current_series = book['series'] + current_series_list.append(book['series']) + + # Add the last book list + add_to_series_by_letter(current_series_list) + + # Add *article* entries for each populated series title letter + for (i,books) in enumerate(series_by_letter): + navPointByLetterTag = Tag(soup, 'navPoint') + navPointByLetterTag['class'] = "article" + navPointByLetterTag['id'] = "%sSeries-ID" % (title_letters[i].upper()) + navPointTag['playOrder'] = self.playOrder + self.playOrder += 1 + navLabelTag = Tag(soup, 'navLabel') + textTag = Tag(soup, 'text') + textTag.insert(0, NavigableString(u"Series beginning with %s" % \ + (title_letters[i] if len(title_letters[i])>1 else "'" + title_letters[i] + "'"))) + navLabelTag.insert(0, textTag) + navPointByLetterTag.insert(0,navLabelTag) + contentTag = Tag(soup, 'content') + contentTag['src'] = "content/%s.html#%s_series" % (output, title_letters[i]) + navPointByLetterTag.insert(1,contentTag) + + if self.generateForKindle: + cmTag = Tag(soup, '%s' % 'calibre:meta') + cmTag['name'] = "description" + cmTag.insert(0, NavigableString(self.formatNCXText(books, dest='description'))) + navPointByLetterTag.insert(2, cmTag) + + navPointTag.insert(nptc, navPointByLetterTag) + nptc += 1 + + # Add this section to the body + body.insert(btc, navPointTag) + btc += 1 + + self.ncxSoup = soup + + def generateNCXByTitle(self, tocTitle): + self.updateProgressFullStep("NCX 'Titles'") + + def add_to_books_by_letter(current_book_list): + current_book_list = " • ".join(current_book_list) + current_book_list = self.formatNCXText(current_book_list, dest="description") + books_by_letter.append(current_book_list) + + soup = self.ncxSoup + output = "ByAlphaTitle" + body = soup.find("navPoint") + btc = len(body.contents) + + # --- Construct the 'Books By Title' section --- + navPointTag = Tag(soup, 'navPoint') + navPointTag['class'] = "section" + navPointTag['id'] = "byalphatitle-ID" + navPointTag['playOrder'] = self.playOrder + self.playOrder += 1 + navLabelTag = Tag(soup, 'navLabel') + textTag = Tag(soup, 'text') + textTag.insert(0, NavigableString(tocTitle)) + navLabelTag.insert(0, textTag) + nptc = 0 + navPointTag.insert(nptc, navLabelTag) + nptc += 1 + contentTag = Tag(soup,"content") + contentTag['src'] = "content/%s.html#section_start" % (output) + navPointTag.insert(nptc, contentTag) + nptc += 1 + + books_by_letter = [] + + # Loop over the titles, find start of each letter, add description_preview_count books + # Special switch for using different title list + if self.useSeriesPrefixInTitlesSection: + title_list = self.booksByTitle + else: + title_list = self.booksByTitle_noSeriesPrefix + current_letter = self.letter_or_symbol(title_list[0]['title_sort'][0]) + title_letters = [current_letter] + current_book_list = [] + current_book = "" + for book in title_list: + if self.letter_or_symbol(book['title_sort'][0]) != current_letter: + # Save the old list + add_to_books_by_letter(current_book_list) + + # Start the new list + current_letter = self.letter_or_symbol(book['title_sort'][0]) + title_letters.append(current_letter) + current_book = book['title'] + current_book_list = [book['title']] + else: + if len(current_book_list) < self.descriptionClip and \ + book['title'] != current_book : + current_book = book['title'] + current_book_list.append(book['title']) + + # Add the last book list + add_to_books_by_letter(current_book_list) + + # Add *article* entries for each populated title letter + for (i,books) in enumerate(books_by_letter): + navPointByLetterTag = Tag(soup, 'navPoint') + navPointByLetterTag['class'] = "article" + navPointByLetterTag['id'] = "%sTitles-ID" % (title_letters[i].upper()) + navPointTag['playOrder'] = self.playOrder + self.playOrder += 1 + navLabelTag = Tag(soup, 'navLabel') + textTag = Tag(soup, 'text') + textTag.insert(0, NavigableString(u"Titles beginning with %s" % \ + (title_letters[i] if len(title_letters[i])>1 else "'" + title_letters[i] + "'"))) + navLabelTag.insert(0, textTag) + navPointByLetterTag.insert(0,navLabelTag) + contentTag = Tag(soup, 'content') + contentTag['src'] = "content/%s.html#%s" % (output, title_letters[i]) + navPointByLetterTag.insert(1,contentTag) + + if self.generateForKindle: + cmTag = Tag(soup, '%s' % 'calibre:meta') + cmTag['name'] = "description" + cmTag.insert(0, NavigableString(self.formatNCXText(books, dest='description'))) + navPointByLetterTag.insert(2, cmTag) + + navPointTag.insert(nptc, navPointByLetterTag) + nptc += 1 + + # Add this section to the body + body.insert(btc, navPointTag) + btc += 1 + + self.ncxSoup = soup + + def generateNCXByAuthor(self, tocTitle): + self.updateProgressFullStep("NCX 'Authors'") + + def add_to_author_list(current_author_list, current_letter): + current_author_list = " • ".join(current_author_list) + current_author_list = self.formatNCXText(current_author_list, dest="description") + master_author_list.append((current_author_list, current_letter)) + + soup = self.ncxSoup + HTML_file = "content/ByAlphaAuthor.html" + body = soup.find("navPoint") + btc = len(body.contents) + + # --- Construct the 'Books By Author' *section* --- + navPointTag = Tag(soup, 'navPoint') + navPointTag['class'] = "section" + file_ID = "%s" % tocTitle.lower() + file_ID = file_ID.replace(" ","") + navPointTag['id'] = "%s-ID" % file_ID + navPointTag['playOrder'] = self.playOrder + self.playOrder += 1 + navLabelTag = Tag(soup, 'navLabel') + textTag = Tag(soup, 'text') + textTag.insert(0, NavigableString('%s' % tocTitle)) + navLabelTag.insert(0, textTag) + nptc = 0 + navPointTag.insert(nptc, navLabelTag) + nptc += 1 + contentTag = Tag(soup,"content") + contentTag['src'] = "%s#section_start" % HTML_file + navPointTag.insert(nptc, contentTag) + nptc += 1 + + # Create an NCX article entry for each populated author index letter + # Loop over the sorted_authors list, find start of each letter, + # add description_preview_count artists + # self.authors[0]:friendly [1]:author_sort [2]:book_count + master_author_list = [] + # self.authors[0][1][0] = Initial letter of author_sort[0] + current_letter = self.letter_or_symbol(self.authors[0][1][0]) + current_author_list = [] + for author in self.authors: + if self.letter_or_symbol(author[1][0]) != current_letter: + # Save the old list + add_to_author_list(current_author_list, current_letter) + + # Start the new list + current_letter = self.letter_or_symbol(author[1][0]) + current_author_list = [author[0]] + else: + if len(current_author_list) < self.descriptionClip: + current_author_list.append(author[0]) + + # Add the last author list + add_to_author_list(current_author_list, current_letter) + + # Add *article* entries for each populated author initial letter + # master_author_list{}: [0]:author list [1]:Initial letter + for authors_by_letter in master_author_list: + navPointByLetterTag = Tag(soup, 'navPoint') + navPointByLetterTag['class'] = "article" + navPointByLetterTag['id'] = "%sauthors-ID" % (authors_by_letter[1]) + navPointTag['playOrder'] = self.playOrder + self.playOrder += 1 + navLabelTag = Tag(soup, 'navLabel') + textTag = Tag(soup, 'text') + textTag.insert(0, NavigableString("Authors beginning with '%s'" % (authors_by_letter[1]))) + navLabelTag.insert(0, textTag) + navPointByLetterTag.insert(0,navLabelTag) + contentTag = Tag(soup, 'content') + contentTag['src'] = "%s#%sauthors" % (HTML_file, authors_by_letter[1]) + + navPointByLetterTag.insert(1,contentTag) + + if self.generateForKindle: + cmTag = Tag(soup, '%s' % 'calibre:meta') + cmTag['name'] = "description" + cmTag.insert(0, NavigableString(authors_by_letter[0])) + navPointByLetterTag.insert(2, cmTag) + + navPointTag.insert(nptc, navPointByLetterTag) + nptc += 1 + + # Add this section to the body + body.insert(btc, navPointTag) + btc += 1 + + self.ncxSoup = soup + + def generateNCXByDateAdded(self, tocTitle): + self.updateProgressFullStep("NCX 'Recently Added'") + + def add_to_master_month_list(current_titles_list): + book_count = len(current_titles_list) + current_titles_list = " • ".join(current_titles_list) + current_titles_list = self.formatNCXText(current_titles_list, dest='description') + master_month_list.append((current_titles_list, current_date, book_count)) + + def add_to_master_date_range_list(current_titles_list): + book_count = len(current_titles_list) + current_titles_list = " • ".join(current_titles_list) + current_titles_list = self.formatNCXText(current_titles_list, dest='description') + master_date_range_list.append((current_titles_list, date_range, book_count)) + + soup = self.ncxSoup + HTML_file = "content/ByDateAdded.html" + body = soup.find("navPoint") + btc = len(body.contents) + + # --- Construct the 'Recently Added' *section* --- + navPointTag = Tag(soup, 'navPoint') + navPointTag['class'] = "section" + file_ID = "%s" % tocTitle.lower() + file_ID = file_ID.replace(" ","") + navPointTag['id'] = "%s-ID" % file_ID + navPointTag['playOrder'] = self.playOrder + self.playOrder += 1 + navLabelTag = Tag(soup, 'navLabel') + textTag = Tag(soup, 'text') + textTag.insert(0, NavigableString('%s' % tocTitle)) + navLabelTag.insert(0, textTag) + nptc = 0 + navPointTag.insert(nptc, navLabelTag) + nptc += 1 + contentTag = Tag(soup,"content") + contentTag['src'] = "%s#section_start" % HTML_file + navPointTag.insert(nptc, contentTag) + nptc += 1 + + # Create an NCX article entry for each date range + current_titles_list = [] + master_date_range_list = [] + today = datetime.datetime.now() + today_time = datetime.datetime(today.year, today.month, today.day) + for (i,date) in enumerate(self.DATE_RANGE): + if i: + date_range = '%d to %d days ago' % (self.DATE_RANGE[i-1], self.DATE_RANGE[i]) + else: + date_range = 'Last %d days' % (self.DATE_RANGE[i]) + date_range_limit = self.DATE_RANGE[i] + for book in self.booksByDateRange: + book_time = datetime.datetime(book['timestamp'].year, book['timestamp'].month, book['timestamp'].day) + if (today_time-book_time).days <= date_range_limit: + #print "generateNCXByDateAdded: %s added %d days ago" % (book['title'], (today_time-book_time).days) + current_titles_list.append(book['title']) + else: + break + if current_titles_list: + add_to_master_date_range_list(current_titles_list) + current_titles_list = [book['title']] + + # Add *article* entries for each populated date range + # master_date_range_list{}: [0]:titles list [1]:datestr + for books_by_date_range in master_date_range_list: + navPointByDateRangeTag = Tag(soup, 'navPoint') + navPointByDateRangeTag['class'] = "article" + navPointByDateRangeTag['id'] = "%s-ID" % books_by_date_range[1].replace(' ','') + navPointTag['playOrder'] = self.playOrder + self.playOrder += 1 + navLabelTag = Tag(soup, 'navLabel') + textTag = Tag(soup, 'text') + textTag.insert(0, NavigableString(books_by_date_range[1])) + navLabelTag.insert(0, textTag) + navPointByDateRangeTag.insert(0,navLabelTag) + contentTag = Tag(soup, 'content') + contentTag['src'] = "%s#bda_%s" % (HTML_file, + books_by_date_range[1].replace(' ','')) + + navPointByDateRangeTag.insert(1,contentTag) + + if self.generateForKindle: + cmTag = Tag(soup, '%s' % 'calibre:meta') + cmTag['name'] = "description" + cmTag.insert(0, NavigableString(books_by_date_range[0])) + navPointByDateRangeTag.insert(2, cmTag) + + cmTag = Tag(soup, '%s' % 'calibre:meta') + cmTag['name'] = "author" + navStr = '%d titles' % books_by_date_range[2] if books_by_date_range[2] > 1 else \ + '%d title' % books_by_date_range[2] + cmTag.insert(0, NavigableString(navStr)) + navPointByDateRangeTag.insert(3, cmTag) + + navPointTag.insert(nptc, navPointByDateRangeTag) + nptc += 1 + + + + # Create an NCX article entry for each populated month + # Loop over the booksByDate list, find start of each month, + # add description_preview_count titles + # master_month_list(list,date,count) + current_titles_list = [] + master_month_list = [] + current_date = self.booksByMonth[0]['timestamp'] + + for book in self.booksByMonth: + if book['timestamp'].month != current_date.month or \ + book['timestamp'].year != current_date.year: + # Save the old lists + add_to_master_month_list(current_titles_list) + + # Start the new list + current_date = book['timestamp'].date() + current_titles_list = [book['title']] + else: + current_titles_list.append(book['title']) + + # Add the last month list + add_to_master_month_list(current_titles_list) + + # Add *article* entries for each populated month + # master_months_list{}: [0]:titles list [1]:date + for books_by_month in master_month_list: + datestr = strftime(u'%B %Y', books_by_month[1].timetuple()) + navPointByMonthTag = Tag(soup, 'navPoint') + navPointByMonthTag['class'] = "article" + navPointByMonthTag['id'] = "bda_%s-%s-ID" % (books_by_month[1].year,books_by_month[1].month ) + navPointTag['playOrder'] = self.playOrder + self.playOrder += 1 + navLabelTag = Tag(soup, 'navLabel') + textTag = Tag(soup, 'text') + textTag.insert(0, NavigableString(datestr)) + navLabelTag.insert(0, textTag) + navPointByMonthTag.insert(0,navLabelTag) + contentTag = Tag(soup, 'content') + contentTag['src'] = "%s#bda_%s-%s" % (HTML_file, + books_by_month[1].year,books_by_month[1].month) + + navPointByMonthTag.insert(1,contentTag) + + if self.generateForKindle: + cmTag = Tag(soup, '%s' % 'calibre:meta') + cmTag['name'] = "description" + cmTag.insert(0, NavigableString(books_by_month[0])) + navPointByMonthTag.insert(2, cmTag) + + cmTag = Tag(soup, '%s' % 'calibre:meta') + cmTag['name'] = "author" + navStr = '%d titles' % books_by_month[2] if books_by_month[2] > 1 else \ + '%d title' % books_by_month[2] + cmTag.insert(0, NavigableString(navStr)) + navPointByMonthTag.insert(3, cmTag) + + navPointTag.insert(nptc, navPointByMonthTag) + nptc += 1 + + # Add this section to the body + body.insert(btc, navPointTag) + btc += 1 + self.ncxSoup = soup + + def generateNCXByDateRead(self, tocTitle): + self.updateProgressFullStep("NCX 'Recently Read'") + if not self.booksByDateRead: + return + + def add_to_master_day_list(current_titles_list): + book_count = len(current_titles_list) + current_titles_list = " • ".join(current_titles_list) + current_titles_list = self.formatNCXText(current_titles_list, dest='description') + master_day_list.append((current_titles_list, current_date, book_count)) + + def add_to_master_date_range_list(current_titles_list): + book_count = len(current_titles_list) + current_titles_list = " • ".join(current_titles_list) + current_titles_list = self.formatNCXText(current_titles_list, dest='description') + master_date_range_list.append((current_titles_list, date_range, book_count)) + + soup = self.ncxSoup + HTML_file = "content/ByDateRead.html" + body = soup.find("navPoint") + btc = len(body.contents) + + # --- Construct the 'Recently Read' *section* --- + navPointTag = Tag(soup, 'navPoint') + navPointTag['class'] = "section" + file_ID = "%s" % tocTitle.lower() + file_ID = file_ID.replace(" ","") + navPointTag['id'] = "%s-ID" % file_ID + navPointTag['playOrder'] = self.playOrder + self.playOrder += 1 + navLabelTag = Tag(soup, 'navLabel') + textTag = Tag(soup, 'text') + textTag.insert(0, NavigableString('%s' % tocTitle)) + navLabelTag.insert(0, textTag) + nptc = 0 + navPointTag.insert(nptc, navLabelTag) + nptc += 1 + contentTag = Tag(soup,"content") + contentTag['src'] = "%s#section_start" % HTML_file + navPointTag.insert(nptc, contentTag) + nptc += 1 + + # Create an NCX article entry for each date range + current_titles_list = [] + master_date_range_list = [] + today = datetime.datetime.now() + today_time = datetime.datetime(today.year, today.month, today.day) + for (i,date) in enumerate(self.DATE_RANGE): + if i: + date_range = '%d to %d days ago' % (self.DATE_RANGE[i-1], self.DATE_RANGE[i]) + else: + date_range = 'Last %d days' % (self.DATE_RANGE[i]) + date_range_limit = self.DATE_RANGE[i] + for book in self.booksByDateRead: + bookmark_time = datetime.datetime.utcfromtimestamp(book['bookmark_timestamp']) + if (today_time-bookmark_time).days <= date_range_limit: + #print "generateNCXByDateAdded: %s added %d days ago" % (book['title'], (today_time-book_time).days) + current_titles_list.append(book['title']) + else: + break + if current_titles_list: + add_to_master_date_range_list(current_titles_list) + current_titles_list = [book['title']] + + # Create an NCX article entry for each populated day + # Loop over the booksByDate list, find start of each month, + # add description_preview_count titles + # master_month_list(list,date,count) + current_titles_list = [] + master_day_list = [] + current_date = datetime.datetime.utcfromtimestamp(self.booksByDateRead[0]['bookmark_timestamp']) + + for book in self.booksByDateRead: + bookmark_time = datetime.datetime.utcfromtimestamp(book['bookmark_timestamp']) + if bookmark_time.day != current_date.day or \ + bookmark_time.month != current_date.month or \ + bookmark_time.year != current_date.year: + # Save the old lists + add_to_master_day_list(current_titles_list) + + # Start the new list + current_date = datetime.datetime.utcfromtimestamp(book['bookmark_timestamp']).date() + current_titles_list = [book['title']] + else: + current_titles_list.append(book['title']) + + # Add the last day list + add_to_master_day_list(current_titles_list) + + # Add *article* entries for each populated day + # master_day_list{}: [0]:titles list [1]:date + for books_by_day in master_day_list: + datestr = strftime(u'%A, %B %d', books_by_day[1].timetuple()) + navPointByDayTag = Tag(soup, 'navPoint') + navPointByDayTag['class'] = "article" + navPointByDayTag['id'] = "bdr_%s-%s-%sID" % (books_by_day[1].year, + books_by_day[1].month, + books_by_day[1].day ) + navPointTag['playOrder'] = self.playOrder + self.playOrder += 1 + navLabelTag = Tag(soup, 'navLabel') + textTag = Tag(soup, 'text') + textTag.insert(0, NavigableString(datestr)) + navLabelTag.insert(0, textTag) + navPointByDayTag.insert(0,navLabelTag) + contentTag = Tag(soup, 'content') + contentTag['src'] = "%s#bdr_%s-%s-%s" % (HTML_file, + books_by_day[1].year, + books_by_day[1].month, + books_by_day[1].day) + + navPointByDayTag.insert(1,contentTag) + + if self.generateForKindle: + cmTag = Tag(soup, '%s' % 'calibre:meta') + cmTag['name'] = "description" + cmTag.insert(0, NavigableString(books_by_day[0])) + navPointByDayTag.insert(2, cmTag) + + cmTag = Tag(soup, '%s' % 'calibre:meta') + cmTag['name'] = "author" + navStr = '%d titles' % books_by_day[2] if books_by_day[2] > 1 else \ + '%d title' % books_by_day[2] + cmTag.insert(0, NavigableString(navStr)) + navPointByDayTag.insert(3, cmTag) + + navPointTag.insert(nptc, navPointByDayTag) + nptc += 1 + + # Add this section to the body + body.insert(btc, navPointTag) + btc += 1 + self.ncxSoup = soup + + def generateNCXByGenre(self, tocTitle): + # Create an NCX section for 'By Genre' + # Add each genre as an article + # 'tag', 'file', 'authors' + + self.updateProgressFullStep("NCX 'Genres'") + + if not len(self.genres): + self.opts.log.warn(" No genres found in tags.\n" + " No Genre section added to Catalog") + return + + ncx_soup = self.ncxSoup + body = ncx_soup.find("navPoint") + btc = len(body.contents) + + # --- Construct the 'Books By Genre' *section* --- + navPointTag = Tag(ncx_soup, 'navPoint') + navPointTag['class'] = "section" + file_ID = "%s" % tocTitle.lower() + file_ID = file_ID.replace(" ","") + navPointTag['id'] = "%s-ID" % file_ID + navPointTag['playOrder'] = self.playOrder + self.playOrder += 1 + navLabelTag = Tag(ncx_soup, 'navLabel') + textTag = Tag(ncx_soup, 'text') + # textTag.insert(0, NavigableString('%s (%d)' % (section_title, len(genre_list)))) + textTag.insert(0, NavigableString('%s' % tocTitle)) + navLabelTag.insert(0, textTag) + nptc = 0 + navPointTag.insert(nptc, navLabelTag) + nptc += 1 + contentTag = Tag(ncx_soup,"content") + contentTag['src'] = "content/Genre_%s.html#section_start" % self.genres[0]['tag'] + navPointTag.insert(nptc, contentTag) + nptc += 1 + + for genre in self.genres: + # Add an article for each genre + navPointVolumeTag = Tag(ncx_soup, 'navPoint') + navPointVolumeTag['class'] = "article" + navPointVolumeTag['id'] = "genre-%s-ID" % genre['tag'] + navPointVolumeTag['playOrder'] = self.playOrder + self.playOrder += 1 + navLabelTag = Tag(ncx_soup, "navLabel") + textTag = Tag(ncx_soup, "text") + + # GwR *** Can this be optimized? + normalized_tag = None + for friendly_tag in self.genre_tags_dict: + if self.genre_tags_dict[friendly_tag] == genre['tag']: + normalized_tag = self.genre_tags_dict[friendly_tag] + break + textTag.insert(0, self.formatNCXText(NavigableString(friendly_tag), dest='description')) + navLabelTag.insert(0,textTag) + navPointVolumeTag.insert(0,navLabelTag) + contentTag = Tag(ncx_soup, "content") + contentTag['src'] = "content/Genre_%s.html#Genre_%s" % (normalized_tag, normalized_tag) + navPointVolumeTag.insert(1, contentTag) + + if self.generateForKindle: + # Build the author tag + cmTag = Tag(ncx_soup, '%s' % 'calibre:meta') + cmTag['name'] = "author" + # First - Last author + + if len(genre['titles_spanned']) > 1 : + author_range = "%s - %s" % (genre['titles_spanned'][0][0], genre['titles_spanned'][1][0]) + else : + author_range = "%s" % (genre['titles_spanned'][0][0]) + + cmTag.insert(0, NavigableString(author_range)) + navPointVolumeTag.insert(2, cmTag) + + # Build the description tag + cmTag = Tag(ncx_soup, '%s' % 'calibre:meta') + cmTag['name'] = "description" + + if False: + # Form 1: Titles spanned + if len(genre['titles_spanned']) > 1: + title_range = "%s -\n%s" % (genre['titles_spanned'][0][1], genre['titles_spanned'][1][1]) + else: + title_range = "%s" % (genre['titles_spanned'][0][1]) + cmTag.insert(0, NavigableString(self.formatNCXText(title_range, dest='description'))) + else: + # Form 2: title • title • title ... + titles = [] + for title in genre['books']: + titles.append(title['title']) + titles = sorted(titles, key=lambda x:(self.generateSortTitle(x),self.generateSortTitle(x))) + titles_list = self.generateShortDescription(u" • ".join(titles), dest="description") + cmTag.insert(0, NavigableString(self.formatNCXText(titles_list, dest='description'))) + + navPointVolumeTag.insert(3, cmTag) + + # Add this volume to the section tag + navPointTag.insert(nptc, navPointVolumeTag) + nptc += 1 + + # Add this section to the body + body.insert(btc, navPointTag) + btc += 1 + self.ncxSoup = ncx_soup + + def writeNCX(self): + self.updateProgressFullStep("Saving NCX") + + outfile = open("%s/%s.ncx" % (self.catalogPath, self.basename), 'w') + outfile.write(self.ncxSoup.prettify()) + + + # ======================== Helpers ======================== + def author_to_author_sort(self, author): + tokens = author.split() + tokens = tokens[-1:] + tokens[:-1] + if len(tokens) > 1: + tokens[0] += ',' + return ' '.join(tokens).capitalize() + + def booksByAuthorSorter_author_sort(self, book): + ''' + Sort non-series books before series books + ''' + if not book['series']: + key = '%s %s' % (capitalize(book['author_sort']), + capitalize(book['title_sort'])) + else: + index = book['series_index'] + integer = int(index) + fraction = index-integer + series_index = '%04d%s' % (integer, str('%0.4f' % fraction).lstrip('0')) + key = '%s ~%s %s' % (capitalize(book['author_sort']), + self.generateSortTitle(book['series']), + series_index) + return key + + def booksByAuthorSorter_author(self, book): + ''' + Sort non-series books before series books + ''' + if not book['series']: + key = '%s %s' % (self.author_to_author_sort(book['author']), + capitalize(book['title_sort'])) + else: + index = book['series_index'] + integer = int(index) + fraction = index-integer + series_index = '%04d%s' % (integer, str('%0.4f' % fraction).lstrip('0')) + key = '%s ~%s %s' % (self.author_to_author_sort(book['author']), + self.generateSortTitle(book['series']), + series_index) + return key + + def calculateThumbnailSize(self): + ''' Calculate thumbnail dimensions based on device DPI. Scale Kindle by 50% ''' + from calibre.customize.ui import output_profiles + for x in output_profiles(): + if x.short_name == self.opts.output_profile: + # aspect ratio: 3:4 + self.thumbWidth = x.dpi * float(self.opts.thumb_width) + self.thumbHeight = self.thumbWidth * 1.33 + if 'kindle' in x.short_name and self.opts.fmt == 'mobi': + # Kindle DPI appears to be off by a factor of 2 + self.thumbWidth = self.thumbWidth/2 + self.thumbHeight = self.thumbHeight/2 + break + if True and self.verbose: + self.opts.log(" DPI = %d; thumbnail dimensions: %d x %d" % \ + (x.dpi, self.thumbWidth, self.thumbHeight)) + + def convertHTMLEntities(self, s): + matches = re.findall("&#\d+;", s) + if len(matches) > 0: + hits = set(matches) + for hit in hits: + name = hit[2:-1] + try: + entnum = int(name) + s = s.replace(hit, unichr(entnum)) + except ValueError: + pass + + matches = re.findall("&\w+;", s) + hits = set(matches) + amp = "&" + if amp in hits: + hits.remove(amp) + for hit in hits: + name = hit[1:-1] + if htmlentitydefs.name2codepoint.has_key(name): + s = s.replace(hit, unichr(htmlentitydefs.name2codepoint[name])) + s = s.replace(amp, "&") + return s + + def createDirectoryStructure(self): + catalogPath = self.catalogPath + self.cleanUp() + + if not os.path.isdir(catalogPath): + os.makedirs(catalogPath) + + # Create /content and /images + content_path = catalogPath + "/content" + if not os.path.isdir(content_path): + os.makedirs(content_path) + images_path = catalogPath + "/images" + if not os.path.isdir(images_path): + os.makedirs(images_path) + + def discoverReadStatus(self, record): + ''' + Given a field:pattern spec, discover if this book marked as read + + if field == tag, scan tags for pattern + if custom field, try regex match for pattern + This allows maximum flexibility with fields of type + datatype bool: #field_name:True + datatype text: #field_name:<string> + datatype datetime: #field_name:.* + + ''' + # Legacy handling of special 'read' tag + field = self.__read_book_marker['field'] + pat = self.__read_book_marker['pattern'] + if field == 'tag' and pat in record['tags']: + return True + + field_contents = self.__db.get_field(record['id'], + field, + index_is_id=True) + if field_contents: + try: + if re.search(pat, unicode(field_contents), + re.IGNORECASE) is not None: + return True + except: + # Compiling of pat failed, ignore it + pass + + return False + + def filterDbTags(self, tags): + # Remove the special marker tags from the database's tag list, + # return sorted list of normalized genre tags + + def format_tag_list(tags, indent=5, line_break=70, header='Tag list'): + def next_tag(sorted_tags): + for (i, tag) in enumerate(sorted_tags): + if i < len(tags) - 1: + yield tag + ", " + else: + yield tag + + ans = '%s%d %s:\n' % (' ' * indent, len(tags), header) + ans += ' ' * (indent + 1) + out_str = '' + sorted_tags = sorted(tags) + for tag in next_tag(sorted_tags): + out_str += tag + if len(out_str) >= line_break: + ans += out_str + '\n' + out_str = ' ' * (indent + 1) + return ans + out_str + + normalized_tags = [] + friendly_tags = [] + excluded_tags = [] + for tag in tags: + if tag in self.markerTags: + excluded_tags.append(tag) + continue + if re.search(self.opts.exclude_genre, tag): + excluded_tags.append(tag) + continue + if tag == ' ': + continue + + normalized_tags.append(re.sub('\W','',tag).lower()) + friendly_tags.append(tag) + + genre_tags_dict = dict(zip(friendly_tags,normalized_tags)) + + # Test for multiple genres resolving to same normalized form + normalized_set = set(normalized_tags) + for normalized in normalized_set: + if normalized_tags.count(normalized) > 1: + self.opts.log.warn(" Warning: multiple tags resolving to genre '%s':" % normalized) + for key in genre_tags_dict: + if genre_tags_dict[key] == normalized: + self.opts.log.warn(" %s" % key) + if self.verbose: + self.opts.log.info('%s' % format_tag_list(genre_tags_dict, header="enabled genre tags in database")) + self.opts.log.info('%s' % format_tag_list(excluded_tags, header="excluded genre tags")) + + return genre_tags_dict + + def formatNCXText(self, description, dest=None): + # Kindle TOC descriptions won't render certain characters + # Fix up + massaged = unicode(BeautifulStoneSoup(description, convertEntities=BeautifulStoneSoup.HTML_ENTITIES)) + + # Replace '&' with '&' + massaged = re.sub("&","&", massaged) + + if massaged.strip() and dest: + #print traceback.print_stack(limit=3) + return self.generateShortDescription(massaged.strip(), dest=dest) + else: + return None + + def generateAuthorAnchor(self, author): + # Strip white space to '' + return re.sub("\W","", author) + + def generateFormatArgs(self, book): + series_index = str(book['series_index']) + if series_index.endswith('.0'): + series_index = series_index[:-2] + args = dict( + title = book['title'], + series = book['series'], + series_index = series_index, + rating = self.generateRatingString(book), + rating_parens = '(%s)' % self.generateRatingString(book) if 'rating' in book else '', + pubyear = book['date'].split()[1] if book['date'] else '', + pubyear_parens = "(%s)" % book['date'].split()[1] if book['date'] else '') + return args + + def generateHTMLByGenre(self, genre, section_head, books, outfile): + # Write an HTML file of this genre's book list + # Return a list with [(first_author, first_book), (last_author, last_book)] + + soup = self.generateHTMLGenreHeader(genre) + body = soup.find('body') + + btc = 0 + + # Insert section tag if this is the section start - first article only + if section_head: + aTag = Tag(soup,'a') + aTag['name'] = 'section_start' + body.insert(btc, aTag) + btc += 1 + + # Create an anchor from the tag + aTag = Tag(soup, 'a') + aTag['name'] = "Genre_%s" % genre + body.insert(btc,aTag) + btc += 1 + + titleTag = body.find(attrs={'class':'title'}) + titleTag.insert(0,NavigableString('%s' % escape(self.getFriendlyGenreTag(genre)))) + + # Insert the books by author list + divTag = body.find(attrs={'class':'authors'}) + dtc = 0 + + current_author = '' + current_series = None + for book in books: + if book['author'] != current_author: + # Start a new author with link + current_author = book['author'] + non_series_books = 0 + current_series = None + pAuthorTag = Tag(soup, "p") + pAuthorTag['class'] = "author_index" + aTag = Tag(soup, "a") + if self.opts.generate_authors: + aTag['href'] = "%s.html#%s" % ("ByAlphaAuthor", self.generateAuthorAnchor(book['author'])) + aTag.insert(0, book['author']) + pAuthorTag.insert(0,aTag) + divTag.insert(dtc,pAuthorTag) + dtc += 1 + + # Check for series + if book['series'] and book['series'] != current_series: + # Start a new series + current_series = book['series'] + pSeriesTag = Tag(soup,'p') + pSeriesTag['class'] = "series" + if self.opts.generate_series: + aTag = Tag(soup,'a') + aTag['href'] = "%s.html#%s_series" % ('BySeries', + re.sub('\W','',book['series']).lower()) + aTag.insert(0, book['series']) + pSeriesTag.insert(0, aTag) + else: + pSeriesTag.insert(0,NavigableString('%s' % book['series'])) + divTag.insert(dtc,pSeriesTag) + dtc += 1 + + if current_series and not book['series']: + current_series = None + + # Add books + pBookTag = Tag(soup, "p") + ptc = 0 + + # book with read|reading|unread symbol or wishlist item + if self.opts.wishlist_tag in book.get('tags', []): + pBookTag['class'] = "wishlist_item" + pBookTag.insert(ptc,NavigableString(self.MISSING_SYMBOL)) + ptc += 1 + else: + if book['read']: + # check mark + pBookTag.insert(ptc,NavigableString(self.READ_SYMBOL)) + pBookTag['class'] = "read_book" + ptc += 1 + elif book['id'] in self.bookmarked_books: + pBookTag.insert(ptc,NavigableString(self.READING_SYMBOL)) + pBookTag['class'] = "read_book" + ptc += 1 + else: + # hidden check mark + pBookTag['class'] = "unread_book" + pBookTag.insert(ptc,NavigableString(self.NOT_READ_SYMBOL)) + ptc += 1 + + # Add the book title + aTag = Tag(soup, "a") + if self.opts.generate_descriptions: + aTag['href'] = "book_%d.html" % (int(float(book['id']))) + + # Generate the title from the template + args = self.generateFormatArgs(book) + if current_series: + #aTag.insert(0,escape(book['title'][len(book['series'])+1:])) + formatted_title = self.by_genres_series_title_template.format(**args).rstrip() + else: + #aTag.insert(0,escape(book['title'])) + formatted_title = self.by_genres_normal_title_template.format(**args).rstrip() + non_series_books += 1 + aTag.insert(0,NavigableString(escape(formatted_title))) + + pBookTag.insert(ptc, aTag) + ptc += 1 + + divTag.insert(dtc, pBookTag) + dtc += 1 + + # Write the generated file to contentdir + outfile = open(outfile, 'w') + outfile.write(soup.prettify()) + outfile.close() + + if len(books) > 1: + titles_spanned = [(books[0]['author'],books[0]['title']), (books[-1]['author'],books[-1]['title'])] + else: + titles_spanned = [(books[0]['author'],books[0]['title'])] + + return titles_spanned + + def generateHTMLDescriptionHeader(self, book): + ''' + Generate description header from template + ''' + from calibre.ebooks.oeb.base import XHTML_NS + + def generate_html(): + args = dict( + author=author, + author_prefix=author_prefix, + comments=comments, + css=css, + formats=formats, + genres=genres, + note_content=note_content, + note_source=note_source, + pubdate=pubdate, + publisher=publisher, + pubmonth=pubmonth, + pubyear=pubyear, + rating=rating, + series=series, + series_index=series_index, + thumb=thumb, + title=title, + title_str=title_str, + xmlns=XHTML_NS, + ) + + generated_html = P('catalog/template.xhtml', + data=True).decode('utf-8').format(**args) + generated_html = substitute_entites(generated_html) + return BeautifulSoup(generated_html) + + # Generate the template arguments + css = P('catalog/stylesheet.css', data=True).decode('utf-8') + title_str = title = escape(book['title']) + series = '' + series_index = '' + if book['series']: + series = escape(book['series']) + series_index = str(book['series_index']) + if series_index.endswith('.0'): + series_index = series_index[:-2] + + # Author, author_prefix (read|reading|none symbol or missing symbol) + author = book['author'] + if self.opts.wishlist_tag in book.get('tags', []): + author_prefix = self.MISSING_SYMBOL + " by " + else: + if book['read']: + author_prefix = self.READ_SYMBOL + " by " + elif self.opts.connected_kindle and book['id'] in self.bookmarked_books: + author_prefix = self.READING_SYMBOL + " by " + else: + author_prefix = "by " + + # Genres + genres = '' + if 'tags' in book: + _soup = BeautifulSoup('') + genresTag = Tag(_soup,'p') + gtc = 0 + for (i, tag) in enumerate(sorted(book.get('tags', []))): + aTag = Tag(_soup,'a') + if self.opts.generate_genres: + aTag['href'] = "Genre_%s.html" % re.sub("\W","",tag.lower()) + aTag.insert(0,escape(NavigableString(tag))) + genresTag.insert(gtc, aTag) + gtc += 1 + if i < len(book['tags'])-1: + genresTag.insert(gtc, NavigableString(' · ')) + gtc += 1 + genres = genresTag.renderContents() + + # Formats + formats = [] + if 'formats' in book: + for format in sorted(book['formats']): + formats.append(format.rpartition('.')[2].upper()) + formats = ' · '.join(formats) + + # Date of publication + if book['date']: + pubdate = book['date'] + pubmonth, pubyear = pubdate.split() + else: + pubdate = pubyear = pubmonth = '' + + # Thumb + _soup = BeautifulSoup('<html>',selfClosingTags=['img']) + thumb = Tag(_soup,"img") + if 'cover' in book and book['cover']: + thumb['src'] = "../images/thumbnail_%d.jpg" % int(book['id']) + else: + thumb['src'] = "../images/thumbnail_default.jpg" + thumb['alt'] = "cover thumbnail" + + # Publisher + publisher = ' ' + if 'publisher' in book: + publisher = book['publisher'] + + # Rating + stars = int(book['rating']) / 2 + rating = '' + if stars: + star_string = self.FULL_RATING_SYMBOL * stars + empty_stars = self.EMPTY_RATING_SYMBOL * (5 - stars) + rating = '%s%s <br/>' % (star_string,empty_stars) + + # Notes + note_source = '' + note_content = '' + if 'notes' in book: + note_source = book['notes']['source'] + note_content = book['notes']['content'] + + # Comments + comments = '' + if 'description' in book and book['description'] > '': + comments = book['description'] + + + # >>>> Populate the template <<<< + soup = generate_html() + + + # >>>> Post-process the template <<<< + body = soup.find('body') + btc = 0 + # Insert the title anchor for inbound links + aTag = Tag(soup, "a") + aTag['name'] = "book%d" % int(book['id']) + body.insert(btc, aTag) + btc += 1 + + # Insert the link to the series or remove <a class="series"> + aTag = body.find('a', attrs={'class':'series_id'}) + if aTag: + if book['series']: + if self.opts.generate_series: + aTag['href'] = "%s.html#%s_series" % ('BySeries', + re.sub('\W','',book['series']).lower()) + else: + aTag.extract() + + # Insert the author link + aTag = body.find('a', attrs={'class':'author'}) + if self.opts.generate_authors and aTag: + aTag['href'] = "%s.html#%s" % ("ByAlphaAuthor", + self.generateAuthorAnchor(book['author'])) + + if publisher == ' ': + publisherTag = body.find('td', attrs={'class':'publisher'}) + if publisherTag: + publisherTag.contents[0].replaceWith(' ') + + if not genres: + genresTag = body.find('p',attrs={'class':'genres'}) + if genresTag: + genresTag.extract() + + if not formats: + formatsTag = body.find('p',attrs={'class':'formats'}) + if formatsTag: + formatsTag.extract() + + if note_content == '': + tdTag = body.find('td', attrs={'class':'notes'}) + if tdTag: + tdTag.contents[0].replaceWith(' ') + + emptyTags = body.findAll('td', attrs={'class':'empty'}) + for mt in emptyTags: + newEmptyTag = Tag(BeautifulSoup(),'td') + newEmptyTag.insert(0,NavigableString(' ')) + mt.replaceWith(newEmptyTag) + + if False: + print soup.prettify() + return soup + + def generateHTMLEmptyHeader(self, title): + header = ''' + <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd"> + <html xmlns="http://www.w3.org/1999/xhtml" xmlns:calibre="http://calibre.kovidgoyal.net/2009/metadata"> + <head> + <meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /> + <link rel="stylesheet" type="text/css" href="stylesheet.css" media="screen" /> + <title> + + + + + ''' + # Insert the supplied title + soup = BeautifulSoup(header) + titleTag = soup.find('title') + titleTag.insert(0,NavigableString(title)) + return soup + + def generateHTMLGenreHeader(self, title): + header = ''' + + + + + + + + +

+
+ + + ''' + # Insert the supplied title + soup = BeautifulSoup(header) + titleTag = soup.find('title') + titleTag.insert(0,escape(NavigableString(title))) + return soup + + def generateMastheadImage(self, out_path): + from calibre.ebooks.conversion.config import load_defaults + from calibre.utils.fonts import fontconfig + font_path = default_font = P('fonts/liberation/LiberationSerif-Bold.ttf') + recs = load_defaults('mobi_output') + masthead_font_family = recs.get('masthead_font', 'Default') + + if masthead_font_family != 'Default': + masthead_font = fontconfig.files_for_family(masthead_font_family) + # Assume 'normal' always in dict, else use default + # {'normal': (path_to_font, friendly name)} + if 'normal' in masthead_font: + font_path = masthead_font['normal'][0] + + if not font_path or not os.access(font_path, os.R_OK): + font_path = default_font + + MI_WIDTH = 600 + MI_HEIGHT = 60 + + try: + from PIL import Image, ImageDraw, ImageFont + Image, ImageDraw, ImageFont + except ImportError: + import Image, ImageDraw, ImageFont + + img = Image.new('RGB', (MI_WIDTH, MI_HEIGHT), 'white') + draw = ImageDraw.Draw(img) + try: + font = ImageFont.truetype(font_path, 48) + except: + self.opts.log.error(" Failed to load user-specifed font '%s'" % font_path) + font = ImageFont.truetype(default_font, 48) + text = self.title.encode('utf-8') + width, height = draw.textsize(text, font=font) + left = max(int((MI_WIDTH - width)/2.), 0) + top = max(int((MI_HEIGHT - height)/2.), 0) + draw.text((left, top), text, fill=(0,0,0), font=font) + img.save(open(out_path, 'wb'), 'GIF') + + def generateRatingString(self, book): + rating = '' + try: + if 'rating' in book: + stars = int(book['rating']) / 2 + if stars: + star_string = self.FULL_RATING_SYMBOL * stars + empty_stars = self.EMPTY_RATING_SYMBOL * (5 - stars) + rating = '%s%s' % (star_string,empty_stars) + except: + # Rating could be None + pass + return rating + + def generateShortDescription(self, description, dest=None): + # Truncate the description, on word boundaries if necessary + # Possible destinations: + # description NCX summary + # title NCX title + # author NCX author + + def shortDescription(description, limit): + short_description = "" + words = description.split() + for word in words: + short_description += word + " " + if len(short_description) > limit: + short_description += "..." + return short_description + + if not description: + return None + + if dest == 'title': + # No truncation for titles, let the device deal with it + return description + elif dest == 'author': + if self.authorClip and len(description) < self.authorClip: + return description + else: + return shortDescription(description, self.authorClip) + elif dest == 'description': + if self.descriptionClip and len(description) < self.descriptionClip: + return description + else: + return shortDescription(description, self.descriptionClip) + else: + print " returning description with unspecified destination '%s'" % description + raise RuntimeError + + def generateSortTitle(self, title): + ''' + Generate a string suitable for sorting from the title + Ignore leading stop words + Optionally convert leading numbers to strings + ''' + from calibre.ebooks.metadata import title_sort + from calibre.library.catalogs.utils import NumberToText + + # Strip stop words + title_words = title_sort(title).split() + translated = [] + + for (i,word) in enumerate(title_words): + # Leading numbers optionally translated to text equivalent + # Capitalize leading sort word + if i==0: + # *** Keep this code in case we need to restore numbers_as_text *** + if False: + #if self.opts.numbers_as_text and re.match('[0-9]+',word[0]): + translated.append(NumberToText(word).text.capitalize()) + else: + if re.match('[0-9]+',word[0]): + word = word.replace(',','') + suffix = re.search('[\D]', word) + if suffix: + word = '%10.0f%s' % (float(word[:suffix.start()]),word[suffix.start():]) + else: + word = '%10.0f' % (float(word)) + + # If leading char > 'A', insert symbol as leading forcing lower sort + # '/' sorts below numbers, g + if self.letter_or_symbol(word[0]) != word[0]: + if word[0] > 'A' or (ord('9') < ord(word[0]) < ord('A')) : + translated.append('/') + translated.append(capitalize(word)) + + else: + if re.search('[0-9]+',word[0]): + word = word.replace(',','') + suffix = re.search('[\D]', word) + if suffix: + word = '%10.0f%s' % (float(word[:suffix.start()]),word[suffix.start():]) + else: + word = '%10.0f' % (float(word)) + translated.append(word) + return ' '.join(translated) + + def generateThumbnail(self, title, image_dir, thumb_file): + ''' + Thumbs are cached with the full cover's crc. If the crc doesn't + match, the cover has been changed since the thumb was cached and needs + to be replaced. + ''' + + def open_archive(mode='r'): + try: + return ZipFile(self.__archive_path, mode=mode) + except: + # Happens on windows if the file is opened by another + # process + pass + + # Generate crc for current cover + #self.opts.log.info(" generateThumbnail():") + with open(title['cover'], 'rb') as f: + data = f.read() + cover_crc = hex(zlib.crc32(data)) + + # Test cache for uuid + zf = open_archive() + if zf is not None: + with zf: + try: + zf.getinfo(title['uuid']+cover_crc) + except: + pass + else: + # uuid found in cache with matching crc + thumb_data = zf.read(title['uuid']+cover_crc) + with open(os.path.join(image_dir, thumb_file), 'wb') as f: + f.write(thumb_data) + return + + + # Save thumb for catalog + thumb_data = thumbnail(data, + width=self.thumbWidth, height=self.thumbHeight)[-1] + with open(os.path.join(image_dir, thumb_file), 'wb') as f: + f.write(thumb_data) + + # Save thumb to archive + if zf is not None: # Ensure that the read succeeded + # If we failed to open the zip file for reading, + # we dont know if it contained the thumb or not + zf = open_archive('a') + if zf is not None: + with zf: + zf.writestr(title['uuid']+cover_crc, thumb_data) + + def getFriendlyGenreTag(self, genre): + # Find the first instance of friendly_tag matching genre + for friendly_tag in self.genre_tags_dict: + if self.genre_tags_dict[friendly_tag] == genre: + return friendly_tag + + def getMarkerTags(self): + ''' Return a list of special marker tags to be excluded from genre list ''' + markerTags = [] + markerTags.extend(self.opts.exclude_tags.split(',')) + return markerTags + + def letter_or_symbol(self,char): + if not re.search('[a-zA-Z]',char): + return 'Symbols' + else: + return char + + def markdownComments(self, comments): + ''' + Convert random comment text to normalized, xml-legal block of

s + 'plain text' returns as +

plain text

+ + 'plain text with minimal markup' returns as +

plain text with minimal markup

+ + '

pre-formatted text

returns untouched + + 'A line of text\n\nFollowed by a line of text' returns as +

A line of text

+

Followed by a line of text

+ + 'A line of text.\nA second line of text.\rA third line of text' returns as +

A line of text.
A second line of text.
A third line of text.

+ + '...end of a paragraph.Somehow the break was lost...' returns as +

...end of a paragraph.

+

Somehow the break was lost...

+ + Deprecated HTML returns as HTML via BeautifulSoup() + + ''' + # Hackish - ignoring sentences ending or beginning in numbers to avoid + # confusion with decimal points. + + # Explode lost CRs to \n\n + for lost_cr in re.finditer('([a-z])([\.\?!])([A-Z])',comments): + comments = comments.replace(lost_cr.group(), + '%s%s\n\n%s' % (lost_cr.group(1), + lost_cr.group(2), + lost_cr.group(3))) + # Extract pre-built elements - annotations, etc. + if not isinstance(comments, unicode): + comments = comments.decode('utf-8', 'replace') + soup = BeautifulSoup(comments) + elems = soup.findAll('div') + for elem in elems: + elem.extract() + + # Reconstruct comments w/o
s + comments = soup.renderContents(None) + + # Convert \n\n to

s + if re.search('\n\n', comments): + soup = BeautifulSoup() + split_ps = comments.split(u'\n\n') + tsc = 0 + for p in split_ps: + pTag = Tag(soup,'p') + pTag.insert(0,p) + soup.insert(tsc,pTag) + tsc += 1 + comments = soup.renderContents(None) + + # Convert solo returns to
+ comments = re.sub('[\r\n]','
', comments) + + # Convert two hypens to emdash + comments = re.sub('--','—',comments) + soup = BeautifulSoup(comments) + result = BeautifulSoup() + rtc = 0 + open_pTag = False + + all_tokens = list(soup.contents) + for token in all_tokens: + if type(token) is NavigableString: + if not open_pTag: + pTag = Tag(result,'p') + open_pTag = True + ptc = 0 + pTag.insert(ptc,prepare_string_for_xml(token)) + ptc += 1 + + elif token.name in ['br','b','i','em']: + if not open_pTag: + pTag = Tag(result,'p') + open_pTag = True + ptc = 0 + pTag.insert(ptc, token) + ptc += 1 + + else: + if open_pTag: + result.insert(rtc, pTag) + rtc += 1 + open_pTag = False + ptc = 0 + # Clean up NavigableStrings for xml + sub_tokens = list(token.contents) + for sub_token in sub_tokens: + if type(sub_token) is NavigableString: + sub_token.replaceWith(prepare_string_for_xml(sub_token)) + result.insert(rtc, token) + rtc += 1 + + if open_pTag: + result.insert(rtc, pTag) + rtc += 1 + + paras = result.findAll('p') + for p in paras: + p['class'] = 'description' + + # Add back

elems initially removed + for elem in elems: + result.insert(rtc,elem) + rtc += 1 + + return result.renderContents(encoding=None) + + def mergeComments(self, record): + ''' + merge ['description'] with custom field contents to be displayed in Descriptions + ''' + merged = '' + if record['description']: + addendum = self.__db.get_field(record['id'], + self.__merge_comments['field'], + index_is_id=True) + if addendum is None: + addendum = '' + include_hr = eval(self.__merge_comments['hr']) + if self.__merge_comments['position'] == 'before': + merged = addendum + if include_hr: + merged += '
' + else: + merged += '\n' + merged += record['description'] + else: + merged = record['description'] + if include_hr: + merged += '
' + else: + merged += '\n' + merged += addendum + else: + # Return the custom field contents + merged = self.__db.get_field(record['id'], + self.__merge_comments['field'], + index_is_id=True) + + return merged + + def processExclusions(self, data_set): + ''' + Remove excluded entries + ''' + field, pat = self.opts.exclude_book_marker.split(':') + if pat == '': + return data_set + filtered_data_set = [] + for record in data_set: + field_contents = self.__db.get_field(record['id'], + field, + index_is_id=True) + if field_contents: + if re.search(pat, unicode(field_contents), + re.IGNORECASE) is not None: + continue + filtered_data_set.append(record) + + return filtered_data_set + + def processSpecialTags(self, tags, this_title, opts): + tag_list = [] + for tag in tags: + tag = self.convertHTMLEntities(tag) + if re.search(opts.exclude_genre, tag): + continue + elif self.__read_book_marker['field'] == 'tag' and \ + tag == self.__read_book_marker['pattern']: + # remove 'read' tag + continue + else: + tag_list.append(tag) + return tag_list + + def updateProgressFullStep(self, description): + self.currentStep += 1 + self.progressString = description + self.progressInt = float((self.currentStep-1)/self.totalSteps) + self.reporter(self.progressInt, self.progressString) + if self.opts.cli_environment: + self.opts.log(u"%3.0f%% %s" % (self.progressInt*100, self.progressString)) + + def updateProgressMicroStep(self, description, micro_step_pct): + step_range = 100/self.totalSteps + self.progressString = description + coarse_progress = float((self.currentStep-1)/self.totalSteps) + fine_progress = float((micro_step_pct*step_range)/100) + self.progressInt = coarse_progress + fine_progress + self.reporter(self.progressInt, self.progressString) + diff --git a/src/calibre/library/catalogs/utils.py b/src/calibre/library/catalogs/utils.py new file mode 100644 index 0000000000..7818268ded --- /dev/null +++ b/src/calibre/library/catalogs/utils.py @@ -0,0 +1,217 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai + +__license__ = 'GPL v3' +__copyright__ = '2010, Greg Riker' +__docformat__ = 'restructuredtext en' + +import re + +from calibre import prints +from calibre.utils.logging import default_log as log + +class NumberToText(object): # {{{ + ''' + Converts numbers to text + 4.56 => four point fifty-six + 456 => four hundred fifty-six + 4:56 => four fifty-six + ''' + ORDINALS = ['zeroth','first','second','third','fourth','fifth','sixth','seventh','eighth','ninth'] + lessThanTwenty = ["","one","two","three","four","five","six","seven","eight","nine", + "ten","eleven","twelve","thirteen","fourteen","fifteen","sixteen","seventeen", + "eighteen","nineteen"] + tens = ["","","twenty","thirty","forty","fifty","sixty","seventy","eighty","ninety"] + hundreds = ["","one","two","three","four","five","six","seven","eight","nine"] + + def __init__(self, number, verbose=False): + self.number = number + self.number_as_float = 0.0 + self.text = '' + self.verbose = verbose + self.log = log + self.numberTranslate() + + def stringFromInt(self, intToTranslate): + # Convert intToTranslate to string + # intToTranslate is a three-digit number + + tensComponentString = "" + hundredsComponent = intToTranslate - (intToTranslate % 100) + tensComponent = intToTranslate % 100 + + # Build the hundreds component + if hundredsComponent: + hundredsComponentString = "%s hundred" % self.hundreds[hundredsComponent/100] + else: + hundredsComponentString = "" + + # Build the tens component + if tensComponent < 20: + tensComponentString = self.lessThanTwenty[tensComponent] + else: + tensPart = "" + onesPart = "" + + # Get the tens part + tensPart = self.tens[tensComponent / 10] + onesPart = self.lessThanTwenty[tensComponent % 10] + + if intToTranslate % 10: + tensComponentString = "%s-%s" % (tensPart, onesPart) + else: + tensComponentString = "%s" % tensPart + + # Concatenate the results + result = '' + if hundredsComponent and not tensComponent: + result = hundredsComponentString + elif not hundredsComponent and tensComponent: + result = tensComponentString + elif hundredsComponent and tensComponent: + result = hundredsComponentString + " " + tensComponentString + else: + prints(" NumberToText.stringFromInt(): empty result translating %d" % intToTranslate) + return result + + def numberTranslate(self): + hundredsNumber = 0 + thousandsNumber = 0 + hundredsString = "" + thousandsString = "" + resultString = "" + self.suffix = '' + + if self.verbose: self.log("numberTranslate(): %s" % self.number) + + # Special case ordinals + if re.search('[st|nd|rd|th]',self.number): + self.number = re.sub(',','',self.number) + ordinal_suffix = re.search('[\D]', self.number) + ordinal_number = re.sub('\D','',re.sub(',','',self.number)) + if self.verbose: self.log("Ordinal: %s" % ordinal_number) + self.number_as_float = ordinal_number + self.suffix = self.number[ordinal_suffix.start():] + if int(ordinal_number) > 9: + # Some typos (e.g., 'twentyth'), acceptable + self.text = '%s' % (NumberToText(ordinal_number).text) + else: + self.text = '%s' % (self.ORDINALS[int(ordinal_number)]) + + # Test for time + elif re.search(':',self.number): + if self.verbose: self.log("Time: %s" % self.number) + self.number_as_float = re.sub(':','.',self.number) + time_strings = self.number.split(":") + hours = NumberToText(time_strings[0]).text + minutes = NumberToText(time_strings[1]).text + self.text = '%s-%s' % (hours.capitalize(), minutes) + + # Test for % + elif re.search('%', self.number): + if self.verbose: self.log("Percent: %s" % self.number) + self.number_as_float = self.number.split('%')[0] + self.text = NumberToText(self.number.replace('%',' percent')).text + + # Test for decimal + elif re.search('\.',self.number): + if self.verbose: self.log("Decimal: %s" % self.number) + self.number_as_float = self.number + decimal_strings = self.number.split(".") + left = NumberToText(decimal_strings[0]).text + right = NumberToText(decimal_strings[1]).text + self.text = '%s point %s' % (left.capitalize(), right) + + # Test for hypenated + elif re.search('-', self.number): + if self.verbose: self.log("Hyphenated: %s" % self.number) + self.number_as_float = self.number.split('-')[0] + strings = self.number.split('-') + if re.search('[0-9]+', strings[0]): + left = NumberToText(strings[0]).text + right = strings[1] + else: + left = strings[0] + right = NumberToText(strings[1]).text + self.text = '%s-%s' % (left, right) + + # Test for only commas and numbers + elif re.search(',', self.number) and not re.search('[^0-9,]',self.number): + if self.verbose: self.log("Comma(s): %s" % self.number) + self.number_as_float = re.sub(',','',self.number) + self.text = NumberToText(self.number_as_float).text + + # Test for hybrid e.g., 'K2, 2nd, 10@10' + elif re.search('[\D]+', self.number): + if self.verbose: self.log("Hybrid: %s" % self.number) + # Split the token into number/text + number_position = re.search('\d',self.number).start() + text_position = re.search('\D',self.number).start() + if number_position < text_position: + number = self.number[:text_position] + text = self.number[text_position:] + self.text = '%s%s' % (NumberToText(number).text,text) + else: + text = self.number[:number_position] + number = self.number[number_position:] + self.text = '%s%s' % (text, NumberToText(number).text) + + else: + if self.verbose: self.log("Clean: %s" % self.number) + try: + self.float_as_number = float(self.number) + number = int(self.number) + except: + return + + if number > 10**9: + self.text = "%d out of range" % number + return + + if number == 10**9: + self.text = "one billion" + else : + # Isolate the three-digit number groups + millionsNumber = number/10**6 + thousandsNumber = (number - (millionsNumber * 10**6))/10**3 + hundredsNumber = number - (millionsNumber * 10**6) - (thousandsNumber * 10**3) + if self.verbose: + print "Converting %s %s %s" % (millionsNumber, thousandsNumber, hundredsNumber) + + # Convert hundredsNumber + if hundredsNumber : + hundredsString = self.stringFromInt(hundredsNumber) + + # Convert thousandsNumber + if thousandsNumber: + if number > 1099 and number < 2000: + resultString = '%s %s' % (self.lessThanTwenty[number/100], + self.stringFromInt(number % 100)) + self.text = resultString.strip().capitalize() + return + else: + thousandsString = self.stringFromInt(thousandsNumber) + + # Convert millionsNumber + if millionsNumber: + millionsString = self.stringFromInt(millionsNumber) + + # Concatenate the strings + resultString = '' + if millionsNumber: + resultString += "%s million " % millionsString + + if thousandsNumber: + resultString += "%s thousand " % thousandsString + + if hundredsNumber: + resultString += "%s" % hundredsString + + if not millionsNumber and not thousandsNumber and not hundredsNumber: + resultString = "zero" + + if self.verbose: + self.log(u'resultString: %s' % resultString) + self.text = resultString.strip().capitalize() +# }}} +