From b9927abdae15d64d60fb9d057827da02d67120d3 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 3 Feb 2012 19:10:28 +0530 Subject: [PATCH 01/31] Ensure font family used in comment viewer/editor is the same as for the rest of the application --- src/calibre/gui2/book_details.py | 15 ++++++++++++--- src/calibre/gui2/comments_editor.py | 8 ++++++-- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/src/calibre/gui2/book_details.py b/src/calibre/gui2/book_details.py index 6ad1aaf0c4..a9a0894345 100644 --- a/src/calibre/gui2/book_details.py +++ b/src/calibre/gui2/book_details.py @@ -38,14 +38,23 @@ def render_html(mi, css, vertical, widget, all_fields=False): # {{{ ans = unicode(col.name()) return ans - f = QFontInfo(QApplication.font(widget)).pixelSize() + fi = QFontInfo(QApplication.font(widget)) + f = fi.pixelSize()+1 + fam = unicode(fi.family()).strip() + if not fam: + fam = 'sans-serif' c = color_to_string(QApplication.palette().color(QPalette.Normal, QPalette.WindowText)) templ = u'''\ - - -
- comic page #%d -
- - - ''') - dir = os.path.dirname(pages[0]) - for i, page in enumerate(pages): - wrapper = WRAPPER%(XHTML_NS, i+1, os.path.basename(page), i+1) - page = os.path.join(dir, 'page_%d.xhtml'%(i+1)) - open(page, 'wb').write(wrapper) - wrappers.append(page) - return wrappers diff --git a/src/calibre/ebooks/conversion/plugins/__init__.py b/src/calibre/ebooks/conversion/plugins/__init__.py new file mode 100644 index 0000000000..dd9615356c --- /dev/null +++ b/src/calibre/ebooks/conversion/plugins/__init__.py @@ -0,0 +1,11 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +__license__ = 'GPL v3' +__copyright__ = '2012, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + + + diff --git a/src/calibre/ebooks/azw4/input.py b/src/calibre/ebooks/conversion/plugins/azw4_input.py similarity index 84% rename from src/calibre/ebooks/azw4/input.py rename to src/calibre/ebooks/conversion/plugins/azw4_input.py index 1ac7657342..6d2b2a917e 100644 --- a/src/calibre/ebooks/azw4/input.py +++ b/src/calibre/ebooks/conversion/plugins/azw4_input.py @@ -7,8 +7,6 @@ __docformat__ = 'restructuredtext en' import os from calibre.customize.conversion import InputFormatPlugin -from calibre.ebooks.pdb.header import PdbHeaderReader -from calibre.ebooks.azw4.reader import Reader class AZW4Input(InputFormatPlugin): @@ -19,6 +17,9 @@ class AZW4Input(InputFormatPlugin): def convert(self, stream, options, file_ext, log, accelerators): + from calibre.ebooks.pdb.header import PdbHeaderReader + from calibre.ebooks.azw4.reader import Reader + header = PdbHeaderReader(stream) reader = Reader(header, stream, log, options) opf = reader.extract_content(os.getcwd()) diff --git a/src/calibre/ebooks/chm/input.py b/src/calibre/ebooks/conversion/plugins/chm_input.py similarity index 98% rename from src/calibre/ebooks/chm/input.py rename to src/calibre/ebooks/conversion/plugins/chm_input.py index f36685bd91..a674735f1d 100644 --- a/src/calibre/ebooks/chm/input.py +++ b/src/calibre/ebooks/conversion/plugins/chm_input.py @@ -3,9 +3,7 @@ __license__ = 'GPL v3' __copyright__ = '2008, Kovid Goyal ,' \ ' and Alex Bramley .' -import os, uuid - -from lxml import html +import os from calibre.customize.conversion import InputFormatPlugin from calibre.ptempfile import TemporaryDirectory @@ -77,7 +75,7 @@ class CHMInput(InputFormatPlugin): def _create_oebbook_html(self, htmlpath, basedir, opts, log, mi): # use HTMLInput plugin to generate book - from calibre.ebooks.html.input import HTMLInput + from calibre.customize.builtins import HTMLInput opts.breadth_first = True htmlinput = HTMLInput(None) oeb = htmlinput.create_oebbook(htmlpath, basedir, opts, log, mi) @@ -85,6 +83,8 @@ class CHMInput(InputFormatPlugin): def _create_oebbook(self, hhcpath, basedir, opts, log, mi): + import uuid + from lxml import html from calibre.ebooks.conversion.plumber import create_oebbook from calibre.ebooks.oeb.base import DirContainer oeb = create_oebbook(log, None, opts, @@ -142,6 +142,7 @@ class CHMInput(InputFormatPlugin): return oeb def _create_html_root(self, hhcpath, log): + from lxml import html hhcdata = self._read_file(hhcpath) hhcroot = html.fromstring(hhcdata) chapters = self._process_nodes(hhcroot) diff --git a/src/calibre/ebooks/conversion/plugins/comic_input.py b/src/calibre/ebooks/conversion/plugins/comic_input.py new file mode 100644 index 0000000000..77ae7d8086 --- /dev/null +++ b/src/calibre/ebooks/conversion/plugins/comic_input.py @@ -0,0 +1,259 @@ +from __future__ import with_statement +__license__ = 'GPL v3' +__copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net' +__docformat__ = 'restructuredtext en' + +''' +Based on ideas from comiclrf created by FangornUK. +''' + +import shutil, textwrap, codecs, os + +from calibre.customize.conversion import InputFormatPlugin, OptionRecommendation +from calibre import CurrentDir +from calibre.ptempfile import PersistentTemporaryDirectory + +class ComicInput(InputFormatPlugin): + + name = 'Comic Input' + author = 'Kovid Goyal' + description = 'Optimize comic files (.cbz, .cbr, .cbc) for viewing on portable devices' + file_types = set(['cbz', 'cbr', 'cbc']) + is_image_collection = True + core_usage = -1 + + options = set([ + OptionRecommendation(name='colors', recommended_value=256, + help=_('Number of colors for grayscale image conversion. Default: ' + '%default. Values of less than 256 may result in blurred text ' + 'on your device if you are creating your comics in EPUB format.')), + OptionRecommendation(name='dont_normalize', recommended_value=False, + help=_('Disable normalize (improve contrast) color range ' + 'for pictures. Default: False')), + OptionRecommendation(name='keep_aspect_ratio', recommended_value=False, + help=_('Maintain picture aspect ratio. Default is to fill the screen.')), + OptionRecommendation(name='dont_sharpen', recommended_value=False, + help=_('Disable sharpening.')), + OptionRecommendation(name='disable_trim', recommended_value=False, + help=_('Disable trimming of comic pages. For some comics, ' + 'trimming might remove content as well as borders.')), + OptionRecommendation(name='landscape', recommended_value=False, + help=_("Don't split landscape images into two portrait images")), + OptionRecommendation(name='wide', recommended_value=False, + help=_("Keep aspect ratio and scale image using screen height as " + "image width for viewing in landscape mode.")), + OptionRecommendation(name='right2left', recommended_value=False, + help=_('Used for right-to-left publications like manga. ' + 'Causes landscape pages to be split into portrait pages ' + 'from right to left.')), + OptionRecommendation(name='despeckle', recommended_value=False, + help=_('Enable Despeckle. Reduces speckle noise. ' + 'May greatly increase processing time.')), + OptionRecommendation(name='no_sort', recommended_value=False, + help=_("Don't sort the files found in the comic " + "alphabetically by name. Instead use the order they were " + "added to the comic.")), + OptionRecommendation(name='output_format', choices=['png', 'jpg'], + recommended_value='png', help=_('The format that images in the created ebook ' + 'are converted to. You can experiment to see which format gives ' + 'you optimal size and look on your device.')), + OptionRecommendation(name='no_process', recommended_value=False, + help=_("Apply no processing to the image")), + OptionRecommendation(name='dont_grayscale', recommended_value=False, + help=_('Do not convert the image to grayscale (black and white)')), + OptionRecommendation(name='comic_image_size', recommended_value=None, + help=_('Specify the image size as widthxheight pixels. Normally,' + ' an image size is automatically calculated from the output ' + 'profile, this option overrides it.')), + OptionRecommendation(name='dont_add_comic_pages_to_toc', recommended_value=False, + help=_('When converting a CBC do not add links to each page to' + ' the TOC. Note this only applies if the TOC has more than one' + ' section')), + ]) + + recommendations = set([ + ('margin_left', 0, OptionRecommendation.HIGH), + ('margin_top', 0, OptionRecommendation.HIGH), + ('margin_right', 0, OptionRecommendation.HIGH), + ('margin_bottom', 0, OptionRecommendation.HIGH), + ('insert_blank_line', False, OptionRecommendation.HIGH), + ('remove_paragraph_spacing', False, OptionRecommendation.HIGH), + ('change_justification', 'left', OptionRecommendation.HIGH), + ('dont_split_on_pagebreaks', True, OptionRecommendation.HIGH), + ('chapter', None, OptionRecommendation.HIGH), + ('page_breaks_brefore', None, OptionRecommendation.HIGH), + ('use_auto_toc', False, OptionRecommendation.HIGH), + ('page_breaks_before', None, OptionRecommendation.HIGH), + ('disable_font_rescaling', True, OptionRecommendation.HIGH), + ('linearize_tables', False, OptionRecommendation.HIGH), + ]) + + def get_comics_from_collection(self, stream): + from calibre.libunzip import extract as zipextract + tdir = PersistentTemporaryDirectory('_comic_collection') + zipextract(stream, tdir) + comics = [] + with CurrentDir(tdir): + if not os.path.exists('comics.txt'): + raise ValueError(( + '%s is not a valid comic collection' + ' no comics.txt was found in the file') + %stream.name) + raw = open('comics.txt', 'rb').read() + if raw.startswith(codecs.BOM_UTF16_BE): + raw = raw.decode('utf-16-be')[1:] + elif raw.startswith(codecs.BOM_UTF16_LE): + raw = raw.decode('utf-16-le')[1:] + elif raw.startswith(codecs.BOM_UTF8): + raw = raw.decode('utf-8')[1:] + else: + raw = raw.decode('utf-8') + for line in raw.splitlines(): + line = line.strip() + if not line: + continue + fname, title = line.partition(':')[0], line.partition(':')[-1] + fname = fname.replace('#', '_') + fname = os.path.join(tdir, *fname.split('/')) + if not title: + title = os.path.basename(fname).rpartition('.')[0] + if os.access(fname, os.R_OK): + comics.append([title, fname]) + if not comics: + raise ValueError('%s has no comics'%stream.name) + return comics + + def get_pages(self, comic, tdir2): + from calibre.ebooks.comic.input import (extract_comic, process_pages, + find_pages) + tdir = extract_comic(comic) + new_pages = find_pages(tdir, sort_on_mtime=self.opts.no_sort, + verbose=self.opts.verbose) + thumbnail = None + if not new_pages: + raise ValueError('Could not find any pages in the comic: %s' + %comic) + if self.opts.no_process: + n2 = [] + for page in new_pages: + n2.append(os.path.join(tdir2, os.path.basename(page))) + shutil.copyfile(page, n2[-1]) + new_pages = n2 + else: + new_pages, failures = process_pages(new_pages, self.opts, + self.report_progress, tdir2) + if failures: + self.log.warning('Could not process the following pages ' + '(run with --verbose to see why):') + for f in failures: + self.log.warning('\t', f) + if not new_pages: + raise ValueError('Could not find any valid pages in comic: %s' + % comic) + thumbnail = os.path.join(tdir2, + 'thumbnail.'+self.opts.output_format.lower()) + if not os.access(thumbnail, os.R_OK): + thumbnail = None + return new_pages + + def get_images(self): + return self._images + + def convert(self, stream, opts, file_ext, log, accelerators): + from calibre.ebooks.metadata import MetaInformation + from calibre.ebooks.metadata.opf2 import OPFCreator + from calibre.ebooks.metadata.toc import TOC + + self.opts, self.log= opts, log + if file_ext == 'cbc': + comics_ = self.get_comics_from_collection(stream) + else: + comics_ = [['Comic', os.path.abspath(stream.name)]] + stream.close() + comics = [] + for i, x in enumerate(comics_): + title, fname = x + cdir = 'comic_%d'%(i+1) if len(comics_) > 1 else '.' + cdir = os.path.abspath(cdir) + if not os.path.exists(cdir): + os.makedirs(cdir) + pages = self.get_pages(fname, cdir) + if not pages: continue + wrappers = self.create_wrappers(pages) + comics.append((title, pages, wrappers)) + + if not comics: + raise ValueError('No comic pages found in %s'%stream.name) + + mi = MetaInformation(os.path.basename(stream.name).rpartition('.')[0], + [_('Unknown')]) + opf = OPFCreator(os.path.abspath('.'), mi) + entries = [] + + def href(x): + if len(comics) == 1: return os.path.basename(x) + return '/'.join(x.split(os.sep)[-2:]) + + for comic in comics: + pages, wrappers = comic[1:] + entries += [(w, None) for w in map(href, wrappers)] + \ + [(x, None) for x in map(href, pages)] + opf.create_manifest(entries) + spine = [] + for comic in comics: + spine.extend(map(href, comic[2])) + self._images = [] + for comic in comics: + self._images.extend(comic[1]) + opf.create_spine(spine) + toc = TOC() + if len(comics) == 1: + wrappers = comics[0][2] + for i, x in enumerate(wrappers): + toc.add_item(href(x), None, _('Page')+' %d'%(i+1), + play_order=i) + else: + po = 0 + for comic in comics: + po += 1 + wrappers = comic[2] + stoc = toc.add_item(href(wrappers[0]), + None, comic[0], play_order=po) + if not opts.dont_add_comic_pages_to_toc: + for i, x in enumerate(wrappers): + stoc.add_item(href(x), None, + _('Page')+' %d'%(i+1), play_order=po) + po += 1 + opf.set_toc(toc) + m, n = open('metadata.opf', 'wb'), open('toc.ncx', 'wb') + opf.render(m, n, 'toc.ncx') + return os.path.abspath('metadata.opf') + + def create_wrappers(self, pages): + from calibre.ebooks.oeb.base import XHTML_NS + wrappers = [] + WRAPPER = textwrap.dedent('''\ + + + Page #%d + + + +
+ comic page #%d +
+ + + ''') + dir = os.path.dirname(pages[0]) + for i, page in enumerate(pages): + wrapper = WRAPPER%(XHTML_NS, i+1, os.path.basename(page), i+1) + page = os.path.join(dir, 'page_%d.xhtml'%(i+1)) + open(page, 'wb').write(wrapper) + wrappers.append(page) + return wrappers + diff --git a/src/calibre/ebooks/djvu/input.py b/src/calibre/ebooks/conversion/plugins/djvu_input.py similarity index 98% rename from src/calibre/ebooks/djvu/input.py rename to src/calibre/ebooks/conversion/plugins/djvu_input.py index 70dbf97f5d..936ef1a702 100644 --- a/src/calibre/ebooks/djvu/input.py +++ b/src/calibre/ebooks/conversion/plugins/djvu_input.py @@ -12,7 +12,6 @@ from subprocess import Popen, PIPE from cStringIO import StringIO from calibre.customize.conversion import InputFormatPlugin, OptionRecommendation -from calibre.ebooks.txt.processor import convert_basic class DJVUInput(InputFormatPlugin): @@ -28,6 +27,8 @@ class DJVUInput(InputFormatPlugin): ]) def convert(self, stream, options, file_ext, log, accelerators): + from calibre.ebooks.txt.processor import convert_basic + stdout = StringIO() ppdjvu = True # using djvutxt is MUCH faster, should make it an option diff --git a/src/calibre/ebooks/epub/input.py b/src/calibre/ebooks/conversion/plugins/epub_input.py similarity index 98% rename from src/calibre/ebooks/epub/input.py rename to src/calibre/ebooks/conversion/plugins/epub_input.py index c2cfedd7d4..47356dbd1f 100644 --- a/src/calibre/ebooks/epub/input.py +++ b/src/calibre/ebooks/conversion/plugins/epub_input.py @@ -3,11 +3,9 @@ __license__ = 'GPL 3' __copyright__ = '2009, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import os, uuid +import os from itertools import cycle -from lxml import etree - from calibre.customize.conversion import InputFormatPlugin, OptionRecommendation class EPUBInput(InputFormatPlugin): @@ -30,6 +28,8 @@ class EPUBInput(InputFormatPlugin): f.write(raw[1024:]) def process_encryption(self, encfile, opf, log): + from lxml import etree + import uuid key = None for item in opf.identifier_iter(): scheme = None @@ -65,6 +65,7 @@ class EPUBInput(InputFormatPlugin): return False def rationalize_cover(self, opf, log): + from lxml import etree guide_cover, guide_elem = None, None for guide_elem in opf.iterguide(): if guide_elem.get('type', '').lower() == 'cover': @@ -110,6 +111,7 @@ class EPUBInput(InputFormatPlugin): renderer) def find_opf(self): + from lxml import etree def attr(n, attr): for k, v in n.attrib.items(): if k.endswith(attr): diff --git a/src/calibre/ebooks/fb2/input.py b/src/calibre/ebooks/conversion/plugins/fb2_input.py similarity index 99% rename from src/calibre/ebooks/fb2/input.py rename to src/calibre/ebooks/conversion/plugins/fb2_input.py index 147e940eb4..747f8f19d8 100644 --- a/src/calibre/ebooks/fb2/input.py +++ b/src/calibre/ebooks/conversion/plugins/fb2_input.py @@ -6,7 +6,6 @@ Convert .fb2 files to .lrf """ import os, re from base64 import b64decode -from lxml import etree from calibre.customize.conversion import InputFormatPlugin, OptionRecommendation from calibre import guess_type @@ -38,6 +37,7 @@ class FB2Input(InputFormatPlugin): def convert(self, stream, options, file_ext, log, accelerators): + from lxml import etree from calibre.ebooks.metadata.opf2 import OPFCreator from calibre.ebooks.metadata.meta import get_metadata from calibre.ebooks.oeb.base import XLINK_NS, XHTML_NS, RECOVER_PARSER diff --git a/src/calibre/ebooks/conversion/plugins/html_input.py b/src/calibre/ebooks/conversion/plugins/html_input.py new file mode 100644 index 0000000000..cfd2ebf8cf --- /dev/null +++ b/src/calibre/ebooks/conversion/plugins/html_input.py @@ -0,0 +1,283 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +__license__ = 'GPL v3' +__copyright__ = '2012, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + +import re, tempfile, os +from functools import partial +from itertools import izip +from urllib import quote + +from calibre.constants import islinux, isbsd +from calibre.customize.conversion import (InputFormatPlugin, + OptionRecommendation) +from calibre.utils.localization import get_lang +from calibre.utils.filenames import ascii_filename + + +class HTMLInput(InputFormatPlugin): + + name = 'HTML Input' + author = 'Kovid Goyal' + description = 'Convert HTML and OPF files to an OEB' + file_types = set(['opf', 'html', 'htm', 'xhtml', 'xhtm', 'shtm', 'shtml']) + + options = set([ + OptionRecommendation(name='breadth_first', + recommended_value=False, level=OptionRecommendation.LOW, + help=_('Traverse links in HTML files breadth first. Normally, ' + 'they are traversed depth first.' + ) + ), + + OptionRecommendation(name='max_levels', + recommended_value=5, level=OptionRecommendation.LOW, + help=_('Maximum levels of recursion when following links in ' + 'HTML files. Must be non-negative. 0 implies that no ' + 'links in the root HTML file are followed. Default is ' + '%default.' + ) + ), + + OptionRecommendation(name='dont_package', + recommended_value=False, level=OptionRecommendation.LOW, + help=_('Normally this input plugin re-arranges all the input ' + 'files into a standard folder hierarchy. Only use this option ' + 'if you know what you are doing as it can result in various ' + 'nasty side effects in the rest of the conversion pipeline.' + ) + ), + + ]) + + def convert(self, stream, opts, file_ext, log, + accelerators): + self._is_case_sensitive = None + basedir = os.getcwd() + self.opts = opts + + fname = None + if hasattr(stream, 'name'): + basedir = os.path.dirname(stream.name) + fname = os.path.basename(stream.name) + + if file_ext != 'opf': + if opts.dont_package: + raise ValueError('The --dont-package option is not supported for an HTML input file') + from calibre.ebooks.metadata.html import get_metadata + mi = get_metadata(stream) + if fname: + from calibre.ebooks.metadata.meta import metadata_from_filename + fmi = metadata_from_filename(fname) + fmi.smart_update(mi) + mi = fmi + oeb = self.create_oebbook(stream.name, basedir, opts, log, mi) + return oeb + + from calibre.ebooks.conversion.plumber import create_oebbook + return create_oebbook(log, stream.name, opts, + encoding=opts.input_encoding) + + def is_case_sensitive(self, path): + if getattr(self, '_is_case_sensitive', None) is not None: + return self._is_case_sensitive + if not path or not os.path.exists(path): + return islinux or isbsd + self._is_case_sensitive = not (os.path.exists(path.lower()) \ + and os.path.exists(path.upper())) + return self._is_case_sensitive + + def create_oebbook(self, htmlpath, basedir, opts, log, mi): + import uuid + from calibre.ebooks.conversion.plumber import create_oebbook + from calibre.ebooks.oeb.base import (DirContainer, + rewrite_links, urlnormalize, urldefrag, BINARY_MIME, OEB_STYLES, + xpath) + from calibre import guess_type + from calibre.ebooks.oeb.transforms.metadata import \ + meta_info_to_oeb_metadata + from calibre.ebooks.html.input import get_filelist + import cssutils, logging + cssutils.log.setLevel(logging.WARN) + self.OEB_STYLES = OEB_STYLES + oeb = create_oebbook(log, None, opts, self, + encoding=opts.input_encoding, populate=False) + self.oeb = oeb + + metadata = oeb.metadata + meta_info_to_oeb_metadata(mi, metadata, log) + if not metadata.language: + oeb.logger.warn(u'Language not specified') + metadata.add('language', get_lang().replace('_', '-')) + if not metadata.creator: + oeb.logger.warn('Creator not specified') + metadata.add('creator', self.oeb.translate(__('Unknown'))) + if not metadata.title: + oeb.logger.warn('Title not specified') + metadata.add('title', self.oeb.translate(__('Unknown'))) + bookid = str(uuid.uuid4()) + metadata.add('identifier', bookid, id='uuid_id', scheme='uuid') + for ident in metadata.identifier: + if 'id' in ident.attrib: + self.oeb.uid = metadata.identifier[0] + break + + filelist = get_filelist(htmlpath, basedir, opts, log) + filelist = [f for f in filelist if not f.is_binary] + htmlfile_map = {} + for f in filelist: + path = f.path + oeb.container = DirContainer(os.path.dirname(path), log, + ignore_opf=True) + bname = os.path.basename(path) + id, href = oeb.manifest.generate(id='html', + href=ascii_filename(bname)) + htmlfile_map[path] = href + item = oeb.manifest.add(id, href, 'text/html') + item.html_input_href = bname + oeb.spine.add(item, True) + + self.added_resources = {} + self.log = log + self.log('Normalizing filename cases') + for path, href in htmlfile_map.items(): + if not self.is_case_sensitive(path): + path = path.lower() + self.added_resources[path] = href + self.urlnormalize, self.DirContainer = urlnormalize, DirContainer + self.urldefrag = urldefrag + self.guess_type, self.BINARY_MIME = guess_type, BINARY_MIME + + self.log('Rewriting HTML links') + for f in filelist: + path = f.path + dpath = os.path.dirname(path) + oeb.container = DirContainer(dpath, log, ignore_opf=True) + item = oeb.manifest.hrefs[htmlfile_map[path]] + rewrite_links(item.data, partial(self.resource_adder, base=dpath)) + + for item in oeb.manifest.values(): + if item.media_type in self.OEB_STYLES: + dpath = None + for path, href in self.added_resources.items(): + if href == item.href: + dpath = os.path.dirname(path) + break + cssutils.replaceUrls(item.data, + partial(self.resource_adder, base=dpath)) + + toc = self.oeb.toc + self.oeb.auto_generated_toc = True + titles = [] + headers = [] + for item in self.oeb.spine: + if not item.linear: continue + html = item.data + title = ''.join(xpath(html, '/h:html/h:head/h:title/text()')) + title = re.sub(r'\s+', ' ', title.strip()) + if title: + titles.append(title) + headers.append('(unlabled)') + for tag in ('h1', 'h2', 'h3', 'h4', 'h5', 'strong'): + expr = '/h:html/h:body//h:%s[position()=1]/text()' + header = ''.join(xpath(html, expr % tag)) + header = re.sub(r'\s+', ' ', header.strip()) + if header: + headers[-1] = header + break + use = titles + if len(titles) > len(set(titles)): + use = headers + for title, item in izip(use, self.oeb.spine): + if not item.linear: continue + toc.add(title, item.href) + + oeb.container = DirContainer(os.getcwdu(), oeb.log, ignore_opf=True) + return oeb + + def link_to_local_path(self, link_, base=None): + from calibre.ebooks.html.input import Link + if not isinstance(link_, unicode): + try: + link_ = link_.decode('utf-8', 'error') + except: + self.log.warn('Failed to decode link %r. Ignoring'%link_) + return None, None + try: + l = Link(link_, base if base else os.getcwdu()) + except: + self.log.exception('Failed to process link: %r'%link_) + return None, None + if l.path is None: + # Not a local resource + return None, None + link = l.path.replace('/', os.sep).strip() + frag = l.fragment + if not link: + return None, None + return link, frag + + def resource_adder(self, link_, base=None): + link, frag = self.link_to_local_path(link_, base=base) + if link is None: + return link_ + try: + if base and not os.path.isabs(link): + link = os.path.join(base, link) + link = os.path.abspath(link) + except: + return link_ + if not os.access(link, os.R_OK): + return link_ + if os.path.isdir(link): + self.log.warn(link_, 'is a link to a directory. Ignoring.') + return link_ + if not self.is_case_sensitive(tempfile.gettempdir()): + link = link.lower() + if link not in self.added_resources: + bhref = os.path.basename(link) + id, href = self.oeb.manifest.generate(id='added', + href=bhref) + guessed = self.guess_type(href)[0] + media_type = guessed or self.BINARY_MIME + if media_type == 'text/plain': + self.log.warn('Ignoring link to text file %r'%link_) + return None + + self.oeb.log.debug('Added', link) + self.oeb.container = self.DirContainer(os.path.dirname(link), + self.oeb.log, ignore_opf=True) + # Load into memory + item = self.oeb.manifest.add(id, href, media_type) + # bhref refers to an already existing file. The read() method of + # DirContainer will call unquote on it before trying to read the + # file, therefore we quote it here. + if isinstance(bhref, unicode): + bhref = bhref.encode('utf-8') + item.html_input_href = quote(bhref).decode('utf-8') + if guessed in self.OEB_STYLES: + item.override_css_fetch = partial( + self.css_import_handler, os.path.dirname(link)) + item.data + self.added_resources[link] = href + + nlink = self.added_resources[link] + if frag: + nlink = '#'.join((nlink, frag)) + return nlink + + def css_import_handler(self, base, href): + link, frag = self.link_to_local_path(href, base=base) + if link is None or not os.access(link, os.R_OK) or os.path.isdir(link): + return (None, None) + try: + raw = open(link, 'rb').read().decode('utf-8', 'replace') + raw = self.oeb.css_preprocessor(raw, add_namespace=True) + except: + self.log.exception('Failed to read CSS file: %r'%link) + return (None, None) + return (None, raw) diff --git a/src/calibre/ebooks/htmlz/input.py b/src/calibre/ebooks/conversion/plugins/htmlz_input.py similarity index 96% rename from src/calibre/ebooks/htmlz/input.py rename to src/calibre/ebooks/conversion/plugins/htmlz_input.py index f0f45f72fe..e9fbb1d7c2 100644 --- a/src/calibre/ebooks/htmlz/input.py +++ b/src/calibre/ebooks/conversion/plugins/htmlz_input.py @@ -10,9 +10,6 @@ import os from calibre import guess_type from calibre.customize.conversion import InputFormatPlugin -from calibre.ebooks.chardet import xml_to_unicode -from calibre.ebooks.metadata.opf2 import OPF -from calibre.utils.zipfile import ZipFile class HTMLZInput(InputFormatPlugin): @@ -23,6 +20,10 @@ class HTMLZInput(InputFormatPlugin): def convert(self, stream, options, file_ext, log, accelerators): + from calibre.ebooks.chardet import xml_to_unicode + from calibre.ebooks.metadata.opf2 import OPF + from calibre.utils.zipfile import ZipFile + self.log = log html = u'' top_levels = [] diff --git a/src/calibre/ebooks/lit/input.py b/src/calibre/ebooks/conversion/plugins/lit_input.py similarity index 100% rename from src/calibre/ebooks/lit/input.py rename to src/calibre/ebooks/conversion/plugins/lit_input.py diff --git a/src/calibre/ebooks/conversion/plugins/lrf_input.py b/src/calibre/ebooks/conversion/plugins/lrf_input.py new file mode 100644 index 0000000000..63af39e1e0 --- /dev/null +++ b/src/calibre/ebooks/conversion/plugins/lrf_input.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +from __future__ import with_statement + +__license__ = 'GPL v3' +__copyright__ = '2009, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + +import os, sys +from calibre.customize.conversion import InputFormatPlugin + +class LRFInput(InputFormatPlugin): + + name = 'LRF Input' + author = 'Kovid Goyal' + description = 'Convert LRF files to HTML' + file_types = set(['lrf']) + + def convert(self, stream, options, file_ext, log, + accelerators): + from lxml import etree + from calibre.ebooks.lrf.input import (MediaType, Styles, TextBlock, + Canvas, ImageBlock, RuledLine) + self.log = log + self.log('Generating XML') + from calibre.ebooks.lrf.lrfparser import LRFDocument + d = LRFDocument(stream) + d.parse() + xml = d.to_xml(write_files=True) + if options.verbose > 2: + open('lrs.xml', 'wb').write(xml.encode('utf-8')) + parser = etree.XMLParser(no_network=True, huge_tree=True) + try: + doc = etree.fromstring(xml, parser=parser) + except: + self.log.warn('Failed to parse XML. Trying to recover') + parser = etree.XMLParser(no_network=True, huge_tree=True, + recover=True) + doc = etree.fromstring(xml, parser=parser) + + + char_button_map = {} + for x in doc.xpath('//CharButton[@refobj]'): + ro = x.get('refobj') + jump_button = doc.xpath('//*[@objid="%s"]'%ro) + if jump_button: + jump_to = jump_button[0].xpath('descendant::JumpTo[@refpage and @refobj]') + if jump_to: + char_button_map[ro] = '%s.xhtml#%s'%(jump_to[0].get('refpage'), + jump_to[0].get('refobj')) + plot_map = {} + for x in doc.xpath('//Plot[@refobj]'): + ro = x.get('refobj') + image = doc.xpath('//Image[@objid="%s" and @refstream]'%ro) + if image: + imgstr = doc.xpath('//ImageStream[@objid="%s" and @file]'% + image[0].get('refstream')) + if imgstr: + plot_map[ro] = imgstr[0].get('file') + + self.log('Converting XML to HTML...') + styledoc = etree.fromstring(P('templates/lrf.xsl', data=True)) + media_type = MediaType() + styles = Styles() + text_block = TextBlock(styles, char_button_map, plot_map, log) + canvas = Canvas(doc, styles, text_block, log) + image_block = ImageBlock(canvas) + ruled_line = RuledLine() + extensions = { + ('calibre', 'media-type') : media_type, + ('calibre', 'text-block') : text_block, + ('calibre', 'ruled-line') : ruled_line, + ('calibre', 'styles') : styles, + ('calibre', 'canvas') : canvas, + ('calibre', 'image-block'): image_block, + } + transform = etree.XSLT(styledoc, extensions=extensions) + try: + result = transform(doc) + except RuntimeError: + sys.setrecursionlimit(5000) + result = transform(doc) + + with open('content.opf', 'wb') as f: + f.write(result) + styles.write() + return os.path.abspath('content.opf') diff --git a/src/calibre/ebooks/mobi/input.py b/src/calibre/ebooks/conversion/plugins/mobi_input.py similarity index 100% rename from src/calibre/ebooks/mobi/input.py rename to src/calibre/ebooks/conversion/plugins/mobi_input.py diff --git a/src/calibre/ebooks/conversion/plugins/odt_input.py b/src/calibre/ebooks/conversion/plugins/odt_input.py new file mode 100644 index 0000000000..5e92ea5163 --- /dev/null +++ b/src/calibre/ebooks/conversion/plugins/odt_input.py @@ -0,0 +1,25 @@ +from __future__ import with_statement +__license__ = 'GPL v3' +__copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net' +__docformat__ = 'restructuredtext en' + +''' +Convert an ODT file into a Open Ebook +''' + +from calibre.customize.conversion import InputFormatPlugin + +class ODTInput(InputFormatPlugin): + + name = 'ODT Input' + author = 'Kovid Goyal' + description = 'Convert ODT (OpenOffice) files to HTML' + file_types = set(['odt']) + + + def convert(self, stream, options, file_ext, log, + accelerators): + from calibre.ebooks.odt.input import Extract + return Extract()(stream, '.', log) + + diff --git a/src/calibre/ebooks/pdb/input.py b/src/calibre/ebooks/conversion/plugins/pdb_input.py similarity index 87% rename from src/calibre/ebooks/pdb/input.py rename to src/calibre/ebooks/conversion/plugins/pdb_input.py index cd861216af..69984ab268 100644 --- a/src/calibre/ebooks/pdb/input.py +++ b/src/calibre/ebooks/conversion/plugins/pdb_input.py @@ -7,8 +7,6 @@ __docformat__ = 'restructuredtext en' import os from calibre.customize.conversion import InputFormatPlugin -from calibre.ebooks.pdb.header import PdbHeaderReader -from calibre.ebooks.pdb import PDBError, IDENTITY_TO_NAME, get_reader class PDBInput(InputFormatPlugin): @@ -19,6 +17,9 @@ class PDBInput(InputFormatPlugin): def convert(self, stream, options, file_ext, log, accelerators): + from calibre.ebooks.pdb.header import PdbHeaderReader + from calibre.ebooks.pdb import PDBError, IDENTITY_TO_NAME, get_reader + header = PdbHeaderReader(stream) Reader = get_reader(header.ident) diff --git a/src/calibre/ebooks/pdf/input.py b/src/calibre/ebooks/conversion/plugins/pdf_input.py similarity index 95% rename from src/calibre/ebooks/pdf/input.py rename to src/calibre/ebooks/conversion/plugins/pdf_input.py index 51f44ba502..0a3821c584 100644 --- a/src/calibre/ebooks/pdf/input.py +++ b/src/calibre/ebooks/conversion/plugins/pdf_input.py @@ -7,8 +7,6 @@ __docformat__ = 'restructuredtext en' import os from calibre.customize.conversion import InputFormatPlugin, OptionRecommendation -from calibre.ebooks.pdf.pdftohtml import pdftohtml -from calibre.ebooks.metadata.opf2 import OPFCreator from calibre.constants import plugins pdfreflow, pdfreflow_err = plugins['pdfreflow'] @@ -43,6 +41,9 @@ class PDFInput(InputFormatPlugin): def convert(self, stream, options, file_ext, log, accelerators): + from calibre.ebooks.metadata.opf2 import OPFCreator + from calibre.ebooks.pdf.pdftohtml import pdftohtml + log.debug('Converting file to html...') # The main html file will be named index.html self.opts, self.log = options, log diff --git a/src/calibre/ebooks/pml/input.py b/src/calibre/ebooks/conversion/plugins/pml_input.py similarity index 96% rename from src/calibre/ebooks/pml/input.py rename to src/calibre/ebooks/conversion/plugins/pml_input.py index 4d59668b12..1351a5c492 100644 --- a/src/calibre/ebooks/pml/input.py +++ b/src/calibre/ebooks/conversion/plugins/pml_input.py @@ -11,9 +11,6 @@ import shutil from calibre.customize.conversion import InputFormatPlugin from calibre.ptempfile import TemporaryDirectory from calibre.utils.zipfile import ZipFile -from calibre.ebooks.pml.pmlconverter import PML_HTMLizer -from calibre.ebooks.metadata.toc import TOC -from calibre.ebooks.metadata.opf2 import OPFCreator class PMLInput(InputFormatPlugin): @@ -24,6 +21,8 @@ class PMLInput(InputFormatPlugin): file_types = set(['pml', 'pmlz']) def process_pml(self, pml_path, html_path, close_all=False): + from calibre.ebooks.pml.pmlconverter import PML_HTMLizer + pclose = False hclose = False @@ -85,6 +84,9 @@ class PMLInput(InputFormatPlugin): def convert(self, stream, options, file_ext, log, accelerators): + from calibre.ebooks.metadata.toc import TOC + from calibre.ebooks.metadata.opf2 import OPFCreator + self.options = options self.log = log pages, images = [], [] diff --git a/src/calibre/ebooks/rb/input.py b/src/calibre/ebooks/conversion/plugins/rb_input.py similarity index 91% rename from src/calibre/ebooks/rb/input.py rename to src/calibre/ebooks/conversion/plugins/rb_input.py index 8b05c1d42e..6a6ca3205a 100644 --- a/src/calibre/ebooks/rb/input.py +++ b/src/calibre/ebooks/conversion/plugins/rb_input.py @@ -6,7 +6,6 @@ __docformat__ = 'restructuredtext en' import os -from calibre.ebooks.rb.reader import Reader from calibre.customize.conversion import InputFormatPlugin class RBInput(InputFormatPlugin): @@ -18,6 +17,8 @@ class RBInput(InputFormatPlugin): def convert(self, stream, options, file_ext, log, accelerators): + from calibre.ebooks.rb.reader import Reader + reader = Reader(stream, log, options.input_encoding) opf = reader.extract_content(os.getcwd()) diff --git a/src/calibre/web/feeds/input.py b/src/calibre/ebooks/conversion/plugins/recipe_input.py similarity index 100% rename from src/calibre/web/feeds/input.py rename to src/calibre/ebooks/conversion/plugins/recipe_input.py diff --git a/src/calibre/ebooks/conversion/plugins/rtf_input.py b/src/calibre/ebooks/conversion/plugins/rtf_input.py new file mode 100644 index 0000000000..91c285c10c --- /dev/null +++ b/src/calibre/ebooks/conversion/plugins/rtf_input.py @@ -0,0 +1,298 @@ +from __future__ import with_statement +__license__ = 'GPL v3' +__copyright__ = '2008, Kovid Goyal ' + +import os, glob, re, textwrap + +from calibre.customize.conversion import InputFormatPlugin + +border_style_map = { + 'single' : 'solid', + 'double-thickness-border' : 'double', + 'shadowed-border': 'outset', + 'double-border': 'double', + 'dotted-border': 'dotted', + 'dashed': 'dashed', + 'hairline': 'solid', + 'inset': 'inset', + 'dash-small': 'dashed', + 'dot-dash': 'dotted', + 'dot-dot-dash': 'dotted', + 'outset': 'outset', + 'tripple': 'double', + 'triple': 'double', + 'thick-thin-small': 'solid', + 'thin-thick-small': 'solid', + 'thin-thick-thin-small': 'solid', + 'thick-thin-medium': 'solid', + 'thin-thick-medium': 'solid', + 'thin-thick-thin-medium': 'solid', + 'thick-thin-large': 'solid', + 'thin-thick-thin-large': 'solid', + 'wavy': 'ridge', + 'double-wavy': 'ridge', + 'striped': 'ridge', + 'emboss': 'inset', + 'engrave': 'inset', + 'frame': 'ridge', +} + + +class RTFInput(InputFormatPlugin): + + name = 'RTF Input' + author = 'Kovid Goyal' + description = 'Convert RTF files to HTML' + file_types = set(['rtf']) + + def generate_xml(self, stream): + from calibre.ebooks.rtf2xml.ParseRtf import ParseRtf + ofile = 'dataxml.xml' + run_lev, debug_dir, indent_out = 1, None, 0 + if getattr(self.opts, 'debug_pipeline', None) is not None: + try: + os.mkdir('rtfdebug') + debug_dir = 'rtfdebug' + run_lev = 4 + indent_out = 1 + self.log('Running RTFParser in debug mode') + except: + self.log.warn('Impossible to run RTFParser in debug mode') + parser = ParseRtf( + in_file = stream, + out_file = ofile, + # Convert symbol fonts to unicode equivalents. Default + # is 1 + convert_symbol = 1, + + # Convert Zapf fonts to unicode equivalents. Default + # is 1. + convert_zapf = 1, + + # Convert Wingding fonts to unicode equivalents. + # Default is 1. + convert_wingdings = 1, + + # Convert RTF caps to real caps. + # Default is 1. + convert_caps = 1, + + # Indent resulting XML. + # Default is 0 (no indent). + indent = indent_out, + + # Form lists from RTF. Default is 1. + form_lists = 1, + + # Convert headings to sections. Default is 0. + headings_to_sections = 1, + + # Group paragraphs with the same style name. Default is 1. + group_styles = 1, + + # Group borders. Default is 1. + group_borders = 1, + + # Write or do not write paragraphs. Default is 0. + empty_paragraphs = 1, + + #debug + deb_dir = debug_dir, + run_level = run_lev, + ) + parser.parse_rtf() + with open(ofile, 'rb') as f: + return f.read() + + def extract_images(self, picts): + import imghdr + self.log('Extracting images...') + + with open(picts, 'rb') as f: + raw = f.read() + picts = filter(len, re.findall(r'\{\\pict([^}]+)\}', raw)) + hex = re.compile(r'[^a-fA-F0-9]') + encs = [hex.sub('', pict) for pict in picts] + + count = 0 + imap = {} + for enc in encs: + if len(enc) % 2 == 1: + enc = enc[:-1] + data = enc.decode('hex') + fmt = imghdr.what(None, data) + if fmt is None: + fmt = 'wmf' + count += 1 + name = '%04d.%s' % (count, fmt) + with open(name, 'wb') as f: + f.write(data) + imap[count] = name + # with open(name+'.hex', 'wb') as f: + # f.write(enc) + return self.convert_images(imap) + + def convert_images(self, imap): + self.default_img = None + for count, val in imap.iteritems(): + try: + imap[count] = self.convert_image(val) + except: + self.log.exception('Failed to convert', val) + return imap + + def convert_image(self, name): + if not name.endswith('.wmf'): + return name + try: + return self.rasterize_wmf(name) + except: + self.log.exception('Failed to convert WMF image %r'%name) + return self.replace_wmf(name) + + def replace_wmf(self, name): + from calibre.ebooks import calibre_cover + if self.default_img is None: + self.default_img = calibre_cover('Conversion of WMF images is not supported', + 'Use Microsoft Word or OpenOffice to save this RTF file' + ' as HTML and convert that in calibre.', title_size=36, + author_size=20) + name = name.replace('.wmf', '.jpg') + with open(name, 'wb') as f: + f.write(self.default_img) + return name + + def rasterize_wmf(self, name): + from calibre.utils.wmf.parse import wmf_unwrap + with open(name, 'rb') as f: + data = f.read() + data = wmf_unwrap(data) + name = name.replace('.wmf', '.png') + with open(name, 'wb') as f: + f.write(data) + return name + + + def write_inline_css(self, ic, border_styles): + font_size_classes = ['span.fs%d { font-size: %spt }'%(i, x) for i, x in + enumerate(ic.font_sizes)] + color_classes = ['span.col%d { color: %s }'%(i, x) for i, x in + enumerate(ic.colors)] + css = textwrap.dedent(''' + span.none { + text-decoration: none; font-weight: normal; + font-style: normal; font-variant: normal + } + + span.italics { font-style: italic } + + span.bold { font-weight: bold } + + span.small-caps { font-variant: small-caps } + + span.underlined { text-decoration: underline } + + span.strike-through { text-decoration: line-through } + + ''') + css += '\n'+'\n'.join(font_size_classes) + css += '\n' +'\n'.join(color_classes) + + for cls, val in border_styles.iteritems(): + css += '\n\n.%s {\n%s\n}'%(cls, val) + + with open('styles.css', 'ab') as f: + f.write(css) + + def convert_borders(self, doc): + border_styles = [] + style_map = {} + for elem in doc.xpath(r'//*[local-name()="cell"]'): + style = ['border-style: hidden', 'border-width: 1px', + 'border-color: black'] + for x in ('bottom', 'top', 'left', 'right'): + bs = elem.get('border-cell-%s-style'%x, None) + if bs: + cbs = border_style_map.get(bs, 'solid') + style.append('border-%s-style: %s'%(x, cbs)) + bw = elem.get('border-cell-%s-line-width'%x, None) + if bw: + style.append('border-%s-width: %spt'%(x, bw)) + bc = elem.get('border-cell-%s-color'%x, None) + if bc: + style.append('border-%s-color: %s'%(x, bc)) + style = ';\n'.join(style) + if style not in border_styles: + border_styles.append(style) + idx = border_styles.index(style) + cls = 'border_style%d'%idx + style_map[cls] = style + elem.set('class', cls) + return style_map + + def convert(self, stream, options, file_ext, log, + accelerators): + from lxml import etree + from calibre.ebooks.metadata.meta import get_metadata + from calibre.ebooks.metadata.opf2 import OPFCreator + from calibre.ebooks.rtf2xml.ParseRtf import RtfInvalidCodeException + from calibre.ebooks.rtf.input import InlineClass + self.opts = options + self.log = log + self.log('Converting RTF to XML...') + try: + xml = self.generate_xml(stream.name) + except RtfInvalidCodeException as e: + raise ValueError(_('This RTF file has a feature calibre does not ' + 'support. Convert it to HTML first and then try it.\n%s')%e) + + d = glob.glob(os.path.join('*_rtf_pict_dir', 'picts.rtf')) + if d: + imap = {} + try: + imap = self.extract_images(d[0]) + except: + self.log.exception('Failed to extract images...') + + self.log('Parsing XML...') + parser = etree.XMLParser(recover=True, no_network=True) + doc = etree.fromstring(xml, parser=parser) + border_styles = self.convert_borders(doc) + for pict in doc.xpath('//rtf:pict[@num]', + namespaces={'rtf':'http://rtf2xml.sourceforge.net/'}): + num = int(pict.get('num')) + name = imap.get(num, None) + if name is not None: + pict.set('num', name) + + self.log('Converting XML to HTML...') + inline_class = InlineClass(self.log) + styledoc = etree.fromstring(P('templates/rtf.xsl', data=True)) + extensions = { ('calibre', 'inline-class') : inline_class } + transform = etree.XSLT(styledoc, extensions=extensions) + result = transform(doc) + html = 'index.xhtml' + with open(html, 'wb') as f: + res = transform.tostring(result) + # res = res[:100].replace('xmlns:html', 'xmlns') + res[100:] + #clean multiple \n + res = re.sub('\n+', '\n', res) + # Replace newlines inserted by the 'empty_paragraphs' option in rtf2xml with html blank lines + # res = re.sub('\s*', '', res) + # res = re.sub('(?<=\n)\n{2}', + # u'

\u00a0

\n'.encode('utf-8'), res) + f.write(res) + self.write_inline_css(inline_class, border_styles) + stream.seek(0) + mi = get_metadata(stream, 'rtf') + if not mi.title: + mi.title = _('Unknown') + if not mi.authors: + mi.authors = [_('Unknown')] + opf = OPFCreator(os.getcwd(), mi) + opf.create_manifest([('index.xhtml', None)]) + opf.create_spine(['index.xhtml']) + opf.render(open('metadata.opf', 'wb')) + return os.path.abspath('metadata.opf') + + diff --git a/src/calibre/ebooks/snb/input.py b/src/calibre/ebooks/conversion/plugins/snb_input.py similarity index 97% rename from src/calibre/ebooks/snb/input.py rename to src/calibre/ebooks/conversion/plugins/snb_input.py index 13b1ca45f9..ae3ab0033c 100755 --- a/src/calibre/ebooks/snb/input.py +++ b/src/calibre/ebooks/conversion/plugins/snb_input.py @@ -4,13 +4,11 @@ __license__ = 'GPL 3' __copyright__ = '2010, Li Fanxi ' __docformat__ = 'restructuredtext en' -import os, uuid +import os from calibre.customize.conversion import InputFormatPlugin -from calibre.ebooks.snb.snbfile import SNBFile from calibre.ptempfile import TemporaryDirectory from calibre.utils.filenames import ascii_filename -from lxml import etree HTML_TEMPLATE = u'%s\n%s\n' @@ -29,7 +27,12 @@ class SNBInput(InputFormatPlugin): def convert(self, stream, options, file_ext, log, accelerators): + import uuid + from lxml import etree + from calibre.ebooks.oeb.base import DirContainer + from calibre.ebooks.snb.snbfile import SNBFile + log.debug("Parsing SNB file...") snbFile = SNBFile() try: diff --git a/src/calibre/ebooks/tcr/input.py b/src/calibre/ebooks/conversion/plugins/tcr_input.py similarity index 94% rename from src/calibre/ebooks/tcr/input.py rename to src/calibre/ebooks/conversion/plugins/tcr_input.py index 4d15fd0923..de4f3f5f40 100644 --- a/src/calibre/ebooks/tcr/input.py +++ b/src/calibre/ebooks/conversion/plugins/tcr_input.py @@ -7,7 +7,6 @@ __docformat__ = 'restructuredtext en' from cStringIO import StringIO from calibre.customize.conversion import InputFormatPlugin -from calibre.ebooks.compression.tcr import decompress class TCRInput(InputFormatPlugin): @@ -17,6 +16,8 @@ class TCRInput(InputFormatPlugin): file_types = set(['tcr']) def convert(self, stream, options, file_ext, log, accelerators): + from calibre.ebooks.compression.tcr import decompress + log.info('Decompressing text...') raw_txt = decompress(stream) diff --git a/src/calibre/ebooks/txt/input.py b/src/calibre/ebooks/conversion/plugins/txt_input.py similarity index 94% rename from src/calibre/ebooks/txt/input.py rename to src/calibre/ebooks/conversion/plugins/txt_input.py index 49c8a2129d..e916b30c29 100644 --- a/src/calibre/ebooks/txt/input.py +++ b/src/calibre/ebooks/conversion/plugins/txt_input.py @@ -8,14 +8,6 @@ import os from calibre import _ent_pat, walk, xml_entity_to_unicode from calibre.customize.conversion import InputFormatPlugin, OptionRecommendation -from calibre.ebooks.conversion.preprocess import DocAnalysis, Dehyphenator -from calibre.ebooks.chardet import detect -from calibre.ebooks.txt.processor import convert_basic, convert_markdown, \ - separate_paragraphs_single_line, separate_paragraphs_print_formatted, \ - preserve_spaces, detect_paragraph_type, detect_formatting_type, \ - normalize_line_endings, convert_textile, remove_indents, block_to_single_line, \ - separate_hard_scene_breaks -from calibre.utils.zipfile import ZipFile class TXTInput(InputFormatPlugin): @@ -61,6 +53,17 @@ class TXTInput(InputFormatPlugin): def convert(self, stream, options, file_ext, log, accelerators): + from calibre.ebooks.conversion.preprocess import DocAnalysis, Dehyphenator + from calibre.ebooks.chardet import detect + from calibre.utils.zipfile import ZipFile + from calibre.ebooks.txt.processor import (convert_basic, + convert_markdown, separate_paragraphs_single_line, + separate_paragraphs_print_formatted, preserve_spaces, + detect_paragraph_type, detect_formatting_type, + normalize_line_endings, convert_textile, remove_indents, + block_to_single_line, separate_hard_scene_breaks) + + self.log = log txt = '' log.debug('Reading text from file...') diff --git a/src/calibre/ebooks/html/input.py b/src/calibre/ebooks/html/input.py index d303dd66a5..6cacb34edc 100644 --- a/src/calibre/ebooks/html/input.py +++ b/src/calibre/ebooks/html/input.py @@ -11,19 +11,13 @@ __docformat__ = 'restructuredtext en' Input plugin for HTML or OPF ebooks. ''' -import os, re, sys, uuid, tempfile, errno as gerrno +import os, re, sys, errno as gerrno from urlparse import urlparse, urlunparse -from urllib import unquote, quote -from functools import partial -from itertools import izip +from urllib import unquote -from calibre.customize.conversion import InputFormatPlugin from calibre.ebooks.chardet import detect_xml_encoding -from calibre.customize.conversion import OptionRecommendation -from calibre.constants import islinux, isbsd, iswindows +from calibre.constants import iswindows from calibre import unicode_path, as_unicode -from calibre.utils.localization import get_lang -from calibre.utils.filenames import ascii_filename class Link(object): ''' @@ -241,262 +235,4 @@ def get_filelist(htmlfile, dir, opts, log): return filelist -class HTMLInput(InputFormatPlugin): - name = 'HTML Input' - author = 'Kovid Goyal' - description = 'Convert HTML and OPF files to an OEB' - file_types = set(['opf', 'html', 'htm', 'xhtml', 'xhtm', 'shtm', 'shtml']) - - options = set([ - OptionRecommendation(name='breadth_first', - recommended_value=False, level=OptionRecommendation.LOW, - help=_('Traverse links in HTML files breadth first. Normally, ' - 'they are traversed depth first.' - ) - ), - - OptionRecommendation(name='max_levels', - recommended_value=5, level=OptionRecommendation.LOW, - help=_('Maximum levels of recursion when following links in ' - 'HTML files. Must be non-negative. 0 implies that no ' - 'links in the root HTML file are followed. Default is ' - '%default.' - ) - ), - - OptionRecommendation(name='dont_package', - recommended_value=False, level=OptionRecommendation.LOW, - help=_('Normally this input plugin re-arranges all the input ' - 'files into a standard folder hierarchy. Only use this option ' - 'if you know what you are doing as it can result in various ' - 'nasty side effects in the rest of the conversion pipeline.' - ) - ), - - ]) - - def convert(self, stream, opts, file_ext, log, - accelerators): - self._is_case_sensitive = None - basedir = os.getcwd() - self.opts = opts - - fname = None - if hasattr(stream, 'name'): - basedir = os.path.dirname(stream.name) - fname = os.path.basename(stream.name) - - if file_ext != 'opf': - if opts.dont_package: - raise ValueError('The --dont-package option is not supported for an HTML input file') - from calibre.ebooks.metadata.html import get_metadata - mi = get_metadata(stream) - if fname: - from calibre.ebooks.metadata.meta import metadata_from_filename - fmi = metadata_from_filename(fname) - fmi.smart_update(mi) - mi = fmi - oeb = self.create_oebbook(stream.name, basedir, opts, log, mi) - return oeb - - from calibre.ebooks.conversion.plumber import create_oebbook - return create_oebbook(log, stream.name, opts, - encoding=opts.input_encoding) - - def is_case_sensitive(self, path): - if getattr(self, '_is_case_sensitive', None) is not None: - return self._is_case_sensitive - if not path or not os.path.exists(path): - return islinux or isbsd - self._is_case_sensitive = not (os.path.exists(path.lower()) \ - and os.path.exists(path.upper())) - return self._is_case_sensitive - - def create_oebbook(self, htmlpath, basedir, opts, log, mi): - from calibre.ebooks.conversion.plumber import create_oebbook - from calibre.ebooks.oeb.base import (DirContainer, - rewrite_links, urlnormalize, urldefrag, BINARY_MIME, OEB_STYLES, - xpath) - from calibre import guess_type - from calibre.ebooks.oeb.transforms.metadata import \ - meta_info_to_oeb_metadata - import cssutils, logging - cssutils.log.setLevel(logging.WARN) - self.OEB_STYLES = OEB_STYLES - oeb = create_oebbook(log, None, opts, self, - encoding=opts.input_encoding, populate=False) - self.oeb = oeb - - metadata = oeb.metadata - meta_info_to_oeb_metadata(mi, metadata, log) - if not metadata.language: - oeb.logger.warn(u'Language not specified') - metadata.add('language', get_lang().replace('_', '-')) - if not metadata.creator: - oeb.logger.warn('Creator not specified') - metadata.add('creator', self.oeb.translate(__('Unknown'))) - if not metadata.title: - oeb.logger.warn('Title not specified') - metadata.add('title', self.oeb.translate(__('Unknown'))) - bookid = str(uuid.uuid4()) - metadata.add('identifier', bookid, id='uuid_id', scheme='uuid') - for ident in metadata.identifier: - if 'id' in ident.attrib: - self.oeb.uid = metadata.identifier[0] - break - - filelist = get_filelist(htmlpath, basedir, opts, log) - filelist = [f for f in filelist if not f.is_binary] - htmlfile_map = {} - for f in filelist: - path = f.path - oeb.container = DirContainer(os.path.dirname(path), log, - ignore_opf=True) - bname = os.path.basename(path) - id, href = oeb.manifest.generate(id='html', - href=ascii_filename(bname)) - htmlfile_map[path] = href - item = oeb.manifest.add(id, href, 'text/html') - item.html_input_href = bname - oeb.spine.add(item, True) - - self.added_resources = {} - self.log = log - self.log('Normalizing filename cases') - for path, href in htmlfile_map.items(): - if not self.is_case_sensitive(path): - path = path.lower() - self.added_resources[path] = href - self.urlnormalize, self.DirContainer = urlnormalize, DirContainer - self.urldefrag = urldefrag - self.guess_type, self.BINARY_MIME = guess_type, BINARY_MIME - - self.log('Rewriting HTML links') - for f in filelist: - path = f.path - dpath = os.path.dirname(path) - oeb.container = DirContainer(dpath, log, ignore_opf=True) - item = oeb.manifest.hrefs[htmlfile_map[path]] - rewrite_links(item.data, partial(self.resource_adder, base=dpath)) - - for item in oeb.manifest.values(): - if item.media_type in self.OEB_STYLES: - dpath = None - for path, href in self.added_resources.items(): - if href == item.href: - dpath = os.path.dirname(path) - break - cssutils.replaceUrls(item.data, - partial(self.resource_adder, base=dpath)) - - toc = self.oeb.toc - self.oeb.auto_generated_toc = True - titles = [] - headers = [] - for item in self.oeb.spine: - if not item.linear: continue - html = item.data - title = ''.join(xpath(html, '/h:html/h:head/h:title/text()')) - title = re.sub(r'\s+', ' ', title.strip()) - if title: - titles.append(title) - headers.append('(unlabled)') - for tag in ('h1', 'h2', 'h3', 'h4', 'h5', 'strong'): - expr = '/h:html/h:body//h:%s[position()=1]/text()' - header = ''.join(xpath(html, expr % tag)) - header = re.sub(r'\s+', ' ', header.strip()) - if header: - headers[-1] = header - break - use = titles - if len(titles) > len(set(titles)): - use = headers - for title, item in izip(use, self.oeb.spine): - if not item.linear: continue - toc.add(title, item.href) - - oeb.container = DirContainer(os.getcwdu(), oeb.log, ignore_opf=True) - return oeb - - def link_to_local_path(self, link_, base=None): - if not isinstance(link_, unicode): - try: - link_ = link_.decode('utf-8', 'error') - except: - self.log.warn('Failed to decode link %r. Ignoring'%link_) - return None, None - try: - l = Link(link_, base if base else os.getcwdu()) - except: - self.log.exception('Failed to process link: %r'%link_) - return None, None - if l.path is None: - # Not a local resource - return None, None - link = l.path.replace('/', os.sep).strip() - frag = l.fragment - if not link: - return None, None - return link, frag - - def resource_adder(self, link_, base=None): - link, frag = self.link_to_local_path(link_, base=base) - if link is None: - return link_ - try: - if base and not os.path.isabs(link): - link = os.path.join(base, link) - link = os.path.abspath(link) - except: - return link_ - if not os.access(link, os.R_OK): - return link_ - if os.path.isdir(link): - self.log.warn(link_, 'is a link to a directory. Ignoring.') - return link_ - if not self.is_case_sensitive(tempfile.gettempdir()): - link = link.lower() - if link not in self.added_resources: - bhref = os.path.basename(link) - id, href = self.oeb.manifest.generate(id='added', - href=bhref) - guessed = self.guess_type(href)[0] - media_type = guessed or self.BINARY_MIME - if media_type == 'text/plain': - self.log.warn('Ignoring link to text file %r'%link_) - return None - - self.oeb.log.debug('Added', link) - self.oeb.container = self.DirContainer(os.path.dirname(link), - self.oeb.log, ignore_opf=True) - # Load into memory - item = self.oeb.manifest.add(id, href, media_type) - # bhref refers to an already existing file. The read() method of - # DirContainer will call unquote on it before trying to read the - # file, therefore we quote it here. - if isinstance(bhref, unicode): - bhref = bhref.encode('utf-8') - item.html_input_href = quote(bhref).decode('utf-8') - if guessed in self.OEB_STYLES: - item.override_css_fetch = partial( - self.css_import_handler, os.path.dirname(link)) - item.data - self.added_resources[link] = href - - nlink = self.added_resources[link] - if frag: - nlink = '#'.join((nlink, frag)) - return nlink - - def css_import_handler(self, base, href): - link, frag = self.link_to_local_path(href, base=base) - if link is None or not os.access(link, os.R_OK) or os.path.isdir(link): - return (None, None) - try: - raw = open(link, 'rb').read().decode('utf-8', 'replace') - raw = self.oeb.css_preprocessor(raw, add_namespace=True) - except: - self.log.exception('Failed to read CSS file: %r'%link) - return (None, None) - return (None, raw) diff --git a/src/calibre/ebooks/lrf/input.py b/src/calibre/ebooks/lrf/input.py index 9777a8a998..e9bf42c6bd 100644 --- a/src/calibre/ebooks/lrf/input.py +++ b/src/calibre/ebooks/lrf/input.py @@ -6,12 +6,11 @@ __license__ = 'GPL v3' __copyright__ = '2009, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import os, textwrap, sys, operator +import textwrap, operator from copy import deepcopy, copy from lxml import etree -from calibre.customize.conversion import InputFormatPlugin from calibre import guess_type class Canvas(etree.XSLTExtension): @@ -406,76 +405,4 @@ class Styles(etree.XSLTExtension): -class LRFInput(InputFormatPlugin): - name = 'LRF Input' - author = 'Kovid Goyal' - description = 'Convert LRF files to HTML' - file_types = set(['lrf']) - - def convert(self, stream, options, file_ext, log, - accelerators): - self.log = log - self.log('Generating XML') - from calibre.ebooks.lrf.lrfparser import LRFDocument - d = LRFDocument(stream) - d.parse() - xml = d.to_xml(write_files=True) - if options.verbose > 2: - open('lrs.xml', 'wb').write(xml.encode('utf-8')) - parser = etree.XMLParser(no_network=True, huge_tree=True) - try: - doc = etree.fromstring(xml, parser=parser) - except: - self.log.warn('Failed to parse XML. Trying to recover') - parser = etree.XMLParser(no_network=True, huge_tree=True, - recover=True) - doc = etree.fromstring(xml, parser=parser) - - - char_button_map = {} - for x in doc.xpath('//CharButton[@refobj]'): - ro = x.get('refobj') - jump_button = doc.xpath('//*[@objid="%s"]'%ro) - if jump_button: - jump_to = jump_button[0].xpath('descendant::JumpTo[@refpage and @refobj]') - if jump_to: - char_button_map[ro] = '%s.xhtml#%s'%(jump_to[0].get('refpage'), - jump_to[0].get('refobj')) - plot_map = {} - for x in doc.xpath('//Plot[@refobj]'): - ro = x.get('refobj') - image = doc.xpath('//Image[@objid="%s" and @refstream]'%ro) - if image: - imgstr = doc.xpath('//ImageStream[@objid="%s" and @file]'% - image[0].get('refstream')) - if imgstr: - plot_map[ro] = imgstr[0].get('file') - - self.log('Converting XML to HTML...') - styledoc = etree.fromstring(P('templates/lrf.xsl', data=True)) - media_type = MediaType() - styles = Styles() - text_block = TextBlock(styles, char_button_map, plot_map, log) - canvas = Canvas(doc, styles, text_block, log) - image_block = ImageBlock(canvas) - ruled_line = RuledLine() - extensions = { - ('calibre', 'media-type') : media_type, - ('calibre', 'text-block') : text_block, - ('calibre', 'ruled-line') : ruled_line, - ('calibre', 'styles') : styles, - ('calibre', 'canvas') : canvas, - ('calibre', 'image-block'): image_block, - } - transform = etree.XSLT(styledoc, extensions=extensions) - try: - result = transform(doc) - except RuntimeError: - sys.setrecursionlimit(5000) - result = transform(doc) - - with open('content.opf', 'wb') as f: - f.write(result) - styles.write() - return os.path.abspath('content.opf') diff --git a/src/calibre/ebooks/odt/input.py b/src/calibre/ebooks/odt/input.py index 214a40c29b..430d95b31f 100644 --- a/src/calibre/ebooks/odt/input.py +++ b/src/calibre/ebooks/odt/input.py @@ -12,7 +12,6 @@ from lxml import etree from odf.odf2xhtml import ODF2XHTML from calibre import CurrentDir, walk -from calibre.customize.conversion import InputFormatPlugin class Extract(ODF2XHTML): @@ -178,16 +177,4 @@ class Extract(ODF2XHTML): return os.path.abspath('metadata.opf') -class ODTInput(InputFormatPlugin): - - name = 'ODT Input' - author = 'Kovid Goyal' - description = 'Convert ODT (OpenOffice) files to HTML' - file_types = set(['odt']) - - - def convert(self, stream, options, file_ext, log, - accelerators): - return Extract()(stream, '.', log) - diff --git a/src/calibre/ebooks/rtf/input.py b/src/calibre/ebooks/rtf/input.py index 5858824434..8e1a5ac775 100644 --- a/src/calibre/ebooks/rtf/input.py +++ b/src/calibre/ebooks/rtf/input.py @@ -2,42 +2,9 @@ from __future__ import with_statement __license__ = 'GPL v3' __copyright__ = '2008, Kovid Goyal ' -import os, glob, re, textwrap from lxml import etree -from calibre.customize.conversion import InputFormatPlugin - -border_style_map = { - 'single' : 'solid', - 'double-thickness-border' : 'double', - 'shadowed-border': 'outset', - 'double-border': 'double', - 'dotted-border': 'dotted', - 'dashed': 'dashed', - 'hairline': 'solid', - 'inset': 'inset', - 'dash-small': 'dashed', - 'dot-dash': 'dotted', - 'dot-dot-dash': 'dotted', - 'outset': 'outset', - 'tripple': 'double', - 'triple': 'double', - 'thick-thin-small': 'solid', - 'thin-thick-small': 'solid', - 'thin-thick-thin-small': 'solid', - 'thick-thin-medium': 'solid', - 'thin-thick-medium': 'solid', - 'thin-thick-thin-medium': 'solid', - 'thick-thin-large': 'solid', - 'thin-thick-thin-large': 'solid', - 'wavy': 'ridge', - 'double-wavy': 'ridge', - 'striped': 'ridge', - 'emboss': 'inset', - 'engrave': 'inset', - 'frame': 'ridge', -} class InlineClass(etree.XSLTExtension): @@ -71,261 +38,3 @@ class InlineClass(etree.XSLTExtension): output_parent.text = ' '.join(classes) -class RTFInput(InputFormatPlugin): - - name = 'RTF Input' - author = 'Kovid Goyal' - description = 'Convert RTF files to HTML' - file_types = set(['rtf']) - - def generate_xml(self, stream): - from calibre.ebooks.rtf2xml.ParseRtf import ParseRtf - ofile = 'dataxml.xml' - run_lev, debug_dir, indent_out = 1, None, 0 - if getattr(self.opts, 'debug_pipeline', None) is not None: - try: - os.mkdir('rtfdebug') - debug_dir = 'rtfdebug' - run_lev = 4 - indent_out = 1 - self.log('Running RTFParser in debug mode') - except: - self.log.warn('Impossible to run RTFParser in debug mode') - parser = ParseRtf( - in_file = stream, - out_file = ofile, - # Convert symbol fonts to unicode equivalents. Default - # is 1 - convert_symbol = 1, - - # Convert Zapf fonts to unicode equivalents. Default - # is 1. - convert_zapf = 1, - - # Convert Wingding fonts to unicode equivalents. - # Default is 1. - convert_wingdings = 1, - - # Convert RTF caps to real caps. - # Default is 1. - convert_caps = 1, - - # Indent resulting XML. - # Default is 0 (no indent). - indent = indent_out, - - # Form lists from RTF. Default is 1. - form_lists = 1, - - # Convert headings to sections. Default is 0. - headings_to_sections = 1, - - # Group paragraphs with the same style name. Default is 1. - group_styles = 1, - - # Group borders. Default is 1. - group_borders = 1, - - # Write or do not write paragraphs. Default is 0. - empty_paragraphs = 1, - - #debug - deb_dir = debug_dir, - run_level = run_lev, - ) - parser.parse_rtf() - with open(ofile, 'rb') as f: - return f.read() - - def extract_images(self, picts): - import imghdr - self.log('Extracting images...') - - with open(picts, 'rb') as f: - raw = f.read() - picts = filter(len, re.findall(r'\{\\pict([^}]+)\}', raw)) - hex = re.compile(r'[^a-fA-F0-9]') - encs = [hex.sub('', pict) for pict in picts] - - count = 0 - imap = {} - for enc in encs: - if len(enc) % 2 == 1: - enc = enc[:-1] - data = enc.decode('hex') - fmt = imghdr.what(None, data) - if fmt is None: - fmt = 'wmf' - count += 1 - name = '%04d.%s' % (count, fmt) - with open(name, 'wb') as f: - f.write(data) - imap[count] = name - # with open(name+'.hex', 'wb') as f: - # f.write(enc) - return self.convert_images(imap) - - def convert_images(self, imap): - self.default_img = None - for count, val in imap.iteritems(): - try: - imap[count] = self.convert_image(val) - except: - self.log.exception('Failed to convert', val) - return imap - - def convert_image(self, name): - if not name.endswith('.wmf'): - return name - try: - return self.rasterize_wmf(name) - except: - self.log.exception('Failed to convert WMF image %r'%name) - return self.replace_wmf(name) - - def replace_wmf(self, name): - from calibre.ebooks import calibre_cover - if self.default_img is None: - self.default_img = calibre_cover('Conversion of WMF images is not supported', - 'Use Microsoft Word or OpenOffice to save this RTF file' - ' as HTML and convert that in calibre.', title_size=36, - author_size=20) - name = name.replace('.wmf', '.jpg') - with open(name, 'wb') as f: - f.write(self.default_img) - return name - - def rasterize_wmf(self, name): - from calibre.utils.wmf.parse import wmf_unwrap - with open(name, 'rb') as f: - data = f.read() - data = wmf_unwrap(data) - name = name.replace('.wmf', '.png') - with open(name, 'wb') as f: - f.write(data) - return name - - - def write_inline_css(self, ic, border_styles): - font_size_classes = ['span.fs%d { font-size: %spt }'%(i, x) for i, x in - enumerate(ic.font_sizes)] - color_classes = ['span.col%d { color: %s }'%(i, x) for i, x in - enumerate(ic.colors)] - css = textwrap.dedent(''' - span.none { - text-decoration: none; font-weight: normal; - font-style: normal; font-variant: normal - } - - span.italics { font-style: italic } - - span.bold { font-weight: bold } - - span.small-caps { font-variant: small-caps } - - span.underlined { text-decoration: underline } - - span.strike-through { text-decoration: line-through } - - ''') - css += '\n'+'\n'.join(font_size_classes) - css += '\n' +'\n'.join(color_classes) - - for cls, val in border_styles.iteritems(): - css += '\n\n.%s {\n%s\n}'%(cls, val) - - with open('styles.css', 'ab') as f: - f.write(css) - - def convert_borders(self, doc): - border_styles = [] - style_map = {} - for elem in doc.xpath(r'//*[local-name()="cell"]'): - style = ['border-style: hidden', 'border-width: 1px', - 'border-color: black'] - for x in ('bottom', 'top', 'left', 'right'): - bs = elem.get('border-cell-%s-style'%x, None) - if bs: - cbs = border_style_map.get(bs, 'solid') - style.append('border-%s-style: %s'%(x, cbs)) - bw = elem.get('border-cell-%s-line-width'%x, None) - if bw: - style.append('border-%s-width: %spt'%(x, bw)) - bc = elem.get('border-cell-%s-color'%x, None) - if bc: - style.append('border-%s-color: %s'%(x, bc)) - style = ';\n'.join(style) - if style not in border_styles: - border_styles.append(style) - idx = border_styles.index(style) - cls = 'border_style%d'%idx - style_map[cls] = style - elem.set('class', cls) - return style_map - - def convert(self, stream, options, file_ext, log, - accelerators): - from calibre.ebooks.metadata.meta import get_metadata - from calibre.ebooks.metadata.opf2 import OPFCreator - from calibre.ebooks.rtf2xml.ParseRtf import RtfInvalidCodeException - self.opts = options - self.log = log - self.log('Converting RTF to XML...') - try: - xml = self.generate_xml(stream.name) - except RtfInvalidCodeException as e: - raise ValueError(_('This RTF file has a feature calibre does not ' - 'support. Convert it to HTML first and then try it.\n%s')%e) - - d = glob.glob(os.path.join('*_rtf_pict_dir', 'picts.rtf')) - if d: - imap = {} - try: - imap = self.extract_images(d[0]) - except: - self.log.exception('Failed to extract images...') - - self.log('Parsing XML...') - parser = etree.XMLParser(recover=True, no_network=True) - doc = etree.fromstring(xml, parser=parser) - border_styles = self.convert_borders(doc) - for pict in doc.xpath('//rtf:pict[@num]', - namespaces={'rtf':'http://rtf2xml.sourceforge.net/'}): - num = int(pict.get('num')) - name = imap.get(num, None) - if name is not None: - pict.set('num', name) - - self.log('Converting XML to HTML...') - inline_class = InlineClass(self.log) - styledoc = etree.fromstring(P('templates/rtf.xsl', data=True)) - extensions = { ('calibre', 'inline-class') : inline_class } - transform = etree.XSLT(styledoc, extensions=extensions) - result = transform(doc) - html = 'index.xhtml' - with open(html, 'wb') as f: - res = transform.tostring(result) - # res = res[:100].replace('xmlns:html', 'xmlns') + res[100:] - #clean multiple \n - res = re.sub('\n+', '\n', res) - # Replace newlines inserted by the 'empty_paragraphs' option in rtf2xml with html blank lines - # res = re.sub('\s*', '', res) - # res = re.sub('(?<=\n)\n{2}', - # u'

\u00a0

\n'.encode('utf-8'), res) - f.write(res) - self.write_inline_css(inline_class, border_styles) - stream.seek(0) - mi = get_metadata(stream, 'rtf') - if not mi.title: - mi.title = _('Unknown') - if not mi.authors: - mi.authors = [_('Unknown')] - opf = OPFCreator(os.getcwd(), mi) - opf.create_manifest([('index.xhtml', None)]) - opf.create_spine(['index.xhtml']) - opf.render(open('metadata.opf', 'wb')) - return os.path.abspath('metadata.opf') - -#ebook-convert "bad.rtf" test.epub -v -d "E:\Mes eBooks\Developpement\debug" -# os.makedirs("E:\\Mes eBooks\\Developpement\\rtfdebug") -# debug_dir = "E:\\Mes eBooks\\Developpement\\rtfdebug" From 08c840cba34a9a4d69d50f83d1ffccce1918abf8 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 5 Feb 2012 23:07:50 +0530 Subject: [PATCH 18/31] TXT Input: Fix self closed title tags being generated --- src/calibre/ebooks/txt/processor.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/calibre/ebooks/txt/processor.py b/src/calibre/ebooks/txt/processor.py index 4cff648fa5..0880eca4ca 100644 --- a/src/calibre/ebooks/txt/processor.py +++ b/src/calibre/ebooks/txt/processor.py @@ -16,7 +16,7 @@ from calibre.ebooks.metadata.opf2 import OPFCreator from calibre.ebooks.conversion.preprocess import DocAnalysis from calibre.utils.cleantext import clean_ascii_chars -HTML_TEMPLATE = u'%s\n%s\n' +HTML_TEMPLATE = u'%s \n%s\n' def clean_txt(txt): ''' @@ -28,7 +28,7 @@ def clean_txt(txt): # Strip whitespace from the end of the line. Also replace # all line breaks with \n. txt = '\n'.join([line.rstrip() for line in txt.splitlines()]) - + # Replace whitespace at the beginning of the line with   txt = re.sub('(?m)(?<=^)([ ]{2,}|\t+)(?=.)', ' ' * 4, txt) @@ -75,7 +75,7 @@ def convert_basic(txt, title='', epub_split_size_kb=0): ''' Converts plain text to html by putting all paragraphs in

tags. It condense and retains blank lines when necessary. - + Requires paragraphs to be in single line format. ''' txt = clean_txt(txt) @@ -215,7 +215,7 @@ def detect_paragraph_type(txt): def detect_formatting_type(txt): ''' Tries to determine the formatting of the document. - + markdown: Markdown formatting is used. textile: Textile formatting is used. heuristic: When none of the above formatting types are From abdb32e1a756bce0b973c45e383717592afc9445 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 5 Feb 2012 23:24:19 +0530 Subject: [PATCH 19/31] RB Input: Do not generate empty title tag --- src/calibre/ebooks/rb/reader.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/calibre/ebooks/rb/reader.py b/src/calibre/ebooks/rb/reader.py index f97c3d78c5..e68cef41d3 100644 --- a/src/calibre/ebooks/rb/reader.py +++ b/src/calibre/ebooks/rb/reader.py @@ -65,7 +65,7 @@ class Reader(object): name = urlunquote(self.stream.read(32).strip('\x00')) size, offset, flags = self.read_i32(), self.read_i32(), self.read_i32() toc.append(RBToc.Item(name=name, size=size, offset=offset, flags=flags)) - + return toc def get_text(self, toc_item, output_dir): @@ -89,7 +89,7 @@ class Reader(object): output += self.stream.read(toc_item.size).decode('cp1252' if self.encoding is None else self.encoding, 'replace') with open(os.path.join(output_dir, toc_item.name), 'wb') as html: - html.write(output.encode('utf-8')) + html.write(output.replace('', '<TITLE> ').encode('utf-8')) def get_image(self, toc_item, output_dir): if toc_item.flags != 0: @@ -105,7 +105,7 @@ class Reader(object): self.log.debug('Extracting content from file...') html = [] images = [] - + for item in self.toc: if item.name.lower().endswith('html'): self.log.debug('HTML item %s found...' % item.name) From e036afe6d003b61dfbe25fb608ab7e50cad91c68 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Sun, 5 Feb 2012 23:44:19 +0530 Subject: [PATCH 20/31] Speed up import of builting output format plugins --- src/calibre/customize/builtins.py | 33 +++++----- .../plugins/epub_output.py} | 7 +-- .../plugins/fb2_output.py} | 2 +- .../plugins/html_output.py} | 23 ++++--- .../plugins/htmlz_output.py} | 8 +-- .../plugins/lit_output.py} | 0 .../plugins/lrf_output.py} | 0 .../plugins/mobi_output.py} | 0 .../plugins/oeb_output.py} | 10 ++-- .../plugins/pdb_output.py} | 0 .../plugins/pdf_output.py} | 60 ++++++++++++++++--- .../plugins/pml_output.py} | 26 ++++---- .../plugins/rb_output.py} | 3 +- .../plugins/rtf_output.py} | 3 +- .../plugins/snb_output.py} | 8 ++- .../ebooks/conversion/plugins/tcr_input.py | 2 +- .../plugins/tcr_output.py} | 5 +- .../plugins/txt_output.py} | 19 +++--- 18 files changed, 129 insertions(+), 80 deletions(-) rename src/calibre/ebooks/{epub/output.py => conversion/plugins/epub_output.py} (99%) rename src/calibre/ebooks/{fb2/output.py => conversion/plugins/fb2_output.py} (99%) rename src/calibre/ebooks/{html/output.py => conversion/plugins/html_output.py} (96%) rename src/calibre/ebooks/{htmlz/output.py => conversion/plugins/htmlz_output.py} (96%) rename src/calibre/ebooks/{lit/output.py => conversion/plugins/lit_output.py} (100%) rename src/calibre/ebooks/{lrf/output.py => conversion/plugins/lrf_output.py} (100%) rename src/calibre/ebooks/{mobi/output.py => conversion/plugins/mobi_output.py} (100%) rename src/calibre/ebooks/{oeb/output.py => conversion/plugins/oeb_output.py} (96%) rename src/calibre/ebooks/{pdb/output.py => conversion/plugins/pdb_output.py} (100%) rename src/calibre/ebooks/{pdf/output.py => conversion/plugins/pdf_output.py} (86%) rename src/calibre/ebooks/{pml/output.py => conversion/plugins/pml_output.py} (88%) rename src/calibre/ebooks/{rb/output.py => conversion/plugins/rb_output.py} (95%) rename src/calibre/ebooks/{rtf/output.py => conversion/plugins/rtf_output.py} (94%) rename src/calibre/ebooks/{snb/output.py => conversion/plugins/snb_output.py} (98%) rename src/calibre/ebooks/{tcr/output.py => conversion/plugins/tcr_output.py} (93%) rename src/calibre/ebooks/{txt/output.py => conversion/plugins/txt_output.py} (93%) diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py index 9cd3271fad..69a867e894 100644 --- a/src/calibre/customize/builtins.py +++ b/src/calibre/customize/builtins.py @@ -523,23 +523,22 @@ from calibre.ebooks.conversion.plugins.lrf_input import LRFInput from calibre.ebooks.conversion.plugins.chm_input import CHMInput from calibre.ebooks.conversion.plugins.snb_input import SNBInput -from calibre.ebooks.epub.output import EPUBOutput -from calibre.ebooks.fb2.output import FB2Output -from calibre.ebooks.lit.output import LITOutput -from calibre.ebooks.lrf.output import LRFOutput -from calibre.ebooks.mobi.output import MOBIOutput -from calibre.ebooks.oeb.output import OEBOutput -from calibre.ebooks.pdb.output import PDBOutput -from calibre.ebooks.pdf.output import PDFOutput -from calibre.ebooks.pml.output import PMLOutput -from calibre.ebooks.rb.output import RBOutput -from calibre.ebooks.rtf.output import RTFOutput -from calibre.ebooks.tcr.output import TCROutput -from calibre.ebooks.txt.output import TXTOutput -from calibre.ebooks.txt.output import TXTZOutput -from calibre.ebooks.html.output import HTMLOutput -from calibre.ebooks.htmlz.output import HTMLZOutput -from calibre.ebooks.snb.output import SNBOutput +from calibre.ebooks.conversion.plugins.epub_output import EPUBOutput +from calibre.ebooks.conversion.plugins.fb2_output import FB2Output +from calibre.ebooks.conversion.plugins.lit_output import LITOutput +from calibre.ebooks.conversion.plugins.lrf_output import LRFOutput +from calibre.ebooks.conversion.plugins.mobi_output import MOBIOutput +from calibre.ebooks.conversion.plugins.oeb_output import OEBOutput +from calibre.ebooks.conversion.plugins.pdb_output import PDBOutput +from calibre.ebooks.conversion.plugins.pdf_output import PDFOutput +from calibre.ebooks.conversion.plugins.pml_output import PMLOutput +from calibre.ebooks.conversion.plugins.rb_output import RBOutput +from calibre.ebooks.conversion.plugins.rtf_output import RTFOutput +from calibre.ebooks.conversion.plugins.tcr_output import TCROutput +from calibre.ebooks.conversion.plugins.txt_output import TXTOutput, TXTZOutput +from calibre.ebooks.conversion.plugins.html_output import HTMLOutput +from calibre.ebooks.conversion.plugins.htmlz_output import HTMLZOutput +from calibre.ebooks.conversion.plugins.snb_output import SNBOutput from calibre.customize.profiles import input_profiles, output_profiles diff --git a/src/calibre/ebooks/epub/output.py b/src/calibre/ebooks/conversion/plugins/epub_output.py similarity index 99% rename from src/calibre/ebooks/epub/output.py rename to src/calibre/ebooks/conversion/plugins/epub_output.py index 2bdfb0d934..44249e49a2 100644 --- a/src/calibre/ebooks/epub/output.py +++ b/src/calibre/ebooks/conversion/plugins/epub_output.py @@ -8,14 +8,12 @@ __docformat__ = 'restructuredtext en' import os, shutil, re -from calibre.customize.conversion import OutputFormatPlugin +from calibre.customize.conversion import (OutputFormatPlugin, + OptionRecommendation) from calibre.ptempfile import TemporaryDirectory from calibre import CurrentDir -from calibre.customize.conversion import OptionRecommendation from calibre.constants import filesystem_encoding -from lxml import etree - block_level_tags = ( 'address', 'body', @@ -289,6 +287,7 @@ class EPUBOutput(OutputFormatPlugin): # }}} def condense_ncx(self, ncx_path): + from lxml import etree if not self.opts.pretty_print: tree = etree.parse(ncx_path) for tag in tree.getroot().iter(tag=etree.Element): diff --git a/src/calibre/ebooks/fb2/output.py b/src/calibre/ebooks/conversion/plugins/fb2_output.py similarity index 99% rename from src/calibre/ebooks/fb2/output.py rename to src/calibre/ebooks/conversion/plugins/fb2_output.py index 2042902724..d7db2a0a33 100644 --- a/src/calibre/ebooks/fb2/output.py +++ b/src/calibre/ebooks/conversion/plugins/fb2_output.py @@ -7,7 +7,6 @@ __docformat__ = 'restructuredtext en' import os from calibre.customize.conversion import OutputFormatPlugin, OptionRecommendation -from calibre.ebooks.fb2.fb2ml import FB2MLizer class FB2Output(OutputFormatPlugin): @@ -162,6 +161,7 @@ class FB2Output(OutputFormatPlugin): def convert(self, oeb_book, output_path, input_plugin, opts, log): from calibre.ebooks.oeb.transforms.jacket import linearize_jacket from calibre.ebooks.oeb.transforms.rasterize import SVGRasterizer, Unavailable + from calibre.ebooks.fb2.fb2ml import FB2MLizer try: rasterizer = SVGRasterizer() diff --git a/src/calibre/ebooks/html/output.py b/src/calibre/ebooks/conversion/plugins/html_output.py similarity index 96% rename from src/calibre/ebooks/html/output.py rename to src/calibre/ebooks/conversion/plugins/html_output.py index fe7b4cf274..3821ba41a4 100644 --- a/src/calibre/ebooks/html/output.py +++ b/src/calibre/ebooks/conversion/plugins/html_output.py @@ -4,22 +4,11 @@ __copyright__ = '2010, Fabian Grassl <fg@jusmeum.de>' __docformat__ = 'restructuredtext en' import os, re, shutil - -from calibre.utils import zipfile - from os.path import dirname, abspath, relpath, exists, basename -from lxml import etree -from templite import Templite - from calibre.customize.conversion import OutputFormatPlugin, OptionRecommendation from calibre import CurrentDir from calibre.ptempfile import PersistentTemporaryDirectory -from calibre.utils.zipfile import ZipFile - -from urllib import unquote - -from calibre.ebooks.html.meta import EasyMeta class HTMLOutput(OutputFormatPlugin): @@ -50,6 +39,9 @@ class HTMLOutput(OutputFormatPlugin): ''' Generate table of contents ''' + from lxml import etree + from urllib import unquote + from calibre.ebooks.oeb.base import element with CurrentDir(output_dir): def build_node(current_node, parent=None): @@ -72,11 +64,18 @@ class HTMLOutput(OutputFormatPlugin): return wrap def generate_html_toc(self, oeb_book, ref_url, output_dir): + from lxml import etree + root = self.generate_toc(oeb_book, ref_url, output_dir) return etree.tostring(root, pretty_print=True, encoding='utf-8', xml_declaration=False) def convert(self, oeb_book, output_path, input_plugin, opts, log): + from lxml import etree + from calibre.utils import zipfile + from templite import Templite + from urllib import unquote + from calibre.ebooks.html.meta import EasyMeta # read template files if opts.template_html_index is not None: @@ -192,7 +191,7 @@ class HTMLOutput(OutputFormatPlugin): f.write(t) item.unload_data_from_memory(memory=path) - zfile = ZipFile(output_path, "w") + zfile = zipfile.ZipFile(output_path, "w") zfile.add_dir(output_dir, basename(output_dir)) zfile.write(output_file, basename(output_file), zipfile.ZIP_DEFLATED) diff --git a/src/calibre/ebooks/htmlz/output.py b/src/calibre/ebooks/conversion/plugins/htmlz_output.py similarity index 96% rename from src/calibre/ebooks/htmlz/output.py rename to src/calibre/ebooks/conversion/plugins/htmlz_output.py index a1ef57af2c..f35dbc4dad 100644 --- a/src/calibre/ebooks/htmlz/output.py +++ b/src/calibre/ebooks/conversion/plugins/htmlz_output.py @@ -9,13 +9,10 @@ __docformat__ = 'restructuredtext en' import os from cStringIO import StringIO -from lxml import etree from calibre.customize.conversion import OutputFormatPlugin, \ OptionRecommendation -from calibre.ebooks.metadata.opf2 import OPF, metadata_to_opf from calibre.ptempfile import TemporaryDirectory -from calibre.utils.zipfile import ZipFile class HTMLZOutput(OutputFormatPlugin): @@ -43,7 +40,10 @@ class HTMLZOutput(OutputFormatPlugin): ]) def convert(self, oeb_book, output_path, input_plugin, opts, log): + from lxml import etree from calibre.ebooks.oeb.base import OEB_IMAGES, SVG_MIME + from calibre.ebooks.metadata.opf2 import OPF, metadata_to_opf + from calibre.utils.zipfile import ZipFile # HTML if opts.htmlz_css_type == 'inline': @@ -81,7 +81,7 @@ class HTMLZOutput(OutputFormatPlugin): fname = os.path.join(tdir, 'images', images[item.href]) with open(fname, 'wb') as img: img.write(data) - + # Cover cover_path = None try: diff --git a/src/calibre/ebooks/lit/output.py b/src/calibre/ebooks/conversion/plugins/lit_output.py similarity index 100% rename from src/calibre/ebooks/lit/output.py rename to src/calibre/ebooks/conversion/plugins/lit_output.py diff --git a/src/calibre/ebooks/lrf/output.py b/src/calibre/ebooks/conversion/plugins/lrf_output.py similarity index 100% rename from src/calibre/ebooks/lrf/output.py rename to src/calibre/ebooks/conversion/plugins/lrf_output.py diff --git a/src/calibre/ebooks/mobi/output.py b/src/calibre/ebooks/conversion/plugins/mobi_output.py similarity index 100% rename from src/calibre/ebooks/mobi/output.py rename to src/calibre/ebooks/conversion/plugins/mobi_output.py diff --git a/src/calibre/ebooks/oeb/output.py b/src/calibre/ebooks/conversion/plugins/oeb_output.py similarity index 96% rename from src/calibre/ebooks/oeb/output.py rename to src/calibre/ebooks/conversion/plugins/oeb_output.py index 38ac2495fd..b69e095d0f 100644 --- a/src/calibre/ebooks/oeb/output.py +++ b/src/calibre/ebooks/conversion/plugins/oeb_output.py @@ -5,13 +5,10 @@ __docformat__ = 'restructuredtext en' import os, re -from lxml import etree -from calibre.customize.conversion import OutputFormatPlugin +from calibre.customize.conversion import (OutputFormatPlugin, + OptionRecommendation) from calibre import CurrentDir -from calibre.customize.conversion import OptionRecommendation - -from urllib import unquote class OEBOutput(OutputFormatPlugin): @@ -23,6 +20,9 @@ class OEBOutput(OutputFormatPlugin): def convert(self, oeb_book, output_path, input_plugin, opts, log): + from urllib import unquote + from lxml import etree + self.log, self.opts = log, opts if not os.path.exists(output_path): os.makedirs(output_path) diff --git a/src/calibre/ebooks/pdb/output.py b/src/calibre/ebooks/conversion/plugins/pdb_output.py similarity index 100% rename from src/calibre/ebooks/pdb/output.py rename to src/calibre/ebooks/conversion/plugins/pdb_output.py diff --git a/src/calibre/ebooks/pdf/output.py b/src/calibre/ebooks/conversion/plugins/pdf_output.py similarity index 86% rename from src/calibre/ebooks/pdf/output.py rename to src/calibre/ebooks/conversion/plugins/pdf_output.py index 14dd27368c..4422265976 100644 --- a/src/calibre/ebooks/pdf/output.py +++ b/src/calibre/ebooks/conversion/plugins/pdf_output.py @@ -13,10 +13,50 @@ import os from calibre.customize.conversion import OutputFormatPlugin, \ OptionRecommendation -from calibre.ebooks.metadata.opf2 import OPF from calibre.ptempfile import TemporaryDirectory -from calibre.ebooks.pdf.pageoptions import UNITS, PAPER_SIZES, \ - ORIENTATIONS + +UNITS = [ + 'millimeter', + 'point', + 'inch' , + 'pica' , + 'didot', + 'cicero', + 'devicepixel', + ] + +PAPER_SIZES = ['b2', + 'a9', + 'executive', + 'tabloid', + 'b4', + 'b5', + 'b6', + 'b7', + 'b0', + 'b1', + 'letter', + 'b3', + 'a7', + 'a8', + 'b8', + 'b9', + 'a3', + 'a1', + 'folio', + 'c5e', + 'dle', + 'a0', + 'ledger', + 'legal', + 'a6', + 'a2', + 'b10', + 'a5', + 'comm10e', + 'a4'] + +ORIENTATIONS = ['portrait', 'landscape'] class PDFOutput(OutputFormatPlugin): @@ -26,23 +66,23 @@ class PDFOutput(OutputFormatPlugin): options = set([ OptionRecommendation(name='unit', recommended_value='inch', - level=OptionRecommendation.LOW, short_switch='u', choices=UNITS.keys(), + level=OptionRecommendation.LOW, short_switch='u', choices=UNITS, help=_('The unit of measure. Default is inch. Choices ' 'are %s ' - 'Note: This does not override the unit for margins!') % UNITS.keys()), + 'Note: This does not override the unit for margins!') % UNITS), OptionRecommendation(name='paper_size', recommended_value='letter', - level=OptionRecommendation.LOW, choices=PAPER_SIZES.keys(), + level=OptionRecommendation.LOW, choices=PAPER_SIZES, help=_('The size of the paper. This size will be overridden when a ' 'non default output profile is used. Default is letter. Choices ' - 'are %s') % PAPER_SIZES.keys()), + 'are %s') % PAPER_SIZES), OptionRecommendation(name='custom_size', recommended_value=None, help=_('Custom size of the document. Use the form widthxheight ' 'EG. `123x321` to specify the width and height. ' 'This overrides any specified paper-size.')), OptionRecommendation(name='orientation', recommended_value='portrait', - level=OptionRecommendation.LOW, choices=ORIENTATIONS.keys(), + level=OptionRecommendation.LOW, choices=ORIENTATIONS, help=_('The orientation of the page. Default is portrait. Choices ' - 'are %s') % ORIENTATIONS.keys()), + 'are %s') % ORIENTATIONS), OptionRecommendation(name='preserve_cover_aspect_ratio', recommended_value=False, help=_('Preserve the aspect ratio of the cover, instead' @@ -105,6 +145,8 @@ class PDFOutput(OutputFormatPlugin): def convert_text(self, oeb_book): from calibre.ebooks.pdf.writer import PDFWriter + from calibre.ebooks.metadata.opf2 import OPF + self.log.debug('Serializing oeb input to disk for processing...') self.get_cover_data() diff --git a/src/calibre/ebooks/pml/output.py b/src/calibre/ebooks/conversion/plugins/pml_output.py similarity index 88% rename from src/calibre/ebooks/pml/output.py rename to src/calibre/ebooks/conversion/plugins/pml_output.py index 63d8a8b220..b406537a98 100644 --- a/src/calibre/ebooks/pml/output.py +++ b/src/calibre/ebooks/conversion/plugins/pml_output.py @@ -4,21 +4,11 @@ __license__ = 'GPL 3' __copyright__ = '2009, John Schember <john@nachtimwald.com>' __docformat__ = 'restructuredtext en' -import os +import os, cStringIO -try: - from PIL import Image - Image -except ImportError: - import Image - -import cStringIO - -from calibre.customize.conversion import OutputFormatPlugin -from calibre.customize.conversion import OptionRecommendation +from calibre.customize.conversion import (OutputFormatPlugin, + OptionRecommendation) from calibre.ptempfile import TemporaryDirectory -from calibre.utils.zipfile import ZipFile -from calibre.ebooks.pml.pmlml import PMLMLizer class PMLOutput(OutputFormatPlugin): @@ -43,6 +33,9 @@ class PMLOutput(OutputFormatPlugin): ]) def convert(self, oeb_book, output_path, input_plugin, opts, log): + from calibre.ebooks.pml.pmlml import PMLMLizer + from calibre.utils.zipfile import ZipFile + with TemporaryDirectory('_pmlz_output') as tdir: pmlmlizer = PMLMLizer(log) pml = unicode(pmlmlizer.extract_content(oeb_book, opts)) @@ -59,6 +52,13 @@ class PMLOutput(OutputFormatPlugin): pmlz.add_dir(tdir) def write_images(self, manifest, image_hrefs, out_dir, opts): + try: + from PIL import Image + Image + except ImportError: + import Image + + from calibre.ebooks.oeb.base import OEB_RASTER_IMAGES for item in manifest: if item.media_type in OEB_RASTER_IMAGES and item.href in image_hrefs.keys(): diff --git a/src/calibre/ebooks/rb/output.py b/src/calibre/ebooks/conversion/plugins/rb_output.py similarity index 95% rename from src/calibre/ebooks/rb/output.py rename to src/calibre/ebooks/conversion/plugins/rb_output.py index a16e408b0f..992843719c 100644 --- a/src/calibre/ebooks/rb/output.py +++ b/src/calibre/ebooks/conversion/plugins/rb_output.py @@ -7,7 +7,6 @@ __docformat__ = 'restructuredtext en' import os from calibre.customize.conversion import OutputFormatPlugin, OptionRecommendation -from calibre.ebooks.rb.writer import RBWriter class RBOutput(OutputFormatPlugin): @@ -22,6 +21,8 @@ class RBOutput(OutputFormatPlugin): ]) def convert(self, oeb_book, output_path, input_plugin, opts, log): + from calibre.ebooks.rb.writer import RBWriter + close = False if not hasattr(output_path, 'write'): close = True diff --git a/src/calibre/ebooks/rtf/output.py b/src/calibre/ebooks/conversion/plugins/rtf_output.py similarity index 94% rename from src/calibre/ebooks/rtf/output.py rename to src/calibre/ebooks/conversion/plugins/rtf_output.py index 5738b7e6f4..ae9e1ea566 100644 --- a/src/calibre/ebooks/rtf/output.py +++ b/src/calibre/ebooks/conversion/plugins/rtf_output.py @@ -6,7 +6,6 @@ __docformat__ = 'restructuredtext en' import os -from calibre.ebooks.rtf.rtfml import RTFMLizer from calibre.customize.conversion import OutputFormatPlugin class RTFOutput(OutputFormatPlugin): @@ -16,6 +15,8 @@ class RTFOutput(OutputFormatPlugin): file_type = 'rtf' def convert(self, oeb_book, output_path, input_plugin, opts, log): + from calibre.ebooks.rtf.rtfml import RTFMLizer + rtfmlitzer = RTFMLizer(log) content = rtfmlitzer.extract_content(oeb_book, opts) diff --git a/src/calibre/ebooks/snb/output.py b/src/calibre/ebooks/conversion/plugins/snb_output.py similarity index 98% rename from src/calibre/ebooks/snb/output.py rename to src/calibre/ebooks/conversion/plugins/snb_output.py index 07a0460c57..e9b8af0db6 100644 --- a/src/calibre/ebooks/snb/output.py +++ b/src/calibre/ebooks/conversion/plugins/snb_output.py @@ -6,12 +6,9 @@ __docformat__ = 'restructuredtext en' import os, string -from lxml import etree from calibre.customize.conversion import OutputFormatPlugin, OptionRecommendation from calibre.ptempfile import TemporaryDirectory from calibre.constants import __appname__, __version__ -from calibre.ebooks.snb.snbfile import SNBFile -from calibre.ebooks.snb.snbml import SNBMLizer, ProcessFileName class SNBOutput(OutputFormatPlugin): @@ -49,6 +46,11 @@ class SNBOutput(OutputFormatPlugin): ]) def convert(self, oeb_book, output_path, input_plugin, opts, log): + from lxml import etree + from calibre.ebooks.snb.snbfile import SNBFile + from calibre.ebooks.snb.snbml import SNBMLizer, ProcessFileName + + self.opts = opts from calibre.ebooks.oeb.transforms.rasterize import SVGRasterizer, Unavailable try: diff --git a/src/calibre/ebooks/conversion/plugins/tcr_input.py b/src/calibre/ebooks/conversion/plugins/tcr_input.py index de4f3f5f40..5ee34285bd 100644 --- a/src/calibre/ebooks/conversion/plugins/tcr_input.py +++ b/src/calibre/ebooks/conversion/plugins/tcr_input.py @@ -29,7 +29,7 @@ class TCRInput(InputFormatPlugin): txt_plugin = plugin_for_input_format('txt') for opt in txt_plugin.options: if not hasattr(self.options, opt.option.name): - setattr(self.options, opt.option.name, opt.recommended_value) + setattr(options, opt.option.name, opt.recommended_value) stream.seek(0) return txt_plugin.convert(stream, options, diff --git a/src/calibre/ebooks/tcr/output.py b/src/calibre/ebooks/conversion/plugins/tcr_output.py similarity index 93% rename from src/calibre/ebooks/tcr/output.py rename to src/calibre/ebooks/conversion/plugins/tcr_output.py index 97c9cae26c..f4dbcce57b 100644 --- a/src/calibre/ebooks/tcr/output.py +++ b/src/calibre/ebooks/conversion/plugins/tcr_output.py @@ -8,8 +8,6 @@ import os from calibre.customize.conversion import OutputFormatPlugin, \ OptionRecommendation -from calibre.ebooks.txt.txtml import TXTMLizer -from calibre.ebooks.compression.tcr import compress class TCROutput(OutputFormatPlugin): @@ -25,6 +23,9 @@ class TCROutput(OutputFormatPlugin): ]) def convert(self, oeb_book, output_path, input_plugin, opts, log): + from calibre.ebooks.txt.txtml import TXTMLizer + from calibre.ebooks.compression.tcr import compress + close = False if not hasattr(output_path, 'write'): close = True diff --git a/src/calibre/ebooks/txt/output.py b/src/calibre/ebooks/conversion/plugins/txt_output.py similarity index 93% rename from src/calibre/ebooks/txt/output.py rename to src/calibre/ebooks/conversion/plugins/txt_output.py index d9c42eb1dc..6cd4c3f801 100644 --- a/src/calibre/ebooks/txt/output.py +++ b/src/calibre/ebooks/conversion/plugins/txt_output.py @@ -7,15 +7,12 @@ __docformat__ = 'restructuredtext en' import os import shutil -from lxml import etree from calibre.customize.conversion import OutputFormatPlugin, \ OptionRecommendation -from calibre.ebooks.txt.txtml import TXTMLizer -from calibre.ebooks.txt.newlines import TxtNewlines, specified_newlines from calibre.ptempfile import TemporaryDirectory, TemporaryFile -from calibre.utils.cleantext import clean_ascii_chars -from calibre.utils.zipfile import ZipFile + +NEWLINE_TYPES = ['system', 'unix', 'old_mac', 'windows'] class TXTOutput(OutputFormatPlugin): @@ -26,11 +23,11 @@ class TXTOutput(OutputFormatPlugin): options = set([ OptionRecommendation(name='newline', recommended_value='system', level=OptionRecommendation.LOW, - short_switch='n', choices=TxtNewlines.NEWLINE_TYPES.keys(), + short_switch='n', choices=NEWLINE_TYPES, help=_('Type of newline to use. Options are %s. Default is \'system\'. ' 'Use \'old_mac\' for compatibility with Mac OS 9 and earlier. ' 'For Mac OS X use \'unix\'. \'system\' will default to the newline ' - 'type used by this OS.') % sorted(TxtNewlines.NEWLINE_TYPES.keys())), + 'type used by this OS.') % sorted(NEWLINE_TYPES)), OptionRecommendation(name='txt_output_encoding', recommended_value='utf-8', level=OptionRecommendation.LOW, help=_('Specify the character encoding of the output document. ' \ @@ -76,6 +73,11 @@ class TXTOutput(OutputFormatPlugin): ]) def convert(self, oeb_book, output_path, input_plugin, opts, log): + from calibre.ebooks.txt.txtml import TXTMLizer + from calibre.utils.cleantext import clean_ascii_chars + from calibre.ebooks.txt.newlines import specified_newlines, TxtNewlines + + if opts.txt_output_formatting.lower() == 'markdown': from calibre.ebooks.txt.markdownml import MarkdownMLizer self.writer = MarkdownMLizer(log) @@ -116,6 +118,9 @@ class TXTZOutput(TXTOutput): def convert(self, oeb_book, output_path, input_plugin, opts, log): from calibre.ebooks.oeb.base import OEB_IMAGES + from calibre.utils.zipfile import ZipFile + from lxml import etree + with TemporaryDirectory('_txtz_output') as tdir: # TXT txt_name = 'index.txt' From e02c57c71b871517c3c21f5a589c12ab28a3325f Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Sun, 5 Feb 2012 23:49:42 +0530 Subject: [PATCH 21/31] ... --- src/calibre/customize/builtins.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py index 69a867e894..9150380537 100644 --- a/src/calibre/customize/builtins.py +++ b/src/calibre/customize/builtins.py @@ -5,11 +5,10 @@ __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>' import os, glob, functools, re from calibre import guess_type -from calibre.customize import FileTypePlugin, MetadataReaderPlugin, \ - MetadataWriterPlugin, PreferencesPlugin, InterfaceActionBase, StoreBase +from calibre.customize import (FileTypePlugin, MetadataReaderPlugin, + MetadataWriterPlugin, PreferencesPlugin, InterfaceActionBase, StoreBase) from calibre.constants import numeric_version from calibre.ebooks.metadata.archive import ArchiveExtract, get_cbz_metadata -from calibre.ebooks.metadata.opf2 import metadata_to_opf from calibre.ebooks.html.to_zip import HTML2ZIP # To archive plugins {{{ @@ -86,6 +85,8 @@ class TXT2TXTZ(FileTypePlugin): return list(set(images)) def run(self, path_to_ebook): + from calibre.ebooks.metadata.opf2 import metadata_to_opf + with open(path_to_ebook, 'rb') as ebf: txt = ebf.read() base_dir = os.path.dirname(path_to_ebook) From a3286903df5e1a51f338ea59a9938900ebbf890a Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Mon, 6 Feb 2012 00:32:10 +0530 Subject: [PATCH 22/31] Speed up import of device plugins --- src/calibre/devices/apple/driver.py | 20 +++++++++++++++---- src/calibre/devices/bambook/libbambookcore.py | 3 ++- src/calibre/devices/kindle/apnx.py | 2 +- src/calibre/devices/kindle/driver.py | 7 +++++-- src/calibre/devices/prs505/sony_cache.py | 7 +++++-- src/calibre/devices/usbms/driver.py | 3 ++- 6 files changed, 31 insertions(+), 11 deletions(-) diff --git a/src/calibre/devices/apple/driver.py b/src/calibre/devices/apple/driver.py index b6d258ad81..b938a6a0a2 100644 --- a/src/calibre/devices/apple/driver.py +++ b/src/calibre/devices/apple/driver.py @@ -8,21 +8,29 @@ __docformat__ = 'restructuredtext en' import cStringIO, ctypes, datetime, os, re, shutil, sys, tempfile, time from calibre.constants import __appname__, __version__, DEBUG -from calibre import fit_image, confirm_config_name +from calibre import fit_image, confirm_config_name, strftime as _strftime from calibre.constants import isosx, iswindows from calibre.devices.errors import OpenFeedback, UserFeedback from calibre.devices.usbms.deviceconfig import DeviceConfig from calibre.devices.interface import DevicePlugin -from calibre.ebooks.BeautifulSoup import BeautifulSoup from calibre.ebooks.metadata import authors_to_string, MetaInformation, title_sort from calibre.ebooks.metadata.book.base import Metadata -from calibre.ebooks.metadata.epub import set_metadata -from calibre.library.server.utils import strftime from calibre.utils.config import config_dir, dynamic, prefs from calibre.utils.date import now, parse_date from calibre.utils.logging import Log from calibre.utils.zipfile import ZipFile +def strftime(fmt='%Y/%m/%d %H:%M:%S', dt=None): + + if not hasattr(dt, 'timetuple'): + dt = now() + dt = dt.timetuple() + try: + return _strftime(fmt, dt) + except: + return _strftime(fmt, now().timetuple()) + + class AppleOpenFeedback(OpenFeedback): @@ -1675,6 +1683,8 @@ class ITUNES(DriverBase): def _dump_epub_metadata(self, fpath): ''' ''' + from calibre.ebooks.BeautifulSoup import BeautifulSoup + self.log.info(" ITUNES.__get_epub_metadata()") title = None author = None @@ -2648,6 +2658,8 @@ class ITUNES(DriverBase): def _update_epub_metadata(self, fpath, metadata): ''' ''' + from calibre.ebooks.metadata.epub import set_metadata + if DEBUG: self.log.info(" ITUNES._update_epub_metadata()") diff --git a/src/calibre/devices/bambook/libbambookcore.py b/src/calibre/devices/bambook/libbambookcore.py index e77ac1da7b..a1c6046df0 100644 --- a/src/calibre/devices/bambook/libbambookcore.py +++ b/src/calibre/devices/bambook/libbambookcore.py @@ -8,7 +8,7 @@ __docformat__ = 'restructuredtext en' Sanda library wrapper ''' -import ctypes, uuid, hashlib, os, sys +import ctypes, hashlib, os, sys from threading import Event, Lock from calibre.constants import iswindows from calibre import load_library @@ -350,6 +350,7 @@ class Bambook: return None def SendFile(self, fileName, guid = None): + import uuid if self.handle: taskID = job.NewJob() if guid: diff --git a/src/calibre/devices/kindle/apnx.py b/src/calibre/devices/kindle/apnx.py index 178c1091f3..a051c84be6 100644 --- a/src/calibre/devices/kindle/apnx.py +++ b/src/calibre/devices/kindle/apnx.py @@ -9,7 +9,6 @@ Generates and writes an APNX page mapping file. ''' import struct -import uuid from calibre.ebooks.mobi.reader import MobiReader from calibre.ebooks.pdb.header import PdbHeaderReader @@ -51,6 +50,7 @@ class APNXBuilder(object): apnxf.write(apnx) def generate_apnx(self, pages): + import uuid apnx = '' content_vals = { diff --git a/src/calibre/devices/kindle/driver.py b/src/calibre/devices/kindle/driver.py index 3c69245cf9..1b10ce3050 100644 --- a/src/calibre/devices/kindle/driver.py +++ b/src/calibre/devices/kindle/driver.py @@ -10,10 +10,8 @@ Device driver for Amazon's Kindle import datetime, os, re, sys, json, hashlib -from calibre.devices.kindle.apnx import APNXBuilder from calibre.devices.kindle.bookmark import Bookmark from calibre.devices.usbms.driver import USBMS -from calibre.ebooks.metadata import MetaInformation from calibre import strftime ''' @@ -152,6 +150,7 @@ class KINDLE(USBMS): path_map, book_ext = resolve_bookmark_paths(storage, path_map) bookmarked_books = {} + for id in path_map: bookmark_ext = path_map[id].rpartition('.')[2] myBookmark = Bookmark(path_map[id], id, book_ext[id], bookmark_ext) @@ -236,6 +235,8 @@ class KINDLE(USBMS): def add_annotation_to_library(self, db, db_id, annotation): from calibre.ebooks.BeautifulSoup import Tag + from calibre.ebooks.metadata import MetaInformation + bm = annotation ignore_tags = set(['Catalog', 'Clippings']) @@ -363,6 +364,8 @@ class KINDLE2(KINDLE): ''' Hijacking this function to write the apnx file. ''' + from calibre.devices.kindle.apnx import APNXBuilder + opts = self.settings() if not opts.extra_customization[self.OPT_APNX]: return diff --git a/src/calibre/devices/prs505/sony_cache.py b/src/calibre/devices/prs505/sony_cache.py index 841f6bc346..979940229a 100644 --- a/src/calibre/devices/prs505/sony_cache.py +++ b/src/calibre/devices/prs505/sony_cache.py @@ -7,8 +7,6 @@ __docformat__ = 'restructuredtext en' import os, time from base64 import b64decode -from uuid import uuid4 -from lxml import etree from datetime import date from calibre import prints, guess_type, isbytestring @@ -78,6 +76,7 @@ def strftime(epoch, zone=time.localtime): return ' '.join(src) def uuid(): + from uuid import uuid4 return str(uuid4()).replace('-', '', 1).upper() # }}} @@ -85,6 +84,8 @@ def uuid(): class XMLCache(object): def __init__(self, paths, ext_paths, prefixes, use_author_sort): + from lxml import etree + if DEBUG: debug_print('Building XMLCache...', paths) self.paths = paths @@ -714,6 +715,8 @@ class XMLCache(object): def write(self): + from lxml import etree + for i, path in self.paths.items(): self.move_playlists_to_bottom() self.cleanup_whitespace(i) diff --git a/src/calibre/devices/usbms/driver.py b/src/calibre/devices/usbms/driver.py index 70b30e98a6..b061bafc03 100644 --- a/src/calibre/devices/usbms/driver.py +++ b/src/calibre/devices/usbms/driver.py @@ -10,7 +10,7 @@ driver. It is intended to be subclassed with the relevant parts implemented for a particular device. ''' -import os, re, time, json, uuid, functools, shutil +import os, re, time, json, functools, shutil from itertools import cycle from calibre.constants import numeric_version @@ -58,6 +58,7 @@ class USBMS(CLI, Device): SCAN_FROM_ROOT = False def _update_driveinfo_record(self, dinfo, prefix, location_code, name=None): + import uuid if not isinstance(dinfo, dict): dinfo = {} if dinfo.get('device_store_uuid', None) is None: From 1f97500e845443660e5652cf5a64b48f0a3e3a1c Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Mon, 6 Feb 2012 09:35:35 +0530 Subject: [PATCH 23/31] EPUB metadata: Extract the raster cover image from malformed EPUB files that specify the href instead of the id in their <meta name=cover> OPF entry --- src/calibre/ebooks/metadata/epub.py | 44 ++++++++++++++++------------- src/calibre/ebooks/metadata/opf2.py | 5 ++++ 2 files changed, 29 insertions(+), 20 deletions(-) diff --git a/src/calibre/ebooks/metadata/epub.py b/src/calibre/ebooks/metadata/epub.py index 30fe53f1a2..b5845144f8 100644 --- a/src/calibre/ebooks/metadata/epub.py +++ b/src/calibre/ebooks/metadata/epub.py @@ -129,28 +129,9 @@ class OCFDirReader(OCFReader): def open(self, path, *args, **kwargs): return open(os.path.join(self.root, path), *args, **kwargs) -def get_cover(opf, opf_path, stream, reader=None): +def render_cover(opf, opf_path, zf, reader=None): from calibre.ebooks import render_html_svg_workaround from calibre.utils.logging import default_log - raster_cover = opf.raster_cover - stream.seek(0) - zf = ZipFile(stream) - if raster_cover: - base = posixpath.dirname(opf_path) - cpath = posixpath.normpath(posixpath.join(base, raster_cover)) - if reader is not None and \ - reader.encryption_meta.is_encrypted(cpath): - return - try: - member = zf.getinfo(cpath) - except: - pass - else: - f = zf.open(member) - data = f.read() - f.close() - zf.close() - return data cpage = opf.first_spine_item() if not cpage: @@ -174,6 +155,29 @@ def get_cover(opf, opf_path, stream, reader=None): return return render_html_svg_workaround(cpage, default_log) +def get_cover(opf, opf_path, stream, reader=None): + raster_cover = opf.raster_cover + stream.seek(0) + zf = ZipFile(stream) + if raster_cover: + base = posixpath.dirname(opf_path) + cpath = posixpath.normpath(posixpath.join(base, raster_cover)) + if reader is not None and \ + reader.encryption_meta.is_encrypted(cpath): + return + try: + member = zf.getinfo(cpath) + except: + pass + else: + f = zf.open(member) + data = f.read() + f.close() + zf.close() + return data + + return render_cover(opf, opf_path, zf, reader=reader) + def get_metadata(stream, extract_cover=True): """ Return metadata as a :class:`Metadata` object """ stream.seek(0) diff --git a/src/calibre/ebooks/metadata/opf2.py b/src/calibre/ebooks/metadata/opf2.py index 9b8ae12b10..8d37e95dc4 100644 --- a/src/calibre/ebooks/metadata/opf2.py +++ b/src/calibre/ebooks/metadata/opf2.py @@ -1019,6 +1019,11 @@ class OPF(object): # {{{ mt = item.get('media-type', '') if 'xml' not in mt: return item.get('href', None) + for item in self.itermanifest(): + if item.get('href', None) == cover_id: + mt = item.get('media-type', '') + if mt.startswith('image/'): + return item.get('href', None) @dynamic_property def cover(self): From 86ad40298a3da073c6c7457779d3c34a6619cfd7 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Mon, 6 Feb 2012 10:10:42 +0530 Subject: [PATCH 24/31] When rendering EPUB covers on OS X remove @font-face rules as well to prevent multi-render crashes --- src/calibre/ebooks/metadata/epub.py | 37 +++++++++++++++++++++++------ 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/src/calibre/ebooks/metadata/epub.py b/src/calibre/ebooks/metadata/epub.py index b5845144f8..5d1661d092 100644 --- a/src/calibre/ebooks/metadata/epub.py +++ b/src/calibre/ebooks/metadata/epub.py @@ -142,17 +142,40 @@ def render_cover(opf, opf_path, zf, reader=None): with TemporaryDirectory('_epub_meta') as tdir: with CurrentDir(tdir): zf.extractall() - if isosx: - # On OS X trying to render an HTML cover which uses embedded - # fonts more than once in the same process causes a crash in Qt - # so be safe and remove the fonts. - for f in walk('.'): - if os.path.splitext(f)[1].lower() in ('.ttf', '.otf'): - os.remove(f) opf_path = opf_path.replace('/', os.sep) cpage = os.path.join(tdir, os.path.dirname(opf_path), cpage) if not os.path.exists(cpage): return + + if isosx: + # On OS X trying to render a HTML cover which uses embedded + # fonts more than once in the same process causes a crash in Qt + # so be safe and remove the fonts as well as any @font-face + # rules + for f in walk('.'): + if os.path.splitext(f)[1].lower() in ('.ttf', '.otf'): + os.remove(f) + ffpat = re.compile(br'@font-face.*?{.*?}', + re.DOTALL|re.IGNORECASE) + with open(cpage, 'r+b') as f: + raw = f.read() + f.truncate(0) + raw = ffpat.sub(b'', raw) + f.write(raw) + from calibre.ebooks.chardet import xml_to_unicode + raw = xml_to_unicode(raw, + strip_encoding_pats=True, resolve_entities=True)[0] + from lxml import html + for link in html.fromstring(raw).xpath('//link'): + href = link.get('href', '') + if href: + path = os.path.join(os.path.dirname(cpage), href) + if os.path.exists(path): + with open(path, 'r+b') as f: + raw = f.read() + f.truncate(0) + raw = ffpat.sub(b'', raw) + f.write(raw) return render_html_svg_workaround(cpage, default_log) def get_cover(opf, opf_path, stream, reader=None): From 6c199a6b58de2f8547a2378cd37dd8d24c7d3c99 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Mon, 6 Feb 2012 10:11:24 +0530 Subject: [PATCH 25/31] ... --- src/calibre/ebooks/metadata/epub.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/calibre/ebooks/metadata/epub.py b/src/calibre/ebooks/metadata/epub.py index 5d1661d092..477b805ba0 100644 --- a/src/calibre/ebooks/metadata/epub.py +++ b/src/calibre/ebooks/metadata/epub.py @@ -176,6 +176,7 @@ def render_cover(opf, opf_path, zf, reader=None): f.truncate(0) raw = ffpat.sub(b'', raw) f.write(raw) + return render_html_svg_workaround(cpage, default_log) def get_cover(opf, opf_path, stream, reader=None): From abbc03eeb9d83c0312cf4fab0162a3db7c4055d2 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Mon, 6 Feb 2012 12:23:45 +0530 Subject: [PATCH 26/31] ... --- src/calibre/gui2/metadata/single_download.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/calibre/gui2/metadata/single_download.py b/src/calibre/gui2/metadata/single_download.py index 1c0f3c8042..091c10462b 100644 --- a/src/calibre/gui2/metadata/single_download.py +++ b/src/calibre/gui2/metadata/single_download.py @@ -323,14 +323,19 @@ class Comments(QWebView): # {{{ ans = unicode(col.name()) return ans - f = QFontInfo(QApplication.font(self.parent())).pixelSize() + fi = QFontInfo(QApplication.font(self.parent())) + f = fi.pixelSize()+1 + fam = unicode(fi.family()).strip().replace('"', '') + if not fam: + fam = 'sans-serif' + c = color_to_string(QApplication.palette().color(QPalette.Normal, QPalette.WindowText)) templ = '''\ <html> <head> <style type="text/css"> - body, td {background-color: transparent; font-size: %dpx; color: %s } + body, td {background-color: transparent; font-family: %s; font-size: %dpx; color: %s } a { text-decoration: none; color: blue } div.description { margin-top: 0; padding-top: 0; text-indent: 0 } table { margin-bottom: 0; padding-bottom: 0; } @@ -342,7 +347,7 @@ class Comments(QWebView): # {{{ </div> </body> <html> - '''%(f, c) + '''%(fam, f, c) self.setHtml(templ%html) # }}} From 891f76d653ac6f4249a8e9bd6e4ae38679532cc7 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Mon, 6 Feb 2012 12:27:09 +0530 Subject: [PATCH 27/31] Delay loading for catalog plugins --- src/calibre/customize/builtins.py | 150 +- src/calibre/gui2/catalog/catalog_bibtex.py | 2 +- src/calibre/gui2/catalog/catalog_csv_xml.py | 2 +- src/calibre/library/catalog.py | 5184 ----------------- src/calibre/library/catalogs/__init__.py | 20 + src/calibre/library/catalogs/bibtex.py | 396 ++ src/calibre/library/catalogs/csv_xml.py | 214 + src/calibre/library/catalogs/epub_mobi.py | 365 ++ .../library/catalogs/epub_mobi_builder.py | 4040 +++++++++++++ src/calibre/library/catalogs/utils.py | 217 + 10 files changed, 5340 insertions(+), 5250 deletions(-) delete mode 100644 src/calibre/library/catalog.py create mode 100644 src/calibre/library/catalogs/__init__.py create mode 100644 src/calibre/library/catalogs/bibtex.py create mode 100644 src/calibre/library/catalogs/csv_xml.py create mode 100644 src/calibre/library/catalogs/epub_mobi.py create mode 100644 src/calibre/library/catalogs/epub_mobi_builder.py create mode 100644 src/calibre/library/catalogs/utils.py diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py index 9150380537..0897199289 100644 --- a/src/calibre/customize/builtins.py +++ b/src/calibre/customize/builtins.py @@ -11,6 +11,8 @@ from calibre.constants import numeric_version from calibre.ebooks.metadata.archive import ArchiveExtract, get_cbz_metadata from calibre.ebooks.html.to_zip import HTML2ZIP +plugins = [] + # To archive plugins {{{ class PML2PMLZ(FileTypePlugin): @@ -118,6 +120,7 @@ class TXT2TXTZ(FileTypePlugin): # No images so just import the TXT file. return path_to_ebook +plugins += [HTML2ZIP, PML2PMLZ, TXT2TXTZ, ArchiveExtract,] # }}} # Metadata reader plugins {{{ @@ -400,6 +403,10 @@ class ZipMetadataReader(MetadataReaderPlugin): def get_metadata(self, stream, ftype): from calibre.ebooks.metadata.zip import get_metadata return get_metadata(stream) + +plugins += [x for x in list(locals().values()) if isinstance(x, type) and \ + x.__name__.endswith('MetadataReader')] + # }}} # Metadata writer plugins {{{ @@ -500,8 +507,12 @@ class TXTZMetadataWriter(MetadataWriterPlugin): from calibre.ebooks.metadata.extz import set_metadata set_metadata(stream, mi) +plugins += [x for x in list(locals().values()) if isinstance(x, type) and \ + x.__name__.endswith('MetadataWriter')] + # }}} +# Conversion plugins {{{ from calibre.ebooks.conversion.plugins.comic_input import ComicInput from calibre.ebooks.conversion.plugins.djvu_input import DJVUInput from calibre.ebooks.conversion.plugins.epub_input import EPUBInput @@ -541,65 +552,6 @@ from calibre.ebooks.conversion.plugins.html_output import HTMLOutput from calibre.ebooks.conversion.plugins.htmlz_output import HTMLZOutput from calibre.ebooks.conversion.plugins.snb_output import SNBOutput -from calibre.customize.profiles import input_profiles, output_profiles - -from calibre.devices.apple.driver import ITUNES -from calibre.devices.hanlin.driver import HANLINV3, HANLINV5, BOOX, SPECTRA -from calibre.devices.blackberry.driver import BLACKBERRY, PLAYBOOK -from calibre.devices.cybook.driver import CYBOOK, ORIZON -from calibre.devices.eb600.driver import (EB600, COOL_ER, SHINEBOOK, - POCKETBOOK360, GER2, ITALICA, ECLICTO, DBOOK, INVESBOOK, - BOOQ, ELONEX, POCKETBOOK301, MENTOR, POCKETBOOK602, - POCKETBOOK701, POCKETBOOK360P, PI2) -from calibre.devices.iliad.driver import ILIAD -from calibre.devices.irexdr.driver import IREXDR1000, IREXDR800 -from calibre.devices.jetbook.driver import JETBOOK, MIBUK, JETBOOK_MINI -from calibre.devices.kindle.driver import (KINDLE, KINDLE2, KINDLE_DX, - KINDLE_FIRE) -from calibre.devices.nook.driver import NOOK, NOOK_COLOR -from calibre.devices.prs505.driver import PRS505 -from calibre.devices.prst1.driver import PRST1 -from calibre.devices.user_defined.driver import USER_DEFINED -from calibre.devices.android.driver import ANDROID, S60, WEBOS -from calibre.devices.nokia.driver import N770, N810, E71X, E52 -from calibre.devices.eslick.driver import ESLICK, EBK52 -from calibre.devices.nuut2.driver import NUUT2 -from calibre.devices.iriver.driver import IRIVER_STORY -from calibre.devices.binatone.driver import README -from calibre.devices.hanvon.driver import (N516, EB511, ALEX, AZBOOKA, THEBOOK, - LIBREAIR, ODYSSEY) -from calibre.devices.edge.driver import EDGE -from calibre.devices.teclast.driver import (TECLAST_K3, NEWSMY, IPAPYRUS, - SOVOS, PICO, SUNSTECH_EB700, ARCHOS7O, STASH, WEXLER) -from calibre.devices.sne.driver import SNE -from calibre.devices.misc import (PALMPRE, AVANT, SWEEX, PDNOVEL, - GEMEI, VELOCITYMICRO, PDNOVEL_KOBO, LUMIREAD, ALURATEK_COLOR, - TREKSTOR, EEEREADER, NEXTBOOK, ADAM, MOOVYBOOK, COBY, EX124G) -from calibre.devices.folder_device.driver import FOLDER_DEVICE_FOR_CONFIG -from calibre.devices.kobo.driver import KOBO -from calibre.devices.bambook.driver import BAMBOOK -from calibre.devices.boeye.driver import BOEYE_BEX, BOEYE_BDX - -from calibre.library.catalog import CSV_XML, EPUB_MOBI, BIBTEX -from calibre.ebooks.epub.fix.unmanifested import Unmanifested -from calibre.ebooks.epub.fix.epubcheck import Epubcheck - -plugins = [HTML2ZIP, PML2PMLZ, TXT2TXTZ, ArchiveExtract, CSV_XML, EPUB_MOBI, BIBTEX, Unmanifested, - Epubcheck, ] - -# New metadata download plugins {{{ -from calibre.ebooks.metadata.sources.google import GoogleBooks -from calibre.ebooks.metadata.sources.amazon import Amazon -from calibre.ebooks.metadata.sources.openlibrary import OpenLibrary -from calibre.ebooks.metadata.sources.isbndb import ISBNDB -from calibre.ebooks.metadata.sources.overdrive import OverDrive -from calibre.ebooks.metadata.sources.douban import Douban -from calibre.ebooks.metadata.sources.ozon import Ozon - -plugins += [GoogleBooks, Amazon, OpenLibrary, ISBNDB, OverDrive, Douban, Ozon] - -# }}} - plugins += [ ComicInput, DJVUInput, @@ -642,6 +594,66 @@ plugins += [ HTMLZOutput, SNBOutput, ] +# }}} + +# Catalog plugins {{{ +from calibre.library.catalogs.csv_xml import CSV_XML +from calibre.library.catalogs.bibtex import BIBTEX +from calibre.library.catalogs.epub_mobi import EPUB_MOBI +plugins += [CSV_XML, BIBTEX, EPUB_MOBI] +# }}} + +# EPUB Fix plugins {{{ +from calibre.ebooks.epub.fix.unmanifested import Unmanifested +from calibre.ebooks.epub.fix.epubcheck import Epubcheck +plugins += [Unmanifested, Epubcheck] +# }}} + +# Profiles {{{ +from calibre.customize.profiles import input_profiles, output_profiles +plugins += input_profiles + output_profiles +# }}} + +# Device driver plugins {{{ +from calibre.devices.apple.driver import ITUNES +from calibre.devices.hanlin.driver import HANLINV3, HANLINV5, BOOX, SPECTRA +from calibre.devices.blackberry.driver import BLACKBERRY, PLAYBOOK +from calibre.devices.cybook.driver import CYBOOK, ORIZON +from calibre.devices.eb600.driver import (EB600, COOL_ER, SHINEBOOK, + POCKETBOOK360, GER2, ITALICA, ECLICTO, DBOOK, INVESBOOK, + BOOQ, ELONEX, POCKETBOOK301, MENTOR, POCKETBOOK602, + POCKETBOOK701, POCKETBOOK360P, PI2) +from calibre.devices.iliad.driver import ILIAD +from calibre.devices.irexdr.driver import IREXDR1000, IREXDR800 +from calibre.devices.jetbook.driver import JETBOOK, MIBUK, JETBOOK_MINI +from calibre.devices.kindle.driver import (KINDLE, KINDLE2, KINDLE_DX, + KINDLE_FIRE) +from calibre.devices.nook.driver import NOOK, NOOK_COLOR +from calibre.devices.prs505.driver import PRS505 +from calibre.devices.prst1.driver import PRST1 +from calibre.devices.user_defined.driver import USER_DEFINED +from calibre.devices.android.driver import ANDROID, S60, WEBOS +from calibre.devices.nokia.driver import N770, N810, E71X, E52 +from calibre.devices.eslick.driver import ESLICK, EBK52 +from calibre.devices.nuut2.driver import NUUT2 +from calibre.devices.iriver.driver import IRIVER_STORY +from calibre.devices.binatone.driver import README +from calibre.devices.hanvon.driver import (N516, EB511, ALEX, AZBOOKA, THEBOOK, + LIBREAIR, ODYSSEY) +from calibre.devices.edge.driver import EDGE +from calibre.devices.teclast.driver import (TECLAST_K3, NEWSMY, IPAPYRUS, + SOVOS, PICO, SUNSTECH_EB700, ARCHOS7O, STASH, WEXLER) +from calibre.devices.sne.driver import SNE +from calibre.devices.misc import (PALMPRE, AVANT, SWEEX, PDNOVEL, + GEMEI, VELOCITYMICRO, PDNOVEL_KOBO, LUMIREAD, ALURATEK_COLOR, + TREKSTOR, EEEREADER, NEXTBOOK, ADAM, MOOVYBOOK, COBY, EX124G) +from calibre.devices.folder_device.driver import FOLDER_DEVICE_FOR_CONFIG +from calibre.devices.kobo.driver import KOBO +from calibre.devices.bambook.driver import BAMBOOK +from calibre.devices.boeye.driver import BOEYE_BEX, BOEYE_BDX + + + # Order here matters. The first matched device is the one used. plugins += [ HANLINV3, @@ -716,11 +728,20 @@ plugins += [ BOEYE_BDX, USER_DEFINED, ] -plugins += [x for x in list(locals().values()) if isinstance(x, type) and \ - x.__name__.endswith('MetadataReader')] -plugins += [x for x in list(locals().values()) if isinstance(x, type) and \ - x.__name__.endswith('MetadataWriter')] -plugins += input_profiles + output_profiles +# }}} + +# New metadata download plugins {{{ +from calibre.ebooks.metadata.sources.google import GoogleBooks +from calibre.ebooks.metadata.sources.amazon import Amazon +from calibre.ebooks.metadata.sources.openlibrary import OpenLibrary +from calibre.ebooks.metadata.sources.isbndb import ISBNDB +from calibre.ebooks.metadata.sources.overdrive import OverDrive +from calibre.ebooks.metadata.sources.douban import Douban +from calibre.ebooks.metadata.sources.ozon import Ozon + +plugins += [GoogleBooks, Amazon, OpenLibrary, ISBNDB, OverDrive, Douban, Ozon] + +# }}} # Interface Actions {{{ @@ -1623,3 +1644,4 @@ plugins += [ ] # }}} + diff --git a/src/calibre/gui2/catalog/catalog_bibtex.py b/src/calibre/gui2/catalog/catalog_bibtex.py index 58dae47a90..65999bbdd3 100644 --- a/src/calibre/gui2/catalog/catalog_bibtex.py +++ b/src/calibre/gui2/catalog/catalog_bibtex.py @@ -31,7 +31,7 @@ class PluginWidget(QWidget, Ui_Form): self.setupUi(self) def initialize(self, name, db): #not working properly to update - from calibre.library.catalog import FIELDS + from calibre.library.catalogs import FIELDS self.all_fields = [x for x in FIELDS if x != 'all'] #add custom columns diff --git a/src/calibre/gui2/catalog/catalog_csv_xml.py b/src/calibre/gui2/catalog/catalog_csv_xml.py index ef322f2db9..96acbdfa9b 100644 --- a/src/calibre/gui2/catalog/catalog_csv_xml.py +++ b/src/calibre/gui2/catalog/catalog_csv_xml.py @@ -21,7 +21,7 @@ class PluginWidget(QWidget, Ui_Form): def __init__(self, parent=None): QWidget.__init__(self, parent) self.setupUi(self) - from calibre.library.catalog import FIELDS + from calibre.library.catalogs import FIELDS self.all_fields = [] for x in FIELDS: if x != 'all': diff --git a/src/calibre/library/catalog.py b/src/calibre/library/catalog.py deleted file mode 100644 index f30e200296..0000000000 --- a/src/calibre/library/catalog.py +++ /dev/null @@ -1,5184 +0,0 @@ -# -*- coding: utf-8 -*- - -__license__ = 'GPL v3' -__copyright__ = '2010, Greg Riker' - -import codecs, datetime, htmlentitydefs, os, re, shutil, zlib -from collections import namedtuple -from copy import deepcopy -from xml.sax.saxutils import escape -from lxml import etree -from types import StringType, UnicodeType - -from calibre import (prints, prepare_string_for_xml, strftime, force_unicode) -from calibre.constants import preferred_encoding, DEBUG -from calibre.customize import CatalogPlugin -from calibre.customize.conversion import OptionRecommendation, DummyReporter -from calibre.ebooks.BeautifulSoup import BeautifulSoup, BeautifulStoneSoup, Tag, NavigableString -from calibre.ebooks.chardet import substitute_entites -from calibre.library.save_to_disk import preprocess_template -from calibre.ptempfile import PersistentTemporaryDirectory -from calibre.utils.bibtex import BibTeX -from calibre.utils.config import config_dir -from calibre.utils.date import format_date, isoformat, is_date_undefined, now as nowf -from calibre.utils.html2text import html2text -from calibre.utils.icu import capitalize -from calibre.utils.logging import default_log as log -from calibre.utils.magick.draw import thumbnail -from calibre.utils.zipfile import ZipFile - -FIELDS = ['all', 'title', 'title_sort', 'author_sort', 'authors', 'comments', - 'cover', 'formats','id', 'isbn', 'ondevice', 'pubdate', 'publisher', - 'rating', 'series_index', 'series', 'size', 'tags', 'timestamp', - 'uuid', 'languages'] - -#Allowed fields for template -TEMPLATE_ALLOWED_FIELDS = [ 'author_sort', 'authors', 'id', 'isbn', 'pubdate', 'title_sort', - 'publisher', 'series_index', 'series', 'tags', 'timestamp', 'title', 'uuid' ] - -class CSV_XML(CatalogPlugin): # {{{ - 'CSV/XML catalog generator' - - Option = namedtuple('Option', 'option, default, dest, action, help') - - name = 'Catalog_CSV_XML' - description = 'CSV/XML catalog generator' - supported_platforms = ['windows', 'osx', 'linux'] - author = 'Greg Riker' - version = (1, 0, 0) - file_types = set(['csv','xml']) - - cli_options = [ - Option('--fields', - default = 'all', - dest = 'fields', - action = None, - help = _('The fields to output when cataloging books in the ' - 'database. Should be a comma-separated list of fields.\n' - 'Available fields: %(fields)s,\n' - 'plus user-created custom fields.\n' - 'Example: %(opt)s=title,authors,tags\n' - "Default: '%%default'\n" - "Applies to: CSV, XML output formats")%dict( - fields=', '.join(FIELDS), opt='--fields')), - - Option('--sort-by', - default = 'id', - dest = 'sort_by', - action = None, - help = _('Output field to sort on.\n' - 'Available fields: author_sort, id, rating, size, timestamp, title_sort\n' - "Default: '%default'\n" - "Applies to: CSV, XML output formats"))] - - def run(self, path_to_output, opts, db, notification=DummyReporter()): - self.fmt = path_to_output.rpartition('.')[2] - self.notification = notification - - if opts.verbose: - opts_dict = vars(opts) - log("%s(): Generating %s" % (self.name,self.fmt.upper())) - if opts.connected_device['is_device_connected']: - log(" connected_device: %s" % opts.connected_device['name']) - if opts_dict['search_text']: - log(" --search='%s'" % opts_dict['search_text']) - - if opts_dict['ids']: - log(" Book count: %d" % len(opts_dict['ids'])) - if opts_dict['search_text']: - log(" (--search ignored when a subset of the database is specified)") - - if opts_dict['fields']: - if opts_dict['fields'] == 'all': - log(" Fields: %s" % ', '.join(FIELDS[1:])) - else: - log(" Fields: %s" % opts_dict['fields']) - - # If a list of ids are provided, don't use search_text - if opts.ids: - opts.search_text = None - - data = self.search_sort_db(db, opts) - - if not len(data): - log.error("\nNo matching database entries for search criteria '%s'" % opts.search_text) - #raise SystemExit(1) - - # Get the requested output fields as a list - fields = self.get_output_fields(db, opts) - - # If connected device, add 'On Device' values to data - if opts.connected_device['is_device_connected'] and 'ondevice' in fields: - for entry in data: - entry['ondevice'] = db.catalog_plugin_on_device_temp_mapping[entry['id']]['ondevice'] - - if self.fmt == 'csv': - outfile = codecs.open(path_to_output, 'w', 'utf8') - - # Write a UTF-8 BOM - outfile.write('\xef\xbb\xbf') - - # Output the field headers - outfile.write(u'%s\n' % u','.join(fields)) - - # Output the entry fields - for entry in data: - outstr = [] - for field in fields: - if field.startswith('#'): - item = db.get_field(entry['id'],field,index_is_id=True) - elif field == 'title_sort': - item = entry['sort'] - else: - item = entry[field] - - if item is None: - outstr.append('""') - continue - elif field == 'formats': - fmt_list = [] - for format in item: - fmt_list.append(format.rpartition('.')[2].lower()) - item = ', '.join(fmt_list) - elif field in ['authors','tags']: - item = ', '.join(item) - elif field == 'isbn': - # Could be 9, 10 or 13 digits - item = u'%s' % re.sub(r'[\D]', '', item) - elif field in ['pubdate', 'timestamp']: - item = isoformat(item) - elif field == 'comments': - item = item.replace(u'\r\n',u' ') - item = item.replace(u'\n',u' ') - - # Convert HTML to markdown text - if type(item) is unicode: - opening_tag = re.search('<(\w+)(\x20|>)',item) - if opening_tag: - closing_tag = re.search('<\/%s>$' % opening_tag.group(1), item) - if closing_tag: - item = html2text(item) - - outstr.append(u'"%s"' % unicode(item).replace('"','""')) - - outfile.write(u','.join(outstr) + u'\n') - outfile.close() - - elif self.fmt == 'xml': - from lxml.builder import E - - root = E.calibredb() - for r in data: - record = E.record() - root.append(record) - - for field in fields: - if field.startswith('#'): - val = db.get_field(r['id'],field,index_is_id=True) - if not isinstance(val, (str, unicode)): - val = unicode(val) - item = getattr(E, field.replace('#','_'))(val) - record.append(item) - - for field in ('id', 'uuid', 'publisher', 'rating', 'size', - 'isbn','ondevice'): - if field in fields: - val = r[field] - if not val: - continue - if not isinstance(val, (str, unicode)): - val = unicode(val) - item = getattr(E, field)(val) - record.append(item) - - if 'title' in fields: - title = E.title(r['title'], sort=r['sort']) - record.append(title) - - if 'authors' in fields: - aus = E.authors(sort=r['author_sort']) - for au in r['authors']: - aus.append(E.author(au)) - record.append(aus) - - for field in ('timestamp', 'pubdate'): - if field in fields: - record.append(getattr(E, field)(r[field].isoformat())) - - if 'tags' in fields and r['tags']: - tags = E.tags() - for tag in r['tags']: - tags.append(E.tag(tag)) - record.append(tags) - - if 'comments' in fields and r['comments']: - record.append(E.comments(r['comments'])) - - if 'series' in fields and r['series']: - record.append(E.series(r['series'], - index=str(r['series_index']))) - - if 'cover' in fields and r['cover']: - record.append(E.cover(r['cover'].replace(os.sep, '/'))) - - if 'formats' in fields and r['formats']: - fmt = E.formats() - for f in r['formats']: - fmt.append(E.format(f.replace(os.sep, '/'))) - record.append(fmt) - - with open(path_to_output, 'w') as f: - f.write(etree.tostring(root, encoding='utf-8', - xml_declaration=True, pretty_print=True)) -# }}} - -class BIBTEX(CatalogPlugin): # {{{ - 'BIBTEX catalog generator' - - Option = namedtuple('Option', 'option, default, dest, action, help') - - name = 'Catalog_BIBTEX' - description = 'BIBTEX catalog generator' - supported_platforms = ['windows', 'osx', 'linux'] - author = 'Sengian' - version = (1, 0, 0) - file_types = set(['bib']) - - cli_options = [ - Option('--fields', - default = 'all', - dest = 'fields', - action = None, - help = _('The fields to output when cataloging books in the ' - 'database. Should be a comma-separated list of fields.\n' - 'Available fields: %(fields)s.\n' - 'plus user-created custom fields.\n' - 'Example: %(opt)s=title,authors,tags\n' - "Default: '%%default'\n" - "Applies to: BIBTEX output format")%dict( - fields=', '.join(FIELDS), opt='--fields')), - - Option('--sort-by', - default = 'id', - dest = 'sort_by', - action = None, - help = _('Output field to sort on.\n' - 'Available fields: author_sort, id, rating, size, timestamp, title.\n' - "Default: '%default'\n" - "Applies to: BIBTEX output format")), - - Option('--create-citation', - default = 'True', - dest = 'impcit', - action = None, - help = _('Create a citation for BibTeX entries.\n' - 'Boolean value: True, False\n' - "Default: '%default'\n" - "Applies to: BIBTEX output format")), - - Option('--add-files-path', - default = 'True', - dest = 'addfiles', - action = None, - help = _('Create a file entry if formats is selected for BibTeX entries.\n' - 'Boolean value: True, False\n' - "Default: '%default'\n" - "Applies to: BIBTEX output format")), - - Option('--citation-template', - default = '{authors}{id}', - dest = 'bib_cit', - action = None, - help = _('The template for citation creation from database fields.\n' - 'Should be a template with {} enclosed fields.\n' - 'Available fields: %s.\n' - "Default: '%%default'\n" - "Applies to: BIBTEX output format")%', '.join(TEMPLATE_ALLOWED_FIELDS)), - - Option('--choose-encoding', - default = 'utf8', - dest = 'bibfile_enc', - action = None, - help = _('BibTeX file encoding output.\n' - 'Available types: utf8, cp1252, ascii.\n' - "Default: '%default'\n" - "Applies to: BIBTEX output format")), - - Option('--choose-encoding-configuration', - default = 'strict', - dest = 'bibfile_enctag', - action = None, - help = _('BibTeX file encoding flag.\n' - 'Available types: strict, replace, ignore, backslashreplace.\n' - "Default: '%default'\n" - "Applies to: BIBTEX output format")), - - Option('--entry-type', - default = 'book', - dest = 'bib_entry', - action = None, - help = _('Entry type for BibTeX catalog.\n' - 'Available types: book, misc, mixed.\n' - "Default: '%default'\n" - "Applies to: BIBTEX output format"))] - - def run(self, path_to_output, opts, db, notification=DummyReporter()): - - def create_bibtex_entry(entry, fields, mode, template_citation, - bibtexdict, db, citation_bibtex=True, calibre_files=True): - - #Bibtex doesn't like UTF-8 but keep unicode until writing - #Define starting chain or if book valid strict and not book return a Fail string - - bibtex_entry = [] - if mode != "misc" and check_entry_book_valid(entry) : - bibtex_entry.append(u'@book{') - elif mode != "book" : - bibtex_entry.append(u'@misc{') - else : - #case strict book - return '' - - if citation_bibtex : - # Citation tag - bibtex_entry.append(make_bibtex_citation(entry, template_citation, - bibtexdict)) - bibtex_entry = [u' '.join(bibtex_entry)] - - for field in fields: - if field.startswith('#'): - item = db.get_field(entry['id'],field,index_is_id=True) - if isinstance(item, (bool, float, int)): - item = repr(item) - elif field == 'title_sort': - item = entry['sort'] - else: - item = entry[field] - - #check if the field should be included (none or empty) - if item is None: - continue - try: - if len(item) == 0 : - continue - except TypeError: - pass - - if field == 'authors' : - bibtex_entry.append(u'author = "%s"' % bibtexdict.bibtex_author_format(item)) - - elif field == 'id' : - bibtex_entry.append(u'calibreid = "%s"' % int(item)) - - elif field == 'rating' : - bibtex_entry.append(u'rating = "%s"' % int(item)) - - elif field == 'size' : - bibtex_entry.append(u'%s = "%s octets"' % (field, int(item))) - - elif field == 'tags' : - #A list to flatten - bibtex_entry.append(u'tags = "%s"' % bibtexdict.utf8ToBibtex(u', '.join(item))) - - elif field == 'comments' : - #\n removal - item = item.replace(u'\r\n',u' ') - item = item.replace(u'\n',u' ') - #html to text - try: - item = html2text(item) - except: - log.warn("Failed to convert comments to text") - bibtex_entry.append(u'note = "%s"' % bibtexdict.utf8ToBibtex(item)) - - elif field == 'isbn' : - # Could be 9, 10 or 13 digits - bibtex_entry.append(u'isbn = "%s"' % re.sub(u'[0-9xX]', u'', item)) - - elif field == 'formats' : - #Add file path if format is selected - formats = [format.rpartition('.')[2].lower() for format in item] - bibtex_entry.append(u'formats = "%s"' % u', '.join(formats)) - if calibre_files: - files = [u':%s:%s' % (format, format.rpartition('.')[2].upper())\ - for format in item] - bibtex_entry.append(u'file = "%s"' % u', '.join(files)) - - elif field == 'series_index' : - bibtex_entry.append(u'volume = "%s"' % int(item)) - - elif field == 'timestamp' : - bibtex_entry.append(u'timestamp = "%s"' % isoformat(item).partition('T')[0]) - - elif field == 'pubdate' : - bibtex_entry.append(u'year = "%s"' % item.year) - bibtex_entry.append(u'month = "%s"' % bibtexdict.utf8ToBibtex(strftime("%b", item))) - - elif field.startswith('#') : - bibtex_entry.append(u'custom_%s = "%s"' % (field[1:], - bibtexdict.utf8ToBibtex(item))) - - else: - # elif field in ['title', 'publisher', 'cover', 'uuid', 'ondevice', - # 'author_sort', 'series', 'title_sort'] : - bibtex_entry.append(u'%s = "%s"' % (field, bibtexdict.utf8ToBibtex(item))) - - bibtex_entry = u',\n '.join(bibtex_entry) - bibtex_entry += u' }\n\n' - - return bibtex_entry - - def check_entry_book_valid(entry): - #Check that the required fields are ok for a book entry - for field in ['title', 'authors', 'publisher'] : - if entry[field] is None or len(entry[field]) == 0 : - return False - if entry['pubdate'] is None : - return False - else : - return True - - def make_bibtex_citation(entry, template_citation, bibtexclass): - - #define a function to replace the template entry by its value - def tpl_replace(objtplname) : - - tpl_field = re.sub(u'[\{\}]', u'', objtplname.group()) - - if tpl_field in TEMPLATE_ALLOWED_FIELDS : - if tpl_field in ['pubdate', 'timestamp'] : - tpl_field = isoformat(entry[tpl_field]).partition('T')[0] - elif tpl_field in ['tags', 'authors'] : - tpl_field =entry[tpl_field][0] - elif tpl_field in ['id', 'series_index'] : - tpl_field = str(entry[tpl_field]) - else : - tpl_field = entry[tpl_field] - return tpl_field - else: - return u'' - - if len(template_citation) >0 : - tpl_citation = bibtexclass.utf8ToBibtex( - bibtexclass.ValidateCitationKey(re.sub(u'\{[^{}]*\}', - tpl_replace, template_citation))) - - if len(tpl_citation) >0 : - return tpl_citation - - if len(entry["isbn"]) > 0 : - template_citation = u'%s' % re.sub(u'[\D]',u'', entry["isbn"]) - - else : - template_citation = u'%s' % str(entry["id"]) - - return bibtexclass.ValidateCitationKey(template_citation) - - self.fmt = path_to_output.rpartition('.')[2] - self.notification = notification - - # Combobox options - bibfile_enc = ['utf8', 'cp1252', 'ascii'] - bibfile_enctag = ['strict', 'replace', 'ignore', 'backslashreplace'] - bib_entry = ['mixed', 'misc', 'book'] - - # Needed beacause CLI return str vs int by widget - try: - bibfile_enc = bibfile_enc[opts.bibfile_enc] - bibfile_enctag = bibfile_enctag[opts.bibfile_enctag] - bib_entry = bib_entry[opts.bib_entry] - except: - if opts.bibfile_enc in bibfile_enc : - bibfile_enc = opts.bibfile_enc - else : - log.warn("Incorrect --choose-encoding flag, revert to default") - bibfile_enc = bibfile_enc[0] - if opts.bibfile_enctag in bibfile_enctag : - bibfile_enctag = opts.bibfile_enctag - else : - log.warn("Incorrect --choose-encoding-configuration flag, revert to default") - bibfile_enctag = bibfile_enctag[0] - if opts.bib_entry in bib_entry : - bib_entry = opts.bib_entry - else : - log.warn("Incorrect --entry-type flag, revert to default") - bib_entry = bib_entry[0] - - if opts.verbose: - opts_dict = vars(opts) - log("%s(): Generating %s" % (self.name,self.fmt)) - if opts.connected_device['is_device_connected']: - log(" connected_device: %s" % opts.connected_device['name']) - if opts_dict['search_text']: - log(" --search='%s'" % opts_dict['search_text']) - - if opts_dict['ids']: - log(" Book count: %d" % len(opts_dict['ids'])) - if opts_dict['search_text']: - log(" (--search ignored when a subset of the database is specified)") - - if opts_dict['fields']: - if opts_dict['fields'] == 'all': - log(" Fields: %s" % ', '.join(FIELDS[1:])) - else: - log(" Fields: %s" % opts_dict['fields']) - - log(" Output file will be encoded in %s with %s flag" % (bibfile_enc, bibfile_enctag)) - - log(" BibTeX entry type is %s with a citation like '%s' flag" % (bib_entry, opts_dict['bib_cit'])) - - # If a list of ids are provided, don't use search_text - if opts.ids: - opts.search_text = None - - data = self.search_sort_db(db, opts) - - if not len(data): - log.error("\nNo matching database entries for search criteria '%s'" % opts.search_text) - - # Get the requested output fields as a list - fields = self.get_output_fields(db, opts) - - if not len(data): - log.error("\nNo matching database entries for search criteria '%s'" % opts.search_text) - - #Initialize BibTeX class - bibtexc = BibTeX() - - #Entries writing after Bibtex formating (or not) - if bibfile_enc != 'ascii' : - bibtexc.ascii_bibtex = False - else : - bibtexc.ascii_bibtex = True - - #Check citation choice and go to default in case of bad CLI - if isinstance(opts.impcit, (StringType, UnicodeType)) : - if opts.impcit == 'False' : - citation_bibtex= False - elif opts.impcit == 'True' : - citation_bibtex= True - else : - log.warn("Incorrect --create-citation, revert to default") - citation_bibtex= True - else : - citation_bibtex= opts.impcit - - #Check add file entry and go to default in case of bad CLI - if isinstance(opts.addfiles, (StringType, UnicodeType)) : - if opts.addfiles == 'False' : - addfiles_bibtex = False - elif opts.addfiles == 'True' : - addfiles_bibtex = True - else : - log.warn("Incorrect --add-files-path, revert to default") - addfiles_bibtex= True - else : - addfiles_bibtex = opts.addfiles - - #Preprocess for error and light correction - template_citation = preprocess_template(opts.bib_cit) - - #Open output and write entries - with codecs.open(path_to_output, 'w', bibfile_enc, bibfile_enctag)\ - as outfile: - #File header - nb_entries = len(data) - - #check in book strict if all is ok else throw a warning into log - if bib_entry == 'book' : - nb_books = len(filter(check_entry_book_valid, data)) - if nb_books < nb_entries : - log.warn("Only %d entries in %d are book compatible" % (nb_books, nb_entries)) - nb_entries = nb_books - - # If connected device, add 'On Device' values to data - if opts.connected_device['is_device_connected'] and 'ondevice' in fields: - for entry in data: - entry['ondevice'] = db.catalog_plugin_on_device_temp_mapping[entry['id']]['ondevice'] - - outfile.write(u'%%%Calibre catalog\n%%%{0} entries in catalog\n\n'.format(nb_entries)) - outfile.write(u'@preamble{"This catalog of %d entries was generated by calibre on %s"}\n\n' - % (nb_entries, nowf().strftime("%A, %d. %B %Y %H:%M").decode(preferred_encoding))) - - for entry in data: - outfile.write(create_bibtex_entry(entry, fields, bib_entry, template_citation, - bibtexc, db, citation_bibtex, addfiles_bibtex)) -# }}} - -class EPUB_MOBI(CatalogPlugin): # {{{ - 'ePub catalog generator' - - Option = namedtuple('Option', 'option, default, dest, action, help') - - name = 'Catalog_EPUB_MOBI' - description = 'EPUB/MOBI catalog generator' - supported_platforms = ['windows', 'osx', 'linux'] - minimum_calibre_version = (0, 7, 40) - author = 'Greg Riker' - version = (1, 0, 0) - file_types = set(['epub','mobi']) - - THUMB_SMALLEST = "1.0" - THUMB_LARGEST = "2.0" - - ''' - # Deprecated, keeping this just in case there are complaints - Option('--numbers-as-text', - default=False, - dest='numbers_as_text', - action = None, - help=_("Sort titles with leading numbers as text, e.g.,\n'2001: A Space Odyssey' sorts as \n'Two Thousand One: A Space Odyssey'.\n" - "Default: '%default'\n" - "Applies to: ePub, MOBI output formats")), - ''' - - cli_options = [Option('--catalog-title', - default = 'My Books', - dest = 'catalog_title', - action = None, - help = _('Title of generated catalog used as title in metadata.\n' - "Default: '%default'\n" - "Applies to: ePub, MOBI output formats")), - Option('--debug-pipeline', - default=None, - dest='debug_pipeline', - action = None, - help=_("Save the output from different stages of the conversion " - "pipeline to the specified " - "directory. Useful if you are unsure at which stage " - "of the conversion process a bug is occurring.\n" - "Default: '%default'\n" - "Applies to: ePub, MOBI output formats")), - Option('--exclude-book-marker', - default=':', - dest='exclude_book_marker', - action = None, - help=_("field:pattern specifying custom field/contents indicating book should be excluded.\n" - "Default: '%default'\n" - "Applies to ePub, MOBI output formats")), - Option('--exclude-genre', - default='\[.+\]', - dest='exclude_genre', - action = None, - help=_("Regex describing tags to exclude as genres.\n" "Default: '%default' excludes bracketed tags, e.g. '[<tag>]'\n" - "Applies to: ePub, MOBI output formats")), - Option('--exclude-tags', - default=('~,'+_('Catalog')), - dest='exclude_tags', - action = None, - help=_("Comma-separated list of tag words indicating book should be excluded from output. " - "For example: 'skip' will match 'skip this book' and 'Skip will like this'. " - "Default: '%default'\n" - "Applies to: ePub, MOBI output formats")), - Option('--generate-authors', - default=False, - dest='generate_authors', - action = 'store_true', - help=_("Include 'Authors' section in catalog.\n" - "Default: '%default'\n" - "Applies to: ePub, MOBI output formats")), - Option('--generate-descriptions', - default=False, - dest='generate_descriptions', - action = 'store_true', - help=_("Include 'Descriptions' section in catalog.\n" - "Default: '%default'\n" - "Applies to: ePub, MOBI output formats")), - Option('--generate-genres', - default=False, - dest='generate_genres', - action = 'store_true', - help=_("Include 'Genres' section in catalog.\n" - "Default: '%default'\n" - "Applies to: ePub, MOBI output formats")), - Option('--generate-titles', - default=False, - dest='generate_titles', - action = 'store_true', - help=_("Include 'Titles' section in catalog.\n" - "Default: '%default'\n" - "Applies to: ePub, MOBI output formats")), - Option('--generate-series', - default=False, - dest='generate_series', - action = 'store_true', - help=_("Include 'Series' section in catalog.\n" - "Default: '%default'\n" - "Applies to: ePub, MOBI output formats")), - Option('--generate-recently-added', - default=False, - dest='generate_recently_added', - action = 'store_true', - help=_("Include 'Recently Added' section in catalog.\n" - "Default: '%default'\n" - "Applies to: ePub, MOBI output formats")), - Option('--header-note-source-field', - default='', - dest='header_note_source_field', - action = None, - help=_("Custom field containing note text to insert in Description header.\n" - "Default: '%default'\n" - "Applies to: ePub, MOBI output formats")), - Option('--merge-comments', - default='::', - dest='merge_comments', - action = None, - help=_("<custom field>:[before|after]:[True|False] specifying:\n" - " <custom field> Custom field containing notes to merge with Comments\n" - " [before|after] Placement of notes with respect to Comments\n" - " [True|False] - A horizontal rule is inserted between notes and Comments\n" - "Default: '%default'\n" - "Applies to ePub, MOBI output formats")), - Option('--output-profile', - default=None, - dest='output_profile', - action = None, - help=_("Specifies the output profile. In some cases, an output profile is required to optimize the catalog for the device. For example, 'kindle' or 'kindle_dx' creates a structured Table of Contents with Sections and Articles.\n" - "Default: '%default'\n" - "Applies to: ePub, MOBI output formats")), - Option('--read-book-marker', - default='tag:+', - dest='read_book_marker', - action = None, - help=_("field:pattern indicating book has been read.\n" "Default: '%default'\n" - "Applies to ePub, MOBI output formats")), - Option('--thumb-width', - default='1.0', - dest='thumb_width', - action = None, - help=_("Size hint (in inches) for book covers in catalog.\n" - "Range: 1.0 - 2.0\n" - "Default: '%default'\n" - "Applies to ePub, MOBI output formats")), - Option('--wishlist-tag', - default='Wishlist', - dest='wishlist_tag', - action = None, - help=_("Tag indicating book to be displayed as wishlist item.\n" "Default: '%default'\n" - "Applies to: ePub, MOBI output formats")), - ] - - class NumberToText(object): - ''' - Converts numbers to text - 4.56 => four point fifty-six - 456 => four hundred fifty-six - 4:56 => four fifty-six - ''' - ORDINALS = ['zeroth','first','second','third','fourth','fifth','sixth','seventh','eighth','ninth'] - lessThanTwenty = ["<zero>","one","two","three","four","five","six","seven","eight","nine", - "ten","eleven","twelve","thirteen","fourteen","fifteen","sixteen","seventeen", - "eighteen","nineteen"] - tens = ["<zero>","<tens>","twenty","thirty","forty","fifty","sixty","seventy","eighty","ninety"] - hundreds = ["<zero>","one","two","three","four","five","six","seven","eight","nine"] - - def __init__(self, number, verbose=False): - self.number = number - self.number_as_float = 0.0 - self.text = '' - self.verbose = verbose - self.log = log - self.numberTranslate() - - def stringFromInt(self, intToTranslate): - # Convert intToTranslate to string - # intToTranslate is a three-digit number - - tensComponentString = "" - hundredsComponent = intToTranslate - (intToTranslate % 100) - tensComponent = intToTranslate % 100 - - # Build the hundreds component - if hundredsComponent: - hundredsComponentString = "%s hundred" % self.hundreds[hundredsComponent/100] - else: - hundredsComponentString = "" - - # Build the tens component - if tensComponent < 20: - tensComponentString = self.lessThanTwenty[tensComponent] - else: - tensPart = "" - onesPart = "" - - # Get the tens part - tensPart = self.tens[tensComponent / 10] - onesPart = self.lessThanTwenty[tensComponent % 10] - - if intToTranslate % 10: - tensComponentString = "%s-%s" % (tensPart, onesPart) - else: - tensComponentString = "%s" % tensPart - - # Concatenate the results - result = '' - if hundredsComponent and not tensComponent: - result = hundredsComponentString - elif not hundredsComponent and tensComponent: - result = tensComponentString - elif hundredsComponent and tensComponent: - result = hundredsComponentString + " " + tensComponentString - else: - prints(" NumberToText.stringFromInt(): empty result translating %d" % intToTranslate) - return result - - def numberTranslate(self): - hundredsNumber = 0 - thousandsNumber = 0 - hundredsString = "" - thousandsString = "" - resultString = "" - self.suffix = '' - - if self.verbose: self.log("numberTranslate(): %s" % self.number) - - # Special case ordinals - if re.search('[st|nd|rd|th]',self.number): - self.number = re.sub(',','',self.number) - ordinal_suffix = re.search('[\D]', self.number) - ordinal_number = re.sub('\D','',re.sub(',','',self.number)) - if self.verbose: self.log("Ordinal: %s" % ordinal_number) - self.number_as_float = ordinal_number - self.suffix = self.number[ordinal_suffix.start():] - if int(ordinal_number) > 9: - # Some typos (e.g., 'twentyth'), acceptable - self.text = '%s' % (EPUB_MOBI.NumberToText(ordinal_number).text) - else: - self.text = '%s' % (self.ORDINALS[int(ordinal_number)]) - - # Test for time - elif re.search(':',self.number): - if self.verbose: self.log("Time: %s" % self.number) - self.number_as_float = re.sub(':','.',self.number) - time_strings = self.number.split(":") - hours = EPUB_MOBI.NumberToText(time_strings[0]).text - minutes = EPUB_MOBI.NumberToText(time_strings[1]).text - self.text = '%s-%s' % (hours.capitalize(), minutes) - - # Test for % - elif re.search('%', self.number): - if self.verbose: self.log("Percent: %s" % self.number) - self.number_as_float = self.number.split('%')[0] - self.text = EPUB_MOBI.NumberToText(self.number.replace('%',' percent')).text - - # Test for decimal - elif re.search('\.',self.number): - if self.verbose: self.log("Decimal: %s" % self.number) - self.number_as_float = self.number - decimal_strings = self.number.split(".") - left = EPUB_MOBI.NumberToText(decimal_strings[0]).text - right = EPUB_MOBI.NumberToText(decimal_strings[1]).text - self.text = '%s point %s' % (left.capitalize(), right) - - # Test for hypenated - elif re.search('-', self.number): - if self.verbose: self.log("Hyphenated: %s" % self.number) - self.number_as_float = self.number.split('-')[0] - strings = self.number.split('-') - if re.search('[0-9]+', strings[0]): - left = EPUB_MOBI.NumberToText(strings[0]).text - right = strings[1] - else: - left = strings[0] - right = EPUB_MOBI.NumberToText(strings[1]).text - self.text = '%s-%s' % (left, right) - - # Test for only commas and numbers - elif re.search(',', self.number) and not re.search('[^0-9,]',self.number): - if self.verbose: self.log("Comma(s): %s" % self.number) - self.number_as_float = re.sub(',','',self.number) - self.text = EPUB_MOBI.NumberToText(self.number_as_float).text - - # Test for hybrid e.g., 'K2, 2nd, 10@10' - elif re.search('[\D]+', self.number): - if self.verbose: self.log("Hybrid: %s" % self.number) - # Split the token into number/text - number_position = re.search('\d',self.number).start() - text_position = re.search('\D',self.number).start() - if number_position < text_position: - number = self.number[:text_position] - text = self.number[text_position:] - self.text = '%s%s' % (EPUB_MOBI.NumberToText(number).text,text) - else: - text = self.number[:number_position] - number = self.number[number_position:] - self.text = '%s%s' % (text, EPUB_MOBI.NumberToText(number).text) - - else: - if self.verbose: self.log("Clean: %s" % self.number) - try: - self.float_as_number = float(self.number) - number = int(self.number) - except: - return - - if number > 10**9: - self.text = "%d out of range" % number - return - - if number == 10**9: - self.text = "one billion" - else : - # Isolate the three-digit number groups - millionsNumber = number/10**6 - thousandsNumber = (number - (millionsNumber * 10**6))/10**3 - hundredsNumber = number - (millionsNumber * 10**6) - (thousandsNumber * 10**3) - if self.verbose: - print "Converting %s %s %s" % (millionsNumber, thousandsNumber, hundredsNumber) - - # Convert hundredsNumber - if hundredsNumber : - hundredsString = self.stringFromInt(hundredsNumber) - - # Convert thousandsNumber - if thousandsNumber: - if number > 1099 and number < 2000: - resultString = '%s %s' % (self.lessThanTwenty[number/100], - self.stringFromInt(number % 100)) - self.text = resultString.strip().capitalize() - return - else: - thousandsString = self.stringFromInt(thousandsNumber) - - # Convert millionsNumber - if millionsNumber: - millionsString = self.stringFromInt(millionsNumber) - - # Concatenate the strings - resultString = '' - if millionsNumber: - resultString += "%s million " % millionsString - - if thousandsNumber: - resultString += "%s thousand " % thousandsString - - if hundredsNumber: - resultString += "%s" % hundredsString - - if not millionsNumber and not thousandsNumber and not hundredsNumber: - resultString = "zero" - - if self.verbose: - self.log(u'resultString: %s' % resultString) - self.text = resultString.strip().capitalize() - - class CatalogBuilder(object): - ''' - Generates catalog source files from calibre database - - Flow of control: - gui2.actions.catalog:generate_catalog() - gui2.tools:generate_catalog() or library.cli:command_catalog() - called from gui2.convert.gui_conversion:gui_catalog() - catalog = Catalog(notification=Reporter()) - catalog.createDirectoryStructure() - catalog.copyResources() - catalog.buildSources() - Options managed in gui2.catalog.catalog_epub_mobi.py - ''' - - # A single number creates 'Last x days' only. - # Multiple numbers create 'Last x days', 'x to y days ago' ... - # e.g, [7,15,30,60], [30] - # [] = No date ranges added - DATE_RANGE=[30] - - # basename output file basename - # creator dc:creator in OPF metadata - # descriptionClip limits size of NCX descriptions (Kindle only) - # includeSources Used in processSpecialTags to skip tags like '[SPL]' - # notification Used to check for cancel, report progress - # stylesheet CSS stylesheet - # title dc:title in OPF metadata, NCX periodical - # verbosity level of diagnostic printout - - def __init__(self, db, opts, plugin, - report_progress=DummyReporter(), - stylesheet="content/stylesheet.css"): - self.__opts = opts - self.__authorClip = opts.authorClip - self.__authors = None - self.__basename = opts.basename - self.__bookmarked_books = None - self.__booksByAuthor = None - self.__booksByDateRead = None - self.__booksByTitle = None - self.__booksByTitle_noSeriesPrefix = None - self.__cache_dir = os.path.join(config_dir, 'caches', 'catalog') - self.__archive_path = os.path.join(self.__cache_dir, "thumbs.zip") - self.__catalogPath = PersistentTemporaryDirectory("_epub_mobi_catalog", prefix='') - self.__contentDir = os.path.join(self.catalogPath, "content") - self.__currentStep = 0.0 - self.__creator = opts.creator - self.__db = db - self.__descriptionClip = opts.descriptionClip - self.__error = [] - self.__generateForKindle = True if (self.opts.fmt == 'mobi' and \ - self.opts.output_profile and \ - self.opts.output_profile.startswith("kindle")) else False - self.__generateRecentlyRead = True if self.opts.generate_recently_added \ - and self.opts.connected_kindle \ - and self.generateForKindle \ - else False - self.__genres = None - self.genres = [] - self.__genre_tags_dict = None - self.__htmlFileList_1 = [] - self.__htmlFileList_2 = [] - self.__markerTags = self.getMarkerTags() - self.__ncxSoup = None - self.__output_profile = None - self.__playOrder = 1 - self.__plugin = plugin - self.__progressInt = 0.0 - self.__progressString = '' - f, _, p = opts.read_book_marker.partition(':') - self.__read_book_marker = {'field':f, 'pattern':p} - f, p, hr = self.opts.merge_comments.split(':') - self.__merge_comments = {'field':f, 'position':p, 'hr':hr} - self.__reporter = report_progress - self.__stylesheet = stylesheet - self.__thumbs = None - self.__thumbWidth = 0 - self.__thumbHeight = 0 - self.__title = opts.catalog_title - self.__totalSteps = 6.0 - self.__useSeriesPrefixInTitlesSection = False - self.__verbose = opts.verbose - - from calibre.customize.ui import output_profiles - for profile in output_profiles(): - if profile.short_name == self.opts.output_profile: - self.__output_profile = profile - break - - # Confirm/create thumbs archive. - if self.opts.generate_descriptions: - if not os.path.exists(self.__cache_dir): - self.opts.log.info(" creating new thumb cache '%s'" % self.__cache_dir) - os.makedirs(self.__cache_dir) - if not os.path.exists(self.__archive_path): - self.opts.log.info(' creating thumbnail archive, thumb_width: %1.2f"' % - float(self.opts.thumb_width)) - with ZipFile(self.__archive_path, mode='w') as zfw: - zfw.writestr("Catalog Thumbs Archive",'') - else: - try: - with ZipFile(self.__archive_path, mode='r') as zfr: - try: - cached_thumb_width = zfr.read('thumb_width') - except: - cached_thumb_width = "-1" - except: - os.remove(self.__archive_path) - cached_thumb_width = '-1' - - if float(cached_thumb_width) != float(self.opts.thumb_width): - self.opts.log.warning(" invalidating cache at '%s'" % self.__archive_path) - self.opts.log.warning(' thumb_width changed: %1.2f" => %1.2f"' % - (float(cached_thumb_width),float(self.opts.thumb_width))) - with ZipFile(self.__archive_path, mode='w') as zfw: - zfw.writestr("Catalog Thumbs Archive",'') - else: - self.opts.log.info(' existing thumb cache at %s, cached_thumb_width: %1.2f"' % - (self.__archive_path, float(cached_thumb_width))) - - # Tweak build steps based on optional sections: 1 call for HTML, 1 for NCX - incremental_jobs = 0 - if self.opts.generate_authors: - incremental_jobs += 2 - if self.opts.generate_titles: - incremental_jobs += 2 - if self.opts.generate_recently_added: - incremental_jobs += 2 - if self.generateRecentlyRead: - incremental_jobs += 2 - if self.opts.generate_series: - incremental_jobs += 2 - if self.opts.generate_descriptions: - # +1 thumbs - incremental_jobs += 3 - self.__totalSteps += incremental_jobs - - # Load section list templates - templates = {} - execfile(P('catalog/section_list_templates.py'), templates) - for name, template in templates.iteritems(): - if name.startswith('by_') and name.endswith('_template'): - setattr(self, name, force_unicode(template, 'utf-8')) - - # Accessors - if True: - ''' - @dynamic_property - def xxxx(self): - def fget(self): - return self.__ - def fset(self, val): - self.__ = val - return property(fget=fget, fset=fset) - ''' - @dynamic_property - def authorClip(self): - def fget(self): - return self.__authorClip - def fset(self, val): - self.__authorClip = val - return property(fget=fget, fset=fset) - @dynamic_property - def authors(self): - def fget(self): - return self.__authors - def fset(self, val): - self.__authors = val - return property(fget=fget, fset=fset) - @dynamic_property - def basename(self): - def fget(self): - return self.__basename - def fset(self, val): - self.__basename = val - return property(fget=fget, fset=fset) - @dynamic_property - def bookmarked_books(self): - def fget(self): - return self.__bookmarked_books - def fset(self, val): - self.__bookmarked_books = val - return property(fget=fget, fset=fset) - @dynamic_property - def booksByAuthor(self): - def fget(self): - return self.__booksByAuthor - def fset(self, val): - self.__booksByAuthor = val - return property(fget=fget, fset=fset) - @dynamic_property - def booksByDateRead(self): - def fget(self): - return self.__booksByDateRead - def fset(self, val): - self.__booksByDateRead = val - return property(fget=fget, fset=fset) - @dynamic_property - def booksByTitle(self): - def fget(self): - return self.__booksByTitle - def fset(self, val): - self.__booksByTitle = val - return property(fget=fget, fset=fset) - @dynamic_property - def booksByTitle_noSeriesPrefix(self): - def fget(self): - return self.__booksByTitle_noSeriesPrefix - def fset(self, val): - self.__booksByTitle_noSeriesPrefix = val - return property(fget=fget, fset=fset) - @dynamic_property - def catalogPath(self): - def fget(self): - return self.__catalogPath - def fset(self, val): - self.__catalogPath = val - return property(fget=fget, fset=fset) - @dynamic_property - def contentDir(self): - def fget(self): - return self.__contentDir - def fset(self, val): - self.__contentDir = val - return property(fget=fget, fset=fset) - @dynamic_property - def currentStep(self): - def fget(self): - return self.__currentStep - def fset(self, val): - self.__currentStep = val - return property(fget=fget, fset=fset) - @dynamic_property - def creator(self): - def fget(self): - return self.__creator - def fset(self, val): - self.__creator = val - return property(fget=fget, fset=fset) - @dynamic_property - def db(self): - def fget(self): - return self.__db - return property(fget=fget) - @dynamic_property - def descriptionClip(self): - def fget(self): - return self.__descriptionClip - def fset(self, val): - self.__descriptionClip = val - return property(fget=fget, fset=fset) - @dynamic_property - def error(self): - def fget(self): - return self.__error - def fset(self, val): - self.__error = val - return property(fget=fget,fset=fset) - @dynamic_property - def generateForKindle(self): - def fget(self): - return self.__generateForKindle - def fset(self, val): - self.__generateForKindle = val - return property(fget=fget, fset=fset) - @dynamic_property - def generateRecentlyRead(self): - def fget(self): - return self.__generateRecentlyRead - def fset(self, val): - self.__generateRecentlyRead = val - return property(fget=fget, fset=fset) - @dynamic_property - def genres(self): - def fget(self): - return self.__genres - def fset(self, val): - self.__genres = val - return property(fget=fget, fset=fset) - @dynamic_property - def genre_tags_dict(self): - def fget(self): - return self.__genre_tags_dict - def fset(self, val): - self.__genre_tags_dict = val - return property(fget=fget, fset=fset) - @dynamic_property - def htmlFileList_1(self): - def fget(self): - return self.__htmlFileList_1 - def fset(self, val): - self.__htmlFileList_1 = val - return property(fget=fget, fset=fset) - @dynamic_property - def htmlFileList_2(self): - def fget(self): - return self.__htmlFileList_2 - def fset(self, val): - self.__htmlFileList_2 = val - return property(fget=fget, fset=fset) - @dynamic_property - def libraryPath(self): - def fget(self): - return self.__libraryPath - def fset(self, val): - self.__libraryPath = val - return property(fget=fget, fset=fset) - @dynamic_property - def markerTags(self): - def fget(self): - return self.__markerTags - def fset(self, val): - self.__markerTags = val - return property(fget=fget, fset=fset) - @dynamic_property - def ncxSoup(self): - def fget(self): - return self.__ncxSoup - def fset(self, val): - self.__ncxSoup = val - return property(fget=fget, fset=fset) - @dynamic_property - def opts(self): - def fget(self): - return self.__opts - return property(fget=fget) - @dynamic_property - def playOrder(self): - def fget(self): - return self.__playOrder - def fset(self,val): - self.__playOrder = val - return property(fget=fget, fset=fset) - @dynamic_property - def plugin(self): - def fget(self): - return self.__plugin - return property(fget=fget) - @dynamic_property - def progressInt(self): - def fget(self): - return self.__progressInt - def fset(self, val): - self.__progressInt = val - return property(fget=fget, fset=fset) - @dynamic_property - def progressString(self): - def fget(self): - return self.__progressString - def fset(self, val): - self.__progressString = val - return property(fget=fget, fset=fset) - @dynamic_property - def reporter(self): - def fget(self): - return self.__reporter - def fset(self, val): - self.__reporter = val - return property(fget=fget, fset=fset) - @dynamic_property - def stylesheet(self): - def fget(self): - return self.__stylesheet - def fset(self, val): - self.__stylesheet = val - return property(fget=fget, fset=fset) - @dynamic_property - def thumbs(self): - def fget(self): - return self.__thumbs - def fset(self, val): - self.__thumbs = val - return property(fget=fget, fset=fset) - def thumbWidth(self): - def fget(self): - return self.__thumbWidth - def fset(self, val): - self.__thumbWidth = val - return property(fget=fget, fset=fset) - def thumbHeight(self): - def fget(self): - return self.__thumbHeight - def fset(self, val): - self.__thumbHeight = val - return property(fget=fget, fset=fset) - @dynamic_property - def title(self): - def fget(self): - return self.__title - def fset(self, val): - self.__title = val - return property(fget=fget, fset=fset) - @dynamic_property - def totalSteps(self): - def fget(self): - return self.__totalSteps - return property(fget=fget) - @dynamic_property - def useSeriesPrefixInTitlesSection(self): - def fget(self): - return self.__useSeriesPrefixInTitlesSection - def fset(self, val): - self.__useSeriesPrefixInTitlesSection = val - return property(fget=fget, fset=fset) - @dynamic_property - def verbose(self): - def fget(self): - return self.__verbose - def fset(self, val): - self.__verbose = val - return property(fget=fget, fset=fset) - - @dynamic_property - def MISSING_SYMBOL(self): - def fget(self): - return self.__output_profile.missing_char - return property(fget=fget) - @dynamic_property - def NOT_READ_SYMBOL(self): - def fget(self): - return '<span style="color:white">%s</span>' % self.__output_profile.read_char - return property(fget=fget) - @dynamic_property - def READING_SYMBOL(self): - def fget(self): - return '<span style="color:black">▷</span>' if self.generateForKindle else \ - '<span style="color:white">+</span>' - return property(fget=fget) - @dynamic_property - def READ_SYMBOL(self): - def fget(self): - return self.__output_profile.read_char - return property(fget=fget) - @dynamic_property - def FULL_RATING_SYMBOL(self): - def fget(self): - return self.__output_profile.ratings_char - return property(fget=fget) - @dynamic_property - def EMPTY_RATING_SYMBOL(self): - def fget(self): - return self.__output_profile.empty_ratings_char - return property(fget=fget) - @dynamic_property - def READ_PROGRESS_SYMBOL(self): - def fget(self): - return "▪" if self.generateForKindle else '+' - return property(fget=fget) - @dynamic_property - def UNREAD_PROGRESS_SYMBOL(self): - def fget(self): - return "▫" if self.generateForKindle else '-' - return property(fget=fget) - - # Methods - def buildSources(self): - if self.booksByTitle is None: - if not self.fetchBooksByTitle(): - return False - if not self.fetchBooksByAuthor(): - return False - self.fetchBookmarks() - if self.opts.generate_descriptions: - self.generateThumbnails() - self.generateHTMLDescriptions() - if self.opts.generate_authors: - self.generateHTMLByAuthor() - if self.opts.generate_titles: - self.generateHTMLByTitle() - if self.opts.generate_series: - self.generateHTMLBySeries() - if self.opts.generate_genres: - self.generateHTMLByTags() - # If this is the only Section, and there are no genres, bail - if self.opts.section_list == ['Genres'] and not self.genres: - error_msg = _("No enabled genres found to catalog.\n") - if not self.opts.cli_environment: - error_msg += "Check 'Excluded genres'\nin E-book options.\n" - self.opts.log.error(error_msg) - self.error.append(_('No books available to catalog')) - self.error.append(error_msg) - return False - if self.opts.generate_recently_added: - self.generateHTMLByDateAdded() - if self.generateRecentlyRead: - self.generateHTMLByDateRead() - - self.generateOPF() - self.generateNCXHeader() - if self.opts.generate_authors: - self.generateNCXByAuthor("Authors") - if self.opts.generate_titles: - self.generateNCXByTitle("Titles") - if self.opts.generate_series: - self.generateNCXBySeries("Series") - if self.opts.generate_genres: - self.generateNCXByGenre("Genres") - if self.opts.generate_recently_added: - self.generateNCXByDateAdded("Recently Added") - if self.generateRecentlyRead: - self.generateNCXByDateRead("Recently Read") - if self.opts.generate_descriptions: - self.generateNCXDescriptions("Descriptions") - - self.writeNCX() - return True - - def cleanUp(self): - pass - - def copyResources(self): - '''Move resource files to self.catalogPath''' - catalog_resources = P("catalog") - - files_to_copy = [('','DefaultCover.jpg'), - ('content','stylesheet.css'), - ('images','mastheadImage.gif')] - - for file in files_to_copy: - if file[0] == '': - shutil.copy(os.path.join(catalog_resources,file[1]), - self.catalogPath) - else: - shutil.copy(os.path.join(catalog_resources,file[1]), - os.path.join(self.catalogPath, file[0])) - - # Create the custom masthead image overwriting default - # If failure, default mastheadImage.gif should still be in place - if self.generateForKindle: - try: - self.generateMastheadImage(os.path.join(self.catalogPath, - 'images/mastheadImage.gif')) - except: - pass - - def fetchBooksByAuthor(self): - ''' - Generate a list of titles sorted by author from the database - return = Success - ''' - - self.updateProgressFullStep("Sorting database") - self.booksByAuthor = list(self.booksByTitle) - - # Test for author_sort mismatches - self.booksByAuthor = sorted(self.booksByAuthor, key=self.booksByAuthorSorter_author) - # Build the unique_authors set from existing data - authors = [(record['author'], record['author_sort']) for record in self.booksByAuthor] - current_author = authors[0] - for (i,author) in enumerate(authors): - if author != current_author and i: - if author[0] == current_author[0]: - if self.opts.fmt == 'mobi': - # Exit if building MOBI - error_msg = _( -'''Inconsistent Author Sort values for -Author '{0}': -'{1}' <> '{2}' -Unable to build MOBI catalog.\n -Select all books by '{0}', apply correct Author Sort value in Edit Metadata dialog, then rebuild the catalog.\n''').format(author[0],author[1],current_author[1]) - self.opts.log.warn('\n*** Metadata error ***') - self.opts.log.warn(error_msg) - - self.error.append('Author Sort mismatch') - self.error.append(error_msg) - return False - else: - # Warning if building non-MOBI - if not self.error: - self.error.append('Author Sort mismatch') - - error_msg = _( -'''Warning: inconsistent Author Sort values for -Author '{0}': -'{1}' <> '{2}'\n''').format(author[0],author[1],current_author[1]) - self.opts.log.warn('\n*** Metadata warning ***') - self.opts.log.warn(error_msg) - self.error.append(error_msg) - - current_author = author - - self.booksByAuthor = sorted(self.booksByAuthor, key=self.booksByAuthorSorter_author_sort) - - # Build the unique_authors set from existing data - authors = [(record['author'], capitalize(record['author_sort'])) for record in self.booksByAuthor] - - # authors[] contains a list of all book authors, with multiple entries for multiple books by author - # authors[]: (([0]:friendly [1]:sort)) - # unique_authors[]: (([0]:friendly [1]:sort [2]:book_count)) - books_by_current_author = 0 - current_author = authors[0] - multiple_authors = False - unique_authors = [] - for (i,author) in enumerate(authors): - if author != current_author: - # Note that current_author and author are tuples: (friendly, sort) - multiple_authors = True - - # New author, save the previous author/sort/count - unique_authors.append((current_author[0], icu_title(current_author[1]), - books_by_current_author)) - current_author = author - books_by_current_author = 1 - elif i==0 and len(authors) == 1: - # Allow for single-book lists - unique_authors.append((current_author[0], icu_title(current_author[1]), - books_by_current_author)) - else: - books_by_current_author += 1 - else: - # Add final author to list or single-author dataset - if (current_author == author and len(authors) > 1) or not multiple_authors: - unique_authors.append((current_author[0], icu_title(current_author[1]), - books_by_current_author)) - - if False and self.verbose: - self.opts.log.info("\nfetchBooksByauthor(): %d unique authors" % len(unique_authors)) - for author in unique_authors: - self.opts.log.info((u" %-50s %-25s %2d" % (author[0][0:45], author[1][0:20], - author[2])).encode('utf-8')) - - self.authors = unique_authors - return True - - def fetchBooksByTitle(self): - self.updateProgressFullStep("Fetching database") - - self.opts.sort_by = 'title' - - # Merge opts.exclude_tags with opts.search_text - # Updated to use exact match syntax - empty_exclude_tags = False if len(self.opts.exclude_tags) else True - search_phrase = '' - if not empty_exclude_tags: - exclude_tags = self.opts.exclude_tags.split(',') - search_terms = [] - for tag in exclude_tags: - search_terms.append("tag:=%s" % tag) - search_phrase = "not (%s)" % " or ".join(search_terms) - # If a list of ids are provided, don't use search_text - if self.opts.ids: - self.opts.search_text = search_phrase - else: - if self.opts.search_text: - self.opts.search_text += " " + search_phrase - else: - self.opts.search_text = search_phrase - - # Fetch the database as a dictionary - data = self.plugin.search_sort_db(self.db, self.opts) - data = self.processExclusions(data) - - # Populate this_title{} from data[{},{}] - titles = [] - for record in data: - this_title = {} - - this_title['id'] = record['id'] - this_title['uuid'] = record['uuid'] - - this_title['title'] = self.convertHTMLEntities(record['title']) - if record['series']: - this_title['series'] = record['series'] - this_title['series_index'] = record['series_index'] - else: - this_title['series'] = None - this_title['series_index'] = 0.0 - - this_title['title_sort'] = self.generateSortTitle(this_title['title']) - if 'authors' in record: - # from calibre.ebooks.metadata import authors_to_string - # return authors_to_string(self.authors) - - this_title['authors'] = record['authors'] - if record['authors']: - this_title['author'] = " & ".join(record['authors']) - else: - this_title['author'] = 'Unknown' - - if 'author_sort' in record and record['author_sort'].strip(): - this_title['author_sort'] = record['author_sort'] - else: - this_title['author_sort'] = self.author_to_author_sort(this_title['author']) - - if record['publisher']: - this_title['publisher'] = re.sub('&', '&', record['publisher']) - - this_title['rating'] = record['rating'] if record['rating'] else 0 - - if is_date_undefined(record['pubdate']): - this_title['date'] = None - else: - this_title['date'] = strftime(u'%B %Y', record['pubdate'].timetuple()) - - this_title['timestamp'] = record['timestamp'] - - if record['comments']: - # Strip annotations - a_offset = record['comments'].find('<div class="user_annotations">') - ad_offset = record['comments'].find('<hr class="annotations_divider" />') - if a_offset >= 0: - record['comments'] = record['comments'][:a_offset] - if ad_offset >= 0: - record['comments'] = record['comments'][:ad_offset] - - this_title['description'] = self.markdownComments(record['comments']) - - # Create short description - paras = BeautifulSoup(this_title['description']).findAll('p') - tokens = [] - for p in paras: - for token in p.contents: - if token.string is not None: - tokens.append(token.string) - this_title['short_description'] = self.generateShortDescription(' '.join(tokens), dest="description") - else: - this_title['description'] = None - this_title['short_description'] = None - - # Merge with custom field/value - if self.__merge_comments['field']: - this_title['description'] = self.mergeComments(this_title) - - if record['cover']: - this_title['cover'] = re.sub('&', '&', record['cover']) - - this_title['read'] = self.discoverReadStatus(record) - - if record['tags']: - this_title['tags'] = self.processSpecialTags(record['tags'], - this_title, self.opts) - if record['formats']: - formats = [] - for format in record['formats']: - formats.append(self.convertHTMLEntities(format)) - this_title['formats'] = formats - - # Add user notes to be displayed in header - # Special case handling for datetime fields and lists - if self.opts.header_note_source_field: - field_md = self.__db.metadata_for_field(self.opts.header_note_source_field) - notes = self.__db.get_field(record['id'], - self.opts.header_note_source_field, - index_is_id=True) - if notes: - if field_md['datatype'] == 'text': - if isinstance(notes,list): - notes = ' · '.join(notes) - elif field_md['datatype'] == 'datetime': - notes = format_date(notes,'dd MMM yyyy') - this_title['notes'] = {'source':field_md['name'], - 'content':notes} - - titles.append(this_title) - - # Re-sort based on title_sort - if len(titles): - self.booksByTitle = sorted(titles, - key=lambda x:(x['title_sort'].upper(), x['title_sort'].upper())) - if False and self.verbose: - self.opts.log.info("fetchBooksByTitle(): %d books" % len(self.booksByTitle)) - self.opts.log.info(" %-40s %-40s" % ('title', 'title_sort')) - for title in self.booksByTitle: - self.opts.log.info((u" %-40s %-40s" % (title['title'][0:40], - title['title_sort'][0:40])).decode('mac-roman')) - return True - else: - error_msg = _("No books found to catalog.\nCheck 'Excluded books' criteria in E-book options.\n") - self.opts.log.error('*** ' + error_msg + ' ***') - self.error.append(_('No books available to include in catalog')) - self.error.append(error_msg) - return False - - def fetchBookmarks(self): - ''' - Collect bookmarks for catalog entries - This will use the system default save_template specified in - Preferences|Add/Save|Sending to device, not a customized one specified in - the Kindle plugin - ''' - from calibre.devices.usbms.device import Device - from calibre.devices.kindle.driver import Bookmark - from calibre.ebooks.metadata import MetaInformation - - MBP_FORMATS = [u'azw', u'mobi', u'prc', u'txt'] - mbp_formats = set(MBP_FORMATS) - PDR_FORMATS = [u'pdf'] - pdr_formats = set(PDR_FORMATS) - TAN_FORMATS = [u'tpz', u'azw1'] - tan_formats = set(TAN_FORMATS) - - class BookmarkDevice(Device): - def initialize(self, save_template): - self._save_template = save_template - self.SUPPORTS_SUB_DIRS = True - def save_template(self): - return self._save_template - - def resolve_bookmark_paths(storage, path_map): - pop_list = [] - book_ext = {} - for id in path_map: - file_fmts = set() - for fmt in path_map[id]['fmts']: - file_fmts.add(fmt) - - bookmark_extension = None - if file_fmts.intersection(mbp_formats): - book_extension = list(file_fmts.intersection(mbp_formats))[0] - bookmark_extension = 'mbp' - elif file_fmts.intersection(tan_formats): - book_extension = list(file_fmts.intersection(tan_formats))[0] - bookmark_extension = 'tan' - elif file_fmts.intersection(pdr_formats): - book_extension = list(file_fmts.intersection(pdr_formats))[0] - bookmark_extension = 'pdr' - - if bookmark_extension: - for vol in storage: - bkmk_path = path_map[id]['path'].replace(os.path.abspath('/<storage>'),vol) - bkmk_path = bkmk_path.replace('bookmark',bookmark_extension) - if os.path.exists(bkmk_path): - path_map[id] = bkmk_path - book_ext[id] = book_extension - break - else: - pop_list.append(id) - else: - pop_list.append(id) - # Remove non-existent bookmark templates - for id in pop_list: - path_map.pop(id) - return path_map, book_ext - - if self.generateRecentlyRead: - self.opts.log.info(" Collecting Kindle bookmarks matching catalog entries") - - d = BookmarkDevice(None) - d.initialize(self.opts.connected_device['save_template']) - - bookmarks = {} - for book in self.booksByTitle: - if 'formats' in book: - path_map = {} - id = book['id'] - original_title = book['title'][book['title'].find(':') + 2:] if book['series'] \ - else book['title'] - myMeta = MetaInformation(original_title, - authors=book['authors']) - myMeta.author_sort = book['author_sort'] - a_path = d.create_upload_path('/<storage>', myMeta, 'x.bookmark', create_dirs=False) - path_map[id] = dict(path=a_path, fmts=[x.rpartition('.')[2] for x in book['formats']]) - - path_map, book_ext = resolve_bookmark_paths(self.opts.connected_device['storage'], path_map) - if path_map: - bookmark_ext = path_map[id].rpartition('.')[2] - myBookmark = Bookmark(path_map[id], id, book_ext[id], bookmark_ext) - try: - book['percent_read'] = min(float(100*myBookmark.last_read / myBookmark.book_length),100) - except: - book['percent_read'] = 0 - dots = int((book['percent_read'] + 5)/10) - dot_string = self.READ_PROGRESS_SYMBOL * dots - empty_dots = self.UNREAD_PROGRESS_SYMBOL * (10 - dots) - book['reading_progress'] = '%s%s' % (dot_string,empty_dots) - bookmarks[id] = ((myBookmark,book)) - - self.bookmarked_books = bookmarks - else: - self.bookmarked_books = {} - - def generateHTMLDescriptions(self): - ''' - Write each title to a separate HTML file in contentdir - ''' - self.updateProgressFullStep("'Descriptions'") - - for (title_num, title) in enumerate(self.booksByTitle): - self.updateProgressMicroStep("Description %d of %d" % \ - (title_num, len(self.booksByTitle)), - float(title_num*100/len(self.booksByTitle))/100) - - # Generate the header from user-customizable template - soup = self.generateHTMLDescriptionHeader(title) - - # Write the book entry to contentdir - outfile = open("%s/book_%d.html" % (self.contentDir, int(title['id'])), 'w') - outfile.write(soup.prettify()) - outfile.close() - - def generateHTMLByTitle(self): - ''' - Write books by title A-Z to HTML file - ''' - self.updateProgressFullStep("'Titles'") - - soup = self.generateHTMLEmptyHeader("Books By Alpha Title") - body = soup.find('body') - btc = 0 - - # Insert section tag - aTag = Tag(soup,'a') - aTag['name'] = 'section_start' - body.insert(btc, aTag) - btc += 1 - - # Insert the anchor - aTag = Tag(soup, "a") - aTag['name'] = "bytitle" - body.insert(btc, aTag) - btc += 1 - - if not self.__generateForKindle: - # We don't need this because the Kindle shows section titles - #<h2><a name="byalphatitle" id="byalphatitle"></a>By Title</h2> - pTag = Tag(soup, "p") - pTag['class'] = 'title' - aTag = Tag(soup, "a") - aTag['name'] = "bytitle" - pTag.insert(0,aTag) - pTag.insert(1,NavigableString('Titles')) - body.insert(btc,pTag) - btc += 1 - - divTag = Tag(soup, "div") - dtc = 0 - current_letter = "" - - # Re-sort title list without leading series/series_index - # Incoming title <series> <series_index>: <title> - if not self.useSeriesPrefixInTitlesSection: - nspt = deepcopy(self.booksByTitle) - nspt = sorted(nspt, key=lambda x:(x['title_sort'].upper(), x['title_sort'].upper())) - self.booksByTitle_noSeriesPrefix = nspt - - # Loop through the books by title - # Generate one divRunningTag per initial letter for the purposes of - # minimizing widows and orphans on readers that can handle large - # <divs> styled as inline-block - title_list = self.booksByTitle - if not self.useSeriesPrefixInTitlesSection: - title_list = self.booksByTitle_noSeriesPrefix - drtc = 0 - divRunningTag = None - for book in title_list: - if self.letter_or_symbol(book['title_sort'][0]) != current_letter : - # Start a new letter - if drtc and divRunningTag is not None: - divTag.insert(dtc, divRunningTag) - dtc += 1 - divRunningTag = Tag(soup, 'div') - if dtc > 0: - divRunningTag['class'] = "initial_letter" - drtc = 0 - current_letter = self.letter_or_symbol(book['title_sort'][0]) - pIndexTag = Tag(soup, "p") - pIndexTag['class'] = "author_title_letter_index" - aTag = Tag(soup, "a") - aTag['name'] = "%s" % self.letter_or_symbol(book['title_sort'][0]) - pIndexTag.insert(0,aTag) - pIndexTag.insert(1,NavigableString(self.letter_or_symbol(book['title_sort'][0]))) - divRunningTag.insert(dtc,pIndexTag) - drtc += 1 - - # Add books - pBookTag = Tag(soup, "p") - ptc = 0 - - # book with read|reading|unread symbol or wishlist item - if self.opts.wishlist_tag in book.get('tags', []): - pBookTag['class'] = "wishlist_item" - pBookTag.insert(ptc,NavigableString(self.MISSING_SYMBOL)) - ptc += 1 - else: - if book['read']: - # check mark - pBookTag.insert(ptc,NavigableString(self.READ_SYMBOL)) - pBookTag['class'] = "read_book" - ptc += 1 - elif book['id'] in self.bookmarked_books: - pBookTag.insert(ptc,NavigableString(self.READING_SYMBOL)) - pBookTag['class'] = "read_book" - ptc += 1 - else: - # hidden check mark - pBookTag['class'] = "unread_book" - pBookTag.insert(ptc,NavigableString(self.NOT_READ_SYMBOL)) - ptc += 1 - - # Link to book - aTag = Tag(soup, "a") - if self.opts.generate_descriptions: - aTag['href'] = "book_%d.html" % (int(float(book['id']))) - - # Generate the title from the template - args = self.generateFormatArgs(book) - if book['series']: - formatted_title = self.by_titles_series_title_template.format(**args).rstrip() - else: - formatted_title = self.by_titles_normal_title_template.format(**args).rstrip() - aTag.insert(0,NavigableString(escape(formatted_title))) - pBookTag.insert(ptc, aTag) - ptc += 1 - - # Dot - pBookTag.insert(ptc, NavigableString(" · ")) - ptc += 1 - - # Link to author - emTag = Tag(soup, "em") - aTag = Tag(soup, "a") - if self.opts.generate_authors: - aTag['href'] = "%s.html#%s" % ("ByAlphaAuthor", self.generateAuthorAnchor(book['author'])) - aTag.insert(0, NavigableString(book['author'])) - emTag.insert(0,aTag) - pBookTag.insert(ptc, emTag) - ptc += 1 - - if divRunningTag is not None: - divRunningTag.insert(drtc, pBookTag) - drtc += 1 - - # Add the last divRunningTag to divTag - if divRunningTag is not None: - divTag.insert(dtc, divRunningTag) - dtc += 1 - - # Add the divTag to the body - body.insert(btc, divTag) - btc += 1 - - # Write the volume to contentdir - outfile_spec = "%s/ByAlphaTitle.html" % (self.contentDir) - outfile = open(outfile_spec, 'w') - outfile.write(soup.prettify()) - outfile.close() - self.htmlFileList_1.append("content/ByAlphaTitle.html") - - def generateHTMLByAuthor(self): - ''' - Write books by author A-Z - ''' - self.updateProgressFullStep("'Authors'") - - friendly_name = "Authors" - - soup = self.generateHTMLEmptyHeader(friendly_name) - body = soup.find('body') - - btc = 0 - - # Insert section tag - aTag = Tag(soup,'a') - aTag['name'] = 'section_start' - body.insert(btc, aTag) - btc += 1 - - # Insert the anchor - aTag = Tag(soup, "a") - anchor_name = friendly_name.lower() - aTag['name'] = anchor_name.replace(" ","") - body.insert(btc, aTag) - btc += 1 - - divTag = Tag(soup, "div") - dtc = 0 - divOpeningTag = None - dotc = 0 - divRunningTag = None - drtc = 0 - - # Loop through booksByAuthor - # Each author/books group goes in an openingTag div (first) or - # a runningTag div (subsequent) - book_count = 0 - current_author = '' - current_letter = '' - current_series = None - #for book in sorted(self.booksByAuthor, key = self.booksByAuthorSorter_author_sort): - for book in self.booksByAuthor: - - book_count += 1 - if self.letter_or_symbol(book['author_sort'][0].upper()) != current_letter : - # Start a new letter with Index letter - if divOpeningTag is not None: - divTag.insert(dtc, divOpeningTag) - dtc += 1 - dotc = 0 - if divRunningTag is not None: - divTag.insert(dtc, divRunningTag) - dtc += 1 - drtc = 0 - divRunningTag = None - - current_letter = self.letter_or_symbol(book['author_sort'][0].upper()) - author_count = 0 - divOpeningTag = Tag(soup, 'div') - if dtc > 0: - divOpeningTag['class'] = "initial_letter" - dotc = 0 - pIndexTag = Tag(soup, "p") - pIndexTag['class'] = "author_title_letter_index" - aTag = Tag(soup, "a") - aTag['name'] = "%sauthors" % self.letter_or_symbol(current_letter) - pIndexTag.insert(0,aTag) - pIndexTag.insert(1,NavigableString(self.letter_or_symbol(book['author_sort'][0].upper()))) - divOpeningTag.insert(dotc,pIndexTag) - dotc += 1 - - if book['author'] != current_author: - # Start a new author - current_author = book['author'] - author_count += 1 - if author_count >= 2: - # Add divOpeningTag to divTag, kill divOpeningTag - if divOpeningTag: - divTag.insert(dtc, divOpeningTag) - dtc += 1 - divOpeningTag = None - dotc = 0 - - # Create a divRunningTag for the next author - if author_count > 2: - divTag.insert(dtc, divRunningTag) - dtc += 1 - - divRunningTag = Tag(soup, 'div') - divRunningTag['class'] = "author_logical_group" - drtc = 0 - - non_series_books = 0 - current_series = None - pAuthorTag = Tag(soup, "p") - pAuthorTag['class'] = "author_index" - aTag = Tag(soup, "a") - aTag['name'] = "%s" % self.generateAuthorAnchor(current_author) - aTag.insert(0,NavigableString(current_author)) - pAuthorTag.insert(0,aTag) - if author_count == 1: - divOpeningTag.insert(dotc, pAuthorTag) - dotc += 1 - else: - divRunningTag.insert(drtc,pAuthorTag) - drtc += 1 - - # Check for series - if book['series'] and book['series'] != current_series: - # Start a new series - current_series = book['series'] - pSeriesTag = Tag(soup,'p') - pSeriesTag['class'] = "series" - - if self.opts.generate_series: - aTag = Tag(soup,'a') - aTag['href'] = "%s.html#%s_series" % ('BySeries', - re.sub('\W','',book['series']).lower()) - aTag.insert(0, book['series']) - #pSeriesTag.insert(0, NavigableString(self.NOT_READ_SYMBOL)) - pSeriesTag.insert(0, aTag) - else: - #pSeriesTag.insert(0,NavigableString(self.NOT_READ_SYMBOL + '%s' % book['series'])) - pSeriesTag.insert(0,NavigableString('%s' % book['series'])) - - if author_count == 1: - divOpeningTag.insert(dotc, pSeriesTag) - dotc += 1 - elif divRunningTag is not None: - divRunningTag.insert(drtc,pSeriesTag) - drtc += 1 - if current_series and not book['series']: - current_series = None - - # Add books - pBookTag = Tag(soup, "p") - ptc = 0 - - # book with read|reading|unread symbol or wishlist item - if self.opts.wishlist_tag in book.get('tags', []): - pBookTag['class'] = "wishlist_item" - pBookTag.insert(ptc,NavigableString(self.MISSING_SYMBOL)) - ptc += 1 - else: - if book['read']: - # check mark - pBookTag.insert(ptc,NavigableString(self.READ_SYMBOL)) - pBookTag['class'] = "read_book" - ptc += 1 - elif book['id'] in self.bookmarked_books: - pBookTag.insert(ptc,NavigableString(self.READING_SYMBOL)) - pBookTag['class'] = "read_book" - ptc += 1 - else: - # hidden check mark - pBookTag['class'] = "unread_book" - pBookTag.insert(ptc,NavigableString(self.NOT_READ_SYMBOL)) - ptc += 1 - - aTag = Tag(soup, "a") - if self.opts.generate_descriptions: - aTag['href'] = "book_%d.html" % (int(float(book['id']))) - - # Generate the title from the template - args = self.generateFormatArgs(book) - if current_series: - #aTag.insert(0,'%s%s' % (escape(book['title'][len(book['series'])+1:]),pubyear)) - formatted_title = self.by_authors_series_title_template.format(**args).rstrip() - else: - #aTag.insert(0,'%s%s' % (escape(book['title']), pubyear)) - formatted_title = self.by_authors_normal_title_template.format(**args).rstrip() - non_series_books += 1 - aTag.insert(0,NavigableString(escape(formatted_title))) - - pBookTag.insert(ptc, aTag) - ptc += 1 - - if author_count == 1: - divOpeningTag.insert(dotc, pBookTag) - dotc += 1 - elif divRunningTag: - divRunningTag.insert(drtc,pBookTag) - drtc += 1 - - # Loop ends here - - if not self.__generateForKindle: - # Insert the <h2> tag with book_count at the head - #<h2><a name="byalphaauthor" id="byalphaauthor"></a>By Author</h2> - pTag = Tag(soup, "p") - pTag['class'] = 'title' - aTag = Tag(soup, "a") - anchor_name = friendly_name.lower() - aTag['name'] = anchor_name.replace(" ","") - pTag.insert(0,aTag) - #h2Tag.insert(1,NavigableString('%s (%d)' % (friendly_name, book_count))) - pTag.insert(1,NavigableString('%s' % (friendly_name))) - body.insert(btc,pTag) - btc += 1 - - if author_count == 1: - divTag.insert(dtc, divOpeningTag) - dtc += 1 - elif divRunningTag is not None: - divTag.insert(dtc, divRunningTag) - dtc += 1 - - # Add the divTag to the body - body.insert(btc, divTag) - - # Write the generated file to contentdir - outfile_spec = "%s/ByAlphaAuthor.html" % (self.contentDir) - outfile = open(outfile_spec, 'w') - outfile.write(soup.prettify()) - outfile.close() - self.htmlFileList_1.append("content/ByAlphaAuthor.html") - - def generateHTMLByDateAdded(self): - ''' - Write books by reverse chronological order - ''' - self.updateProgressFullStep("'Recently Added'") - - def add_books_to_HTML_by_month(this_months_list, dtc): - if len(this_months_list): - - #this_months_list = sorted(this_months_list, key=self.booksByAuthorSorter_author_sort) - - # Create a new month anchor - date_string = strftime(u'%B %Y', current_date.timetuple()) - pIndexTag = Tag(soup, "p") - pIndexTag['class'] = "date_index" - aTag = Tag(soup, "a") - aTag['name'] = "bda_%s-%s" % (current_date.year, current_date.month) - pIndexTag.insert(0,aTag) - pIndexTag.insert(1,NavigableString(date_string)) - divTag.insert(dtc,pIndexTag) - dtc += 1 - current_author = None - current_series = None - - for new_entry in this_months_list: - if new_entry['author'] != current_author: - # Start a new author - current_author = new_entry['author'] - non_series_books = 0 - current_series = None - pAuthorTag = Tag(soup, "p") - pAuthorTag['class'] = "author_index" - aTag = Tag(soup, "a") - if self.opts.generate_authors: - aTag['name'] = "%s" % self.generateAuthorAnchor(current_author) - aTag.insert(0,NavigableString(current_author)) - pAuthorTag.insert(0,aTag) - divTag.insert(dtc,pAuthorTag) - dtc += 1 - - # Check for series - if new_entry['series'] and new_entry['series'] != current_series: - # Start a new series - current_series = new_entry['series'] - pSeriesTag = Tag(soup,'p') - pSeriesTag['class'] = "series" - if self.opts.generate_series: - aTag = Tag(soup,'a') - aTag['href'] = "%s.html#%s_series" % ('BySeries', - re.sub('\W','',new_entry['series']).lower()) - aTag.insert(0, new_entry['series']) - pSeriesTag.insert(0, aTag) - else: - pSeriesTag.insert(0,NavigableString('%s' % new_entry['series'])) - divTag.insert(dtc,pSeriesTag) - dtc += 1 - if current_series and not new_entry['series']: - current_series = None - - # Add books - pBookTag = Tag(soup, "p") - ptc = 0 - - # book with read|reading|unread symbol or wishlist item - if self.opts.wishlist_tag in new_entry.get('tags', []): - pBookTag['class'] = "wishlist_item" - pBookTag.insert(ptc,NavigableString(self.MISSING_SYMBOL)) - ptc += 1 - else: - if new_entry['read']: - # check mark - pBookTag.insert(ptc,NavigableString(self.READ_SYMBOL)) - pBookTag['class'] = "read_book" - ptc += 1 - elif new_entry['id'] in self.bookmarked_books: - pBookTag.insert(ptc,NavigableString(self.READING_SYMBOL)) - pBookTag['class'] = "read_book" - ptc += 1 - else: - # hidden check mark - pBookTag['class'] = "unread_book" - pBookTag.insert(ptc,NavigableString(self.NOT_READ_SYMBOL)) - ptc += 1 - - aTag = Tag(soup, "a") - if self.opts.generate_descriptions: - aTag['href'] = "book_%d.html" % (int(float(new_entry['id']))) - - # Generate the title from the template - args = self.generateFormatArgs(new_entry) - if current_series: - formatted_title = self.by_month_added_series_title_template.format(**args).rstrip() - else: - formatted_title = self.by_month_added_normal_title_template.format(**args).rstrip() - non_series_books += 1 - aTag.insert(0,NavigableString(escape(formatted_title))) - pBookTag.insert(ptc, aTag) - ptc += 1 - - divTag.insert(dtc, pBookTag) - dtc += 1 - return dtc - - def add_books_to_HTML_by_date_range(date_range_list, date_range, dtc): - if len(date_range_list): - pIndexTag = Tag(soup, "p") - pIndexTag['class'] = "date_index" - aTag = Tag(soup, "a") - aTag['name'] = "bda_%s" % date_range.replace(' ','') - pIndexTag.insert(0,aTag) - pIndexTag.insert(1,NavigableString(date_range)) - divTag.insert(dtc,pIndexTag) - dtc += 1 - - for new_entry in date_range_list: - # Add books - pBookTag = Tag(soup, "p") - ptc = 0 - - # book with read|reading|unread symbol or wishlist item - if self.opts.wishlist_tag in new_entry.get('tags', []): - pBookTag['class'] = "wishlist_item" - pBookTag.insert(ptc,NavigableString(self.MISSING_SYMBOL)) - ptc += 1 - else: - if new_entry['read']: - # check mark - pBookTag.insert(ptc,NavigableString(self.READ_SYMBOL)) - pBookTag['class'] = "read_book" - ptc += 1 - elif new_entry['id'] in self.bookmarked_books: - pBookTag.insert(ptc,NavigableString(self.READING_SYMBOL)) - pBookTag['class'] = "read_book" - ptc += 1 - else: - # hidden check mark - pBookTag['class'] = "unread_book" - pBookTag.insert(ptc,NavigableString(self.NOT_READ_SYMBOL)) - ptc += 1 - - aTag = Tag(soup, "a") - if self.opts.generate_descriptions: - aTag['href'] = "book_%d.html" % (int(float(new_entry['id']))) - - # Generate the title from the template - args = self.generateFormatArgs(new_entry) - if new_entry['series']: - formatted_title = self.by_recently_added_series_title_template.format(**args).rstrip() - else: - formatted_title = self.by_recently_added_normal_title_template.format(**args).rstrip() - aTag.insert(0,NavigableString(escape(formatted_title))) - pBookTag.insert(ptc, aTag) - ptc += 1 - - # Dot - pBookTag.insert(ptc, NavigableString(" · ")) - ptc += 1 - - # Link to author - emTag = Tag(soup, "em") - aTag = Tag(soup, "a") - if self.opts.generate_authors: - aTag['href'] = "%s.html#%s" % ("ByAlphaAuthor", self.generateAuthorAnchor(new_entry['author'])) - aTag.insert(0, NavigableString(new_entry['author'])) - emTag.insert(0,aTag) - pBookTag.insert(ptc, emTag) - ptc += 1 - - divTag.insert(dtc, pBookTag) - dtc += 1 - return dtc - - friendly_name = "Recently Added" - - soup = self.generateHTMLEmptyHeader(friendly_name) - body = soup.find('body') - - btc = 0 - - # Insert section tag - aTag = Tag(soup,'a') - aTag['name'] = 'section_start' - body.insert(btc, aTag) - btc += 1 - - # Insert the anchor - aTag = Tag(soup, "a") - anchor_name = friendly_name.lower() - aTag['name'] = anchor_name.replace(" ","") - body.insert(btc, aTag) - btc += 1 - - if not self.__generateForKindle: - #<h2><a name="byalphaauthor" id="byalphaauthor"></a>By Author</h2> - pTag = Tag(soup, "p") - pTag['class'] = 'title' - aTag = Tag(soup, "a") - anchor_name = friendly_name.lower() - aTag['name'] = anchor_name.replace(" ","") - pTag.insert(0,aTag) - pTag.insert(1,NavigableString('%s' % friendly_name)) - body.insert(btc,pTag) - btc += 1 - - divTag = Tag(soup, "div") - dtc = 0 - - # >>> Books by date range <<< - if self.useSeriesPrefixInTitlesSection: - self.booksByDateRange = sorted(self.booksByTitle, - key=lambda x:(x['timestamp'], x['timestamp']),reverse=True) - else: - nspt = deepcopy(self.booksByTitle) - self.booksByDateRange = sorted(nspt, key=lambda x:(x['timestamp'], x['timestamp']),reverse=True) - - date_range_list = [] - today_time = nowf().replace(hour=23, minute=59, second=59) - for (i, date) in enumerate(self.DATE_RANGE): - date_range_limit = self.DATE_RANGE[i] - if i: - date_range = '%d to %d days ago' % (self.DATE_RANGE[i-1], self.DATE_RANGE[i]) - else: - date_range = 'Last %d days' % (self.DATE_RANGE[i]) - - for book in self.booksByDateRange: - book_time = book['timestamp'] - delta = today_time-book_time - if delta.days <= date_range_limit: - date_range_list.append(book) - else: - break - - dtc = add_books_to_HTML_by_date_range(date_range_list, date_range, dtc) - date_range_list = [book] - - # >>>> Books by month <<<< - # Sort titles case-insensitive for by month using series prefix - self.booksByMonth = sorted(self.booksByTitle, - key=lambda x:(x['timestamp'], x['timestamp']),reverse=True) - - # Loop through books by date - current_date = datetime.date.fromordinal(1) - this_months_list = [] - for book in self.booksByMonth: - if book['timestamp'].month != current_date.month or \ - book['timestamp'].year != current_date.year: - dtc = add_books_to_HTML_by_month(this_months_list, dtc) - this_months_list = [] - current_date = book['timestamp'].date() - this_months_list.append(book) - - # Add the last month's list - add_books_to_HTML_by_month(this_months_list, dtc) - - # Add the divTag to the body - body.insert(btc, divTag) - - # Write the generated file to contentdir - outfile_spec = "%s/ByDateAdded.html" % (self.contentDir) - outfile = open(outfile_spec, 'w') - outfile.write(soup.prettify()) - outfile.close() - self.htmlFileList_2.append("content/ByDateAdded.html") - - def generateHTMLByDateRead(self): - ''' - Write books by active bookmarks - ''' - friendly_name = 'Recently Read' - self.updateProgressFullStep("'%s'" % friendly_name) - if not self.bookmarked_books: - return - - def add_books_to_HTML_by_day(todays_list, dtc): - if len(todays_list): - # Create a new day anchor - date_string = strftime(u'%A, %B %d', current_date.timetuple()) - pIndexTag = Tag(soup, "p") - pIndexTag['class'] = "date_index" - aTag = Tag(soup, "a") - aTag['name'] = "bdr_%s-%s-%s" % (current_date.year, current_date.month, current_date.day) - pIndexTag.insert(0,aTag) - pIndexTag.insert(1,NavigableString(date_string)) - divTag.insert(dtc,pIndexTag) - dtc += 1 - - for new_entry in todays_list: - pBookTag = Tag(soup, "p") - pBookTag['class'] = "date_read" - ptc = 0 - - # Percent read - pBookTag.insert(ptc, NavigableString(new_entry['reading_progress'])) - ptc += 1 - - aTag = Tag(soup, "a") - if self.opts.generate_descriptions: - aTag['href'] = "book_%d.html" % (int(float(new_entry['id']))) - aTag.insert(0,escape(new_entry['title'])) - pBookTag.insert(ptc, aTag) - ptc += 1 - - # Dot - pBookTag.insert(ptc, NavigableString(" · ")) - ptc += 1 - - # Link to author - emTag = Tag(soup, "em") - aTag = Tag(soup, "a") - if self.opts.generate_authors: - aTag['href'] = "%s.html#%s" % ("ByAlphaAuthor", self.generateAuthorAnchor(new_entry['author'])) - aTag.insert(0, NavigableString(new_entry['author'])) - emTag.insert(0,aTag) - pBookTag.insert(ptc, emTag) - ptc += 1 - - divTag.insert(dtc, pBookTag) - dtc += 1 - return dtc - - def add_books_to_HTML_by_date_range(date_range_list, date_range, dtc): - if len(date_range_list): - pIndexTag = Tag(soup, "p") - pIndexTag['class'] = "date_index" - aTag = Tag(soup, "a") - aTag['name'] = "bdr_%s" % date_range.replace(' ','') - pIndexTag.insert(0,aTag) - pIndexTag.insert(1,NavigableString(date_range)) - divTag.insert(dtc,pIndexTag) - dtc += 1 - - for new_entry in date_range_list: - # Add books - pBookTag = Tag(soup, "p") - pBookTag['class'] = "date_read" - ptc = 0 - - # Percent read - dots = int((new_entry['percent_read'] + 5)/10) - dot_string = self.READ_PROGRESS_SYMBOL * dots - empty_dots = self.UNREAD_PROGRESS_SYMBOL * (10 - dots) - pBookTag.insert(ptc, NavigableString('%s%s' % (dot_string,empty_dots))) - ptc += 1 - - aTag = Tag(soup, "a") - if self.opts.generate_descriptions: - aTag['href'] = "book_%d.html" % (int(float(new_entry['id']))) - aTag.insert(0,escape(new_entry['title'])) - pBookTag.insert(ptc, aTag) - ptc += 1 - - # Dot - pBookTag.insert(ptc, NavigableString(" · ")) - ptc += 1 - - # Link to author - emTag = Tag(soup, "em") - aTag = Tag(soup, "a") - if self.opts.generate_authors: - aTag['href'] = "%s.html#%s" % ("ByAlphaAuthor", self.generateAuthorAnchor(new_entry['author'])) - aTag.insert(0, NavigableString(new_entry['author'])) - emTag.insert(0,aTag) - pBookTag.insert(ptc, emTag) - ptc += 1 - - divTag.insert(dtc, pBookTag) - dtc += 1 - return dtc - - soup = self.generateHTMLEmptyHeader(friendly_name) - body = soup.find('body') - - btc = 0 - - # Insert section tag - aTag = Tag(soup,'a') - aTag['name'] = 'section_start' - body.insert(btc, aTag) - btc += 1 - - # Insert the anchor - aTag = Tag(soup, "a") - anchor_name = friendly_name.lower() - aTag['name'] = anchor_name.replace(" ","") - body.insert(btc, aTag) - btc += 1 - - divTag = Tag(soup, "div") - dtc = 0 - - # self.bookmarked_books: (Bookmark, book) - bookmarked_books = [] - for bm_book in self.bookmarked_books: - book = self.bookmarked_books[bm_book] - #print "bm_book: %s" % bm_book - book[1]['bookmark_timestamp'] = book[0].timestamp - try: - book[1]['percent_read'] = min(float(100*book[0].last_read / book[0].book_length),100) - except: - book[1]['percent_read'] = 0 - bookmarked_books.append(book[1]) - - self.booksByDateRead = sorted(bookmarked_books, - key=lambda x:(x['bookmark_timestamp'], x['bookmark_timestamp']),reverse=True) - - # >>>> Recently read by day <<<< - current_date = datetime.date.fromordinal(1) - todays_list = [] - for book in self.booksByDateRead: - bookmark_time = datetime.datetime.utcfromtimestamp(book['bookmark_timestamp']) - if bookmark_time.day != current_date.day or \ - bookmark_time.month != current_date.month or \ - bookmark_time.year != current_date.year: - dtc = add_books_to_HTML_by_day(todays_list, dtc) - todays_list = [] - current_date = datetime.datetime.utcfromtimestamp(book['bookmark_timestamp']).date() - todays_list.append(book) - - # Add the last day's list - add_books_to_HTML_by_day(todays_list, dtc) - - # Add the divTag to the body - body.insert(btc, divTag) - - # Write the generated file to contentdir - outfile_spec = "%s/ByDateRead.html" % (self.contentDir) - outfile = open(outfile_spec, 'w') - outfile.write(soup.prettify()) - outfile.close() - self.htmlFileList_2.append("content/ByDateRead.html") - - def generateHTMLBySeries(self): - ''' - Generate a list of series - ''' - self.updateProgressFullStep("Fetching series") - - self.opts.sort_by = 'series' - - # Merge opts.exclude_tags with opts.search_text - # Updated to use exact match syntax - empty_exclude_tags = False if len(self.opts.exclude_tags) else True - search_phrase = 'series:true ' - if not empty_exclude_tags: - exclude_tags = self.opts.exclude_tags.split(',') - search_terms = [] - for tag in exclude_tags: - search_terms.append("tag:=%s" % tag) - search_phrase += "not (%s)" % " or ".join(search_terms) - - # If a list of ids are provided, don't use search_text - if self.opts.ids: - self.opts.search_text = search_phrase - else: - if self.opts.search_text: - self.opts.search_text += " " + search_phrase - else: - self.opts.search_text = search_phrase - - # Fetch the database as a dictionary - data = self.plugin.search_sort_db(self.db, self.opts) - self.booksBySeries = self.processExclusions(data) - - if not self.booksBySeries: - self.opts.generate_series = False - self.opts.log(" no series found in selected books, cancelling series generation") - return - - friendly_name = "Series" - - soup = self.generateHTMLEmptyHeader(friendly_name) - body = soup.find('body') - - btc = 0 - - # Insert section tag - aTag = Tag(soup,'a') - aTag['name'] = 'section_start' - body.insert(btc, aTag) - btc += 1 - - # Insert the anchor - aTag = Tag(soup, "a") - anchor_name = friendly_name.lower() - aTag['name'] = anchor_name.replace(" ","") - body.insert(btc, aTag) - btc += 1 - - divTag = Tag(soup, "div") - dtc = 0 - current_letter = "" - current_series = None - - # Loop through booksBySeries - series_count = 0 - for book in self.booksBySeries: - # Check for initial letter change - sort_title = self.generateSortTitle(book['series']) - if self.letter_or_symbol(sort_title[0].upper()) != current_letter : - # Start a new letter with Index letter - current_letter = self.letter_or_symbol(sort_title[0].upper()) - pIndexTag = Tag(soup, "p") - pIndexTag['class'] = "series_letter_index" - aTag = Tag(soup, "a") - aTag['name'] = "%s_series" % self.letter_or_symbol(current_letter) - pIndexTag.insert(0,aTag) - pIndexTag.insert(1,NavigableString(self.letter_or_symbol(sort_title[0].upper()))) - divTag.insert(dtc,pIndexTag) - dtc += 1 - - # Check for series change - if book['series'] != current_series: - # Start a new series - series_count += 1 - current_series = book['series'] - pSeriesTag = Tag(soup,'p') - pSeriesTag['class'] = "series" - aTag = Tag(soup, 'a') - aTag['name'] = "%s_series" % re.sub('\W','',book['series']).lower() - pSeriesTag.insert(0,aTag) - pSeriesTag.insert(1,NavigableString('%s' % book['series'])) - divTag.insert(dtc,pSeriesTag) - dtc += 1 - - # Add books - pBookTag = Tag(soup, "p") - ptc = 0 - - book['read'] = self.discoverReadStatus(book) - - # book with read|reading|unread symbol or wishlist item - if self.opts.wishlist_tag in book.get('tags', []): - pBookTag['class'] = "wishlist_item" - pBookTag.insert(ptc,NavigableString(self.MISSING_SYMBOL)) - ptc += 1 - else: - if book.get('read', False): - # check mark - pBookTag.insert(ptc,NavigableString(self.READ_SYMBOL)) - pBookTag['class'] = "read_book" - ptc += 1 - elif book['id'] in self.bookmarked_books: - pBookTag.insert(ptc,NavigableString(self.READING_SYMBOL)) - pBookTag['class'] = "read_book" - ptc += 1 - else: - # hidden check mark - pBookTag['class'] = "unread_book" - pBookTag.insert(ptc,NavigableString(self.NOT_READ_SYMBOL)) - ptc += 1 - - aTag = Tag(soup, "a") - if self.opts.generate_descriptions: - aTag['href'] = "book_%d.html" % (int(float(book['id']))) - # Use series, series index if avail else just title - #aTag.insert(0,'%d. %s · %s' % (book['series_index'],escape(book['title']), ' & '.join(book['authors']))) - - if is_date_undefined(book['pubdate']): - book['date'] = None - else: - book['date'] = strftime(u'%B %Y', book['pubdate'].timetuple()) - - args = self.generateFormatArgs(book) - formatted_title = self.by_series_title_template.format(**args).rstrip() - aTag.insert(0,NavigableString(escape(formatted_title))) - pBookTag.insert(ptc, aTag) - ptc += 1 - - # · - pBookTag.insert(ptc, NavigableString(' · ')) - ptc += 1 - - # Link to author - aTag = Tag(soup, "a") - if self.opts.generate_authors: - aTag['href'] = "%s.html#%s" % ("ByAlphaAuthor", - self.generateAuthorAnchor(escape(' & '.join(book['authors'])))) - aTag.insert(0, NavigableString(' & '.join(book['authors']))) - pBookTag.insert(ptc, aTag) - ptc += 1 - - divTag.insert(dtc, pBookTag) - dtc += 1 - - if not self.__generateForKindle: - # Insert the <h2> tag with book_count at the head - #<h2><a name="byseries" id="byseries"></a>By Series</h2> - pTag = Tag(soup, "p") - pTag['class'] = 'title' - aTag = Tag(soup, "a") - anchor_name = friendly_name.lower() - aTag['name'] = anchor_name.replace(" ","") - pTag.insert(0,aTag) - #h2Tag.insert(1,NavigableString('%s (%d)' % (friendly_name, series_count))) - pTag.insert(1,NavigableString('%s' % friendly_name)) - body.insert(btc,pTag) - btc += 1 - - # Add the divTag to the body - body.insert(btc, divTag) - - # Write the generated file to contentdir - outfile_spec = "%s/BySeries.html" % (self.contentDir) - outfile = open(outfile_spec, 'w') - outfile.write(soup.prettify()) - outfile.close() - self.htmlFileList_1.append("content/BySeries.html") - - def generateHTMLByTags(self): - ''' - Generate individual HTML files for each tag, e.g. Fiction, Nonfiction ... - Note that special tags - have already been filtered from books[] - There may be synonomous tags - ''' - self.updateProgressFullStep("'Genres'") - - self.genre_tags_dict = self.filterDbTags(self.db.all_tags()) - - # Extract books matching filtered_tags - genre_list = [] - for friendly_tag in sorted(self.genre_tags_dict): - #print "\ngenerateHTMLByTags(): looking for books with friendly_tag '%s'" % friendly_tag - # tag_list => { normalized_genre_tag : [{book},{},{}], - # normalized_genre_tag : [{book},{},{}] } - - tag_list = {} - for book in self.booksByAuthor: - # Scan each book for tag matching friendly_tag - if 'tags' in book and friendly_tag in book['tags']: - this_book = {} - this_book['author'] = book['author'] - this_book['title'] = book['title'] - this_book['author_sort'] = capitalize(book['author_sort']) - this_book['read'] = book['read'] - this_book['tags'] = book['tags'] - this_book['id'] = book['id'] - this_book['series'] = book['series'] - this_book['series_index'] = book['series_index'] - this_book['date'] = book['date'] - normalized_tag = self.genre_tags_dict[friendly_tag] - genre_tag_list = [key for genre in genre_list for key in genre] - if normalized_tag in genre_tag_list: - for existing_genre in genre_list: - for key in existing_genre: - new_book = None - if key == normalized_tag: - for book in existing_genre[key]: - if book['title'] == this_book['title']: - new_book = False - break - else: - new_book = True - if new_book: - existing_genre[key].append(this_book) - else: - tag_list[normalized_tag] = [this_book] - genre_list.append(tag_list) - - if self.opts.verbose: - if len(genre_list): - self.opts.log.info(" Genre summary: %d active genre tags used in generating catalog with %d titles" % - (len(genre_list), len(self.booksByTitle))) - - for genre in genre_list: - for key in genre: - self.opts.log.info(" %s: %d %s" % (self.getFriendlyGenreTag(key), - len(genre[key]), - 'titles' if len(genre[key]) > 1 else 'title')) - - - # Write the results - # genre_list = [ {friendly_tag:[{book},{book}]}, {friendly_tag:[{book},{book}]}, ...] - master_genre_list = [] - for genre_tag_set in genre_list: - for (index, genre) in enumerate(genre_tag_set): - #print "genre: %s \t genre_tag_set[genre]: %s" % (genre, genre_tag_set[genre]) - - # Create sorted_authors[0] = friendly, [1] = author_sort for NCX creation - authors = [] - for book in genre_tag_set[genre]: - authors.append((book['author'],book['author_sort'])) - - # authors[] contains a list of all book authors, with multiple entries for multiple books by author - # Create unique_authors with a count of books per author as the third tuple element - books_by_current_author = 1 - current_author = authors[0] - unique_authors = [] - for (i,author) in enumerate(authors): - if author != current_author and i: - unique_authors.append((current_author[0], current_author[1], books_by_current_author)) - current_author = author - books_by_current_author = 1 - elif i==0 and len(authors) == 1: - # Allow for single-book lists - unique_authors.append((current_author[0], current_author[1], books_by_current_author)) - else: - books_by_current_author += 1 - - # Write the genre book list as an article - titles_spanned = self.generateHTMLByGenre(genre, True if index==0 else False, - genre_tag_set[genre], - "%s/Genre_%s.html" % (self.contentDir, - genre)) - - tag_file = "content/Genre_%s.html" % genre - master_genre_list.append({'tag':genre, - 'file':tag_file, - 'authors':unique_authors, - 'books':genre_tag_set[genre], - 'titles_spanned':titles_spanned}) - - self.genres = master_genre_list - - def generateThumbnails(self): - ''' - Generate a thumbnail per cover. If a current thumbnail exists, skip - If a cover doesn't exist, use default - Return list of active thumbs - ''' - self.updateProgressFullStep("'Thumbnails'") - thumbs = ['thumbnail_default.jpg'] - image_dir = "%s/images" % self.catalogPath - for (i,title) in enumerate(self.booksByTitle): - # Update status - self.updateProgressMicroStep("Thumbnail %d of %d" % \ - (i,len(self.booksByTitle)), - i/float(len(self.booksByTitle))) - - thumb_file = 'thumbnail_%d.jpg' % int(title['id']) - thumb_generated = True - valid_cover = True - try: - self.generateThumbnail(title, image_dir, thumb_file) - thumbs.append("thumbnail_%d.jpg" % int(title['id'])) - except: - if 'cover' in title and os.path.exists(title['cover']): - valid_cover = False - self.opts.log.warn(" *** Invalid cover file for '%s'***" % - (title['title'])) - if not self.error: - self.error.append('Invalid cover files') - self.error.append("Warning: invalid cover file for '%s', default cover substituted.\n" % (title['title'])) - - thumb_generated = False - - if not thumb_generated: - self.opts.log.warn(" using default cover for '%s' (%d)" % (title['title'], title['id'])) - # Confirm thumb exists, default is current - default_thumb_fp = os.path.join(image_dir,"thumbnail_default.jpg") - cover = os.path.join(self.catalogPath, "DefaultCover.png") - title['cover'] = cover - - if not os.path.exists(cover): - shutil.copyfile(I('book.png'), cover) - - if os.path.isfile(default_thumb_fp): - # Check to see if default cover is newer than thumbnail - # os.path.getmtime() = modified time - # os.path.ctime() = creation time - cover_timestamp = os.path.getmtime(cover) - thumb_timestamp = os.path.getmtime(default_thumb_fp) - if thumb_timestamp < cover_timestamp: - if False and self.verbose: - self.opts.log.warn("updating thumbnail_default for %s" % title['title']) - self.generateThumbnail(title, image_dir, - "thumbnail_default.jpg" if valid_cover else thumb_file) - else: - if False and self.verbose: - self.opts.log.warn(" generating new thumbnail_default.jpg") - self.generateThumbnail(title, image_dir, - "thumbnail_default.jpg" if valid_cover else thumb_file) - # Clear the book's cover property - title['cover'] = None - - - # Write thumb_width to the file, validating cache contents - # Allows detection of aborted catalog builds - with ZipFile(self.__archive_path, mode='a') as zfw: - zfw.writestr('thumb_width', self.opts.thumb_width) - - self.thumbs = thumbs - - def generateOPF(self): - - self.updateProgressFullStep("Generating OPF") - - header = ''' - <?xml version="1.0" encoding="UTF-8"?> - <package xmlns="http://www.idpf.org/2007/opf" version="2.0" unique-identifier="calibre_id"> - <metadata xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:opf="http://www.idpf.org/2007/opf" xmlns:calibre="http://calibre.kovidgoyal.net/2009/metadata" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"> - <dc:language>en-US</dc:language> - <meta name="calibre:publication_type" content="periodical:default"/> - </metadata> - <manifest></manifest> - <spine toc="ncx"></spine> - <guide></guide> - </package> - ''' - # Add the supplied metadata tags - soup = BeautifulStoneSoup(header, selfClosingTags=['item','itemref', 'reference']) - metadata = soup.find('metadata') - mtc = 0 - - titleTag = Tag(soup, "dc:title") - titleTag.insert(0,self.title) - metadata.insert(mtc, titleTag) - mtc += 1 - - creatorTag = Tag(soup, "dc:creator") - creatorTag.insert(0, self.creator) - metadata.insert(mtc, creatorTag) - mtc += 1 - - # Create the OPF tags - manifest = soup.find('manifest') - mtc = 0 - spine = soup.find('spine') - stc = 0 - guide = soup.find('guide') - - itemTag = Tag(soup, "item") - itemTag['id'] = "ncx" - itemTag['href'] = '%s.ncx' % self.basename - itemTag['media-type'] = "application/x-dtbncx+xml" - manifest.insert(mtc, itemTag) - mtc += 1 - - itemTag = Tag(soup, "item") - itemTag['id'] = 'stylesheet' - itemTag['href'] = self.stylesheet - itemTag['media-type'] = 'text/css' - manifest.insert(mtc, itemTag) - mtc += 1 - - itemTag = Tag(soup, "item") - itemTag['id'] = 'mastheadimage-image' - itemTag['href'] = "images/mastheadImage.gif" - itemTag['media-type'] = 'image/gif' - manifest.insert(mtc, itemTag) - mtc += 1 - - # Write the thumbnail images, descriptions to the manifest - sort_descriptions_by = [] - if self.opts.generate_descriptions: - for thumb in self.thumbs: - itemTag = Tag(soup, "item") - itemTag['href'] = "images/%s" % (thumb) - end = thumb.find('.jpg') - itemTag['id'] = "%s-image" % thumb[:end] - itemTag['media-type'] = 'image/jpeg' - manifest.insert(mtc, itemTag) - mtc += 1 - - # HTML files - add descriptions to manifest and spine - sort_descriptions_by = self.booksByAuthor if self.opts.sort_descriptions_by_author \ - else self.booksByTitle - # Add html_files to manifest and spine - - for file in self.htmlFileList_1: - # By Author, By Title, By Series, - itemTag = Tag(soup, "item") - start = file.find('/') + 1 - end = file.find('.') - itemTag['href'] = file - itemTag['id'] = file[start:end].lower() - itemTag['media-type'] = "application/xhtml+xml" - manifest.insert(mtc, itemTag) - mtc += 1 - - # spine - itemrefTag = Tag(soup, "itemref") - itemrefTag['idref'] = file[start:end].lower() - spine.insert(stc, itemrefTag) - stc += 1 - - # Add genre files to manifest and spine - for genre in self.genres: - if False: self.opts.log.info("adding %s to manifest and spine" % genre['tag']) - itemTag = Tag(soup, "item") - start = genre['file'].find('/') + 1 - end = genre['file'].find('.') - itemTag['href'] = genre['file'] - itemTag['id'] = genre['file'][start:end].lower() - itemTag['media-type'] = "application/xhtml+xml" - manifest.insert(mtc, itemTag) - mtc += 1 - - # spine - itemrefTag = Tag(soup, "itemref") - itemrefTag['idref'] = genre['file'][start:end].lower() - spine.insert(stc, itemrefTag) - stc += 1 - - for file in self.htmlFileList_2: - # By Date Added, By Date Read - itemTag = Tag(soup, "item") - start = file.find('/') + 1 - end = file.find('.') - itemTag['href'] = file - itemTag['id'] = file[start:end].lower() - itemTag['media-type'] = "application/xhtml+xml" - manifest.insert(mtc, itemTag) - mtc += 1 - - # spine - itemrefTag = Tag(soup, "itemref") - itemrefTag['idref'] = file[start:end].lower() - spine.insert(stc, itemrefTag) - stc += 1 - - for book in sort_descriptions_by: - # manifest - itemTag = Tag(soup, "item") - itemTag['href'] = "content/book_%d.html" % int(book['id']) - itemTag['id'] = "book%d" % int(book['id']) - itemTag['media-type'] = "application/xhtml+xml" - manifest.insert(mtc, itemTag) - mtc += 1 - - # spine - itemrefTag = Tag(soup, "itemref") - itemrefTag['idref'] = "book%d" % int(book['id']) - spine.insert(stc, itemrefTag) - stc += 1 - - # Guide - referenceTag = Tag(soup, "reference") - referenceTag['type'] = 'masthead' - referenceTag['title'] = 'mastheadimage-image' - referenceTag['href'] = 'images/mastheadImage.gif' - guide.insert(0,referenceTag) - - # Write the OPF file - outfile = open("%s/%s.opf" % (self.catalogPath, self.basename), 'w') - outfile.write(soup.prettify()) - - def generateNCXHeader(self): - - self.updateProgressFullStep("NCX header") - - header = ''' - <?xml version="1.0" encoding="utf-8"?> - <ncx xmlns="http://www.daisy.org/z3986/2005/ncx/" xmlns:calibre="http://calibre.kovidgoyal.net/2009/metadata" version="2005-1" xml:lang="en"> - </ncx> - ''' - soup = BeautifulStoneSoup(header, selfClosingTags=['content','calibre:meta-img']) - - ncx = soup.find('ncx') - navMapTag = Tag(soup, 'navMap') - navPointTag = Tag(soup, 'navPoint') - navPointTag['class'] = "periodical" - navPointTag['id'] = "title" - navPointTag['playOrder'] = self.playOrder - self.playOrder += 1 - navLabelTag = Tag(soup, 'navLabel') - textTag = Tag(soup, 'text') - textTag.insert(0, NavigableString(self.title)) - navLabelTag.insert(0, textTag) - navPointTag.insert(0, navLabelTag) - - if self.opts.generate_authors: - contentTag = Tag(soup, 'content') - contentTag['src'] = "content/ByAlphaAuthor.html" - navPointTag.insert(1, contentTag) - elif self.opts.generate_titles: - contentTag = Tag(soup, 'content') - contentTag['src'] = "content/ByAlphaTitle.html" - navPointTag.insert(1, contentTag) - elif self.opts.generate_series: - contentTag = Tag(soup, 'content') - contentTag['src'] = "content/BySeries.html" - navPointTag.insert(1, contentTag) - elif self.opts.generate_genres: - contentTag = Tag(soup, 'content') - #contentTag['src'] = "content/ByGenres.html" - contentTag['src'] = "%s" % self.genres[0]['file'] - navPointTag.insert(1, contentTag) - elif self.opts.generate_recently_added: - contentTag = Tag(soup, 'content') - contentTag['src'] = "content/ByDateAdded.html" - navPointTag.insert(1, contentTag) - else: - # Descriptions only - sort_descriptions_by = self.booksByAuthor if self.opts.sort_descriptions_by_author \ - else self.booksByTitle - contentTag = Tag(soup, 'content') - contentTag['src'] = "content/book_%d.html" % int(sort_descriptions_by[0]['id']) - navPointTag.insert(1, contentTag) - - cmiTag = Tag(soup, '%s' % 'calibre:meta-img') - cmiTag['name'] = "mastheadImage" - cmiTag['src'] = "images/mastheadImage.gif" - navPointTag.insert(2,cmiTag) - navMapTag.insert(0,navPointTag) - - ncx.insert(0,navMapTag) - self.ncxSoup = soup - - def generateNCXDescriptions(self, tocTitle): - - self.updateProgressFullStep("NCX 'Descriptions'") - - # --- Construct the 'Books by Title' section --- - ncx_soup = self.ncxSoup - body = ncx_soup.find("navPoint") - btc = len(body.contents) - - # Add the section navPoint - navPointTag = Tag(ncx_soup, 'navPoint') - navPointTag['class'] = "section" - navPointTag['id'] = "bytitle-ID" - navPointTag['playOrder'] = self.playOrder - self.playOrder += 1 - navLabelTag = Tag(ncx_soup, 'navLabel') - textTag = Tag(ncx_soup, 'text') - textTag.insert(0, NavigableString(tocTitle)) - navLabelTag.insert(0, textTag) - nptc = 0 - navPointTag.insert(nptc, navLabelTag) - nptc += 1 - contentTag = Tag(ncx_soup,"content") - contentTag['src'] = "content/book_%d.html" % int(self.booksByTitle[0]['id']) - navPointTag.insert(nptc, contentTag) - nptc += 1 - - # Loop over the titles - sort_descriptions_by = self.booksByAuthor if self.opts.sort_descriptions_by_author \ - else self.booksByTitle - - for book in sort_descriptions_by: - navPointVolumeTag = Tag(ncx_soup, 'navPoint') - navPointVolumeTag['class'] = "article" - navPointVolumeTag['id'] = "book%dID" % int(book['id']) - navPointVolumeTag['playOrder'] = self.playOrder - self.playOrder += 1 - navLabelTag = Tag(ncx_soup, "navLabel") - textTag = Tag(ncx_soup, "text") - if book['series']: - series_index = str(book['series_index']) - if series_index.endswith('.0'): - series_index = series_index[:-2] - if self.generateForKindle: - # Don't include Author for Kindle - textTag.insert(0, NavigableString(self.formatNCXText('%s (%s [%s])' % - (book['title'], book['series'], series_index), dest='title'))) - else: - # Include Author for non-Kindle - textTag.insert(0, NavigableString(self.formatNCXText('%s (%s [%s]) · %s ' % - (book['title'], book['series'], series_index, book['author']), dest='title'))) - else: - if self.generateForKindle: - # Don't include Author for Kindle - title_str = self.formatNCXText('%s' % (book['title']), dest='title') - if self.opts.connected_kindle and book['id'] in self.bookmarked_books: - ''' - dots = int((book['percent_read'] + 5)/10) - dot_string = '+' * dots - empty_dots = '-' * (10 - dots) - title_str += ' %s%s' % (dot_string,empty_dots) - ''' - title_str += '*' - textTag.insert(0, NavigableString(title_str)) - else: - # Include Author for non-Kindle - textTag.insert(0, NavigableString(self.formatNCXText('%s · %s' % \ - (book['title'], book['author']), dest='title'))) - navLabelTag.insert(0,textTag) - navPointVolumeTag.insert(0,navLabelTag) - - contentTag = Tag(ncx_soup, "content") - contentTag['src'] = "content/book_%d.html#book%d" % (int(book['id']), int(book['id'])) - navPointVolumeTag.insert(1, contentTag) - - if self.generateForKindle: - # Add the author tag - cmTag = Tag(ncx_soup, '%s' % 'calibre:meta') - cmTag['name'] = "author" - - if book['date']: - navStr = '%s | %s' % (self.formatNCXText(book['author'], dest='author'), - book['date'].split()[1]) - else: - navStr = '%s' % (self.formatNCXText(book['author'], dest='author')) - - if 'tags' in book and len(book['tags']): - navStr = self.formatNCXText(navStr + ' | ' + ' · '.join(sorted(book['tags'])), dest='author') - cmTag.insert(0, NavigableString(navStr)) - navPointVolumeTag.insert(2, cmTag) - - # Add the description tag - if book['short_description']: - cmTag = Tag(ncx_soup, '%s' % 'calibre:meta') - cmTag['name'] = "description" - cmTag.insert(0, NavigableString(self.formatNCXText(book['short_description'], dest='description'))) - navPointVolumeTag.insert(3, cmTag) - - # Add this volume to the section tag - navPointTag.insert(nptc, navPointVolumeTag) - nptc += 1 - - # Add this section to the body - body.insert(btc, navPointTag) - btc += 1 - - self.ncxSoup = ncx_soup - - def generateNCXBySeries(self, tocTitle): - self.updateProgressFullStep("NCX 'Series'") - - def add_to_series_by_letter(current_series_list): - current_series_list = " • ".join(current_series_list) - current_series_list = self.formatNCXText(current_series_list, dest="description") - series_by_letter.append(current_series_list) - - soup = self.ncxSoup - output = "BySeries" - body = soup.find("navPoint") - btc = len(body.contents) - - # --- Construct the 'Books By Series' section --- - navPointTag = Tag(soup, 'navPoint') - navPointTag['class'] = "section" - navPointTag['id'] = "byseries-ID" - navPointTag['playOrder'] = self.playOrder - self.playOrder += 1 - navLabelTag = Tag(soup, 'navLabel') - textTag = Tag(soup, 'text') - textTag.insert(0, NavigableString(tocTitle)) - navLabelTag.insert(0, textTag) - nptc = 0 - navPointTag.insert(nptc, navLabelTag) - nptc += 1 - contentTag = Tag(soup,"content") - contentTag['src'] = "content/%s.html#section_start" % (output) - navPointTag.insert(nptc, contentTag) - nptc += 1 - - series_by_letter = [] - - # Loop over the series titles, find start of each letter, add description_preview_count books - # Special switch for using different title list - title_list = self.booksBySeries - current_letter = self.letter_or_symbol(self.generateSortTitle(title_list[0]['series'])[0]) - title_letters = [current_letter] - current_series_list = [] - current_series = "" - for book in title_list: - sort_title = self.generateSortTitle(book['series']) - if self.letter_or_symbol(sort_title[0]) != current_letter: - # Save the old list - add_to_series_by_letter(current_series_list) - - # Start the new list - current_letter = self.letter_or_symbol(sort_title[0]) - title_letters.append(current_letter) - current_series = book['series'] - current_series_list = [book['series']] - else: - if len(current_series_list) < self.descriptionClip and \ - book['series'] != current_series : - current_series = book['series'] - current_series_list.append(book['series']) - - # Add the last book list - add_to_series_by_letter(current_series_list) - - # Add *article* entries for each populated series title letter - for (i,books) in enumerate(series_by_letter): - navPointByLetterTag = Tag(soup, 'navPoint') - navPointByLetterTag['class'] = "article" - navPointByLetterTag['id'] = "%sSeries-ID" % (title_letters[i].upper()) - navPointTag['playOrder'] = self.playOrder - self.playOrder += 1 - navLabelTag = Tag(soup, 'navLabel') - textTag = Tag(soup, 'text') - textTag.insert(0, NavigableString(u"Series beginning with %s" % \ - (title_letters[i] if len(title_letters[i])>1 else "'" + title_letters[i] + "'"))) - navLabelTag.insert(0, textTag) - navPointByLetterTag.insert(0,navLabelTag) - contentTag = Tag(soup, 'content') - contentTag['src'] = "content/%s.html#%s_series" % (output, title_letters[i]) - navPointByLetterTag.insert(1,contentTag) - - if self.generateForKindle: - cmTag = Tag(soup, '%s' % 'calibre:meta') - cmTag['name'] = "description" - cmTag.insert(0, NavigableString(self.formatNCXText(books, dest='description'))) - navPointByLetterTag.insert(2, cmTag) - - navPointTag.insert(nptc, navPointByLetterTag) - nptc += 1 - - # Add this section to the body - body.insert(btc, navPointTag) - btc += 1 - - self.ncxSoup = soup - - def generateNCXByTitle(self, tocTitle): - self.updateProgressFullStep("NCX 'Titles'") - - def add_to_books_by_letter(current_book_list): - current_book_list = " • ".join(current_book_list) - current_book_list = self.formatNCXText(current_book_list, dest="description") - books_by_letter.append(current_book_list) - - soup = self.ncxSoup - output = "ByAlphaTitle" - body = soup.find("navPoint") - btc = len(body.contents) - - # --- Construct the 'Books By Title' section --- - navPointTag = Tag(soup, 'navPoint') - navPointTag['class'] = "section" - navPointTag['id'] = "byalphatitle-ID" - navPointTag['playOrder'] = self.playOrder - self.playOrder += 1 - navLabelTag = Tag(soup, 'navLabel') - textTag = Tag(soup, 'text') - textTag.insert(0, NavigableString(tocTitle)) - navLabelTag.insert(0, textTag) - nptc = 0 - navPointTag.insert(nptc, navLabelTag) - nptc += 1 - contentTag = Tag(soup,"content") - contentTag['src'] = "content/%s.html#section_start" % (output) - navPointTag.insert(nptc, contentTag) - nptc += 1 - - books_by_letter = [] - - # Loop over the titles, find start of each letter, add description_preview_count books - # Special switch for using different title list - if self.useSeriesPrefixInTitlesSection: - title_list = self.booksByTitle - else: - title_list = self.booksByTitle_noSeriesPrefix - current_letter = self.letter_or_symbol(title_list[0]['title_sort'][0]) - title_letters = [current_letter] - current_book_list = [] - current_book = "" - for book in title_list: - if self.letter_or_symbol(book['title_sort'][0]) != current_letter: - # Save the old list - add_to_books_by_letter(current_book_list) - - # Start the new list - current_letter = self.letter_or_symbol(book['title_sort'][0]) - title_letters.append(current_letter) - current_book = book['title'] - current_book_list = [book['title']] - else: - if len(current_book_list) < self.descriptionClip and \ - book['title'] != current_book : - current_book = book['title'] - current_book_list.append(book['title']) - - # Add the last book list - add_to_books_by_letter(current_book_list) - - # Add *article* entries for each populated title letter - for (i,books) in enumerate(books_by_letter): - navPointByLetterTag = Tag(soup, 'navPoint') - navPointByLetterTag['class'] = "article" - navPointByLetterTag['id'] = "%sTitles-ID" % (title_letters[i].upper()) - navPointTag['playOrder'] = self.playOrder - self.playOrder += 1 - navLabelTag = Tag(soup, 'navLabel') - textTag = Tag(soup, 'text') - textTag.insert(0, NavigableString(u"Titles beginning with %s" % \ - (title_letters[i] if len(title_letters[i])>1 else "'" + title_letters[i] + "'"))) - navLabelTag.insert(0, textTag) - navPointByLetterTag.insert(0,navLabelTag) - contentTag = Tag(soup, 'content') - contentTag['src'] = "content/%s.html#%s" % (output, title_letters[i]) - navPointByLetterTag.insert(1,contentTag) - - if self.generateForKindle: - cmTag = Tag(soup, '%s' % 'calibre:meta') - cmTag['name'] = "description" - cmTag.insert(0, NavigableString(self.formatNCXText(books, dest='description'))) - navPointByLetterTag.insert(2, cmTag) - - navPointTag.insert(nptc, navPointByLetterTag) - nptc += 1 - - # Add this section to the body - body.insert(btc, navPointTag) - btc += 1 - - self.ncxSoup = soup - - def generateNCXByAuthor(self, tocTitle): - self.updateProgressFullStep("NCX 'Authors'") - - def add_to_author_list(current_author_list, current_letter): - current_author_list = " • ".join(current_author_list) - current_author_list = self.formatNCXText(current_author_list, dest="description") - master_author_list.append((current_author_list, current_letter)) - - soup = self.ncxSoup - HTML_file = "content/ByAlphaAuthor.html" - body = soup.find("navPoint") - btc = len(body.contents) - - # --- Construct the 'Books By Author' *section* --- - navPointTag = Tag(soup, 'navPoint') - navPointTag['class'] = "section" - file_ID = "%s" % tocTitle.lower() - file_ID = file_ID.replace(" ","") - navPointTag['id'] = "%s-ID" % file_ID - navPointTag['playOrder'] = self.playOrder - self.playOrder += 1 - navLabelTag = Tag(soup, 'navLabel') - textTag = Tag(soup, 'text') - textTag.insert(0, NavigableString('%s' % tocTitle)) - navLabelTag.insert(0, textTag) - nptc = 0 - navPointTag.insert(nptc, navLabelTag) - nptc += 1 - contentTag = Tag(soup,"content") - contentTag['src'] = "%s#section_start" % HTML_file - navPointTag.insert(nptc, contentTag) - nptc += 1 - - # Create an NCX article entry for each populated author index letter - # Loop over the sorted_authors list, find start of each letter, - # add description_preview_count artists - # self.authors[0]:friendly [1]:author_sort [2]:book_count - master_author_list = [] - # self.authors[0][1][0] = Initial letter of author_sort[0] - current_letter = self.letter_or_symbol(self.authors[0][1][0]) - current_author_list = [] - for author in self.authors: - if self.letter_or_symbol(author[1][0]) != current_letter: - # Save the old list - add_to_author_list(current_author_list, current_letter) - - # Start the new list - current_letter = self.letter_or_symbol(author[1][0]) - current_author_list = [author[0]] - else: - if len(current_author_list) < self.descriptionClip: - current_author_list.append(author[0]) - - # Add the last author list - add_to_author_list(current_author_list, current_letter) - - # Add *article* entries for each populated author initial letter - # master_author_list{}: [0]:author list [1]:Initial letter - for authors_by_letter in master_author_list: - navPointByLetterTag = Tag(soup, 'navPoint') - navPointByLetterTag['class'] = "article" - navPointByLetterTag['id'] = "%sauthors-ID" % (authors_by_letter[1]) - navPointTag['playOrder'] = self.playOrder - self.playOrder += 1 - navLabelTag = Tag(soup, 'navLabel') - textTag = Tag(soup, 'text') - textTag.insert(0, NavigableString("Authors beginning with '%s'" % (authors_by_letter[1]))) - navLabelTag.insert(0, textTag) - navPointByLetterTag.insert(0,navLabelTag) - contentTag = Tag(soup, 'content') - contentTag['src'] = "%s#%sauthors" % (HTML_file, authors_by_letter[1]) - - navPointByLetterTag.insert(1,contentTag) - - if self.generateForKindle: - cmTag = Tag(soup, '%s' % 'calibre:meta') - cmTag['name'] = "description" - cmTag.insert(0, NavigableString(authors_by_letter[0])) - navPointByLetterTag.insert(2, cmTag) - - navPointTag.insert(nptc, navPointByLetterTag) - nptc += 1 - - # Add this section to the body - body.insert(btc, navPointTag) - btc += 1 - - self.ncxSoup = soup - - def generateNCXByDateAdded(self, tocTitle): - self.updateProgressFullStep("NCX 'Recently Added'") - - def add_to_master_month_list(current_titles_list): - book_count = len(current_titles_list) - current_titles_list = " • ".join(current_titles_list) - current_titles_list = self.formatNCXText(current_titles_list, dest='description') - master_month_list.append((current_titles_list, current_date, book_count)) - - def add_to_master_date_range_list(current_titles_list): - book_count = len(current_titles_list) - current_titles_list = " • ".join(current_titles_list) - current_titles_list = self.formatNCXText(current_titles_list, dest='description') - master_date_range_list.append((current_titles_list, date_range, book_count)) - - soup = self.ncxSoup - HTML_file = "content/ByDateAdded.html" - body = soup.find("navPoint") - btc = len(body.contents) - - # --- Construct the 'Recently Added' *section* --- - navPointTag = Tag(soup, 'navPoint') - navPointTag['class'] = "section" - file_ID = "%s" % tocTitle.lower() - file_ID = file_ID.replace(" ","") - navPointTag['id'] = "%s-ID" % file_ID - navPointTag['playOrder'] = self.playOrder - self.playOrder += 1 - navLabelTag = Tag(soup, 'navLabel') - textTag = Tag(soup, 'text') - textTag.insert(0, NavigableString('%s' % tocTitle)) - navLabelTag.insert(0, textTag) - nptc = 0 - navPointTag.insert(nptc, navLabelTag) - nptc += 1 - contentTag = Tag(soup,"content") - contentTag['src'] = "%s#section_start" % HTML_file - navPointTag.insert(nptc, contentTag) - nptc += 1 - - # Create an NCX article entry for each date range - current_titles_list = [] - master_date_range_list = [] - today = datetime.datetime.now() - today_time = datetime.datetime(today.year, today.month, today.day) - for (i,date) in enumerate(self.DATE_RANGE): - if i: - date_range = '%d to %d days ago' % (self.DATE_RANGE[i-1], self.DATE_RANGE[i]) - else: - date_range = 'Last %d days' % (self.DATE_RANGE[i]) - date_range_limit = self.DATE_RANGE[i] - for book in self.booksByDateRange: - book_time = datetime.datetime(book['timestamp'].year, book['timestamp'].month, book['timestamp'].day) - if (today_time-book_time).days <= date_range_limit: - #print "generateNCXByDateAdded: %s added %d days ago" % (book['title'], (today_time-book_time).days) - current_titles_list.append(book['title']) - else: - break - if current_titles_list: - add_to_master_date_range_list(current_titles_list) - current_titles_list = [book['title']] - - # Add *article* entries for each populated date range - # master_date_range_list{}: [0]:titles list [1]:datestr - for books_by_date_range in master_date_range_list: - navPointByDateRangeTag = Tag(soup, 'navPoint') - navPointByDateRangeTag['class'] = "article" - navPointByDateRangeTag['id'] = "%s-ID" % books_by_date_range[1].replace(' ','') - navPointTag['playOrder'] = self.playOrder - self.playOrder += 1 - navLabelTag = Tag(soup, 'navLabel') - textTag = Tag(soup, 'text') - textTag.insert(0, NavigableString(books_by_date_range[1])) - navLabelTag.insert(0, textTag) - navPointByDateRangeTag.insert(0,navLabelTag) - contentTag = Tag(soup, 'content') - contentTag['src'] = "%s#bda_%s" % (HTML_file, - books_by_date_range[1].replace(' ','')) - - navPointByDateRangeTag.insert(1,contentTag) - - if self.generateForKindle: - cmTag = Tag(soup, '%s' % 'calibre:meta') - cmTag['name'] = "description" - cmTag.insert(0, NavigableString(books_by_date_range[0])) - navPointByDateRangeTag.insert(2, cmTag) - - cmTag = Tag(soup, '%s' % 'calibre:meta') - cmTag['name'] = "author" - navStr = '%d titles' % books_by_date_range[2] if books_by_date_range[2] > 1 else \ - '%d title' % books_by_date_range[2] - cmTag.insert(0, NavigableString(navStr)) - navPointByDateRangeTag.insert(3, cmTag) - - navPointTag.insert(nptc, navPointByDateRangeTag) - nptc += 1 - - - - # Create an NCX article entry for each populated month - # Loop over the booksByDate list, find start of each month, - # add description_preview_count titles - # master_month_list(list,date,count) - current_titles_list = [] - master_month_list = [] - current_date = self.booksByMonth[0]['timestamp'] - - for book in self.booksByMonth: - if book['timestamp'].month != current_date.month or \ - book['timestamp'].year != current_date.year: - # Save the old lists - add_to_master_month_list(current_titles_list) - - # Start the new list - current_date = book['timestamp'].date() - current_titles_list = [book['title']] - else: - current_titles_list.append(book['title']) - - # Add the last month list - add_to_master_month_list(current_titles_list) - - # Add *article* entries for each populated month - # master_months_list{}: [0]:titles list [1]:date - for books_by_month in master_month_list: - datestr = strftime(u'%B %Y', books_by_month[1].timetuple()) - navPointByMonthTag = Tag(soup, 'navPoint') - navPointByMonthTag['class'] = "article" - navPointByMonthTag['id'] = "bda_%s-%s-ID" % (books_by_month[1].year,books_by_month[1].month ) - navPointTag['playOrder'] = self.playOrder - self.playOrder += 1 - navLabelTag = Tag(soup, 'navLabel') - textTag = Tag(soup, 'text') - textTag.insert(0, NavigableString(datestr)) - navLabelTag.insert(0, textTag) - navPointByMonthTag.insert(0,navLabelTag) - contentTag = Tag(soup, 'content') - contentTag['src'] = "%s#bda_%s-%s" % (HTML_file, - books_by_month[1].year,books_by_month[1].month) - - navPointByMonthTag.insert(1,contentTag) - - if self.generateForKindle: - cmTag = Tag(soup, '%s' % 'calibre:meta') - cmTag['name'] = "description" - cmTag.insert(0, NavigableString(books_by_month[0])) - navPointByMonthTag.insert(2, cmTag) - - cmTag = Tag(soup, '%s' % 'calibre:meta') - cmTag['name'] = "author" - navStr = '%d titles' % books_by_month[2] if books_by_month[2] > 1 else \ - '%d title' % books_by_month[2] - cmTag.insert(0, NavigableString(navStr)) - navPointByMonthTag.insert(3, cmTag) - - navPointTag.insert(nptc, navPointByMonthTag) - nptc += 1 - - # Add this section to the body - body.insert(btc, navPointTag) - btc += 1 - self.ncxSoup = soup - - def generateNCXByDateRead(self, tocTitle): - self.updateProgressFullStep("NCX 'Recently Read'") - if not self.booksByDateRead: - return - - def add_to_master_day_list(current_titles_list): - book_count = len(current_titles_list) - current_titles_list = " • ".join(current_titles_list) - current_titles_list = self.formatNCXText(current_titles_list, dest='description') - master_day_list.append((current_titles_list, current_date, book_count)) - - def add_to_master_date_range_list(current_titles_list): - book_count = len(current_titles_list) - current_titles_list = " • ".join(current_titles_list) - current_titles_list = self.formatNCXText(current_titles_list, dest='description') - master_date_range_list.append((current_titles_list, date_range, book_count)) - - soup = self.ncxSoup - HTML_file = "content/ByDateRead.html" - body = soup.find("navPoint") - btc = len(body.contents) - - # --- Construct the 'Recently Read' *section* --- - navPointTag = Tag(soup, 'navPoint') - navPointTag['class'] = "section" - file_ID = "%s" % tocTitle.lower() - file_ID = file_ID.replace(" ","") - navPointTag['id'] = "%s-ID" % file_ID - navPointTag['playOrder'] = self.playOrder - self.playOrder += 1 - navLabelTag = Tag(soup, 'navLabel') - textTag = Tag(soup, 'text') - textTag.insert(0, NavigableString('%s' % tocTitle)) - navLabelTag.insert(0, textTag) - nptc = 0 - navPointTag.insert(nptc, navLabelTag) - nptc += 1 - contentTag = Tag(soup,"content") - contentTag['src'] = "%s#section_start" % HTML_file - navPointTag.insert(nptc, contentTag) - nptc += 1 - - # Create an NCX article entry for each date range - current_titles_list = [] - master_date_range_list = [] - today = datetime.datetime.now() - today_time = datetime.datetime(today.year, today.month, today.day) - for (i,date) in enumerate(self.DATE_RANGE): - if i: - date_range = '%d to %d days ago' % (self.DATE_RANGE[i-1], self.DATE_RANGE[i]) - else: - date_range = 'Last %d days' % (self.DATE_RANGE[i]) - date_range_limit = self.DATE_RANGE[i] - for book in self.booksByDateRead: - bookmark_time = datetime.datetime.utcfromtimestamp(book['bookmark_timestamp']) - if (today_time-bookmark_time).days <= date_range_limit: - #print "generateNCXByDateAdded: %s added %d days ago" % (book['title'], (today_time-book_time).days) - current_titles_list.append(book['title']) - else: - break - if current_titles_list: - add_to_master_date_range_list(current_titles_list) - current_titles_list = [book['title']] - - # Create an NCX article entry for each populated day - # Loop over the booksByDate list, find start of each month, - # add description_preview_count titles - # master_month_list(list,date,count) - current_titles_list = [] - master_day_list = [] - current_date = datetime.datetime.utcfromtimestamp(self.booksByDateRead[0]['bookmark_timestamp']) - - for book in self.booksByDateRead: - bookmark_time = datetime.datetime.utcfromtimestamp(book['bookmark_timestamp']) - if bookmark_time.day != current_date.day or \ - bookmark_time.month != current_date.month or \ - bookmark_time.year != current_date.year: - # Save the old lists - add_to_master_day_list(current_titles_list) - - # Start the new list - current_date = datetime.datetime.utcfromtimestamp(book['bookmark_timestamp']).date() - current_titles_list = [book['title']] - else: - current_titles_list.append(book['title']) - - # Add the last day list - add_to_master_day_list(current_titles_list) - - # Add *article* entries for each populated day - # master_day_list{}: [0]:titles list [1]:date - for books_by_day in master_day_list: - datestr = strftime(u'%A, %B %d', books_by_day[1].timetuple()) - navPointByDayTag = Tag(soup, 'navPoint') - navPointByDayTag['class'] = "article" - navPointByDayTag['id'] = "bdr_%s-%s-%sID" % (books_by_day[1].year, - books_by_day[1].month, - books_by_day[1].day ) - navPointTag['playOrder'] = self.playOrder - self.playOrder += 1 - navLabelTag = Tag(soup, 'navLabel') - textTag = Tag(soup, 'text') - textTag.insert(0, NavigableString(datestr)) - navLabelTag.insert(0, textTag) - navPointByDayTag.insert(0,navLabelTag) - contentTag = Tag(soup, 'content') - contentTag['src'] = "%s#bdr_%s-%s-%s" % (HTML_file, - books_by_day[1].year, - books_by_day[1].month, - books_by_day[1].day) - - navPointByDayTag.insert(1,contentTag) - - if self.generateForKindle: - cmTag = Tag(soup, '%s' % 'calibre:meta') - cmTag['name'] = "description" - cmTag.insert(0, NavigableString(books_by_day[0])) - navPointByDayTag.insert(2, cmTag) - - cmTag = Tag(soup, '%s' % 'calibre:meta') - cmTag['name'] = "author" - navStr = '%d titles' % books_by_day[2] if books_by_day[2] > 1 else \ - '%d title' % books_by_day[2] - cmTag.insert(0, NavigableString(navStr)) - navPointByDayTag.insert(3, cmTag) - - navPointTag.insert(nptc, navPointByDayTag) - nptc += 1 - - # Add this section to the body - body.insert(btc, navPointTag) - btc += 1 - self.ncxSoup = soup - - def generateNCXByGenre(self, tocTitle): - # Create an NCX section for 'By Genre' - # Add each genre as an article - # 'tag', 'file', 'authors' - - self.updateProgressFullStep("NCX 'Genres'") - - if not len(self.genres): - self.opts.log.warn(" No genres found in tags.\n" - " No Genre section added to Catalog") - return - - ncx_soup = self.ncxSoup - body = ncx_soup.find("navPoint") - btc = len(body.contents) - - # --- Construct the 'Books By Genre' *section* --- - navPointTag = Tag(ncx_soup, 'navPoint') - navPointTag['class'] = "section" - file_ID = "%s" % tocTitle.lower() - file_ID = file_ID.replace(" ","") - navPointTag['id'] = "%s-ID" % file_ID - navPointTag['playOrder'] = self.playOrder - self.playOrder += 1 - navLabelTag = Tag(ncx_soup, 'navLabel') - textTag = Tag(ncx_soup, 'text') - # textTag.insert(0, NavigableString('%s (%d)' % (section_title, len(genre_list)))) - textTag.insert(0, NavigableString('%s' % tocTitle)) - navLabelTag.insert(0, textTag) - nptc = 0 - navPointTag.insert(nptc, navLabelTag) - nptc += 1 - contentTag = Tag(ncx_soup,"content") - contentTag['src'] = "content/Genre_%s.html#section_start" % self.genres[0]['tag'] - navPointTag.insert(nptc, contentTag) - nptc += 1 - - for genre in self.genres: - # Add an article for each genre - navPointVolumeTag = Tag(ncx_soup, 'navPoint') - navPointVolumeTag['class'] = "article" - navPointVolumeTag['id'] = "genre-%s-ID" % genre['tag'] - navPointVolumeTag['playOrder'] = self.playOrder - self.playOrder += 1 - navLabelTag = Tag(ncx_soup, "navLabel") - textTag = Tag(ncx_soup, "text") - - # GwR *** Can this be optimized? - normalized_tag = None - for friendly_tag in self.genre_tags_dict: - if self.genre_tags_dict[friendly_tag] == genre['tag']: - normalized_tag = self.genre_tags_dict[friendly_tag] - break - textTag.insert(0, self.formatNCXText(NavigableString(friendly_tag), dest='description')) - navLabelTag.insert(0,textTag) - navPointVolumeTag.insert(0,navLabelTag) - contentTag = Tag(ncx_soup, "content") - contentTag['src'] = "content/Genre_%s.html#Genre_%s" % (normalized_tag, normalized_tag) - navPointVolumeTag.insert(1, contentTag) - - if self.generateForKindle: - # Build the author tag - cmTag = Tag(ncx_soup, '%s' % 'calibre:meta') - cmTag['name'] = "author" - # First - Last author - - if len(genre['titles_spanned']) > 1 : - author_range = "%s - %s" % (genre['titles_spanned'][0][0], genre['titles_spanned'][1][0]) - else : - author_range = "%s" % (genre['titles_spanned'][0][0]) - - cmTag.insert(0, NavigableString(author_range)) - navPointVolumeTag.insert(2, cmTag) - - # Build the description tag - cmTag = Tag(ncx_soup, '%s' % 'calibre:meta') - cmTag['name'] = "description" - - if False: - # Form 1: Titles spanned - if len(genre['titles_spanned']) > 1: - title_range = "%s -\n%s" % (genre['titles_spanned'][0][1], genre['titles_spanned'][1][1]) - else: - title_range = "%s" % (genre['titles_spanned'][0][1]) - cmTag.insert(0, NavigableString(self.formatNCXText(title_range, dest='description'))) - else: - # Form 2: title • title • title ... - titles = [] - for title in genre['books']: - titles.append(title['title']) - titles = sorted(titles, key=lambda x:(self.generateSortTitle(x),self.generateSortTitle(x))) - titles_list = self.generateShortDescription(u" • ".join(titles), dest="description") - cmTag.insert(0, NavigableString(self.formatNCXText(titles_list, dest='description'))) - - navPointVolumeTag.insert(3, cmTag) - - # Add this volume to the section tag - navPointTag.insert(nptc, navPointVolumeTag) - nptc += 1 - - # Add this section to the body - body.insert(btc, navPointTag) - btc += 1 - self.ncxSoup = ncx_soup - - def writeNCX(self): - self.updateProgressFullStep("Saving NCX") - - outfile = open("%s/%s.ncx" % (self.catalogPath, self.basename), 'w') - outfile.write(self.ncxSoup.prettify()) - - - # ======================== Helpers ======================== - def author_to_author_sort(self, author): - tokens = author.split() - tokens = tokens[-1:] + tokens[:-1] - if len(tokens) > 1: - tokens[0] += ',' - return ' '.join(tokens).capitalize() - - def booksByAuthorSorter_author_sort(self, book): - ''' - Sort non-series books before series books - ''' - if not book['series']: - key = '%s %s' % (capitalize(book['author_sort']), - capitalize(book['title_sort'])) - else: - index = book['series_index'] - integer = int(index) - fraction = index-integer - series_index = '%04d%s' % (integer, str('%0.4f' % fraction).lstrip('0')) - key = '%s ~%s %s' % (capitalize(book['author_sort']), - self.generateSortTitle(book['series']), - series_index) - return key - - def booksByAuthorSorter_author(self, book): - ''' - Sort non-series books before series books - ''' - if not book['series']: - key = '%s %s' % (self.author_to_author_sort(book['author']), - capitalize(book['title_sort'])) - else: - index = book['series_index'] - integer = int(index) - fraction = index-integer - series_index = '%04d%s' % (integer, str('%0.4f' % fraction).lstrip('0')) - key = '%s ~%s %s' % (self.author_to_author_sort(book['author']), - self.generateSortTitle(book['series']), - series_index) - return key - - def calculateThumbnailSize(self): - ''' Calculate thumbnail dimensions based on device DPI. Scale Kindle by 50% ''' - from calibre.customize.ui import output_profiles - for x in output_profiles(): - if x.short_name == self.opts.output_profile: - # aspect ratio: 3:4 - self.thumbWidth = x.dpi * float(self.opts.thumb_width) - self.thumbHeight = self.thumbWidth * 1.33 - if 'kindle' in x.short_name and self.opts.fmt == 'mobi': - # Kindle DPI appears to be off by a factor of 2 - self.thumbWidth = self.thumbWidth/2 - self.thumbHeight = self.thumbHeight/2 - break - if True and self.verbose: - self.opts.log(" DPI = %d; thumbnail dimensions: %d x %d" % \ - (x.dpi, self.thumbWidth, self.thumbHeight)) - - def convertHTMLEntities(self, s): - matches = re.findall("&#\d+;", s) - if len(matches) > 0: - hits = set(matches) - for hit in hits: - name = hit[2:-1] - try: - entnum = int(name) - s = s.replace(hit, unichr(entnum)) - except ValueError: - pass - - matches = re.findall("&\w+;", s) - hits = set(matches) - amp = "&" - if amp in hits: - hits.remove(amp) - for hit in hits: - name = hit[1:-1] - if htmlentitydefs.name2codepoint.has_key(name): - s = s.replace(hit, unichr(htmlentitydefs.name2codepoint[name])) - s = s.replace(amp, "&") - return s - - def createDirectoryStructure(self): - catalogPath = self.catalogPath - self.cleanUp() - - if not os.path.isdir(catalogPath): - os.makedirs(catalogPath) - - # Create /content and /images - content_path = catalogPath + "/content" - if not os.path.isdir(content_path): - os.makedirs(content_path) - images_path = catalogPath + "/images" - if not os.path.isdir(images_path): - os.makedirs(images_path) - - def discoverReadStatus(self, record): - ''' - Given a field:pattern spec, discover if this book marked as read - - if field == tag, scan tags for pattern - if custom field, try regex match for pattern - This allows maximum flexibility with fields of type - datatype bool: #field_name:True - datatype text: #field_name:<string> - datatype datetime: #field_name:.* - - ''' - # Legacy handling of special 'read' tag - field = self.__read_book_marker['field'] - pat = self.__read_book_marker['pattern'] - if field == 'tag' and pat in record['tags']: - return True - - field_contents = self.__db.get_field(record['id'], - field, - index_is_id=True) - if field_contents: - try: - if re.search(pat, unicode(field_contents), - re.IGNORECASE) is not None: - return True - except: - # Compiling of pat failed, ignore it - pass - - return False - - def filterDbTags(self, tags): - # Remove the special marker tags from the database's tag list, - # return sorted list of normalized genre tags - - def format_tag_list(tags, indent=5, line_break=70, header='Tag list'): - def next_tag(sorted_tags): - for (i, tag) in enumerate(sorted_tags): - if i < len(tags) - 1: - yield tag + ", " - else: - yield tag - - ans = '%s%d %s:\n' % (' ' * indent, len(tags), header) - ans += ' ' * (indent + 1) - out_str = '' - sorted_tags = sorted(tags) - for tag in next_tag(sorted_tags): - out_str += tag - if len(out_str) >= line_break: - ans += out_str + '\n' - out_str = ' ' * (indent + 1) - return ans + out_str - - normalized_tags = [] - friendly_tags = [] - excluded_tags = [] - for tag in tags: - if tag in self.markerTags: - excluded_tags.append(tag) - continue - if re.search(self.opts.exclude_genre, tag): - excluded_tags.append(tag) - continue - if tag == ' ': - continue - - normalized_tags.append(re.sub('\W','',tag).lower()) - friendly_tags.append(tag) - - genre_tags_dict = dict(zip(friendly_tags,normalized_tags)) - - # Test for multiple genres resolving to same normalized form - normalized_set = set(normalized_tags) - for normalized in normalized_set: - if normalized_tags.count(normalized) > 1: - self.opts.log.warn(" Warning: multiple tags resolving to genre '%s':" % normalized) - for key in genre_tags_dict: - if genre_tags_dict[key] == normalized: - self.opts.log.warn(" %s" % key) - if self.verbose: - self.opts.log.info('%s' % format_tag_list(genre_tags_dict, header="enabled genre tags in database")) - self.opts.log.info('%s' % format_tag_list(excluded_tags, header="excluded genre tags")) - - return genre_tags_dict - - def formatNCXText(self, description, dest=None): - # Kindle TOC descriptions won't render certain characters - # Fix up - massaged = unicode(BeautifulStoneSoup(description, convertEntities=BeautifulStoneSoup.HTML_ENTITIES)) - - # Replace '&' with '&' - massaged = re.sub("&","&", massaged) - - if massaged.strip() and dest: - #print traceback.print_stack(limit=3) - return self.generateShortDescription(massaged.strip(), dest=dest) - else: - return None - - def generateAuthorAnchor(self, author): - # Strip white space to '' - return re.sub("\W","", author) - - def generateFormatArgs(self, book): - series_index = str(book['series_index']) - if series_index.endswith('.0'): - series_index = series_index[:-2] - args = dict( - title = book['title'], - series = book['series'], - series_index = series_index, - rating = self.generateRatingString(book), - rating_parens = '(%s)' % self.generateRatingString(book) if 'rating' in book else '', - pubyear = book['date'].split()[1] if book['date'] else '', - pubyear_parens = "(%s)" % book['date'].split()[1] if book['date'] else '') - return args - - def generateHTMLByGenre(self, genre, section_head, books, outfile): - # Write an HTML file of this genre's book list - # Return a list with [(first_author, first_book), (last_author, last_book)] - - soup = self.generateHTMLGenreHeader(genre) - body = soup.find('body') - - btc = 0 - - # Insert section tag if this is the section start - first article only - if section_head: - aTag = Tag(soup,'a') - aTag['name'] = 'section_start' - body.insert(btc, aTag) - btc += 1 - - # Create an anchor from the tag - aTag = Tag(soup, 'a') - aTag['name'] = "Genre_%s" % genre - body.insert(btc,aTag) - btc += 1 - - titleTag = body.find(attrs={'class':'title'}) - titleTag.insert(0,NavigableString('%s' % escape(self.getFriendlyGenreTag(genre)))) - - # Insert the books by author list - divTag = body.find(attrs={'class':'authors'}) - dtc = 0 - - current_author = '' - current_series = None - for book in books: - if book['author'] != current_author: - # Start a new author with link - current_author = book['author'] - non_series_books = 0 - current_series = None - pAuthorTag = Tag(soup, "p") - pAuthorTag['class'] = "author_index" - aTag = Tag(soup, "a") - if self.opts.generate_authors: - aTag['href'] = "%s.html#%s" % ("ByAlphaAuthor", self.generateAuthorAnchor(book['author'])) - aTag.insert(0, book['author']) - pAuthorTag.insert(0,aTag) - divTag.insert(dtc,pAuthorTag) - dtc += 1 - - # Check for series - if book['series'] and book['series'] != current_series: - # Start a new series - current_series = book['series'] - pSeriesTag = Tag(soup,'p') - pSeriesTag['class'] = "series" - if self.opts.generate_series: - aTag = Tag(soup,'a') - aTag['href'] = "%s.html#%s_series" % ('BySeries', - re.sub('\W','',book['series']).lower()) - aTag.insert(0, book['series']) - pSeriesTag.insert(0, aTag) - else: - pSeriesTag.insert(0,NavigableString('%s' % book['series'])) - divTag.insert(dtc,pSeriesTag) - dtc += 1 - - if current_series and not book['series']: - current_series = None - - # Add books - pBookTag = Tag(soup, "p") - ptc = 0 - - # book with read|reading|unread symbol or wishlist item - if self.opts.wishlist_tag in book.get('tags', []): - pBookTag['class'] = "wishlist_item" - pBookTag.insert(ptc,NavigableString(self.MISSING_SYMBOL)) - ptc += 1 - else: - if book['read']: - # check mark - pBookTag.insert(ptc,NavigableString(self.READ_SYMBOL)) - pBookTag['class'] = "read_book" - ptc += 1 - elif book['id'] in self.bookmarked_books: - pBookTag.insert(ptc,NavigableString(self.READING_SYMBOL)) - pBookTag['class'] = "read_book" - ptc += 1 - else: - # hidden check mark - pBookTag['class'] = "unread_book" - pBookTag.insert(ptc,NavigableString(self.NOT_READ_SYMBOL)) - ptc += 1 - - # Add the book title - aTag = Tag(soup, "a") - if self.opts.generate_descriptions: - aTag['href'] = "book_%d.html" % (int(float(book['id']))) - - # Generate the title from the template - args = self.generateFormatArgs(book) - if current_series: - #aTag.insert(0,escape(book['title'][len(book['series'])+1:])) - formatted_title = self.by_genres_series_title_template.format(**args).rstrip() - else: - #aTag.insert(0,escape(book['title'])) - formatted_title = self.by_genres_normal_title_template.format(**args).rstrip() - non_series_books += 1 - aTag.insert(0,NavigableString(escape(formatted_title))) - - pBookTag.insert(ptc, aTag) - ptc += 1 - - divTag.insert(dtc, pBookTag) - dtc += 1 - - # Write the generated file to contentdir - outfile = open(outfile, 'w') - outfile.write(soup.prettify()) - outfile.close() - - if len(books) > 1: - titles_spanned = [(books[0]['author'],books[0]['title']), (books[-1]['author'],books[-1]['title'])] - else: - titles_spanned = [(books[0]['author'],books[0]['title'])] - - return titles_spanned - - def generateHTMLDescriptionHeader(self, book): - ''' - Generate description header from template - ''' - from calibre.ebooks.oeb.base import XHTML_NS - - def generate_html(): - args = dict( - author=author, - author_prefix=author_prefix, - comments=comments, - css=css, - formats=formats, - genres=genres, - note_content=note_content, - note_source=note_source, - pubdate=pubdate, - publisher=publisher, - pubmonth=pubmonth, - pubyear=pubyear, - rating=rating, - series=series, - series_index=series_index, - thumb=thumb, - title=title, - title_str=title_str, - xmlns=XHTML_NS, - ) - - generated_html = P('catalog/template.xhtml', - data=True).decode('utf-8').format(**args) - generated_html = substitute_entites(generated_html) - return BeautifulSoup(generated_html) - - # Generate the template arguments - css = P('catalog/stylesheet.css', data=True).decode('utf-8') - title_str = title = escape(book['title']) - series = '' - series_index = '' - if book['series']: - series = escape(book['series']) - series_index = str(book['series_index']) - if series_index.endswith('.0'): - series_index = series_index[:-2] - - # Author, author_prefix (read|reading|none symbol or missing symbol) - author = book['author'] - if self.opts.wishlist_tag in book.get('tags', []): - author_prefix = self.MISSING_SYMBOL + " by " - else: - if book['read']: - author_prefix = self.READ_SYMBOL + " by " - elif self.opts.connected_kindle and book['id'] in self.bookmarked_books: - author_prefix = self.READING_SYMBOL + " by " - else: - author_prefix = "by " - - # Genres - genres = '' - if 'tags' in book: - _soup = BeautifulSoup('') - genresTag = Tag(_soup,'p') - gtc = 0 - for (i, tag) in enumerate(sorted(book.get('tags', []))): - aTag = Tag(_soup,'a') - if self.opts.generate_genres: - aTag['href'] = "Genre_%s.html" % re.sub("\W","",tag.lower()) - aTag.insert(0,escape(NavigableString(tag))) - genresTag.insert(gtc, aTag) - gtc += 1 - if i < len(book['tags'])-1: - genresTag.insert(gtc, NavigableString(' · ')) - gtc += 1 - genres = genresTag.renderContents() - - # Formats - formats = [] - if 'formats' in book: - for format in sorted(book['formats']): - formats.append(format.rpartition('.')[2].upper()) - formats = ' · '.join(formats) - - # Date of publication - if book['date']: - pubdate = book['date'] - pubmonth, pubyear = pubdate.split() - else: - pubdate = pubyear = pubmonth = '' - - # Thumb - _soup = BeautifulSoup('<html>',selfClosingTags=['img']) - thumb = Tag(_soup,"img") - if 'cover' in book and book['cover']: - thumb['src'] = "../images/thumbnail_%d.jpg" % int(book['id']) - else: - thumb['src'] = "../images/thumbnail_default.jpg" - thumb['alt'] = "cover thumbnail" - - # Publisher - publisher = ' ' - if 'publisher' in book: - publisher = book['publisher'] - - # Rating - stars = int(book['rating']) / 2 - rating = '' - if stars: - star_string = self.FULL_RATING_SYMBOL * stars - empty_stars = self.EMPTY_RATING_SYMBOL * (5 - stars) - rating = '%s%s <br/>' % (star_string,empty_stars) - - # Notes - note_source = '' - note_content = '' - if 'notes' in book: - note_source = book['notes']['source'] - note_content = book['notes']['content'] - - # Comments - comments = '' - if 'description' in book and book['description'] > '': - comments = book['description'] - - - # >>>> Populate the template <<<< - soup = generate_html() - - - # >>>> Post-process the template <<<< - body = soup.find('body') - btc = 0 - # Insert the title anchor for inbound links - aTag = Tag(soup, "a") - aTag['name'] = "book%d" % int(book['id']) - body.insert(btc, aTag) - btc += 1 - - # Insert the link to the series or remove <a class="series"> - aTag = body.find('a', attrs={'class':'series_id'}) - if aTag: - if book['series']: - if self.opts.generate_series: - aTag['href'] = "%s.html#%s_series" % ('BySeries', - re.sub('\W','',book['series']).lower()) - else: - aTag.extract() - - # Insert the author link - aTag = body.find('a', attrs={'class':'author'}) - if self.opts.generate_authors and aTag: - aTag['href'] = "%s.html#%s" % ("ByAlphaAuthor", - self.generateAuthorAnchor(book['author'])) - - if publisher == ' ': - publisherTag = body.find('td', attrs={'class':'publisher'}) - if publisherTag: - publisherTag.contents[0].replaceWith(' ') - - if not genres: - genresTag = body.find('p',attrs={'class':'genres'}) - if genresTag: - genresTag.extract() - - if not formats: - formatsTag = body.find('p',attrs={'class':'formats'}) - if formatsTag: - formatsTag.extract() - - if note_content == '': - tdTag = body.find('td', attrs={'class':'notes'}) - if tdTag: - tdTag.contents[0].replaceWith(' ') - - emptyTags = body.findAll('td', attrs={'class':'empty'}) - for mt in emptyTags: - newEmptyTag = Tag(BeautifulSoup(),'td') - newEmptyTag.insert(0,NavigableString(' ')) - mt.replaceWith(newEmptyTag) - - if False: - print soup.prettify() - return soup - - def generateHTMLEmptyHeader(self, title): - header = ''' - <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd"> - <html xmlns="http://www.w3.org/1999/xhtml" xmlns:calibre="http://calibre.kovidgoyal.net/2009/metadata"> - <head> - <meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /> - <link rel="stylesheet" type="text/css" href="stylesheet.css" media="screen" /> - <title> - - - - - ''' - # Insert the supplied title - soup = BeautifulSoup(header) - titleTag = soup.find('title') - titleTag.insert(0,NavigableString(title)) - return soup - - def generateHTMLGenreHeader(self, title): - header = ''' - - - - - - - - -

-
- - - ''' - # Insert the supplied title - soup = BeautifulSoup(header) - titleTag = soup.find('title') - titleTag.insert(0,escape(NavigableString(title))) - return soup - - def generateMastheadImage(self, out_path): - from calibre.ebooks.conversion.config import load_defaults - from calibre.utils.fonts import fontconfig - font_path = default_font = P('fonts/liberation/LiberationSerif-Bold.ttf') - recs = load_defaults('mobi_output') - masthead_font_family = recs.get('masthead_font', 'Default') - - if masthead_font_family != 'Default': - masthead_font = fontconfig.files_for_family(masthead_font_family) - # Assume 'normal' always in dict, else use default - # {'normal': (path_to_font, friendly name)} - if 'normal' in masthead_font: - font_path = masthead_font['normal'][0] - - if not font_path or not os.access(font_path, os.R_OK): - font_path = default_font - - MI_WIDTH = 600 - MI_HEIGHT = 60 - - try: - from PIL import Image, ImageDraw, ImageFont - Image, ImageDraw, ImageFont - except ImportError: - import Image, ImageDraw, ImageFont - - img = Image.new('RGB', (MI_WIDTH, MI_HEIGHT), 'white') - draw = ImageDraw.Draw(img) - try: - font = ImageFont.truetype(font_path, 48) - except: - self.opts.log.error(" Failed to load user-specifed font '%s'" % font_path) - font = ImageFont.truetype(default_font, 48) - text = self.title.encode('utf-8') - width, height = draw.textsize(text, font=font) - left = max(int((MI_WIDTH - width)/2.), 0) - top = max(int((MI_HEIGHT - height)/2.), 0) - draw.text((left, top), text, fill=(0,0,0), font=font) - img.save(open(out_path, 'wb'), 'GIF') - - def generateRatingString(self, book): - rating = '' - try: - if 'rating' in book: - stars = int(book['rating']) / 2 - if stars: - star_string = self.FULL_RATING_SYMBOL * stars - empty_stars = self.EMPTY_RATING_SYMBOL * (5 - stars) - rating = '%s%s' % (star_string,empty_stars) - except: - # Rating could be None - pass - return rating - - def generateShortDescription(self, description, dest=None): - # Truncate the description, on word boundaries if necessary - # Possible destinations: - # description NCX summary - # title NCX title - # author NCX author - - def shortDescription(description, limit): - short_description = "" - words = description.split() - for word in words: - short_description += word + " " - if len(short_description) > limit: - short_description += "..." - return short_description - - if not description: - return None - - if dest == 'title': - # No truncation for titles, let the device deal with it - return description - elif dest == 'author': - if self.authorClip and len(description) < self.authorClip: - return description - else: - return shortDescription(description, self.authorClip) - elif dest == 'description': - if self.descriptionClip and len(description) < self.descriptionClip: - return description - else: - return shortDescription(description, self.descriptionClip) - else: - print " returning description with unspecified destination '%s'" % description - raise RuntimeError - - def generateSortTitle(self, title): - ''' - Generate a string suitable for sorting from the title - Ignore leading stop words - Optionally convert leading numbers to strings - ''' - from calibre.ebooks.metadata import title_sort - - # Strip stop words - title_words = title_sort(title).split() - translated = [] - - for (i,word) in enumerate(title_words): - # Leading numbers optionally translated to text equivalent - # Capitalize leading sort word - if i==0: - # *** Keep this code in case we need to restore numbers_as_text *** - if False: - #if self.opts.numbers_as_text and re.match('[0-9]+',word[0]): - translated.append(EPUB_MOBI.NumberToText(word).text.capitalize()) - else: - if re.match('[0-9]+',word[0]): - word = word.replace(',','') - suffix = re.search('[\D]', word) - if suffix: - word = '%10.0f%s' % (float(word[:suffix.start()]),word[suffix.start():]) - else: - word = '%10.0f' % (float(word)) - - # If leading char > 'A', insert symbol as leading forcing lower sort - # '/' sorts below numbers, g - if self.letter_or_symbol(word[0]) != word[0]: - if word[0] > 'A' or (ord('9') < ord(word[0]) < ord('A')) : - translated.append('/') - translated.append(capitalize(word)) - - else: - if re.search('[0-9]+',word[0]): - word = word.replace(',','') - suffix = re.search('[\D]', word) - if suffix: - word = '%10.0f%s' % (float(word[:suffix.start()]),word[suffix.start():]) - else: - word = '%10.0f' % (float(word)) - translated.append(word) - return ' '.join(translated) - - def generateThumbnail(self, title, image_dir, thumb_file): - ''' - Thumbs are cached with the full cover's crc. If the crc doesn't - match, the cover has been changed since the thumb was cached and needs - to be replaced. - ''' - - def open_archive(mode='r'): - try: - return ZipFile(self.__archive_path, mode=mode) - except: - # Happens on windows if the file is opened by another - # process - pass - - # Generate crc for current cover - #self.opts.log.info(" generateThumbnail():") - with open(title['cover'], 'rb') as f: - data = f.read() - cover_crc = hex(zlib.crc32(data)) - - # Test cache for uuid - zf = open_archive() - if zf is not None: - with zf: - try: - zf.getinfo(title['uuid']+cover_crc) - except: - pass - else: - # uuid found in cache with matching crc - thumb_data = zf.read(title['uuid']+cover_crc) - with open(os.path.join(image_dir, thumb_file), 'wb') as f: - f.write(thumb_data) - return - - - # Save thumb for catalog - thumb_data = thumbnail(data, - width=self.thumbWidth, height=self.thumbHeight)[-1] - with open(os.path.join(image_dir, thumb_file), 'wb') as f: - f.write(thumb_data) - - # Save thumb to archive - if zf is not None: # Ensure that the read succeeded - # If we failed to open the zip file for reading, - # we dont know if it contained the thumb or not - zf = open_archive('a') - if zf is not None: - with zf: - zf.writestr(title['uuid']+cover_crc, thumb_data) - - def getFriendlyGenreTag(self, genre): - # Find the first instance of friendly_tag matching genre - for friendly_tag in self.genre_tags_dict: - if self.genre_tags_dict[friendly_tag] == genre: - return friendly_tag - - def getMarkerTags(self): - ''' Return a list of special marker tags to be excluded from genre list ''' - markerTags = [] - markerTags.extend(self.opts.exclude_tags.split(',')) - return markerTags - - def letter_or_symbol(self,char): - if not re.search('[a-zA-Z]',char): - return 'Symbols' - else: - return char - - def markdownComments(self, comments): - ''' - Convert random comment text to normalized, xml-legal block of

s - 'plain text' returns as -

plain text

- - 'plain text with minimal markup' returns as -

plain text with minimal markup

- - '

pre-formatted text

returns untouched - - 'A line of text\n\nFollowed by a line of text' returns as -

A line of text

-

Followed by a line of text

- - 'A line of text.\nA second line of text.\rA third line of text' returns as -

A line of text.
A second line of text.
A third line of text.

- - '...end of a paragraph.Somehow the break was lost...' returns as -

...end of a paragraph.

-

Somehow the break was lost...

- - Deprecated HTML returns as HTML via BeautifulSoup() - - ''' - # Hackish - ignoring sentences ending or beginning in numbers to avoid - # confusion with decimal points. - - # Explode lost CRs to \n\n - for lost_cr in re.finditer('([a-z])([\.\?!])([A-Z])',comments): - comments = comments.replace(lost_cr.group(), - '%s%s\n\n%s' % (lost_cr.group(1), - lost_cr.group(2), - lost_cr.group(3))) - # Extract pre-built elements - annotations, etc. - if not isinstance(comments, unicode): - comments = comments.decode('utf-8', 'replace') - soup = BeautifulSoup(comments) - elems = soup.findAll('div') - for elem in elems: - elem.extract() - - # Reconstruct comments w/o
s - comments = soup.renderContents(None) - - # Convert \n\n to

s - if re.search('\n\n', comments): - soup = BeautifulSoup() - split_ps = comments.split(u'\n\n') - tsc = 0 - for p in split_ps: - pTag = Tag(soup,'p') - pTag.insert(0,p) - soup.insert(tsc,pTag) - tsc += 1 - comments = soup.renderContents(None) - - # Convert solo returns to
- comments = re.sub('[\r\n]','
', comments) - - # Convert two hypens to emdash - comments = re.sub('--','—',comments) - soup = BeautifulSoup(comments) - result = BeautifulSoup() - rtc = 0 - open_pTag = False - - all_tokens = list(soup.contents) - for token in all_tokens: - if type(token) is NavigableString: - if not open_pTag: - pTag = Tag(result,'p') - open_pTag = True - ptc = 0 - pTag.insert(ptc,prepare_string_for_xml(token)) - ptc += 1 - - elif token.name in ['br','b','i','em']: - if not open_pTag: - pTag = Tag(result,'p') - open_pTag = True - ptc = 0 - pTag.insert(ptc, token) - ptc += 1 - - else: - if open_pTag: - result.insert(rtc, pTag) - rtc += 1 - open_pTag = False - ptc = 0 - # Clean up NavigableStrings for xml - sub_tokens = list(token.contents) - for sub_token in sub_tokens: - if type(sub_token) is NavigableString: - sub_token.replaceWith(prepare_string_for_xml(sub_token)) - result.insert(rtc, token) - rtc += 1 - - if open_pTag: - result.insert(rtc, pTag) - rtc += 1 - - paras = result.findAll('p') - for p in paras: - p['class'] = 'description' - - # Add back

elems initially removed - for elem in elems: - result.insert(rtc,elem) - rtc += 1 - - return result.renderContents(encoding=None) - - def mergeComments(self, record): - ''' - merge ['description'] with custom field contents to be displayed in Descriptions - ''' - merged = '' - if record['description']: - addendum = self.__db.get_field(record['id'], - self.__merge_comments['field'], - index_is_id=True) - if addendum is None: - addendum = '' - include_hr = eval(self.__merge_comments['hr']) - if self.__merge_comments['position'] == 'before': - merged = addendum - if include_hr: - merged += '
' - else: - merged += '\n' - merged += record['description'] - else: - merged = record['description'] - if include_hr: - merged += '
' - else: - merged += '\n' - merged += addendum - else: - # Return the custom field contents - merged = self.__db.get_field(record['id'], - self.__merge_comments['field'], - index_is_id=True) - - return merged - - def processExclusions(self, data_set): - ''' - Remove excluded entries - ''' - field, pat = self.opts.exclude_book_marker.split(':') - if pat == '': - return data_set - filtered_data_set = [] - for record in data_set: - field_contents = self.__db.get_field(record['id'], - field, - index_is_id=True) - if field_contents: - if re.search(pat, unicode(field_contents), - re.IGNORECASE) is not None: - continue - filtered_data_set.append(record) - - return filtered_data_set - - def processSpecialTags(self, tags, this_title, opts): - tag_list = [] - for tag in tags: - tag = self.convertHTMLEntities(tag) - if re.search(opts.exclude_genre, tag): - continue - elif self.__read_book_marker['field'] == 'tag' and \ - tag == self.__read_book_marker['pattern']: - # remove 'read' tag - continue - else: - tag_list.append(tag) - return tag_list - - def updateProgressFullStep(self, description): - self.currentStep += 1 - self.progressString = description - self.progressInt = float((self.currentStep-1)/self.totalSteps) - self.reporter(self.progressInt, self.progressString) - if self.opts.cli_environment: - self.opts.log(u"%3.0f%% %s" % (self.progressInt*100, self.progressString)) - - def updateProgressMicroStep(self, description, micro_step_pct): - step_range = 100/self.totalSteps - self.progressString = description - coarse_progress = float((self.currentStep-1)/self.totalSteps) - fine_progress = float((micro_step_pct*step_range)/100) - self.progressInt = coarse_progress + fine_progress - self.reporter(self.progressInt, self.progressString) - - def run(self, path_to_output, opts, db, notification=DummyReporter()): - opts.log = log - opts.fmt = self.fmt = path_to_output.rpartition('.')[2] - - # Add local options - opts.creator = '%s, %s %s, %s' % (strftime('%A'), strftime('%B'), strftime('%d').lstrip('0'), strftime('%Y')) - opts.creator_sort_as = '%s %s' % ('calibre', strftime('%Y-%m-%d')) - opts.connected_kindle = False - - # Finalize output_profile - op = opts.output_profile - if op is None: - op = 'default' - - if opts.connected_device['name'] and 'kindle' in opts.connected_device['name'].lower(): - opts.connected_kindle = True - if opts.connected_device['serial'] and \ - opts.connected_device['serial'][:4] in ['B004','B005']: - op = "kindle_dx" - else: - op = "kindle" - opts.descriptionClip = 380 if op.endswith('dx') or 'kindle' not in op else 100 - opts.authorClip = 100 if op.endswith('dx') or 'kindle' not in op else 60 - opts.output_profile = op - - opts.basename = "Catalog" - opts.cli_environment = not hasattr(opts,'sync') - - # Hard-wired to always sort descriptions by author, with series after non-series - opts.sort_descriptions_by_author = True - - build_log = [] - - build_log.append(u"%s(): Generating %s %sin %s environment" % - (self.name,self.fmt,'for %s ' % opts.output_profile if opts.output_profile else '', - 'CLI' if opts.cli_environment else 'GUI')) - - # If exclude_genre is blank, assume user wants all genre tags included - if opts.exclude_genre.strip() == '': - opts.exclude_genre = '\[^.\]' - build_log.append(" converting empty exclude_genre to '\[^.\]'") - - if opts.connected_device['is_device_connected'] and \ - opts.connected_device['kind'] == 'device': - if opts.connected_device['serial']: - build_log.append(u" connected_device: '%s' #%s%s " % \ - (opts.connected_device['name'], - opts.connected_device['serial'][0:4], - 'x' * (len(opts.connected_device['serial']) - 4))) - for storage in opts.connected_device['storage']: - if storage: - build_log.append(u" mount point: %s" % storage) - else: - build_log.append(u" connected_device: '%s'" % opts.connected_device['name']) - try: - for storage in opts.connected_device['storage']: - if storage: - build_log.append(u" mount point: %s" % storage) - except: - build_log.append(u" (no mount points)") - else: - build_log.append(u" connected_device: '%s'" % opts.connected_device['name']) - - opts_dict = vars(opts) - if opts_dict['ids']: - build_log.append(" book count: %d" % len(opts_dict['ids'])) - - sections_list = [] - if opts.generate_authors: - sections_list.append('Authors') - if opts.generate_titles: - sections_list.append('Titles') - if opts.generate_series: - sections_list.append('Series') - if opts.generate_genres: - sections_list.append('Genres') - if opts.generate_recently_added: - sections_list.append('Recently Added') - if opts.generate_descriptions: - sections_list.append('Descriptions') - - if not sections_list: - if opts.cli_environment: - opts.log.warn('*** No Section switches specified, enabling all Sections ***') - opts.generate_authors = True - opts.generate_titles = True - opts.generate_series = True - opts.generate_genres = True - opts.generate_recently_added = True - opts.generate_descriptions = True - sections_list = ['Authors','Titles','Series','Genres','Recently Added','Descriptions'] - else: - opts.log.warn('\n*** No enabled Sections, terminating catalog generation ***') - return ["No Included Sections","No enabled Sections.\nCheck E-book options tab\n'Included sections'\n"] - if opts.fmt == 'mobi' and sections_list == ['Descriptions']: - warning = _("\n*** Adding 'By Authors' Section required for MOBI output ***") - opts.log.warn(warning) - sections_list.insert(0,'Authors') - opts.generate_authors = True - - opts.log(u" Sections: %s" % ', '.join(sections_list)) - opts.section_list = sections_list - - # Limit thumb_width to 1.0" - 2.0" - try: - if float(opts.thumb_width) < float(self.THUMB_SMALLEST): - log.warning("coercing thumb_width from '%s' to '%s'" % (opts.thumb_width,self.THUMB_SMALLEST)) - opts.thumb_width = self.THUMB_SMALLEST - if float(opts.thumb_width) > float(self.THUMB_LARGEST): - log.warning("coercing thumb_width from '%s' to '%s'" % (opts.thumb_width,self.THUMB_LARGEST)) - opts.thumb_width = self.THUMB_LARGEST - opts.thumb_width = "%.2f" % float(opts.thumb_width) - except: - log.error("coercing thumb_width from '%s' to '%s'" % (opts.thumb_width,self.THUMB_SMALLEST)) - opts.thumb_width = "1.0" - - # Display opts - keys = opts_dict.keys() - keys.sort() - build_log.append(" opts:") - for key in keys: - if key in ['catalog_title','authorClip','connected_kindle','descriptionClip', - 'exclude_book_marker','exclude_genre','exclude_tags', - 'header_note_source_field','merge_comments', - 'output_profile','read_book_marker', - 'search_text','sort_by','sort_descriptions_by_author','sync', - 'thumb_width','wishlist_tag']: - build_log.append(" %s: %s" % (key, repr(opts_dict[key]))) - - if opts.verbose: - log('\n'.join(line for line in build_log)) - - self.opts = opts - - # Launch the Catalog builder - catalog = self.CatalogBuilder(db, opts, self, report_progress=notification) - - if opts.verbose: - log.info(" Begin catalog source generation") - catalog.createDirectoryStructure() - catalog.copyResources() - catalog.calculateThumbnailSize() - catalog_source_built = catalog.buildSources() - - if opts.verbose: - if catalog_source_built: - log.info(" Completed catalog source generation\n") - else: - log.error(" *** Terminated catalog generation, check log for details ***") - - if catalog_source_built: - recommendations = [] - recommendations.append(('remove_fake_margins', False, - OptionRecommendation.HIGH)) - if DEBUG: - recommendations.append(('comments', '\n'.join(line for line in build_log), - OptionRecommendation.HIGH)) - else: - recommendations.append(('comments', '', OptionRecommendation.HIGH)) - - dp = getattr(opts, 'debug_pipeline', None) - if dp is not None: - recommendations.append(('debug_pipeline', dp, - OptionRecommendation.HIGH)) - - if opts.fmt == 'mobi' and opts.output_profile and opts.output_profile.startswith("kindle"): - recommendations.append(('output_profile', opts.output_profile, - OptionRecommendation.HIGH)) - recommendations.append(('no_inline_toc', True, - OptionRecommendation.HIGH)) - recommendations.append(('book_producer',opts.output_profile, - OptionRecommendation.HIGH)) - - # If cover exists, use it - cpath = None - try: - search_text = 'title:"%s" author:%s' % ( - opts.catalog_title.replace('"', '\\"'), 'calibre') - matches = db.search(search_text, return_matches=True) - if matches: - cpath = db.cover(matches[0], index_is_id=True, as_path=True) - if cpath and os.path.exists(cpath): - recommendations.append(('cover', cpath, - OptionRecommendation.HIGH)) - except: - pass - - # Run ebook-convert - from calibre.ebooks.conversion.plumber import Plumber - plumber = Plumber(os.path.join(catalog.catalogPath, - opts.basename + '.opf'), path_to_output, log, report_progress=notification, - abort_after_input_dump=False) - plumber.merge_ui_recommendations(recommendations) - plumber.run() - - try: - os.remove(cpath) - except: - pass - - # returns to gui2.actions.catalog:catalog_generated() - return catalog.error -# }}} diff --git a/src/calibre/library/catalogs/__init__.py b/src/calibre/library/catalogs/__init__.py new file mode 100644 index 0000000000..a53fffb344 --- /dev/null +++ b/src/calibre/library/catalogs/__init__.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +__license__ = 'GPL v3' +__copyright__ = '2012, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + + +FIELDS = ['all', 'title', 'title_sort', 'author_sort', 'authors', 'comments', + 'cover', 'formats','id', 'isbn', 'ondevice', 'pubdate', 'publisher', + 'rating', 'series_index', 'series', 'size', 'tags', 'timestamp', + 'uuid', 'languages'] + +#Allowed fields for template +TEMPLATE_ALLOWED_FIELDS = [ 'author_sort', 'authors', 'id', 'isbn', 'pubdate', 'title_sort', + 'publisher', 'series_index', 'series', 'tags', 'timestamp', 'title', 'uuid' ] + + diff --git a/src/calibre/library/catalogs/bibtex.py b/src/calibre/library/catalogs/bibtex.py new file mode 100644 index 0000000000..1815940931 --- /dev/null +++ b/src/calibre/library/catalogs/bibtex.py @@ -0,0 +1,396 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai + +__license__ = 'GPL v3' +__copyright__ = '2012, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + +import re, codecs +from collections import namedtuple +from types import StringType, UnicodeType + +from calibre import (strftime) +from calibre.customize import CatalogPlugin +from calibre.library.catalogs import FIELDS, TEMPLATE_ALLOWED_FIELDS +from calibre.utils.logging import default_log as log +from calibre.customize.conversion import DummyReporter +from calibre.constants import preferred_encoding + + +class BIBTEX(CatalogPlugin): + 'BIBTEX catalog generator' + + Option = namedtuple('Option', 'option, default, dest, action, help') + + name = 'Catalog_BIBTEX' + description = 'BIBTEX catalog generator' + supported_platforms = ['windows', 'osx', 'linux'] + author = 'Sengian' + version = (1, 0, 0) + file_types = set(['bib']) + + cli_options = [ + Option('--fields', + default = 'all', + dest = 'fields', + action = None, + help = _('The fields to output when cataloging books in the ' + 'database. Should be a comma-separated list of fields.\n' + 'Available fields: %(fields)s.\n' + 'plus user-created custom fields.\n' + 'Example: %(opt)s=title,authors,tags\n' + "Default: '%%default'\n" + "Applies to: BIBTEX output format")%dict( + fields=', '.join(FIELDS), opt='--fields')), + + Option('--sort-by', + default = 'id', + dest = 'sort_by', + action = None, + help = _('Output field to sort on.\n' + 'Available fields: author_sort, id, rating, size, timestamp, title.\n' + "Default: '%default'\n" + "Applies to: BIBTEX output format")), + + Option('--create-citation', + default = 'True', + dest = 'impcit', + action = None, + help = _('Create a citation for BibTeX entries.\n' + 'Boolean value: True, False\n' + "Default: '%default'\n" + "Applies to: BIBTEX output format")), + + Option('--add-files-path', + default = 'True', + dest = 'addfiles', + action = None, + help = _('Create a file entry if formats is selected for BibTeX entries.\n' + 'Boolean value: True, False\n' + "Default: '%default'\n" + "Applies to: BIBTEX output format")), + + Option('--citation-template', + default = '{authors}{id}', + dest = 'bib_cit', + action = None, + help = _('The template for citation creation from database fields.\n' + 'Should be a template with {} enclosed fields.\n' + 'Available fields: %s.\n' + "Default: '%%default'\n" + "Applies to: BIBTEX output format")%', '.join(TEMPLATE_ALLOWED_FIELDS)), + + Option('--choose-encoding', + default = 'utf8', + dest = 'bibfile_enc', + action = None, + help = _('BibTeX file encoding output.\n' + 'Available types: utf8, cp1252, ascii.\n' + "Default: '%default'\n" + "Applies to: BIBTEX output format")), + + Option('--choose-encoding-configuration', + default = 'strict', + dest = 'bibfile_enctag', + action = None, + help = _('BibTeX file encoding flag.\n' + 'Available types: strict, replace, ignore, backslashreplace.\n' + "Default: '%default'\n" + "Applies to: BIBTEX output format")), + + Option('--entry-type', + default = 'book', + dest = 'bib_entry', + action = None, + help = _('Entry type for BibTeX catalog.\n' + 'Available types: book, misc, mixed.\n' + "Default: '%default'\n" + "Applies to: BIBTEX output format"))] + + def run(self, path_to_output, opts, db, notification=DummyReporter()): + from calibre.utils.date import isoformat + from calibre.utils.html2text import html2text + from calibre.utils.bibtex import BibTeX + from calibre.library.save_to_disk import preprocess_template + from calibre.utils.date import now as nowf + + def create_bibtex_entry(entry, fields, mode, template_citation, + bibtexdict, db, citation_bibtex=True, calibre_files=True): + + #Bibtex doesn't like UTF-8 but keep unicode until writing + #Define starting chain or if book valid strict and not book return a Fail string + + bibtex_entry = [] + if mode != "misc" and check_entry_book_valid(entry) : + bibtex_entry.append(u'@book{') + elif mode != "book" : + bibtex_entry.append(u'@misc{') + else : + #case strict book + return '' + + if citation_bibtex : + # Citation tag + bibtex_entry.append(make_bibtex_citation(entry, template_citation, + bibtexdict)) + bibtex_entry = [u' '.join(bibtex_entry)] + + for field in fields: + if field.startswith('#'): + item = db.get_field(entry['id'],field,index_is_id=True) + if isinstance(item, (bool, float, int)): + item = repr(item) + elif field == 'title_sort': + item = entry['sort'] + else: + item = entry[field] + + #check if the field should be included (none or empty) + if item is None: + continue + try: + if len(item) == 0 : + continue + except TypeError: + pass + + if field == 'authors' : + bibtex_entry.append(u'author = "%s"' % bibtexdict.bibtex_author_format(item)) + + elif field == 'id' : + bibtex_entry.append(u'calibreid = "%s"' % int(item)) + + elif field == 'rating' : + bibtex_entry.append(u'rating = "%s"' % int(item)) + + elif field == 'size' : + bibtex_entry.append(u'%s = "%s octets"' % (field, int(item))) + + elif field == 'tags' : + #A list to flatten + bibtex_entry.append(u'tags = "%s"' % bibtexdict.utf8ToBibtex(u', '.join(item))) + + elif field == 'comments' : + #\n removal + item = item.replace(u'\r\n',u' ') + item = item.replace(u'\n',u' ') + #html to text + try: + item = html2text(item) + except: + log.warn("Failed to convert comments to text") + bibtex_entry.append(u'note = "%s"' % bibtexdict.utf8ToBibtex(item)) + + elif field == 'isbn' : + # Could be 9, 10 or 13 digits + bibtex_entry.append(u'isbn = "%s"' % re.sub(u'[0-9xX]', u'', item)) + + elif field == 'formats' : + #Add file path if format is selected + formats = [format.rpartition('.')[2].lower() for format in item] + bibtex_entry.append(u'formats = "%s"' % u', '.join(formats)) + if calibre_files: + files = [u':%s:%s' % (format, format.rpartition('.')[2].upper())\ + for format in item] + bibtex_entry.append(u'file = "%s"' % u', '.join(files)) + + elif field == 'series_index' : + bibtex_entry.append(u'volume = "%s"' % int(item)) + + elif field == 'timestamp' : + bibtex_entry.append(u'timestamp = "%s"' % isoformat(item).partition('T')[0]) + + elif field == 'pubdate' : + bibtex_entry.append(u'year = "%s"' % item.year) + bibtex_entry.append(u'month = "%s"' % bibtexdict.utf8ToBibtex(strftime("%b", item))) + + elif field.startswith('#') and isinstance(item, basestring): + bibtex_entry.append(u'custom_%s = "%s"' % (field[1:], + bibtexdict.utf8ToBibtex(item))) + + elif isinstance(item, basestring): + # elif field in ['title', 'publisher', 'cover', 'uuid', 'ondevice', + # 'author_sort', 'series', 'title_sort'] : + bibtex_entry.append(u'%s = "%s"' % (field, bibtexdict.utf8ToBibtex(item))) + + bibtex_entry = u',\n '.join(bibtex_entry) + bibtex_entry += u' }\n\n' + + return bibtex_entry + + def check_entry_book_valid(entry): + #Check that the required fields are ok for a book entry + for field in ['title', 'authors', 'publisher'] : + if entry[field] is None or len(entry[field]) == 0 : + return False + if entry['pubdate'] is None : + return False + else : + return True + + def make_bibtex_citation(entry, template_citation, bibtexclass): + + #define a function to replace the template entry by its value + def tpl_replace(objtplname) : + + tpl_field = re.sub(u'[\{\}]', u'', objtplname.group()) + + if tpl_field in TEMPLATE_ALLOWED_FIELDS : + if tpl_field in ['pubdate', 'timestamp'] : + tpl_field = isoformat(entry[tpl_field]).partition('T')[0] + elif tpl_field in ['tags', 'authors'] : + tpl_field =entry[tpl_field][0] + elif tpl_field in ['id', 'series_index'] : + tpl_field = str(entry[tpl_field]) + else : + tpl_field = entry[tpl_field] + return tpl_field + else: + return u'' + + if len(template_citation) >0 : + tpl_citation = bibtexclass.utf8ToBibtex( + bibtexclass.ValidateCitationKey(re.sub(u'\{[^{}]*\}', + tpl_replace, template_citation))) + + if len(tpl_citation) >0 : + return tpl_citation + + if len(entry["isbn"]) > 0 : + template_citation = u'%s' % re.sub(u'[\D]',u'', entry["isbn"]) + + else : + template_citation = u'%s' % str(entry["id"]) + + return bibtexclass.ValidateCitationKey(template_citation) + + self.fmt = path_to_output.rpartition('.')[2] + self.notification = notification + + # Combobox options + bibfile_enc = ['utf8', 'cp1252', 'ascii'] + bibfile_enctag = ['strict', 'replace', 'ignore', 'backslashreplace'] + bib_entry = ['mixed', 'misc', 'book'] + + # Needed beacause CLI return str vs int by widget + try: + bibfile_enc = bibfile_enc[opts.bibfile_enc] + bibfile_enctag = bibfile_enctag[opts.bibfile_enctag] + bib_entry = bib_entry[opts.bib_entry] + except: + if opts.bibfile_enc in bibfile_enc : + bibfile_enc = opts.bibfile_enc + else : + log.warn("Incorrect --choose-encoding flag, revert to default") + bibfile_enc = bibfile_enc[0] + if opts.bibfile_enctag in bibfile_enctag : + bibfile_enctag = opts.bibfile_enctag + else : + log.warn("Incorrect --choose-encoding-configuration flag, revert to default") + bibfile_enctag = bibfile_enctag[0] + if opts.bib_entry in bib_entry : + bib_entry = opts.bib_entry + else : + log.warn("Incorrect --entry-type flag, revert to default") + bib_entry = bib_entry[0] + + if opts.verbose: + opts_dict = vars(opts) + log("%s(): Generating %s" % (self.name,self.fmt)) + if opts.connected_device['is_device_connected']: + log(" connected_device: %s" % opts.connected_device['name']) + if opts_dict['search_text']: + log(" --search='%s'" % opts_dict['search_text']) + + if opts_dict['ids']: + log(" Book count: %d" % len(opts_dict['ids'])) + if opts_dict['search_text']: + log(" (--search ignored when a subset of the database is specified)") + + if opts_dict['fields']: + if opts_dict['fields'] == 'all': + log(" Fields: %s" % ', '.join(FIELDS[1:])) + else: + log(" Fields: %s" % opts_dict['fields']) + + log(" Output file will be encoded in %s with %s flag" % (bibfile_enc, bibfile_enctag)) + + log(" BibTeX entry type is %s with a citation like '%s' flag" % (bib_entry, opts_dict['bib_cit'])) + + # If a list of ids are provided, don't use search_text + if opts.ids: + opts.search_text = None + + data = self.search_sort_db(db, opts) + + if not len(data): + log.error("\nNo matching database entries for search criteria '%s'" % opts.search_text) + + # Get the requested output fields as a list + fields = self.get_output_fields(db, opts) + + if not len(data): + log.error("\nNo matching database entries for search criteria '%s'" % opts.search_text) + + #Initialize BibTeX class + bibtexc = BibTeX() + + #Entries writing after Bibtex formating (or not) + if bibfile_enc != 'ascii' : + bibtexc.ascii_bibtex = False + else : + bibtexc.ascii_bibtex = True + + #Check citation choice and go to default in case of bad CLI + if isinstance(opts.impcit, (StringType, UnicodeType)) : + if opts.impcit == 'False' : + citation_bibtex= False + elif opts.impcit == 'True' : + citation_bibtex= True + else : + log.warn("Incorrect --create-citation, revert to default") + citation_bibtex= True + else : + citation_bibtex= opts.impcit + + #Check add file entry and go to default in case of bad CLI + if isinstance(opts.addfiles, (StringType, UnicodeType)) : + if opts.addfiles == 'False' : + addfiles_bibtex = False + elif opts.addfiles == 'True' : + addfiles_bibtex = True + else : + log.warn("Incorrect --add-files-path, revert to default") + addfiles_bibtex= True + else : + addfiles_bibtex = opts.addfiles + + #Preprocess for error and light correction + template_citation = preprocess_template(opts.bib_cit) + + #Open output and write entries + with codecs.open(path_to_output, 'w', bibfile_enc, bibfile_enctag)\ + as outfile: + #File header + nb_entries = len(data) + + #check in book strict if all is ok else throw a warning into log + if bib_entry == 'book' : + nb_books = len(filter(check_entry_book_valid, data)) + if nb_books < nb_entries : + log.warn("Only %d entries in %d are book compatible" % (nb_books, nb_entries)) + nb_entries = nb_books + + # If connected device, add 'On Device' values to data + if opts.connected_device['is_device_connected'] and 'ondevice' in fields: + for entry in data: + entry['ondevice'] = db.catalog_plugin_on_device_temp_mapping[entry['id']]['ondevice'] + + outfile.write(u'%%%Calibre catalog\n%%%{0} entries in catalog\n\n'.format(nb_entries)) + outfile.write(u'@preamble{"This catalog of %d entries was generated by calibre on %s"}\n\n' + % (nb_entries, nowf().strftime("%A, %d. %B %Y %H:%M").decode(preferred_encoding))) + + for entry in data: + outfile.write(create_bibtex_entry(entry, fields, bib_entry, template_citation, + bibtexc, db, citation_bibtex, addfiles_bibtex)) + diff --git a/src/calibre/library/catalogs/csv_xml.py b/src/calibre/library/catalogs/csv_xml.py new file mode 100644 index 0000000000..a09084889b --- /dev/null +++ b/src/calibre/library/catalogs/csv_xml.py @@ -0,0 +1,214 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai + +__license__ = 'GPL v3' +__copyright__ = '2012, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + +import re, codecs, os +from collections import namedtuple + +from calibre.customize import CatalogPlugin +from calibre.library.catalogs import FIELDS +from calibre.utils.logging import default_log as log +from calibre.customize.conversion import DummyReporter + +class CSV_XML(CatalogPlugin): + 'CSV/XML catalog generator' + + Option = namedtuple('Option', 'option, default, dest, action, help') + + name = 'Catalog_CSV_XML' + description = 'CSV/XML catalog generator' + supported_platforms = ['windows', 'osx', 'linux'] + author = 'Greg Riker' + version = (1, 0, 0) + file_types = set(['csv','xml']) + + cli_options = [ + Option('--fields', + default = 'all', + dest = 'fields', + action = None, + help = _('The fields to output when cataloging books in the ' + 'database. Should be a comma-separated list of fields.\n' + 'Available fields: %(fields)s,\n' + 'plus user-created custom fields.\n' + 'Example: %(opt)s=title,authors,tags\n' + "Default: '%%default'\n" + "Applies to: CSV, XML output formats")%dict( + fields=', '.join(FIELDS), opt='--fields')), + + Option('--sort-by', + default = 'id', + dest = 'sort_by', + action = None, + help = _('Output field to sort on.\n' + 'Available fields: author_sort, id, rating, size, timestamp, title_sort\n' + "Default: '%default'\n" + "Applies to: CSV, XML output formats"))] + + def run(self, path_to_output, opts, db, notification=DummyReporter()): + from calibre.utils.date import isoformat + from calibre.utils.html2text import html2text + from lxml import etree + + self.fmt = path_to_output.rpartition('.')[2] + self.notification = notification + + if opts.verbose: + opts_dict = vars(opts) + log("%s(): Generating %s" % (self.name,self.fmt.upper())) + if opts.connected_device['is_device_connected']: + log(" connected_device: %s" % opts.connected_device['name']) + if opts_dict['search_text']: + log(" --search='%s'" % opts_dict['search_text']) + + if opts_dict['ids']: + log(" Book count: %d" % len(opts_dict['ids'])) + if opts_dict['search_text']: + log(" (--search ignored when a subset of the database is specified)") + + if opts_dict['fields']: + if opts_dict['fields'] == 'all': + log(" Fields: %s" % ', '.join(FIELDS[1:])) + else: + log(" Fields: %s" % opts_dict['fields']) + + # If a list of ids are provided, don't use search_text + if opts.ids: + opts.search_text = None + + data = self.search_sort_db(db, opts) + + if not len(data): + log.error("\nNo matching database entries for search criteria '%s'" % opts.search_text) + #raise SystemExit(1) + + # Get the requested output fields as a list + fields = self.get_output_fields(db, opts) + + # If connected device, add 'On Device' values to data + if opts.connected_device['is_device_connected'] and 'ondevice' in fields: + for entry in data: + entry['ondevice'] = db.catalog_plugin_on_device_temp_mapping[entry['id']]['ondevice'] + + if self.fmt == 'csv': + outfile = codecs.open(path_to_output, 'w', 'utf8') + + # Write a UTF-8 BOM + outfile.write('\xef\xbb\xbf') + + # Output the field headers + outfile.write(u'%s\n' % u','.join(fields)) + + # Output the entry fields + for entry in data: + outstr = [] + for field in fields: + if field.startswith('#'): + item = db.get_field(entry['id'],field,index_is_id=True) + elif field == 'title_sort': + item = entry['sort'] + else: + item = entry[field] + + if item is None: + outstr.append('""') + continue + elif field == 'formats': + fmt_list = [] + for format in item: + fmt_list.append(format.rpartition('.')[2].lower()) + item = ', '.join(fmt_list) + elif field in ['authors','tags']: + item = ', '.join(item) + elif field == 'isbn': + # Could be 9, 10 or 13 digits + item = u'%s' % re.sub(r'[\D]', '', item) + elif field in ['pubdate', 'timestamp']: + item = isoformat(item) + elif field == 'comments': + item = item.replace(u'\r\n',u' ') + item = item.replace(u'\n',u' ') + + # Convert HTML to markdown text + if type(item) is unicode: + opening_tag = re.search('<(\w+)(\x20|>)',item) + if opening_tag: + closing_tag = re.search('<\/%s>$' % opening_tag.group(1), item) + if closing_tag: + item = html2text(item) + + outstr.append(u'"%s"' % unicode(item).replace('"','""')) + + outfile.write(u','.join(outstr) + u'\n') + outfile.close() + + elif self.fmt == 'xml': + from lxml.builder import E + + root = E.calibredb() + for r in data: + record = E.record() + root.append(record) + + for field in fields: + if field.startswith('#'): + val = db.get_field(r['id'],field,index_is_id=True) + if not isinstance(val, (str, unicode)): + val = unicode(val) + item = getattr(E, field.replace('#','_'))(val) + record.append(item) + + for field in ('id', 'uuid', 'publisher', 'rating', 'size', + 'isbn','ondevice'): + if field in fields: + val = r[field] + if not val: + continue + if not isinstance(val, (str, unicode)): + val = unicode(val) + item = getattr(E, field)(val) + record.append(item) + + if 'title' in fields: + title = E.title(r['title'], sort=r['sort']) + record.append(title) + + if 'authors' in fields: + aus = E.authors(sort=r['author_sort']) + for au in r['authors']: + aus.append(E.author(au)) + record.append(aus) + + for field in ('timestamp', 'pubdate'): + if field in fields: + record.append(getattr(E, field)(r[field].isoformat())) + + if 'tags' in fields and r['tags']: + tags = E.tags() + for tag in r['tags']: + tags.append(E.tag(tag)) + record.append(tags) + + if 'comments' in fields and r['comments']: + record.append(E.comments(r['comments'])) + + if 'series' in fields and r['series']: + record.append(E.series(r['series'], + index=str(r['series_index']))) + + if 'cover' in fields and r['cover']: + record.append(E.cover(r['cover'].replace(os.sep, '/'))) + + if 'formats' in fields and r['formats']: + fmt = E.formats() + for f in r['formats']: + fmt.append(E.format(f.replace(os.sep, '/'))) + record.append(fmt) + + with open(path_to_output, 'w') as f: + f.write(etree.tostring(root, encoding='utf-8', + xml_declaration=True, pretty_print=True)) + diff --git a/src/calibre/library/catalogs/epub_mobi.py b/src/calibre/library/catalogs/epub_mobi.py new file mode 100644 index 0000000000..97669f6ad8 --- /dev/null +++ b/src/calibre/library/catalogs/epub_mobi.py @@ -0,0 +1,365 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +__license__ = 'GPL v3' +__copyright__ = '2012, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + +import os +from collections import namedtuple + +from calibre import strftime +from calibre.constants import DEBUG +from calibre.customize import CatalogPlugin +from calibre.customize.conversion import OptionRecommendation, DummyReporter +from calibre.utils.logging import default_log as log + +Option = namedtuple('Option', 'option, default, dest, action, help') + +class EPUB_MOBI(CatalogPlugin): + 'ePub catalog generator' + + name = 'Catalog_EPUB_MOBI' + description = 'EPUB/MOBI catalog generator' + supported_platforms = ['windows', 'osx', 'linux'] + minimum_calibre_version = (0, 7, 40) + author = 'Greg Riker' + version = (1, 0, 0) + file_types = set(['epub','mobi']) + + THUMB_SMALLEST = "1.0" + THUMB_LARGEST = "2.0" + + cli_options = [Option('--catalog-title', # {{{ + default = 'My Books', + dest = 'catalog_title', + action = None, + help = _('Title of generated catalog used as title in metadata.\n' + "Default: '%default'\n" + "Applies to: ePub, MOBI output formats")), + Option('--debug-pipeline', + default=None, + dest='debug_pipeline', + action = None, + help=_("Save the output from different stages of the conversion " + "pipeline to the specified " + "directory. Useful if you are unsure at which stage " + "of the conversion process a bug is occurring.\n" + "Default: '%default'\n" + "Applies to: ePub, MOBI output formats")), + Option('--exclude-book-marker', + default=':', + dest='exclude_book_marker', + action = None, + help=_("field:pattern specifying custom field/contents indicating book should be excluded.\n" + "Default: '%default'\n" + "Applies to ePub, MOBI output formats")), + Option('--exclude-genre', + default='\[.+\]', + dest='exclude_genre', + action = None, + help=_("Regex describing tags to exclude as genres.\n" "Default: '%default' excludes bracketed tags, e.g. '[]'\n" + "Applies to: ePub, MOBI output formats")), + Option('--exclude-tags', + default=('~,'+_('Catalog')), + dest='exclude_tags', + action = None, + help=_("Comma-separated list of tag words indicating book should be excluded from output. " + "For example: 'skip' will match 'skip this book' and 'Skip will like this'. " + "Default: '%default'\n" + "Applies to: ePub, MOBI output formats")), + Option('--generate-authors', + default=False, + dest='generate_authors', + action = 'store_true', + help=_("Include 'Authors' section in catalog.\n" + "Default: '%default'\n" + "Applies to: ePub, MOBI output formats")), + Option('--generate-descriptions', + default=False, + dest='generate_descriptions', + action = 'store_true', + help=_("Include 'Descriptions' section in catalog.\n" + "Default: '%default'\n" + "Applies to: ePub, MOBI output formats")), + Option('--generate-genres', + default=False, + dest='generate_genres', + action = 'store_true', + help=_("Include 'Genres' section in catalog.\n" + "Default: '%default'\n" + "Applies to: ePub, MOBI output formats")), + Option('--generate-titles', + default=False, + dest='generate_titles', + action = 'store_true', + help=_("Include 'Titles' section in catalog.\n" + "Default: '%default'\n" + "Applies to: ePub, MOBI output formats")), + Option('--generate-series', + default=False, + dest='generate_series', + action = 'store_true', + help=_("Include 'Series' section in catalog.\n" + "Default: '%default'\n" + "Applies to: ePub, MOBI output formats")), + Option('--generate-recently-added', + default=False, + dest='generate_recently_added', + action = 'store_true', + help=_("Include 'Recently Added' section in catalog.\n" + "Default: '%default'\n" + "Applies to: ePub, MOBI output formats")), + Option('--header-note-source-field', + default='', + dest='header_note_source_field', + action = None, + help=_("Custom field containing note text to insert in Description header.\n" + "Default: '%default'\n" + "Applies to: ePub, MOBI output formats")), + Option('--merge-comments', + default='::', + dest='merge_comments', + action = None, + help=_(":[before|after]:[True|False] specifying:\n" + " Custom field containing notes to merge with Comments\n" + " [before|after] Placement of notes with respect to Comments\n" + " [True|False] - A horizontal rule is inserted between notes and Comments\n" + "Default: '%default'\n" + "Applies to ePub, MOBI output formats")), + Option('--output-profile', + default=None, + dest='output_profile', + action = None, + help=_("Specifies the output profile. In some cases, an output profile is required to optimize the catalog for the device. For example, 'kindle' or 'kindle_dx' creates a structured Table of Contents with Sections and Articles.\n" + "Default: '%default'\n" + "Applies to: ePub, MOBI output formats")), + Option('--read-book-marker', + default='tag:+', + dest='read_book_marker', + action = None, + help=_("field:pattern indicating book has been read.\n" "Default: '%default'\n" + "Applies to ePub, MOBI output formats")), + Option('--thumb-width', + default='1.0', + dest='thumb_width', + action = None, + help=_("Size hint (in inches) for book covers in catalog.\n" + "Range: 1.0 - 2.0\n" + "Default: '%default'\n" + "Applies to ePub, MOBI output formats")), + Option('--wishlist-tag', + default='Wishlist', + dest='wishlist_tag', + action = None, + help=_("Tag indicating book to be displayed as wishlist item.\n" "Default: '%default'\n" + "Applies to: ePub, MOBI output formats")), + ] + # }}} + + def run(self, path_to_output, opts, db, notification=DummyReporter()): + from calibre.library.catalogs.epub_mobi_builder import CatalogBuilder + opts.log = log + opts.fmt = self.fmt = path_to_output.rpartition('.')[2] + + # Add local options + opts.creator = '%s, %s %s, %s' % (strftime('%A'), strftime('%B'), strftime('%d').lstrip('0'), strftime('%Y')) + opts.creator_sort_as = '%s %s' % ('calibre', strftime('%Y-%m-%d')) + opts.connected_kindle = False + + # Finalize output_profile + op = opts.output_profile + if op is None: + op = 'default' + + if opts.connected_device['name'] and 'kindle' in opts.connected_device['name'].lower(): + opts.connected_kindle = True + if opts.connected_device['serial'] and \ + opts.connected_device['serial'][:4] in ['B004','B005']: + op = "kindle_dx" + else: + op = "kindle" + opts.descriptionClip = 380 if op.endswith('dx') or 'kindle' not in op else 100 + opts.authorClip = 100 if op.endswith('dx') or 'kindle' not in op else 60 + opts.output_profile = op + + opts.basename = "Catalog" + opts.cli_environment = not hasattr(opts,'sync') + + # Hard-wired to always sort descriptions by author, with series after non-series + opts.sort_descriptions_by_author = True + + build_log = [] + + build_log.append(u"%s(): Generating %s %sin %s environment" % + (self.name,self.fmt,'for %s ' % opts.output_profile if opts.output_profile else '', + 'CLI' if opts.cli_environment else 'GUI')) + + # If exclude_genre is blank, assume user wants all genre tags included + if opts.exclude_genre.strip() == '': + opts.exclude_genre = '\[^.\]' + build_log.append(" converting empty exclude_genre to '\[^.\]'") + + if opts.connected_device['is_device_connected'] and \ + opts.connected_device['kind'] == 'device': + if opts.connected_device['serial']: + build_log.append(u" connected_device: '%s' #%s%s " % \ + (opts.connected_device['name'], + opts.connected_device['serial'][0:4], + 'x' * (len(opts.connected_device['serial']) - 4))) + for storage in opts.connected_device['storage']: + if storage: + build_log.append(u" mount point: %s" % storage) + else: + build_log.append(u" connected_device: '%s'" % opts.connected_device['name']) + try: + for storage in opts.connected_device['storage']: + if storage: + build_log.append(u" mount point: %s" % storage) + except: + build_log.append(u" (no mount points)") + else: + build_log.append(u" connected_device: '%s'" % opts.connected_device['name']) + + opts_dict = vars(opts) + if opts_dict['ids']: + build_log.append(" book count: %d" % len(opts_dict['ids'])) + + sections_list = [] + if opts.generate_authors: + sections_list.append('Authors') + if opts.generate_titles: + sections_list.append('Titles') + if opts.generate_series: + sections_list.append('Series') + if opts.generate_genres: + sections_list.append('Genres') + if opts.generate_recently_added: + sections_list.append('Recently Added') + if opts.generate_descriptions: + sections_list.append('Descriptions') + + if not sections_list: + if opts.cli_environment: + opts.log.warn('*** No Section switches specified, enabling all Sections ***') + opts.generate_authors = True + opts.generate_titles = True + opts.generate_series = True + opts.generate_genres = True + opts.generate_recently_added = True + opts.generate_descriptions = True + sections_list = ['Authors','Titles','Series','Genres','Recently Added','Descriptions'] + else: + opts.log.warn('\n*** No enabled Sections, terminating catalog generation ***') + return ["No Included Sections","No enabled Sections.\nCheck E-book options tab\n'Included sections'\n"] + if opts.fmt == 'mobi' and sections_list == ['Descriptions']: + warning = _("\n*** Adding 'By Authors' Section required for MOBI output ***") + opts.log.warn(warning) + sections_list.insert(0,'Authors') + opts.generate_authors = True + + opts.log(u" Sections: %s" % ', '.join(sections_list)) + opts.section_list = sections_list + + # Limit thumb_width to 1.0" - 2.0" + try: + if float(opts.thumb_width) < float(self.THUMB_SMALLEST): + log.warning("coercing thumb_width from '%s' to '%s'" % (opts.thumb_width,self.THUMB_SMALLEST)) + opts.thumb_width = self.THUMB_SMALLEST + if float(opts.thumb_width) > float(self.THUMB_LARGEST): + log.warning("coercing thumb_width from '%s' to '%s'" % (opts.thumb_width,self.THUMB_LARGEST)) + opts.thumb_width = self.THUMB_LARGEST + opts.thumb_width = "%.2f" % float(opts.thumb_width) + except: + log.error("coercing thumb_width from '%s' to '%s'" % (opts.thumb_width,self.THUMB_SMALLEST)) + opts.thumb_width = "1.0" + + # Display opts + keys = opts_dict.keys() + keys.sort() + build_log.append(" opts:") + for key in keys: + if key in ['catalog_title','authorClip','connected_kindle','descriptionClip', + 'exclude_book_marker','exclude_genre','exclude_tags', + 'header_note_source_field','merge_comments', + 'output_profile','read_book_marker', + 'search_text','sort_by','sort_descriptions_by_author','sync', + 'thumb_width','wishlist_tag']: + build_log.append(" %s: %s" % (key, repr(opts_dict[key]))) + + if opts.verbose: + log('\n'.join(line for line in build_log)) + + self.opts = opts + + # Launch the Catalog builder + catalog = CatalogBuilder(db, opts, self, report_progress=notification) + + if opts.verbose: + log.info(" Begin catalog source generation") + catalog.createDirectoryStructure() + catalog.copyResources() + catalog.calculateThumbnailSize() + catalog_source_built = catalog.buildSources() + + if opts.verbose: + if catalog_source_built: + log.info(" Completed catalog source generation\n") + else: + log.error(" *** Terminated catalog generation, check log for details ***") + + if catalog_source_built: + recommendations = [] + recommendations.append(('remove_fake_margins', False, + OptionRecommendation.HIGH)) + if DEBUG: + recommendations.append(('comments', '\n'.join(line for line in build_log), + OptionRecommendation.HIGH)) + else: + recommendations.append(('comments', '', OptionRecommendation.HIGH)) + + dp = getattr(opts, 'debug_pipeline', None) + if dp is not None: + recommendations.append(('debug_pipeline', dp, + OptionRecommendation.HIGH)) + + if opts.fmt == 'mobi' and opts.output_profile and opts.output_profile.startswith("kindle"): + recommendations.append(('output_profile', opts.output_profile, + OptionRecommendation.HIGH)) + recommendations.append(('no_inline_toc', True, + OptionRecommendation.HIGH)) + recommendations.append(('book_producer',opts.output_profile, + OptionRecommendation.HIGH)) + + # If cover exists, use it + cpath = None + try: + search_text = 'title:"%s" author:%s' % ( + opts.catalog_title.replace('"', '\\"'), 'calibre') + matches = db.search(search_text, return_matches=True) + if matches: + cpath = db.cover(matches[0], index_is_id=True, as_path=True) + if cpath and os.path.exists(cpath): + recommendations.append(('cover', cpath, + OptionRecommendation.HIGH)) + except: + pass + + # Run ebook-convert + from calibre.ebooks.conversion.plumber import Plumber + plumber = Plumber(os.path.join(catalog.catalogPath, + opts.basename + '.opf'), path_to_output, log, report_progress=notification, + abort_after_input_dump=False) + plumber.merge_ui_recommendations(recommendations) + plumber.run() + + try: + os.remove(cpath) + except: + pass + + # returns to gui2.actions.catalog:catalog_generated() + return catalog.error + diff --git a/src/calibre/library/catalogs/epub_mobi_builder.py b/src/calibre/library/catalogs/epub_mobi_builder.py new file mode 100644 index 0000000000..1383379db2 --- /dev/null +++ b/src/calibre/library/catalogs/epub_mobi_builder.py @@ -0,0 +1,4040 @@ +# -*- coding: utf-8 -*- + +__license__ = 'GPL v3' +__copyright__ = '2010, Greg Riker' + +import datetime, htmlentitydefs, os, re, shutil, zlib +from copy import deepcopy +from xml.sax.saxutils import escape + +from calibre import (prepare_string_for_xml, strftime, force_unicode) +from calibre.customize.conversion import DummyReporter +from calibre.ebooks.BeautifulSoup import BeautifulSoup, BeautifulStoneSoup, Tag, NavigableString +from calibre.ebooks.chardet import substitute_entites +from calibre.ptempfile import PersistentTemporaryDirectory +from calibre.utils.config import config_dir +from calibre.utils.date import format_date, is_date_undefined, now as nowf +from calibre.utils.icu import capitalize +from calibre.utils.magick.draw import thumbnail +from calibre.utils.zipfile import ZipFile + + + +class CatalogBuilder(object): + ''' + Generates catalog source files from calibre database + + Flow of control: + gui2.actions.catalog:generate_catalog() + gui2.tools:generate_catalog() or library.cli:command_catalog() + called from gui2.convert.gui_conversion:gui_catalog() + catalog = Catalog(notification=Reporter()) + catalog.createDirectoryStructure() + catalog.copyResources() + catalog.buildSources() + Options managed in gui2.catalog.catalog_epub_mobi.py + ''' + + # A single number creates 'Last x days' only. + # Multiple numbers create 'Last x days', 'x to y days ago' ... + # e.g, [7,15,30,60], [30] + # [] = No date ranges added + DATE_RANGE=[30] + + # basename output file basename + # creator dc:creator in OPF metadata + # descriptionClip limits size of NCX descriptions (Kindle only) + # includeSources Used in processSpecialTags to skip tags like '[SPL]' + # notification Used to check for cancel, report progress + # stylesheet CSS stylesheet + # title dc:title in OPF metadata, NCX periodical + # verbosity level of diagnostic printout + + def __init__(self, db, opts, plugin, + report_progress=DummyReporter(), + stylesheet="content/stylesheet.css"): + self.__opts = opts + self.__authorClip = opts.authorClip + self.__authors = None + self.__basename = opts.basename + self.__bookmarked_books = None + self.__booksByAuthor = None + self.__booksByDateRead = None + self.__booksByTitle = None + self.__booksByTitle_noSeriesPrefix = None + self.__cache_dir = os.path.join(config_dir, 'caches', 'catalog') + self.__archive_path = os.path.join(self.__cache_dir, "thumbs.zip") + self.__catalogPath = PersistentTemporaryDirectory("_epub_mobi_catalog", prefix='') + self.__contentDir = os.path.join(self.catalogPath, "content") + self.__currentStep = 0.0 + self.__creator = opts.creator + self.__db = db + self.__descriptionClip = opts.descriptionClip + self.__error = [] + self.__generateForKindle = True if (self.opts.fmt == 'mobi' and \ + self.opts.output_profile and \ + self.opts.output_profile.startswith("kindle")) else False + self.__generateRecentlyRead = True if self.opts.generate_recently_added \ + and self.opts.connected_kindle \ + and self.generateForKindle \ + else False + self.__genres = None + self.genres = [] + self.__genre_tags_dict = None + self.__htmlFileList_1 = [] + self.__htmlFileList_2 = [] + self.__markerTags = self.getMarkerTags() + self.__ncxSoup = None + self.__output_profile = None + self.__playOrder = 1 + self.__plugin = plugin + self.__progressInt = 0.0 + self.__progressString = '' + f, _, p = opts.read_book_marker.partition(':') + self.__read_book_marker = {'field':f, 'pattern':p} + f, p, hr = self.opts.merge_comments.split(':') + self.__merge_comments = {'field':f, 'position':p, 'hr':hr} + self.__reporter = report_progress + self.__stylesheet = stylesheet + self.__thumbs = None + self.__thumbWidth = 0 + self.__thumbHeight = 0 + self.__title = opts.catalog_title + self.__totalSteps = 6.0 + self.__useSeriesPrefixInTitlesSection = False + self.__verbose = opts.verbose + + from calibre.customize.ui import output_profiles + for profile in output_profiles(): + if profile.short_name == self.opts.output_profile: + self.__output_profile = profile + break + + # Confirm/create thumbs archive. + if self.opts.generate_descriptions: + if not os.path.exists(self.__cache_dir): + self.opts.log.info(" creating new thumb cache '%s'" % self.__cache_dir) + os.makedirs(self.__cache_dir) + if not os.path.exists(self.__archive_path): + self.opts.log.info(' creating thumbnail archive, thumb_width: %1.2f"' % + float(self.opts.thumb_width)) + with ZipFile(self.__archive_path, mode='w') as zfw: + zfw.writestr("Catalog Thumbs Archive",'') + else: + try: + with ZipFile(self.__archive_path, mode='r') as zfr: + try: + cached_thumb_width = zfr.read('thumb_width') + except: + cached_thumb_width = "-1" + except: + os.remove(self.__archive_path) + cached_thumb_width = '-1' + + if float(cached_thumb_width) != float(self.opts.thumb_width): + self.opts.log.warning(" invalidating cache at '%s'" % self.__archive_path) + self.opts.log.warning(' thumb_width changed: %1.2f" => %1.2f"' % + (float(cached_thumb_width),float(self.opts.thumb_width))) + with ZipFile(self.__archive_path, mode='w') as zfw: + zfw.writestr("Catalog Thumbs Archive",'') + else: + self.opts.log.info(' existing thumb cache at %s, cached_thumb_width: %1.2f"' % + (self.__archive_path, float(cached_thumb_width))) + + # Tweak build steps based on optional sections: 1 call for HTML, 1 for NCX + incremental_jobs = 0 + if self.opts.generate_authors: + incremental_jobs += 2 + if self.opts.generate_titles: + incremental_jobs += 2 + if self.opts.generate_recently_added: + incremental_jobs += 2 + if self.generateRecentlyRead: + incremental_jobs += 2 + if self.opts.generate_series: + incremental_jobs += 2 + if self.opts.generate_descriptions: + # +1 thumbs + incremental_jobs += 3 + self.__totalSteps += incremental_jobs + + # Load section list templates + templates = {} + execfile(P('catalog/section_list_templates.py'), templates) + for name, template in templates.iteritems(): + if name.startswith('by_') and name.endswith('_template'): + setattr(self, name, force_unicode(template, 'utf-8')) + + # Accessors + if True: + ''' + @dynamic_property + def xxxx(self): + def fget(self): + return self.__ + def fset(self, val): + self.__ = val + return property(fget=fget, fset=fset) + ''' + @dynamic_property + def authorClip(self): + def fget(self): + return self.__authorClip + def fset(self, val): + self.__authorClip = val + return property(fget=fget, fset=fset) + @dynamic_property + def authors(self): + def fget(self): + return self.__authors + def fset(self, val): + self.__authors = val + return property(fget=fget, fset=fset) + @dynamic_property + def basename(self): + def fget(self): + return self.__basename + def fset(self, val): + self.__basename = val + return property(fget=fget, fset=fset) + @dynamic_property + def bookmarked_books(self): + def fget(self): + return self.__bookmarked_books + def fset(self, val): + self.__bookmarked_books = val + return property(fget=fget, fset=fset) + @dynamic_property + def booksByAuthor(self): + def fget(self): + return self.__booksByAuthor + def fset(self, val): + self.__booksByAuthor = val + return property(fget=fget, fset=fset) + @dynamic_property + def booksByDateRead(self): + def fget(self): + return self.__booksByDateRead + def fset(self, val): + self.__booksByDateRead = val + return property(fget=fget, fset=fset) + @dynamic_property + def booksByTitle(self): + def fget(self): + return self.__booksByTitle + def fset(self, val): + self.__booksByTitle = val + return property(fget=fget, fset=fset) + @dynamic_property + def booksByTitle_noSeriesPrefix(self): + def fget(self): + return self.__booksByTitle_noSeriesPrefix + def fset(self, val): + self.__booksByTitle_noSeriesPrefix = val + return property(fget=fget, fset=fset) + @dynamic_property + def catalogPath(self): + def fget(self): + return self.__catalogPath + def fset(self, val): + self.__catalogPath = val + return property(fget=fget, fset=fset) + @dynamic_property + def contentDir(self): + def fget(self): + return self.__contentDir + def fset(self, val): + self.__contentDir = val + return property(fget=fget, fset=fset) + @dynamic_property + def currentStep(self): + def fget(self): + return self.__currentStep + def fset(self, val): + self.__currentStep = val + return property(fget=fget, fset=fset) + @dynamic_property + def creator(self): + def fget(self): + return self.__creator + def fset(self, val): + self.__creator = val + return property(fget=fget, fset=fset) + @dynamic_property + def db(self): + def fget(self): + return self.__db + return property(fget=fget) + @dynamic_property + def descriptionClip(self): + def fget(self): + return self.__descriptionClip + def fset(self, val): + self.__descriptionClip = val + return property(fget=fget, fset=fset) + @dynamic_property + def error(self): + def fget(self): + return self.__error + def fset(self, val): + self.__error = val + return property(fget=fget,fset=fset) + @dynamic_property + def generateForKindle(self): + def fget(self): + return self.__generateForKindle + def fset(self, val): + self.__generateForKindle = val + return property(fget=fget, fset=fset) + @dynamic_property + def generateRecentlyRead(self): + def fget(self): + return self.__generateRecentlyRead + def fset(self, val): + self.__generateRecentlyRead = val + return property(fget=fget, fset=fset) + @dynamic_property + def genres(self): + def fget(self): + return self.__genres + def fset(self, val): + self.__genres = val + return property(fget=fget, fset=fset) + @dynamic_property + def genre_tags_dict(self): + def fget(self): + return self.__genre_tags_dict + def fset(self, val): + self.__genre_tags_dict = val + return property(fget=fget, fset=fset) + @dynamic_property + def htmlFileList_1(self): + def fget(self): + return self.__htmlFileList_1 + def fset(self, val): + self.__htmlFileList_1 = val + return property(fget=fget, fset=fset) + @dynamic_property + def htmlFileList_2(self): + def fget(self): + return self.__htmlFileList_2 + def fset(self, val): + self.__htmlFileList_2 = val + return property(fget=fget, fset=fset) + @dynamic_property + def libraryPath(self): + def fget(self): + return self.__libraryPath + def fset(self, val): + self.__libraryPath = val + return property(fget=fget, fset=fset) + @dynamic_property + def markerTags(self): + def fget(self): + return self.__markerTags + def fset(self, val): + self.__markerTags = val + return property(fget=fget, fset=fset) + @dynamic_property + def ncxSoup(self): + def fget(self): + return self.__ncxSoup + def fset(self, val): + self.__ncxSoup = val + return property(fget=fget, fset=fset) + @dynamic_property + def opts(self): + def fget(self): + return self.__opts + return property(fget=fget) + @dynamic_property + def playOrder(self): + def fget(self): + return self.__playOrder + def fset(self,val): + self.__playOrder = val + return property(fget=fget, fset=fset) + @dynamic_property + def plugin(self): + def fget(self): + return self.__plugin + return property(fget=fget) + @dynamic_property + def progressInt(self): + def fget(self): + return self.__progressInt + def fset(self, val): + self.__progressInt = val + return property(fget=fget, fset=fset) + @dynamic_property + def progressString(self): + def fget(self): + return self.__progressString + def fset(self, val): + self.__progressString = val + return property(fget=fget, fset=fset) + @dynamic_property + def reporter(self): + def fget(self): + return self.__reporter + def fset(self, val): + self.__reporter = val + return property(fget=fget, fset=fset) + @dynamic_property + def stylesheet(self): + def fget(self): + return self.__stylesheet + def fset(self, val): + self.__stylesheet = val + return property(fget=fget, fset=fset) + @dynamic_property + def thumbs(self): + def fget(self): + return self.__thumbs + def fset(self, val): + self.__thumbs = val + return property(fget=fget, fset=fset) + def thumbWidth(self): + def fget(self): + return self.__thumbWidth + def fset(self, val): + self.__thumbWidth = val + return property(fget=fget, fset=fset) + def thumbHeight(self): + def fget(self): + return self.__thumbHeight + def fset(self, val): + self.__thumbHeight = val + return property(fget=fget, fset=fset) + @dynamic_property + def title(self): + def fget(self): + return self.__title + def fset(self, val): + self.__title = val + return property(fget=fget, fset=fset) + @dynamic_property + def totalSteps(self): + def fget(self): + return self.__totalSteps + return property(fget=fget) + @dynamic_property + def useSeriesPrefixInTitlesSection(self): + def fget(self): + return self.__useSeriesPrefixInTitlesSection + def fset(self, val): + self.__useSeriesPrefixInTitlesSection = val + return property(fget=fget, fset=fset) + @dynamic_property + def verbose(self): + def fget(self): + return self.__verbose + def fset(self, val): + self.__verbose = val + return property(fget=fget, fset=fset) + + @dynamic_property + def MISSING_SYMBOL(self): + def fget(self): + return self.__output_profile.missing_char + return property(fget=fget) + @dynamic_property + def NOT_READ_SYMBOL(self): + def fget(self): + return '%s' % self.__output_profile.read_char + return property(fget=fget) + @dynamic_property + def READING_SYMBOL(self): + def fget(self): + return '▷' if self.generateForKindle else \ + '+' + return property(fget=fget) + @dynamic_property + def READ_SYMBOL(self): + def fget(self): + return self.__output_profile.read_char + return property(fget=fget) + @dynamic_property + def FULL_RATING_SYMBOL(self): + def fget(self): + return self.__output_profile.ratings_char + return property(fget=fget) + @dynamic_property + def EMPTY_RATING_SYMBOL(self): + def fget(self): + return self.__output_profile.empty_ratings_char + return property(fget=fget) + @dynamic_property + def READ_PROGRESS_SYMBOL(self): + def fget(self): + return "▪" if self.generateForKindle else '+' + return property(fget=fget) + @dynamic_property + def UNREAD_PROGRESS_SYMBOL(self): + def fget(self): + return "▫" if self.generateForKindle else '-' + return property(fget=fget) + + # Methods + def buildSources(self): + if self.booksByTitle is None: + if not self.fetchBooksByTitle(): + return False + if not self.fetchBooksByAuthor(): + return False + self.fetchBookmarks() + if self.opts.generate_descriptions: + self.generateThumbnails() + self.generateHTMLDescriptions() + if self.opts.generate_authors: + self.generateHTMLByAuthor() + if self.opts.generate_titles: + self.generateHTMLByTitle() + if self.opts.generate_series: + self.generateHTMLBySeries() + if self.opts.generate_genres: + self.generateHTMLByTags() + # If this is the only Section, and there are no genres, bail + if self.opts.section_list == ['Genres'] and not self.genres: + error_msg = _("No enabled genres found to catalog.\n") + if not self.opts.cli_environment: + error_msg += "Check 'Excluded genres'\nin E-book options.\n" + self.opts.log.error(error_msg) + self.error.append(_('No books available to catalog')) + self.error.append(error_msg) + return False + if self.opts.generate_recently_added: + self.generateHTMLByDateAdded() + if self.generateRecentlyRead: + self.generateHTMLByDateRead() + + self.generateOPF() + self.generateNCXHeader() + if self.opts.generate_authors: + self.generateNCXByAuthor("Authors") + if self.opts.generate_titles: + self.generateNCXByTitle("Titles") + if self.opts.generate_series: + self.generateNCXBySeries("Series") + if self.opts.generate_genres: + self.generateNCXByGenre("Genres") + if self.opts.generate_recently_added: + self.generateNCXByDateAdded("Recently Added") + if self.generateRecentlyRead: + self.generateNCXByDateRead("Recently Read") + if self.opts.generate_descriptions: + self.generateNCXDescriptions("Descriptions") + + self.writeNCX() + return True + + def cleanUp(self): + pass + + def copyResources(self): + '''Move resource files to self.catalogPath''' + catalog_resources = P("catalog") + + files_to_copy = [('','DefaultCover.jpg'), + ('content','stylesheet.css'), + ('images','mastheadImage.gif')] + + for file in files_to_copy: + if file[0] == '': + shutil.copy(os.path.join(catalog_resources,file[1]), + self.catalogPath) + else: + shutil.copy(os.path.join(catalog_resources,file[1]), + os.path.join(self.catalogPath, file[0])) + + # Create the custom masthead image overwriting default + # If failure, default mastheadImage.gif should still be in place + if self.generateForKindle: + try: + self.generateMastheadImage(os.path.join(self.catalogPath, + 'images/mastheadImage.gif')) + except: + pass + + def fetchBooksByAuthor(self): + ''' + Generate a list of titles sorted by author from the database + return = Success + ''' + + self.updateProgressFullStep("Sorting database") + self.booksByAuthor = list(self.booksByTitle) + + # Test for author_sort mismatches + self.booksByAuthor = sorted(self.booksByAuthor, key=self.booksByAuthorSorter_author) + # Build the unique_authors set from existing data + authors = [(record['author'], record['author_sort']) for record in self.booksByAuthor] + current_author = authors[0] + for (i,author) in enumerate(authors): + if author != current_author and i: + if author[0] == current_author[0]: + if self.opts.fmt == 'mobi': + # Exit if building MOBI + error_msg = _( +'''Inconsistent Author Sort values for +Author '{0}': +'{1}' <> '{2}' +Unable to build MOBI catalog.\n +Select all books by '{0}', apply correct Author Sort value in Edit Metadata dialog, then rebuild the catalog.\n''').format(author[0],author[1],current_author[1]) + self.opts.log.warn('\n*** Metadata error ***') + self.opts.log.warn(error_msg) + + self.error.append('Author Sort mismatch') + self.error.append(error_msg) + return False + else: + # Warning if building non-MOBI + if not self.error: + self.error.append('Author Sort mismatch') + + error_msg = _( +'''Warning: inconsistent Author Sort values for +Author '{0}': +'{1}' <> '{2}'\n''').format(author[0],author[1],current_author[1]) + self.opts.log.warn('\n*** Metadata warning ***') + self.opts.log.warn(error_msg) + self.error.append(error_msg) + + current_author = author + + self.booksByAuthor = sorted(self.booksByAuthor, key=self.booksByAuthorSorter_author_sort) + + # Build the unique_authors set from existing data + authors = [(record['author'], capitalize(record['author_sort'])) for record in self.booksByAuthor] + + # authors[] contains a list of all book authors, with multiple entries for multiple books by author + # authors[]: (([0]:friendly [1]:sort)) + # unique_authors[]: (([0]:friendly [1]:sort [2]:book_count)) + books_by_current_author = 0 + current_author = authors[0] + multiple_authors = False + unique_authors = [] + for (i,author) in enumerate(authors): + if author != current_author: + # Note that current_author and author are tuples: (friendly, sort) + multiple_authors = True + + # New author, save the previous author/sort/count + unique_authors.append((current_author[0], icu_title(current_author[1]), + books_by_current_author)) + current_author = author + books_by_current_author = 1 + elif i==0 and len(authors) == 1: + # Allow for single-book lists + unique_authors.append((current_author[0], icu_title(current_author[1]), + books_by_current_author)) + else: + books_by_current_author += 1 + else: + # Add final author to list or single-author dataset + if (current_author == author and len(authors) > 1) or not multiple_authors: + unique_authors.append((current_author[0], icu_title(current_author[1]), + books_by_current_author)) + + if False and self.verbose: + self.opts.log.info("\nfetchBooksByauthor(): %d unique authors" % len(unique_authors)) + for author in unique_authors: + self.opts.log.info((u" %-50s %-25s %2d" % (author[0][0:45], author[1][0:20], + author[2])).encode('utf-8')) + + self.authors = unique_authors + return True + + def fetchBooksByTitle(self): + self.updateProgressFullStep("Fetching database") + + self.opts.sort_by = 'title' + + # Merge opts.exclude_tags with opts.search_text + # Updated to use exact match syntax + empty_exclude_tags = False if len(self.opts.exclude_tags) else True + search_phrase = '' + if not empty_exclude_tags: + exclude_tags = self.opts.exclude_tags.split(',') + search_terms = [] + for tag in exclude_tags: + search_terms.append("tag:=%s" % tag) + search_phrase = "not (%s)" % " or ".join(search_terms) + # If a list of ids are provided, don't use search_text + if self.opts.ids: + self.opts.search_text = search_phrase + else: + if self.opts.search_text: + self.opts.search_text += " " + search_phrase + else: + self.opts.search_text = search_phrase + + # Fetch the database as a dictionary + data = self.plugin.search_sort_db(self.db, self.opts) + data = self.processExclusions(data) + + # Populate this_title{} from data[{},{}] + titles = [] + for record in data: + this_title = {} + + this_title['id'] = record['id'] + this_title['uuid'] = record['uuid'] + + this_title['title'] = self.convertHTMLEntities(record['title']) + if record['series']: + this_title['series'] = record['series'] + this_title['series_index'] = record['series_index'] + else: + this_title['series'] = None + this_title['series_index'] = 0.0 + + this_title['title_sort'] = self.generateSortTitle(this_title['title']) + if 'authors' in record: + # from calibre.ebooks.metadata import authors_to_string + # return authors_to_string(self.authors) + + this_title['authors'] = record['authors'] + if record['authors']: + this_title['author'] = " & ".join(record['authors']) + else: + this_title['author'] = 'Unknown' + + if 'author_sort' in record and record['author_sort'].strip(): + this_title['author_sort'] = record['author_sort'] + else: + this_title['author_sort'] = self.author_to_author_sort(this_title['author']) + + if record['publisher']: + this_title['publisher'] = re.sub('&', '&', record['publisher']) + + this_title['rating'] = record['rating'] if record['rating'] else 0 + + if is_date_undefined(record['pubdate']): + this_title['date'] = None + else: + this_title['date'] = strftime(u'%B %Y', record['pubdate'].timetuple()) + + this_title['timestamp'] = record['timestamp'] + + if record['comments']: + # Strip annotations + a_offset = record['comments'].find('
') + ad_offset = record['comments'].find('
') + if a_offset >= 0: + record['comments'] = record['comments'][:a_offset] + if ad_offset >= 0: + record['comments'] = record['comments'][:ad_offset] + + this_title['description'] = self.markdownComments(record['comments']) + + # Create short description + paras = BeautifulSoup(this_title['description']).findAll('p') + tokens = [] + for p in paras: + for token in p.contents: + if token.string is not None: + tokens.append(token.string) + this_title['short_description'] = self.generateShortDescription(' '.join(tokens), dest="description") + else: + this_title['description'] = None + this_title['short_description'] = None + + # Merge with custom field/value + if self.__merge_comments['field']: + this_title['description'] = self.mergeComments(this_title) + + if record['cover']: + this_title['cover'] = re.sub('&', '&', record['cover']) + + this_title['read'] = self.discoverReadStatus(record) + + if record['tags']: + this_title['tags'] = self.processSpecialTags(record['tags'], + this_title, self.opts) + if record['formats']: + formats = [] + for format in record['formats']: + formats.append(self.convertHTMLEntities(format)) + this_title['formats'] = formats + + # Add user notes to be displayed in header + # Special case handling for datetime fields and lists + if self.opts.header_note_source_field: + field_md = self.__db.metadata_for_field(self.opts.header_note_source_field) + notes = self.__db.get_field(record['id'], + self.opts.header_note_source_field, + index_is_id=True) + if notes: + if field_md['datatype'] == 'text': + if isinstance(notes,list): + notes = ' · '.join(notes) + elif field_md['datatype'] == 'datetime': + notes = format_date(notes,'dd MMM yyyy') + this_title['notes'] = {'source':field_md['name'], + 'content':notes} + + titles.append(this_title) + + # Re-sort based on title_sort + if len(titles): + self.booksByTitle = sorted(titles, + key=lambda x:(x['title_sort'].upper(), x['title_sort'].upper())) + if False and self.verbose: + self.opts.log.info("fetchBooksByTitle(): %d books" % len(self.booksByTitle)) + self.opts.log.info(" %-40s %-40s" % ('title', 'title_sort')) + for title in self.booksByTitle: + self.opts.log.info((u" %-40s %-40s" % (title['title'][0:40], + title['title_sort'][0:40])).decode('mac-roman')) + return True + else: + error_msg = _("No books found to catalog.\nCheck 'Excluded books' criteria in E-book options.\n") + self.opts.log.error('*** ' + error_msg + ' ***') + self.error.append(_('No books available to include in catalog')) + self.error.append(error_msg) + return False + + def fetchBookmarks(self): + ''' + Collect bookmarks for catalog entries + This will use the system default save_template specified in + Preferences|Add/Save|Sending to device, not a customized one specified in + the Kindle plugin + ''' + from calibre.devices.usbms.device import Device + from calibre.devices.kindle.driver import Bookmark + from calibre.ebooks.metadata import MetaInformation + + MBP_FORMATS = [u'azw', u'mobi', u'prc', u'txt'] + mbp_formats = set(MBP_FORMATS) + PDR_FORMATS = [u'pdf'] + pdr_formats = set(PDR_FORMATS) + TAN_FORMATS = [u'tpz', u'azw1'] + tan_formats = set(TAN_FORMATS) + + class BookmarkDevice(Device): + def initialize(self, save_template): + self._save_template = save_template + self.SUPPORTS_SUB_DIRS = True + def save_template(self): + return self._save_template + + def resolve_bookmark_paths(storage, path_map): + pop_list = [] + book_ext = {} + for id in path_map: + file_fmts = set() + for fmt in path_map[id]['fmts']: + file_fmts.add(fmt) + + bookmark_extension = None + if file_fmts.intersection(mbp_formats): + book_extension = list(file_fmts.intersection(mbp_formats))[0] + bookmark_extension = 'mbp' + elif file_fmts.intersection(tan_formats): + book_extension = list(file_fmts.intersection(tan_formats))[0] + bookmark_extension = 'tan' + elif file_fmts.intersection(pdr_formats): + book_extension = list(file_fmts.intersection(pdr_formats))[0] + bookmark_extension = 'pdr' + + if bookmark_extension: + for vol in storage: + bkmk_path = path_map[id]['path'].replace(os.path.abspath('/'),vol) + bkmk_path = bkmk_path.replace('bookmark',bookmark_extension) + if os.path.exists(bkmk_path): + path_map[id] = bkmk_path + book_ext[id] = book_extension + break + else: + pop_list.append(id) + else: + pop_list.append(id) + # Remove non-existent bookmark templates + for id in pop_list: + path_map.pop(id) + return path_map, book_ext + + if self.generateRecentlyRead: + self.opts.log.info(" Collecting Kindle bookmarks matching catalog entries") + + d = BookmarkDevice(None) + d.initialize(self.opts.connected_device['save_template']) + + bookmarks = {} + for book in self.booksByTitle: + if 'formats' in book: + path_map = {} + id = book['id'] + original_title = book['title'][book['title'].find(':') + 2:] if book['series'] \ + else book['title'] + myMeta = MetaInformation(original_title, + authors=book['authors']) + myMeta.author_sort = book['author_sort'] + a_path = d.create_upload_path('/', myMeta, 'x.bookmark', create_dirs=False) + path_map[id] = dict(path=a_path, fmts=[x.rpartition('.')[2] for x in book['formats']]) + + path_map, book_ext = resolve_bookmark_paths(self.opts.connected_device['storage'], path_map) + if path_map: + bookmark_ext = path_map[id].rpartition('.')[2] + myBookmark = Bookmark(path_map[id], id, book_ext[id], bookmark_ext) + try: + book['percent_read'] = min(float(100*myBookmark.last_read / myBookmark.book_length),100) + except: + book['percent_read'] = 0 + dots = int((book['percent_read'] + 5)/10) + dot_string = self.READ_PROGRESS_SYMBOL * dots + empty_dots = self.UNREAD_PROGRESS_SYMBOL * (10 - dots) + book['reading_progress'] = '%s%s' % (dot_string,empty_dots) + bookmarks[id] = ((myBookmark,book)) + + self.bookmarked_books = bookmarks + else: + self.bookmarked_books = {} + + def generateHTMLDescriptions(self): + ''' + Write each title to a separate HTML file in contentdir + ''' + self.updateProgressFullStep("'Descriptions'") + + for (title_num, title) in enumerate(self.booksByTitle): + self.updateProgressMicroStep("Description %d of %d" % \ + (title_num, len(self.booksByTitle)), + float(title_num*100/len(self.booksByTitle))/100) + + # Generate the header from user-customizable template + soup = self.generateHTMLDescriptionHeader(title) + + # Write the book entry to contentdir + outfile = open("%s/book_%d.html" % (self.contentDir, int(title['id'])), 'w') + outfile.write(soup.prettify()) + outfile.close() + + def generateHTMLByTitle(self): + ''' + Write books by title A-Z to HTML file + ''' + self.updateProgressFullStep("'Titles'") + + soup = self.generateHTMLEmptyHeader("Books By Alpha Title") + body = soup.find('body') + btc = 0 + + # Insert section tag + aTag = Tag(soup,'a') + aTag['name'] = 'section_start' + body.insert(btc, aTag) + btc += 1 + + # Insert the anchor + aTag = Tag(soup, "a") + aTag['name'] = "bytitle" + body.insert(btc, aTag) + btc += 1 + + if not self.__generateForKindle: + # We don't need this because the Kindle shows section titles + #

By Title

+ pTag = Tag(soup, "p") + pTag['class'] = 'title' + aTag = Tag(soup, "a") + aTag['name'] = "bytitle" + pTag.insert(0,aTag) + pTag.insert(1,NavigableString('Titles')) + body.insert(btc,pTag) + btc += 1 + + divTag = Tag(soup, "div") + dtc = 0 + current_letter = "" + + # Re-sort title list without leading series/series_index + # Incoming title : + if not self.useSeriesPrefixInTitlesSection: + nspt = deepcopy(self.booksByTitle) + nspt = sorted(nspt, key=lambda x:(x['title_sort'].upper(), x['title_sort'].upper())) + self.booksByTitle_noSeriesPrefix = nspt + + # Loop through the books by title + # Generate one divRunningTag per initial letter for the purposes of + # minimizing widows and orphans on readers that can handle large + # <divs> styled as inline-block + title_list = self.booksByTitle + if not self.useSeriesPrefixInTitlesSection: + title_list = self.booksByTitle_noSeriesPrefix + drtc = 0 + divRunningTag = None + for book in title_list: + if self.letter_or_symbol(book['title_sort'][0]) != current_letter : + # Start a new letter + if drtc and divRunningTag is not None: + divTag.insert(dtc, divRunningTag) + dtc += 1 + divRunningTag = Tag(soup, 'div') + if dtc > 0: + divRunningTag['class'] = "initial_letter" + drtc = 0 + current_letter = self.letter_or_symbol(book['title_sort'][0]) + pIndexTag = Tag(soup, "p") + pIndexTag['class'] = "author_title_letter_index" + aTag = Tag(soup, "a") + aTag['name'] = "%s" % self.letter_or_symbol(book['title_sort'][0]) + pIndexTag.insert(0,aTag) + pIndexTag.insert(1,NavigableString(self.letter_or_symbol(book['title_sort'][0]))) + divRunningTag.insert(dtc,pIndexTag) + drtc += 1 + + # Add books + pBookTag = Tag(soup, "p") + ptc = 0 + + # book with read|reading|unread symbol or wishlist item + if self.opts.wishlist_tag in book.get('tags', []): + pBookTag['class'] = "wishlist_item" + pBookTag.insert(ptc,NavigableString(self.MISSING_SYMBOL)) + ptc += 1 + else: + if book['read']: + # check mark + pBookTag.insert(ptc,NavigableString(self.READ_SYMBOL)) + pBookTag['class'] = "read_book" + ptc += 1 + elif book['id'] in self.bookmarked_books: + pBookTag.insert(ptc,NavigableString(self.READING_SYMBOL)) + pBookTag['class'] = "read_book" + ptc += 1 + else: + # hidden check mark + pBookTag['class'] = "unread_book" + pBookTag.insert(ptc,NavigableString(self.NOT_READ_SYMBOL)) + ptc += 1 + + # Link to book + aTag = Tag(soup, "a") + if self.opts.generate_descriptions: + aTag['href'] = "book_%d.html" % (int(float(book['id']))) + + # Generate the title from the template + args = self.generateFormatArgs(book) + if book['series']: + formatted_title = self.by_titles_series_title_template.format(**args).rstrip() + else: + formatted_title = self.by_titles_normal_title_template.format(**args).rstrip() + aTag.insert(0,NavigableString(escape(formatted_title))) + pBookTag.insert(ptc, aTag) + ptc += 1 + + # Dot + pBookTag.insert(ptc, NavigableString(" · ")) + ptc += 1 + + # Link to author + emTag = Tag(soup, "em") + aTag = Tag(soup, "a") + if self.opts.generate_authors: + aTag['href'] = "%s.html#%s" % ("ByAlphaAuthor", self.generateAuthorAnchor(book['author'])) + aTag.insert(0, NavigableString(book['author'])) + emTag.insert(0,aTag) + pBookTag.insert(ptc, emTag) + ptc += 1 + + if divRunningTag is not None: + divRunningTag.insert(drtc, pBookTag) + drtc += 1 + + # Add the last divRunningTag to divTag + if divRunningTag is not None: + divTag.insert(dtc, divRunningTag) + dtc += 1 + + # Add the divTag to the body + body.insert(btc, divTag) + btc += 1 + + # Write the volume to contentdir + outfile_spec = "%s/ByAlphaTitle.html" % (self.contentDir) + outfile = open(outfile_spec, 'w') + outfile.write(soup.prettify()) + outfile.close() + self.htmlFileList_1.append("content/ByAlphaTitle.html") + + def generateHTMLByAuthor(self): + ''' + Write books by author A-Z + ''' + self.updateProgressFullStep("'Authors'") + + friendly_name = "Authors" + + soup = self.generateHTMLEmptyHeader(friendly_name) + body = soup.find('body') + + btc = 0 + + # Insert section tag + aTag = Tag(soup,'a') + aTag['name'] = 'section_start' + body.insert(btc, aTag) + btc += 1 + + # Insert the anchor + aTag = Tag(soup, "a") + anchor_name = friendly_name.lower() + aTag['name'] = anchor_name.replace(" ","") + body.insert(btc, aTag) + btc += 1 + + divTag = Tag(soup, "div") + dtc = 0 + divOpeningTag = None + dotc = 0 + divRunningTag = None + drtc = 0 + + # Loop through booksByAuthor + # Each author/books group goes in an openingTag div (first) or + # a runningTag div (subsequent) + book_count = 0 + current_author = '' + current_letter = '' + current_series = None + #for book in sorted(self.booksByAuthor, key = self.booksByAuthorSorter_author_sort): + for book in self.booksByAuthor: + + book_count += 1 + if self.letter_or_symbol(book['author_sort'][0].upper()) != current_letter : + # Start a new letter with Index letter + if divOpeningTag is not None: + divTag.insert(dtc, divOpeningTag) + dtc += 1 + dotc = 0 + if divRunningTag is not None: + divTag.insert(dtc, divRunningTag) + dtc += 1 + drtc = 0 + divRunningTag = None + + current_letter = self.letter_or_symbol(book['author_sort'][0].upper()) + author_count = 0 + divOpeningTag = Tag(soup, 'div') + if dtc > 0: + divOpeningTag['class'] = "initial_letter" + dotc = 0 + pIndexTag = Tag(soup, "p") + pIndexTag['class'] = "author_title_letter_index" + aTag = Tag(soup, "a") + aTag['name'] = "%sauthors" % self.letter_or_symbol(current_letter) + pIndexTag.insert(0,aTag) + pIndexTag.insert(1,NavigableString(self.letter_or_symbol(book['author_sort'][0].upper()))) + divOpeningTag.insert(dotc,pIndexTag) + dotc += 1 + + if book['author'] != current_author: + # Start a new author + current_author = book['author'] + author_count += 1 + if author_count >= 2: + # Add divOpeningTag to divTag, kill divOpeningTag + if divOpeningTag: + divTag.insert(dtc, divOpeningTag) + dtc += 1 + divOpeningTag = None + dotc = 0 + + # Create a divRunningTag for the next author + if author_count > 2: + divTag.insert(dtc, divRunningTag) + dtc += 1 + + divRunningTag = Tag(soup, 'div') + divRunningTag['class'] = "author_logical_group" + drtc = 0 + + non_series_books = 0 + current_series = None + pAuthorTag = Tag(soup, "p") + pAuthorTag['class'] = "author_index" + aTag = Tag(soup, "a") + aTag['name'] = "%s" % self.generateAuthorAnchor(current_author) + aTag.insert(0,NavigableString(current_author)) + pAuthorTag.insert(0,aTag) + if author_count == 1: + divOpeningTag.insert(dotc, pAuthorTag) + dotc += 1 + else: + divRunningTag.insert(drtc,pAuthorTag) + drtc += 1 + + # Check for series + if book['series'] and book['series'] != current_series: + # Start a new series + current_series = book['series'] + pSeriesTag = Tag(soup,'p') + pSeriesTag['class'] = "series" + + if self.opts.generate_series: + aTag = Tag(soup,'a') + aTag['href'] = "%s.html#%s_series" % ('BySeries', + re.sub('\W','',book['series']).lower()) + aTag.insert(0, book['series']) + #pSeriesTag.insert(0, NavigableString(self.NOT_READ_SYMBOL)) + pSeriesTag.insert(0, aTag) + else: + #pSeriesTag.insert(0,NavigableString(self.NOT_READ_SYMBOL + '%s' % book['series'])) + pSeriesTag.insert(0,NavigableString('%s' % book['series'])) + + if author_count == 1: + divOpeningTag.insert(dotc, pSeriesTag) + dotc += 1 + elif divRunningTag is not None: + divRunningTag.insert(drtc,pSeriesTag) + drtc += 1 + if current_series and not book['series']: + current_series = None + + # Add books + pBookTag = Tag(soup, "p") + ptc = 0 + + # book with read|reading|unread symbol or wishlist item + if self.opts.wishlist_tag in book.get('tags', []): + pBookTag['class'] = "wishlist_item" + pBookTag.insert(ptc,NavigableString(self.MISSING_SYMBOL)) + ptc += 1 + else: + if book['read']: + # check mark + pBookTag.insert(ptc,NavigableString(self.READ_SYMBOL)) + pBookTag['class'] = "read_book" + ptc += 1 + elif book['id'] in self.bookmarked_books: + pBookTag.insert(ptc,NavigableString(self.READING_SYMBOL)) + pBookTag['class'] = "read_book" + ptc += 1 + else: + # hidden check mark + pBookTag['class'] = "unread_book" + pBookTag.insert(ptc,NavigableString(self.NOT_READ_SYMBOL)) + ptc += 1 + + aTag = Tag(soup, "a") + if self.opts.generate_descriptions: + aTag['href'] = "book_%d.html" % (int(float(book['id']))) + + # Generate the title from the template + args = self.generateFormatArgs(book) + if current_series: + #aTag.insert(0,'%s%s' % (escape(book['title'][len(book['series'])+1:]),pubyear)) + formatted_title = self.by_authors_series_title_template.format(**args).rstrip() + else: + #aTag.insert(0,'%s%s' % (escape(book['title']), pubyear)) + formatted_title = self.by_authors_normal_title_template.format(**args).rstrip() + non_series_books += 1 + aTag.insert(0,NavigableString(escape(formatted_title))) + + pBookTag.insert(ptc, aTag) + ptc += 1 + + if author_count == 1: + divOpeningTag.insert(dotc, pBookTag) + dotc += 1 + elif divRunningTag: + divRunningTag.insert(drtc,pBookTag) + drtc += 1 + + # Loop ends here + + if not self.__generateForKindle: + # Insert the <h2> tag with book_count at the head + #<h2><a name="byalphaauthor" id="byalphaauthor"></a>By Author</h2> + pTag = Tag(soup, "p") + pTag['class'] = 'title' + aTag = Tag(soup, "a") + anchor_name = friendly_name.lower() + aTag['name'] = anchor_name.replace(" ","") + pTag.insert(0,aTag) + #h2Tag.insert(1,NavigableString('%s (%d)' % (friendly_name, book_count))) + pTag.insert(1,NavigableString('%s' % (friendly_name))) + body.insert(btc,pTag) + btc += 1 + + if author_count == 1: + divTag.insert(dtc, divOpeningTag) + dtc += 1 + elif divRunningTag is not None: + divTag.insert(dtc, divRunningTag) + dtc += 1 + + # Add the divTag to the body + body.insert(btc, divTag) + + # Write the generated file to contentdir + outfile_spec = "%s/ByAlphaAuthor.html" % (self.contentDir) + outfile = open(outfile_spec, 'w') + outfile.write(soup.prettify()) + outfile.close() + self.htmlFileList_1.append("content/ByAlphaAuthor.html") + + def generateHTMLByDateAdded(self): + ''' + Write books by reverse chronological order + ''' + self.updateProgressFullStep("'Recently Added'") + + def add_books_to_HTML_by_month(this_months_list, dtc): + if len(this_months_list): + + #this_months_list = sorted(this_months_list, key=self.booksByAuthorSorter_author_sort) + + # Create a new month anchor + date_string = strftime(u'%B %Y', current_date.timetuple()) + pIndexTag = Tag(soup, "p") + pIndexTag['class'] = "date_index" + aTag = Tag(soup, "a") + aTag['name'] = "bda_%s-%s" % (current_date.year, current_date.month) + pIndexTag.insert(0,aTag) + pIndexTag.insert(1,NavigableString(date_string)) + divTag.insert(dtc,pIndexTag) + dtc += 1 + current_author = None + current_series = None + + for new_entry in this_months_list: + if new_entry['author'] != current_author: + # Start a new author + current_author = new_entry['author'] + non_series_books = 0 + current_series = None + pAuthorTag = Tag(soup, "p") + pAuthorTag['class'] = "author_index" + aTag = Tag(soup, "a") + if self.opts.generate_authors: + aTag['name'] = "%s" % self.generateAuthorAnchor(current_author) + aTag.insert(0,NavigableString(current_author)) + pAuthorTag.insert(0,aTag) + divTag.insert(dtc,pAuthorTag) + dtc += 1 + + # Check for series + if new_entry['series'] and new_entry['series'] != current_series: + # Start a new series + current_series = new_entry['series'] + pSeriesTag = Tag(soup,'p') + pSeriesTag['class'] = "series" + if self.opts.generate_series: + aTag = Tag(soup,'a') + aTag['href'] = "%s.html#%s_series" % ('BySeries', + re.sub('\W','',new_entry['series']).lower()) + aTag.insert(0, new_entry['series']) + pSeriesTag.insert(0, aTag) + else: + pSeriesTag.insert(0,NavigableString('%s' % new_entry['series'])) + divTag.insert(dtc,pSeriesTag) + dtc += 1 + if current_series and not new_entry['series']: + current_series = None + + # Add books + pBookTag = Tag(soup, "p") + ptc = 0 + + # book with read|reading|unread symbol or wishlist item + if self.opts.wishlist_tag in new_entry.get('tags', []): + pBookTag['class'] = "wishlist_item" + pBookTag.insert(ptc,NavigableString(self.MISSING_SYMBOL)) + ptc += 1 + else: + if new_entry['read']: + # check mark + pBookTag.insert(ptc,NavigableString(self.READ_SYMBOL)) + pBookTag['class'] = "read_book" + ptc += 1 + elif new_entry['id'] in self.bookmarked_books: + pBookTag.insert(ptc,NavigableString(self.READING_SYMBOL)) + pBookTag['class'] = "read_book" + ptc += 1 + else: + # hidden check mark + pBookTag['class'] = "unread_book" + pBookTag.insert(ptc,NavigableString(self.NOT_READ_SYMBOL)) + ptc += 1 + + aTag = Tag(soup, "a") + if self.opts.generate_descriptions: + aTag['href'] = "book_%d.html" % (int(float(new_entry['id']))) + + # Generate the title from the template + args = self.generateFormatArgs(new_entry) + if current_series: + formatted_title = self.by_month_added_series_title_template.format(**args).rstrip() + else: + formatted_title = self.by_month_added_normal_title_template.format(**args).rstrip() + non_series_books += 1 + aTag.insert(0,NavigableString(escape(formatted_title))) + pBookTag.insert(ptc, aTag) + ptc += 1 + + divTag.insert(dtc, pBookTag) + dtc += 1 + return dtc + + def add_books_to_HTML_by_date_range(date_range_list, date_range, dtc): + if len(date_range_list): + pIndexTag = Tag(soup, "p") + pIndexTag['class'] = "date_index" + aTag = Tag(soup, "a") + aTag['name'] = "bda_%s" % date_range.replace(' ','') + pIndexTag.insert(0,aTag) + pIndexTag.insert(1,NavigableString(date_range)) + divTag.insert(dtc,pIndexTag) + dtc += 1 + + for new_entry in date_range_list: + # Add books + pBookTag = Tag(soup, "p") + ptc = 0 + + # book with read|reading|unread symbol or wishlist item + if self.opts.wishlist_tag in new_entry.get('tags', []): + pBookTag['class'] = "wishlist_item" + pBookTag.insert(ptc,NavigableString(self.MISSING_SYMBOL)) + ptc += 1 + else: + if new_entry['read']: + # check mark + pBookTag.insert(ptc,NavigableString(self.READ_SYMBOL)) + pBookTag['class'] = "read_book" + ptc += 1 + elif new_entry['id'] in self.bookmarked_books: + pBookTag.insert(ptc,NavigableString(self.READING_SYMBOL)) + pBookTag['class'] = "read_book" + ptc += 1 + else: + # hidden check mark + pBookTag['class'] = "unread_book" + pBookTag.insert(ptc,NavigableString(self.NOT_READ_SYMBOL)) + ptc += 1 + + aTag = Tag(soup, "a") + if self.opts.generate_descriptions: + aTag['href'] = "book_%d.html" % (int(float(new_entry['id']))) + + # Generate the title from the template + args = self.generateFormatArgs(new_entry) + if new_entry['series']: + formatted_title = self.by_recently_added_series_title_template.format(**args).rstrip() + else: + formatted_title = self.by_recently_added_normal_title_template.format(**args).rstrip() + aTag.insert(0,NavigableString(escape(formatted_title))) + pBookTag.insert(ptc, aTag) + ptc += 1 + + # Dot + pBookTag.insert(ptc, NavigableString(" · ")) + ptc += 1 + + # Link to author + emTag = Tag(soup, "em") + aTag = Tag(soup, "a") + if self.opts.generate_authors: + aTag['href'] = "%s.html#%s" % ("ByAlphaAuthor", self.generateAuthorAnchor(new_entry['author'])) + aTag.insert(0, NavigableString(new_entry['author'])) + emTag.insert(0,aTag) + pBookTag.insert(ptc, emTag) + ptc += 1 + + divTag.insert(dtc, pBookTag) + dtc += 1 + return dtc + + friendly_name = "Recently Added" + + soup = self.generateHTMLEmptyHeader(friendly_name) + body = soup.find('body') + + btc = 0 + + # Insert section tag + aTag = Tag(soup,'a') + aTag['name'] = 'section_start' + body.insert(btc, aTag) + btc += 1 + + # Insert the anchor + aTag = Tag(soup, "a") + anchor_name = friendly_name.lower() + aTag['name'] = anchor_name.replace(" ","") + body.insert(btc, aTag) + btc += 1 + + if not self.__generateForKindle: + #<h2><a name="byalphaauthor" id="byalphaauthor"></a>By Author</h2> + pTag = Tag(soup, "p") + pTag['class'] = 'title' + aTag = Tag(soup, "a") + anchor_name = friendly_name.lower() + aTag['name'] = anchor_name.replace(" ","") + pTag.insert(0,aTag) + pTag.insert(1,NavigableString('%s' % friendly_name)) + body.insert(btc,pTag) + btc += 1 + + divTag = Tag(soup, "div") + dtc = 0 + + # >>> Books by date range <<< + if self.useSeriesPrefixInTitlesSection: + self.booksByDateRange = sorted(self.booksByTitle, + key=lambda x:(x['timestamp'], x['timestamp']),reverse=True) + else: + nspt = deepcopy(self.booksByTitle) + self.booksByDateRange = sorted(nspt, key=lambda x:(x['timestamp'], x['timestamp']),reverse=True) + + date_range_list = [] + today_time = nowf().replace(hour=23, minute=59, second=59) + for (i, date) in enumerate(self.DATE_RANGE): + date_range_limit = self.DATE_RANGE[i] + if i: + date_range = '%d to %d days ago' % (self.DATE_RANGE[i-1], self.DATE_RANGE[i]) + else: + date_range = 'Last %d days' % (self.DATE_RANGE[i]) + + for book in self.booksByDateRange: + book_time = book['timestamp'] + delta = today_time-book_time + if delta.days <= date_range_limit: + date_range_list.append(book) + else: + break + + dtc = add_books_to_HTML_by_date_range(date_range_list, date_range, dtc) + date_range_list = [book] + + # >>>> Books by month <<<< + # Sort titles case-insensitive for by month using series prefix + self.booksByMonth = sorted(self.booksByTitle, + key=lambda x:(x['timestamp'], x['timestamp']),reverse=True) + + # Loop through books by date + current_date = datetime.date.fromordinal(1) + this_months_list = [] + for book in self.booksByMonth: + if book['timestamp'].month != current_date.month or \ + book['timestamp'].year != current_date.year: + dtc = add_books_to_HTML_by_month(this_months_list, dtc) + this_months_list = [] + current_date = book['timestamp'].date() + this_months_list.append(book) + + # Add the last month's list + add_books_to_HTML_by_month(this_months_list, dtc) + + # Add the divTag to the body + body.insert(btc, divTag) + + # Write the generated file to contentdir + outfile_spec = "%s/ByDateAdded.html" % (self.contentDir) + outfile = open(outfile_spec, 'w') + outfile.write(soup.prettify()) + outfile.close() + self.htmlFileList_2.append("content/ByDateAdded.html") + + def generateHTMLByDateRead(self): + ''' + Write books by active bookmarks + ''' + friendly_name = 'Recently Read' + self.updateProgressFullStep("'%s'" % friendly_name) + if not self.bookmarked_books: + return + + def add_books_to_HTML_by_day(todays_list, dtc): + if len(todays_list): + # Create a new day anchor + date_string = strftime(u'%A, %B %d', current_date.timetuple()) + pIndexTag = Tag(soup, "p") + pIndexTag['class'] = "date_index" + aTag = Tag(soup, "a") + aTag['name'] = "bdr_%s-%s-%s" % (current_date.year, current_date.month, current_date.day) + pIndexTag.insert(0,aTag) + pIndexTag.insert(1,NavigableString(date_string)) + divTag.insert(dtc,pIndexTag) + dtc += 1 + + for new_entry in todays_list: + pBookTag = Tag(soup, "p") + pBookTag['class'] = "date_read" + ptc = 0 + + # Percent read + pBookTag.insert(ptc, NavigableString(new_entry['reading_progress'])) + ptc += 1 + + aTag = Tag(soup, "a") + if self.opts.generate_descriptions: + aTag['href'] = "book_%d.html" % (int(float(new_entry['id']))) + aTag.insert(0,escape(new_entry['title'])) + pBookTag.insert(ptc, aTag) + ptc += 1 + + # Dot + pBookTag.insert(ptc, NavigableString(" · ")) + ptc += 1 + + # Link to author + emTag = Tag(soup, "em") + aTag = Tag(soup, "a") + if self.opts.generate_authors: + aTag['href'] = "%s.html#%s" % ("ByAlphaAuthor", self.generateAuthorAnchor(new_entry['author'])) + aTag.insert(0, NavigableString(new_entry['author'])) + emTag.insert(0,aTag) + pBookTag.insert(ptc, emTag) + ptc += 1 + + divTag.insert(dtc, pBookTag) + dtc += 1 + return dtc + + def add_books_to_HTML_by_date_range(date_range_list, date_range, dtc): + if len(date_range_list): + pIndexTag = Tag(soup, "p") + pIndexTag['class'] = "date_index" + aTag = Tag(soup, "a") + aTag['name'] = "bdr_%s" % date_range.replace(' ','') + pIndexTag.insert(0,aTag) + pIndexTag.insert(1,NavigableString(date_range)) + divTag.insert(dtc,pIndexTag) + dtc += 1 + + for new_entry in date_range_list: + # Add books + pBookTag = Tag(soup, "p") + pBookTag['class'] = "date_read" + ptc = 0 + + # Percent read + dots = int((new_entry['percent_read'] + 5)/10) + dot_string = self.READ_PROGRESS_SYMBOL * dots + empty_dots = self.UNREAD_PROGRESS_SYMBOL * (10 - dots) + pBookTag.insert(ptc, NavigableString('%s%s' % (dot_string,empty_dots))) + ptc += 1 + + aTag = Tag(soup, "a") + if self.opts.generate_descriptions: + aTag['href'] = "book_%d.html" % (int(float(new_entry['id']))) + aTag.insert(0,escape(new_entry['title'])) + pBookTag.insert(ptc, aTag) + ptc += 1 + + # Dot + pBookTag.insert(ptc, NavigableString(" · ")) + ptc += 1 + + # Link to author + emTag = Tag(soup, "em") + aTag = Tag(soup, "a") + if self.opts.generate_authors: + aTag['href'] = "%s.html#%s" % ("ByAlphaAuthor", self.generateAuthorAnchor(new_entry['author'])) + aTag.insert(0, NavigableString(new_entry['author'])) + emTag.insert(0,aTag) + pBookTag.insert(ptc, emTag) + ptc += 1 + + divTag.insert(dtc, pBookTag) + dtc += 1 + return dtc + + soup = self.generateHTMLEmptyHeader(friendly_name) + body = soup.find('body') + + btc = 0 + + # Insert section tag + aTag = Tag(soup,'a') + aTag['name'] = 'section_start' + body.insert(btc, aTag) + btc += 1 + + # Insert the anchor + aTag = Tag(soup, "a") + anchor_name = friendly_name.lower() + aTag['name'] = anchor_name.replace(" ","") + body.insert(btc, aTag) + btc += 1 + + divTag = Tag(soup, "div") + dtc = 0 + + # self.bookmarked_books: (Bookmark, book) + bookmarked_books = [] + for bm_book in self.bookmarked_books: + book = self.bookmarked_books[bm_book] + #print "bm_book: %s" % bm_book + book[1]['bookmark_timestamp'] = book[0].timestamp + try: + book[1]['percent_read'] = min(float(100*book[0].last_read / book[0].book_length),100) + except: + book[1]['percent_read'] = 0 + bookmarked_books.append(book[1]) + + self.booksByDateRead = sorted(bookmarked_books, + key=lambda x:(x['bookmark_timestamp'], x['bookmark_timestamp']),reverse=True) + + # >>>> Recently read by day <<<< + current_date = datetime.date.fromordinal(1) + todays_list = [] + for book in self.booksByDateRead: + bookmark_time = datetime.datetime.utcfromtimestamp(book['bookmark_timestamp']) + if bookmark_time.day != current_date.day or \ + bookmark_time.month != current_date.month or \ + bookmark_time.year != current_date.year: + dtc = add_books_to_HTML_by_day(todays_list, dtc) + todays_list = [] + current_date = datetime.datetime.utcfromtimestamp(book['bookmark_timestamp']).date() + todays_list.append(book) + + # Add the last day's list + add_books_to_HTML_by_day(todays_list, dtc) + + # Add the divTag to the body + body.insert(btc, divTag) + + # Write the generated file to contentdir + outfile_spec = "%s/ByDateRead.html" % (self.contentDir) + outfile = open(outfile_spec, 'w') + outfile.write(soup.prettify()) + outfile.close() + self.htmlFileList_2.append("content/ByDateRead.html") + + def generateHTMLBySeries(self): + ''' + Generate a list of series + ''' + self.updateProgressFullStep("Fetching series") + + self.opts.sort_by = 'series' + + # Merge opts.exclude_tags with opts.search_text + # Updated to use exact match syntax + empty_exclude_tags = False if len(self.opts.exclude_tags) else True + search_phrase = 'series:true ' + if not empty_exclude_tags: + exclude_tags = self.opts.exclude_tags.split(',') + search_terms = [] + for tag in exclude_tags: + search_terms.append("tag:=%s" % tag) + search_phrase += "not (%s)" % " or ".join(search_terms) + + # If a list of ids are provided, don't use search_text + if self.opts.ids: + self.opts.search_text = search_phrase + else: + if self.opts.search_text: + self.opts.search_text += " " + search_phrase + else: + self.opts.search_text = search_phrase + + # Fetch the database as a dictionary + data = self.plugin.search_sort_db(self.db, self.opts) + self.booksBySeries = self.processExclusions(data) + + if not self.booksBySeries: + self.opts.generate_series = False + self.opts.log(" no series found in selected books, cancelling series generation") + return + + friendly_name = "Series" + + soup = self.generateHTMLEmptyHeader(friendly_name) + body = soup.find('body') + + btc = 0 + + # Insert section tag + aTag = Tag(soup,'a') + aTag['name'] = 'section_start' + body.insert(btc, aTag) + btc += 1 + + # Insert the anchor + aTag = Tag(soup, "a") + anchor_name = friendly_name.lower() + aTag['name'] = anchor_name.replace(" ","") + body.insert(btc, aTag) + btc += 1 + + divTag = Tag(soup, "div") + dtc = 0 + current_letter = "" + current_series = None + + # Loop through booksBySeries + series_count = 0 + for book in self.booksBySeries: + # Check for initial letter change + sort_title = self.generateSortTitle(book['series']) + if self.letter_or_symbol(sort_title[0].upper()) != current_letter : + # Start a new letter with Index letter + current_letter = self.letter_or_symbol(sort_title[0].upper()) + pIndexTag = Tag(soup, "p") + pIndexTag['class'] = "series_letter_index" + aTag = Tag(soup, "a") + aTag['name'] = "%s_series" % self.letter_or_symbol(current_letter) + pIndexTag.insert(0,aTag) + pIndexTag.insert(1,NavigableString(self.letter_or_symbol(sort_title[0].upper()))) + divTag.insert(dtc,pIndexTag) + dtc += 1 + + # Check for series change + if book['series'] != current_series: + # Start a new series + series_count += 1 + current_series = book['series'] + pSeriesTag = Tag(soup,'p') + pSeriesTag['class'] = "series" + aTag = Tag(soup, 'a') + aTag['name'] = "%s_series" % re.sub('\W','',book['series']).lower() + pSeriesTag.insert(0,aTag) + pSeriesTag.insert(1,NavigableString('%s' % book['series'])) + divTag.insert(dtc,pSeriesTag) + dtc += 1 + + # Add books + pBookTag = Tag(soup, "p") + ptc = 0 + + book['read'] = self.discoverReadStatus(book) + + # book with read|reading|unread symbol or wishlist item + if self.opts.wishlist_tag in book.get('tags', []): + pBookTag['class'] = "wishlist_item" + pBookTag.insert(ptc,NavigableString(self.MISSING_SYMBOL)) + ptc += 1 + else: + if book.get('read', False): + # check mark + pBookTag.insert(ptc,NavigableString(self.READ_SYMBOL)) + pBookTag['class'] = "read_book" + ptc += 1 + elif book['id'] in self.bookmarked_books: + pBookTag.insert(ptc,NavigableString(self.READING_SYMBOL)) + pBookTag['class'] = "read_book" + ptc += 1 + else: + # hidden check mark + pBookTag['class'] = "unread_book" + pBookTag.insert(ptc,NavigableString(self.NOT_READ_SYMBOL)) + ptc += 1 + + aTag = Tag(soup, "a") + if self.opts.generate_descriptions: + aTag['href'] = "book_%d.html" % (int(float(book['id']))) + # Use series, series index if avail else just title + #aTag.insert(0,'%d. %s · %s' % (book['series_index'],escape(book['title']), ' & '.join(book['authors']))) + + if is_date_undefined(book['pubdate']): + book['date'] = None + else: + book['date'] = strftime(u'%B %Y', book['pubdate'].timetuple()) + + args = self.generateFormatArgs(book) + formatted_title = self.by_series_title_template.format(**args).rstrip() + aTag.insert(0,NavigableString(escape(formatted_title))) + pBookTag.insert(ptc, aTag) + ptc += 1 + + # · + pBookTag.insert(ptc, NavigableString(' · ')) + ptc += 1 + + # Link to author + aTag = Tag(soup, "a") + if self.opts.generate_authors: + aTag['href'] = "%s.html#%s" % ("ByAlphaAuthor", + self.generateAuthorAnchor(escape(' & '.join(book['authors'])))) + aTag.insert(0, NavigableString(' & '.join(book['authors']))) + pBookTag.insert(ptc, aTag) + ptc += 1 + + divTag.insert(dtc, pBookTag) + dtc += 1 + + if not self.__generateForKindle: + # Insert the <h2> tag with book_count at the head + #<h2><a name="byseries" id="byseries"></a>By Series</h2> + pTag = Tag(soup, "p") + pTag['class'] = 'title' + aTag = Tag(soup, "a") + anchor_name = friendly_name.lower() + aTag['name'] = anchor_name.replace(" ","") + pTag.insert(0,aTag) + #h2Tag.insert(1,NavigableString('%s (%d)' % (friendly_name, series_count))) + pTag.insert(1,NavigableString('%s' % friendly_name)) + body.insert(btc,pTag) + btc += 1 + + # Add the divTag to the body + body.insert(btc, divTag) + + # Write the generated file to contentdir + outfile_spec = "%s/BySeries.html" % (self.contentDir) + outfile = open(outfile_spec, 'w') + outfile.write(soup.prettify()) + outfile.close() + self.htmlFileList_1.append("content/BySeries.html") + + def generateHTMLByTags(self): + ''' + Generate individual HTML files for each tag, e.g. Fiction, Nonfiction ... + Note that special tags - have already been filtered from books[] + There may be synonomous tags + ''' + self.updateProgressFullStep("'Genres'") + + self.genre_tags_dict = self.filterDbTags(self.db.all_tags()) + + # Extract books matching filtered_tags + genre_list = [] + for friendly_tag in sorted(self.genre_tags_dict): + #print "\ngenerateHTMLByTags(): looking for books with friendly_tag '%s'" % friendly_tag + # tag_list => { normalized_genre_tag : [{book},{},{}], + # normalized_genre_tag : [{book},{},{}] } + + tag_list = {} + for book in self.booksByAuthor: + # Scan each book for tag matching friendly_tag + if 'tags' in book and friendly_tag in book['tags']: + this_book = {} + this_book['author'] = book['author'] + this_book['title'] = book['title'] + this_book['author_sort'] = capitalize(book['author_sort']) + this_book['read'] = book['read'] + this_book['tags'] = book['tags'] + this_book['id'] = book['id'] + this_book['series'] = book['series'] + this_book['series_index'] = book['series_index'] + this_book['date'] = book['date'] + normalized_tag = self.genre_tags_dict[friendly_tag] + genre_tag_list = [key for genre in genre_list for key in genre] + if normalized_tag in genre_tag_list: + for existing_genre in genre_list: + for key in existing_genre: + new_book = None + if key == normalized_tag: + for book in existing_genre[key]: + if book['title'] == this_book['title']: + new_book = False + break + else: + new_book = True + if new_book: + existing_genre[key].append(this_book) + else: + tag_list[normalized_tag] = [this_book] + genre_list.append(tag_list) + + if self.opts.verbose: + if len(genre_list): + self.opts.log.info(" Genre summary: %d active genre tags used in generating catalog with %d titles" % + (len(genre_list), len(self.booksByTitle))) + + for genre in genre_list: + for key in genre: + self.opts.log.info(" %s: %d %s" % (self.getFriendlyGenreTag(key), + len(genre[key]), + 'titles' if len(genre[key]) > 1 else 'title')) + + + # Write the results + # genre_list = [ {friendly_tag:[{book},{book}]}, {friendly_tag:[{book},{book}]}, ...] + master_genre_list = [] + for genre_tag_set in genre_list: + for (index, genre) in enumerate(genre_tag_set): + #print "genre: %s \t genre_tag_set[genre]: %s" % (genre, genre_tag_set[genre]) + + # Create sorted_authors[0] = friendly, [1] = author_sort for NCX creation + authors = [] + for book in genre_tag_set[genre]: + authors.append((book['author'],book['author_sort'])) + + # authors[] contains a list of all book authors, with multiple entries for multiple books by author + # Create unique_authors with a count of books per author as the third tuple element + books_by_current_author = 1 + current_author = authors[0] + unique_authors = [] + for (i,author) in enumerate(authors): + if author != current_author and i: + unique_authors.append((current_author[0], current_author[1], books_by_current_author)) + current_author = author + books_by_current_author = 1 + elif i==0 and len(authors) == 1: + # Allow for single-book lists + unique_authors.append((current_author[0], current_author[1], books_by_current_author)) + else: + books_by_current_author += 1 + + # Write the genre book list as an article + titles_spanned = self.generateHTMLByGenre(genre, True if index==0 else False, + genre_tag_set[genre], + "%s/Genre_%s.html" % (self.contentDir, + genre)) + + tag_file = "content/Genre_%s.html" % genre + master_genre_list.append({'tag':genre, + 'file':tag_file, + 'authors':unique_authors, + 'books':genre_tag_set[genre], + 'titles_spanned':titles_spanned}) + + self.genres = master_genre_list + + def generateThumbnails(self): + ''' + Generate a thumbnail per cover. If a current thumbnail exists, skip + If a cover doesn't exist, use default + Return list of active thumbs + ''' + self.updateProgressFullStep("'Thumbnails'") + thumbs = ['thumbnail_default.jpg'] + image_dir = "%s/images" % self.catalogPath + for (i,title) in enumerate(self.booksByTitle): + # Update status + self.updateProgressMicroStep("Thumbnail %d of %d" % \ + (i,len(self.booksByTitle)), + i/float(len(self.booksByTitle))) + + thumb_file = 'thumbnail_%d.jpg' % int(title['id']) + thumb_generated = True + valid_cover = True + try: + self.generateThumbnail(title, image_dir, thumb_file) + thumbs.append("thumbnail_%d.jpg" % int(title['id'])) + except: + if 'cover' in title and os.path.exists(title['cover']): + valid_cover = False + self.opts.log.warn(" *** Invalid cover file for '%s'***" % + (title['title'])) + if not self.error: + self.error.append('Invalid cover files') + self.error.append("Warning: invalid cover file for '%s', default cover substituted.\n" % (title['title'])) + + thumb_generated = False + + if not thumb_generated: + self.opts.log.warn(" using default cover for '%s' (%d)" % (title['title'], title['id'])) + # Confirm thumb exists, default is current + default_thumb_fp = os.path.join(image_dir,"thumbnail_default.jpg") + cover = os.path.join(self.catalogPath, "DefaultCover.png") + title['cover'] = cover + + if not os.path.exists(cover): + shutil.copyfile(I('book.png'), cover) + + if os.path.isfile(default_thumb_fp): + # Check to see if default cover is newer than thumbnail + # os.path.getmtime() = modified time + # os.path.ctime() = creation time + cover_timestamp = os.path.getmtime(cover) + thumb_timestamp = os.path.getmtime(default_thumb_fp) + if thumb_timestamp < cover_timestamp: + if False and self.verbose: + self.opts.log.warn("updating thumbnail_default for %s" % title['title']) + self.generateThumbnail(title, image_dir, + "thumbnail_default.jpg" if valid_cover else thumb_file) + else: + if False and self.verbose: + self.opts.log.warn(" generating new thumbnail_default.jpg") + self.generateThumbnail(title, image_dir, + "thumbnail_default.jpg" if valid_cover else thumb_file) + # Clear the book's cover property + title['cover'] = None + + + # Write thumb_width to the file, validating cache contents + # Allows detection of aborted catalog builds + with ZipFile(self.__archive_path, mode='a') as zfw: + zfw.writestr('thumb_width', self.opts.thumb_width) + + self.thumbs = thumbs + + def generateOPF(self): + + self.updateProgressFullStep("Generating OPF") + + header = ''' + <?xml version="1.0" encoding="UTF-8"?> + <package xmlns="http://www.idpf.org/2007/opf" version="2.0" unique-identifier="calibre_id"> + <metadata xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:opf="http://www.idpf.org/2007/opf" xmlns:calibre="http://calibre.kovidgoyal.net/2009/metadata" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"> + <dc:language>en-US</dc:language> + <meta name="calibre:publication_type" content="periodical:default"/> + </metadata> + <manifest></manifest> + <spine toc="ncx"></spine> + <guide></guide> + </package> + ''' + # Add the supplied metadata tags + soup = BeautifulStoneSoup(header, selfClosingTags=['item','itemref', 'reference']) + metadata = soup.find('metadata') + mtc = 0 + + titleTag = Tag(soup, "dc:title") + titleTag.insert(0,self.title) + metadata.insert(mtc, titleTag) + mtc += 1 + + creatorTag = Tag(soup, "dc:creator") + creatorTag.insert(0, self.creator) + metadata.insert(mtc, creatorTag) + mtc += 1 + + # Create the OPF tags + manifest = soup.find('manifest') + mtc = 0 + spine = soup.find('spine') + stc = 0 + guide = soup.find('guide') + + itemTag = Tag(soup, "item") + itemTag['id'] = "ncx" + itemTag['href'] = '%s.ncx' % self.basename + itemTag['media-type'] = "application/x-dtbncx+xml" + manifest.insert(mtc, itemTag) + mtc += 1 + + itemTag = Tag(soup, "item") + itemTag['id'] = 'stylesheet' + itemTag['href'] = self.stylesheet + itemTag['media-type'] = 'text/css' + manifest.insert(mtc, itemTag) + mtc += 1 + + itemTag = Tag(soup, "item") + itemTag['id'] = 'mastheadimage-image' + itemTag['href'] = "images/mastheadImage.gif" + itemTag['media-type'] = 'image/gif' + manifest.insert(mtc, itemTag) + mtc += 1 + + # Write the thumbnail images, descriptions to the manifest + sort_descriptions_by = [] + if self.opts.generate_descriptions: + for thumb in self.thumbs: + itemTag = Tag(soup, "item") + itemTag['href'] = "images/%s" % (thumb) + end = thumb.find('.jpg') + itemTag['id'] = "%s-image" % thumb[:end] + itemTag['media-type'] = 'image/jpeg' + manifest.insert(mtc, itemTag) + mtc += 1 + + # HTML files - add descriptions to manifest and spine + sort_descriptions_by = self.booksByAuthor if self.opts.sort_descriptions_by_author \ + else self.booksByTitle + # Add html_files to manifest and spine + + for file in self.htmlFileList_1: + # By Author, By Title, By Series, + itemTag = Tag(soup, "item") + start = file.find('/') + 1 + end = file.find('.') + itemTag['href'] = file + itemTag['id'] = file[start:end].lower() + itemTag['media-type'] = "application/xhtml+xml" + manifest.insert(mtc, itemTag) + mtc += 1 + + # spine + itemrefTag = Tag(soup, "itemref") + itemrefTag['idref'] = file[start:end].lower() + spine.insert(stc, itemrefTag) + stc += 1 + + # Add genre files to manifest and spine + for genre in self.genres: + if False: self.opts.log.info("adding %s to manifest and spine" % genre['tag']) + itemTag = Tag(soup, "item") + start = genre['file'].find('/') + 1 + end = genre['file'].find('.') + itemTag['href'] = genre['file'] + itemTag['id'] = genre['file'][start:end].lower() + itemTag['media-type'] = "application/xhtml+xml" + manifest.insert(mtc, itemTag) + mtc += 1 + + # spine + itemrefTag = Tag(soup, "itemref") + itemrefTag['idref'] = genre['file'][start:end].lower() + spine.insert(stc, itemrefTag) + stc += 1 + + for file in self.htmlFileList_2: + # By Date Added, By Date Read + itemTag = Tag(soup, "item") + start = file.find('/') + 1 + end = file.find('.') + itemTag['href'] = file + itemTag['id'] = file[start:end].lower() + itemTag['media-type'] = "application/xhtml+xml" + manifest.insert(mtc, itemTag) + mtc += 1 + + # spine + itemrefTag = Tag(soup, "itemref") + itemrefTag['idref'] = file[start:end].lower() + spine.insert(stc, itemrefTag) + stc += 1 + + for book in sort_descriptions_by: + # manifest + itemTag = Tag(soup, "item") + itemTag['href'] = "content/book_%d.html" % int(book['id']) + itemTag['id'] = "book%d" % int(book['id']) + itemTag['media-type'] = "application/xhtml+xml" + manifest.insert(mtc, itemTag) + mtc += 1 + + # spine + itemrefTag = Tag(soup, "itemref") + itemrefTag['idref'] = "book%d" % int(book['id']) + spine.insert(stc, itemrefTag) + stc += 1 + + # Guide + referenceTag = Tag(soup, "reference") + referenceTag['type'] = 'masthead' + referenceTag['title'] = 'mastheadimage-image' + referenceTag['href'] = 'images/mastheadImage.gif' + guide.insert(0,referenceTag) + + # Write the OPF file + outfile = open("%s/%s.opf" % (self.catalogPath, self.basename), 'w') + outfile.write(soup.prettify()) + + def generateNCXHeader(self): + + self.updateProgressFullStep("NCX header") + + header = ''' + <?xml version="1.0" encoding="utf-8"?> + <ncx xmlns="http://www.daisy.org/z3986/2005/ncx/" xmlns:calibre="http://calibre.kovidgoyal.net/2009/metadata" version="2005-1" xml:lang="en"> + </ncx> + ''' + soup = BeautifulStoneSoup(header, selfClosingTags=['content','calibre:meta-img']) + + ncx = soup.find('ncx') + navMapTag = Tag(soup, 'navMap') + navPointTag = Tag(soup, 'navPoint') + navPointTag['class'] = "periodical" + navPointTag['id'] = "title" + navPointTag['playOrder'] = self.playOrder + self.playOrder += 1 + navLabelTag = Tag(soup, 'navLabel') + textTag = Tag(soup, 'text') + textTag.insert(0, NavigableString(self.title)) + navLabelTag.insert(0, textTag) + navPointTag.insert(0, navLabelTag) + + if self.opts.generate_authors: + contentTag = Tag(soup, 'content') + contentTag['src'] = "content/ByAlphaAuthor.html" + navPointTag.insert(1, contentTag) + elif self.opts.generate_titles: + contentTag = Tag(soup, 'content') + contentTag['src'] = "content/ByAlphaTitle.html" + navPointTag.insert(1, contentTag) + elif self.opts.generate_series: + contentTag = Tag(soup, 'content') + contentTag['src'] = "content/BySeries.html" + navPointTag.insert(1, contentTag) + elif self.opts.generate_genres: + contentTag = Tag(soup, 'content') + #contentTag['src'] = "content/ByGenres.html" + contentTag['src'] = "%s" % self.genres[0]['file'] + navPointTag.insert(1, contentTag) + elif self.opts.generate_recently_added: + contentTag = Tag(soup, 'content') + contentTag['src'] = "content/ByDateAdded.html" + navPointTag.insert(1, contentTag) + else: + # Descriptions only + sort_descriptions_by = self.booksByAuthor if self.opts.sort_descriptions_by_author \ + else self.booksByTitle + contentTag = Tag(soup, 'content') + contentTag['src'] = "content/book_%d.html" % int(sort_descriptions_by[0]['id']) + navPointTag.insert(1, contentTag) + + cmiTag = Tag(soup, '%s' % 'calibre:meta-img') + cmiTag['name'] = "mastheadImage" + cmiTag['src'] = "images/mastheadImage.gif" + navPointTag.insert(2,cmiTag) + navMapTag.insert(0,navPointTag) + + ncx.insert(0,navMapTag) + self.ncxSoup = soup + + def generateNCXDescriptions(self, tocTitle): + + self.updateProgressFullStep("NCX 'Descriptions'") + + # --- Construct the 'Books by Title' section --- + ncx_soup = self.ncxSoup + body = ncx_soup.find("navPoint") + btc = len(body.contents) + + # Add the section navPoint + navPointTag = Tag(ncx_soup, 'navPoint') + navPointTag['class'] = "section" + navPointTag['id'] = "bytitle-ID" + navPointTag['playOrder'] = self.playOrder + self.playOrder += 1 + navLabelTag = Tag(ncx_soup, 'navLabel') + textTag = Tag(ncx_soup, 'text') + textTag.insert(0, NavigableString(tocTitle)) + navLabelTag.insert(0, textTag) + nptc = 0 + navPointTag.insert(nptc, navLabelTag) + nptc += 1 + contentTag = Tag(ncx_soup,"content") + contentTag['src'] = "content/book_%d.html" % int(self.booksByTitle[0]['id']) + navPointTag.insert(nptc, contentTag) + nptc += 1 + + # Loop over the titles + sort_descriptions_by = self.booksByAuthor if self.opts.sort_descriptions_by_author \ + else self.booksByTitle + + for book in sort_descriptions_by: + navPointVolumeTag = Tag(ncx_soup, 'navPoint') + navPointVolumeTag['class'] = "article" + navPointVolumeTag['id'] = "book%dID" % int(book['id']) + navPointVolumeTag['playOrder'] = self.playOrder + self.playOrder += 1 + navLabelTag = Tag(ncx_soup, "navLabel") + textTag = Tag(ncx_soup, "text") + if book['series']: + series_index = str(book['series_index']) + if series_index.endswith('.0'): + series_index = series_index[:-2] + if self.generateForKindle: + # Don't include Author for Kindle + textTag.insert(0, NavigableString(self.formatNCXText('%s (%s [%s])' % + (book['title'], book['series'], series_index), dest='title'))) + else: + # Include Author for non-Kindle + textTag.insert(0, NavigableString(self.formatNCXText('%s (%s [%s]) · %s ' % + (book['title'], book['series'], series_index, book['author']), dest='title'))) + else: + if self.generateForKindle: + # Don't include Author for Kindle + title_str = self.formatNCXText('%s' % (book['title']), dest='title') + if self.opts.connected_kindle and book['id'] in self.bookmarked_books: + ''' + dots = int((book['percent_read'] + 5)/10) + dot_string = '+' * dots + empty_dots = '-' * (10 - dots) + title_str += ' %s%s' % (dot_string,empty_dots) + ''' + title_str += '*' + textTag.insert(0, NavigableString(title_str)) + else: + # Include Author for non-Kindle + textTag.insert(0, NavigableString(self.formatNCXText('%s · %s' % \ + (book['title'], book['author']), dest='title'))) + navLabelTag.insert(0,textTag) + navPointVolumeTag.insert(0,navLabelTag) + + contentTag = Tag(ncx_soup, "content") + contentTag['src'] = "content/book_%d.html#book%d" % (int(book['id']), int(book['id'])) + navPointVolumeTag.insert(1, contentTag) + + if self.generateForKindle: + # Add the author tag + cmTag = Tag(ncx_soup, '%s' % 'calibre:meta') + cmTag['name'] = "author" + + if book['date']: + navStr = '%s | %s' % (self.formatNCXText(book['author'], dest='author'), + book['date'].split()[1]) + else: + navStr = '%s' % (self.formatNCXText(book['author'], dest='author')) + + if 'tags' in book and len(book['tags']): + navStr = self.formatNCXText(navStr + ' | ' + ' · '.join(sorted(book['tags'])), dest='author') + cmTag.insert(0, NavigableString(navStr)) + navPointVolumeTag.insert(2, cmTag) + + # Add the description tag + if book['short_description']: + cmTag = Tag(ncx_soup, '%s' % 'calibre:meta') + cmTag['name'] = "description" + cmTag.insert(0, NavigableString(self.formatNCXText(book['short_description'], dest='description'))) + navPointVolumeTag.insert(3, cmTag) + + # Add this volume to the section tag + navPointTag.insert(nptc, navPointVolumeTag) + nptc += 1 + + # Add this section to the body + body.insert(btc, navPointTag) + btc += 1 + + self.ncxSoup = ncx_soup + + def generateNCXBySeries(self, tocTitle): + self.updateProgressFullStep("NCX 'Series'") + + def add_to_series_by_letter(current_series_list): + current_series_list = " • ".join(current_series_list) + current_series_list = self.formatNCXText(current_series_list, dest="description") + series_by_letter.append(current_series_list) + + soup = self.ncxSoup + output = "BySeries" + body = soup.find("navPoint") + btc = len(body.contents) + + # --- Construct the 'Books By Series' section --- + navPointTag = Tag(soup, 'navPoint') + navPointTag['class'] = "section" + navPointTag['id'] = "byseries-ID" + navPointTag['playOrder'] = self.playOrder + self.playOrder += 1 + navLabelTag = Tag(soup, 'navLabel') + textTag = Tag(soup, 'text') + textTag.insert(0, NavigableString(tocTitle)) + navLabelTag.insert(0, textTag) + nptc = 0 + navPointTag.insert(nptc, navLabelTag) + nptc += 1 + contentTag = Tag(soup,"content") + contentTag['src'] = "content/%s.html#section_start" % (output) + navPointTag.insert(nptc, contentTag) + nptc += 1 + + series_by_letter = [] + + # Loop over the series titles, find start of each letter, add description_preview_count books + # Special switch for using different title list + title_list = self.booksBySeries + current_letter = self.letter_or_symbol(self.generateSortTitle(title_list[0]['series'])[0]) + title_letters = [current_letter] + current_series_list = [] + current_series = "" + for book in title_list: + sort_title = self.generateSortTitle(book['series']) + if self.letter_or_symbol(sort_title[0]) != current_letter: + # Save the old list + add_to_series_by_letter(current_series_list) + + # Start the new list + current_letter = self.letter_or_symbol(sort_title[0]) + title_letters.append(current_letter) + current_series = book['series'] + current_series_list = [book['series']] + else: + if len(current_series_list) < self.descriptionClip and \ + book['series'] != current_series : + current_series = book['series'] + current_series_list.append(book['series']) + + # Add the last book list + add_to_series_by_letter(current_series_list) + + # Add *article* entries for each populated series title letter + for (i,books) in enumerate(series_by_letter): + navPointByLetterTag = Tag(soup, 'navPoint') + navPointByLetterTag['class'] = "article" + navPointByLetterTag['id'] = "%sSeries-ID" % (title_letters[i].upper()) + navPointTag['playOrder'] = self.playOrder + self.playOrder += 1 + navLabelTag = Tag(soup, 'navLabel') + textTag = Tag(soup, 'text') + textTag.insert(0, NavigableString(u"Series beginning with %s" % \ + (title_letters[i] if len(title_letters[i])>1 else "'" + title_letters[i] + "'"))) + navLabelTag.insert(0, textTag) + navPointByLetterTag.insert(0,navLabelTag) + contentTag = Tag(soup, 'content') + contentTag['src'] = "content/%s.html#%s_series" % (output, title_letters[i]) + navPointByLetterTag.insert(1,contentTag) + + if self.generateForKindle: + cmTag = Tag(soup, '%s' % 'calibre:meta') + cmTag['name'] = "description" + cmTag.insert(0, NavigableString(self.formatNCXText(books, dest='description'))) + navPointByLetterTag.insert(2, cmTag) + + navPointTag.insert(nptc, navPointByLetterTag) + nptc += 1 + + # Add this section to the body + body.insert(btc, navPointTag) + btc += 1 + + self.ncxSoup = soup + + def generateNCXByTitle(self, tocTitle): + self.updateProgressFullStep("NCX 'Titles'") + + def add_to_books_by_letter(current_book_list): + current_book_list = " • ".join(current_book_list) + current_book_list = self.formatNCXText(current_book_list, dest="description") + books_by_letter.append(current_book_list) + + soup = self.ncxSoup + output = "ByAlphaTitle" + body = soup.find("navPoint") + btc = len(body.contents) + + # --- Construct the 'Books By Title' section --- + navPointTag = Tag(soup, 'navPoint') + navPointTag['class'] = "section" + navPointTag['id'] = "byalphatitle-ID" + navPointTag['playOrder'] = self.playOrder + self.playOrder += 1 + navLabelTag = Tag(soup, 'navLabel') + textTag = Tag(soup, 'text') + textTag.insert(0, NavigableString(tocTitle)) + navLabelTag.insert(0, textTag) + nptc = 0 + navPointTag.insert(nptc, navLabelTag) + nptc += 1 + contentTag = Tag(soup,"content") + contentTag['src'] = "content/%s.html#section_start" % (output) + navPointTag.insert(nptc, contentTag) + nptc += 1 + + books_by_letter = [] + + # Loop over the titles, find start of each letter, add description_preview_count books + # Special switch for using different title list + if self.useSeriesPrefixInTitlesSection: + title_list = self.booksByTitle + else: + title_list = self.booksByTitle_noSeriesPrefix + current_letter = self.letter_or_symbol(title_list[0]['title_sort'][0]) + title_letters = [current_letter] + current_book_list = [] + current_book = "" + for book in title_list: + if self.letter_or_symbol(book['title_sort'][0]) != current_letter: + # Save the old list + add_to_books_by_letter(current_book_list) + + # Start the new list + current_letter = self.letter_or_symbol(book['title_sort'][0]) + title_letters.append(current_letter) + current_book = book['title'] + current_book_list = [book['title']] + else: + if len(current_book_list) < self.descriptionClip and \ + book['title'] != current_book : + current_book = book['title'] + current_book_list.append(book['title']) + + # Add the last book list + add_to_books_by_letter(current_book_list) + + # Add *article* entries for each populated title letter + for (i,books) in enumerate(books_by_letter): + navPointByLetterTag = Tag(soup, 'navPoint') + navPointByLetterTag['class'] = "article" + navPointByLetterTag['id'] = "%sTitles-ID" % (title_letters[i].upper()) + navPointTag['playOrder'] = self.playOrder + self.playOrder += 1 + navLabelTag = Tag(soup, 'navLabel') + textTag = Tag(soup, 'text') + textTag.insert(0, NavigableString(u"Titles beginning with %s" % \ + (title_letters[i] if len(title_letters[i])>1 else "'" + title_letters[i] + "'"))) + navLabelTag.insert(0, textTag) + navPointByLetterTag.insert(0,navLabelTag) + contentTag = Tag(soup, 'content') + contentTag['src'] = "content/%s.html#%s" % (output, title_letters[i]) + navPointByLetterTag.insert(1,contentTag) + + if self.generateForKindle: + cmTag = Tag(soup, '%s' % 'calibre:meta') + cmTag['name'] = "description" + cmTag.insert(0, NavigableString(self.formatNCXText(books, dest='description'))) + navPointByLetterTag.insert(2, cmTag) + + navPointTag.insert(nptc, navPointByLetterTag) + nptc += 1 + + # Add this section to the body + body.insert(btc, navPointTag) + btc += 1 + + self.ncxSoup = soup + + def generateNCXByAuthor(self, tocTitle): + self.updateProgressFullStep("NCX 'Authors'") + + def add_to_author_list(current_author_list, current_letter): + current_author_list = " • ".join(current_author_list) + current_author_list = self.formatNCXText(current_author_list, dest="description") + master_author_list.append((current_author_list, current_letter)) + + soup = self.ncxSoup + HTML_file = "content/ByAlphaAuthor.html" + body = soup.find("navPoint") + btc = len(body.contents) + + # --- Construct the 'Books By Author' *section* --- + navPointTag = Tag(soup, 'navPoint') + navPointTag['class'] = "section" + file_ID = "%s" % tocTitle.lower() + file_ID = file_ID.replace(" ","") + navPointTag['id'] = "%s-ID" % file_ID + navPointTag['playOrder'] = self.playOrder + self.playOrder += 1 + navLabelTag = Tag(soup, 'navLabel') + textTag = Tag(soup, 'text') + textTag.insert(0, NavigableString('%s' % tocTitle)) + navLabelTag.insert(0, textTag) + nptc = 0 + navPointTag.insert(nptc, navLabelTag) + nptc += 1 + contentTag = Tag(soup,"content") + contentTag['src'] = "%s#section_start" % HTML_file + navPointTag.insert(nptc, contentTag) + nptc += 1 + + # Create an NCX article entry for each populated author index letter + # Loop over the sorted_authors list, find start of each letter, + # add description_preview_count artists + # self.authors[0]:friendly [1]:author_sort [2]:book_count + master_author_list = [] + # self.authors[0][1][0] = Initial letter of author_sort[0] + current_letter = self.letter_or_symbol(self.authors[0][1][0]) + current_author_list = [] + for author in self.authors: + if self.letter_or_symbol(author[1][0]) != current_letter: + # Save the old list + add_to_author_list(current_author_list, current_letter) + + # Start the new list + current_letter = self.letter_or_symbol(author[1][0]) + current_author_list = [author[0]] + else: + if len(current_author_list) < self.descriptionClip: + current_author_list.append(author[0]) + + # Add the last author list + add_to_author_list(current_author_list, current_letter) + + # Add *article* entries for each populated author initial letter + # master_author_list{}: [0]:author list [1]:Initial letter + for authors_by_letter in master_author_list: + navPointByLetterTag = Tag(soup, 'navPoint') + navPointByLetterTag['class'] = "article" + navPointByLetterTag['id'] = "%sauthors-ID" % (authors_by_letter[1]) + navPointTag['playOrder'] = self.playOrder + self.playOrder += 1 + navLabelTag = Tag(soup, 'navLabel') + textTag = Tag(soup, 'text') + textTag.insert(0, NavigableString("Authors beginning with '%s'" % (authors_by_letter[1]))) + navLabelTag.insert(0, textTag) + navPointByLetterTag.insert(0,navLabelTag) + contentTag = Tag(soup, 'content') + contentTag['src'] = "%s#%sauthors" % (HTML_file, authors_by_letter[1]) + + navPointByLetterTag.insert(1,contentTag) + + if self.generateForKindle: + cmTag = Tag(soup, '%s' % 'calibre:meta') + cmTag['name'] = "description" + cmTag.insert(0, NavigableString(authors_by_letter[0])) + navPointByLetterTag.insert(2, cmTag) + + navPointTag.insert(nptc, navPointByLetterTag) + nptc += 1 + + # Add this section to the body + body.insert(btc, navPointTag) + btc += 1 + + self.ncxSoup = soup + + def generateNCXByDateAdded(self, tocTitle): + self.updateProgressFullStep("NCX 'Recently Added'") + + def add_to_master_month_list(current_titles_list): + book_count = len(current_titles_list) + current_titles_list = " • ".join(current_titles_list) + current_titles_list = self.formatNCXText(current_titles_list, dest='description') + master_month_list.append((current_titles_list, current_date, book_count)) + + def add_to_master_date_range_list(current_titles_list): + book_count = len(current_titles_list) + current_titles_list = " • ".join(current_titles_list) + current_titles_list = self.formatNCXText(current_titles_list, dest='description') + master_date_range_list.append((current_titles_list, date_range, book_count)) + + soup = self.ncxSoup + HTML_file = "content/ByDateAdded.html" + body = soup.find("navPoint") + btc = len(body.contents) + + # --- Construct the 'Recently Added' *section* --- + navPointTag = Tag(soup, 'navPoint') + navPointTag['class'] = "section" + file_ID = "%s" % tocTitle.lower() + file_ID = file_ID.replace(" ","") + navPointTag['id'] = "%s-ID" % file_ID + navPointTag['playOrder'] = self.playOrder + self.playOrder += 1 + navLabelTag = Tag(soup, 'navLabel') + textTag = Tag(soup, 'text') + textTag.insert(0, NavigableString('%s' % tocTitle)) + navLabelTag.insert(0, textTag) + nptc = 0 + navPointTag.insert(nptc, navLabelTag) + nptc += 1 + contentTag = Tag(soup,"content") + contentTag['src'] = "%s#section_start" % HTML_file + navPointTag.insert(nptc, contentTag) + nptc += 1 + + # Create an NCX article entry for each date range + current_titles_list = [] + master_date_range_list = [] + today = datetime.datetime.now() + today_time = datetime.datetime(today.year, today.month, today.day) + for (i,date) in enumerate(self.DATE_RANGE): + if i: + date_range = '%d to %d days ago' % (self.DATE_RANGE[i-1], self.DATE_RANGE[i]) + else: + date_range = 'Last %d days' % (self.DATE_RANGE[i]) + date_range_limit = self.DATE_RANGE[i] + for book in self.booksByDateRange: + book_time = datetime.datetime(book['timestamp'].year, book['timestamp'].month, book['timestamp'].day) + if (today_time-book_time).days <= date_range_limit: + #print "generateNCXByDateAdded: %s added %d days ago" % (book['title'], (today_time-book_time).days) + current_titles_list.append(book['title']) + else: + break + if current_titles_list: + add_to_master_date_range_list(current_titles_list) + current_titles_list = [book['title']] + + # Add *article* entries for each populated date range + # master_date_range_list{}: [0]:titles list [1]:datestr + for books_by_date_range in master_date_range_list: + navPointByDateRangeTag = Tag(soup, 'navPoint') + navPointByDateRangeTag['class'] = "article" + navPointByDateRangeTag['id'] = "%s-ID" % books_by_date_range[1].replace(' ','') + navPointTag['playOrder'] = self.playOrder + self.playOrder += 1 + navLabelTag = Tag(soup, 'navLabel') + textTag = Tag(soup, 'text') + textTag.insert(0, NavigableString(books_by_date_range[1])) + navLabelTag.insert(0, textTag) + navPointByDateRangeTag.insert(0,navLabelTag) + contentTag = Tag(soup, 'content') + contentTag['src'] = "%s#bda_%s" % (HTML_file, + books_by_date_range[1].replace(' ','')) + + navPointByDateRangeTag.insert(1,contentTag) + + if self.generateForKindle: + cmTag = Tag(soup, '%s' % 'calibre:meta') + cmTag['name'] = "description" + cmTag.insert(0, NavigableString(books_by_date_range[0])) + navPointByDateRangeTag.insert(2, cmTag) + + cmTag = Tag(soup, '%s' % 'calibre:meta') + cmTag['name'] = "author" + navStr = '%d titles' % books_by_date_range[2] if books_by_date_range[2] > 1 else \ + '%d title' % books_by_date_range[2] + cmTag.insert(0, NavigableString(navStr)) + navPointByDateRangeTag.insert(3, cmTag) + + navPointTag.insert(nptc, navPointByDateRangeTag) + nptc += 1 + + + + # Create an NCX article entry for each populated month + # Loop over the booksByDate list, find start of each month, + # add description_preview_count titles + # master_month_list(list,date,count) + current_titles_list = [] + master_month_list = [] + current_date = self.booksByMonth[0]['timestamp'] + + for book in self.booksByMonth: + if book['timestamp'].month != current_date.month or \ + book['timestamp'].year != current_date.year: + # Save the old lists + add_to_master_month_list(current_titles_list) + + # Start the new list + current_date = book['timestamp'].date() + current_titles_list = [book['title']] + else: + current_titles_list.append(book['title']) + + # Add the last month list + add_to_master_month_list(current_titles_list) + + # Add *article* entries for each populated month + # master_months_list{}: [0]:titles list [1]:date + for books_by_month in master_month_list: + datestr = strftime(u'%B %Y', books_by_month[1].timetuple()) + navPointByMonthTag = Tag(soup, 'navPoint') + navPointByMonthTag['class'] = "article" + navPointByMonthTag['id'] = "bda_%s-%s-ID" % (books_by_month[1].year,books_by_month[1].month ) + navPointTag['playOrder'] = self.playOrder + self.playOrder += 1 + navLabelTag = Tag(soup, 'navLabel') + textTag = Tag(soup, 'text') + textTag.insert(0, NavigableString(datestr)) + navLabelTag.insert(0, textTag) + navPointByMonthTag.insert(0,navLabelTag) + contentTag = Tag(soup, 'content') + contentTag['src'] = "%s#bda_%s-%s" % (HTML_file, + books_by_month[1].year,books_by_month[1].month) + + navPointByMonthTag.insert(1,contentTag) + + if self.generateForKindle: + cmTag = Tag(soup, '%s' % 'calibre:meta') + cmTag['name'] = "description" + cmTag.insert(0, NavigableString(books_by_month[0])) + navPointByMonthTag.insert(2, cmTag) + + cmTag = Tag(soup, '%s' % 'calibre:meta') + cmTag['name'] = "author" + navStr = '%d titles' % books_by_month[2] if books_by_month[2] > 1 else \ + '%d title' % books_by_month[2] + cmTag.insert(0, NavigableString(navStr)) + navPointByMonthTag.insert(3, cmTag) + + navPointTag.insert(nptc, navPointByMonthTag) + nptc += 1 + + # Add this section to the body + body.insert(btc, navPointTag) + btc += 1 + self.ncxSoup = soup + + def generateNCXByDateRead(self, tocTitle): + self.updateProgressFullStep("NCX 'Recently Read'") + if not self.booksByDateRead: + return + + def add_to_master_day_list(current_titles_list): + book_count = len(current_titles_list) + current_titles_list = " • ".join(current_titles_list) + current_titles_list = self.formatNCXText(current_titles_list, dest='description') + master_day_list.append((current_titles_list, current_date, book_count)) + + def add_to_master_date_range_list(current_titles_list): + book_count = len(current_titles_list) + current_titles_list = " • ".join(current_titles_list) + current_titles_list = self.formatNCXText(current_titles_list, dest='description') + master_date_range_list.append((current_titles_list, date_range, book_count)) + + soup = self.ncxSoup + HTML_file = "content/ByDateRead.html" + body = soup.find("navPoint") + btc = len(body.contents) + + # --- Construct the 'Recently Read' *section* --- + navPointTag = Tag(soup, 'navPoint') + navPointTag['class'] = "section" + file_ID = "%s" % tocTitle.lower() + file_ID = file_ID.replace(" ","") + navPointTag['id'] = "%s-ID" % file_ID + navPointTag['playOrder'] = self.playOrder + self.playOrder += 1 + navLabelTag = Tag(soup, 'navLabel') + textTag = Tag(soup, 'text') + textTag.insert(0, NavigableString('%s' % tocTitle)) + navLabelTag.insert(0, textTag) + nptc = 0 + navPointTag.insert(nptc, navLabelTag) + nptc += 1 + contentTag = Tag(soup,"content") + contentTag['src'] = "%s#section_start" % HTML_file + navPointTag.insert(nptc, contentTag) + nptc += 1 + + # Create an NCX article entry for each date range + current_titles_list = [] + master_date_range_list = [] + today = datetime.datetime.now() + today_time = datetime.datetime(today.year, today.month, today.day) + for (i,date) in enumerate(self.DATE_RANGE): + if i: + date_range = '%d to %d days ago' % (self.DATE_RANGE[i-1], self.DATE_RANGE[i]) + else: + date_range = 'Last %d days' % (self.DATE_RANGE[i]) + date_range_limit = self.DATE_RANGE[i] + for book in self.booksByDateRead: + bookmark_time = datetime.datetime.utcfromtimestamp(book['bookmark_timestamp']) + if (today_time-bookmark_time).days <= date_range_limit: + #print "generateNCXByDateAdded: %s added %d days ago" % (book['title'], (today_time-book_time).days) + current_titles_list.append(book['title']) + else: + break + if current_titles_list: + add_to_master_date_range_list(current_titles_list) + current_titles_list = [book['title']] + + # Create an NCX article entry for each populated day + # Loop over the booksByDate list, find start of each month, + # add description_preview_count titles + # master_month_list(list,date,count) + current_titles_list = [] + master_day_list = [] + current_date = datetime.datetime.utcfromtimestamp(self.booksByDateRead[0]['bookmark_timestamp']) + + for book in self.booksByDateRead: + bookmark_time = datetime.datetime.utcfromtimestamp(book['bookmark_timestamp']) + if bookmark_time.day != current_date.day or \ + bookmark_time.month != current_date.month or \ + bookmark_time.year != current_date.year: + # Save the old lists + add_to_master_day_list(current_titles_list) + + # Start the new list + current_date = datetime.datetime.utcfromtimestamp(book['bookmark_timestamp']).date() + current_titles_list = [book['title']] + else: + current_titles_list.append(book['title']) + + # Add the last day list + add_to_master_day_list(current_titles_list) + + # Add *article* entries for each populated day + # master_day_list{}: [0]:titles list [1]:date + for books_by_day in master_day_list: + datestr = strftime(u'%A, %B %d', books_by_day[1].timetuple()) + navPointByDayTag = Tag(soup, 'navPoint') + navPointByDayTag['class'] = "article" + navPointByDayTag['id'] = "bdr_%s-%s-%sID" % (books_by_day[1].year, + books_by_day[1].month, + books_by_day[1].day ) + navPointTag['playOrder'] = self.playOrder + self.playOrder += 1 + navLabelTag = Tag(soup, 'navLabel') + textTag = Tag(soup, 'text') + textTag.insert(0, NavigableString(datestr)) + navLabelTag.insert(0, textTag) + navPointByDayTag.insert(0,navLabelTag) + contentTag = Tag(soup, 'content') + contentTag['src'] = "%s#bdr_%s-%s-%s" % (HTML_file, + books_by_day[1].year, + books_by_day[1].month, + books_by_day[1].day) + + navPointByDayTag.insert(1,contentTag) + + if self.generateForKindle: + cmTag = Tag(soup, '%s' % 'calibre:meta') + cmTag['name'] = "description" + cmTag.insert(0, NavigableString(books_by_day[0])) + navPointByDayTag.insert(2, cmTag) + + cmTag = Tag(soup, '%s' % 'calibre:meta') + cmTag['name'] = "author" + navStr = '%d titles' % books_by_day[2] if books_by_day[2] > 1 else \ + '%d title' % books_by_day[2] + cmTag.insert(0, NavigableString(navStr)) + navPointByDayTag.insert(3, cmTag) + + navPointTag.insert(nptc, navPointByDayTag) + nptc += 1 + + # Add this section to the body + body.insert(btc, navPointTag) + btc += 1 + self.ncxSoup = soup + + def generateNCXByGenre(self, tocTitle): + # Create an NCX section for 'By Genre' + # Add each genre as an article + # 'tag', 'file', 'authors' + + self.updateProgressFullStep("NCX 'Genres'") + + if not len(self.genres): + self.opts.log.warn(" No genres found in tags.\n" + " No Genre section added to Catalog") + return + + ncx_soup = self.ncxSoup + body = ncx_soup.find("navPoint") + btc = len(body.contents) + + # --- Construct the 'Books By Genre' *section* --- + navPointTag = Tag(ncx_soup, 'navPoint') + navPointTag['class'] = "section" + file_ID = "%s" % tocTitle.lower() + file_ID = file_ID.replace(" ","") + navPointTag['id'] = "%s-ID" % file_ID + navPointTag['playOrder'] = self.playOrder + self.playOrder += 1 + navLabelTag = Tag(ncx_soup, 'navLabel') + textTag = Tag(ncx_soup, 'text') + # textTag.insert(0, NavigableString('%s (%d)' % (section_title, len(genre_list)))) + textTag.insert(0, NavigableString('%s' % tocTitle)) + navLabelTag.insert(0, textTag) + nptc = 0 + navPointTag.insert(nptc, navLabelTag) + nptc += 1 + contentTag = Tag(ncx_soup,"content") + contentTag['src'] = "content/Genre_%s.html#section_start" % self.genres[0]['tag'] + navPointTag.insert(nptc, contentTag) + nptc += 1 + + for genre in self.genres: + # Add an article for each genre + navPointVolumeTag = Tag(ncx_soup, 'navPoint') + navPointVolumeTag['class'] = "article" + navPointVolumeTag['id'] = "genre-%s-ID" % genre['tag'] + navPointVolumeTag['playOrder'] = self.playOrder + self.playOrder += 1 + navLabelTag = Tag(ncx_soup, "navLabel") + textTag = Tag(ncx_soup, "text") + + # GwR *** Can this be optimized? + normalized_tag = None + for friendly_tag in self.genre_tags_dict: + if self.genre_tags_dict[friendly_tag] == genre['tag']: + normalized_tag = self.genre_tags_dict[friendly_tag] + break + textTag.insert(0, self.formatNCXText(NavigableString(friendly_tag), dest='description')) + navLabelTag.insert(0,textTag) + navPointVolumeTag.insert(0,navLabelTag) + contentTag = Tag(ncx_soup, "content") + contentTag['src'] = "content/Genre_%s.html#Genre_%s" % (normalized_tag, normalized_tag) + navPointVolumeTag.insert(1, contentTag) + + if self.generateForKindle: + # Build the author tag + cmTag = Tag(ncx_soup, '%s' % 'calibre:meta') + cmTag['name'] = "author" + # First - Last author + + if len(genre['titles_spanned']) > 1 : + author_range = "%s - %s" % (genre['titles_spanned'][0][0], genre['titles_spanned'][1][0]) + else : + author_range = "%s" % (genre['titles_spanned'][0][0]) + + cmTag.insert(0, NavigableString(author_range)) + navPointVolumeTag.insert(2, cmTag) + + # Build the description tag + cmTag = Tag(ncx_soup, '%s' % 'calibre:meta') + cmTag['name'] = "description" + + if False: + # Form 1: Titles spanned + if len(genre['titles_spanned']) > 1: + title_range = "%s -\n%s" % (genre['titles_spanned'][0][1], genre['titles_spanned'][1][1]) + else: + title_range = "%s" % (genre['titles_spanned'][0][1]) + cmTag.insert(0, NavigableString(self.formatNCXText(title_range, dest='description'))) + else: + # Form 2: title • title • title ... + titles = [] + for title in genre['books']: + titles.append(title['title']) + titles = sorted(titles, key=lambda x:(self.generateSortTitle(x),self.generateSortTitle(x))) + titles_list = self.generateShortDescription(u" • ".join(titles), dest="description") + cmTag.insert(0, NavigableString(self.formatNCXText(titles_list, dest='description'))) + + navPointVolumeTag.insert(3, cmTag) + + # Add this volume to the section tag + navPointTag.insert(nptc, navPointVolumeTag) + nptc += 1 + + # Add this section to the body + body.insert(btc, navPointTag) + btc += 1 + self.ncxSoup = ncx_soup + + def writeNCX(self): + self.updateProgressFullStep("Saving NCX") + + outfile = open("%s/%s.ncx" % (self.catalogPath, self.basename), 'w') + outfile.write(self.ncxSoup.prettify()) + + + # ======================== Helpers ======================== + def author_to_author_sort(self, author): + tokens = author.split() + tokens = tokens[-1:] + tokens[:-1] + if len(tokens) > 1: + tokens[0] += ',' + return ' '.join(tokens).capitalize() + + def booksByAuthorSorter_author_sort(self, book): + ''' + Sort non-series books before series books + ''' + if not book['series']: + key = '%s %s' % (capitalize(book['author_sort']), + capitalize(book['title_sort'])) + else: + index = book['series_index'] + integer = int(index) + fraction = index-integer + series_index = '%04d%s' % (integer, str('%0.4f' % fraction).lstrip('0')) + key = '%s ~%s %s' % (capitalize(book['author_sort']), + self.generateSortTitle(book['series']), + series_index) + return key + + def booksByAuthorSorter_author(self, book): + ''' + Sort non-series books before series books + ''' + if not book['series']: + key = '%s %s' % (self.author_to_author_sort(book['author']), + capitalize(book['title_sort'])) + else: + index = book['series_index'] + integer = int(index) + fraction = index-integer + series_index = '%04d%s' % (integer, str('%0.4f' % fraction).lstrip('0')) + key = '%s ~%s %s' % (self.author_to_author_sort(book['author']), + self.generateSortTitle(book['series']), + series_index) + return key + + def calculateThumbnailSize(self): + ''' Calculate thumbnail dimensions based on device DPI. Scale Kindle by 50% ''' + from calibre.customize.ui import output_profiles + for x in output_profiles(): + if x.short_name == self.opts.output_profile: + # aspect ratio: 3:4 + self.thumbWidth = x.dpi * float(self.opts.thumb_width) + self.thumbHeight = self.thumbWidth * 1.33 + if 'kindle' in x.short_name and self.opts.fmt == 'mobi': + # Kindle DPI appears to be off by a factor of 2 + self.thumbWidth = self.thumbWidth/2 + self.thumbHeight = self.thumbHeight/2 + break + if True and self.verbose: + self.opts.log(" DPI = %d; thumbnail dimensions: %d x %d" % \ + (x.dpi, self.thumbWidth, self.thumbHeight)) + + def convertHTMLEntities(self, s): + matches = re.findall("&#\d+;", s) + if len(matches) > 0: + hits = set(matches) + for hit in hits: + name = hit[2:-1] + try: + entnum = int(name) + s = s.replace(hit, unichr(entnum)) + except ValueError: + pass + + matches = re.findall("&\w+;", s) + hits = set(matches) + amp = "&" + if amp in hits: + hits.remove(amp) + for hit in hits: + name = hit[1:-1] + if htmlentitydefs.name2codepoint.has_key(name): + s = s.replace(hit, unichr(htmlentitydefs.name2codepoint[name])) + s = s.replace(amp, "&") + return s + + def createDirectoryStructure(self): + catalogPath = self.catalogPath + self.cleanUp() + + if not os.path.isdir(catalogPath): + os.makedirs(catalogPath) + + # Create /content and /images + content_path = catalogPath + "/content" + if not os.path.isdir(content_path): + os.makedirs(content_path) + images_path = catalogPath + "/images" + if not os.path.isdir(images_path): + os.makedirs(images_path) + + def discoverReadStatus(self, record): + ''' + Given a field:pattern spec, discover if this book marked as read + + if field == tag, scan tags for pattern + if custom field, try regex match for pattern + This allows maximum flexibility with fields of type + datatype bool: #field_name:True + datatype text: #field_name:<string> + datatype datetime: #field_name:.* + + ''' + # Legacy handling of special 'read' tag + field = self.__read_book_marker['field'] + pat = self.__read_book_marker['pattern'] + if field == 'tag' and pat in record['tags']: + return True + + field_contents = self.__db.get_field(record['id'], + field, + index_is_id=True) + if field_contents: + try: + if re.search(pat, unicode(field_contents), + re.IGNORECASE) is not None: + return True + except: + # Compiling of pat failed, ignore it + pass + + return False + + def filterDbTags(self, tags): + # Remove the special marker tags from the database's tag list, + # return sorted list of normalized genre tags + + def format_tag_list(tags, indent=5, line_break=70, header='Tag list'): + def next_tag(sorted_tags): + for (i, tag) in enumerate(sorted_tags): + if i < len(tags) - 1: + yield tag + ", " + else: + yield tag + + ans = '%s%d %s:\n' % (' ' * indent, len(tags), header) + ans += ' ' * (indent + 1) + out_str = '' + sorted_tags = sorted(tags) + for tag in next_tag(sorted_tags): + out_str += tag + if len(out_str) >= line_break: + ans += out_str + '\n' + out_str = ' ' * (indent + 1) + return ans + out_str + + normalized_tags = [] + friendly_tags = [] + excluded_tags = [] + for tag in tags: + if tag in self.markerTags: + excluded_tags.append(tag) + continue + if re.search(self.opts.exclude_genre, tag): + excluded_tags.append(tag) + continue + if tag == ' ': + continue + + normalized_tags.append(re.sub('\W','',tag).lower()) + friendly_tags.append(tag) + + genre_tags_dict = dict(zip(friendly_tags,normalized_tags)) + + # Test for multiple genres resolving to same normalized form + normalized_set = set(normalized_tags) + for normalized in normalized_set: + if normalized_tags.count(normalized) > 1: + self.opts.log.warn(" Warning: multiple tags resolving to genre '%s':" % normalized) + for key in genre_tags_dict: + if genre_tags_dict[key] == normalized: + self.opts.log.warn(" %s" % key) + if self.verbose: + self.opts.log.info('%s' % format_tag_list(genre_tags_dict, header="enabled genre tags in database")) + self.opts.log.info('%s' % format_tag_list(excluded_tags, header="excluded genre tags")) + + return genre_tags_dict + + def formatNCXText(self, description, dest=None): + # Kindle TOC descriptions won't render certain characters + # Fix up + massaged = unicode(BeautifulStoneSoup(description, convertEntities=BeautifulStoneSoup.HTML_ENTITIES)) + + # Replace '&' with '&' + massaged = re.sub("&","&", massaged) + + if massaged.strip() and dest: + #print traceback.print_stack(limit=3) + return self.generateShortDescription(massaged.strip(), dest=dest) + else: + return None + + def generateAuthorAnchor(self, author): + # Strip white space to '' + return re.sub("\W","", author) + + def generateFormatArgs(self, book): + series_index = str(book['series_index']) + if series_index.endswith('.0'): + series_index = series_index[:-2] + args = dict( + title = book['title'], + series = book['series'], + series_index = series_index, + rating = self.generateRatingString(book), + rating_parens = '(%s)' % self.generateRatingString(book) if 'rating' in book else '', + pubyear = book['date'].split()[1] if book['date'] else '', + pubyear_parens = "(%s)" % book['date'].split()[1] if book['date'] else '') + return args + + def generateHTMLByGenre(self, genre, section_head, books, outfile): + # Write an HTML file of this genre's book list + # Return a list with [(first_author, first_book), (last_author, last_book)] + + soup = self.generateHTMLGenreHeader(genre) + body = soup.find('body') + + btc = 0 + + # Insert section tag if this is the section start - first article only + if section_head: + aTag = Tag(soup,'a') + aTag['name'] = 'section_start' + body.insert(btc, aTag) + btc += 1 + + # Create an anchor from the tag + aTag = Tag(soup, 'a') + aTag['name'] = "Genre_%s" % genre + body.insert(btc,aTag) + btc += 1 + + titleTag = body.find(attrs={'class':'title'}) + titleTag.insert(0,NavigableString('%s' % escape(self.getFriendlyGenreTag(genre)))) + + # Insert the books by author list + divTag = body.find(attrs={'class':'authors'}) + dtc = 0 + + current_author = '' + current_series = None + for book in books: + if book['author'] != current_author: + # Start a new author with link + current_author = book['author'] + non_series_books = 0 + current_series = None + pAuthorTag = Tag(soup, "p") + pAuthorTag['class'] = "author_index" + aTag = Tag(soup, "a") + if self.opts.generate_authors: + aTag['href'] = "%s.html#%s" % ("ByAlphaAuthor", self.generateAuthorAnchor(book['author'])) + aTag.insert(0, book['author']) + pAuthorTag.insert(0,aTag) + divTag.insert(dtc,pAuthorTag) + dtc += 1 + + # Check for series + if book['series'] and book['series'] != current_series: + # Start a new series + current_series = book['series'] + pSeriesTag = Tag(soup,'p') + pSeriesTag['class'] = "series" + if self.opts.generate_series: + aTag = Tag(soup,'a') + aTag['href'] = "%s.html#%s_series" % ('BySeries', + re.sub('\W','',book['series']).lower()) + aTag.insert(0, book['series']) + pSeriesTag.insert(0, aTag) + else: + pSeriesTag.insert(0,NavigableString('%s' % book['series'])) + divTag.insert(dtc,pSeriesTag) + dtc += 1 + + if current_series and not book['series']: + current_series = None + + # Add books + pBookTag = Tag(soup, "p") + ptc = 0 + + # book with read|reading|unread symbol or wishlist item + if self.opts.wishlist_tag in book.get('tags', []): + pBookTag['class'] = "wishlist_item" + pBookTag.insert(ptc,NavigableString(self.MISSING_SYMBOL)) + ptc += 1 + else: + if book['read']: + # check mark + pBookTag.insert(ptc,NavigableString(self.READ_SYMBOL)) + pBookTag['class'] = "read_book" + ptc += 1 + elif book['id'] in self.bookmarked_books: + pBookTag.insert(ptc,NavigableString(self.READING_SYMBOL)) + pBookTag['class'] = "read_book" + ptc += 1 + else: + # hidden check mark + pBookTag['class'] = "unread_book" + pBookTag.insert(ptc,NavigableString(self.NOT_READ_SYMBOL)) + ptc += 1 + + # Add the book title + aTag = Tag(soup, "a") + if self.opts.generate_descriptions: + aTag['href'] = "book_%d.html" % (int(float(book['id']))) + + # Generate the title from the template + args = self.generateFormatArgs(book) + if current_series: + #aTag.insert(0,escape(book['title'][len(book['series'])+1:])) + formatted_title = self.by_genres_series_title_template.format(**args).rstrip() + else: + #aTag.insert(0,escape(book['title'])) + formatted_title = self.by_genres_normal_title_template.format(**args).rstrip() + non_series_books += 1 + aTag.insert(0,NavigableString(escape(formatted_title))) + + pBookTag.insert(ptc, aTag) + ptc += 1 + + divTag.insert(dtc, pBookTag) + dtc += 1 + + # Write the generated file to contentdir + outfile = open(outfile, 'w') + outfile.write(soup.prettify()) + outfile.close() + + if len(books) > 1: + titles_spanned = [(books[0]['author'],books[0]['title']), (books[-1]['author'],books[-1]['title'])] + else: + titles_spanned = [(books[0]['author'],books[0]['title'])] + + return titles_spanned + + def generateHTMLDescriptionHeader(self, book): + ''' + Generate description header from template + ''' + from calibre.ebooks.oeb.base import XHTML_NS + + def generate_html(): + args = dict( + author=author, + author_prefix=author_prefix, + comments=comments, + css=css, + formats=formats, + genres=genres, + note_content=note_content, + note_source=note_source, + pubdate=pubdate, + publisher=publisher, + pubmonth=pubmonth, + pubyear=pubyear, + rating=rating, + series=series, + series_index=series_index, + thumb=thumb, + title=title, + title_str=title_str, + xmlns=XHTML_NS, + ) + + generated_html = P('catalog/template.xhtml', + data=True).decode('utf-8').format(**args) + generated_html = substitute_entites(generated_html) + return BeautifulSoup(generated_html) + + # Generate the template arguments + css = P('catalog/stylesheet.css', data=True).decode('utf-8') + title_str = title = escape(book['title']) + series = '' + series_index = '' + if book['series']: + series = escape(book['series']) + series_index = str(book['series_index']) + if series_index.endswith('.0'): + series_index = series_index[:-2] + + # Author, author_prefix (read|reading|none symbol or missing symbol) + author = book['author'] + if self.opts.wishlist_tag in book.get('tags', []): + author_prefix = self.MISSING_SYMBOL + " by " + else: + if book['read']: + author_prefix = self.READ_SYMBOL + " by " + elif self.opts.connected_kindle and book['id'] in self.bookmarked_books: + author_prefix = self.READING_SYMBOL + " by " + else: + author_prefix = "by " + + # Genres + genres = '' + if 'tags' in book: + _soup = BeautifulSoup('') + genresTag = Tag(_soup,'p') + gtc = 0 + for (i, tag) in enumerate(sorted(book.get('tags', []))): + aTag = Tag(_soup,'a') + if self.opts.generate_genres: + aTag['href'] = "Genre_%s.html" % re.sub("\W","",tag.lower()) + aTag.insert(0,escape(NavigableString(tag))) + genresTag.insert(gtc, aTag) + gtc += 1 + if i < len(book['tags'])-1: + genresTag.insert(gtc, NavigableString(' · ')) + gtc += 1 + genres = genresTag.renderContents() + + # Formats + formats = [] + if 'formats' in book: + for format in sorted(book['formats']): + formats.append(format.rpartition('.')[2].upper()) + formats = ' · '.join(formats) + + # Date of publication + if book['date']: + pubdate = book['date'] + pubmonth, pubyear = pubdate.split() + else: + pubdate = pubyear = pubmonth = '' + + # Thumb + _soup = BeautifulSoup('<html>',selfClosingTags=['img']) + thumb = Tag(_soup,"img") + if 'cover' in book and book['cover']: + thumb['src'] = "../images/thumbnail_%d.jpg" % int(book['id']) + else: + thumb['src'] = "../images/thumbnail_default.jpg" + thumb['alt'] = "cover thumbnail" + + # Publisher + publisher = ' ' + if 'publisher' in book: + publisher = book['publisher'] + + # Rating + stars = int(book['rating']) / 2 + rating = '' + if stars: + star_string = self.FULL_RATING_SYMBOL * stars + empty_stars = self.EMPTY_RATING_SYMBOL * (5 - stars) + rating = '%s%s <br/>' % (star_string,empty_stars) + + # Notes + note_source = '' + note_content = '' + if 'notes' in book: + note_source = book['notes']['source'] + note_content = book['notes']['content'] + + # Comments + comments = '' + if 'description' in book and book['description'] > '': + comments = book['description'] + + + # >>>> Populate the template <<<< + soup = generate_html() + + + # >>>> Post-process the template <<<< + body = soup.find('body') + btc = 0 + # Insert the title anchor for inbound links + aTag = Tag(soup, "a") + aTag['name'] = "book%d" % int(book['id']) + body.insert(btc, aTag) + btc += 1 + + # Insert the link to the series or remove <a class="series"> + aTag = body.find('a', attrs={'class':'series_id'}) + if aTag: + if book['series']: + if self.opts.generate_series: + aTag['href'] = "%s.html#%s_series" % ('BySeries', + re.sub('\W','',book['series']).lower()) + else: + aTag.extract() + + # Insert the author link + aTag = body.find('a', attrs={'class':'author'}) + if self.opts.generate_authors and aTag: + aTag['href'] = "%s.html#%s" % ("ByAlphaAuthor", + self.generateAuthorAnchor(book['author'])) + + if publisher == ' ': + publisherTag = body.find('td', attrs={'class':'publisher'}) + if publisherTag: + publisherTag.contents[0].replaceWith(' ') + + if not genres: + genresTag = body.find('p',attrs={'class':'genres'}) + if genresTag: + genresTag.extract() + + if not formats: + formatsTag = body.find('p',attrs={'class':'formats'}) + if formatsTag: + formatsTag.extract() + + if note_content == '': + tdTag = body.find('td', attrs={'class':'notes'}) + if tdTag: + tdTag.contents[0].replaceWith(' ') + + emptyTags = body.findAll('td', attrs={'class':'empty'}) + for mt in emptyTags: + newEmptyTag = Tag(BeautifulSoup(),'td') + newEmptyTag.insert(0,NavigableString(' ')) + mt.replaceWith(newEmptyTag) + + if False: + print soup.prettify() + return soup + + def generateHTMLEmptyHeader(self, title): + header = ''' + <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd"> + <html xmlns="http://www.w3.org/1999/xhtml" xmlns:calibre="http://calibre.kovidgoyal.net/2009/metadata"> + <head> + <meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /> + <link rel="stylesheet" type="text/css" href="stylesheet.css" media="screen" /> + <title> + + + + + ''' + # Insert the supplied title + soup = BeautifulSoup(header) + titleTag = soup.find('title') + titleTag.insert(0,NavigableString(title)) + return soup + + def generateHTMLGenreHeader(self, title): + header = ''' + + + + + + + + +

+
+ + + ''' + # Insert the supplied title + soup = BeautifulSoup(header) + titleTag = soup.find('title') + titleTag.insert(0,escape(NavigableString(title))) + return soup + + def generateMastheadImage(self, out_path): + from calibre.ebooks.conversion.config import load_defaults + from calibre.utils.fonts import fontconfig + font_path = default_font = P('fonts/liberation/LiberationSerif-Bold.ttf') + recs = load_defaults('mobi_output') + masthead_font_family = recs.get('masthead_font', 'Default') + + if masthead_font_family != 'Default': + masthead_font = fontconfig.files_for_family(masthead_font_family) + # Assume 'normal' always in dict, else use default + # {'normal': (path_to_font, friendly name)} + if 'normal' in masthead_font: + font_path = masthead_font['normal'][0] + + if not font_path or not os.access(font_path, os.R_OK): + font_path = default_font + + MI_WIDTH = 600 + MI_HEIGHT = 60 + + try: + from PIL import Image, ImageDraw, ImageFont + Image, ImageDraw, ImageFont + except ImportError: + import Image, ImageDraw, ImageFont + + img = Image.new('RGB', (MI_WIDTH, MI_HEIGHT), 'white') + draw = ImageDraw.Draw(img) + try: + font = ImageFont.truetype(font_path, 48) + except: + self.opts.log.error(" Failed to load user-specifed font '%s'" % font_path) + font = ImageFont.truetype(default_font, 48) + text = self.title.encode('utf-8') + width, height = draw.textsize(text, font=font) + left = max(int((MI_WIDTH - width)/2.), 0) + top = max(int((MI_HEIGHT - height)/2.), 0) + draw.text((left, top), text, fill=(0,0,0), font=font) + img.save(open(out_path, 'wb'), 'GIF') + + def generateRatingString(self, book): + rating = '' + try: + if 'rating' in book: + stars = int(book['rating']) / 2 + if stars: + star_string = self.FULL_RATING_SYMBOL * stars + empty_stars = self.EMPTY_RATING_SYMBOL * (5 - stars) + rating = '%s%s' % (star_string,empty_stars) + except: + # Rating could be None + pass + return rating + + def generateShortDescription(self, description, dest=None): + # Truncate the description, on word boundaries if necessary + # Possible destinations: + # description NCX summary + # title NCX title + # author NCX author + + def shortDescription(description, limit): + short_description = "" + words = description.split() + for word in words: + short_description += word + " " + if len(short_description) > limit: + short_description += "..." + return short_description + + if not description: + return None + + if dest == 'title': + # No truncation for titles, let the device deal with it + return description + elif dest == 'author': + if self.authorClip and len(description) < self.authorClip: + return description + else: + return shortDescription(description, self.authorClip) + elif dest == 'description': + if self.descriptionClip and len(description) < self.descriptionClip: + return description + else: + return shortDescription(description, self.descriptionClip) + else: + print " returning description with unspecified destination '%s'" % description + raise RuntimeError + + def generateSortTitle(self, title): + ''' + Generate a string suitable for sorting from the title + Ignore leading stop words + Optionally convert leading numbers to strings + ''' + from calibre.ebooks.metadata import title_sort + from calibre.library.catalogs.utils import NumberToText + + # Strip stop words + title_words = title_sort(title).split() + translated = [] + + for (i,word) in enumerate(title_words): + # Leading numbers optionally translated to text equivalent + # Capitalize leading sort word + if i==0: + # *** Keep this code in case we need to restore numbers_as_text *** + if False: + #if self.opts.numbers_as_text and re.match('[0-9]+',word[0]): + translated.append(NumberToText(word).text.capitalize()) + else: + if re.match('[0-9]+',word[0]): + word = word.replace(',','') + suffix = re.search('[\D]', word) + if suffix: + word = '%10.0f%s' % (float(word[:suffix.start()]),word[suffix.start():]) + else: + word = '%10.0f' % (float(word)) + + # If leading char > 'A', insert symbol as leading forcing lower sort + # '/' sorts below numbers, g + if self.letter_or_symbol(word[0]) != word[0]: + if word[0] > 'A' or (ord('9') < ord(word[0]) < ord('A')) : + translated.append('/') + translated.append(capitalize(word)) + + else: + if re.search('[0-9]+',word[0]): + word = word.replace(',','') + suffix = re.search('[\D]', word) + if suffix: + word = '%10.0f%s' % (float(word[:suffix.start()]),word[suffix.start():]) + else: + word = '%10.0f' % (float(word)) + translated.append(word) + return ' '.join(translated) + + def generateThumbnail(self, title, image_dir, thumb_file): + ''' + Thumbs are cached with the full cover's crc. If the crc doesn't + match, the cover has been changed since the thumb was cached and needs + to be replaced. + ''' + + def open_archive(mode='r'): + try: + return ZipFile(self.__archive_path, mode=mode) + except: + # Happens on windows if the file is opened by another + # process + pass + + # Generate crc for current cover + #self.opts.log.info(" generateThumbnail():") + with open(title['cover'], 'rb') as f: + data = f.read() + cover_crc = hex(zlib.crc32(data)) + + # Test cache for uuid + zf = open_archive() + if zf is not None: + with zf: + try: + zf.getinfo(title['uuid']+cover_crc) + except: + pass + else: + # uuid found in cache with matching crc + thumb_data = zf.read(title['uuid']+cover_crc) + with open(os.path.join(image_dir, thumb_file), 'wb') as f: + f.write(thumb_data) + return + + + # Save thumb for catalog + thumb_data = thumbnail(data, + width=self.thumbWidth, height=self.thumbHeight)[-1] + with open(os.path.join(image_dir, thumb_file), 'wb') as f: + f.write(thumb_data) + + # Save thumb to archive + if zf is not None: # Ensure that the read succeeded + # If we failed to open the zip file for reading, + # we dont know if it contained the thumb or not + zf = open_archive('a') + if zf is not None: + with zf: + zf.writestr(title['uuid']+cover_crc, thumb_data) + + def getFriendlyGenreTag(self, genre): + # Find the first instance of friendly_tag matching genre + for friendly_tag in self.genre_tags_dict: + if self.genre_tags_dict[friendly_tag] == genre: + return friendly_tag + + def getMarkerTags(self): + ''' Return a list of special marker tags to be excluded from genre list ''' + markerTags = [] + markerTags.extend(self.opts.exclude_tags.split(',')) + return markerTags + + def letter_or_symbol(self,char): + if not re.search('[a-zA-Z]',char): + return 'Symbols' + else: + return char + + def markdownComments(self, comments): + ''' + Convert random comment text to normalized, xml-legal block of

s + 'plain text' returns as +

plain text

+ + 'plain text with minimal markup' returns as +

plain text with minimal markup

+ + '

pre-formatted text

returns untouched + + 'A line of text\n\nFollowed by a line of text' returns as +

A line of text

+

Followed by a line of text

+ + 'A line of text.\nA second line of text.\rA third line of text' returns as +

A line of text.
A second line of text.
A third line of text.

+ + '...end of a paragraph.Somehow the break was lost...' returns as +

...end of a paragraph.

+

Somehow the break was lost...

+ + Deprecated HTML returns as HTML via BeautifulSoup() + + ''' + # Hackish - ignoring sentences ending or beginning in numbers to avoid + # confusion with decimal points. + + # Explode lost CRs to \n\n + for lost_cr in re.finditer('([a-z])([\.\?!])([A-Z])',comments): + comments = comments.replace(lost_cr.group(), + '%s%s\n\n%s' % (lost_cr.group(1), + lost_cr.group(2), + lost_cr.group(3))) + # Extract pre-built elements - annotations, etc. + if not isinstance(comments, unicode): + comments = comments.decode('utf-8', 'replace') + soup = BeautifulSoup(comments) + elems = soup.findAll('div') + for elem in elems: + elem.extract() + + # Reconstruct comments w/o
s + comments = soup.renderContents(None) + + # Convert \n\n to

s + if re.search('\n\n', comments): + soup = BeautifulSoup() + split_ps = comments.split(u'\n\n') + tsc = 0 + for p in split_ps: + pTag = Tag(soup,'p') + pTag.insert(0,p) + soup.insert(tsc,pTag) + tsc += 1 + comments = soup.renderContents(None) + + # Convert solo returns to
+ comments = re.sub('[\r\n]','
', comments) + + # Convert two hypens to emdash + comments = re.sub('--','—',comments) + soup = BeautifulSoup(comments) + result = BeautifulSoup() + rtc = 0 + open_pTag = False + + all_tokens = list(soup.contents) + for token in all_tokens: + if type(token) is NavigableString: + if not open_pTag: + pTag = Tag(result,'p') + open_pTag = True + ptc = 0 + pTag.insert(ptc,prepare_string_for_xml(token)) + ptc += 1 + + elif token.name in ['br','b','i','em']: + if not open_pTag: + pTag = Tag(result,'p') + open_pTag = True + ptc = 0 + pTag.insert(ptc, token) + ptc += 1 + + else: + if open_pTag: + result.insert(rtc, pTag) + rtc += 1 + open_pTag = False + ptc = 0 + # Clean up NavigableStrings for xml + sub_tokens = list(token.contents) + for sub_token in sub_tokens: + if type(sub_token) is NavigableString: + sub_token.replaceWith(prepare_string_for_xml(sub_token)) + result.insert(rtc, token) + rtc += 1 + + if open_pTag: + result.insert(rtc, pTag) + rtc += 1 + + paras = result.findAll('p') + for p in paras: + p['class'] = 'description' + + # Add back

elems initially removed + for elem in elems: + result.insert(rtc,elem) + rtc += 1 + + return result.renderContents(encoding=None) + + def mergeComments(self, record): + ''' + merge ['description'] with custom field contents to be displayed in Descriptions + ''' + merged = '' + if record['description']: + addendum = self.__db.get_field(record['id'], + self.__merge_comments['field'], + index_is_id=True) + if addendum is None: + addendum = '' + include_hr = eval(self.__merge_comments['hr']) + if self.__merge_comments['position'] == 'before': + merged = addendum + if include_hr: + merged += '
' + else: + merged += '\n' + merged += record['description'] + else: + merged = record['description'] + if include_hr: + merged += '
' + else: + merged += '\n' + merged += addendum + else: + # Return the custom field contents + merged = self.__db.get_field(record['id'], + self.__merge_comments['field'], + index_is_id=True) + + return merged + + def processExclusions(self, data_set): + ''' + Remove excluded entries + ''' + field, pat = self.opts.exclude_book_marker.split(':') + if pat == '': + return data_set + filtered_data_set = [] + for record in data_set: + field_contents = self.__db.get_field(record['id'], + field, + index_is_id=True) + if field_contents: + if re.search(pat, unicode(field_contents), + re.IGNORECASE) is not None: + continue + filtered_data_set.append(record) + + return filtered_data_set + + def processSpecialTags(self, tags, this_title, opts): + tag_list = [] + for tag in tags: + tag = self.convertHTMLEntities(tag) + if re.search(opts.exclude_genre, tag): + continue + elif self.__read_book_marker['field'] == 'tag' and \ + tag == self.__read_book_marker['pattern']: + # remove 'read' tag + continue + else: + tag_list.append(tag) + return tag_list + + def updateProgressFullStep(self, description): + self.currentStep += 1 + self.progressString = description + self.progressInt = float((self.currentStep-1)/self.totalSteps) + self.reporter(self.progressInt, self.progressString) + if self.opts.cli_environment: + self.opts.log(u"%3.0f%% %s" % (self.progressInt*100, self.progressString)) + + def updateProgressMicroStep(self, description, micro_step_pct): + step_range = 100/self.totalSteps + self.progressString = description + coarse_progress = float((self.currentStep-1)/self.totalSteps) + fine_progress = float((micro_step_pct*step_range)/100) + self.progressInt = coarse_progress + fine_progress + self.reporter(self.progressInt, self.progressString) + diff --git a/src/calibre/library/catalogs/utils.py b/src/calibre/library/catalogs/utils.py new file mode 100644 index 0000000000..7818268ded --- /dev/null +++ b/src/calibre/library/catalogs/utils.py @@ -0,0 +1,217 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai + +__license__ = 'GPL v3' +__copyright__ = '2010, Greg Riker' +__docformat__ = 'restructuredtext en' + +import re + +from calibre import prints +from calibre.utils.logging import default_log as log + +class NumberToText(object): # {{{ + ''' + Converts numbers to text + 4.56 => four point fifty-six + 456 => four hundred fifty-six + 4:56 => four fifty-six + ''' + ORDINALS = ['zeroth','first','second','third','fourth','fifth','sixth','seventh','eighth','ninth'] + lessThanTwenty = ["","one","two","three","four","five","six","seven","eight","nine", + "ten","eleven","twelve","thirteen","fourteen","fifteen","sixteen","seventeen", + "eighteen","nineteen"] + tens = ["","","twenty","thirty","forty","fifty","sixty","seventy","eighty","ninety"] + hundreds = ["","one","two","three","four","five","six","seven","eight","nine"] + + def __init__(self, number, verbose=False): + self.number = number + self.number_as_float = 0.0 + self.text = '' + self.verbose = verbose + self.log = log + self.numberTranslate() + + def stringFromInt(self, intToTranslate): + # Convert intToTranslate to string + # intToTranslate is a three-digit number + + tensComponentString = "" + hundredsComponent = intToTranslate - (intToTranslate % 100) + tensComponent = intToTranslate % 100 + + # Build the hundreds component + if hundredsComponent: + hundredsComponentString = "%s hundred" % self.hundreds[hundredsComponent/100] + else: + hundredsComponentString = "" + + # Build the tens component + if tensComponent < 20: + tensComponentString = self.lessThanTwenty[tensComponent] + else: + tensPart = "" + onesPart = "" + + # Get the tens part + tensPart = self.tens[tensComponent / 10] + onesPart = self.lessThanTwenty[tensComponent % 10] + + if intToTranslate % 10: + tensComponentString = "%s-%s" % (tensPart, onesPart) + else: + tensComponentString = "%s" % tensPart + + # Concatenate the results + result = '' + if hundredsComponent and not tensComponent: + result = hundredsComponentString + elif not hundredsComponent and tensComponent: + result = tensComponentString + elif hundredsComponent and tensComponent: + result = hundredsComponentString + " " + tensComponentString + else: + prints(" NumberToText.stringFromInt(): empty result translating %d" % intToTranslate) + return result + + def numberTranslate(self): + hundredsNumber = 0 + thousandsNumber = 0 + hundredsString = "" + thousandsString = "" + resultString = "" + self.suffix = '' + + if self.verbose: self.log("numberTranslate(): %s" % self.number) + + # Special case ordinals + if re.search('[st|nd|rd|th]',self.number): + self.number = re.sub(',','',self.number) + ordinal_suffix = re.search('[\D]', self.number) + ordinal_number = re.sub('\D','',re.sub(',','',self.number)) + if self.verbose: self.log("Ordinal: %s" % ordinal_number) + self.number_as_float = ordinal_number + self.suffix = self.number[ordinal_suffix.start():] + if int(ordinal_number) > 9: + # Some typos (e.g., 'twentyth'), acceptable + self.text = '%s' % (NumberToText(ordinal_number).text) + else: + self.text = '%s' % (self.ORDINALS[int(ordinal_number)]) + + # Test for time + elif re.search(':',self.number): + if self.verbose: self.log("Time: %s" % self.number) + self.number_as_float = re.sub(':','.',self.number) + time_strings = self.number.split(":") + hours = NumberToText(time_strings[0]).text + minutes = NumberToText(time_strings[1]).text + self.text = '%s-%s' % (hours.capitalize(), minutes) + + # Test for % + elif re.search('%', self.number): + if self.verbose: self.log("Percent: %s" % self.number) + self.number_as_float = self.number.split('%')[0] + self.text = NumberToText(self.number.replace('%',' percent')).text + + # Test for decimal + elif re.search('\.',self.number): + if self.verbose: self.log("Decimal: %s" % self.number) + self.number_as_float = self.number + decimal_strings = self.number.split(".") + left = NumberToText(decimal_strings[0]).text + right = NumberToText(decimal_strings[1]).text + self.text = '%s point %s' % (left.capitalize(), right) + + # Test for hypenated + elif re.search('-', self.number): + if self.verbose: self.log("Hyphenated: %s" % self.number) + self.number_as_float = self.number.split('-')[0] + strings = self.number.split('-') + if re.search('[0-9]+', strings[0]): + left = NumberToText(strings[0]).text + right = strings[1] + else: + left = strings[0] + right = NumberToText(strings[1]).text + self.text = '%s-%s' % (left, right) + + # Test for only commas and numbers + elif re.search(',', self.number) and not re.search('[^0-9,]',self.number): + if self.verbose: self.log("Comma(s): %s" % self.number) + self.number_as_float = re.sub(',','',self.number) + self.text = NumberToText(self.number_as_float).text + + # Test for hybrid e.g., 'K2, 2nd, 10@10' + elif re.search('[\D]+', self.number): + if self.verbose: self.log("Hybrid: %s" % self.number) + # Split the token into number/text + number_position = re.search('\d',self.number).start() + text_position = re.search('\D',self.number).start() + if number_position < text_position: + number = self.number[:text_position] + text = self.number[text_position:] + self.text = '%s%s' % (NumberToText(number).text,text) + else: + text = self.number[:number_position] + number = self.number[number_position:] + self.text = '%s%s' % (text, NumberToText(number).text) + + else: + if self.verbose: self.log("Clean: %s" % self.number) + try: + self.float_as_number = float(self.number) + number = int(self.number) + except: + return + + if number > 10**9: + self.text = "%d out of range" % number + return + + if number == 10**9: + self.text = "one billion" + else : + # Isolate the three-digit number groups + millionsNumber = number/10**6 + thousandsNumber = (number - (millionsNumber * 10**6))/10**3 + hundredsNumber = number - (millionsNumber * 10**6) - (thousandsNumber * 10**3) + if self.verbose: + print "Converting %s %s %s" % (millionsNumber, thousandsNumber, hundredsNumber) + + # Convert hundredsNumber + if hundredsNumber : + hundredsString = self.stringFromInt(hundredsNumber) + + # Convert thousandsNumber + if thousandsNumber: + if number > 1099 and number < 2000: + resultString = '%s %s' % (self.lessThanTwenty[number/100], + self.stringFromInt(number % 100)) + self.text = resultString.strip().capitalize() + return + else: + thousandsString = self.stringFromInt(thousandsNumber) + + # Convert millionsNumber + if millionsNumber: + millionsString = self.stringFromInt(millionsNumber) + + # Concatenate the strings + resultString = '' + if millionsNumber: + resultString += "%s million " % millionsString + + if thousandsNumber: + resultString += "%s thousand " % thousandsString + + if hundredsNumber: + resultString += "%s" % hundredsString + + if not millionsNumber and not thousandsNumber and not hundredsNumber: + resultString = "zero" + + if self.verbose: + self.log(u'resultString: %s' % resultString) + self.text = resultString.strip().capitalize() +# }}} + From 12d81e629ef977ae62b143fe4c40542fe0c346e5 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 6 Feb 2012 12:30:45 +0530 Subject: [PATCH 28/31] ... --- src/calibre/devices/apple/driver.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/calibre/devices/apple/driver.py b/src/calibre/devices/apple/driver.py index b938a6a0a2..5ed389a7b1 100644 --- a/src/calibre/devices/apple/driver.py +++ b/src/calibre/devices/apple/driver.py @@ -97,8 +97,6 @@ class AppleOpenFeedback(OpenFeedback): return Dialog(parent, self) -from PIL import Image as PILImage -from lxml import etree if isosx: try: @@ -1356,6 +1354,8 @@ class ITUNES(DriverBase): assumes pythoncom wrapper for db_added as of iTunes 9.2, iBooks 1.1, can't set artwork for PDF files via automation ''' + from PIL import Image as PILImage + if DEBUG: self.log.info(" ITUNES._cover_to_thumb()") @@ -1936,6 +1936,7 @@ class ITUNES(DriverBase): cache_dir = os.path.join(config_dir, 'caches', 'itunes') as of iTunes 9.2, iBooks 1.1, can't set artwork for PDF files via automation ''' + from PIL import Image as PILImage if not self.settings().extra_customization[self.CACHE_COVERS]: thumb_data = None @@ -2659,6 +2660,7 @@ class ITUNES(DriverBase): ''' ''' from calibre.ebooks.metadata.epub import set_metadata + from lxml import etree if DEBUG: self.log.info(" ITUNES._update_epub_metadata()") From 9fb9e89e91bde46a04b81ea70f462e064eab8afc Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 6 Feb 2012 14:01:40 +0530 Subject: [PATCH 29/31] More delay load optimizations. Time taken to import all builtin plugins now reduced by 60% from before I started. --- src/calibre/customize/builtins.py | 30 + src/calibre/customize/profiles.py | 2 +- src/calibre/devices/apple/driver.py | 596 +++++++++--------- src/calibre/devices/cybook/t2b.py | 6 +- src/calibre/devices/kobo/bookmark.py | 3 +- src/calibre/devices/kobo/driver.py | 8 +- src/calibre/devices/prst1/driver.py | 10 +- src/calibre/ebooks/chardet.py | 6 +- .../ebooks/conversion/plugins/pdf_input.py | 5 +- src/calibre/ebooks/epub/fix/epubcheck.py | 3 +- src/calibre/ebooks/metadata/book/base.py | 2 +- .../ebooks/metadata/book/json_codec.py | 3 +- src/calibre/ebooks/metadata/sources/amazon.py | 28 +- src/calibre/ebooks/metadata/sources/base.py | 2 +- src/calibre/ebooks/metadata/sources/douban.py | 47 +- src/calibre/ebooks/metadata/sources/google.py | 42 +- src/calibre/ebooks/metadata/sources/isbndb.py | 8 +- .../ebooks/metadata/sources/overdrive.py | 15 +- src/calibre/ebooks/metadata/sources/ozon.py | 13 +- src/calibre/library/catalogs/bibtex.py | 2 +- src/calibre/library/catalogs/csv_xml.py | 2 +- src/calibre/library/catalogs/epub_mobi.py | 3 +- src/calibre/utils/date.py | 2 +- 23 files changed, 459 insertions(+), 379 deletions(-) diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py index 0897199289..fd1ccd0349 100644 --- a/src/calibre/customize/builtins.py +++ b/src/calibre/customize/builtins.py @@ -1645,3 +1645,33 @@ plugins += [ # }}} +if __name__ == '__main__': + # Test load speed + import subprocess, textwrap + try: + subprocess.check_call(['python', '-c', textwrap.dedent( + ''' + from __future__ import print_function + import time, sys, init_calibre + st = time.time() + import calibre.customize.builtins + t = time.time() - st + ret = 0 + + for x in ('lxml', 'calibre.ebooks.BeautifulSoup', 'uuid', + 'calibre.utils.terminfo', 'calibre.utils.magick', 'PIL', 'Image', + 'sqlite3', 'mechanize', 'httplib', 'xml'): + if x in sys.modules: + ret = 1 + print (x, 'has been loaded by a plugin') + if ret: + print ('\\nA good way to trackdown what is loading something is to run' + ' python -c "import init_calibre; import calibre.customize.builtins"') + print() + print ('Time taken to import all plugins: %.2f'%t) + sys.exit(ret) + + ''')]) + except subprocess.CalledProcessError: + raise SystemExit(1) + diff --git a/src/calibre/customize/profiles.py b/src/calibre/customize/profiles.py index 8bb0e55f5e..f6ed6ce3ec 100644 --- a/src/calibre/customize/profiles.py +++ b/src/calibre/customize/profiles.py @@ -5,7 +5,6 @@ __copyright__ = '2009, Kovid Goyal ' __docformat__ = 'restructuredtext en' from itertools import izip -from xml.sax.saxutils import escape from calibre.customize import Plugin as _Plugin @@ -268,6 +267,7 @@ class OutputProfile(Plugin): @classmethod def tags_to_string(cls, tags): + from xml.sax.saxutils import escape return escape(', '.join(tags)) class iPadOutput(OutputProfile): diff --git a/src/calibre/devices/apple/driver.py b/src/calibre/devices/apple/driver.py index 5ed389a7b1..524a62224f 100644 --- a/src/calibre/devices/apple/driver.py +++ b/src/calibre/devices/apple/driver.py @@ -17,7 +17,6 @@ from calibre.ebooks.metadata import authors_to_string, MetaInformation, title_so from calibre.ebooks.metadata.book.base import Metadata from calibre.utils.config import config_dir, dynamic, prefs from calibre.utils.date import now, parse_date -from calibre.utils.logging import Log from calibre.utils.zipfile import ZipFile def strftime(fmt='%Y/%m/%d %H:%M:%S', dt=None): @@ -30,13 +29,18 @@ def strftime(fmt='%Y/%m/%d %H:%M:%S', dt=None): except: return _strftime(fmt, now().timetuple()) - +_log = None +def logger(): + global _log + if _log is None: + from calibre.utils.logging import ThreadSafeLog + _log = ThreadSafeLog() + return _log class AppleOpenFeedback(OpenFeedback): def __init__(self, plugin): OpenFeedback.__init__(self, u'') - self.log = plugin.log self.plugin = plugin def custom_dialog(self, parent): @@ -86,12 +90,13 @@ class AppleOpenFeedback(OpenFeedback): self.finished.connect(self.do_it) def do_it(self, return_code): + from calibre.utils.logging import default_log if return_code == self.Accepted: - self.cd.log.info(" Apple driver ENABLED") + default_log.info(" Apple driver ENABLED") dynamic[confirm_config_name(self.cd.plugin.DISPLAY_DISABLE_DIALOG)] = False else: from calibre.customize.ui import disable_plugin - self.cd.log.info(" Apple driver DISABLED") + default_log.info(" Apple driver DISABLED") disable_plugin(self.cd.plugin) return Dialog(parent, self) @@ -303,7 +308,6 @@ class ITUNES(DriverBase): iTunes= None iTunes_local_storage = None library_orphans = None - log = Log() manual_sync_mode = False path_template = 'iTunes/%s - %s.%s' plugboards = None @@ -329,7 +333,7 @@ class ITUNES(DriverBase): L{books}(oncard='cardb')). ''' if DEBUG: - self.log.info("ITUNES.add_books_to_metadata()") + logger().info("ITUNES.add_books_to_metadata()") task_count = float(len(self.update_list)) @@ -343,10 +347,10 @@ class ITUNES(DriverBase): for (j,p_book) in enumerate(self.update_list): if False: if isosx: - self.log.info(" looking for '%s' by %s uuid:%s" % + logger().info(" looking for '%s' by %s uuid:%s" % (p_book['title'],p_book['author'], p_book['uuid'])) elif iswindows: - self.log.info(" looking for '%s' by %s (%s)" % + logger().info(" looking for '%s' by %s (%s)" % (p_book['title'],p_book['author'], p_book['uuid'])) # Purge the booklist, self.cached_books @@ -356,10 +360,10 @@ class ITUNES(DriverBase): booklists[0].pop(i) if False: if isosx: - self.log.info(" removing old %s %s from booklists[0]" % + logger().info(" removing old %s %s from booklists[0]" % (p_book['title'], str(p_book['lib_book'])[-9:])) elif iswindows: - self.log.info(" removing old '%s' from booklists[0]" % + logger().info(" removing old '%s' from booklists[0]" % (p_book['title'])) # If >1 matching uuid, remove old title @@ -389,7 +393,7 @@ class ITUNES(DriverBase): # for new_book in metadata[0]: for new_book in locations[0]: if DEBUG: - self.log.info(" adding '%s' by '%s' to booklists[0]" % + logger().info(" adding '%s' by '%s' to booklists[0]" % (new_book.title, new_book.author)) booklists[0].append(new_book) @@ -414,15 +418,15 @@ class ITUNES(DriverBase): """ if not oncard: if DEBUG: - self.log.info("ITUNES:books():") + logger().info("ITUNES:books():") if self.settings().extra_customization[self.CACHE_COVERS]: - self.log.info(" Cover fetching/caching enabled") + logger().info(" Cover fetching/caching enabled") else: - self.log.info(" Cover fetching/caching disabled") + logger().info(" Cover fetching/caching disabled") # Fetch a list of books from iPod device connected to iTunes if 'iPod' in self.sources: - booklist = BookList(self.log) + booklist = BookList(logger()) cached_books = {} if isosx: @@ -513,7 +517,7 @@ class ITUNES(DriverBase): self._dump_cached_books('returning from books()',indent=2) return booklist else: - return BookList(self.log) + return BookList(logger()) def can_handle(self, device_info, debug=False): ''' @@ -550,7 +554,7 @@ class ITUNES(DriverBase): # We need to know if iTunes sees the iPad # It may have been ejected if DEBUG: - self.log.info("ITUNES.can_handle()") + logger().info("ITUNES.can_handle()") self._launch_iTunes() self.sources = self._get_sources() @@ -563,15 +567,15 @@ class ITUNES(DriverBase): attempts -= 1 time.sleep(0.5) if DEBUG: - self.log.warning(" waiting for connected iPad, attempt #%d" % (10 - attempts)) + logger().warning(" waiting for connected iPad, attempt #%d" % (10 - attempts)) else: if DEBUG: - self.log.info(' found connected iPad') + logger().info(' found connected iPad') break else: # iTunes running, but not connected iPad if DEBUG: - self.log.info(' self.ejected = True') + logger().info(' self.ejected = True') self.ejected = True return False @@ -605,26 +609,26 @@ class ITUNES(DriverBase): sys.stdout.write('.') sys.stdout.flush() if DEBUG: - self.log.info('ITUNES.can_handle_windows:\n confirming connected iPad') + logger().info('ITUNES.can_handle_windows:\n confirming connected iPad') self.ejected = False self._discover_manual_sync_mode() return True else: if DEBUG: - self.log.info("ITUNES.can_handle_windows():\n device ejected") + logger().info("ITUNES.can_handle_windows():\n device ejected") self.ejected = True return False except: # iTunes connection failed, probably not running anymore - self.log.error("ITUNES.can_handle_windows():\n lost connection to iTunes") + logger().error("ITUNES.can_handle_windows():\n lost connection to iTunes") return False finally: pythoncom.CoUninitialize() else: if DEBUG: - self.log.info("ITUNES:can_handle_windows():\n Launching iTunes") + logger().info("ITUNES:can_handle_windows():\n Launching iTunes") try: pythoncom.CoInitialize() @@ -639,19 +643,19 @@ class ITUNES(DriverBase): attempts -= 1 time.sleep(0.5) if DEBUG: - self.log.warning(" waiting for connected iPad, attempt #%d" % (10 - attempts)) + logger().warning(" waiting for connected iPad, attempt #%d" % (10 - attempts)) else: if DEBUG: - self.log.info(' found connected iPad in iTunes') + logger().info(' found connected iPad in iTunes') break else: # iTunes running, but not connected iPad if DEBUG: - self.log.info(' iDevice has been ejected') + logger().info(' iDevice has been ejected') self.ejected = True return False - self.log.info(' found connected iPad in sources') + logger().info(' found connected iPad in sources') self._discover_manual_sync_mode(wait=1.0) finally: @@ -694,11 +698,11 @@ class ITUNES(DriverBase): self.problem_msg = _("Some books not found in iTunes database.\n" "Delete using the iBooks app.\n" "Click 'Show Details' for a list.") - self.log.info("ITUNES:delete_books()") + logger().info("ITUNES:delete_books()") for path in paths: if self.cached_books[path]['lib_book']: if DEBUG: - self.log.info(" Deleting '%s' from iTunes library" % (path)) + logger().info(" Deleting '%s' from iTunes library" % (path)) if isosx: self._remove_from_iTunes(self.cached_books[path]) @@ -718,7 +722,7 @@ class ITUNES(DriverBase): self.update_needed = True self.update_msg = "Deleted books from device" else: - self.log.info(" skipping sync phase, manual_sync_mode: True") + logger().info(" skipping sync phase, manual_sync_mode: True") else: if self.manual_sync_mode: metadata = MetaInformation(self.cached_books[path]['title'], @@ -745,7 +749,7 @@ class ITUNES(DriverBase): are pending GUI jobs that need to communicate with the device. ''' if DEBUG: - self.log.info("ITUNES:eject(): ejecting '%s'" % self.sources['iPod']) + logger().info("ITUNES:eject(): ejecting '%s'" % self.sources['iPod']) if isosx: self.iTunes.eject(self.sources['iPod']) elif iswindows: @@ -774,7 +778,7 @@ class ITUNES(DriverBase): In Windows, a sync-in-progress blocks this call until sync is complete """ if DEBUG: - self.log.info("ITUNES:free_space()") + logger().info("ITUNES:free_space()") free_space = 0 if isosx: @@ -796,7 +800,7 @@ class ITUNES(DriverBase): pythoncom.CoUninitialize() break except: - self.log.error(' waiting for free_space() call to go through') + logger().error(' waiting for free_space() call to go through') return (free_space,-1,-1) @@ -806,7 +810,7 @@ class ITUNES(DriverBase): @return: (device name, device version, software version on device, mime type) """ if DEBUG: - self.log.info("ITUNES:get_device_information()") + logger().info("ITUNES:get_device_information()") return (self.sources['iPod'],'hw v1.0','sw v1.0', 'mime type normally goes here') @@ -816,7 +820,7 @@ class ITUNES(DriverBase): @param outfile: file object like C{sys.stdout} or the result of an C{open} call ''' if DEBUG: - self.log.info("ITUNES.get_file(): exporting '%s'" % path) + logger().info("ITUNES.get_file(): exporting '%s'" % path) outfile.write(open(self.cached_books[path]['lib_book'].location().path).read()) @@ -836,7 +840,7 @@ class ITUNES(DriverBase): ''' if DEBUG: - self.log.info("ITUNES.open(connected_device: %s)" % repr(connected_device)) + logger().info("ITUNES.open(connected_device: %s)" % repr(connected_device)) # Display a dialog recommending using 'Connect to iTunes' if user hasn't # previously disabled the dialog @@ -844,33 +848,33 @@ class ITUNES(DriverBase): raise AppleOpenFeedback(self) else: if DEBUG: - self.log.warning(" %s" % self.UNSUPPORTED_DIRECT_CONNECT_MODE_MESSAGE) + logger().warning(" %s" % self.UNSUPPORTED_DIRECT_CONNECT_MODE_MESSAGE) # Confirm/create thumbs archive if not os.path.exists(self.cache_dir): if DEBUG: - self.log.info(" creating thumb cache at '%s'" % self.cache_dir) + logger().info(" creating thumb cache at '%s'" % self.cache_dir) os.makedirs(self.cache_dir) if not os.path.exists(self.archive_path): - self.log.info(" creating zip archive") + logger().info(" creating zip archive") zfw = ZipFile(self.archive_path, mode='w') zfw.writestr("iTunes Thumbs Archive",'') zfw.close() else: if DEBUG: - self.log.info(" existing thumb cache at '%s'" % self.archive_path) + logger().info(" existing thumb cache at '%s'" % self.archive_path) # If enabled in config options, create/confirm an iTunes storage folder if not self.settings().extra_customization[self.USE_ITUNES_STORAGE]: self.iTunes_local_storage = os.path.join(config_dir,'iTunes storage') if not os.path.exists(self.iTunes_local_storage): if DEBUG: - self.log(" creating iTunes_local_storage at '%s'" % self.iTunes_local_storage) + logger()(" creating iTunes_local_storage at '%s'" % self.iTunes_local_storage) os.mkdir(self.iTunes_local_storage) else: if DEBUG: - self.log(" existing iTunes_local_storage at '%s'" % self.iTunes_local_storage) + logger()(" existing iTunes_local_storage at '%s'" % self.iTunes_local_storage) def remove_books_from_metadata(self, paths, booklists): ''' @@ -885,11 +889,11 @@ class ITUNES(DriverBase): as uuids are different ''' if DEBUG: - self.log.info("ITUNES.remove_books_from_metadata()") + logger().info("ITUNES.remove_books_from_metadata()") for path in paths: if DEBUG: self._dump_cached_book(self.cached_books[path], indent=2) - self.log.info(" looking for '%s' by '%s' uuid:%s" % + logger().info(" looking for '%s' by '%s' uuid:%s" % (self.cached_books[path]['title'], self.cached_books[path]['author'], self.cached_books[path]['uuid'])) @@ -897,19 +901,19 @@ class ITUNES(DriverBase): # Purge the booklist, self.cached_books, thumb cache for i,bl_book in enumerate(booklists[0]): if False: - self.log.info(" evaluating '%s' by '%s' uuid:%s" % + logger().info(" evaluating '%s' by '%s' uuid:%s" % (bl_book.title, bl_book.author,bl_book.uuid)) found = False if bl_book.uuid == self.cached_books[path]['uuid']: if False: - self.log.info(" matched with uuid") + logger().info(" matched with uuid") booklists[0].pop(i) found = True elif bl_book.title == self.cached_books[path]['title'] and \ bl_book.author[0] == self.cached_books[path]['author']: if False: - self.log.info(" matched with title + author") + logger().info(" matched with title + author") booklists[0].pop(i) found = True @@ -930,17 +934,17 @@ class ITUNES(DriverBase): thumb = None if thumb: if DEBUG: - self.log.info(" deleting '%s' from cover cache" % (thumb_path)) + logger().info(" deleting '%s' from cover cache" % (thumb_path)) zf.delete(thumb_path) else: if DEBUG: - self.log.info(" '%s' not found in cover cache" % thumb_path) + logger().info(" '%s' not found in cover cache" % thumb_path) zf.close() break else: if DEBUG: - self.log.error(" unable to find '%s' by '%s' (%s)" % + logger().error(" unable to find '%s' by '%s' (%s)" % (bl_book.title, bl_book.author,bl_book.uuid)) if False: @@ -959,7 +963,7 @@ class ITUNES(DriverBase): :detected_device: Device information from the device scanner """ if DEBUG: - self.log.info("ITUNES.reset()") + logger().info("ITUNES.reset()") if report_progress: self.set_progress_reporter(report_progress) @@ -971,7 +975,7 @@ class ITUNES(DriverBase): task does not have any progress information ''' if DEBUG: - self.log.info("ITUNES.set_progress_reporter()") + logger().info("ITUNES.set_progress_reporter()") self.report_progress = report_progress @@ -979,8 +983,8 @@ class ITUNES(DriverBase): # This method is called with the plugboard that matches the format # declared in use_plugboard_ext and a device name of ITUNES if DEBUG: - self.log.info("ITUNES.set_plugboard()") - #self.log.info(' plugboard: %s' % plugboards) + logger().info("ITUNES.set_plugboard()") + #logger().info(' plugboard: %s' % plugboards) self.plugboards = plugboards self.plugboard_func = pb_func @@ -993,11 +997,11 @@ class ITUNES(DriverBase): ''' if DEBUG: - self.log.info("ITUNES.sync_booklists()") + logger().info("ITUNES.sync_booklists()") if self.update_needed: if DEBUG: - self.log.info(' calling _update_device') + logger().info(' calling _update_device') self._update_device(msg=self.update_msg, wait=False) self.update_needed = False @@ -1020,7 +1024,7 @@ class ITUNES(DriverBase): particular device doesn't have any of these locations it should return 0. """ if DEBUG: - self.log.info("ITUNES:total_space()") + logger().info("ITUNES:total_space()") capacity = 0 if isosx: if 'iPod' in self.sources: @@ -1058,7 +1062,7 @@ class ITUNES(DriverBase): "Click 'Show Details' for a list.") if DEBUG: - self.log.info("ITUNES.upload_books()") + logger().info("ITUNES.upload_books()") if isosx: for (i,fpath) in enumerate(files): @@ -1075,8 +1079,8 @@ class ITUNES(DriverBase): # Add new_book to self.cached_books if DEBUG: - self.log.info("ITUNES.upload_books()") - self.log.info(" adding '%s' by '%s' uuid:%s to self.cached_books" % + logger().info("ITUNES.upload_books()") + logger().info(" adding '%s' by '%s' uuid:%s to self.cached_books" % (metadata[i].title, authors_to_string(metadata[i].authors), metadata[i].uuid)) @@ -1119,8 +1123,8 @@ class ITUNES(DriverBase): # Add new_book to self.cached_books if DEBUG: - self.log.info("ITUNES.upload_books()") - self.log.info(" adding '%s' by '%s' uuid:%s to self.cached_books" % + logger().info("ITUNES.upload_books()") + logger().info(" adding '%s' by '%s' uuid:%s to self.cached_books" % (metadata[i].title, authors_to_string(metadata[i].authors), metadata[i].uuid)) @@ -1157,7 +1161,7 @@ class ITUNES(DriverBase): ''' assumes pythoncom wrapper for windows ''' - self.log.info(" ITUNES._add_device_book()") + logger().info(" ITUNES._add_device_book()") if isosx: if 'iPod' in self.sources: connected_device = self.sources['iPod'] @@ -1167,12 +1171,12 @@ class ITUNES(DriverBase): break else: if DEBUG: - self.log.error(" Device|Books playlist not found") + logger().error(" Device|Books playlist not found") # Add the passed book to the Device|Books playlist added = pl.add(appscript.mactypes.File(fpath),to=pl) if False: - self.log.info(" '%s' added to Device|Books" % metadata.title) + logger().info(" '%s' added to Device|Books" % metadata.title) self._wait_for_writable_metadata(added) return added @@ -1189,7 +1193,7 @@ class ITUNES(DriverBase): break else: if DEBUG: - self.log.info(" no Books playlist found") + logger().info(" no Books playlist found") # Add the passed book to the Device|Books playlist if pl: @@ -1251,7 +1255,7 @@ class ITUNES(DriverBase): windows assumes pythoncom wrapper ''' if DEBUG: - self.log.info(" ITUNES._add_library_book()") + logger().info(" ITUNES._add_library_book()") if isosx: added = self.iTunes.add(appscript.mactypes.File(file)) @@ -1262,9 +1266,9 @@ class ITUNES(DriverBase): fa = FileArray(file_s) op_status = lib.AddFiles(fa) if DEBUG: - self.log.info(" file added to Library|Books") + logger().info(" file added to Library|Books") - self.log.info(" iTunes adding '%s'" % file) + logger().info(" iTunes adding '%s'" % file) if DEBUG: sys.stdout.write(" iTunes copying '%s' ..." % metadata.title) @@ -1318,7 +1322,7 @@ class ITUNES(DriverBase): fp = cached_book['lib_book'].Location ''' if DEBUG: - self.log.info(" ITUNES._add_new_copy()") + logger().info(" ITUNES._add_new_copy()") if fpath.rpartition('.')[2].lower() == 'epub': self._update_epub_metadata(fpath, metadata) @@ -1339,7 +1343,7 @@ class ITUNES(DriverBase): db_added = self._add_device_book(fpath, metadata) lb_added = self._add_library_book(fpath, metadata) if not lb_added and DEBUG: - self.log.warn(" failed to add '%s' to iTunes, iTunes Media folder inaccessible" % metadata.title) + logger().warn(" failed to add '%s' to iTunes, iTunes Media folder inaccessible" % metadata.title) else: lb_added = self._add_library_book(fpath, metadata) if not lb_added: @@ -1357,7 +1361,7 @@ class ITUNES(DriverBase): from PIL import Image as PILImage if DEBUG: - self.log.info(" ITUNES._cover_to_thumb()") + logger().info(" ITUNES._cover_to_thumb()") thumb = None if metadata.cover: @@ -1374,7 +1378,7 @@ class ITUNES(DriverBase): scaled, nwidth, nheight = fit_image(width, height, self.MAX_COVER_WIDTH, self.MAX_COVER_HEIGHT) if scaled: if DEBUG: - self.log.info(" cover scaled from %sx%s to %sx%s" % + logger().info(" cover scaled from %sx%s to %sx%s" % (width,height,nwidth,nheight)) img = img.resize((nwidth, nheight), PILImage.ANTIALIAS) cd = cStringIO.StringIO() @@ -1386,7 +1390,7 @@ class ITUNES(DriverBase): cover_data = cd.read() except: self.problem_titles.append("'%s' by %s" % (metadata.title, authors_to_string(metadata.authors))) - self.log.error(" error scaling '%s' for '%s'" % (metadata.cover,metadata.title)) + logger().error(" error scaling '%s' for '%s'" % (metadata.cover,metadata.title)) import traceback traceback.print_exc() @@ -1404,17 +1408,17 @@ class ITUNES(DriverBase): lb_added.artworks[1].data_.set(cover_data) except: if DEBUG: - self.log.warning(" iTunes automation interface reported an error" + logger().warning(" iTunes automation interface reported an error" " adding artwork to '%s' in the iTunes Library" % metadata.title) pass if db_added: try: db_added.artworks[1].data_.set(cover_data) - self.log.info(" writing '%s' cover to iDevice" % metadata.title) + logger().info(" writing '%s' cover to iDevice" % metadata.title) except: if DEBUG: - self.log.warning(" iTunes automation interface reported an error" + logger().warning(" iTunes automation interface reported an error" " adding artwork to '%s' on the iDevice" % metadata.title) #import traceback #traceback.print_exc() @@ -1436,7 +1440,7 @@ class ITUNES(DriverBase): lb_added.AddArtworkFromFile(tc) except: if DEBUG: - self.log.warning(" iTunes automation interface reported an error" + logger().warning(" iTunes automation interface reported an error" " when adding artwork to '%s' in the iTunes Library" % metadata.title) pass @@ -1448,7 +1452,7 @@ class ITUNES(DriverBase): elif format == 'pdf': if DEBUG: - self.log.info(" unable to set PDF cover via automation interface") + logger().info(" unable to set PDF cover via automation interface") try: # Resize for thumb @@ -1463,13 +1467,13 @@ class ITUNES(DriverBase): # Refresh the thumbnail cache if DEBUG: - self.log.info( " refreshing cached thumb for '%s'" % metadata.title) + logger().info( " refreshing cached thumb for '%s'" % metadata.title) zfw = ZipFile(self.archive_path, mode='a') thumb_path = path.rpartition('.')[0] + '.jpg' zfw.writestr(thumb_path, thumb) except: self.problem_titles.append("'%s' by %s" % (metadata.title, authors_to_string(metadata.authors))) - self.log.error(" error converting '%s' to thumb for '%s'" % (metadata.cover,metadata.title)) + logger().error(" error converting '%s' to thumb for '%s'" % (metadata.cover,metadata.title)) finally: try: zfw.close() @@ -1477,14 +1481,14 @@ class ITUNES(DriverBase): pass else: if DEBUG: - self.log.info(" no cover defined in metadata for '%s'" % metadata.title) + logger().info(" no cover defined in metadata for '%s'" % metadata.title) return thumb def _create_new_book(self,fpath, metadata, path, db_added, lb_added, thumb, format): ''' ''' if DEBUG: - self.log.info(" ITUNES._create_new_book()") + logger().info(" ITUNES._create_new_book()") this_book = Book(metadata.title, authors_to_string(metadata.authors)) this_book.datetime = time.gmtime() @@ -1533,7 +1537,7 @@ class ITUNES(DriverBase): wait is passed when launching iTunes, as it seems to need a moment to come to its senses ''' if DEBUG: - self.log.info(" ITUNES._discover_manual_sync_mode()") + logger().info(" ITUNES._discover_manual_sync_mode()") if wait: time.sleep(wait) if isosx: @@ -1545,12 +1549,12 @@ class ITUNES(DriverBase): dev_books = pl.file_tracks() break else: - self.log.error(" book_playlist not found") + logger().error(" book_playlist not found") if dev_books is not None and len(dev_books): first_book = dev_books[0] if False: - self.log.info(" determing manual mode by modifying '%s' by %s" % (first_book.name(), first_book.artist())) + logger().info(" determing manual mode by modifying '%s' by %s" % (first_book.name(), first_book.artist())) try: first_book.bpm.set(0) self.manual_sync_mode = True @@ -1558,7 +1562,7 @@ class ITUNES(DriverBase): self.manual_sync_mode = False else: if DEBUG: - self.log.info(" adding tracer to empty Books|Playlist") + logger().info(" adding tracer to empty Books|Playlist") try: added = pl.add(appscript.mactypes.File(P('tracer.epub')),to=pl) time.sleep(0.5) @@ -1581,7 +1585,7 @@ class ITUNES(DriverBase): if dev_books is not None and dev_books.Count: first_book = dev_books.Item(1) #if DEBUG: - #self.log.info(" determing manual mode by modifying '%s' by %s" % (first_book.Name, first_book.Artist)) + #logger().info(" determing manual mode by modifying '%s' by %s" % (first_book.Name, first_book.Artist)) try: first_book.BPM = 0 self.manual_sync_mode = True @@ -1589,7 +1593,7 @@ class ITUNES(DriverBase): self.manual_sync_mode = False else: if DEBUG: - self.log.info(" sending tracer to empty Books|Playlist") + logger().info(" sending tracer to empty Books|Playlist") fpath = P('tracer.epub') mi = MetaInformation('Tracer',['calibre']) try: @@ -1600,24 +1604,24 @@ class ITUNES(DriverBase): except: self.manual_sync_mode = False - self.log.info(" iTunes.manual_sync_mode: %s" % self.manual_sync_mode) + logger().info(" iTunes.manual_sync_mode: %s" % self.manual_sync_mode) def _dump_booklist(self, booklist, header=None,indent=0): ''' ''' if header: msg = '\n%sbooklist %s:' % (' '*indent,header) - self.log.info(msg) - self.log.info('%s%s' % (' '*indent,'-' * len(msg))) + logger().info(msg) + logger().info('%s%s' % (' '*indent,'-' * len(msg))) for book in booklist: if isosx: - self.log.info("%s%-40.40s %-30.30s %-10.10s %s" % + logger().info("%s%-40.40s %-30.30s %-10.10s %s" % (' '*indent,book.title, book.author, str(book.library_id)[-9:], book.uuid)) elif iswindows: - self.log.info("%s%-40.40s %-30.30s" % + logger().info("%s%-40.40s %-30.30s" % (' '*indent,book.title, book.author)) - self.log.info() + logger().info() def _dump_cached_book(self, cached_book, header=None,indent=0): ''' @@ -1625,16 +1629,16 @@ class ITUNES(DriverBase): if isosx: if header: msg = '%s%s' % (' '*indent,header) - self.log.info(msg) - self.log.info( "%s%s" % (' '*indent, '-' * len(msg))) - self.log.info("%s%-40.40s %-30.30s %-10.10s %-10.10s %s" % + logger().info(msg) + logger().info( "%s%s" % (' '*indent, '-' * len(msg))) + logger().info("%s%-40.40s %-30.30s %-10.10s %-10.10s %s" % (' '*indent, 'title', 'author', 'lib_book', 'dev_book', 'uuid')) - self.log.info("%s%-40.40s %-30.30s %-10.10s %-10.10s %s" % + logger().info("%s%-40.40s %-30.30s %-10.10s %-10.10s %s" % (' '*indent, cached_book['title'], cached_book['author'], @@ -1644,10 +1648,10 @@ class ITUNES(DriverBase): elif iswindows: if header: msg = '%s%s' % (' '*indent,header) - self.log.info(msg) - self.log.info( "%s%s" % (' '*indent, '-' * len(msg))) + logger().info(msg) + logger().info( "%s%s" % (' '*indent, '-' * len(msg))) - self.log.info("%s%-40.40s %-30.30s %s" % + logger().info("%s%-40.40s %-30.30s %s" % (' '*indent, cached_book['title'], cached_book['author'], @@ -1658,11 +1662,11 @@ class ITUNES(DriverBase): ''' if header: msg = '\n%sself.cached_books %s:' % (' '*indent,header) - self.log.info(msg) - self.log.info( "%s%s" % (' '*indent,'-' * len(msg))) + logger().info(msg) + logger().info( "%s%s" % (' '*indent,'-' * len(msg))) if isosx: for cb in self.cached_books.keys(): - self.log.info("%s%-40.40s %-30.30s %-10.10s %-10.10s %s" % + logger().info("%s%-40.40s %-30.30s %-10.10s %-10.10s %s" % (' '*indent, self.cached_books[cb]['title'], self.cached_books[cb]['author'], @@ -1671,21 +1675,21 @@ class ITUNES(DriverBase): self.cached_books[cb]['uuid'])) elif iswindows: for cb in self.cached_books.keys(): - self.log.info("%s%-40.40s %-30.30s %-4.4s %s" % + logger().info("%s%-40.40s %-30.30s %-4.4s %s" % (' '*indent, self.cached_books[cb]['title'], self.cached_books[cb]['author'], self.cached_books[cb]['format'], self.cached_books[cb]['uuid'])) - self.log.info() + logger().info() def _dump_epub_metadata(self, fpath): ''' ''' from calibre.ebooks.BeautifulSoup import BeautifulSoup - self.log.info(" ITUNES.__get_epub_metadata()") + logger().info(" ITUNES.__get_epub_metadata()") title = None author = None timestamp = None @@ -1705,11 +1709,11 @@ class ITUNES(DriverBase): if not title or not author: if DEBUG: - self.log.error(" couldn't extract title/author from %s in %s" % (opf,fpath)) - self.log.error(" title: %s author: %s timestamp: %s" % (title, author, timestamp)) + logger().error(" couldn't extract title/author from %s in %s" % (opf,fpath)) + logger().error(" title: %s author: %s timestamp: %s" % (title, author, timestamp)) else: if DEBUG: - self.log.error(" can't find .opf in %s" % fpath) + logger().error(" can't find .opf in %s" % fpath) zf.close() return (title, author, timestamp) @@ -1730,20 +1734,20 @@ class ITUNES(DriverBase): ''' ''' if DEBUG: - self.log.info("\n library_books:") + logger().info("\n library_books:") for book in library_books: - self.log.info(" %s" % book) - self.log.info() + logger().info(" %s" % book) + logger().info() def _dump_update_list(self,header=None,indent=0): if header and self.update_list: msg = '\n%sself.update_list %s' % (' '*indent,header) - self.log.info(msg) - self.log.info( "%s%s" % (' '*indent,'-' * len(msg))) + logger().info(msg) + logger().info( "%s%s" % (' '*indent,'-' * len(msg))) if isosx: for ub in self.update_list: - self.log.info("%s%-40.40s %-30.30s %-10.10s %s" % + logger().info("%s%-40.40s %-30.30s %-10.10s %s" % (' '*indent, ub['title'], ub['author'], @@ -1751,7 +1755,7 @@ class ITUNES(DriverBase): ub['uuid'])) elif iswindows: for ub in self.update_list: - self.log.info("%s%-40.40s %-30.30s" % + logger().info("%s%-40.40s %-30.30s" % (' '*indent, ub['title'], ub['author'])) @@ -1763,42 +1767,42 @@ class ITUNES(DriverBase): if iswindows: dev_books = self._get_device_books_playlist() if DEBUG: - self.log.info(" ITUNES._find_device_book()") - self.log.info(" searching for '%s' by '%s' (%s)" % + logger().info(" ITUNES._find_device_book()") + logger().info(" searching for '%s' by '%s' (%s)" % (search['title'], search['author'],search['uuid'])) attempts = 9 while attempts: # Try by uuid - only one hit if 'uuid' in search and search['uuid']: if DEBUG: - self.log.info(" searching by uuid '%s' ..." % search['uuid']) + logger().info(" searching by uuid '%s' ..." % search['uuid']) hits = dev_books.Search(search['uuid'],self.SearchField.index('All')) if hits: hit = hits[0] - self.log.info(" found '%s' by %s (%s)" % (hit.Name, hit.Artist, hit.Composer)) + logger().info(" found '%s' by %s (%s)" % (hit.Name, hit.Artist, hit.Composer)) return hit # Try by author - there could be multiple hits if search['author']: if DEBUG: - self.log.info(" searching by author '%s' ..." % search['author']) + logger().info(" searching by author '%s' ..." % search['author']) hits = dev_books.Search(search['author'],self.SearchField.index('Artists')) if hits: for hit in hits: if hit.Name == search['title']: if DEBUG: - self.log.info(" found '%s' by %s (%s)" % (hit.Name, hit.Artist, hit.Composer)) + logger().info(" found '%s' by %s (%s)" % (hit.Name, hit.Artist, hit.Composer)) return hit # Search by title if no author available if DEBUG: - self.log.info(" searching by title '%s' ..." % search['title']) + logger().info(" searching by title '%s' ..." % search['title']) hits = dev_books.Search(search['title'],self.SearchField.index('All')) if hits: for hit in hits: if hit.Name == search['title']: if DEBUG: - self.log.info(" found '%s'" % (hit.Name)) + logger().info(" found '%s'" % (hit.Name)) return hit # PDF just sent, title not updated yet, look for export pattern @@ -1807,24 +1811,24 @@ class ITUNES(DriverBase): title = re.sub(r'[^0-9a-zA-Z ]', '_', search['title']) author = re.sub(r'[^0-9a-zA-Z ]', '_', search['author']) if DEBUG: - self.log.info(" searching by name: '%s - %s'" % (title,author)) + logger().info(" searching by name: '%s - %s'" % (title,author)) hits = dev_books.Search('%s - %s' % (title,author), self.SearchField.index('All')) if hits: hit = hits[0] - self.log.info(" found '%s' by %s (%s)" % (hit.Name, hit.Artist, hit.Composer)) + logger().info(" found '%s' by %s (%s)" % (hit.Name, hit.Artist, hit.Composer)) return hit else: if DEBUG: - self.log.info(" no PDF hits") + logger().info(" no PDF hits") attempts -= 1 time.sleep(0.5) if DEBUG: - self.log.warning(" attempt #%d" % (10 - attempts)) + logger().warning(" attempt #%d" % (10 - attempts)) if DEBUG: - self.log.error(" no hits") + logger().error(" no hits") return None def _find_library_book(self, search): @@ -1833,13 +1837,13 @@ class ITUNES(DriverBase): ''' if iswindows: if DEBUG: - self.log.info(" ITUNES._find_library_book()") + logger().info(" ITUNES._find_library_book()") ''' if 'uuid' in search: - self.log.info(" looking for '%s' by %s (%s)" % + logger().info(" looking for '%s' by %s (%s)" % (search['title'], search['author'], search['uuid'])) else: - self.log.info(" looking for '%s' by %s" % + logger().info(" looking for '%s' by %s" % (search['title'], search['author'])) ''' @@ -1847,11 +1851,11 @@ class ITUNES(DriverBase): if source.Kind == self.Sources.index('Library'): lib = source if DEBUG: - self.log.info(" Library source: '%s' kind: %s" % (lib.Name, self.Sources[lib.Kind])) + logger().info(" Library source: '%s' kind: %s" % (lib.Name, self.Sources[lib.Kind])) break else: if DEBUG: - self.log.info(" Library source not found") + logger().info(" Library source not found") if lib is not None: lib_books = None @@ -1859,12 +1863,12 @@ class ITUNES(DriverBase): if pl.Kind == self.PlaylistKind.index('User') and \ pl.SpecialKind == self.PlaylistSpecialKind.index('Books'): if DEBUG: - self.log.info(" Books playlist: '%s'" % (pl.Name)) + logger().info(" Books playlist: '%s'" % (pl.Name)) lib_books = pl break else: if DEBUG: - self.log.error(" no Books playlist found") + logger().error(" no Books playlist found") attempts = 9 @@ -1872,35 +1876,35 @@ class ITUNES(DriverBase): # Find book whose Album field = search['uuid'] if 'uuid' in search and search['uuid']: if DEBUG: - self.log.info(" searching by uuid '%s' ..." % search['uuid']) + logger().info(" searching by uuid '%s' ..." % search['uuid']) hits = lib_books.Search(search['uuid'],self.SearchField.index('All')) if hits: hit = hits[0] if DEBUG: - self.log.info(" found '%s' by %s (%s)" % (hit.Name, hit.Artist, hit.Composer)) + logger().info(" found '%s' by %s (%s)" % (hit.Name, hit.Artist, hit.Composer)) return hit # Search by author if known if search['author']: if DEBUG: - self.log.info(" searching by author '%s' ..." % search['author']) + logger().info(" searching by author '%s' ..." % search['author']) hits = lib_books.Search(search['author'],self.SearchField.index('Artists')) if hits: for hit in hits: if hit.Name == search['title']: if DEBUG: - self.log.info(" found '%s' by %s (%s)" % (hit.Name, hit.Artist, hit.Composer)) + logger().info(" found '%s' by %s (%s)" % (hit.Name, hit.Artist, hit.Composer)) return hit # Search by title if no author available if DEBUG: - self.log.info(" searching by title '%s' ..." % search['title']) + logger().info(" searching by title '%s' ..." % search['title']) hits = lib_books.Search(search['title'],self.SearchField.index('All')) if hits: for hit in hits: if hit.Name == search['title']: if DEBUG: - self.log.info(" found '%s'" % (hit.Name)) + logger().info(" found '%s'" % (hit.Name)) return hit # PDF just sent, title not updated yet, look for export pattern @@ -1909,24 +1913,24 @@ class ITUNES(DriverBase): title = re.sub(r'[^0-9a-zA-Z ]', '_', search['title']) author = re.sub(r'[^0-9a-zA-Z ]', '_', search['author']) if DEBUG: - self.log.info(" searching by name: %s - %s" % (title,author)) + logger().info(" searching by name: %s - %s" % (title,author)) hits = lib_books.Search('%s - %s' % (title,author), self.SearchField.index('All')) if hits: hit = hits[0] - self.log.info(" found '%s' by %s (%s)" % (hit.Name, hit.Artist, hit.Composer)) + logger().info(" found '%s' by %s (%s)" % (hit.Name, hit.Artist, hit.Composer)) return hit else: if DEBUG: - self.log.info(" no PDF hits") + logger().info(" no PDF hits") attempts -= 1 time.sleep(0.5) if DEBUG: - self.log.warning(" attempt #%d" % (10 - attempts)) + logger().warning(" attempt #%d" % (10 - attempts)) if DEBUG: - self.log.error(" search for '%s' yielded no hits" % search['title']) + logger().error(" search for '%s' yielded no hits" % search['title']) return None def _generate_thumbnail(self, book_path, book): @@ -1953,18 +1957,18 @@ class ITUNES(DriverBase): thumb_data = zfr.read(thumb_path) if thumb_data == 'None': if False: - self.log.info(" ITUNES._generate_thumbnail()\n returning None from cover cache for '%s'" % title) + logger().info(" ITUNES._generate_thumbnail()\n returning None from cover cache for '%s'" % title) zfr.close() return None except: zfw = ZipFile(self.archive_path, mode='a') else: if False: - self.log.info(" returning thumb from cache for '%s'" % title) + logger().info(" returning thumb from cache for '%s'" % title) return thumb_data if DEBUG: - self.log.info(" ITUNES._generate_thumbnail('%s'):" % title) + logger().info(" ITUNES._generate_thumbnail('%s'):" % title) if isosx: # Fetch the artwork from iTunes @@ -1973,7 +1977,7 @@ class ITUNES(DriverBase): except: # If no artwork, write an empty marker to cache if DEBUG: - self.log.error(" error fetching iTunes artwork for '%s'" % title) + logger().error(" error fetching iTunes artwork for '%s'" % title) zfw.writestr(thumb_path, 'None') zfw.close() return None @@ -1990,12 +1994,12 @@ class ITUNES(DriverBase): thumb_data = thumb.getvalue() thumb.close() if False: - self.log.info(" generated thumb for '%s', caching" % title) + logger().info(" generated thumb for '%s', caching" % title) # Cache the tagged thumb zfw.writestr(thumb_path, thumb_data) except: if DEBUG: - self.log.error(" error generating thumb for '%s', caching empty marker" % book.name()) + logger().error(" error generating thumb for '%s', caching empty marker" % book.name()) self._dump_hex(data[:32]) thumb_data = None # Cache the empty cover @@ -2010,7 +2014,7 @@ class ITUNES(DriverBase): elif iswindows: if not book.Artwork.Count: if DEBUG: - self.log.info(" no artwork available for '%s'" % book.Name) + logger().info(" no artwork available for '%s'" % book.Name) zfw.writestr(thumb_path, 'None') zfw.close() return None @@ -2030,12 +2034,12 @@ class ITUNES(DriverBase): os.remove(tmp_thumb) thumb.close() if False: - self.log.info(" generated thumb for '%s', caching" % book.Name) + logger().info(" generated thumb for '%s', caching" % book.Name) # Cache the tagged thumb zfw.writestr(thumb_path, thumb_data) except: if DEBUG: - self.log.error(" error generating thumb for '%s', caching empty marker" % book.Name) + logger().error(" error generating thumb for '%s', caching empty marker" % book.Name) thumb_data = None # Cache the empty cover zfw.writestr(thumb_path,'None') @@ -2058,9 +2062,9 @@ class ITUNES(DriverBase): for file in myZipList: exploded_file_size += file.file_size if False: - self.log.info(" ITUNES._get_device_book_size()") - self.log.info(" %d items in archive" % len(myZipList)) - self.log.info(" compressed: %d exploded: %d" % (compressed_size, exploded_file_size)) + logger().info(" ITUNES._get_device_book_size()") + logger().info(" %d items in archive" % len(myZipList)) + logger().info(" compressed: %d exploded: %d" % (compressed_size, exploded_file_size)) myZip.close() return exploded_file_size @@ -2069,7 +2073,7 @@ class ITUNES(DriverBase): Assumes pythoncom wrapper for Windows ''' if DEBUG: - self.log.info("\n ITUNES._get_device_books()") + logger().info("\n ITUNES._get_device_books()") device_books = [] if isosx: @@ -2080,24 +2084,24 @@ class ITUNES(DriverBase): for pl in device.playlists(): if pl.special_kind() == appscript.k.Books: if DEBUG: - self.log.info(" Book playlist: '%s'" % (pl.name())) + logger().info(" Book playlist: '%s'" % (pl.name())) dev_books = pl.file_tracks() break else: - self.log.error(" book_playlist not found") + logger().error(" book_playlist not found") for book in dev_books: # This may need additional entries for international iTunes users if book.kind() in self.Audiobooks: if DEBUG: - self.log.info(" ignoring '%s' of type '%s'" % (book.name(), book.kind())) + logger().info(" ignoring '%s' of type '%s'" % (book.name(), book.kind())) else: if DEBUG: - self.log.info(" %-30.30s %-30.30s %-40.40s [%s]" % + logger().info(" %-30.30s %-30.30s %-40.40s [%s]" % (book.name(), book.artist(), book.album(), book.kind())) device_books.append(book) if DEBUG: - self.log.info() + logger().info() elif iswindows: if 'iPod' in self.sources: @@ -2111,24 +2115,24 @@ class ITUNES(DriverBase): if pl.Kind == self.PlaylistKind.index('User') and \ pl.SpecialKind == self.PlaylistSpecialKind.index('Books'): if DEBUG: - self.log.info(" Books playlist: '%s'" % (pl.Name)) + logger().info(" Books playlist: '%s'" % (pl.Name)) dev_books = pl.Tracks break else: if DEBUG: - self.log.info(" no Books playlist found") + logger().info(" no Books playlist found") for book in dev_books: # This may need additional entries for international iTunes users if book.KindAsString in self.Audiobooks: if DEBUG: - self.log.info(" ignoring '%s' of type '%s'" % (book.Name, book.KindAsString)) + logger().info(" ignoring '%s' of type '%s'" % (book.Name, book.KindAsString)) else: if DEBUG: - self.log.info(" %-30.30s %-30.30s %-40.40s [%s]" % (book.Name, book.Artist, book.Album, book.KindAsString)) + logger().info(" %-30.30s %-30.30s %-40.40s [%s]" % (book.Name, book.Artist, book.Album, book.KindAsString)) device_books.append(book) if DEBUG: - self.log.info() + logger().info() finally: pythoncom.CoUninitialize() @@ -2151,7 +2155,7 @@ class ITUNES(DriverBase): break else: if DEBUG: - self.log.error(" no iPad|Books playlist found") + logger().error(" no iPad|Books playlist found") return pl def _get_library_books(self): @@ -2160,7 +2164,7 @@ class ITUNES(DriverBase): Windows assumes pythoncom wrapper ''' if DEBUG: - self.log.info("\n ITUNES._get_library_books()") + logger().info("\n ITUNES._get_library_books()") library_books = {} library_orphans = {} @@ -2171,11 +2175,11 @@ class ITUNES(DriverBase): if source.kind() == appscript.k.library: lib = source if DEBUG: - self.log.info(" Library source: '%s'" % (lib.name())) + logger().info(" Library source: '%s'" % (lib.name())) break else: if DEBUG: - self.log.error(' Library source not found') + logger().error(' Library source not found') if lib is not None: lib_books = None @@ -2183,18 +2187,18 @@ class ITUNES(DriverBase): for pl in lib.playlists(): if pl.special_kind() == appscript.k.Books: if DEBUG: - self.log.info(" Books playlist: '%s'" % (pl.name())) + logger().info(" Books playlist: '%s'" % (pl.name())) break else: if DEBUG: - self.log.info(" no Library|Books playlist found") + logger().info(" no Library|Books playlist found") lib_books = pl.file_tracks() for book in lib_books: # This may need additional entries for international iTunes users if book.kind() in self.Audiobooks: if DEBUG: - self.log.info(" ignoring '%s' of type '%s'" % (book.name(), book.kind())) + logger().info(" ignoring '%s' of type '%s'" % (book.name(), book.kind())) else: # Collect calibre orphans - remnants of recipe uploads format = 'pdf' if book.kind().startswith('PDF') else 'epub' @@ -2204,31 +2208,31 @@ class ITUNES(DriverBase): if book.location() == appscript.k.missing_value: library_orphans[path] = book if False: - self.log.info(" found iTunes PTF '%s' in Library|Books" % book.name()) + logger().info(" found iTunes PTF '%s' in Library|Books" % book.name()) except: if DEBUG: - self.log.error(" iTunes returned an error returning .location() with %s" % book.name()) + logger().error(" iTunes returned an error returning .location() with %s" % book.name()) library_books[path] = book if DEBUG: - self.log.info(" %-30.30s %-30.30s %-40.40s [%s]" % + logger().info(" %-30.30s %-30.30s %-40.40s [%s]" % (book.name(), book.artist(), book.album(), book.kind())) else: if DEBUG: - self.log.info(' no Library playlists') + logger().info(' no Library playlists') else: if DEBUG: - self.log.info(' no Library found') + logger().info(' no Library found') elif iswindows: lib = None for source in self.iTunes.sources: if source.Kind == self.Sources.index('Library'): lib = source - self.log.info(" Library source: '%s' kind: %s" % (lib.Name, self.Sources[lib.Kind])) + logger().info(" Library source: '%s' kind: %s" % (lib.Name, self.Sources[lib.Kind])) break else: - self.log.error(" Library source not found") + logger().error(" Library source not found") if lib is not None: lib_books = None @@ -2237,22 +2241,22 @@ class ITUNES(DriverBase): if pl.Kind == self.PlaylistKind.index('User') and \ pl.SpecialKind == self.PlaylistSpecialKind.index('Books'): if DEBUG: - self.log.info(" Books playlist: '%s'" % (pl.Name)) + logger().info(" Books playlist: '%s'" % (pl.Name)) lib_books = pl.Tracks break else: if DEBUG: - self.log.error(" no Library|Books playlist found") + logger().error(" no Library|Books playlist found") else: if DEBUG: - self.log.error(" no Library playlists found") + logger().error(" no Library playlists found") try: for book in lib_books: # This may need additional entries for international iTunes users if book.KindAsString in self.Audiobooks: if DEBUG: - self.log.info(" ignoring %-30.30s of type '%s'" % (book.Name, book.KindAsString)) + logger().info(" ignoring %-30.30s of type '%s'" % (book.Name, book.KindAsString)) else: format = 'pdf' if book.KindAsString.startswith('PDF') else 'epub' path = self.path_template % (book.Name, book.Artist,format) @@ -2262,14 +2266,14 @@ class ITUNES(DriverBase): if not book.Location: library_orphans[path] = book if False: - self.log.info(" found iTunes PTF '%s' in Library|Books" % book.Name) + logger().info(" found iTunes PTF '%s' in Library|Books" % book.Name) library_books[path] = book if DEBUG: - self.log.info(" %-30.30s %-30.30s %-40.40s [%s]" % (book.Name, book.Artist, book.Album, book.KindAsString)) + logger().info(" %-30.30s %-30.30s %-40.40s [%s]" % (book.Name, book.Artist, book.Album, book.KindAsString)) except: if DEBUG: - self.log.info(" no books in library") + logger().info(" no books in library") self.library_orphans = library_orphans return library_books @@ -2314,7 +2318,7 @@ class ITUNES(DriverBase): # If more than one connected iDevice, remove all from list to prevent driver initialization if kinds.count('iPod') > 1: if DEBUG: - self.log.error(" %d connected iPod devices detected, calibre supports a single connected iDevice" % kinds.count('iPod')) + logger().error(" %d connected iPod devices detected, calibre supports a single connected iDevice" % kinds.count('iPod')) while kinds.count('iPod'): index = kinds.index('iPod') kinds.pop(index) @@ -2334,7 +2338,7 @@ class ITUNES(DriverBase): ''' ''' if DEBUG: - self.log.info(" ITUNES:_launch_iTunes():\n Instantiating iTunes") + logger().info(" ITUNES:_launch_iTunes():\n Instantiating iTunes") if isosx: ''' @@ -2344,7 +2348,7 @@ class ITUNES(DriverBase): running_apps = appscript.app('System Events') if not 'iTunes' in running_apps.processes.name(): if DEBUG: - self.log.info( "ITUNES:_launch_iTunes(): Launching iTunes" ) + logger().info( "ITUNES:_launch_iTunes(): Launching iTunes" ) try: self.iTunes = iTunes= appscript.app('iTunes', hide=True) except: @@ -2366,16 +2370,16 @@ class ITUNES(DriverBase): if os.path.exists(media_dir): self.iTunes_media = media_dir else: - self.log.error(" could not confirm valid iTunes.media_dir from %s" % 'com.apple.itunes') - self.log.error(" media_dir: %s" % media_dir) + logger().error(" could not confirm valid iTunes.media_dir from %s" % 'com.apple.itunes') + logger().error(" media_dir: %s" % media_dir) ''' if DEBUG: - self.log.info(" %s %s" % (__appname__, __version__)) - self.log.info(" [OSX %s - %s (%s), driver version %d.%d.%d]" % + logger().info(" %s %s" % (__appname__, __version__)) + logger().info(" [OSX %s - %s (%s), driver version %d.%d.%d]" % (self.iTunes.name(), self.iTunes.version(), self.initial_status, self.version[0],self.version[1],self.version[2])) - self.log.info(" calibre_library_path: %s" % self.calibre_library_path) + logger().info(" calibre_library_path: %s" % self.calibre_library_path) if iswindows: ''' @@ -2428,19 +2432,19 @@ class ITUNES(DriverBase): if os.path.exists(media_dir): self.iTunes_media = media_dir elif hasattr(string,'parent'): - self.log.error(" could not extract valid iTunes.media_dir from %s" % self.iTunes.LibraryXMLPath) - self.log.error(" %s" % string.parent.prettify()) - self.log.error(" '%s' not found" % media_dir) + logger().error(" could not extract valid iTunes.media_dir from %s" % self.iTunes.LibraryXMLPath) + logger().error(" %s" % string.parent.prettify()) + logger().error(" '%s' not found" % media_dir) else: - self.log.error(" no media dir found: string: %s" % string) + logger().error(" no media dir found: string: %s" % string) ''' if DEBUG: - self.log.info(" %s %s" % (__appname__, __version__)) - self.log.info(" [Windows %s - %s (%s), driver version %d.%d.%d]" % + logger().info(" %s %s" % (__appname__, __version__)) + logger().info(" [Windows %s - %s (%s), driver version %d.%d.%d]" % (self.iTunes.Windows[0].name, self.iTunes.Version, self.initial_status, self.version[0],self.version[1],self.version[2])) - self.log.info(" calibre_library_path: %s" % self.calibre_library_path) + logger().info(" calibre_library_path: %s" % self.calibre_library_path) def _purge_orphans(self,library_books, cached_books): ''' @@ -2449,16 +2453,16 @@ class ITUNES(DriverBase): This occurs when the user deletes a book in iBooks while disconnected ''' if DEBUG: - self.log.info(" ITUNES._purge_orphans()") + logger().info(" ITUNES._purge_orphans()") #self._dump_library_books(library_books) - #self.log.info(" cached_books:\n %s" % "\n ".join(cached_books.keys())) + #logger().info(" cached_books:\n %s" % "\n ".join(cached_books.keys())) for book in library_books: if isosx: if book not in cached_books and \ str(library_books[book].description()).startswith(self.description_prefix): if DEBUG: - self.log.info(" '%s' not found on iDevice, removing from iTunes" % book) + logger().info(" '%s' not found on iDevice, removing from iTunes" % book) btr = { 'title':library_books[book].name(), 'author':library_books[book].artist(), 'lib_book':library_books[book]} @@ -2467,19 +2471,19 @@ class ITUNES(DriverBase): if book not in cached_books and \ library_books[book].Description.startswith(self.description_prefix): if DEBUG: - self.log.info(" '%s' not found on iDevice, removing from iTunes" % book) + logger().info(" '%s' not found on iDevice, removing from iTunes" % book) btr = { 'title':library_books[book].Name, 'author':library_books[book].Artist, 'lib_book':library_books[book]} self._remove_from_iTunes(btr) if DEBUG: - self.log.info() + logger().info() def _remove_existing_copy(self, path, metadata): ''' ''' if DEBUG: - self.log.info(" ITUNES._remove_existing_copy()") + logger().info(" ITUNES._remove_existing_copy()") if self.manual_sync_mode: # Delete existing from Device|Books, add to self.update_list @@ -2491,16 +2495,16 @@ class ITUNES(DriverBase): self.update_list.append(self.cached_books[book]) if DEBUG: - self.log.info( " deleting device book '%s'" % (metadata.title)) + logger().info( " deleting device book '%s'" % (metadata.title)) self._remove_from_device(self.cached_books[book]) if DEBUG: - self.log.info(" deleting library book '%s'" % metadata.title) + logger().info(" deleting library book '%s'" % metadata.title) self._remove_from_iTunes(self.cached_books[book]) break else: if DEBUG: - self.log.info(" '%s' not in cached_books" % metadata.title) + logger().info(" '%s' not in cached_books" % metadata.title) else: # Delete existing from Library|Books, add to self.update_list # for deletion from booklist[0] during add_books_to_metadata @@ -2510,35 +2514,35 @@ class ITUNES(DriverBase): self.cached_books[book]['author'] == authors_to_string(metadata.authors)): self.update_list.append(self.cached_books[book]) if DEBUG: - self.log.info( " deleting library book '%s'" % metadata.title) + logger().info( " deleting library book '%s'" % metadata.title) self._remove_from_iTunes(self.cached_books[book]) break else: if DEBUG: - self.log.info(" '%s' not found in cached_books" % metadata.title) + logger().info(" '%s' not found in cached_books" % metadata.title) def _remove_from_device(self, cached_book): ''' Windows assumes pythoncom wrapper ''' if DEBUG: - self.log.info(" ITUNES._remove_from_device()") + logger().info(" ITUNES._remove_from_device()") if isosx: if DEBUG: - self.log.info(" deleting '%s' from iDevice" % cached_book['title']) + logger().info(" deleting '%s' from iDevice" % cached_book['title']) try: cached_book['dev_book'].delete() except: - self.log.error(" error deleting '%s'" % cached_book['title']) + logger().error(" error deleting '%s'" % cached_book['title']) elif iswindows: hit = self._find_device_book(cached_book) if hit: if DEBUG: - self.log.info(" deleting '%s' from iDevice" % cached_book['title']) + logger().info(" deleting '%s' from iDevice" % cached_book['title']) hit.Delete() else: if DEBUG: - self.log.warning(" unable to remove '%s' by '%s' (%s) from device" % + logger().warning(" unable to remove '%s' by '%s' (%s) from device" % (cached_book['title'],cached_book['author'],cached_book['uuid'])) def _remove_from_iTunes(self, cached_book): @@ -2546,34 +2550,34 @@ class ITUNES(DriverBase): iTunes does not delete books from storage when removing from database via automation ''' if DEBUG: - self.log.info(" ITUNES._remove_from_iTunes():") + logger().info(" ITUNES._remove_from_iTunes():") if isosx: ''' Manually remove the book from iTunes storage ''' try: fp = cached_book['lib_book'].location().path if DEBUG: - self.log.info(" processing %s" % fp) + logger().info(" processing %s" % fp) if fp.startswith(prefs['library_path']): - self.log.info(" '%s' stored in calibre database, not removed" % cached_book['title']) + logger().info(" '%s' stored in calibre database, not removed" % cached_book['title']) elif not self.settings().extra_customization[self.USE_ITUNES_STORAGE] and \ fp.startswith(self.iTunes_local_storage) and \ os.path.exists(fp): # Delete the copy in iTunes_local_storage os.remove(fp) if DEBUG: - self.log(" removing from iTunes_local_storage") + logger()(" removing from iTunes_local_storage") else: # Delete from iTunes Media folder if os.path.exists(fp): os.remove(fp) if DEBUG: - self.log.info(" deleting from iTunes storage") + logger().info(" deleting from iTunes storage") author_storage_path = os.path.split(fp)[0] try: os.rmdir(author_storage_path) if DEBUG: - self.log.info(" removing empty author directory") + logger().info(" removing empty author directory") except: author_files = os.listdir(author_storage_path) if '.DS_Store' in author_files: @@ -2581,23 +2585,23 @@ class ITUNES(DriverBase): if not author_files: os.rmdir(author_storage_path) if DEBUG: - self.log.info(" removing empty author directory") + logger().info(" removing empty author directory") else: - self.log.info(" '%s' does not exist at storage location" % cached_book['title']) + logger().info(" '%s' does not exist at storage location" % cached_book['title']) except: # We get here if there was an error with .location().path if DEBUG: - self.log.info(" '%s' not found in iTunes storage" % cached_book['title']) + logger().info(" '%s' not found in iTunes storage" % cached_book['title']) # Delete the book from the iTunes database try: self.iTunes.delete(cached_book['lib_book']) if DEBUG: - self.log.info(" removing from iTunes database") + logger().info(" removing from iTunes database") except: if DEBUG: - self.log.info(" unable to remove from iTunes database") + logger().info(" unable to remove from iTunes database") elif iswindows: ''' @@ -2615,43 +2619,43 @@ class ITUNES(DriverBase): if book: if DEBUG: - self.log.info(" processing %s" % fp) + logger().info(" processing %s" % fp) if fp.startswith(prefs['library_path']): - self.log.info(" '%s' stored in calibre database, not removed" % cached_book['title']) + logger().info(" '%s' stored in calibre database, not removed" % cached_book['title']) elif not self.settings().extra_customization[self.USE_ITUNES_STORAGE] and \ fp.startswith(self.iTunes_local_storage) and \ os.path.exists(fp): # Delete the copy in iTunes_local_storage os.remove(fp) if DEBUG: - self.log(" removing from iTunes_local_storage") + logger()(" removing from iTunes_local_storage") else: # Delete from iTunes Media folder if os.path.exists(fp): os.remove(fp) if DEBUG: - self.log.info(" deleting from iTunes storage") + logger().info(" deleting from iTunes storage") author_storage_path = os.path.split(fp)[0] try: os.rmdir(author_storage_path) if DEBUG: - self.log.info(" removing empty author directory") + logger().info(" removing empty author directory") except: pass else: - self.log.info(" '%s' does not exist at storage location" % cached_book['title']) + logger().info(" '%s' does not exist at storage location" % cached_book['title']) else: if DEBUG: - self.log.info(" '%s' not found in iTunes storage" % cached_book['title']) + logger().info(" '%s' not found in iTunes storage" % cached_book['title']) # Delete the book from the iTunes database try: book.Delete() if DEBUG: - self.log.info(" removing from iTunes database") + logger().info(" removing from iTunes database") except: if DEBUG: - self.log.info(" unable to remove from iTunes database") + logger().info(" unable to remove from iTunes database") def title_sorter(self, title): return re.sub('^\s*A\s+|^\s*The\s+|^\s*An\s+', '', title).rstrip() @@ -2663,7 +2667,7 @@ class ITUNES(DriverBase): from lxml import etree if DEBUG: - self.log.info(" ITUNES._update_epub_metadata()") + logger().info(" ITUNES._update_epub_metadata()") # Fetch plugboard updates metadata_x = self._xform_metadata_via_plugboard(metadata, 'epub') @@ -2691,17 +2695,17 @@ class ITUNES(DriverBase): metadata.timestamp = datetime.datetime(old_ts.year, old_ts.month, old_ts.day, old_ts.hour, old_ts.minute, old_ts.second, old_ts.microsecond+1, old_ts.tzinfo) if DEBUG: - self.log.info(" existing timestamp: %s" % metadata.timestamp) + logger().info(" existing timestamp: %s" % metadata.timestamp) else: metadata.timestamp = now() if DEBUG: - self.log.info(" add timestamp: %s" % metadata.timestamp) + logger().info(" add timestamp: %s" % metadata.timestamp) else: metadata.timestamp = now() if DEBUG: - self.log.warning(" missing block in OPF file") - self.log.info(" add timestamp: %s" % metadata.timestamp) + logger().warning(" missing block in OPF file") + logger().info(" add timestamp: %s" % metadata.timestamp) zf_opf.close() @@ -2731,7 +2735,7 @@ class ITUNES(DriverBase): Trigger a sync, wait for completion ''' if DEBUG: - self.log.info(" ITUNES:_update_device():\n %s" % msg) + logger().info(" ITUNES:_update_device():\n %s" % msg) if isosx: self.iTunes.update() @@ -2777,7 +2781,7 @@ class ITUNES(DriverBase): ''' ''' if DEBUG: - self.log.info(" ITUNES._update_iTunes_metadata()") + logger().info(" ITUNES._update_iTunes_metadata()") STRIP_TAGS = re.compile(r'<[^<]*?/?>') @@ -2829,8 +2833,8 @@ class ITUNES(DriverBase): # If title_sort applied in plugboard, that overrides using series/index as title_sort if metadata_x.series and self.settings().extra_customization[self.USE_SERIES_AS_CATEGORY]: if DEBUG: - self.log.info(" ITUNES._update_iTunes_metadata()") - self.log.info(" using Series name '%s' as Genre" % metadata_x.series) + logger().info(" ITUNES._update_iTunes_metadata()") + logger().info(" using Series name '%s' as Genre" % metadata_x.series) # Format the index as a sort key index = metadata_x.series_index @@ -2854,7 +2858,7 @@ class ITUNES(DriverBase): break if db_added: - self.log.warning(" waiting for db_added to become writeable ") + logger().warning(" waiting for db_added to become writeable ") time.sleep(1.0) # If no title_sort plugboard tweak, create sort_name from series/index if metadata.title_sort == metadata_x.title_sort: @@ -2874,7 +2878,7 @@ class ITUNES(DriverBase): elif metadata_x.tags is not None: if DEBUG: - self.log.info(" %susing Tag as Genre" % + logger().info(" %susing Tag as Genre" % "no Series name available, " if self.settings().extra_customization[self.USE_SERIES_AS_CATEGORY] else '') for tag in metadata_x.tags: if self._is_alpha(tag[0]): @@ -2897,7 +2901,7 @@ class ITUNES(DriverBase): lb_added.Year = metadata_x.pubdate.year if db_added: - self.log.warning(" waiting for db_added to become writeable ") + logger().warning(" waiting for db_added to become writeable ") time.sleep(1.0) db_added.Name = metadata_x.title db_added.Album = metadata_x.title @@ -2924,7 +2928,7 @@ class ITUNES(DriverBase): db_added.AlbumRating = (metadata_x.rating*10) except: if DEBUG: - self.log.warning(" iTunes automation interface reported an error" + logger().warning(" iTunes automation interface reported an error" " setting AlbumRating on iDevice") # Set Genre from first alpha tag, overwrite with series if available @@ -2933,7 +2937,7 @@ class ITUNES(DriverBase): if metadata_x.series and self.settings().extra_customization[self.USE_SERIES_AS_CATEGORY]: if DEBUG: - self.log.info(" using Series name as Genre") + logger().info(" using Series name as Genre") # Format the index as a sort key index = metadata_x.series_index integer = int(index) @@ -2949,13 +2953,13 @@ class ITUNES(DriverBase): lb_added.TrackNumber = metadata_x.series_index except: if DEBUG: - self.log.warning(" iTunes automation interface reported an error" + logger().warning(" iTunes automation interface reported an error" " setting TrackNumber in iTunes") try: lb_added.EpisodeNumber = metadata_x.series_index except: if DEBUG: - self.log.warning(" iTunes automation interface reported an error" + logger().warning(" iTunes automation interface reported an error" " setting EpisodeNumber in iTunes") # If no plugboard transform applied to tags, change the Genre/Category to Series @@ -2977,13 +2981,13 @@ class ITUNES(DriverBase): db_added.TrackNumber = metadata_x.series_index except: if DEBUG: - self.log.warning(" iTunes automation interface reported an error" + logger().warning(" iTunes automation interface reported an error" " setting TrackNumber on iDevice") try: db_added.EpisodeNumber = metadata_x.series_index except: if DEBUG: - self.log.warning(" iTunes automation interface reported an error" + logger().warning(" iTunes automation interface reported an error" " setting EpisodeNumber on iDevice") # If no plugboard transform applied to tags, change the Genre/Category to Series @@ -2997,7 +3001,7 @@ class ITUNES(DriverBase): elif metadata_x.tags is not None: if DEBUG: - self.log.info(" using Tag as Genre") + logger().info(" using Tag as Genre") for tag in metadata_x.tags: if self._is_alpha(tag[0]): if lb_added: @@ -3011,8 +3015,8 @@ class ITUNES(DriverBase): Ensure iDevice metadata is writable. Direct connect mode only ''' if DEBUG: - self.log.info(" ITUNES._wait_for_writable_metadata()") - self.log.warning(" %s" % self.UNSUPPORTED_DIRECT_CONNECT_MODE_MESSAGE) + logger().info(" ITUNES._wait_for_writable_metadata()") + logger().warning(" %s" % self.UNSUPPORTED_DIRECT_CONNECT_MODE_MESSAGE) attempts = 9 while attempts: @@ -3026,40 +3030,40 @@ class ITUNES(DriverBase): attempts -= 1 time.sleep(delay) if DEBUG: - self.log.warning(" waiting %.1f seconds for iDevice metadata to become writable (attempt #%d)" % + logger().warning(" waiting %.1f seconds for iDevice metadata to become writable (attempt #%d)" % (delay, (10 - attempts))) else: if DEBUG: - self.log.error(" failed to write device metadata") + logger().error(" failed to write device metadata") def _xform_metadata_via_plugboard(self, book, format): ''' Transform book metadata from plugboard templates ''' if DEBUG: - self.log.info(" ITUNES._xform_metadata_via_plugboard()") + logger().info(" ITUNES._xform_metadata_via_plugboard()") if self.plugboard_func: pb = self.plugboard_func(self.DEVICE_PLUGBOARD_NAME, format, self.plugboards) newmi = book.deepcopy_metadata() newmi.template_to_attribute(book, pb) if pb is not None and DEBUG: - #self.log.info(" transforming %s using %s:" % (format, pb)) - self.log.info(" title: '%s' %s" % (book.title, ">>> '%s'" % + #logger().info(" transforming %s using %s:" % (format, pb)) + logger().info(" title: '%s' %s" % (book.title, ">>> '%s'" % newmi.title if book.title != newmi.title else '')) - self.log.info(" title_sort: %s %s" % (book.title_sort, ">>> %s" % + logger().info(" title_sort: %s %s" % (book.title_sort, ">>> %s" % newmi.title_sort if book.title_sort != newmi.title_sort else '')) - self.log.info(" authors: %s %s" % (book.authors, ">>> %s" % + logger().info(" authors: %s %s" % (book.authors, ">>> %s" % newmi.authors if book.authors != newmi.authors else '')) - self.log.info(" author_sort: %s %s" % (book.author_sort, ">>> %s" % + logger().info(" author_sort: %s %s" % (book.author_sort, ">>> %s" % newmi.author_sort if book.author_sort != newmi.author_sort else '')) - self.log.info(" language: %s %s" % (book.language, ">>> %s" % + logger().info(" language: %s %s" % (book.language, ">>> %s" % newmi.language if book.language != newmi.language else '')) - self.log.info(" publisher: %s %s" % (book.publisher, ">>> %s" % + logger().info(" publisher: %s %s" % (book.publisher, ">>> %s" % newmi.publisher if book.publisher != newmi.publisher else '')) - self.log.info(" tags: %s %s" % (book.tags, ">>> %s" % + logger().info(" tags: %s %s" % (book.tags, ">>> %s" % newmi.tags if book.tags != newmi.tags else '')) else: if DEBUG: - self.log(" matching plugboard not found") + logger()(" matching plugboard not found") else: newmi = book @@ -3082,7 +3086,7 @@ class ITUNES_ASYNC(ITUNES): def __init__(self,path): if DEBUG: - self.log.info("ITUNES_ASYNC:__init__()") + logger().info("ITUNES_ASYNC:__init__()") if isosx and appscript is None: self.connected = False @@ -3124,15 +3128,15 @@ class ITUNES_ASYNC(ITUNES): """ if not oncard: if DEBUG: - self.log.info("ITUNES_ASYNC:books()") + logger().info("ITUNES_ASYNC:books()") if self.settings().extra_customization[self.CACHE_COVERS]: - self.log.info(" Cover fetching/caching enabled") + logger().info(" Cover fetching/caching enabled") else: - self.log.info(" Cover fetching/caching disabled") + logger().info(" Cover fetching/caching disabled") # Fetch a list of books from iTunes - booklist = BookList(self.log) + booklist = BookList(logger()) cached_books = {} if isosx: @@ -3228,7 +3232,7 @@ class ITUNES_ASYNC(ITUNES): return booklist else: - return BookList(self.log) + return BookList(logger()) def eject(self): ''' @@ -3236,7 +3240,7 @@ class ITUNES_ASYNC(ITUNES): are pending GUI jobs that need to communicate with the device. ''' if DEBUG: - self.log.info("ITUNES_ASYNC:eject()") + logger().info("ITUNES_ASYNC:eject()") self.iTunes = None self.connected = False @@ -3251,7 +3255,7 @@ class ITUNES_ASYNC(ITUNES): particular device doesn't have any of these locations it should return -1. """ if DEBUG: - self.log.info("ITUNES_ASYNC:free_space()") + logger().info("ITUNES_ASYNC:free_space()") free_space = 0 if isosx: s = os.statvfs(os.sep) @@ -3268,7 +3272,7 @@ class ITUNES_ASYNC(ITUNES): @return: (device name, device version, software version on device, mime type) """ if DEBUG: - self.log.info("ITUNES_ASYNC:get_device_information()") + logger().info("ITUNES_ASYNC:get_device_information()") return ('iTunes','hw v1.0','sw v1.0', 'mime type normally goes here') @@ -3291,33 +3295,33 @@ class ITUNES_ASYNC(ITUNES): we need to talk to iTunes to discover if there's a connected iPod ''' if DEBUG: - self.log.info("ITUNES_ASYNC.open(connected_device: %s)" % repr(connected_device)) + logger().info("ITUNES_ASYNC.open(connected_device: %s)" % repr(connected_device)) # Confirm/create thumbs archive if not os.path.exists(self.cache_dir): if DEBUG: - self.log.info(" creating thumb cache '%s'" % self.cache_dir) + logger().info(" creating thumb cache '%s'" % self.cache_dir) os.makedirs(self.cache_dir) if not os.path.exists(self.archive_path): - self.log.info(" creating zip archive") + logger().info(" creating zip archive") zfw = ZipFile(self.archive_path, mode='w') zfw.writestr("iTunes Thumbs Archive",'') zfw.close() else: if DEBUG: - self.log.info(" existing thumb cache at '%s'" % self.archive_path) + logger().info(" existing thumb cache at '%s'" % self.archive_path) # If enabled in config options, create/confirm an iTunes storage folder if not self.settings().extra_customization[self.USE_ITUNES_STORAGE]: self.iTunes_local_storage = os.path.join(config_dir,'iTunes storage') if not os.path.exists(self.iTunes_local_storage): if DEBUG: - self.log(" creating iTunes_local_storage at '%s'" % self.iTunes_local_storage) + logger()(" creating iTunes_local_storage at '%s'" % self.iTunes_local_storage) os.mkdir(self.iTunes_local_storage) else: if DEBUG: - self.log(" existing iTunes_local_storage at '%s'" % self.iTunes_local_storage) + logger()(" existing iTunes_local_storage at '%s'" % self.iTunes_local_storage) def sync_booklists(self, booklists, end_session=True): ''' @@ -3328,7 +3332,7 @@ class ITUNES_ASYNC(ITUNES): ''' if DEBUG: - self.log.info("ITUNES_ASYNC.sync_booklists()") + logger().info("ITUNES_ASYNC.sync_booklists()") # Inform user of any problem books if self.problem_titles: @@ -3342,7 +3346,7 @@ class ITUNES_ASYNC(ITUNES): ''' ''' if DEBUG: - self.log.info("ITUNES_ASYNC:unmount_device()") + logger().info("ITUNES_ASYNC:unmount_device()") self.connected = False class BookList(list): diff --git a/src/calibre/devices/cybook/t2b.py b/src/calibre/devices/cybook/t2b.py index 7aaeeb63d7..fc0c772bf7 100644 --- a/src/calibre/devices/cybook/t2b.py +++ b/src/calibre/devices/cybook/t2b.py @@ -5,7 +5,6 @@ Write a t2b file to disk. ''' import StringIO -from PIL import Image DEFAULT_T2B_DATA = '\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x00\x0f\xff\xff\xff\xf0\xff\x0f\xc3\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xf8\x00\x00\xff\xff\xff\xf0\xff\x0f\xc3\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xe0\xff\xf0\xff\xff\xff\xf0\xff\xff\xc3\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xc3\xff\xff\xff\xff\xff\xf0\xff\xff\xc3\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x07\xff\xff\xfc\x00?\xf0\xff\x0f\xc3\x00?\xf0\xc0\xfe\x00?\xff\xff\xff\xff\xff\xff\xff\x0f\xff\xff\xf0<\x0f\xf0\xff\x0f\xc0,\x0f\xf0\x0e\xf0,\x0f\xff\xff\xff\xff\xff\xff\xff\x0f\xff\xff\xff\xff\xc3\xf0\xff\x0f\xc0\xff\x0f\xf0\xff\xf0\xff\xc7\xff\xff\xff\xff\xff\xff\xff\x0f\xff\xff\xff\xff\xc3\xf0\xff\x0f\xc3\xff\xc3\xf0\xff\xc3\xff\xc3\xff\xff\xff\xff\xff\xff\xff\x0f\xff\xff\xff\x00\x03\xf0\xff\x0f\xc3\xff\xc3\xf0\xff\xc3\xff\xc3\xff\xff\xff\xff\xff\xff\xff\x0f\xff\xff\xf0\x1f\xc3\xf0\xff\x0f\xc3\xff\xc3\xf0\xff\xc0\x00\x03\xff\xff\xff\xff\xff\xff\xff\x0b\xff\xff\xf0\xff\xc3\xf0\xff\x0f\xc3\xff\xc3\xf0\xff\xc3\xff\xff\xff\xff\xff\xff\xff\xff\xff\xc3\xff\xff\xf3\xff\xc3\xf0\xff\x0f\xc3\xff\xc3\xf0\xff\xc3\xff\xff\xff\xff\xff\xff\xff\xff\xff\xc0\xff\xfc\xf0\xff\x03\xf0\xff\x0f\xc0\xff\x0f\xf0\xff\xf0\xff\xff\xff\xff\xff\xff\xff\xff\xff\xf0\x0f\x00\xf08\x03\xf0\xff\x0f\xc0,\x0f\xf0\xff\xf0\x1f\x03\xff\xff\xff\xff\xff\xff\xff\xff\x00\x0f\xfc\x00\xc3\xf0\xff\x0f\xc3\x00?\xf0\xff\xff\x00\x0f\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xf0\x0f\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xf0\x00\x00\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x03\xfe\x94\xff\xff\xff\xff\xff\xff\xff\xff\xff\xc0\x00\x00\x00\x0f\xff\xff\xff\xff\xff\xff\xfc\x7f\xfe\x94\xff\xff\xff\xff\xff\xff\xff\xff\xfc\x0f\xff\xfe\xa9@\xff\xff\xff\xff\xff\xff\xfc?\xfe\xa4\xff\xff\xff\xff\xff\xff\xff\xff\xfc\xff\xff\xff\xe9P\xff\xff\xff\xff\xff\xff\xfe/\xfe\xa8\xff\xff\xff\xff\xff\xff\xff\xff\xfc\xff\xff\xff\xf9T\xff\xff\xff\xff\xf0@\x00+\xfa\xa8?\xff\xff\xff\xff\xff\xff\xff\xfc\xbf\xff\xff\xf9T\xff\xff\xff\xff\xcb\xe4}*\xaa\xaa?\xff\xff\xff\xff\xff\xff\xff\xfc\xbf\xff\xff\xe9T\xff\xff\xff\xff\xc7\xe4\xfd\x1a\xaa\xaa?\xff\xff\xff\xff\xff\xff\xff\xfc\xaf\xea\xaa\xa6\xa4\xff@\x00\x0f\xc3\xe8\xfe\x1a\xaa\xaa?\xff\xff\xff\xff\xff\xff\xff\xfcj\x95UZ\xa4\x00\x7f\xfe\x90\x03\xe8\xfe\n\xaa\xaa?\xff\xff\xff\xff\xff\xff\xff\xfcj\x95UZ\xa4?\xff\xff\xa5C\xe8\xfe\x06\xaa\xaa?\xff\xff\xff\xff\xff\xff\xff\xfcj\x95UZ\xa4?\xff\xff\xeaC\xe8\xbe\x06\xaa\xaa\x0f\xff\xff\xff\xff\xff\xff\xff\xfcj\x95UZ\xa4/\xff\xff\xea\x82\xe8j\x06\xaa\xaa\x0f\xff\xff\xff\xff\xff\xff\xff\xfcj\x95UZ\xa4/\xff\xff\xaa\x82\xe8*F\xaa\xaa\x8f\xff\xff\xff\xff\xff\xff\xff\xfcj\x95UZ\xa4+\xff\xfe\xaa\x82\xe8*\x86\xaa\xaa\x8f\xff\xff\x80\xff\xff\xff\xff\xfcj\x95UV\xa4\x1a\xfa\xaa\xaa\x82\xe8*\x86\xaa\xaa\x8f\xf0\x00T?\xff\xff\xff\xfcj\x95UV\xa4\x1a\xfa\xaa\xaa\x82\xe8*\x81\xaa\xaa\x8c\x03\xff\x95?\xff\xff\xff\xfcj\x95UV\xa4\x1a\xfa\xaa\xaa\x82\xe8*\x81\xaa\xaa\x80\xbf\xff\x95?\xff\xff\xff\xfcj\x95UV\xa4\x1a\xfa\xaa\xaa\x82\xe8*\x81\xaa\xaa\x9b\xff\xff\x95\x0f\xff\xff\xff\xfcj\x95UV\xa4\x1a\xfa\xaa\xaa\x82\xe8\x1a\x81\xaa\xaa\x9a\xff\xfe\x95\x0f\xff\xff\xff\xfcj\x95UV\xa4\x1a\xfa\xaa\xaa\x82\xe8\n\x81\xaa\xaa\xa6\xbf\xfeUO\xff\xff\xff\xfcj\x95UV\xa4\x1a\xfa\xaa\xaa\x82\xa8\n\x91j\xaa\xa5\xaa\xa9ZO\xff\xff\xff\xfcj\x95UV\xa4\x1a\xfa\xaa\xaa\x82\xa8\n\xa0j\xaa\xa5Z\x95ZO\xff\xff\xff\xfcj\x95UV\xa4*\xfa\xaa\xaa\x82\xa9\n\xa0j\xaa\xa5UUZC\xff\xff\xff\xfcj\x95UV\xa4*\xfa\xaa\xaa\x82\xaa\n\xa0j\xaa\xa4UUZS\xff\xff\xff\xfcZ\x95UV\xa4*\xfa\xaa\xaa\x82\xaa\n\xa0j\xaa\xa4UUZS\xff\xff\xff\xfcZ\x95UU\xa4*\xfa\xaa\xaa\x82\xaa\n\xa0j\xaa\xa8UUVS\xff\xff\xff\xfcZ\x95UU\xa4*\xea\xaa\xaa\x82\xaa\x06\xa0Z\xaa\xa8UUV\x93\xff\xff\xff\xfcZ\x95UU\xa4*\xaa\xaa\xaa\x81\xaa\x02\xa0\x1a\xaa\xa8UUV\x90\xff\xff\xff\xfcZ\x95UU\xa4*\xaa\xaa\xaa\x80\xaa\x02\xa0\x1a\xaa\xa8\x15UU\x94\xff\xff\xff\xfcZ\x95UU\xa4*\xaa\xaa\xaa\x80\xaa"\xa0\x1a\xaa\xa8\x15UU\x94\xff\xff\xff\xfcZ\x95UU\xa4*\xaa\xaa\xaa\x80\xaa2\xa4\x16\xaa\xa8\x15UU\x94\xff\xff\xff\xfcZ\x95UU\xa4*\xaa\xaa\xaa\x80\xaa2\xa8\x16\xa6\xa9\x15UU\x94\xff\xff\xff\xfcZ\x95UU\xa4*\xaa\xaa\xaa\x80\xaa2\xa8\x16\xa6\xa9\x05UUT?\xff\xff\xfcZ\x95UU\xa4*\xaa\xaa\xaa\x84\xaa2\xa8\x16\xaa\xaa\x05UUU?\xff\xff\xfcZ\x95UU\xa4*\xaa\xaa\xaa\x88\xaa2\xa8\x06\xaa\xaa\x05UUU?\xff\xff\xfcZ\x95UU\xa4*\xaa\xaa\xaa\x8c\xaa1\xa8\xc5\xaa\xaa\x05UUU?\xff\xff\xfcZ\x95UU\xa4*\xaa\xaa\xaa\x8c\xaa0\xa8E\xa9\xaa\x05UUU/\xff\xff\xfcZ\x95UU\xa4*\xaa\xaa\xaa\x8c\xaa<\xa8\x05\xa9\xaaAUUU\x0f\xff\xff\xfcZ\x95UU\xa4*\xaa\xaa\xaa\x8c\xaa<\xa8\x05\xa9\xaaAUUUO\xff\xff\xfcZ\x95UU\xa4*\xaa\xaa\xaa\x8c\xaa<\xa9\x05\xaa\xaaAUUUO\xff\xff\xfcZ\x95UU\xa4*\xaa\xaa\xaa\x8c\xaa\x1c\xaa\x01\xaa\xaa\x81UUUO\xff\xff\xfcZ\x95UU\xa4*\xaa\xaa\xaa\x8c\xaa\x0c\xaa\x01\xaa\xaa\x81UUUO\xff\xff\xfcZ\x95UU\xa4*\xaa\xaa\xaa\x8c\xaa\x0c\xaa1j\xaa\x80UUUC\xff\xff\xfcZ\x95UU\xa4*\xaa\xaa\xaa\x8c\xaa\x0cj1jj\x90UUUS\xff\xff\xfcZ\x95UU\xa4*\xaa\xaa\xaa\x8c\xaa\x0c*1jj\x90UUUS\xff\xff\xfcZ\x95UU\xa4*\xaa\xaa\xaa\x8c\xaaL*1jj\xa0UUUS\xff\xff\xfcZ\x95UU\xa4*\xaa\xaa\xaa\x8c\xaa\x8f* j\xaa\xa0\x15UUS\xff\xff\xfcZ\x95UU\xa4*\xaa\xaa\xaa\x8c\xaa\x8f*@j\xaa\xa0\x15UUP\xff\xff\xfcZ\x95UU\xa4*\xaa\xaa\xaa\x8c\xaa\x8f*\x8cZ\xaa\xa1\x15UUT\xff\xff\xfcZ\x95UU\xa4j\xaa\xaa\xaa\x8c\xaa\x8f*\x8cZ\x9a\xa0\x15UUT\xff\xff\xfcZ\x95UU\xa4j\xaa\xaa\xaa\x8c\xaa\x8f*\x8cZ\x9a\xa0\x15UUT\xff\xff\xfcZ\x95UU\xa4j\xaa\xaa\xaa\x8c\xaa\x8f\x1a\x8cZ\x9a\xa4\x15UUT?\xff\xfcZ\x95UU\x94j\xaa\xaa\xaa\x8cj\x8f\n\x8cVj\xa4\x05UU\xa4?\xff\xfcVUUU\xa4j\xaa\xaa\xaa\x8cj\x8fJ\x8c\x16\xaa\xa8\xc5UZ\xa5?\xff\xfcUUUV\xa4j\xaa\xaa\xaa\x8cj\x8f\xca\x8f\x16\xaa\xa8\xc5V\xaa\xa5?\xff\xfcUj\xaa\xaa\xa4j\xaa\xaa\xaa\x8cj\x8f\xca\x8f\x1a\xaa\xa8\x05Z\xaaU?\xff\xfcV\xaa\xaa\xaa\xa5j\xaa\xaa\xaa\x8e*\x8f\xca\x83\x1a\xaa\xa4\x01eUU?\xff\xfcZ\xaa\xaa\xaa\xa5j\xaa\xaa\xaa\x8f*\x8f\xca\x83\x1a\xa5U\x01U\x00\x00\x0f\xff\xfcUUUUUZ\xaa\xaa\xaaO%\x8f\xc6\x93\x15\x00\x001@\x0f\xff\xff\xff\xfcP\x00\x00\x00\x15\x00\x00\x00\x00\x0f\x00\x07\xc0\x03\x00\xff\xff0\x1f\xff\xff\xff\xff\xfc\x00\xff\xff\xf8\x00?\xff\xff\xff\x0f?\xc7\xc3\xf7\x0f\xff\xff\xf1\xff\xff\xff\xff\xff\xfc\xff\xff\xff\xff\xf4\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff' @@ -18,7 +17,7 @@ def reduce_color(c): return 2 else: return 3 - + def i2b(n): return "".join([str((n >> y) & 1) for y in range(1, -1, -1)]) @@ -27,12 +26,13 @@ def write_t2b(t2bfile, coverdata=None): t2bfile is a file handle ready to write binary data to disk. coverdata is a string representation of a JPEG file. ''' + from PIL import Image if coverdata != None: coverdata = StringIO.StringIO(coverdata) cover = Image.open(coverdata).convert("L") cover.thumbnail((96, 144), Image.ANTIALIAS) t2bcover = Image.new('L', (96, 144), 'white') - + x, y = cover.size t2bcover.paste(cover, ((96-x)/2, (144-y)/2)) diff --git a/src/calibre/devices/kobo/bookmark.py b/src/calibre/devices/kobo/bookmark.py index 8e199f77a6..afb392403d 100644 --- a/src/calibre/devices/kobo/bookmark.py +++ b/src/calibre/devices/kobo/bookmark.py @@ -7,7 +7,6 @@ __docformat__ = 'restructuredtext en' import os from contextlib import closing -import sqlite3 as sqlite class Bookmark(): # {{{ ''' @@ -32,7 +31,7 @@ class Bookmark(): # {{{ def get_bookmark_data(self): ''' Return the timestamp and last_read_location ''' - + import sqlite3 as sqlite user_notes = {} self.timestamp = os.path.getmtime(self.path) with closing(sqlite.connect(self.db_path)) as connection: diff --git a/src/calibre/devices/kobo/driver.py b/src/calibre/devices/kobo/driver.py index 0bc578155d..f68ea8feff 100644 --- a/src/calibre/devices/kobo/driver.py +++ b/src/calibre/devices/kobo/driver.py @@ -6,7 +6,6 @@ __copyright__ = '2010, Timothy Legge and Kovid Goyal ]+encoding\s*=\s*[\'"](.*?)[\'"][^<>]*>', @@ -34,8 +33,13 @@ def substitute_entites(raw): _CHARSET_ALIASES = { "macintosh" : "mac-roman", "x-sjis" : "shift-jis" } +def detect(*args, **kwargs): + from chardet import detect + return detect(*args, **kwargs) + def force_encoding(raw, verbose, assume_utf8=False): from calibre.constants import preferred_encoding + try: chardet = detect(raw[:1024*50]) except: diff --git a/src/calibre/ebooks/conversion/plugins/pdf_input.py b/src/calibre/ebooks/conversion/plugins/pdf_input.py index 0a3821c584..be0150834b 100644 --- a/src/calibre/ebooks/conversion/plugins/pdf_input.py +++ b/src/calibre/ebooks/conversion/plugins/pdf_input.py @@ -7,8 +7,6 @@ __docformat__ = 'restructuredtext en' import os from calibre.customize.conversion import InputFormatPlugin, OptionRecommendation -from calibre.constants import plugins -pdfreflow, pdfreflow_err = plugins['pdfreflow'] class PDFInput(InputFormatPlugin): @@ -29,6 +27,9 @@ class PDFInput(InputFormatPlugin): ]) def convert_new(self, stream, accelerators): + from calibre.constants import plugins + pdfreflow, pdfreflow_err = plugins['pdfreflow'] + from calibre.ebooks.pdf.reflow import PDFDocument from calibre.utils.cleantext import clean_ascii_chars if pdfreflow_err: diff --git a/src/calibre/ebooks/epub/fix/epubcheck.py b/src/calibre/ebooks/epub/fix/epubcheck.py index 9e812e1cf4..0029868c23 100644 --- a/src/calibre/ebooks/epub/fix/epubcheck.py +++ b/src/calibre/ebooks/epub/fix/epubcheck.py @@ -6,7 +6,6 @@ __copyright__ = '2010, Kovid Goyal ' __docformat__ = 'restructuredtext en' from calibre.ebooks.epub.fix import ePubFixer, InvalidEpub -from calibre.utils.date import parse_date, strptime class Epubcheck(ePubFixer): @@ -35,6 +34,8 @@ class Epubcheck(ePubFixer): return 'epubcheck' def fix_pubdates(self): + from calibre.utils.date import parse_date, strptime + dirtied = False opf = self.container.opf for dcdate in opf.xpath('//dc:date', diff --git a/src/calibre/ebooks/metadata/book/base.py b/src/calibre/ebooks/metadata/book/base.py index 286bcee9d0..0312a7db6a 100644 --- a/src/calibre/ebooks/metadata/book/base.py +++ b/src/calibre/ebooks/metadata/book/base.py @@ -710,7 +710,7 @@ class Metadata(object): fmt('Title sort', self.title_sort) if self.authors: fmt('Author(s)', authors_to_string(self.authors) + \ - ((' [' + self.author_sort + ']') + ((' [' + self.author_sort + ']') if self.author_sort and self.author_sort != _('Unknown') else '')) if self.publisher: fmt('Publisher', self.publisher) diff --git a/src/calibre/ebooks/metadata/book/json_codec.py b/src/calibre/ebooks/metadata/book/json_codec.py index a14e18569a..c0c3900a5d 100644 --- a/src/calibre/ebooks/metadata/book/json_codec.py +++ b/src/calibre/ebooks/metadata/book/json_codec.py @@ -12,7 +12,6 @@ from calibre.ebooks.metadata.book import SERIALIZABLE_FIELDS from calibre.constants import filesystem_encoding, preferred_encoding from calibre.library.field_metadata import FieldMetadata from calibre.utils.date import parse_date, isoformat, UNDEFINED_DATE, local_tz -from calibre.utils.magick import Image from calibre import isbytestring # Translate datetimes to and from strings. The string form is the datetime in @@ -37,6 +36,8 @@ def encode_thumbnail(thumbnail): ''' Encode the image part of a thumbnail, then return the 3 part tuple ''' + from calibre.utils.magick import Image + if thumbnail is None: return None if not isinstance(thumbnail, (tuple, list)): diff --git a/src/calibre/ebooks/metadata/sources/amazon.py b/src/calibre/ebooks/metadata/sources/amazon.py index cae31abe09..3d08b96c5f 100644 --- a/src/calibre/ebooks/metadata/sources/amazon.py +++ b/src/calibre/ebooks/metadata/sources/amazon.py @@ -12,19 +12,14 @@ from urllib import urlencode from threading import Thread from Queue import Queue, Empty -from lxml.html import tostring from calibre import as_unicode from calibre.ebooks.metadata import check_isbn from calibre.ebooks.metadata.sources.base import (Source, Option, fixcase, fixauthors) -from calibre.utils.cleantext import clean_ascii_chars -from calibre.ebooks.chardet import xml_to_unicode from calibre.ebooks.metadata.book.base import Metadata -from calibre.library.comments import sanitize_comments_html from calibre.utils.date import parse_date from calibre.utils.localization import canonicalize_lang -from calibre.utils.soupparser import fromstring class Worker(Thread): # Get details {{{ @@ -43,6 +38,8 @@ class Worker(Thread): # Get details {{{ self.browser = browser.clone_browser() self.cover_url = self.amazon_id = self.isbn = None self.domain = domain + from lxml.html import tostring + self.tostring = tostring months = { 'de': { @@ -176,6 +173,10 @@ class Worker(Thread): # Get details {{{ self.log.exception('get_details failed for url: %r'%self.url) def get_details(self): + from calibre.utils.cleantext import clean_ascii_chars + from calibre.utils.soupparser import fromstring + from calibre.ebooks.chardet import xml_to_unicode + try: raw = self.browser.open_novisit(self.url, timeout=self.timeout).read().strip() except Exception as e: @@ -210,7 +211,7 @@ class Worker(Thread): # Get details {{{ errmsg = root.xpath('//*[@id="errorMessage"]') if errmsg: msg = 'Failed to parse amazon details page: %r'%self.url - msg += tostring(errmsg, method='text', encoding=unicode).strip() + msg += self.tostring(errmsg, method='text', encoding=unicode).strip() self.log.error(msg) return @@ -322,10 +323,10 @@ class Worker(Thread): # Get details {{{ tdiv = root.xpath('//h1[contains(@class, "parseasinTitle")]')[0] actual_title = tdiv.xpath('descendant::*[@id="btAsinTitle"]') if actual_title: - title = tostring(actual_title[0], encoding=unicode, + title = self.tostring(actual_title[0], encoding=unicode, method='text').strip() else: - title = tostring(tdiv, encoding=unicode, method='text').strip() + title = self.tostring(tdiv, encoding=unicode, method='text').strip() return re.sub(r'[(\[].*[)\]]', '', title).strip() def parse_authors(self, root): @@ -337,7 +338,7 @@ class Worker(Thread): # Get details {{{ ''') for x in aname: x.tail = '' - authors = [tostring(x, encoding=unicode, method='text').strip() for x + authors = [self.tostring(x, encoding=unicode, method='text').strip() for x in aname] authors = [a for a in authors if a] return authors @@ -356,6 +357,8 @@ class Worker(Thread): # Get details {{{ return float(m.group(1))/float(m.group(3)) * 5 def parse_comments(self, root): + from calibre.library.comments import sanitize_comments_html + desc = root.xpath('//div[@id="productDescription"]/*[@class="content"]') if desc: desc = desc[0] @@ -365,7 +368,7 @@ class Worker(Thread): # Get details {{{ for a in desc.xpath('descendant::a[@href]'): del a.attrib['href'] a.tag = 'span' - desc = tostring(desc, method='html', encoding=unicode).strip() + desc = self.tostring(desc, method='html', encoding=unicode).strip() # Encoding bug in Amazon data U+fffd (replacement char) # in some examples it is present in place of ' @@ -602,6 +605,11 @@ class Amazon(Source): Note this method will retry without identifiers automatically if no match is found with identifiers. ''' + from lxml.html import tostring + from calibre.utils.cleantext import clean_ascii_chars + from calibre.utils.soupparser import fromstring + from calibre.ebooks.chardet import xml_to_unicode + query, domain = self.create_query(log, title=title, authors=authors, identifiers=identifiers) if query is None: diff --git a/src/calibre/ebooks/metadata/sources/base.py b/src/calibre/ebooks/metadata/sources/base.py index 9ae8902671..4c334f4e46 100644 --- a/src/calibre/ebooks/metadata/sources/base.py +++ b/src/calibre/ebooks/metadata/sources/base.py @@ -12,7 +12,6 @@ from future_builtins import map from calibre import browser, random_user_agent from calibre.customize import Plugin -from calibre.utils.logging import ThreadSafeLog, FileStream from calibre.utils.config import JSONConfig from calibre.utils.titlecase import titlecase from calibre.utils.icu import capitalize, lower, upper @@ -34,6 +33,7 @@ msprefs.defaults['fewer_tags'] = True msprefs.defaults['cover_priorities'] = {'Google':2} def create_log(ostream=None): + from calibre.utils.logging import ThreadSafeLog, FileStream log = ThreadSafeLog(level=ThreadSafeLog.DEBUG) log.outputs = [FileStream(ostream)] return log diff --git a/src/calibre/ebooks/metadata/sources/douban.py b/src/calibre/ebooks/metadata/sources/douban.py index 06e874e8ca..6857d62d4d 100644 --- a/src/calibre/ebooks/metadata/sources/douban.py +++ b/src/calibre/ebooks/metadata/sources/douban.py @@ -12,14 +12,10 @@ from urllib import urlencode from functools import partial from Queue import Queue, Empty -from lxml import etree from calibre.ebooks.metadata import check_isbn from calibre.ebooks.metadata.sources.base import Source from calibre.ebooks.metadata.book.base import Metadata -from calibre.ebooks.chardet import xml_to_unicode -from calibre.utils.date import parse_date, utcnow -from calibre.utils.cleantext import clean_ascii_chars from calibre import as_unicode NAMESPACES = { @@ -28,22 +24,6 @@ NAMESPACES = { 'db': 'http://www.douban.com/xmlns/', 'gd': 'http://schemas.google.com/g/2005' } -XPath = partial(etree.XPath, namespaces=NAMESPACES) -total_results = XPath('//openSearch:totalResults') -start_index = XPath('//openSearch:startIndex') -items_per_page = XPath('//openSearch:itemsPerPage') -entry = XPath('//atom:entry') -entry_id = XPath('descendant::atom:id') -title = XPath('descendant::atom:title') -description = XPath('descendant::atom:summary') -publisher = XPath("descendant::db:attribute[@name='publisher']") -isbn = XPath("descendant::db:attribute[@name='isbn13']") -date = XPath("descendant::db:attribute[@name='pubdate']") -creator = XPath("descendant::db:attribute[@name='author']") -booktag = XPath("descendant::db:tag/attribute::name") -rating = XPath("descendant::gd:rating/attribute::average") -cover_url = XPath("descendant::atom:link[@rel='image']/attribute::href") - def get_details(browser, url, timeout): # {{{ try: if Douban.DOUBAN_API_KEY and Douban.DOUBAN_API_KEY != '': @@ -61,6 +41,25 @@ def get_details(browser, url, timeout): # {{{ # }}} def to_metadata(browser, log, entry_, timeout): # {{{ + from lxml import etree + from calibre.ebooks.chardet import xml_to_unicode + from calibre.utils.date import parse_date, utcnow + from calibre.utils.cleantext import clean_ascii_chars + + XPath = partial(etree.XPath, namespaces=NAMESPACES) + entry = XPath('//atom:entry') + entry_id = XPath('descendant::atom:id') + title = XPath('descendant::atom:title') + description = XPath('descendant::atom:summary') + publisher = XPath("descendant::db:attribute[@name='publisher']") + isbn = XPath("descendant::db:attribute[@name='isbn13']") + date = XPath("descendant::db:attribute[@name='pubdate']") + creator = XPath("descendant::db:attribute[@name='author']") + booktag = XPath("descendant::db:tag/attribute::name") + rating = XPath("descendant::gd:rating/attribute::average") + cover_url = XPath("descendant::atom:link[@rel='image']/attribute::href") + + def get_text(extra, x): try: ans = x(extra) @@ -275,6 +274,7 @@ class Douban(Source): def get_all_details(self, br, log, entries, abort, # {{{ result_queue, timeout): + from lxml import etree for relevance, i in enumerate(entries): try: ans = to_metadata(br, log, i, timeout) @@ -298,6 +298,13 @@ class Douban(Source): def identify(self, log, result_queue, abort, title=None, authors=None, # {{{ identifiers={}, timeout=30): + from lxml import etree + from calibre.ebooks.chardet import xml_to_unicode + from calibre.utils.cleantext import clean_ascii_chars + + XPath = partial(etree.XPath, namespaces=NAMESPACES) + entry = XPath('//atom:entry') + query = self.create_query(log, title=title, authors=authors, identifiers=identifiers) if not query: diff --git a/src/calibre/ebooks/metadata/sources/google.py b/src/calibre/ebooks/metadata/sources/google.py index f9c43d86cc..3962afcb5e 100644 --- a/src/calibre/ebooks/metadata/sources/google.py +++ b/src/calibre/ebooks/metadata/sources/google.py @@ -12,8 +12,6 @@ from urllib import urlencode from functools import partial from Queue import Queue, Empty -from lxml import etree - from calibre.ebooks.metadata import check_isbn from calibre.ebooks.metadata.sources.base import Source from calibre.ebooks.metadata.book.base import Metadata @@ -29,23 +27,6 @@ NAMESPACES = { 'dc' : 'http://purl.org/dc/terms', 'gd' : 'http://schemas.google.com/g/2005' } -XPath = partial(etree.XPath, namespaces=NAMESPACES) - -total_results = XPath('//openSearch:totalResults') -start_index = XPath('//openSearch:startIndex') -items_per_page = XPath('//openSearch:itemsPerPage') -entry = XPath('//atom:entry') -entry_id = XPath('descendant::atom:id') -creator = XPath('descendant::dc:creator') -identifier = XPath('descendant::dc:identifier') -title = XPath('descendant::dc:title') -date = XPath('descendant::dc:date') -publisher = XPath('descendant::dc:publisher') -subject = XPath('descendant::dc:subject') -description = XPath('descendant::dc:description') -language = XPath('descendant::dc:language') -rating = XPath('descendant::gd:rating[@average]') - def get_details(browser, url, timeout): # {{{ try: raw = browser.open_novisit(url, timeout=timeout).read() @@ -61,6 +42,24 @@ def get_details(browser, url, timeout): # {{{ # }}} def to_metadata(browser, log, entry_, timeout): # {{{ + from lxml import etree + XPath = partial(etree.XPath, namespaces=NAMESPACES) + + # total_results = XPath('//openSearch:totalResults') + # start_index = XPath('//openSearch:startIndex') + # items_per_page = XPath('//openSearch:itemsPerPage') + entry = XPath('//atom:entry') + entry_id = XPath('descendant::atom:id') + creator = XPath('descendant::dc:creator') + identifier = XPath('descendant::dc:identifier') + title = XPath('descendant::dc:title') + date = XPath('descendant::dc:date') + publisher = XPath('descendant::dc:publisher') + subject = XPath('descendant::dc:subject') + description = XPath('descendant::dc:description') + language = XPath('descendant::dc:language') + rating = XPath('descendant::gd:rating[@average]') + def get_text(extra, x): try: @@ -266,6 +265,7 @@ class GoogleBooks(Source): def get_all_details(self, br, log, entries, abort, # {{{ result_queue, timeout): + from lxml import etree for relevance, i in enumerate(entries): try: ans = to_metadata(br, log, i, timeout) @@ -289,6 +289,10 @@ class GoogleBooks(Source): def identify(self, log, result_queue, abort, title=None, authors=None, # {{{ identifiers={}, timeout=30): + from lxml import etree + XPath = partial(etree.XPath, namespaces=NAMESPACES) + entry = XPath('//atom:entry') + query = self.create_query(log, title=title, authors=authors, identifiers=identifiers) if not query: diff --git a/src/calibre/ebooks/metadata/sources/isbndb.py b/src/calibre/ebooks/metadata/sources/isbndb.py index 1da7f906bb..7e15ad275e 100644 --- a/src/calibre/ebooks/metadata/sources/isbndb.py +++ b/src/calibre/ebooks/metadata/sources/isbndb.py @@ -9,12 +9,9 @@ __docformat__ = 'restructuredtext en' from urllib import quote -from lxml import etree from calibre.ebooks.metadata import check_isbn from calibre.ebooks.metadata.sources.base import Source, Option -from calibre.ebooks.chardet import xml_to_unicode -from calibre.utils.cleantext import clean_ascii_chars from calibre.utils.icu import lower from calibre.ebooks.metadata.book.base import Metadata @@ -122,6 +119,7 @@ class ISBNDB(Source): result_queue.put(result) def parse_feed(self, feed, seen, orig_title, orig_authors, identifiers): + from lxml import etree def tostring(x): if x is None: @@ -198,6 +196,10 @@ class ISBNDB(Source): def make_query(self, q, abort, title=None, authors=None, identifiers={}, max_pages=10, timeout=30): + from lxml import etree + from calibre.ebooks.chardet import xml_to_unicode + from calibre.utils.cleantext import clean_ascii_chars + page_num = 1 parser = etree.XMLParser(recover=True, no_network=True) br = self.browser diff --git a/src/calibre/ebooks/metadata/sources/overdrive.py b/src/calibre/ebooks/metadata/sources/overdrive.py index 1164567ff5..bb1bbb9d42 100755 --- a/src/calibre/ebooks/metadata/sources/overdrive.py +++ b/src/calibre/ebooks/metadata/sources/overdrive.py @@ -9,18 +9,14 @@ __docformat__ = 'restructuredtext en' ''' Fetch metadata using Overdrive Content Reserve ''' -import re, random, mechanize, copy, json +import re, random, copy, json from threading import RLock from Queue import Queue, Empty -from lxml import html from calibre.ebooks.metadata import check_isbn from calibre.ebooks.metadata.sources.base import Source, Option from calibre.ebooks.metadata.book.base import Metadata -from calibre.ebooks.chardet import xml_to_unicode -from calibre.library.comments import sanitize_comments_html -from calibre.utils.soupparser import fromstring ovrdrv_data_cache = {} cache_lock = RLock() @@ -80,6 +76,7 @@ class OverDrive(Source): def download_cover(self, log, result_queue, abort, # {{{ title=None, authors=None, identifiers={}, timeout=30): + import mechanize cached_url = self.get_cached_cover_url(identifiers) if cached_url is None: log.info('No cached cover found, running identify') @@ -170,6 +167,7 @@ class OverDrive(Source): this page attempts to set a cookie that Mechanize doesn't like copy the cookiejar to a separate instance and make a one-off request with the temp cookiejar ''' + import mechanize goodcookies = br._ua_handlers['_cookies'].cookiejar clean_cj = mechanize.CookieJar() cookies_to_copy = [] @@ -187,6 +185,7 @@ class OverDrive(Source): br.set_cookiejar(clean_cj) def overdrive_search(self, br, log, q, title, author): + import mechanize # re-initialize the cookiejar to so that it's clean clean_cj = mechanize.CookieJar() br.set_cookiejar(clean_cj) @@ -303,6 +302,7 @@ class OverDrive(Source): return '' def overdrive_get_record(self, br, log, q, ovrdrv_id): + import mechanize search_url = q+'SearchResults.aspx?ReserveID={'+ovrdrv_id+'}' results_url = q+'SearchResults.svc/GetResults?sEcho=1&iColumns=18&sColumns=ReserveID%2CTitle%2CSubtitle%2CEdition%2CSeries%2CPublisher%2CFormat%2CFormatID%2CCreators%2CThumbImage%2CShortDescription%2CWorldCatLink%2CExcerptLink%2CCreatorFile%2CSortTitle%2CAvailableToLibrary%2CAvailableToRetailer%2CRelevancyRank&iDisplayStart=0&iDisplayLength=10&sSearch=&bEscapeRegex=true&iSortingCols=1&iSortCol_0=17&sSortDir_0=asc' @@ -393,6 +393,11 @@ class OverDrive(Source): def get_book_detail(self, br, metadata_url, mi, ovrdrv_id, log): + from lxml import html + from calibre.ebooks.chardet import xml_to_unicode + from calibre.utils.soupparser import fromstring + from calibre.library.comments import sanitize_comments_html + try: raw = br.open_novisit(metadata_url).read() except Exception, e: diff --git a/src/calibre/ebooks/metadata/sources/ozon.py b/src/calibre/ebooks/metadata/sources/ozon.py index de45e0b8db..d40e43d582 100644 --- a/src/calibre/ebooks/metadata/sources/ozon.py +++ b/src/calibre/ebooks/metadata/sources/ozon.py @@ -6,15 +6,11 @@ __copyright__ = '2011, Roman Mukhin ' __docformat__ = 'restructuredtext en' import re -import urllib2 import datetime from urllib import quote_plus from Queue import Queue, Empty -from lxml import etree, html + from calibre import as_unicode - -from calibre.ebooks.chardet import xml_to_unicode - from calibre.ebooks.metadata import check_isbn from calibre.ebooks.metadata.sources.base import Source from calibre.ebooks.metadata.book.base import Metadata @@ -43,6 +39,7 @@ class Ozon(Source): isbnRegex = re.compile(isbnPattern) def get_book_url(self, identifiers): # {{{ + import urllib2 ozon_id = identifiers.get('ozon', None) res = None if ozon_id: @@ -81,6 +78,9 @@ class Ozon(Source): def identify(self, log, result_queue, abort, title=None, authors=None, identifiers={}, timeout=30): # {{{ + from lxml import etree + from calibre.ebooks.chardet import xml_to_unicode + if not self.is_configured(): return query = self.create_query(log, title=title, authors=authors, identifiers=identifiers) @@ -283,6 +283,9 @@ class Ozon(Source): # }}} def get_book_details(self, log, metadata, timeout): # {{{ + from lxml import html, etree + from calibre.ebooks.chardet import xml_to_unicode + url = self.get_book_url(metadata.get_identifiers())[2] raw = self.browser.open_novisit(url, timeout=timeout).read() diff --git a/src/calibre/library/catalogs/bibtex.py b/src/calibre/library/catalogs/bibtex.py index 1815940931..b959f22eac 100644 --- a/src/calibre/library/catalogs/bibtex.py +++ b/src/calibre/library/catalogs/bibtex.py @@ -12,7 +12,6 @@ from types import StringType, UnicodeType from calibre import (strftime) from calibre.customize import CatalogPlugin from calibre.library.catalogs import FIELDS, TEMPLATE_ALLOWED_FIELDS -from calibre.utils.logging import default_log as log from calibre.customize.conversion import DummyReporter from calibre.constants import preferred_encoding @@ -113,6 +112,7 @@ class BIBTEX(CatalogPlugin): from calibre.utils.bibtex import BibTeX from calibre.library.save_to_disk import preprocess_template from calibre.utils.date import now as nowf + from calibre.utils.logging import default_log as log def create_bibtex_entry(entry, fields, mode, template_citation, bibtexdict, db, citation_bibtex=True, calibre_files=True): diff --git a/src/calibre/library/catalogs/csv_xml.py b/src/calibre/library/catalogs/csv_xml.py index a09084889b..f4a05d86a9 100644 --- a/src/calibre/library/catalogs/csv_xml.py +++ b/src/calibre/library/catalogs/csv_xml.py @@ -10,7 +10,6 @@ from collections import namedtuple from calibre.customize import CatalogPlugin from calibre.library.catalogs import FIELDS -from calibre.utils.logging import default_log as log from calibre.customize.conversion import DummyReporter class CSV_XML(CatalogPlugin): @@ -52,6 +51,7 @@ class CSV_XML(CatalogPlugin): from calibre.utils.date import isoformat from calibre.utils.html2text import html2text from lxml import etree + from calibre.utils.logging import default_log as log self.fmt = path_to_output.rpartition('.')[2] self.notification = notification diff --git a/src/calibre/library/catalogs/epub_mobi.py b/src/calibre/library/catalogs/epub_mobi.py index 97669f6ad8..cdc27b5e60 100644 --- a/src/calibre/library/catalogs/epub_mobi.py +++ b/src/calibre/library/catalogs/epub_mobi.py @@ -14,7 +14,6 @@ from calibre import strftime from calibre.constants import DEBUG from calibre.customize import CatalogPlugin from calibre.customize.conversion import OptionRecommendation, DummyReporter -from calibre.utils.logging import default_log as log Option = namedtuple('Option', 'option, default, dest, action, help') @@ -161,6 +160,8 @@ class EPUB_MOBI(CatalogPlugin): def run(self, path_to_output, opts, db, notification=DummyReporter()): from calibre.library.catalogs.epub_mobi_builder import CatalogBuilder + from calibre.utils.logging import default_log as log + opts.log = log opts.fmt = self.fmt = path_to_output.rpartition('.')[2] diff --git a/src/calibre/utils/date.py b/src/calibre/utils/date.py index faac8795d4..8741cb6a9e 100644 --- a/src/calibre/utils/date.py +++ b/src/calibre/utils/date.py @@ -10,7 +10,6 @@ import re from datetime import datetime, time from functools import partial -from dateutil.parser import parse from dateutil.tz import tzlocal, tzutc from calibre import strftime @@ -71,6 +70,7 @@ def parse_date(date_string, assume_utc=False, as_utc=True, default=None): :param default: Missing fields are filled in from default. If None, the current date is used. ''' + from dateutil.parser import parse if not date_string: return UNDEFINED_DATE if default is None: From efcb589b192f4aaf303ec2c4295a3b3498005f70 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 6 Feb 2012 14:02:11 +0530 Subject: [PATCH 30/31] ... --- src/calibre/customize/builtins.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py index fd1ccd0349..855d105e15 100644 --- a/src/calibre/customize/builtins.py +++ b/src/calibre/customize/builtins.py @@ -1665,7 +1665,7 @@ if __name__ == '__main__': ret = 1 print (x, 'has been loaded by a plugin') if ret: - print ('\\nA good way to trackdown what is loading something is to run' + print ('\\nA good way to track down what is loading something is to run' ' python -c "import init_calibre; import calibre.customize.builtins"') print() print ('Time taken to import all plugins: %.2f'%t) From 7396d7a69de36d2c842905518890ed731b9c093d Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 6 Feb 2012 14:44:31 +0530 Subject: [PATCH 31/31] Do not initialize the user defined driver plugin in worker processes --- src/calibre/customize/ui.py | 16 +++++++++++++++- src/calibre/devices/user_defined/driver.py | 8 +++++++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/src/calibre/customize/ui.py b/src/calibre/customize/ui.py index dbde947e42..b365eb1346 100644 --- a/src/calibre/customize/ui.py +++ b/src/calibre/customize/ui.py @@ -452,6 +452,9 @@ def device_plugins(include_disabled=False): # {{{ if isinstance(plugin, DevicePlugin): if include_disabled or not is_disabled(plugin): if platform in plugin.supported_platforms: + if getattr(plugin, 'plugin_needs_delayed_initialization', + False): + plugin.do_delayed_plugin_initialization() yield plugin # }}} @@ -496,7 +499,7 @@ def initialize_plugin(plugin, path_to_zip_file): def has_external_plugins(): return bool(config['plugins']) -def initialize_plugins(): +def initialize_plugins(perf=False): global _initialized_plugins _initialized_plugins = [] conflicts = [name for name in config['plugins'] if name in @@ -505,6 +508,10 @@ def initialize_plugins(): remove_plugin(p) external_plugins = config['plugins'] ostdout, ostderr = sys.stdout, sys.stderr + if perf: + from collections import defaultdict + import time + times = defaultdict(lambda:0) for zfp in list(external_plugins) + builtin_plugins: try: if not isinstance(zfp, type): @@ -517,7 +524,11 @@ def initialize_plugins(): plugin = load_plugin(zfp) if not isinstance(zfp, type) else zfp except PluginNotFound: continue + if perf: + st = time.time() plugin = initialize_plugin(plugin, None if isinstance(zfp, type) else zfp) + if perf: + times[plugin.name] = time.time() - st _initialized_plugins.append(plugin) except: print 'Failed to initialize plugin:', repr(zfp) @@ -526,6 +537,9 @@ def initialize_plugins(): # Prevent a custom plugin from overriding stdout/stderr as this breaks # ipython sys.stdout, sys.stderr = ostdout, ostderr + if perf: + for x in sorted(times, key=lambda x:times[x]): + print ('%50s: %.3f'%(x, times[x])) _initialized_plugins.sort(cmp=lambda x,y:cmp(x.priority, y.priority), reverse=True) reread_filetype_plugins() reread_metadata_plugins() diff --git a/src/calibre/devices/user_defined/driver.py b/src/calibre/devices/user_defined/driver.py index d613a09508..6c4e1f77d9 100644 --- a/src/calibre/devices/user_defined/driver.py +++ b/src/calibre/devices/user_defined/driver.py @@ -90,6 +90,10 @@ class USER_DEFINED(USBMS): OPT_CARD_A_FOLDER = 9 def initialize(self): + self.plugin_needs_delayed_initialization = True + USBMS.initialize(self) + + def do_delayed_plugin_initialization(self): try: e = self.settings().extra_customization self.VENDOR_ID = int(e[self.OPT_USB_VENDOR_ID], 16) @@ -107,4 +111,6 @@ class USER_DEFINED(USBMS): except: import traceback traceback.print_exc() - USBMS.initialize(self) + self.plugin_needs_delayed_initialization = False + +