From ec863926661d2bc9366d2fe1b74bd091138e0495 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 14 May 2013 08:29:58 +0530 Subject: [PATCH 01/18] Fix #1179697 (write a device driver for my device) --- src/calibre/devices/android/driver.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/calibre/devices/android/driver.py b/src/calibre/devices/android/driver.py index 9d5ce152d3..2855de16ae 100644 --- a/src/calibre/devices/android/driver.py +++ b/src/calibre/devices/android/driver.py @@ -240,7 +240,8 @@ class ANDROID(USBMS): 'ADVANCED', 'SGH-I727', 'USB_FLASH_DRIVER', 'ANDROID', 'S5830I_CARD', 'MID7042', 'LINK-CREATE', '7035', 'VIEWPAD_7E', 'NOVO7', 'MB526', '_USB#WYK7MSF8KE', 'TABLET_PC', 'F', 'MT65XX_MS', - 'ICS', 'E400', '__FILE-STOR_GADG', 'ST80208-1', 'GT-S5660M_CARD', 'XT894'] + 'ICS', 'E400', '__FILE-STOR_GADG', 'ST80208-1', 'GT-S5660M_CARD', 'XT894', '_USB', + ] WINDOWS_CARD_A_MEM = ['ANDROID_PHONE', 'GT-I9000_CARD', 'SGH-I897', 'FILE-STOR_GADGET', 'SGH-T959_CARD', 'SGH-T959', 'SAMSUNG_ANDROID', 'GT-P1000_CARD', 'A70S', 'A101IT', '7', 'INCREDIBLE', 'A7EB', 'SGH-T849_CARD', @@ -251,7 +252,9 @@ class ANDROID(USBMS): 'FILE-CD_GADGET', 'GT-I9001_CARD', 'USB_2.0', 'XT875', 'UMS_COMPOSITE', 'PRO', '.KOBO_VOX', 'SGH-T989_CARD', 'SGH-I727', 'USB_FLASH_DRIVER', 'ANDROID', 'MID7042', '7035', 'VIEWPAD_7E', - 'NOVO7', 'ADVANCED', 'TABLET_PC', 'F', 'E400_SD_CARD', 'ST80208-1', 'XT894'] + 'NOVO7', 'ADVANCED', 'TABLET_PC', 'F', 'E400_SD_CARD', 'ST80208-1', 'XT894', + '_USB', + ] OSX_MAIN_MEM = 'Android Device Main Memory' From e33ac985b4b5e6485a5ce53970093ce6e04a78ad Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 14 May 2013 09:30:36 +0530 Subject: [PATCH 02/18] On linux when searching the system for fonts, search all directories returned by fontconfig, if available, instead of a default list of directories --- src/calibre/utils/fonts/scanner.py | 84 ++++++++++++++++++++++++++---- 1 file changed, 75 insertions(+), 9 deletions(-) diff --git a/src/calibre/utils/fonts/scanner.py b/src/calibre/utils/fonts/scanner.py index 827e5536d5..b5628989c2 100644 --- a/src/calibre/utils/fonts/scanner.py +++ b/src/calibre/utils/fonts/scanner.py @@ -13,13 +13,82 @@ from threading import Thread from calibre import walk, prints, as_unicode from calibre.constants import (config_dir, iswindows, isosx, plugins, DEBUG, - isworker) + isworker, filesystem_encoding) from calibre.utils.fonts.metadata import FontMetadata, UnsupportedFont from calibre.utils.icu import sort_key class NoFonts(ValueError): pass + +def default_font_dirs(): + return [ + '/opt/share/fonts', + '/usr/share/fonts', + '/usr/local/share/fonts', + os.path.expanduser('~/.local/share/fonts'), + os.path.expanduser('~/.fonts') + ] + + +def fc_list(): + import ctypes + from ctypes.util import find_library + + lib = find_library('fontconfig') + if lib is None: + return default_font_dirs() + try: + lib = ctypes.CDLL(lib) + except: + return default_font_dirs() + + prototype = ctypes.CFUNCTYPE(ctypes.c_void_p, ctypes.c_void_p) + try: + get_font_dirs = prototype(('FcConfigGetFontDirs', lib)) + except (AttributeError): + return default_font_dirs() + prototype = ctypes.CFUNCTYPE(ctypes.c_char_p, ctypes.c_void_p) + try: + next_dir = prototype(('FcStrListNext', lib)) + except (AttributeError): + return default_font_dirs() + + prototype = ctypes.CFUNCTYPE(None, ctypes.c_void_p) + try: + end = prototype(('FcStrListDone', lib)) + except (AttributeError): + return default_font_dirs() + + str_list = get_font_dirs(ctypes.c_void_p()) + if not str_list: + return default_font_dirs() + + ans = [] + while True: + d = next_dir(str_list) + if not d: + break + if d: + try: + ans.append(d.decode(filesystem_encoding)) + except ValueError: + return default_font_dirs + end(str_list) + if len(ans) < 3: + return default_font_dirs() + parents = [] + for f in ans: + found = False + for p in parents: + if f.startswith(p): + found = True + break + if not found: + parents.append(f) + return parents + + def font_dirs(): if iswindows: winutil, err = plugins['winutil'] @@ -35,12 +104,7 @@ def font_dirs(): os.path.expanduser('~/.fonts'), os.path.expanduser('~/Library/Fonts'), ] - return [ - '/opt/share/fonts', - '/usr/share/fonts', - '/usr/local/share/fonts', - os.path.expanduser('~/.fonts') - ] + return fc_list() class Scanner(Thread): @@ -133,7 +197,8 @@ class Scanner(Thread): for family in self.find_font_families(): faces = filter(filter_faces, self.fonts_for_family(family)) - if not faces: continue + if not faces: + continue generic_family = panose_to_css_generic_family(faces[0]['panose']) if generic_family in allowed_families or generic_family == preferred_families[0]: return (family, faces) @@ -233,7 +298,8 @@ class Scanner(Thread): def build_families(self): families = defaultdict(list) for f in self.cached_fonts.itervalues(): - if not f: continue + if not f: + continue lf = icu_lower(f['font-family'] or '') if lf: families[lf].append(f) From 802e4c52fb841f7bf3ef92476b29796f04774595 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 14 May 2013 09:53:42 +0530 Subject: [PATCH 03/18] Change the filesystem encoding used by python to utf-8 if it is ascii --- src/calibre/constants.py | 6 ++---- src/calibre/utils/icu.c | 15 +++++++++++++++ src/calibre/utils/icu.py | 15 +++++++++++++-- 3 files changed, 30 insertions(+), 6 deletions(-) diff --git a/src/calibre/constants.py b/src/calibre/constants.py index 6526c2e289..4c17a90122 100644 --- a/src/calibre/constants.py +++ b/src/calibre/constants.py @@ -66,10 +66,8 @@ else: filesystem_encoding = 'utf-8' # On linux, unicode arguments to os file functions are coerced to an ascii # bytestring if sys.getfilesystemencoding() == 'ascii', which is - # just plain dumb. So issue a warning. - print ('WARNING: You do not have the LANG environment variable set correctly. ' - 'This will cause problems with non-ascii filenames. ' - 'Set it to something like en_US.UTF-8.\n') + # just plain dumb. This is fixed by the icu.py module which, when + # imported changes ascii to utf-8 except: filesystem_encoding = 'utf-8' diff --git a/src/calibre/utils/icu.c b/src/calibre/utils/icu.c index ccb1cfb5b9..aee47448fd 100644 --- a/src/calibre/utils/icu.c +++ b/src/calibre/utils/icu.c @@ -661,6 +661,17 @@ icu_set_default_encoding(PyObject *self, PyObject *args) { } // }}} +// set_default_encoding {{{ +static PyObject * +icu_set_filesystem_encoding(PyObject *self, PyObject *args) { + char *encoding; + if (!PyArg_ParseTuple(args, "s:setfilesystemencoding", &encoding)) + return NULL; + Py_FileSystemDefaultEncoding = strdup(encoding); + Py_RETURN_NONE; + +} +// }}} // set_default_encoding {{{ static PyObject * icu_get_available_transliterators(PyObject *self, PyObject *args) { @@ -707,6 +718,10 @@ static PyMethodDef icu_methods[] = { "set_default_encoding(encoding) -> Set the default encoding for the python unicode implementation." }, + {"set_filesystem_encoding", icu_set_filesystem_encoding, METH_VARARGS, + "set_filesystem_encoding(encoding) -> Set the filesystem encoding for python." + }, + {"get_available_transliterators", icu_get_available_transliterators, METH_VARARGS, "get_available_transliterators() -> Return list of available transliterators. This list is rather limited on OS X." }, diff --git a/src/calibre/utils/icu.py b/src/calibre/utils/icu.py index e1e6c1a1c6..1f54a04646 100644 --- a/src/calibre/utils/icu.py +++ b/src/calibre/utils/icu.py @@ -163,11 +163,22 @@ load_collator() _icu_not_ok = _icu is None or _collator is None try: - if sys.getdefaultencoding().lower() == 'ascii': + senc = sys.getdefaultencoding() + if not senc or senc.lower() == 'ascii': _icu.set_default_encoding('utf-8') + del senc except: pass +try: + fenc = sys.getfilesystemencoding() + if not fenc or fenc.lower() == 'ascii': + _icu.set_filesystem_encoding('utf-8') + del fenc +except: + pass + + # }}} ################# The string functions ######################################## @@ -247,7 +258,7 @@ def collation_order(a): ################################################################################ -def test(): # {{{ +def test(): # {{{ from calibre import prints # Data {{{ german = ''' From ffdc9d377c7540f7bab6ac68303c744676828597 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 14 May 2013 11:42:31 +0530 Subject: [PATCH 04/18] ... --- src/calibre/ebooks/docx/dump.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/ebooks/docx/dump.py b/src/calibre/ebooks/docx/dump.py index f6432125c5..6ebc2e8871 100644 --- a/src/calibre/ebooks/docx/dump.py +++ b/src/calibre/ebooks/docx/dump.py @@ -22,7 +22,7 @@ def dump(path): zf.extractall(dest) for f in walk(dest): - if f.endswith('.xml'): + if f.endswith('.xml') or f.endswith('.rels'): with open(f, 'r+b') as stream: raw = stream.read() root = etree.fromstring(raw) From d8a896616a34432bc7c4ae00ce8018619881ae7a Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 14 May 2013 16:09:12 +0530 Subject: [PATCH 05/18] DOCX Input: Fonts --- src/calibre/ebooks/docx/char_styles.py | 15 ++- src/calibre/ebooks/docx/container.py | 4 +- src/calibre/ebooks/docx/fonts.py | 132 +++++++++++++++++++++++++ src/calibre/ebooks/docx/names.py | 1 + src/calibre/ebooks/docx/styles.py | 11 ++- src/calibre/ebooks/docx/to_html.py | 18 +++- 6 files changed, 173 insertions(+), 8 deletions(-) create mode 100644 src/calibre/ebooks/docx/fonts.py diff --git a/src/calibre/ebooks/docx/char_styles.py b/src/calibre/ebooks/docx/char_styles.py index a9d2a43cdb..b65766e494 100644 --- a/src/calibre/ebooks/docx/char_styles.py +++ b/src/calibre/ebooks/docx/char_styles.py @@ -113,6 +113,14 @@ def read_vert_align(parent, dest): if val and val in {'baseline', 'subscript', 'superscript'}: ans = val setattr(dest, 'vert_align', ans) + +def read_font_family(parent, dest): + ans = inherit + for col in XPath('./w:rFonts[@w:ascii]')(parent): + val = get(col, 'w:ascii') + if val: + ans = val + setattr(dest, 'font_family', ans) # }}} class RunStyle(object): @@ -122,7 +130,7 @@ class RunStyle(object): 'rtl', 'shadow', 'smallCaps', 'strike', 'vanish', 'border_color', 'border_style', 'border_width', 'padding', 'color', 'highlight', 'background_color', - 'letter_spacing', 'font_size', 'text_decoration', 'vert_align', 'lang', + 'letter_spacing', 'font_size', 'text_decoration', 'vert_align', 'lang', 'font_family' } toggle_properties = { @@ -141,7 +149,7 @@ class RunStyle(object): ): setattr(self, p, binary_property(rPr, p)) - for x in ('text_border', 'color', 'highlight', 'shd', 'letter_spacing', 'sz', 'underline', 'vert_align', 'lang'): + for x in ('text_border', 'color', 'highlight', 'shd', 'letter_spacing', 'sz', 'underline', 'vert_align', 'lang', 'font_family'): f = globals()['read_%s' % x] f(rPr, self) @@ -212,6 +220,9 @@ class RunStyle(object): if self.b: c['font-weight'] = 'bold' + + if self.font_family is not inherit: + c['font-family'] = self.font_family return self._css def same_border(self, other): diff --git a/src/calibre/ebooks/docx/container.py b/src/calibre/ebooks/docx/container.py index ec0decacef..bcca336474 100644 --- a/src/calibre/ebooks/docx/container.py +++ b/src/calibre/ebooks/docx/container.py @@ -167,7 +167,9 @@ class DOCX(object): @property def document_relationships(self): - name = self.document_name + return self.get_relationships(self.document_name) + + def get_relationships(self, name): base = '/'.join(name.split('/')[:-1]) by_id, by_type = {}, {} parts = name.split('/') diff --git a/src/calibre/ebooks/docx/fonts.py b/src/calibre/ebooks/docx/fonts.py new file mode 100644 index 0000000000..4ed602c71d --- /dev/null +++ b/src/calibre/ebooks/docx/fonts.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python +# vim:fileencoding=utf-8 +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +__license__ = 'GPL v3' +__copyright__ = '2013, Kovid Goyal ' + +import os, re +from collections import namedtuple + +from calibre.ebooks.docx.block_styles import binary_property, inherit +from calibre.ebooks.docx.names import XPath, get +from calibre.utils.filenames import ascii_filename +from calibre.utils.fonts.scanner import font_scanner, NoFonts +from calibre.utils.fonts.utils import panose_to_css_generic_family, is_truetype_font + +Embed = namedtuple('Embed', 'name key subsetted') + +def has_system_fonts(name): + try: + return bool(font_scanner.fonts_for_family(name)) + except NoFonts: + return False + +def get_variant(bold=False, italic=False): + return {(False, False):'Regular', (False, True):'Italic', + (True, False):'Bold', (True, True):'BoldItalic'}[(bold, italic)] + +class Family(object): + + def __init__(self, elem, embed_relationships): + self.name = self.family_name = get(elem, 'w:name') + self.alt_names = tuple(get(x, 'w:val') for x in XPath('./w:altName')(elem)) + if self.alt_names and not has_system_fonts(self.name): + for x in self.alt_names: + if has_system_fonts(x): + self.family_name = x + break + + self.embedded = {} + for x in ('Regular', 'Bold', 'Italic', 'BoldItalic'): + for y in XPath('./w:embed%s[@r:id]' % x)(elem): + rid = get(y, 'r:id') + key = get(y, 'w:fontKey') + subsetted = get(y, 'w:subsetted') in {'1', 'true', 'on'} + if rid in embed_relationships: + self.embedded[x] = Embed(embed_relationships[rid], key, subsetted) + + self.generic_family = 'auto' + for x in XPath('./w:family[@w:val]')(elem): + self.generic_family = get(x, 'w:val', 'auto') + + ntt = binary_property(elem, 'notTrueType') + self.is_ttf = ntt is inherit or not ntt + + self.panose1 = None + self.panose_name = None + for x in XPath('./w:panose1[@w:val]')(elem): + try: + v = get(x, 'w:val') + v = tuple(int(v[i:i+2], 16) for i in xrange(0, len(v), 2)) + except (TypeError, ValueError, IndexError): + pass + else: + self.panose1 = v + self.panose_name = panose_to_css_generic_family(v) + + self.css_generic_family = {'roman':'serif', 'swiss':'sans-serif', 'modern':'monospace', + 'decorative':'fantasy', 'script':'cursive'}.get(self.generic_family, None) + self.css_generic_family = self.css_generic_family or self.panose_name or 'serif' + + +class Fonts(object): + + def __init__(self): + self.fonts = {} + self.used = set() + + def __call__(self, root, embed_relationships, docx, dest_dir): + for elem in XPath('//w:font[@w:name]')(root): + self.fonts[get(elem, 'w:name')] = Family(elem, embed_relationships) + + def family_for(self, name, bold=False, italic=False): + f = self.fonts.get(name, None) + if f is None: + return 'serif' + variant = get_variant(bold, italic) + self.used.add((name, variant)) + name = f.name if variant in f.embedded else f.family_name + return '"%s", %s' % (name.replace('"', ''), f.css_generic_family) + + def embed_fonts(self, dest_dir, docx): + defs = [] + dest_dir = os.path.join(dest_dir, 'fonts') + for name, variant in self.used: + f = self.fonts[name] + if variant in f.embedded: + if not os.path.exists(dest_dir): + os.mkdir(dest_dir) + fname = self.write(name, dest_dir, docx, variant) + if fname is not None: + d = {'font-family':'"%s"' % name.replace('"', ''), 'src': 'url("fonts/%s")' % fname} + if 'Bold' in variant: + d['font-weight'] = 'bold' + if 'Italic' in variant: + d['font-style'] = 'italic' + d = ['%s: %s' % (k, v) for k, v in d.iteritems()] + d = ';\n\t'.join(d) + defs.append('@font-face {\n\t%s\n}\n' % d) + return '\n'.join(defs) + + def write(self, name, dest_dir, docx, variant): + f = self.fonts[name] + ef = f.embedded[variant] + raw = docx.read(ef.name) + prefix = raw[:32] + if ef.key: + key = re.sub(r'[^A-Fa-f0-9]', '', ef.key) + key = bytearray(reversed(tuple(int(key[i:i+2], 16) for i in xrange(0, len(key), 2)))) + prefix = bytearray(prefix) + prefix = bytes(bytearray(prefix[i]^key[i % len(key)] for i in xrange(len(prefix)))) + if not is_truetype_font(prefix): + return None + ext = 'otf' if prefix.startswith(b'OTTO') else 'ttf' + fname = ascii_filename('%s - %s.%s' % (name, variant, ext)) + with open(os.path.join(dest_dir, fname), 'wb') as dest: + dest.write(prefix) + dest.write(raw[32:]) + + return fname + diff --git a/src/calibre/ebooks/docx/names.py b/src/calibre/ebooks/docx/names.py index 91b051d691..da643dcc2c 100644 --- a/src/calibre/ebooks/docx/names.py +++ b/src/calibre/ebooks/docx/names.py @@ -13,6 +13,7 @@ DOCPROPS = 'http://schemas.openxmlformats.org/package/2006/relationships/metada APPPROPS = 'http://schemas.openxmlformats.org/officeDocument/2006/relationships/extended-properties' STYLES = 'http://schemas.openxmlformats.org/officeDocument/2006/relationships/styles' NUMBERING = 'http://schemas.openxmlformats.org/officeDocument/2006/relationships/numbering' +FONTS = 'http://schemas.openxmlformats.org/officeDocument/2006/relationships/fontTable' namespaces = { 'mo': 'http://schemas.microsoft.com/office/mac/office/2008/main', diff --git a/src/calibre/ebooks/docx/styles.py b/src/calibre/ebooks/docx/styles.py index 44ae2cea89..13b9ebe58f 100644 --- a/src/calibre/ebooks/docx/styles.py +++ b/src/calibre/ebooks/docx/styles.py @@ -97,7 +97,8 @@ class Styles(object): def get(self, key, default=None): return self.id_map.get(key, default) - def __call__(self, root): + def __call__(self, root, fonts): + self.fonts = fonts for s in XPath('//w:style')(root): s = Style(s) if s.style_id: @@ -246,6 +247,9 @@ class Styles(object): for attr in ans.all_properties: setattr(ans, attr, self.run_val(parent_styles, direct_formatting, attr)) + if ans.font_family is not inherit: + ans.font_family = self.fonts.family_for(ans.font_family, ans.b, ans.i) + return ans def resolve(self, obj): @@ -290,13 +294,16 @@ class Styles(object): h = hash(frozenset(css.iteritems())) return self.classes.get(h, (None, None))[0] - def generate_css(self): + def generate_css(self, dest_dir, docx): + ef = self.fonts.embed_fonts(dest_dir, docx) prefix = textwrap.dedent( '''\ p { text-indent: 1.5em } ul, ol, p { margin: 0; padding: 0 } ''') + if ef: + prefix += '\n' + ef ans = [] for (cls, css) in sorted(self.classes.itervalues(), key=lambda x:x[0]): diff --git a/src/calibre/ebooks/docx/to_html.py b/src/calibre/ebooks/docx/to_html.py index 8cd79074e3..dbd6dce043 100644 --- a/src/calibre/ebooks/docx/to_html.py +++ b/src/calibre/ebooks/docx/to_html.py @@ -14,9 +14,10 @@ from lxml.html.builder import ( HTML, HEAD, TITLE, BODY, LINK, META, P, SPAN, BR) from calibre.ebooks.docx.container import DOCX, fromstring -from calibre.ebooks.docx.names import XPath, is_tag, barename, XML, STYLES, NUMBERING +from calibre.ebooks.docx.names import XPath, is_tag, barename, XML, STYLES, NUMBERING, FONTS from calibre.ebooks.docx.styles import Styles, inherit from calibre.ebooks.docx.numbering import Numbering +from calibre.ebooks.docx.fonts import Fonts from calibre.utils.localization import canonicalize_lang, lang_as_iso639_1 class Text: @@ -116,7 +117,18 @@ class Convert(object): nname = get_name(NUMBERING, 'numbering.xml') sname = get_name(STYLES, 'styles.xml') + fname = get_name(FONTS, 'fontTable.xml') numbering = self.numbering = Numbering() + fonts = self.fonts = Fonts() + + if fname is not None: + embed_relationships = self.docx.get_relationships(fname)[0] + try: + raw = self.docx.read(fname) + except KeyError: + self.log.warn('Fonts table %s does not exist' % fname) + else: + fonts(fromstring(raw), embed_relationships, self.docx, self.dest_dir) if sname is not None: try: @@ -124,7 +136,7 @@ class Convert(object): except KeyError: self.log.warn('Styles %s do not exist' % sname) else: - self.styles(fromstring(raw)) + self.styles(fromstring(raw), fonts) if nname is not None: try: @@ -140,7 +152,7 @@ class Convert(object): raw = html.tostring(self.html, encoding='utf-8', doctype='') with open(os.path.join(self.dest_dir, 'index.html'), 'wb') as f: f.write(raw) - css = self.styles.generate_css() + css = self.styles.generate_css(self.dest_dir, self.docx) if css: with open(os.path.join(self.dest_dir, 'docx.css'), 'wb') as f: f.write(css.encode('utf-8')) From aa2aa3d2ef8bb89acf7a6e943be9a91391d9cdd0 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 14 May 2013 16:36:09 +0530 Subject: [PATCH 06/18] Ignore line height of 1 --- src/calibre/ebooks/docx/block_styles.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/calibre/ebooks/docx/block_styles.py b/src/calibre/ebooks/docx/block_styles.py index b501580042..eef68a184f 100644 --- a/src/calibre/ebooks/docx/block_styles.py +++ b/src/calibre/ebooks/docx/block_styles.py @@ -271,7 +271,10 @@ class ParagraphStyle(object): if val is not inherit: c['margin-%s' % edge] = val - for x in ('text_indent', 'text_align', 'line_height', 'background_color'): + if self.line_height not in {inherit, '1'}: + c['line-height'] = self.line_height + + for x in ('text_indent', 'text_align', 'background_color'): val = getattr(self, x) if val is not inherit: c[x.replace('_', '-')] = val From 5ec61a6b299ab2114e0b7b7ae5848b733d512371 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 14 May 2013 16:45:03 +0530 Subject: [PATCH 07/18] Dont ignore the content in tables, just extarct the content as linear blocks for now --- src/calibre/ebooks/docx/to_html.py | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/src/calibre/ebooks/docx/to_html.py b/src/calibre/ebooks/docx/to_html.py index dbd6dce043..b4e5b0e5f7 100644 --- a/src/calibre/ebooks/docx/to_html.py +++ b/src/calibre/ebooks/docx/to_html.py @@ -14,7 +14,7 @@ from lxml.html.builder import ( HTML, HEAD, TITLE, BODY, LINK, META, P, SPAN, BR) from calibre.ebooks.docx.container import DOCX, fromstring -from calibre.ebooks.docx.names import XPath, is_tag, barename, XML, STYLES, NUMBERING, FONTS +from calibre.ebooks.docx.names import XPath, is_tag, XML, STYLES, NUMBERING, FONTS from calibre.ebooks.docx.styles import Styles, inherit from calibre.ebooks.docx.numbering import Numbering from calibre.ebooks.docx.fonts import Fonts @@ -64,16 +64,11 @@ class Convert(object): doc = self.docx.document relationships_by_id, relationships_by_type = self.docx.document_relationships self.read_styles(relationships_by_type) - for top_level in XPath('/w:document/w:body/*')(doc): - if is_tag(top_level, 'w:p'): - p = self.convert_p(top_level) - self.body.append(p) - elif is_tag(top_level, 'w:tbl'): - pass # TODO: tables - elif is_tag(top_level, 'w:sectPr'): - pass # TODO: Last section properties - else: - self.log.debug('Unknown top-level tag: %s, ignoring' % barename(top_level.tag)) + for wp in XPath('//w:p')(doc): + p = self.convert_p(wp) + self.body.append(p) + # TODO: tables child of (nested tables?) + # TODO: Last section properties child of numbered = [] for html_obj, obj in self.object_map.iteritems(): From 33793ff0d1135729832cd4d2c10f1c2a2a37516f Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 14 May 2013 18:01:55 +0530 Subject: [PATCH 08/18] Driver for SONY PRS-T2N --- src/calibre/devices/prst1/driver.py | 34 ++++++++++++++--------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/src/calibre/devices/prst1/driver.py b/src/calibre/devices/prst1/driver.py index 72533860d4..0431ca7bfd 100644 --- a/src/calibre/devices/prst1/driver.py +++ b/src/calibre/devices/prst1/driver.py @@ -39,8 +39,8 @@ class PRST1(USBMS): path_sep = '/' booklist_class = CollectionsBookList - FORMATS = ['epub', 'pdf', 'txt', 'book', 'zbf'] # The last two are - # used in japan + FORMATS = ['epub', 'pdf', 'txt', 'book', 'zbf'] # The last two are + # used in japan CAN_SET_METADATA = ['collections'] CAN_DO_DEVICE_DB_PLUGBOARD = True @@ -50,10 +50,10 @@ class PRST1(USBMS): VENDOR_NAME = 'SONY' WINDOWS_MAIN_MEM = re.compile( - r'(PRS-T(1|2)&)' + r'(PRS-T(1|2|2N)&)' ) WINDOWS_CARD_A_MEM = re.compile( - r'(PRS-T(1|2)__SD&)' + r'(PRS-T(1|2|2N)__SD&)' ) MAIN_MEMORY_VOLUME_LABEL = 'SONY Reader Main Memory' STORAGE_CARD_VOLUME_LABEL = 'SONY Reader Storage Card' @@ -66,7 +66,7 @@ class PRST1(USBMS): EXTRA_CUSTOMIZATION_MESSAGE = [ _('Comma separated list of metadata fields ' - 'to turn into collections on the device. Possibilities include: ')+\ + 'to turn into collections on the device. Possibilities include: ')+ 'series, tags, authors', _('Upload separate cover thumbnails for books') + ':::'+_('Normally, the SONY readers get the cover image from the' @@ -194,17 +194,17 @@ class PRST1(USBMS): time_offsets = {} for i, row in enumerate(cursor): try: - comp_date = int(os.path.getmtime(self.normalize_path(prefix + row[0])) * 1000); + comp_date = int(os.path.getmtime(self.normalize_path(prefix + row[0])) * 1000) except (OSError, IOError, TypeError): # In case the db has incorrect path info continue - device_date = int(row[1]); + device_date = int(row[1]) offset = device_date - comp_date time_offsets.setdefault(offset, 0) time_offsets[offset] = time_offsets[offset] + 1 try: - device_offset = max(time_offsets,key = lambda a: time_offsets.get(a)) + device_offset = max(time_offsets, key=lambda a: time_offsets.get(a)) debug_print("Device Offset: %d ms"%device_offset) self.device_offset = device_offset except ValueError: @@ -213,7 +213,7 @@ class PRST1(USBMS): for idx, book in enumerate(bl): query = 'SELECT _id, thumbnail FROM books WHERE file_path = ?' t = (book.lpath,) - cursor.execute (query, t) + cursor.execute(query, t) for i, row in enumerate(cursor): book.device_collections = bl_collections.get(row[0], None) @@ -318,14 +318,14 @@ class PRST1(USBMS): ' any notes/highlights, etc.')%dbpath)+' Underlying error:' '\n'+tb) - def get_lastrowid(self, cursor): - # SQLite3 + Python has a fun issue on 32-bit systems with integer overflows. - # Issue a SQL query instead, getting the value as a string, and then converting to a long python int manually. - query = 'SELECT last_insert_rowid()' - cursor.execute(query) - row = cursor.fetchone() + def get_lastrowid(self, cursor): + # SQLite3 + Python has a fun issue on 32-bit systems with integer overflows. + # Issue a SQL query instead, getting the value as a string, and then converting to a long python int manually. + query = 'SELECT last_insert_rowid()' + cursor.execute(query) + row = cursor.fetchone() - return long(row[0]) + return long(row[0]) def get_database_min_id(self, source_id): sequence_min = 0L @@ -345,7 +345,7 @@ class PRST1(USBMS): # Insert the sequence Id if it doesn't query = ('INSERT INTO sqlite_sequence (name, seq) ' 'SELECT ?, ? ' - 'WHERE NOT EXISTS (SELECT 1 FROM sqlite_sequence WHERE name = ?)'); + 'WHERE NOT EXISTS (SELECT 1 FROM sqlite_sequence WHERE name = ?)') cursor.execute(query, (table, sequence_id, table,)) cursor.close() From a597fe76bb40aa170af740b269e5cc48f8e5e633 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 14 May 2013 18:39:58 +0530 Subject: [PATCH 09/18] DOCX Input: Cascade the font css --- src/calibre/ebooks/docx/block_styles.py | 9 +++- src/calibre/ebooks/docx/char_styles.py | 20 ++++++--- src/calibre/ebooks/docx/styles.py | 55 ++++++++++++++++++++++++- src/calibre/ebooks/docx/to_html.py | 12 +++--- 4 files changed, 81 insertions(+), 15 deletions(-) diff --git a/src/calibre/ebooks/docx/block_styles.py b/src/calibre/ebooks/docx/block_styles.py index eef68a184f..10dc416eec 100644 --- a/src/calibre/ebooks/docx/block_styles.py +++ b/src/calibre/ebooks/docx/block_styles.py @@ -208,7 +208,7 @@ class ParagraphStyle(object): # Misc. 'text_indent', 'text_align', 'line_height', 'direction', 'background_color', - 'numbering', + 'numbering', 'font_family', 'font_size', ) def __init__(self, pPr=None): @@ -232,6 +232,8 @@ class ParagraphStyle(object): for s in XPath('./w:pStyle[@w:val]')(pPr): self.linked_style = get(s, 'w:val') + self.font_family = self.font_size = inherit + self._css = None def update(self, other): @@ -274,10 +276,13 @@ class ParagraphStyle(object): if self.line_height not in {inherit, '1'}: c['line-height'] = self.line_height - for x in ('text_indent', 'text_align', 'background_color'): + for x in ('text_indent', 'text_align', 'background_color', 'font_family', 'font_size'): val = getattr(self, x) if val is not inherit: + if x == 'font_size': + val = '%.3gpt' % val c[x.replace('_', '-')] = val + return self._css # TODO: keepNext must be done at markup level diff --git a/src/calibre/ebooks/docx/char_styles.py b/src/calibre/ebooks/docx/char_styles.py index b65766e494..ca023e23af 100644 --- a/src/calibre/ebooks/docx/char_styles.py +++ b/src/calibre/ebooks/docx/char_styles.py @@ -172,6 +172,18 @@ class RunStyle(object): if val is inherit: setattr(self, p, getattr(parent, p)) + def get_border_css(self, ans): + for x in ('color', 'style', 'width'): + val = getattr(self, 'border_'+x) + if x == 'width' and val is not inherit: + val = '%.3gpt' % val + if val is not inherit: + ans['border-%s' % x] = val + + def clear_border_css(self): + for x in ('color', 'style', 'width'): + setattr(self, 'border_'+x, inherit) + @property def css(self): if self._css is None: @@ -196,12 +208,7 @@ class RunStyle(object): if self.vanish is True: c['display'] = 'none' - for x in ('color', 'style', 'width'): - val = getattr(self, 'border_'+x) - if x == 'width' and val is not inherit: - val = '%.3gpt' % val - if val is not inherit: - c['border-%s' % x] = val + self.get_border_css(c) if self.padding is not inherit: c['padding'] = '%.3gpt' % self.padding @@ -223,6 +230,7 @@ class RunStyle(object): if self.font_family is not inherit: c['font-family'] = self.font_family + return self._css def same_border(self, other): diff --git a/src/calibre/ebooks/docx/styles.py b/src/calibre/ebooks/docx/styles.py index 13b9ebe58f..c17418d0dd 100644 --- a/src/calibre/ebooks/docx/styles.py +++ b/src/calibre/ebooks/docx/styles.py @@ -258,6 +258,55 @@ class Styles(object): if obj.tag.endswith('}r'): return self.resolve_run(obj) + def cascade(self, layers): + self.body_font_family = 'serif' + self.body_font_size = '10pt' + + for p, runs in layers.iteritems(): + char_styles = [self.resolve_run(r) for r in runs] + block_style = self.resolve_paragraph(p) + c = Counter() + for s in char_styles: + if s.font_family is not inherit: + c[s.font_family] += 1 + if c: + family = c.most_common(1)[0][0] + block_style.font_family = family + for s in char_styles: + if s.font_family == family: + s.font_family = inherit + + sizes = [s.font_size for s in char_styles if s.font_size is not inherit] + if sizes: + sz = block_style.font_size = sizes[0] + for s in char_styles: + if s.font_size == sz: + s.font_size = inherit + + block_styles = [self.resolve_paragraph(p) for p in layers] + c = Counter() + for s in block_styles: + if s.font_family is not inherit: + c[s.font_family] += 1 + + if c: + self.body_font_family = family = c.most_common(1)[0][0] + for s in block_styles: + if s.font_family == family: + s.font_family = inherit + + c = Counter() + for s in block_styles: + if s.font_size is not inherit: + c[s.font_size] += 1 + + if c: + sz = c.most_common(1)[0][0] + for s in block_styles: + if s.font_size == sz: + s.font_size = inherit + self.body_font_size = '%.3gpt' % sz + def resolve_numbering(self, numbering): # When a numPr element appears inside a paragraph style, the lvl info # must be discarder and pStyle used instead. @@ -298,12 +347,14 @@ class Styles(object): ef = self.fonts.embed_fonts(dest_dir, docx) prefix = textwrap.dedent( '''\ + body { font-family: %s; font-size: %s } + p { text-indent: 1.5em } ul, ol, p { margin: 0; padding: 0 } - ''') + ''') % (self.body_font_family, self.body_font_size) if ef: - prefix += '\n' + ef + prefix = ef + '\n' + prefix ans = [] for (cls, css) in sorted(self.classes.itervalues(), key=lambda x:x[0]): diff --git a/src/calibre/ebooks/docx/to_html.py b/src/calibre/ebooks/docx/to_html.py index b4e5b0e5f7..902952ca4a 100644 --- a/src/calibre/ebooks/docx/to_html.py +++ b/src/calibre/ebooks/docx/to_html.py @@ -64,12 +64,15 @@ class Convert(object): doc = self.docx.document relationships_by_id, relationships_by_type = self.docx.document_relationships self.read_styles(relationships_by_type) + self.layers = OrderedDict() for wp in XPath('//w:p')(doc): p = self.convert_p(wp) self.body.append(p) # TODO: tables child of (nested tables?) # TODO: Last section properties child of + self.styles.cascade(self.layers) + numbered = [] for html_obj, obj in self.object_map.iteritems(): raw = obj.get('calibre_num_id', None) @@ -156,9 +159,11 @@ class Convert(object): dest = P() self.object_map[dest] = p style = self.styles.resolve_paragraph(p) + self.layers[p] = [] for run in XPath('descendant::w:r')(p): span = self.convert_run(run) dest.append(span) + self.layers[p].append(run) m = re.match(r'heading\s+(\d+)$', style.style_name or '', re.IGNORECASE) if m is not None: @@ -184,12 +189,9 @@ class Convert(object): spans = [] bs = {} for span, style in border_run: - c = style.css + style.get_border_css(bs) + style.clear_border_css() spans.append(span) - for x in ('width', 'color', 'style'): - val = c.pop('border-%s' % x, None) - if val is not None: - bs['border-%s' % x] = val if bs: cls = self.styles.register(bs, 'text_border') wrapper = self.wrap_elems(spans, SPAN()) From 61dac94abe24497d5f4260afb98437e77226ac89 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Tue, 14 May 2013 15:12:21 +0200 Subject: [PATCH 10/18] First pass at advanced icon selection rules --- src/calibre/gui2/dialogs/template_dialog.py | 117 ++++++++++++-- src/calibre/gui2/dialogs/template_dialog.ui | 171 +++++++++++++++----- src/calibre/gui2/preferences/coloring.py | 50 ++++-- src/calibre/library/server/browse.py | 1 + 4 files changed, 271 insertions(+), 68 deletions(-) diff --git a/src/calibre/gui2/dialogs/template_dialog.py b/src/calibre/gui2/dialogs/template_dialog.py index b60449512b..a7331f8c91 100644 --- a/src/calibre/gui2/dialogs/template_dialog.py +++ b/src/calibre/gui2/dialogs/template_dialog.py @@ -3,17 +3,21 @@ __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net' __docformat__ = 'restructuredtext en' __license__ = 'GPL v3' -import json +import json, os, traceback from PyQt4.Qt import (Qt, QDialog, QDialogButtonBox, QSyntaxHighlighter, QFont, - QRegExp, QApplication, QTextCharFormat, QColor, QCursor) + QRegExp, QApplication, QTextCharFormat, QColor, QCursor, + QIcon, QSize) -from calibre.gui2 import error_dialog +from calibre import sanitize_file_name_unicode +from calibre.constants import config_dir from calibre.gui2.dialogs.template_dialog_ui import Ui_TemplateDialog from calibre.utils.formatter_functions import formatter_functions +from calibre.utils.icu import sort_key from calibre.ebooks.metadata.book.base import Metadata from calibre.ebooks.metadata.book.formatter import SafeFormat -from calibre.library.coloring import (displayable_columns) +from calibre.library.coloring import (displayable_columns, color_row_key) +from calibre.gui2 import error_dialog, choose_files, pixmap_to_data class ParenPosition: @@ -198,25 +202,55 @@ class TemplateHighlighter(QSyntaxHighlighter): class TemplateDialog(QDialog, Ui_TemplateDialog): - def __init__(self, parent, text, mi=None, fm=None, color_field=None): + def __init__(self, parent, text, mi=None, fm=None, color_field=None, + icon_file=None, icon_rule_kind=None): QDialog.__init__(self, parent) Ui_TemplateDialog.__init__(self) self.setupUi(self) self.coloring = color_field is not None + self.iconing = icon_file is not None + + cols = [] + if fm is not None: + for key in sorted(displayable_columns(fm), + key=lambda(k): sort_key(fm[k]['name']) if k != color_row_key else 0): + if key == color_row_key and not self.coloring: + continue + from calibre.gui2.preferences.coloring import all_columns_string + name = all_columns_string if key == color_row_key else fm[key]['name'] + if name: + cols.append((name, key)) + + self.color_layout.setVisible(False) + self.icon_layout.setVisible(False) + if self.coloring: - cols = sorted([k for k in displayable_columns(fm)]) - self.colored_field.addItems(cols) + self.color_layout.setVisible(True) + for n1, k1 in cols: + self.colored_field.addItem(n1, k1) self.colored_field.setCurrentIndex(self.colored_field.findText(color_field)) colors = QColor.colorNames() colors.sort() self.color_name.addItems(colors) - else: - self.colored_field.setVisible(False) - self.colored_field_label.setVisible(False) - self.color_chooser_label.setVisible(False) - self.color_name.setVisible(False) - self.color_copy_button.setVisible(False) + elif self.iconing: + self.icon_layout.setVisible(True) + for n1, k1 in cols: + self.icon_field.addItem(n1, k1) + self.icon_file_names = [] + d = os.path.join(config_dir, 'cc_icons') + if os.path.exists(d): + for icon_file in os.listdir(d): + icon_file = icu_lower(icon_file) + if os.path.exists(os.path.join(d, icon_file)): + if icon_file.endswith('.png'): + self.icon_file_names.append(icon_file) + self.icon_file_names.sort(key=sort_key) + self.update_filename_box() + self.icon_with_text.setChecked(True) + if icon_rule_kind == 'icon_only': + self.icon_without_text.setChecked(True) + if mi: self.mi = mi else: @@ -248,6 +282,8 @@ class TemplateDialog(QDialog, Ui_TemplateDialog): self.buttonBox.button(QDialogButtonBox.Ok).setText(_('&OK')) self.buttonBox.button(QDialogButtonBox.Cancel).setText(_('&Cancel')) self.color_copy_button.clicked.connect(self.color_to_clipboard) + self.filename_button.clicked.connect(self.filename_button_clicked) + self.icon_copy_button.clicked.connect(self.icon_to_clipboard) try: with open(P('template-functions.json'), 'rb') as f: @@ -276,11 +312,55 @@ class TemplateDialog(QDialog, Ui_TemplateDialog): '' '%s'%tt) + def filename_button_clicked(self): + try: + path = choose_files(self, 'choose_category_icon', + _('Select Icon'), filters=[ + ('Images', ['png', 'gif', 'jpg', 'jpeg'])], + all_files=False, select_only_single_file=True) + if path: + icon_path = path[0] + icon_name = sanitize_file_name_unicode( + os.path.splitext( + os.path.basename(icon_path))[0]+'.png') + if icon_name not in self.icon_file_names: + self.icon_file_names.append(icon_name) + self.update_filename_box() + try: + p = QIcon(icon_path).pixmap(QSize(128, 128)) + d = os.path.join(config_dir, 'cc_icons') + if not os.path.exists(os.path.join(d, icon_name)): + if not os.path.exists(d): + os.makedirs(d) + with open(os.path.join(d, icon_name), 'wb') as f: + f.write(pixmap_to_data(p, format='PNG')) + except: + traceback.print_exc() + self.icon_files.setCurrentIndex(self.icon_files.findText(icon_name)) + self.icon_files.adjustSize() + except: + traceback.print_exc() + return + + def update_filename_box(self): + self.icon_files.clear() + self.icon_file_names.sort(key=sort_key) + self.icon_files.addItem('') + self.icon_files.addItems(self.icon_file_names) + for i,filename in enumerate(self.icon_file_names): + icon = QIcon(os.path.join(config_dir, 'cc_icons', filename)) + self.icon_files.setItemIcon(i+1, icon) + def color_to_clipboard(self): app = QApplication.instance() c = app.clipboard() c.setText(unicode(self.color_name.currentText())) + def icon_to_clipboard(self): + app = QApplication.instance() + c = app.clipboard() + c.setText(unicode(self.icon_files.currentText())) + def textbox_changed(self): cur_text = unicode(self.textbox.toPlainText()) if self.last_text != cur_text: @@ -324,5 +404,14 @@ class TemplateDialog(QDialog, Ui_TemplateDialog): _('The template box cannot be empty'), show=True) return - self.rule = (unicode(self.colored_field.currentText()), txt) + self.rule = (unicode(self.colored_field.itemData( + self.colored_field.currentIndex()).toString()), txt) + elif self.iconing: + rt = 'icon' if self.icon_with_text.isChecked() else 'icon_only' + self.rule = (rt, + unicode(self.icon_field.itemData( + self.icon_field.currentIndex()).toString()), + txt) + else: + self.rule = ('', txt) QDialog.accept(self) diff --git a/src/calibre/gui2/dialogs/template_dialog.ui b/src/calibre/gui2/dialogs/template_dialog.ui index 0acfc0f0f8..18c2a5ee35 100644 --- a/src/calibre/gui2/dialogs/template_dialog.ui +++ b/src/calibre/gui2/dialogs/template_dialog.ui @@ -21,47 +21,136 @@ - - - - - Set the color of the column: - - - colored_field - - - - - - - - - - - Copy a color name to the clipboard: - - - color_name - - - - - - - - - - - - :/images/edit-copy.png:/images/edit-copy.png - - - Copy the selected color name to the clipboard - - - - + + + + + + Set the color of the column: + + + colored_field + + + + + + + + + + + Copy a color name to the clipboard: + + + color_name + + + + + + + + + + + + :/images/edit-copy.png:/images/edit-copy.png + + + Copy the selected color name to the clipboard + + + + + + + + + + + + + Kind + + + + + + icon with no text + + + + + + + icon with text + + + + + + + 100 + 0 + + + + + + + + Apply the icon to column: + + + icon_field + + + + + + + + + + + Copy a icon file name to the clipboard: + + + color_name + + + + + + + + + + + + + + + :/images/edit-copy.png:/images/edit-copy.png + + + Copy the selected color name to the clipboard + + + + + + + Add file + + + + + + + + diff --git a/src/calibre/gui2/preferences/coloring.py b/src/calibre/gui2/preferences/coloring.py index 8d27d14e5b..422c0ba012 100644 --- a/src/calibre/gui2/preferences/coloring.py +++ b/src/calibre/gui2/preferences/coloring.py @@ -636,10 +636,20 @@ class RulesModel(QAbstractListModel): # {{{ def rule_to_html(self, kind, col, rule): if not isinstance(rule, Rule): - return _(''' -

Advanced Rule for column %(col)s: - 

%(rule)s
- ''')%dict(col=col, rule=prepare_string_for_xml(rule)) + if kind == 'color': + return _(''' +

Advanced Rule for column %(col)s: + 

%(rule)s
+ ''')%dict(col=col, rule=prepare_string_for_xml(rule)) + else: + return _(''' +

Advanced Rule: set %(typ)s for column %(col)s: + 

%(rule)s
+ ''')%dict(col=col, + typ=icon_rule_kinds[0][0] + if kind == icon_rule_kinds[0][1] else icon_rule_kinds[1][0], + rule=prepare_string_for_xml(rule)) + conditions = [self.condition_to_html(c) for c in rule.conditions] trans_kind = 'not found' @@ -761,7 +771,7 @@ class EditRules(QWidget): # {{{ ' what icon to use. Click the Add Rule button below' ' to get started.

You can change an existing rule by' ' double clicking it.')) - self.add_advanced_button.setVisible(False) +# self.add_advanced_button.setVisible(False) def add_rule(self): d = RuleEditor(self.model.fm, self.pref_name) @@ -774,13 +784,23 @@ class EditRules(QWidget): # {{{ self.changed.emit() def add_advanced(self): - td = TemplateDialog(self, '', mi=self.mi, fm=self.fm, color_field='') - if td.exec_() == td.Accepted: - col, r = td.rule - if r and col: - idx = self.model.add_rule('color', col, r) - self.rules_view.scrollTo(idx) - self.changed.emit() + if self.pref_name == 'column_color_rules': + td = TemplateDialog(self, '', mi=self.mi, fm=self.fm, color_field='') + if td.exec_() == td.Accepted: + col, r = td.rule + if r and col: + idx = self.model.add_rule('color', col, r) + self.rules_view.scrollTo(idx) + self.changed.emit() + else: + td = TemplateDialog(self, '', mi=self.mi, fm=self.fm, icon_file='') + if td.exec_() == td.Accepted: + print(td.rule) + typ, col, r = td.rule + if typ and r and col: + idx = self.model.add_rule(typ, col, r) + self.rules_view.scrollTo(idx) + self.changed.emit() def edit_rule(self, index): try: @@ -790,8 +810,12 @@ class EditRules(QWidget): # {{{ if isinstance(rule, Rule): d = RuleEditor(self.model.fm, self.pref_name) d.apply_rule(kind, col, rule) - else: + elif self.pref_name == 'column_color_rules': d = TemplateDialog(self, rule, mi=self.mi, fm=self.fm, color_field=col) + else: + d = TemplateDialog(self, rule, mi=self.mi, fm=self.fm, icon_file=col, + icon_rule_kind=kind) + if d.exec_() == d.Accepted: if len(d.rule) == 2: # Convert template dialog rules to a triple d.rule = ('color', d.rule[0], d.rule[1]) diff --git a/src/calibre/library/server/browse.py b/src/calibre/library/server/browse.py index d25c34d52b..959248fdf8 100644 --- a/src/calibre/library/server/browse.py +++ b/src/calibre/library/server/browse.py @@ -749,6 +749,7 @@ class BrowseServer(object): @Endpoint(mimetype='application/json; charset=utf-8') def browse_booklist_page(self, ids=None, sort=None): + print('here') if sort == 'null': sort = None if ids is None: From eafa02f6f9ffc7991e92acb6b9fdf7324954bd93 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Tue, 14 May 2013 16:44:28 +0200 Subject: [PATCH 11/18] Add user-setable device name to wireless device --- src/calibre/devices/smart_device_app/driver.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/calibre/devices/smart_device_app/driver.py b/src/calibre/devices/smart_device_app/driver.py index 6187f94b31..5fe60862e1 100644 --- a/src/calibre/devices/smart_device_app/driver.py +++ b/src/calibre/devices/smart_device_app/driver.py @@ -875,6 +875,9 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): self.client_device_kind = result.get('deviceKind', '') self._debug('Client device kind', self.client_device_kind) + self.client_device_name = result.get('deviceName', self.client_device_kind) + self._debug('Client device name', self.client_device_name) + self.max_book_packet_len = result.get('maxBookContentPacketLen', self.BASE_PACKET_LEN) self._debug('max_book_packet_len', self.max_book_packet_len) @@ -946,6 +949,8 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): return False def get_gui_name(self): + if getattr(self, 'client_device_name', None): + return self.gui_name_template%(self.gui_name, self.client_device_name) if getattr(self, 'client_device_kind', None): return self.gui_name_template%(self.gui_name, self.client_device_kind) return self.gui_name From 2add5649efcd28d6dc2e5a8444ceb54d7148d131 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Tue, 14 May 2013 16:45:23 +0200 Subject: [PATCH 12/18] Remove silly print statement --- src/calibre/library/server/browse.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/calibre/library/server/browse.py b/src/calibre/library/server/browse.py index 959248fdf8..d25c34d52b 100644 --- a/src/calibre/library/server/browse.py +++ b/src/calibre/library/server/browse.py @@ -749,7 +749,6 @@ class BrowseServer(object): @Endpoint(mimetype='application/json; charset=utf-8') def browse_booklist_page(self, ids=None, sort=None): - print('here') if sort == 'null': sort = None if ids is None: From 4b3c9ba5402fed439b48700773ac8f3e7432a7a1 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Tue, 14 May 2013 16:45:56 +0200 Subject: [PATCH 13/18] Changes after testing advance icon rules --- src/calibre/gui2/dialogs/template_dialog.py | 7 ++++--- src/calibre/gui2/dialogs/template_dialog.ui | 9 ++++++--- src/calibre/gui2/preferences/coloring.py | 4 ++-- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/src/calibre/gui2/dialogs/template_dialog.py b/src/calibre/gui2/dialogs/template_dialog.py index a7331f8c91..2bafc2812a 100644 --- a/src/calibre/gui2/dialogs/template_dialog.py +++ b/src/calibre/gui2/dialogs/template_dialog.py @@ -203,13 +203,13 @@ class TemplateHighlighter(QSyntaxHighlighter): class TemplateDialog(QDialog, Ui_TemplateDialog): def __init__(self, parent, text, mi=None, fm=None, color_field=None, - icon_file=None, icon_rule_kind=None): + icon_field_key=None, icon_rule_kind=None): QDialog.__init__(self, parent) Ui_TemplateDialog.__init__(self) self.setupUi(self) self.coloring = color_field is not None - self.iconing = icon_file is not None + self.iconing = icon_field_key is not None cols = [] if fm is not None: @@ -229,7 +229,7 @@ class TemplateDialog(QDialog, Ui_TemplateDialog): self.color_layout.setVisible(True) for n1, k1 in cols: self.colored_field.addItem(n1, k1) - self.colored_field.setCurrentIndex(self.colored_field.findText(color_field)) + self.colored_field.setCurrentIndex(self.colored_field.findData(color_field)) colors = QColor.colorNames() colors.sort() self.color_name.addItems(colors) @@ -250,6 +250,7 @@ class TemplateDialog(QDialog, Ui_TemplateDialog): self.icon_with_text.setChecked(True) if icon_rule_kind == 'icon_only': self.icon_without_text.setChecked(True) + self.icon_field.setCurrentIndex(self.icon_field.findData(icon_field_key)) if mi: self.mi = mi diff --git a/src/calibre/gui2/dialogs/template_dialog.ui b/src/calibre/gui2/dialogs/template_dialog.ui index 18c2a5ee35..db9cc16dd9 100644 --- a/src/calibre/gui2/dialogs/template_dialog.ui +++ b/src/calibre/gui2/dialogs/template_dialog.ui @@ -114,7 +114,7 @@ - Copy a icon file name to the clipboard: + Copy an icon file name to the clipboard: color_name @@ -135,14 +135,17 @@ :/images/edit-copy.png:/images/edit-copy.png - Copy the selected color name to the clipboard + Copy the selected icon file name to the clipboard - Add file + Add icon + + + Add an icon file to the set of choices diff --git a/src/calibre/gui2/preferences/coloring.py b/src/calibre/gui2/preferences/coloring.py index 422c0ba012..a195c948e5 100644 --- a/src/calibre/gui2/preferences/coloring.py +++ b/src/calibre/gui2/preferences/coloring.py @@ -793,7 +793,7 @@ class EditRules(QWidget): # {{{ self.rules_view.scrollTo(idx) self.changed.emit() else: - td = TemplateDialog(self, '', mi=self.mi, fm=self.fm, icon_file='') + td = TemplateDialog(self, '', mi=self.mi, fm=self.fm, icon_field_key='') if td.exec_() == td.Accepted: print(td.rule) typ, col, r = td.rule @@ -813,7 +813,7 @@ class EditRules(QWidget): # {{{ elif self.pref_name == 'column_color_rules': d = TemplateDialog(self, rule, mi=self.mi, fm=self.fm, color_field=col) else: - d = TemplateDialog(self, rule, mi=self.mi, fm=self.fm, icon_file=col, + d = TemplateDialog(self, rule, mi=self.mi, fm=self.fm, icon_field_key=col, icon_rule_kind=kind) if d.exec_() == d.Accepted: From 55c7eb963d3784e6e6c5e470a5947bb4e3afb756 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 14 May 2013 21:42:17 +0530 Subject: [PATCH 14/18] Fix lists implementad as tables having too large a left margin --- src/calibre/ebooks/docx/numbering.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/calibre/ebooks/docx/numbering.py b/src/calibre/ebooks/docx/numbering.py index 8693e2a9a1..8a177ae40a 100644 --- a/src/calibre/ebooks/docx/numbering.py +++ b/src/calibre/ebooks/docx/numbering.py @@ -274,6 +274,11 @@ class Numbering(object): for wrap in body.xpath('//ol[@lvlid]'): wrap.attrib.pop('lvlid') wrap.tag = 'div' + text = '' + for li in wrap.iterchildren('li'): + t = li[0].text + if t and len(t) > len(text): + text = t for i, li in enumerate(wrap.iterchildren('li')): li.tag = 'div' li.attrib.pop('value', None) @@ -281,7 +286,8 @@ class Numbering(object): obj = object_map[li] bs = styles.para_cache[obj] if i == 0: - wrap.set('style', 'display:table; margin-left: %s' % (bs.css.get('margin-left', 0))) + m = len(text)//2 # Move the table left to simulate the behavior of a list (number is to the left of text margin) + wrap.set('style', 'display:table; margin-left: -%dem; padding-left: %s' % (m, bs.css.get('margin-left', 0))) bs.css.pop('margin-left', None) for child in li: child.set('style', 'display:table-cell') From 0bca6e903b9e6f731a71d5b5cc44f02f9b95bda2 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 14 May 2013 23:25:29 +0530 Subject: [PATCH 15/18] ... --- src/calibre/ebooks/docx/numbering.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/calibre/ebooks/docx/numbering.py b/src/calibre/ebooks/docx/numbering.py index 8a177ae40a..0178df3227 100644 --- a/src/calibre/ebooks/docx/numbering.py +++ b/src/calibre/ebooks/docx/numbering.py @@ -214,6 +214,8 @@ class Numbering(object): p.set('list-template', val) self.update_counter(counter, ilvl, d.levels) + templates = {} + def commit(current_run): if not current_run: return @@ -244,6 +246,9 @@ class Numbering(object): span.append(gc) child.append(span) span = SPAN(child.get('list-template')) + last = templates.get(lvlid, '') + if span.text and len(span.text) > len(last): + templates[lvlid] = span.text child.insert(0, span) for attr in ('list-lvl', 'list-id', 'list-template'): child.attrib.pop(attr, None) @@ -272,9 +277,10 @@ class Numbering(object): commit(current_run) for wrap in body.xpath('//ol[@lvlid]'): - wrap.attrib.pop('lvlid') + lvlid = wrap.attrib.pop('lvlid') wrap.tag = 'div' text = '' + maxtext = templates.get(lvlid, '').replace('.', '')[:-1] for li in wrap.iterchildren('li'): t = li[0].text if t and len(t) > len(text): @@ -286,7 +292,7 @@ class Numbering(object): obj = object_map[li] bs = styles.para_cache[obj] if i == 0: - m = len(text)//2 # Move the table left to simulate the behavior of a list (number is to the left of text margin) + m = len(maxtext) # Move the table left to simulate the behavior of a list (number is to the left of text margin) wrap.set('style', 'display:table; margin-left: -%dem; padding-left: %s' % (m, bs.css.get('margin-left', 0))) bs.css.pop('margin-left', None) for child in li: From ed652e1a955f7227f0fcfe97719ec1a75f722a80 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 15 May 2013 08:46:33 +0530 Subject: [PATCH 16/18] Update Weblogs SL --- recipes/weblogs_sl.recipe | 28 ++++++---------------------- 1 file changed, 6 insertions(+), 22 deletions(-) diff --git a/recipes/weblogs_sl.recipe b/recipes/weblogs_sl.recipe index b260d2dde5..7c09b79c4a 100644 --- a/recipes/weblogs_sl.recipe +++ b/recipes/weblogs_sl.recipe @@ -3,7 +3,7 @@ __license__ = 'GPL v3' __copyright__ = '4 February 2011, desUBIKado' __author__ = 'desUBIKado' __version__ = 'v0.09' -__date__ = '02, December 2012' +__date__ = '14, May 2013' ''' http://www.weblogssl.com/ ''' @@ -56,15 +56,16 @@ class weblogssl(BasicNewsRecipe): ,(u'Zona FandoM', u'http://feeds.weblogssl.com/zonafandom') ,(u'Fandemia', u'http://feeds.weblogssl.com/fandemia') ,(u'Tendencias', u'http://feeds.weblogssl.com/trendencias') - ,(u'Beb\xe9s y m\xe1s', u'http://feeds.weblogssl.com/bebesymas') + ,(u'Tendencias Belleza', u'http://feeds.weblogssl.com/trendenciasbelleza') + ,(u'Tendencias Hombre', u'http://feeds.weblogssl.com/trendenciashombre') + ,(u'Tendencias Shopping', u'http://feeds.weblogssl.com/trendenciasshopping') ,(u'Directo al paladar', u'http://feeds.weblogssl.com/directoalpaladar') ,(u'Compradicci\xf3n', u'http://feeds.weblogssl.com/compradiccion') ,(u'Decoesfera', u'http://feeds.weblogssl.com/decoesfera') ,(u'Embelezzia', u'http://feeds.weblogssl.com/embelezzia') ,(u'Vit\xf3nica', u'http://feeds.weblogssl.com/vitonica') ,(u'Ambiente G', u'http://feeds.weblogssl.com/ambienteg') - ,(u'Tendencias Belleza', u'http://feeds.weblogssl.com/trendenciasbelleza') - ,(u'Tendencias Hombre', u'http://feeds.weblogssl.com/trendenciashombre') + ,(u'Beb\xe9s y m\xe1s', u'http://feeds.weblogssl.com/bebesymas') ,(u'Peques y m\xe1s', u'http://feeds.weblogssl.com/pequesymas') ,(u'Motorpasi\xf3n', u'http://feeds.weblogssl.com/motorpasion') ,(u'Motorpasi\xf3n F1', u'http://feeds.weblogssl.com/motorpasionf1') @@ -90,7 +91,7 @@ class weblogssl(BasicNewsRecipe): dict(name='section' , attrs={'class':'comments'}), #m.xataka.com dict(name='div' , attrs={'class':'article-comments'}), #m.xataka.com dict(name='nav' , attrs={'class':'article-taxonomy'}) #m.xataka.com - ] + ] remove_tags_after = dict(name='section' , attrs={'class':'comments'}) @@ -119,23 +120,6 @@ class weblogssl(BasicNewsRecipe): return soup - # Para obtener la url original del articulo a partir de la de "feedsportal" - # El siguiente código es gracias al usuario "bosplans" de www.mobileread.com - # http://www.mobileread.com/forums/showthread.php?t=130297 def get_article_url(self, article): - link = article.get('link', None) - if link is None: - return article - # if link.split('/')[-4]=="xataka2": - # return article.get('feedburner_origlink', article.get('link', article.get('guid'))) - if link.split('/')[-4]=="xataka2": return article.get('guid', None) - if link.split('/')[-1]=="story01.htm": - link=link.split('/')[-2] - a=['0B','0C','0D','0E','0F','0G','0N' ,'0L0S','0A'] - b=['.' ,'/' ,'?' ,'-' ,'=' ,'&' ,'.com','www.','0'] - for i in range(0,len(a)): - link=link.replace(a[i],b[i]) - link="http://"+link - return link From 316b44c764c78fca0f2d59792c451ec3cb602f03 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 15 May 2013 11:13:49 +0530 Subject: [PATCH 17/18] PDF Input: Fix crashes on some malformed files, by updating the PDF library calibre uses (poppler 0.22.4) --- setup/installer/linux/freeze2.py | 2 +- setup/installer/osx/app/main.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup/installer/linux/freeze2.py b/setup/installer/linux/freeze2.py index e43bfa193e..4e7f2728dc 100644 --- a/setup/installer/linux/freeze2.py +++ b/setup/installer/linux/freeze2.py @@ -38,7 +38,7 @@ binary_includes = [ '/lib/libz.so.1', '/usr/lib/libtiff.so.5', '/lib/libbz2.so.1', - '/usr/lib/libpoppler.so.28', + '/usr/lib/libpoppler.so.37', '/usr/lib/libxml2.so.2', '/usr/lib/libopenjpeg.so.2', '/usr/lib/libxslt.so.1', diff --git a/setup/installer/osx/app/main.py b/setup/installer/osx/app/main.py index f72928360b..9618b90232 100644 --- a/setup/installer/osx/app/main.py +++ b/setup/installer/osx/app/main.py @@ -378,7 +378,7 @@ class Py2App(object): @flush def add_poppler(self): info('\nAdding poppler') - for x in ('libpoppler.28.dylib',): + for x in ('libpoppler.37.dylib',): self.install_dylib(os.path.join(SW, 'lib', x)) for x in ('pdftohtml', 'pdftoppm', 'pdfinfo'): self.install_dylib(os.path.join(SW, 'bin', x), False) From a930622f1141faf9d242d07e1af3ef5a5efa4752 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 15 May 2013 11:21:06 +0530 Subject: [PATCH 18/18] ToC Editor: Fix incorrect playOrders in the generated toc.ncx when editing the toc in an epub file. This apparently affects FBReader. --- src/calibre/ebooks/oeb/polish/toc.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/calibre/ebooks/oeb/polish/toc.py b/src/calibre/ebooks/oeb/polish/toc.py index c84dd1b094..8be23bdc38 100644 --- a/src/calibre/ebooks/oeb/polish/toc.py +++ b/src/calibre/ebooks/oeb/polish/toc.py @@ -9,7 +9,7 @@ __docformat__ = 'restructuredtext en' import re from urlparse import urlparse -from collections import deque +from collections import deque, Counter from functools import partial from lxml import etree @@ -29,7 +29,8 @@ class TOC(object): def __init__(self, title=None, dest=None, frag=None): self.title, self.dest, self.frag = title, dest, frag self.dest_exists = self.dest_error = None - if self.title: self.title = self.title.strip() + if self.title: + self.title = self.title.strip() self.parent = None self.children = [] @@ -326,11 +327,13 @@ def create_ncx(toc, to_href, btitle, lang, uid): navmap = etree.SubElement(ncx, NCX('navMap')) spat = re.compile(r'\s+') - def process_node(xml_parent, toc_parent, play_order=0): + play_order = Counter() + + def process_node(xml_parent, toc_parent): for child in toc_parent: - play_order += 1 + play_order['c'] += 1 point = etree.SubElement(xml_parent, NCX('navPoint'), id=uuid_id(), - playOrder=str(play_order)) + playOrder=str(play_order['c'])) label = etree.SubElement(point, NCX('navLabel')) title = child.title if title: @@ -341,7 +344,7 @@ def create_ncx(toc, to_href, btitle, lang, uid): if child.frag: href += '#'+child.frag etree.SubElement(point, NCX('content'), src=href) - process_node(point, child, play_order) + process_node(point, child) process_node(navmap, toc) return ncx