From ae13c3430f9a68f0e26472b0ccf03d55ec6cef53 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 19 Oct 2012 12:12:18 +0530 Subject: [PATCH 01/23] ... --- src/calibre/library/cli.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/calibre/library/cli.py b/src/calibre/library/cli.py index 615da6d71d..10265dd773 100644 --- a/src/calibre/library/cli.py +++ b/src/calibre/library/cli.py @@ -695,13 +695,13 @@ datatype is one of: {0} '--display="{\\"enum_values\\":[\\"val1\\", \\"val2\\"]}"' '\n' 'There are many options that can go into the display variable.' - 'The options by column type are:' + 'The options by column type are:\n' 'composite: composite_template, composite_sort, make_category,' - ' contains_html, use_decorations' - 'datetime: date_format' - 'enumeration: enum_values, enum_colors, use_decorations' - 'int, float: number_format' - 'text: is_names, use_decorations' + 'contains_html, use_decorations\n' + 'datetime: date_format\n' + 'enumeration: enum_values, enum_colors, use_decorations\n' + 'int, float: number_format\n' + 'text: is_names, use_decorations\n' '\n' 'The best way to find legal combinations is to create a custom' 'column of the appropriate type in the GUI then look at the' From be84b61294ce5e0e639794d42ce29938598bdb75 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 19 Oct 2012 16:38:15 +0530 Subject: [PATCH 02/23] Fix Boston Globe --- recipes/boston.com.recipe | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/recipes/boston.com.recipe b/recipes/boston.com.recipe index 48add6112c..e8eb7d1a8b 100644 --- a/recipes/boston.com.recipe +++ b/recipes/boston.com.recipe @@ -15,7 +15,8 @@ class BusinessStandard(BasicNewsRecipe): no_stylesheets = True delay = 1 use_embedded_content = False - encoding = 'cp1252' + auto_cleanup = True + encoding = 'utf-8' publisher = 'Boston' category = 'news, boston, usa, world' language = 'en' @@ -30,23 +31,23 @@ class BusinessStandard(BasicNewsRecipe): ,'publisher' : publisher } - keep_only_tags = [dict(attrs={'id':['INDblogEntry','blogEntry','articleHeader','articleGraphs','galleryShell']})] - remove_tags = [ - dict(name=['object','link','script','iframe']) - ,dict(attrs={'id':['blogheadTools','bdc_emailWidget','tools','relatedContent']}) - ] + #keep_only_tags = [dict(attrs={'id':['INDblogEntry','blogEntry','articleHeader','articleGraphs','galleryShell']})] + #remove_tags = [ + #dict(name=['object','link','script','iframe']) + #,dict(attrs={'id':['blogheadTools','bdc_emailWidget','tools','relatedContent']}) + #] feeds = [ (u'Top Stories' , u'http://feeds.boston.com/boston/topstories' ) - ,(u'Patriots news', u'http://feeds.boston.com/boston/sports/football/patriots') + ,(u'Patriots news', u'http://feeds.boston.com/boston/sports/football/patriots/patriots_rss') ,(u'National news', u'http://feeds.boston.com/boston/news/nation' ) ,(u'World news' , u'http://feeds.boston.com/boston/news/world' ) ] - def print_version(self, url): - return url + '?page=full' + #def print_version(self, url): + #return url + '?page=full' - def get_article_url(self, article): - rawarticle = article.get('guid', None) - return rawarticle.rpartition('?')[0] + #def get_article_url(self, article): + #rawarticle = article.get('guid', None) + #return rawarticle.rpartition('?')[0] From d5c07a7daf630fc053b39be9bceae8a1dd23e3c6 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 19 Oct 2012 20:49:32 +0530 Subject: [PATCH 03/23] Update Daily Mirror --- recipes/daily_mirror.recipe | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/recipes/daily_mirror.recipe b/recipes/daily_mirror.recipe index b53a22b648..bff337bcf7 100644 --- a/recipes/daily_mirror.recipe +++ b/recipes/daily_mirror.recipe @@ -7,7 +7,7 @@ class AdvancedUserRecipe1306061239(BasicNewsRecipe): description = 'News as provided by The Daily Mirror -UK' __author__ = 'Dave Asbury' - # last updated 8/6/12 + # last updated 19/10/12 language = 'en_GB' #cover_url = 'http://yookeo.com/screens/m/i/mirror.co.uk.jpg' @@ -15,10 +15,12 @@ class AdvancedUserRecipe1306061239(BasicNewsRecipe): oldest_article = 1 - max_articles_per_feed = 12 + max_articles_per_feed = 1 remove_empty_feeds = True remove_javascript = True no_stylesheets = True + ignore_duplicate_articles = {'title'} + # auto_cleanup = True #conversion_options = { 'linearize_tables' : True } @@ -60,11 +62,12 @@ class AdvancedUserRecipe1306061239(BasicNewsRecipe): # example of commented out feed not needed ,(u'Travel','http://www.mirror.co.uk/advice/travel/rss.xml') ] - extra_css = ''' - h1{ font-size:medium;} - body{ text-align: justify; font-family:Arial,Helvetica,sans-serif; font-size:11px; font-size-adjust:none; font-stretch:normal; font-style:normal; font-variant:normal; font-weight:normal;} - img { display:block} - '''# + extra_css = ''' + h1{font-family:Arial,Helvetica,sans-serif; font-weight:bold;font-size:large;} + h2{font-family:Arial,Helvetica,sans-serif; font-weight:normal;font-size:small;} + p{font-family:Arial,Helvetica,sans-serif;font-size:small;} + body{font-family:Helvetica,Arial,sans-serif;font-size:small;} + ''' def get_cover_url(self): soup = self.index_to_soup('http://www.politicshome.com/uk/latest_frontpage.html') @@ -75,8 +78,10 @@ class AdvancedUserRecipe1306061239(BasicNewsRecipe): #cov2 now contains url of the page containing pic soup = self.index_to_soup(cov2) cov = soup.find(attrs={'id' : 'large'}) - cov2 = str(cov) - cov2=cov2[27:-18] + cov=str(cov) + cov2 = re.findall('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+', cov) + cov2 = str(cov2) + cov2=cov2[2:len(cov2)-2] #cov2 now is pic url, now go back to original function br = browser() br.set_handle_redirect(False) From 950a288d9ab698c42e2d33bb460461a16722aee8 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 19 Oct 2012 21:00:37 +0530 Subject: [PATCH 04/23] ... --- recipes/the_sun.recipe | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/recipes/the_sun.recipe b/recipes/the_sun.recipe index d7966c8289..3155bce3f2 100644 --- a/recipes/the_sun.recipe +++ b/recipes/the_sun.recipe @@ -1,4 +1,4 @@ -import random +import re, random from calibre import browser from calibre.web.feeds.recipes import BasicNewsRecipe @@ -8,7 +8,7 @@ class AdvancedUserRecipe1325006965(BasicNewsRecipe): title = u'The Sun UK' description = 'Articles from The Sun tabloid UK' __author__ = 'Dave Asbury' - # last updated 12/10/12 added starsons remove article code + # last updated 19/10/12 better cover fetch language = 'en_GB' oldest_article = 1 max_articles_per_feed = 15 @@ -19,7 +19,7 @@ class AdvancedUserRecipe1325006965(BasicNewsRecipe): remove_javascript = True no_stylesheets = True - ignore_duplicate_articles = {'title'} + ignore_duplicate_articles = {'title','url'} extra_css = ''' @@ -72,9 +72,10 @@ class AdvancedUserRecipe1325006965(BasicNewsRecipe): #cov2 now contains url of the page containing pic soup = self.index_to_soup(cov2) cov = soup.find(attrs={'id' : 'large'}) - cov2 = str(cov) - cov2=cov2[27:-18] - #cov2 now is pic url, now go back to original function + cov=str(cov) + cov2 = re.findall('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+', cov) + cov2 = str(cov2) + cov2=cov2[2:len(cov2)-2] br = browser() br.set_handle_redirect(False) try: From 524e0119edac00c8fa9dbe00362f6bbfac5cad21 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 20 Oct 2012 08:59:49 +0530 Subject: [PATCH 05/23] Fix regression that broke certain CSS selectors. Fixes #1068937 (Regression: CSS not interpreted correctly) --- src/calibre/ebooks/oeb/stylizer.py | 7 ++++++- src/calibre/ebooks/oeb/transforms/split.py | 5 +++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/calibre/ebooks/oeb/stylizer.py b/src/calibre/ebooks/oeb/stylizer.py index 68978b9637..1579dde481 100644 --- a/src/calibre/ebooks/oeb/stylizer.py +++ b/src/calibre/ebooks/oeb/stylizer.py @@ -125,6 +125,11 @@ class CaseInsensitiveAttributesTranslator(HTMLTranslator): (id_selector.id.lower())) ci_css_to_xpath = CaseInsensitiveAttributesTranslator().css_to_xpath +NULL_NAMESPACE_REGEX = re.compile(ur'''name\(\) = ['"]h:''') +def fix_namespace(raw): + ans = NULL_NAMESPACE_REGEX.sub(lambda + m:m.group().replace(u'h:', u''), raw) + return ans class CSSSelector(object): @@ -136,7 +141,7 @@ class CSSSelector(object): def build_selector(self, css, log, func=css_to_xpath): try: - return etree.XPath(func(css), namespaces=self.namespaces) + return etree.XPath(fix_namespace(func(css)), namespaces=self.namespaces) except: if log is not None: log.exception('Failed to parse CSS selector: %r'%css) diff --git a/src/calibre/ebooks/oeb/transforms/split.py b/src/calibre/ebooks/oeb/transforms/split.py index e46ddb5fb5..5b00400dbf 100644 --- a/src/calibre/ebooks/oeb/transforms/split.py +++ b/src/calibre/ebooks/oeb/transforms/split.py @@ -73,6 +73,7 @@ class Split(object): def find_page_breaks(self, item): if self.page_break_selectors is None: + from calibre.ebooks.oeb.stylizer import fix_namespace css_to_xpath = HTMLTranslator().css_to_xpath self.page_break_selectors = set([]) stylesheets = [x.data for x in self.oeb.manifest if x.media_type in @@ -84,7 +85,7 @@ class Split(object): 'page-break-after'), 'cssText', '').strip().lower() try: if before and before not in {'avoid', 'auto', 'inherit'}: - self.page_break_selectors.add((XPath(css_to_xpath(rule.selectorText)), + self.page_break_selectors.add((XPath(fix_namespace(css_to_xpath(rule.selectorText))), True)) if self.remove_css_pagebreaks: rule.style.removeProperty('page-break-before') @@ -92,7 +93,7 @@ class Split(object): pass try: if after and after not in {'avoid', 'auto', 'inherit'}: - self.page_break_selectors.add((XPath(css_to_xpath(rule.selectorText)), + self.page_break_selectors.add((XPath(fix_namespace(css_to_xpath(rule.selectorText))), False)) if self.remove_css_pagebreaks: rule.style.removeProperty('page-break-after') From ade7c9280a2241fb033669e596a8b89f72743fda Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 20 Oct 2012 09:07:03 +0530 Subject: [PATCH 06/23] Faster implementation of namespace hack --- src/calibre/ebooks/oeb/stylizer.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/calibre/ebooks/oeb/stylizer.py b/src/calibre/ebooks/oeb/stylizer.py index 1579dde481..6b82f2f801 100644 --- a/src/calibre/ebooks/oeb/stylizer.py +++ b/src/calibre/ebooks/oeb/stylizer.py @@ -125,11 +125,17 @@ class CaseInsensitiveAttributesTranslator(HTMLTranslator): (id_selector.id.lower())) ci_css_to_xpath = CaseInsensitiveAttributesTranslator().css_to_xpath -NULL_NAMESPACE_REGEX = re.compile(ur'''name\(\) = ['"]h:''') + +NULL_NAMESPACE_REGEX = re.compile(ur'''(name\(\) = ['"])h:''') def fix_namespace(raw): - ans = NULL_NAMESPACE_REGEX.sub(lambda - m:m.group().replace(u'h:', u''), raw) - return ans + ''' + cssselect uses name() = 'h:p' to select tags for some CSS selectors (e.g. + h|p+h|p). + However, since for us the XHTML namespace is the default namespace (with no + prefix), name() is the same as local-name(). So this is a hack to + workaround the problem. + ''' + return NULL_NAMESPACE_REGEX.sub(ur'\1', raw) class CSSSelector(object): From 42f2f945e7975451bcfd48242b8ba1f3bef56424 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 20 Oct 2012 13:28:05 +0530 Subject: [PATCH 07/23] Start a python interface to FreeType --- session.vim | 1 + setup/build_environment.py | 8 + setup/extensions.py | 8 +- src/calibre/constants.py | 1 + src/calibre/test_build.py | 8 +- src/calibre/utils/fonts/__init__.py | 2 +- src/calibre/utils/fonts/freetype.cpp | 289 +++++++++++++++++++++++++++ src/calibre/utils/fonts/freetype.py | 69 +++++++ 8 files changed, 383 insertions(+), 3 deletions(-) create mode 100644 src/calibre/utils/fonts/freetype.cpp create mode 100644 src/calibre/utils/fonts/freetype.py diff --git a/session.vim b/session.vim index 6b8878b84d..c60bcbdf4f 100644 --- a/session.vim +++ b/session.vim @@ -8,6 +8,7 @@ let g:syntastic_cpp_include_dirs = [ \'/usr/include/qt4/QtCore', \'/usr/include/qt4/QtGui', \'/usr/include/qt4', + \'/usr/include/freetype2', \'src/qtcurve/common', 'src/qtcurve', \'/usr/include/ImageMagick', \] diff --git a/setup/build_environment.py b/setup/build_environment.py index 58d2006ee7..69b1dc231b 100644 --- a/setup/build_environment.py +++ b/setup/build_environment.py @@ -84,6 +84,7 @@ qt_inc = pyqt.qt_inc_dir qt_lib = pyqt.qt_lib_dir ft_lib_dirs = [] ft_libs = [] +ft_inc_dirs = [] jpg_libs = [] jpg_lib_dirs = [] fc_inc = '/usr/include/fontconfig' @@ -116,6 +117,7 @@ if iswindows: jpg_libs = ['jpeg'] ft_lib_dirs = [sw_lib_dir] ft_libs = ['freetype'] + ft_inc_dirs = [sw_inc_dir] magick_inc_dirs = [os.path.join(prefix, 'build', 'ImageMagick-6.7.6')] magick_lib_dirs = [os.path.join(magick_inc_dirs[0], 'VisualMagick', 'lib')] @@ -135,6 +137,8 @@ elif isosx: png_inc_dirs = consolidate('PNG_INC_DIR', '/sw/include') png_lib_dirs = consolidate('PNG_LIB_DIR', '/sw/lib') png_libs = ['png12'] + ft_libs = ['freetype'] + ft_inc_dirs = ['/sw/include/freetype2'] else: # Include directories png_inc_dirs = pkgconfig_include_dirs('libpng', 'PNG_INC_DIR', @@ -150,6 +154,10 @@ else: if not magick_libs: magick_libs = ['MagickWand', 'MagickCore'] png_libs = ['png'] + ft_inc_dirs = pkgconfig_include_dirs('freetype2', 'FT_INC_DIR', + '/usr/include/freetype2') + ft_lib_dirs = pkgconfig_lib_dirs('freetype2', 'FT_LIB_DIR', '/usr/lib') + ft_libs = pkgconfig_libs('freetype2', '', '') fc_inc = os.environ.get('FC_INC_DIR', fc_inc) diff --git a/setup/extensions.py b/setup/extensions.py index 1827d32f4a..2cae3d857f 100644 --- a/setup/extensions.py +++ b/setup/extensions.py @@ -17,7 +17,7 @@ from setup.build_environment import (fc_inc, fc_lib, chmlib_inc_dirs, fc_error, podofo_inc, podofo_lib, podofo_error, pyqt, OSX_SDK, NMAKE, QMAKE, msvc, MT, win_inc, win_lib, win_ddk, magick_inc_dirs, magick_lib_dirs, magick_libs, chmlib_lib_dirs, sqlite_inc_dirs, icu_inc_dirs, - icu_lib_dirs, win_ddk_lib_dirs) + icu_lib_dirs, win_ddk_lib_dirs, ft_libs, ft_lib_dirs, ft_inc_dirs) MT isunix = islinux or isosx or isbsd @@ -119,6 +119,12 @@ extensions = [ 'calibre/utils/lzx/mspack.h'], inc_dirs=['calibre/utils/lzx']), + Extension('freetype', + ['calibre/utils/fonts/freetype.cpp'], + inc_dirs = ft_inc_dirs, + libraries=ft_libs, + lib_dirs=ft_lib_dirs), + Extension('fontconfig', ['calibre/utils/fonts/fontconfig.c'], inc_dirs = [fc_inc], diff --git a/src/calibre/constants.py b/src/calibre/constants.py index 35b03f2b2f..fe3c40309e 100644 --- a/src/calibre/constants.py +++ b/src/calibre/constants.py @@ -89,6 +89,7 @@ class Plugins(collections.Mapping): 'chm_extra', 'icu', 'speedup', + 'freetype', ] if iswindows: plugins.extend(['winutil', 'wpd', 'winfonts']) diff --git a/src/calibre/test_build.py b/src/calibre/test_build.py index 35e050cf19..e6e8611ac5 100644 --- a/src/calibre/test_build.py +++ b/src/calibre/test_build.py @@ -32,6 +32,11 @@ def test_lxml(): else: raise RuntimeError('lxml failed') +def test_freetype(): + from calibre.utils.fonts.freetype import test + test() + print ('FreeType OK!') + def test_fontconfig(): from calibre.utils.fonts import fontconfig families = fontconfig.find_font_families() @@ -103,7 +108,7 @@ def test_icu(): def test_wpd(): wpd = plugins['wpd'][0] try: - wpd.init() + wpd.init('calibre', 1, 1, 1) except wpd.NoWPD: print ('This computer does not have WPD') else: @@ -112,6 +117,7 @@ def test_wpd(): def test(): test_plugins() test_lxml() + test_freetype() test_fontconfig() test_sqlite() test_qt() diff --git a/src/calibre/utils/fonts/__init__.py b/src/calibre/utils/fonts/__init__.py index a5563acd4e..45a665b75a 100644 --- a/src/calibre/utils/fonts/__init__.py +++ b/src/calibre/utils/fonts/__init__.py @@ -55,7 +55,7 @@ class Fonts(object): files = self.backend.files_for_family(family, normalize=normalize) ans = {} for ft, val in files.iteritems(): - name, f = val + f, name = val ext = f.rpartition('.')[-1].lower() ans[ft] = (ext, name, open(f, 'rb').read()) return ans diff --git a/src/calibre/utils/fonts/freetype.cpp b/src/calibre/utils/fonts/freetype.cpp new file mode 100644 index 0000000000..a5529b3200 --- /dev/null +++ b/src/calibre/utils/fonts/freetype.cpp @@ -0,0 +1,289 @@ +/* + * freetype.cpp + * Copyright (C) 2012 Kovid Goyal + * + * Distributed under terms of the GPL3 license. + */ + +#define _UNICODE +#define UNICODE +#define PY_SSIZE_T_CLEAN +#include + +#include +#include FT_FREETYPE_H + +static PyObject *FreeTypeError = NULL; + +typedef struct { + PyObject_HEAD + FT_Face *face; + // Every face must keep a reference to the FreeType library object to + // ensure it is garbage collected before the library object, to prevent + // segfaults. + PyObject *library; +} Face; + +typedef struct { + PyObject_HEAD + FT_Library *library; +} FreeType; + +// Face.__init__() {{{ +static void +Face_dealloc(Face* self) +{ + if (self->face != NULL) { + Py_BEGIN_ALLOW_THREADS; + FT_Done_Face(*(self->face)); + free(self->face); + Py_END_ALLOW_THREADS; + } + self->face = NULL; + + Py_DECREF(self->library); + self->library = NULL; + + self->ob_type->tp_free((PyObject*)self); +} + +static int +Face_init(Face *self, PyObject *args, PyObject *kwds) +{ + FT_Error error = 0; + char *data; + Py_ssize_t sz; + PyObject *ft; + + if (!PyArg_ParseTuple(args, "Os#", &ft, &data, &sz)) return -1; + self->face = (FT_Face*)calloc(1, sizeof(FT_Face)); + if (self->face == NULL) { PyErr_NoMemory(); return -1; } + self->library = ft; + Py_XINCREF(ft); + + Py_BEGIN_ALLOW_THREADS; + error = FT_New_Memory_Face( *(( (FreeType*)ft )->library), + (const FT_Byte*)data, (FT_Long)sz, 0, self->face); + Py_END_ALLOW_THREADS; + if (error) { + free(self->face); self->face = NULL; + if ( error == FT_Err_Unknown_File_Format || error == FT_Err_Invalid_Stream_Operation) + PyErr_SetString(FreeTypeError, "Not a supported font format"); + else + PyErr_Format(FreeTypeError, "Failed to initialize the Font with error: 0x%x", error); + return -1; + } + return 0; +} + +// }}} + +static PyObject* +supports_text(Face *self, PyObject *args) { + PyObject *chars, *fast, *ret = Py_True; + Py_ssize_t sz, i; + FT_ULong code; + FT_Face face; + + if (!PyArg_ParseTuple(args, "O", &chars)) return NULL; + fast = PySequence_Fast(chars, "List of chars is not a sequence"); + if (fast == NULL) return NULL; + sz = PySequence_Fast_GET_SIZE(fast); + face = *(self->face); + + for (i = 0; i < sz; i++) { + code = (FT_ULong)PyNumber_AsSsize_t(PySequence_Fast_GET_ITEM(fast, i), NULL); + if (FT_Get_Char_Index(face, code) == 0) { + ret = Py_False; + break; + } + } + + Py_DECREF(fast); + Py_XINCREF(ret); + return ret; +} + +static PyMethodDef Face_methods[] = { + {"supports_text", (PyCFunction)supports_text, METH_VARARGS, + "supports_text(sequence of unicode character codes) -> Return True iff this font has glyphs for all the specified characters." + }, + + {NULL} /* Sentinel */ +}; + +// FreeType.__init__() {{{ +static void +dealloc(FreeType* self) +{ + if (self->library != NULL) { + Py_BEGIN_ALLOW_THREADS; + FT_Done_FreeType(*(self->library)); + free(self->library); + Py_END_ALLOW_THREADS; + } + self->library = NULL; + + self->ob_type->tp_free((PyObject*)self); +} + +static int +init(FreeType *self, PyObject *args, PyObject *kwds) +{ + FT_Error error = 0; + self->library = (FT_Library*)calloc(1, sizeof(FT_Library)); + if (self->library == NULL) { PyErr_NoMemory(); return -1; } + Py_BEGIN_ALLOW_THREADS; + error = FT_Init_FreeType(self->library); + Py_END_ALLOW_THREADS; + if (error) { + free(self->library); + self->library = NULL; + PyErr_Format(FreeTypeError, "Failed to initialize the FreeType library with error: %d", error); + return -1; + } + return 0; +} + +// }}} + +static PyTypeObject FaceType = { // {{{ + PyObject_HEAD_INIT(NULL) + 0, /*ob_size*/ + "freetype.Face", /*tp_name*/ + sizeof(Face), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + (destructor)Face_dealloc, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_compare*/ + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash */ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE, /*tp_flags*/ + "Face", /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + Face_methods, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + (initproc)Face_init, /* tp_init */ + 0, /* tp_alloc */ + 0, /* tp_new */ +}; // }}} + +static PyObject* +load_font(FreeType *self, PyObject *args) { + PyObject *ret, *arg_list, *bytes; + + if (!PyArg_ParseTuple(args, "O", &bytes)) return NULL; + + arg_list = Py_BuildValue("OO", self, bytes); + if (arg_list == NULL) return NULL; + + ret = PyObject_CallObject((PyObject *) &FaceType, arg_list); + Py_DECREF(arg_list); + + return ret; +} + +static PyMethodDef FreeType_methods[] = { + {"load_font", (PyCFunction)load_font, METH_VARARGS, + "load_font(bytestring) -> Load a font from font data." + }, + + {NULL} /* Sentinel */ +}; + + +static PyTypeObject FreeTypeType = { // {{{ + PyObject_HEAD_INIT(NULL) + 0, /*ob_size*/ + "freetype.FreeType", /*tp_name*/ + sizeof(FreeType), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + (destructor)dealloc, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_compare*/ + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash */ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE, /*tp_flags*/ + "FreeType", /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + FreeType_methods, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + (initproc)init, /* tp_init */ + 0, /* tp_alloc */ + 0, /* tp_new */ +}; // }}} + +static +PyMethodDef methods[] = { + {NULL, NULL, 0, NULL} +}; + +PyMODINIT_FUNC +initfreetype(void) { + PyObject *m; + + FreeTypeType.tp_new = PyType_GenericNew; + if (PyType_Ready(&FreeTypeType) < 0) + return; + + FaceType.tp_new = PyType_GenericNew; + if (PyType_Ready(&FaceType) < 0) + return; + + m = Py_InitModule3( + "freetype", methods, + "FreeType API" + ); + if (m == NULL) return; + + FreeTypeError = PyErr_NewException((char*)"freetype.FreeTypeError", NULL, NULL); + if (FreeTypeError == NULL) return; + PyModule_AddObject(m, "FreeTypeError", FreeTypeError); + + Py_INCREF(&FreeTypeType); + PyModule_AddObject(m, "FreeType", (PyObject *)&FreeTypeType); + PyModule_AddObject(m, "Face", (PyObject *)&FaceType); +} + diff --git a/src/calibre/utils/fonts/freetype.py b/src/calibre/utils/fonts/freetype.py new file mode 100644 index 0000000000..41e9592971 --- /dev/null +++ b/src/calibre/utils/fonts/freetype.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:fdm=marker:ai +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +__license__ = 'GPL v3' +__copyright__ = '2012, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + +import threading +from functools import wraps +from future_builtins import map + +from calibre.constants import plugins + +class ThreadingViolation(Exception): + + def __init__(self): + Exception.__init__(self, + 'You cannot use the MTP driver from a thread other than the ' + ' thread in which startup() was called') + +def same_thread(func): + @wraps(func) + def check_thread(self, *args, **kwargs): + if self.start_thread is not threading.current_thread(): + raise ThreadingViolation() + return func(self, *args, **kwargs) + return check_thread + +class Face(object): + + def __init__(self, face): + self.start_thread = threading.current_thread() + self.face = face + + @same_thread + def supports_text(self, text): + if not isinstance(text, unicode): + raise TypeError('%r is not a unicode object'%text) + chars = tuple(frozenset(map(ord, text))) + return self.face.supports_text(chars) + +class FreeType(object): + + def __init__(self): + self.start_thread = threading.current_thread() + ft, ft_err = plugins['freetype'] + if ft_err: + raise RuntimeError('Failed to load FreeType module with error: %s' + % ft_err) + self.ft = ft.FreeType() + + @same_thread + def load_font(self, data): + return Face(self.ft.load_font(data)) + +def test(): + data = P('fonts/calibreSymbols.otf', data=True) + ft = FreeType() + font = ft.load_font(data) + if not font.supports_text('.\u2605★'): + raise RuntimeError('Incorrectly returning that text is not supported') + if font.supports_text('abc'): + raise RuntimeError('Incorrectly claiming that text is supported') + +if __name__ == '__main__': + test() + From 407cd60ddaa998efbaebcaf2e6af712ba710795c Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 20 Oct 2012 14:14:24 +0530 Subject: [PATCH 08/23] Get family and style names from FreeType --- src/calibre/utils/fonts/freetype.cpp | 53 ++++++++++++++++++---------- src/calibre/utils/fonts/freetype.py | 7 ++++ 2 files changed, 41 insertions(+), 19 deletions(-) diff --git a/src/calibre/utils/fonts/freetype.cpp b/src/calibre/utils/fonts/freetype.cpp index a5529b3200..e4e30000a0 100644 --- a/src/calibre/utils/fonts/freetype.cpp +++ b/src/calibre/utils/fonts/freetype.cpp @@ -17,7 +17,7 @@ static PyObject *FreeTypeError = NULL; typedef struct { PyObject_HEAD - FT_Face *face; + FT_Face face; // Every face must keep a reference to the FreeType library object to // ensure it is garbage collected before the library object, to prevent // segfaults. @@ -26,7 +26,7 @@ typedef struct { typedef struct { PyObject_HEAD - FT_Library *library; + FT_Library library; } FreeType; // Face.__init__() {{{ @@ -35,8 +35,7 @@ Face_dealloc(Face* self) { if (self->face != NULL) { Py_BEGIN_ALLOW_THREADS; - FT_Done_Face(*(self->face)); - free(self->face); + FT_Done_Face(self->face); Py_END_ALLOW_THREADS; } self->face = NULL; @@ -56,17 +55,15 @@ Face_init(Face *self, PyObject *args, PyObject *kwds) PyObject *ft; if (!PyArg_ParseTuple(args, "Os#", &ft, &data, &sz)) return -1; - self->face = (FT_Face*)calloc(1, sizeof(FT_Face)); - if (self->face == NULL) { PyErr_NoMemory(); return -1; } self->library = ft; Py_XINCREF(ft); Py_BEGIN_ALLOW_THREADS; - error = FT_New_Memory_Face( *(( (FreeType*)ft )->library), - (const FT_Byte*)data, (FT_Long)sz, 0, self->face); + error = FT_New_Memory_Face( ( (FreeType*)ft )->library, + (const FT_Byte*)data, (FT_Long)sz, 0, &self->face); Py_END_ALLOW_THREADS; if (error) { - free(self->face); self->face = NULL; + self->face = NULL; if ( error == FT_Err_Unknown_File_Format || error == FT_Err_Invalid_Stream_Operation) PyErr_SetString(FreeTypeError, "Not a supported font format"); else @@ -78,22 +75,30 @@ Face_init(Face *self, PyObject *args, PyObject *kwds) // }}} +static PyObject * +family_name(Face *self, void *closure) { + return Py_BuildValue("s", self->face->family_name); +} + +static PyObject * +style_name(Face *self, void *closure) { + return Py_BuildValue("s", self->face->style_name); +} + static PyObject* supports_text(Face *self, PyObject *args) { PyObject *chars, *fast, *ret = Py_True; Py_ssize_t sz, i; FT_ULong code; - FT_Face face; if (!PyArg_ParseTuple(args, "O", &chars)) return NULL; fast = PySequence_Fast(chars, "List of chars is not a sequence"); if (fast == NULL) return NULL; sz = PySequence_Fast_GET_SIZE(fast); - face = *(self->face); for (i = 0; i < sz; i++) { code = (FT_ULong)PyNumber_AsSsize_t(PySequence_Fast_GET_ITEM(fast, i), NULL); - if (FT_Get_Char_Index(face, code) == 0) { + if (FT_Get_Char_Index(self->face, code) == 0) { ret = Py_False; break; } @@ -104,6 +109,20 @@ supports_text(Face *self, PyObject *args) { return ret; } +static PyGetSetDef Face_getsetters[] = { + {(char *)"family_name", + (getter)family_name, NULL, + (char *)"The family name of this font.", + NULL}, + + {(char *)"style_name", + (getter)style_name, NULL, + (char *)"The style name of this font.", + NULL}, + + {NULL} /* Sentinel */ +}; + static PyMethodDef Face_methods[] = { {"supports_text", (PyCFunction)supports_text, METH_VARARGS, "supports_text(sequence of unicode character codes) -> Return True iff this font has glyphs for all the specified characters." @@ -118,8 +137,7 @@ dealloc(FreeType* self) { if (self->library != NULL) { Py_BEGIN_ALLOW_THREADS; - FT_Done_FreeType(*(self->library)); - free(self->library); + FT_Done_FreeType(self->library); Py_END_ALLOW_THREADS; } self->library = NULL; @@ -131,13 +149,10 @@ static int init(FreeType *self, PyObject *args, PyObject *kwds) { FT_Error error = 0; - self->library = (FT_Library*)calloc(1, sizeof(FT_Library)); - if (self->library == NULL) { PyErr_NoMemory(); return -1; } Py_BEGIN_ALLOW_THREADS; - error = FT_Init_FreeType(self->library); + error = FT_Init_FreeType(&self->library); Py_END_ALLOW_THREADS; if (error) { - free(self->library); self->library = NULL; PyErr_Format(FreeTypeError, "Failed to initialize the FreeType library with error: %d", error); return -1; @@ -178,7 +193,7 @@ static PyTypeObject FaceType = { // {{{ 0, /* tp_iternext */ Face_methods, /* tp_methods */ 0, /* tp_members */ - 0, /* tp_getset */ + Face_getsetters, /* tp_getset */ 0, /* tp_base */ 0, /* tp_dict */ 0, /* tp_descr_get */ diff --git a/src/calibre/utils/fonts/freetype.py b/src/calibre/utils/fonts/freetype.py index 41e9592971..ac5385ea98 100644 --- a/src/calibre/utils/fonts/freetype.py +++ b/src/calibre/utils/fonts/freetype.py @@ -33,6 +33,13 @@ class Face(object): def __init__(self, face): self.start_thread = threading.current_thread() self.face = face + for x in ('family_name', 'style_name'): + val = getattr(self.face, x) + try: + val = val.decode('utf-8') + except UnicodeDecodeError: + val = repr(val).decode('utf-8') + setattr(self, x, val) @same_thread def supports_text(self, text): From 135123ad4662b990971da861481e5fb7e02861f0 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 20 Oct 2012 17:14:17 +0530 Subject: [PATCH 09/23] Generate cover: If the default font cannot render characters in the metadata (for example for east asian languages) try to automatically find a font on the system that is capable of rendering the characters --- src/calibre/ebooks/__init__.py | 29 ++++++++++-- src/calibre/utils/fonts/__init__.py | 46 +++++++++++++++++++ .../utils/fonts/{freetype.py => free_type.py} | 26 ++++++++++- src/calibre/utils/fonts/utils.py | 43 +++++++++++++++-- src/calibre/utils/magick/draw.py | 20 +++++--- 5 files changed, 148 insertions(+), 16 deletions(-) rename src/calibre/utils/fonts/{freetype.py => free_type.py} (72%) diff --git a/src/calibre/ebooks/__init__.py b/src/calibre/ebooks/__init__.py index ee880000f0..9cf0e51e7e 100644 --- a/src/calibre/ebooks/__init__.py +++ b/src/calibre/ebooks/__init__.py @@ -178,18 +178,41 @@ def normalize(x): def calibre_cover(title, author_string, series_string=None, output_format='jpg', title_size=46, author_size=36, logo_path=None): + from calibre.utils.config_base import tweaks title = normalize(title) author_string = normalize(author_string) series_string = normalize(series_string) from calibre.utils.magick.draw import create_cover_page, TextLine - lines = [TextLine(title, title_size), TextLine(author_string, author_size)] + text = title + author_string + (series_string or u'') + font_path = tweaks['generate_cover_title_font'] + if font_path is None: + font_path = P('fonts/liberation/LiberationSerif-Bold.ttf') + + from calibre.utils.fonts.utils import get_font_for_text + font = open(font_path, 'rb').read() + c = get_font_for_text(text, font) + cleanup = False + if c is not None and c != font: + from calibre.ptempfile import PersistentTemporaryFile + pt = PersistentTemporaryFile('.ttf') + pt.write(c) + pt.close() + font_path = pt.name + cleanup = True + + lines = [TextLine(title, title_size, font_path=font_path), + TextLine(author_string, author_size, font_path=font_path)] if series_string: - lines.append(TextLine(series_string, author_size)) + lines.append(TextLine(series_string, author_size, font_path=font_path)) if logo_path is None: logo_path = I('library.png') - return create_cover_page(lines, logo_path, output_format='jpg', + try: + return create_cover_page(lines, logo_path, output_format='jpg', texture_opacity=0.3, texture_data=I('cover_texture.png', data=True)) + finally: + if cleanup: + os.remove(font_path) UNIT_RE = re.compile(r'^(-*[0-9]*[.]?[0-9]*)\s*(%|em|ex|en|px|mm|cm|in|pt|pc)$') diff --git a/src/calibre/utils/fonts/__init__.py b/src/calibre/utils/fonts/__init__.py index 45a665b75a..e245b1536c 100644 --- a/src/calibre/utils/fonts/__init__.py +++ b/src/calibre/utils/fonts/__init__.py @@ -60,6 +60,52 @@ class Fonts(object): ans[ft] = (ext, name, open(f, 'rb').read()) return ans + def find_font_for_text(self, text, allowed_families={'serif', 'sans-serif'}, + preferred_families=('serif', 'sans-serif', 'monospace', 'cursive', 'fantasy')): + ''' + Find a font on the system capable of rendering the given text. + + Returns a font family (as given by fonts_for_family()) that has a + "normal" font and that can render the supplied text. If no such font + exists, returns None. + + :return: (family name, faces) or None, None + ''' + from calibre.utils.fonts.free_type import FreeType, get_printable_characters, FreeTypeError + from calibre.utils.fonts.utils import panose_to_css_generic_family, get_font_characteristics + ft = FreeType() + found = {} + if not isinstance(text, unicode): + raise TypeError(u'%r is not unicode'%text) + text = get_printable_characters(text) + + def filter_faces(faces): + ans = {} + for k, v in faces.iteritems(): + try: + font = ft.load_font(v[2]) + except FreeTypeError: + continue + if font.supports_text(text, has_non_printable_chars=False): + ans[k] = v + return ans + + for family in sorted(self.find_font_families()): + faces = filter_faces(self.fonts_for_family(family)) + if 'normal' not in faces: + continue + panose = get_font_characteristics(faces['normal'][2])[5] + generic_family = panose_to_css_generic_family(panose) + if generic_family in allowed_families or generic_family == preferred_families[0]: + return (family, faces) + elif generic_family not in found: + found[generic_family] = (family, faces) + + for f in preferred_families: + if f in found: + return found[f] + return None, None + fontconfig = Fonts() def test(): diff --git a/src/calibre/utils/fonts/freetype.py b/src/calibre/utils/fonts/free_type.py similarity index 72% rename from src/calibre/utils/fonts/freetype.py rename to src/calibre/utils/fonts/free_type.py index ac5385ea98..a2e8eca213 100644 --- a/src/calibre/utils/fonts/freetype.py +++ b/src/calibre/utils/fonts/free_type.py @@ -7,7 +7,7 @@ __license__ = 'GPL v3' __copyright__ = '2012, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import threading +import threading, unicodedata from functools import wraps from future_builtins import map @@ -20,6 +20,10 @@ class ThreadingViolation(Exception): 'You cannot use the MTP driver from a thread other than the ' ' thread in which startup() was called') +def get_printable_characters(text): + return u''.join(x for x in unicodedata.normalize('NFC', text) + if unicodedata.category(x)[0] not in {'C', 'Z', 'M'}) + def same_thread(func): @wraps(func) def check_thread(self, *args, **kwargs): @@ -28,6 +32,8 @@ def same_thread(func): return func(self, *args, **kwargs) return check_thread +FreeTypeError = getattr(plugins['freetype'][0], 'FreeTypeError', Exception) + class Face(object): def __init__(self, face): @@ -42,9 +48,14 @@ class Face(object): setattr(self, x, val) @same_thread - def supports_text(self, text): + def supports_text(self, text, has_non_printable_chars=True): + ''' + Returns True if all the characters in text have glyphs in this font. + ''' if not isinstance(text, unicode): raise TypeError('%r is not a unicode object'%text) + if has_non_printable_chars: + text = get_printable_characters(text) chars = tuple(frozenset(map(ord, text))) return self.face.supports_text(chars) @@ -71,6 +82,17 @@ def test(): if font.supports_text('abc'): raise RuntimeError('Incorrectly claiming that text is supported') +def test_find_font(): + from calibre.utils.fonts import fontconfig + abcd = '诶比西迪' + family = fontconfig.find_font_for_text(abcd)[0] + print ('Family for Chinese text:', family) + family = fontconfig.find_font_for_text(abcd)[0] + abcd = 'لوحة المفاتيح العربية' + print ('Family for Arabic text:', family) + + if __name__ == '__main__': test() + test_find_font() diff --git a/src/calibre/utils/fonts/utils.py b/src/calibre/utils/fonts/utils.py index f20f238481..4fcaa20c44 100644 --- a/src/calibre/utils/fonts/utils.py +++ b/src/calibre/utils/fonts/utils.py @@ -38,8 +38,8 @@ def get_table(raw, name): def get_font_characteristics(raw): ''' - Return (weight, is_italic, is_bold, is_regular, fs_type). These values are taken - from the OS/2 table of the font. See + Return (weight, is_italic, is_bold, is_regular, fs_type, panose). These + values are taken from the OS/2 table of the font. See http://www.microsoft.com/typography/otspec/os2.htm for details ''' os2_table = get_table(raw, 'os/2')[0] @@ -54,7 +54,6 @@ def get_font_characteristics(raw): family_class) = struct.unpack_from(common_fields, os2_table) offset = struct.calcsize(common_fields) panose = struct.unpack_from(b'>10B', os2_table, offset) - panose offset += 10 (range1,) = struct.unpack_from(b'>L', os2_table, offset) offset += struct.calcsize(b'>L') @@ -69,7 +68,21 @@ def get_font_characteristics(raw): is_italic = (selection & 0b1) != 0 is_bold = (selection & 0b100000) != 0 is_regular = (selection & 0b1000000) != 0 - return weight, is_italic, is_bold, is_regular, fs_type + return weight, is_italic, is_bold, is_regular, fs_type, panose + +def panose_to_css_generic_family(panose): + proportion = panose[3] + if proportion == 9: + return 'monospace' + family_type = panose[0] + if family_type == 3: + return 'cursive' + if family_type == 4: + return 'fantasy' + serif_style = panose[1] + if serif_style in (11, 12, 13): + return 'sans-serif' + return 'serif' def decode_name_record(recs): ''' @@ -225,13 +238,33 @@ def remove_embed_restriction(raw): verify_checksums(raw) return raw +def get_font_for_text(text, candidate_font_data=None): + ok = False + if candidate_font_data is not None: + from calibre.utils.fonts.free_type import FreeType, FreeTypeError + ft = FreeType() + try: + font = ft.load_font(candidate_font_data) + ok = font.supports_text(text) + except FreeTypeError: + ok = True + if not ok: + from calibre.utils.fonts import fontconfig + family, faces = fontconfig.find_font_for_text(text) + if family is not None: + f = faces.get('bold', faces['normal']) + candidate_font_data = f[2] + return candidate_font_data + def test(): import sys, os for f in sys.argv[1:]: print (os.path.basename(f)) raw = open(f, 'rb').read() print (get_font_names(raw)) - print (get_font_characteristics(raw)) + characs = get_font_characteristics(raw) + print (characs) + print (panose_to_css_generic_family(characs[5])) verify_checksums(raw) remove_embed_restriction(raw) diff --git a/src/calibre/utils/magick/draw.py b/src/calibre/utils/magick/draw.py index 046d0d5224..9d8cfdfcbf 100644 --- a/src/calibre/utils/magick/draw.py +++ b/src/calibre/utils/magick/draw.py @@ -10,7 +10,7 @@ import os from calibre.utils.magick import Image, DrawingWand, create_canvas from calibre.constants import __appname__, __version__ from calibre.utils.config import tweaks -from calibre import fit_image +from calibre import fit_image, force_unicode def _data_to_image(data): if isinstance(data, Image): @@ -166,12 +166,9 @@ def add_borders_to_image(img_data, left=0, top=0, right=0, bottom=0, return canvas.export(fmt) def create_text_wand(font_size, font_path=None): - if font_path is None: - font_path = tweaks['generate_cover_title_font'] - if font_path is None: - font_path = P('fonts/liberation/LiberationSerif-Bold.ttf') ans = DrawingWand() - ans.font = font_path + if font_path is not None: + ans.font = font_path ans.font_size = font_size ans.gravity = 'CenterGravity' ans.text_alias = True @@ -238,6 +235,17 @@ class TextLine(object): def __init__(self, text, font_size, bottom_margin=30, font_path=None): self.text, self.font_size, = text, font_size self.bottom_margin = bottom_margin + if font_path is None: + if not isinstance(text, unicode): + text = force_unicode(text) + from calibre.utils.fonts.utils import get_font_for_text + fd = get_font_for_text(text) + if fd is not None: + from calibre.ptempfile import PersistentTemporaryFile + pt = PersistentTemporaryFile('.ttf') + pt.write(fd) + pt.close() + font_path = pt.name self.font_path = font_path def __repr__(self): From 76312ec4e7a82b25071f4e94a329d095d1ec4bfc Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 20 Oct 2012 18:00:21 +0530 Subject: [PATCH 10/23] Yazihane by A Erdogan --- recipes/yazihane.recipe | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 recipes/yazihane.recipe diff --git a/recipes/yazihane.recipe b/recipes/yazihane.recipe new file mode 100644 index 0000000000..941e3e8b3b --- /dev/null +++ b/recipes/yazihane.recipe @@ -0,0 +1,19 @@ +from calibre.web.feeds.news import BasicNewsRecipe +import re + +class AdvancedUserRecipe1350731826(BasicNewsRecipe): + title = u'Yazihane' + oldest_article = 7 + max_articles_per_feed = 100 + __author__ = 'A Erdogan' + description = 'Sports Blog' + publisher = 'yazihaneden.com' + category = 'sports, basketball, nba, cycling, euroleague' + no_stylesheets = True + use_embedded_content = False + masthead_url = 'http://www.yazihaneden.com/wp-content/uploads/Untitled-1.png' + language = 'tr' + + keep_only_tags = [ dict(name='div', attrs={'id':re.compile('(^|| )post-($|| )', re.DOTALL)})] + remove_tags_after = dict(name='div', attrs={'class':'post-footer clear'}) + feeds = [(u'Yazihane', u'http://www.yazihaneden.com/feed/')] From 765296c2adf3927755a32331fc2b66c25720a955 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 20 Oct 2012 21:37:07 +0530 Subject: [PATCH 11/23] Get Books: Handle website change that broke the SONY Store plugin --- src/calibre/gui2/store/stores/sony_plugin.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/calibre/gui2/store/stores/sony_plugin.py b/src/calibre/gui2/store/stores/sony_plugin.py index 2ad344e82c..aa0c65bcde 100644 --- a/src/calibre/gui2/store/stores/sony_plugin.py +++ b/src/calibre/gui2/store/stores/sony_plugin.py @@ -66,6 +66,8 @@ class SonyStore(BasicStoreConfig, StorePlugin): detail_url = ''.join(item.xpath('descendant::h3[@class="item"]' '/descendant::a[@class="fn" and @href]/@href')) if not detail_url: continue + if detail_url.startswith('/'): + detail_url = 'http:'+detail_url s.detail_item = detail_url counter -= 1 From dee25540472c8add3e077cb85d06118caad915b3 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 21 Oct 2012 11:46:36 +0530 Subject: [PATCH 12/23] ... --- src/calibre/gui2/convert/look_and_feel.ui | 141 ++++++++++++---------- 1 file changed, 77 insertions(+), 64 deletions(-) diff --git a/src/calibre/gui2/convert/look_and_feel.ui b/src/calibre/gui2/convert/look_and_feel.ui index fae1cf2331..d6ecb1b61a 100644 --- a/src/calibre/gui2/convert/look_and_feel.ui +++ b/src/calibre/gui2/convert/look_and_feel.ui @@ -7,27 +7,53 @@ 0 0 655 - 522 + 619 Form - - + + - &Disable font size rescaling + Minimum &line height: + + + opt_minimum_line_height - - + + + + % + + + 1 + + + 900.000000000000000 + + + + + - Base &font size: + Line &height: - opt_base_font_size + opt_line_height + + + + + + + pt + + + 1 @@ -97,49 +123,6 @@ - - - - Minimum &line height: - - - opt_minimum_line_height - - - - - - - % - - - 1 - - - 900.000000000000000 - - - - - - - Line &height: - - - opt_line_height - - - - - - - pt - - - 1 - - - @@ -206,19 +189,6 @@ - - - - &Line size: - - - Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter - - - opt_insert_blank_line_size - - - @@ -378,10 +348,53 @@ + + + + Qt::Vertical + + + + 20 + 40 + + + + + + + + &Disable font size rescaling + + + + + + + Base &font size: + + + opt_base_font_size + + + + + + + &Line size: + + + Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter + + + opt_insert_blank_line_size + + + From 21f8c69496d6ab3ca8d9a07db00fdf49ec34167c Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 21 Oct 2012 11:47:27 +0530 Subject: [PATCH 13/23] Use mimetype for fonts from the epub 3 spec --- resources/mime.types | 8 ++++---- src/calibre/ebooks/oeb/base.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/resources/mime.types b/resources/mime.types index 75452bb17a..f54443f30c 100644 --- a/resources/mime.types +++ b/resources/mime.types @@ -787,12 +787,10 @@ application/x-font-framemaker application/x-font-ghostscript gsf application/x-font-libgrx application/x-font-linux-psf psf -application/x-font-otf otf application/x-font-pcf pcf application/x-font-snf snf application/x-font-speedo application/x-font-sunos-news -application/x-font-ttf ttc ttf application/x-font-type1 afm pfa pfb pfm application/x-font-vfont application/x-freemind mm @@ -1368,8 +1366,6 @@ text/fb2+xml fb2 text/x-sony-bbeb+xml lrs application/x-sony-bbeb lrf lrx application/adobe-page-template+xml xpgt -application/x-font-opentype otf -application/x-font-truetype ttf application/x-mobipocket-ebook mobi prc azw application/x-topaz-ebook tpz azw1 application/x-mobipocket-subscription pobi @@ -1381,3 +1377,7 @@ application/x-cb7 cb7 application/x-koboreader-ebook kobo image/wmf wmf application/ereader pdb +# See http://idpf.org/epub/30/spec/epub30-publications.html#sec-core-media-types +application/vnd.ms-opentype otf +application/font-woff woff +application/x-font-truetype ttf diff --git a/src/calibre/ebooks/oeb/base.py b/src/calibre/ebooks/oeb/base.py index 2af72a07fb..50df05ed16 100644 --- a/src/calibre/ebooks/oeb/base.py +++ b/src/calibre/ebooks/oeb/base.py @@ -258,7 +258,7 @@ OPF_MIME = types_map['.opf'] PAGE_MAP_MIME = 'application/oebps-page-map+xml' OEB_DOC_MIME = 'text/x-oeb1-document' OEB_CSS_MIME = 'text/x-oeb1-css' -OPENTYPE_MIME = 'application/x-font-opentype' +OPENTYPE_MIME = types_map['.otf'] GIF_MIME = types_map['.gif'] JPEG_MIME = types_map['.jpeg'] PNG_MIME = types_map['.png'] From 7e96216b58304927fe85537f894f52d07d9d921b Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 21 Oct 2012 13:31:07 +0530 Subject: [PATCH 14/23] Conversion: Add an option to embed a font family into the book. The embedded font is used as the base font for all text that does not specify its own font family in the input document. Works only with output formats that support font embedding, principally EPUB/AZW3. Option is found under Look & Feel in the conversion dialog --- src/calibre/ebooks/conversion/cli.py | 2 +- src/calibre/ebooks/conversion/plumber.py | 11 + src/calibre/ebooks/oeb/transforms/flatcss.py | 57 +++++- src/calibre/gui2/convert/look_and_feel.py | 1 + src/calibre/gui2/convert/look_and_feel.ui | 204 ++++++++++--------- 5 files changed, 180 insertions(+), 95 deletions(-) diff --git a/src/calibre/ebooks/conversion/cli.py b/src/calibre/ebooks/conversion/cli.py index fb1974f93b..2aa0add3ee 100644 --- a/src/calibre/ebooks/conversion/cli.py +++ b/src/calibre/ebooks/conversion/cli.py @@ -132,7 +132,7 @@ def add_pipeline_options(parser, plumber): _('Options to control the look and feel of the output'), [ 'base_font_size', 'disable_font_rescaling', - 'font_size_mapping', + 'font_size_mapping', 'embed_font_family', 'line_height', 'minimum_line_height', 'linearize_tables', 'extra_css', 'filter_css', diff --git a/src/calibre/ebooks/conversion/plumber.py b/src/calibre/ebooks/conversion/plumber.py index 60cce24121..bfd2e36359 100644 --- a/src/calibre/ebooks/conversion/plumber.py +++ b/src/calibre/ebooks/conversion/plumber.py @@ -193,6 +193,17 @@ OptionRecommendation(name='line_height', ) ), +OptionRecommendation(name='embed_font_family', + recommended_value=None, level=OptionRecommendation.LOW, + help=_( + 'Embed the specified font family into the book. This specifies ' + 'the "base" font used for the book. If the input document ' + 'specifies its own fonts, they may override this base font. ' + 'You can use the filter style information option to remove fonts from the ' + 'input document. Note that font embedding only works ' + 'with some output formats, principally EPUB and AZW3.') + ), + OptionRecommendation(name='linearize_tables', recommended_value=False, level=OptionRecommendation.LOW, help=_('Some badly designed documents use tables to control the ' diff --git a/src/calibre/ebooks/oeb/transforms/flatcss.py b/src/calibre/ebooks/oeb/transforms/flatcss.py index 10b7e259ac..2f2fd727c0 100644 --- a/src/calibre/ebooks/oeb/transforms/flatcss.py +++ b/src/calibre/ebooks/oeb/transforms/flatcss.py @@ -14,9 +14,11 @@ from lxml import etree import cssutils from cssutils.css import Property +from calibre import guess_type from calibre.ebooks.oeb.base import (XHTML, XHTML_NS, CSS_MIME, OEB_STYLES, namespace, barename, XPath) from calibre.ebooks.oeb.stylizer import Stylizer +from calibre.utils.filenames import ascii_filename COLLAPSE = re.compile(r'[ \t\r\n\v]+') STRIPNUM = re.compile(r'[-0-9]+$') @@ -144,11 +146,61 @@ class CSSFlattener(object): cssutils.replaceUrls(item.data, item.abshref, ignoreImportRules=True) + self.body_font_family, self.embed_font_rules = self.get_embed_font_info( + self.opts.embed_font_family) self.stylize_spine() self.sbase = self.baseline_spine() if self.fbase else None self.fmap = FontMapper(self.sbase, self.fbase, self.fkey) self.flatten_spine() + def get_embed_font_info(self, family, failure_critical=True): + efi = [] + body_font_family = None + if not family: + return body_font_family, efi + from calibre.utils.fonts import fontconfig + from calibre.utils.fonts.utils import (get_font_characteristics, + panose_to_css_generic_family, get_font_names) + faces = fontconfig.fonts_for_family(family) + if not faces or not u'normal' in faces: + msg = (u'No embeddable fonts found for family: %r'%self.opts.embed_font_family) + if failure_critical: + raise ValueError(msg) + self.oeb.log.warn(msg) + return body_font_family, efi + + for k, v in faces.iteritems(): + ext, data = v[0::2] + weight, is_italic, is_bold, is_regular, fs_type, panose = \ + get_font_characteristics(data) + generic_family = panose_to_css_generic_family(panose) + family_name, subfamily_name, full_name = get_font_names(data) + if k == u'normal': + body_font_family = u"'%s',%s"%(family_name, generic_family) + if family_name.lower() != family.lower(): + self.oeb.log.warn(u'Failed to find an exact match for font:' + u' %r, using %r instead'%(family, family_name)) + else: + self.oeb.log(u'Embedding font: %s'%family_name) + font = {u'font-family':u'"%s"'%family_name} + if is_italic: + font[u'font-style'] = u'italic' + if is_bold: + font[u'font-weight'] = u'bold' + fid, href = self.oeb.manifest.generate(id=u'font', + href=u'%s.%s'%(ascii_filename(full_name).replace(u' ', u'-'), ext)) + item = self.oeb.manifest.add(fid, href, + guess_type(full_name+'.'+ext)[0], + data=data) + item.unload_data_from_memory() + font[u'src'] = u'url(%s)'%item.href + rule = '@font-face { %s }'%('; '.join(u'%s:%s'%(k, v) for k, v in + font.iteritems())) + rule = cssutils.parseString(rule) + efi.append(rule) + + return body_font_family, efi + def stylize_spine(self): self.stylizers = {} profile = self.context.source @@ -170,6 +222,8 @@ class CSSFlattener(object): bs.extend(['page-break-before: always']) if self.context.change_justification != 'original': bs.append('text-align: '+ self.context.change_justification) + if self.body_font_family: + bs.append(u'font-family: '+self.body_font_family) body.set('style', '; '.join(bs)) stylizer = Stylizer(html, item.href, self.oeb, self.context, profile, user_css=self.context.extra_css, @@ -450,7 +504,8 @@ class CSSFlattener(object): items.sort() css = ';\n'.join("%s: %s" % (key, val) for key, val in items) css = ('@page {\n%s\n}\n'%css) if items else '' - rules = [r.cssText for r in stylizer.font_face_rules] + rules = [r.cssText for r in stylizer.font_face_rules + + self.embed_font_rules] raw = '\n\n'.join(rules) css += '\n\n' + raw global_css[css].append(item) diff --git a/src/calibre/gui2/convert/look_and_feel.py b/src/calibre/gui2/convert/look_and_feel.py index ad604ec4e3..1609a0add3 100644 --- a/src/calibre/gui2/convert/look_and_feel.py +++ b/src/calibre/gui2/convert/look_and_feel.py @@ -32,6 +32,7 @@ class LookAndFeelWidget(Widget, Ui_Form): Widget.__init__(self, parent, ['change_justification', 'extra_css', 'base_font_size', 'font_size_mapping', 'line_height', 'minimum_line_height', + 'embed_font_family', 'smarten_punctuation', 'unsmarten_punctuation', 'disable_font_rescaling', 'insert_blank_line', 'remove_paragraph_spacing', diff --git a/src/calibre/gui2/convert/look_and_feel.ui b/src/calibre/gui2/convert/look_and_feel.ui index d6ecb1b61a..039625bc25 100644 --- a/src/calibre/gui2/convert/look_and_feel.ui +++ b/src/calibre/gui2/convert/look_and_feel.ui @@ -14,6 +14,26 @@ Form + + + + pt + + + 1 + + + + + + + Line &height: + + + opt_line_height + + + @@ -37,26 +57,6 @@ - - - - Line &height: - - - opt_line_height - - - - - - - pt - - - 1 - - - @@ -140,14 +140,14 @@ - + Remove &spacing between paragraphs - + &Indent size: @@ -160,7 +160,7 @@ - + <p>When calibre removes inter paragraph spacing, it automatically sets a paragraph indent, to ensure that paragraphs can be easily distinguished. This option controls the width of that indent. @@ -182,72 +182,7 @@ - - - - Insert &blank line between paragraphs - - - - - - - em - - - 1 - - - - - - - Text &justification: - - - opt_change_justification - - - - - - - - - - Smarten &punctuation - - - - - - - &Transliterate unicode characters to ASCII - - - - - - - &UnSmarten punctuation - - - - - - - Keep &ligatures - - - - - - - &Linearize tables - - - - + 0 @@ -365,10 +300,68 @@ - - + + - &Disable font size rescaling + Insert &blank line between paragraphs + + + + + + + em + + + 1 + + + + + + + Text &justification: + + + opt_change_justification + + + + + + + + + + Smarten &punctuation + + + + + + + &Transliterate unicode characters to ASCII + + + + + + + &UnSmarten punctuation + + + + + + + Keep &ligatures + + + + + + + &Linearize tables @@ -382,7 +375,7 @@ - + &Line size: @@ -395,6 +388,26 @@ + + + + &Embed font family: + + + opt_embed_font_family + + + + + + + &Disable font size rescaling + + + + + + @@ -403,6 +416,11 @@ QComboBox
widgets.h
+ + FontFamilyChooser + QComboBox +
calibre/gui2/font_family_chooser.h
+
From 93b2f860f9fa5c73773013b29d152b1723108140 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 21 Oct 2012 15:09:07 +0530 Subject: [PATCH 15/23] Add code to convert between WOFF<->sfnt font files --- setup/extensions.py | 14 +- src/calibre/constants.py | 1 + src/calibre/test_build.py | 5 + src/calibre/utils/fonts/woff/__init__.py | 36 + src/calibre/utils/fonts/woff/main.c | 108 ++ src/calibre/utils/fonts/woff/woff-private.h | 151 +++ src/calibre/utils/fonts/woff/woff.c | 1170 +++++++++++++++++++ src/calibre/utils/fonts/woff/woff.h | 211 ++++ 8 files changed, 1694 insertions(+), 2 deletions(-) create mode 100644 src/calibre/utils/fonts/woff/__init__.py create mode 100644 src/calibre/utils/fonts/woff/main.c create mode 100644 src/calibre/utils/fonts/woff/woff-private.h create mode 100644 src/calibre/utils/fonts/woff/woff.c create mode 100644 src/calibre/utils/fonts/woff/woff.h diff --git a/setup/extensions.py b/setup/extensions.py index 2cae3d857f..8d49f904b7 100644 --- a/setup/extensions.py +++ b/setup/extensions.py @@ -50,9 +50,9 @@ class Extension(object): reflow_sources = glob.glob(os.path.join(SRC, 'calibre', 'ebooks', 'pdf', '*.cpp')) reflow_headers = glob.glob(os.path.join(SRC, 'calibre', 'ebooks', 'pdf', '*.h')) -pdfreflow_libs = [] +woff_libs = ['z'] if iswindows: - pdfreflow_libs = ['advapi32', 'User32', 'Gdi32', 'zlib'] + woff_libs = ['zlib'] icu_libs = ['icudata', 'icui18n', 'icuuc', 'icuio'] icu_cflags = [] @@ -132,6 +132,16 @@ extensions = [ lib_dirs=[fc_lib], error=fc_error), + Extension('woff', + ['calibre/utils/fonts/woff/main.c', + 'calibre/utils/fonts/woff/woff.c'], + headers=[ + 'calibre/utils/fonts/woff/woff.h', + 'calibre/utils/fonts/woff/woff-private.h'], + libraries=woff_libs + ), + + Extension('msdes', ['calibre/utils/msdes/msdesmodule.c', 'calibre/utils/msdes/des.c'], diff --git a/src/calibre/constants.py b/src/calibre/constants.py index fe3c40309e..6bc9995384 100644 --- a/src/calibre/constants.py +++ b/src/calibre/constants.py @@ -90,6 +90,7 @@ class Plugins(collections.Mapping): 'icu', 'speedup', 'freetype', + 'woff', ] if iswindows: plugins.extend(['winutil', 'wpd', 'winfonts']) diff --git a/src/calibre/test_build.py b/src/calibre/test_build.py index e6e8611ac5..8bf3427f8c 100644 --- a/src/calibre/test_build.py +++ b/src/calibre/test_build.py @@ -114,6 +114,10 @@ def test_wpd(): else: wpd.uninit() +def test_woff(): + from calibre.utils.fonts.woff import test + test() + def test(): test_plugins() test_lxml() @@ -124,6 +128,7 @@ def test(): test_imaging() test_unrar() test_icu() + test_woff() if iswindows: test_win32() test_winutil() diff --git a/src/calibre/utils/fonts/woff/__init__.py b/src/calibre/utils/fonts/woff/__init__.py new file mode 100644 index 0000000000..8684c03489 --- /dev/null +++ b/src/calibre/utils/fonts/woff/__init__.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:fdm=marker:ai +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +__license__ = 'GPL v3' +__copyright__ = '2012, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + +from calibre.constants import plugins + +def get_woff(): + woff, woff_err = plugins['woff'] + if woff_err: + raise RuntimeError('Failed to load the WOFF plugin: %s'%woff_err) + return woff + +def to_woff(raw): + woff = get_woff() + return woff.to_woff(raw) + +def from_woff(raw): + woff = get_woff() + return woff.from_woff(raw) + +def test(): + sfnt = P('fonts/calibreSymbols.otf', data=True) + woff = to_woff(sfnt) + recon = from_woff(woff) + if recon != sfnt: + raise ValueError('WOFF roundtrip resulted in different sfnt') + +if __name__ == '__main__': + test() + + diff --git a/src/calibre/utils/fonts/woff/main.c b/src/calibre/utils/fonts/woff/main.c new file mode 100644 index 0000000000..827c1365d5 --- /dev/null +++ b/src/calibre/utils/fonts/woff/main.c @@ -0,0 +1,108 @@ +/* + * main.c + * Copyright (C) 2012 Kovid Goyal + * + * Distributed under terms of the GPL3 license. + */ + + +#define _UNICODE +#define UNICODE +#define PY_SSIZE_T_CLEAN +#include + +#include "woff.h" + +static PyObject *WOFFError = NULL; + +static PyObject* woff_err(uint32_t status) { + const char *msg; + switch(status) { + case eWOFF_out_of_memory: + return PyErr_NoMemory(); + case eWOFF_invalid: + msg = "Invalid input data"; break; + case eWOFF_compression_failure: + msg = "Compression failed"; break; + case eWOFF_bad_signature: + msg = "Bad font signature"; break; + case eWOFF_buffer_too_small: + msg = "Buffer too small"; break; + case eWOFF_bad_parameter: + msg = "Bad parameter"; break; + case eWOFF_illegal_order: + msg = "Illegal order of WOFF chunks"; break; + default: + msg = "Unknown Error"; + } + PyErr_SetString(WOFFError, msg); + return NULL; +} + +static PyObject* +to_woff(PyObject *self, PyObject *args) { + const char *sfnt; + char *woff = NULL; + Py_ssize_t sz; + uint32_t wofflen = 0, status = eWOFF_ok; + PyObject *ans; + + if (!PyArg_ParseTuple(args, "s#", &sfnt, &sz)) return NULL; + + woff = (char*)woffEncode((uint8_t*)sfnt, sz, 0, 0, &wofflen, &status); + + if (WOFF_FAILURE(status) || woff == NULL) return woff_err(status); + + ans = Py_BuildValue("s#", woff, wofflen); + free(woff); + return ans; +} + +static PyObject* +from_woff(PyObject *self, PyObject *args) { + const char *woff; + char *sfnt; + Py_ssize_t sz; + uint32_t sfntlen = 0, status = eWOFF_ok; + PyObject *ans; + + if (!PyArg_ParseTuple(args, "s#", &woff, &sz)) return NULL; + + sfnt = (char*)woffDecode((uint8_t*)woff, sz, &sfntlen, &status); + + if (WOFF_FAILURE(status) || sfnt == NULL) return woff_err(status); + ans = Py_BuildValue("s#", sfnt, sfntlen); + free(sfnt); + return ans; +} + +static +PyMethodDef methods[] = { + {"to_woff", (PyCFunction)to_woff, METH_VARARGS, + "to_woff(bytestring) -> Convert the sfnt data in bytestring to WOFF format (returned as a bytestring)." + }, + + {"from_woff", (PyCFunction)from_woff, METH_VARARGS, + "from_woff(bytestring) -> Convert the woff data in bytestring to SFNT format (returned as a bytestring)." + }, + + + {NULL, NULL, 0, NULL} +}; + +PyMODINIT_FUNC +initwoff(void) { + PyObject *m; + + m = Py_InitModule3( + "woff", methods, + "Convert to/from the WOFF<->sfnt font formats" + ); + if (m == NULL) return; + + WOFFError = PyErr_NewException((char*)"woff.WOFFError", NULL, NULL); + if (WOFFError == NULL) return; + PyModule_AddObject(m, "WOFFError", WOFFError); +} + + diff --git a/src/calibre/utils/fonts/woff/woff-private.h b/src/calibre/utils/fonts/woff/woff-private.h new file mode 100644 index 0000000000..8a41aa0c92 --- /dev/null +++ b/src/calibre/utils/fonts/woff/woff-private.h @@ -0,0 +1,151 @@ +/* -*- Mode: C; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * ***** BEGIN LICENSE BLOCK ***** + * Version: MPL 1.1/GPL 2.0/LGPL 2.1 + * + * The contents of this file are subject to the Mozilla Public License Version + * 1.1 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * http://www.mozilla.org/MPL/ + * + * Software distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License + * for the specific language governing rights and limitations under the + * License. + * + * The Original Code is WOFF font packaging code. + * + * The Initial Developer of the Original Code is Mozilla Corporation. + * Portions created by the Initial Developer are Copyright (C) 2009 + * the Initial Developer. All Rights Reserved. + * + * Contributor(s): + * Jonathan Kew + * + * Alternatively, the contents of this file may be used under the terms of + * either the GNU General Public License Version 2 or later (the "GPL"), or + * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), + * in which case the provisions of the GPL or the LGPL are applicable instead + * of those above. If you wish to allow use of your version of this file only + * under the terms of either the GPL or the LGPL, and not to allow others to + * use your version of this file under the terms of the MPL, indicate your + * decision by deleting the provisions above and replace them with the notice + * and other provisions required by the GPL or the LGPL. If you do not delete + * the provisions above, a recipient may use your version of this file under + * the terms of any one of the MPL, the GPL or the LGPL. + * + * ***** END LICENSE BLOCK ***** */ + +#ifndef WOFF_PRIVATE_H_ +#define WOFF_PRIVATE_H_ + +#include "woff.h" + +/* private definitions used in the WOFF encoder/decoder functions */ + +/* create an OT tag from 4 characters */ +#define TAG(a,b,c,d) ((a)<<24 | (b)<<16 | (c)<<8 | (d)) + +#define WOFF_SIGNATURE TAG('w','O','F','F') + +#define SFNT_VERSION_CFF TAG('O','T','T','O') +#define SFNT_VERSION_TT 0x00010000 +#define SFNT_VERSION_true TAG('t','r','u','e') + +#define TABLE_TAG_DSIG TAG('D','S','I','G') +#define TABLE_TAG_head TAG('h','e','a','d') +#define TABLE_TAG_bhed TAG('b','h','e','d') + +#define SFNT_CHECKSUM_CALC_CONST 0xB1B0AFBAU /* from the TT/OT spec */ + +#ifdef WOFF_MOZILLA_CLIENT +# include +# define READ32BE(x) PR_ntohl(x) +# define READ16BE(x) PR_ntohs(x) +#else +/* These macros to read values as big-endian only work on "real" variables, + not general expressions, because of the use of &(x), but they are + designed to work on both BE and LE machines without the need for a + configure check. For production code, we might want to replace this + with something more efficient. */ +/* read a 32-bit BigEndian value */ +# define READ32BE(x) ( ( (uint32_t) ((uint8_t*)&(x))[0] << 24 ) + \ + ( (uint32_t) ((uint8_t*)&(x))[1] << 16 ) + \ + ( (uint32_t) ((uint8_t*)&(x))[2] << 8 ) + \ + (uint32_t) ((uint8_t*)&(x))[3] ) +/* read a 16-bit BigEndian value */ +# define READ16BE(x) ( ( (uint16_t) ((uint8_t*)&(x))[0] << 8 ) + \ + (uint16_t) ((uint8_t*)&(x))[1] ) +#endif + +#pragma pack(push,1) + +typedef struct { + uint32_t version; + uint16_t numTables; + uint16_t searchRange; + uint16_t entrySelector; + uint16_t rangeShift; +} sfntHeader; + +typedef struct { + uint32_t tag; + uint32_t checksum; + uint32_t offset; + uint32_t length; +} sfntDirEntry; + +typedef struct { + uint32_t signature; + uint32_t flavor; + uint32_t length; + uint16_t numTables; + uint16_t reserved; + uint32_t totalSfntSize; + uint16_t majorVersion; + uint16_t minorVersion; + uint32_t metaOffset; + uint32_t metaCompLen; + uint32_t metaOrigLen; + uint32_t privOffset; + uint32_t privLen; +} woffHeader; + +typedef struct { + uint32_t tag; + uint32_t offset; + uint32_t compLen; + uint32_t origLen; + uint32_t checksum; +} woffDirEntry; + +typedef struct { + uint32_t version; + uint32_t fontRevision; + uint32_t checkSumAdjustment; + uint32_t magicNumber; + uint16_t flags; + uint16_t unitsPerEm; + uint32_t created[2]; + uint32_t modified[2]; + int16_t xMin; + int16_t yMin; + int16_t xMax; + int16_t yMax; + uint16_t macStyle; + uint16_t lowestRecPpem; + int16_t fontDirectionHint; + int16_t indexToLocFormat; + int16_t glyphDataFormat; +} sfntHeadTable; + +#define HEAD_TABLE_SIZE 54 /* sizeof(sfntHeadTable) may report 56 because of alignment */ + +typedef struct { + uint32_t offset; + uint16_t oldIndex; + uint16_t newIndex; +} tableOrderRec; + +#pragma pack(pop) + +#endif diff --git a/src/calibre/utils/fonts/woff/woff.c b/src/calibre/utils/fonts/woff/woff.c new file mode 100644 index 0000000000..4dcaadb00f --- /dev/null +++ b/src/calibre/utils/fonts/woff/woff.c @@ -0,0 +1,1170 @@ +/* -*- Mode: C; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * ***** BEGIN LICENSE BLOCK ***** + * Version: MPL 1.1/GPL 2.0/LGPL 2.1 + * + * The contents of this file are subject to the Mozilla Public License Version + * 1.1 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * http://www.mozilla.org/MPL/ + * + * Software distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License + * for the specific language governing rights and limitations under the + * License. + * + * The Original Code is WOFF font packaging code. + * + * The Initial Developer of the Original Code is Mozilla Corporation. + * Portions created by the Initial Developer are Copyright (C) 2009 + * the Initial Developer. All Rights Reserved. + * + * Contributor(s): + * Jonathan Kew + * + * Alternatively, the contents of this file may be used under the terms of + * either the GNU General Public License Version 2 or later (the "GPL"), or + * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), + * in which case the provisions of the GPL or the LGPL are applicable instead + * of those above. If you wish to allow use of your version of this file only + * under the terms of either the GPL or the LGPL, and not to allow others to + * use your version of this file under the terms of the MPL, indicate your + * decision by deleting the provisions above and replace them with the notice + * and other provisions required by the GPL or the LGPL. If you do not delete + * the provisions above, a recipient may use your version of this file under + * the terms of any one of the MPL, the GPL or the LGPL. + * + * ***** END LICENSE BLOCK ***** */ + +#include "woff-private.h" + +#include +#include +#include +#include + +#ifdef WOFF_MOZILLA_CLIENT /* define this when building as part of Gecko */ +# include "prmem.h" +# define malloc PR_Malloc +# define realloc PR_Realloc +# define free PR_Free +#endif + +/* + * Just simple whole-file encoding and decoding functions; a more extensive + * WOFF library could provide support for accessing individual tables from a + * compressed font, alternative options for memory allocation/ownership and + * error handling, etc. + */ + +/* on errors, each function sets a status variable and jumps to failure: */ +#undef FAIL +#define FAIL(err) do { status |= err; goto failure; } while (0) + +/* adjust an offset for longword alignment */ +#define LONGALIGN(x) (((x) + 3) & ~3) + +static int +compareOffsets(const void * lhs, const void * rhs) +{ + const tableOrderRec * a = (const tableOrderRec *) lhs; + const tableOrderRec * b = (const tableOrderRec *) rhs; + /* don't simply return a->offset - b->offset because these are unsigned + offset values; could convert to int, but possible integer overflow */ + return a->offset > b->offset ? 1 : + a->offset < b->offset ? -1 : + 0; +} + +#ifndef WOFF_MOZILLA_CLIENT + +/******************************************************************/ +/* * * * * * * * * * * * * * ENCODING * * * * * * * * * * * * * * */ +/******************************************************************/ + +static uint32_t +calcChecksum(const sfntDirEntry * dirEntry, + const uint8_t * sfntData, uint32_t sfntLen) +{ + /* just returns zero on errors, they will be detected again elsewhere */ + const uint32_t * csumPtr; + const uint32_t * csumEnd; + uint32_t csum = 0; + uint32_t length = LONGALIGN(READ32BE(dirEntry->length)); + uint32_t offset = READ32BE(dirEntry->offset); + uint32_t tag; + if ((offset & 3) != 0) { + return csum; + } + if (length > sfntLen || offset > sfntLen - length) { + return csum; + } + csumPtr = (const uint32_t *) (sfntData + offset); + csumEnd = csumPtr + length / 4; + while (csumPtr < csumEnd) { + csum += READ32BE(*csumPtr); + csumPtr++; + } + tag = READ32BE(dirEntry->tag); + if (tag == TABLE_TAG_head || tag == TABLE_TAG_bhed) { + const sfntHeadTable * head; + if (length < HEAD_TABLE_SIZE) { + return 0; + } + head = (const sfntHeadTable *)(sfntData + offset); + csum -= READ32BE(head->checkSumAdjustment); + } + return csum; +} + +const uint8_t * +woffEncode(const uint8_t * sfntData, uint32_t sfntLen, + uint16_t majorVersion, uint16_t minorVersion, + uint32_t * woffLen, uint32_t * pStatus) +{ + uint8_t * woffData = NULL; + tableOrderRec * tableOrder = NULL; + + uint32_t tableOffset; + uint32_t totalSfntSize; + + uint16_t numOrigTables; + uint16_t numTables; + uint16_t tableIndex; + uint16_t order; + const sfntDirEntry * sfntDir; + uint32_t tableBase; + uint32_t checkSumAdjustment = 0; + woffHeader * newHeader; + uint32_t tag = 0; + uint32_t removedDsigSize = 0; + uint32_t status = eWOFF_ok; + + const sfntHeader * header = (const sfntHeader *) (sfntData); + const sfntHeadTable * head = NULL; + + if (pStatus && WOFF_FAILURE(*pStatus)) { + return NULL; + } + + if (READ32BE(header->version) != SFNT_VERSION_TT && + READ32BE(header->version) != SFNT_VERSION_CFF && + READ32BE(header->version) != SFNT_VERSION_true) { + status |= eWOFF_warn_unknown_version; + } + + numOrigTables = READ16BE(header->numTables); + sfntDir = (const sfntDirEntry *) (sfntData + sizeof(sfntHeader)); + + for (tableIndex = 0; tableIndex < numOrigTables; ++tableIndex) { + /* validate table checksums, to figure out if we need to drop DSIG; + also check that table directory is correctly sorted */ + uint32_t prevTag = tag; + uint32_t csum = calcChecksum(&sfntDir[tableIndex], sfntData, sfntLen); + if (csum != READ32BE(sfntDir[tableIndex].checksum)) { + status |= eWOFF_warn_checksum_mismatch; + } + checkSumAdjustment += csum; + tag = READ32BE(sfntDir[tableIndex].tag); + if (tag <= prevTag) { + FAIL(eWOFF_invalid); + } + if (tag == TABLE_TAG_head || tag == TABLE_TAG_bhed) { + if (READ32BE(sfntDir[tableIndex].length) < HEAD_TABLE_SIZE) { + FAIL(eWOFF_invalid); + } + head = (const sfntHeadTable *)(sfntData + + READ32BE(sfntDir[tableIndex].offset)); + } + } + if (!head) { + FAIL(eWOFF_invalid); + } + if ((status & eWOFF_warn_checksum_mismatch) == 0) { + /* no point even checking if we already have an error, + as fixing that will change the overall checksum too */ + const uint32_t * csumPtr = (const uint32_t *) sfntData; + const uint32_t * csumEnd = csumPtr + 3 + 4 * numOrigTables; + while (csumPtr < csumEnd) { + checkSumAdjustment += READ32BE(*csumPtr); + ++csumPtr; + } + checkSumAdjustment = 0xB1B0AFBA - checkSumAdjustment; + if (checkSumAdjustment != READ32BE(head->checkSumAdjustment)) { + status |= eWOFF_warn_checksum_mismatch; + } + } + + /* Fixing checkSumAdjustment is tricky, because if there's a DSIG table, + we're going to have to remove that, which in turn means that table + offsets in the directory will all change. + And recalculating checkSumAdjustment requires taking account of any + individual table checksum corrections, but they have not actually been + applied to the sfnt data at this point. + And finally, we'd need to get the corrected checkSumAdjustment into the + encoded head table (but we can't modify the original sfnt data). + An easier way out seems to be to go ahead and encode the font, knowing + that checkSumAdjustment will be wrong; then (if the status flag + eWOFF_warn_checksum_mismatch is set) we'll decode the font back to + sfnt format. This will fix up the checkSumAdjustment (and return a + warning status). We'll ignore that warning, and then re-encode the + new, cleaned-up sfnt to get the final WOFF data. Perhaps not the most + efficient approach, but it seems simpler than trying to predict the + correct final checkSumAdjustment and incorporate it into the head + table on the fly. */ + + tableOrder = (tableOrderRec *) malloc(numOrigTables * sizeof(tableOrderRec)); + if (!tableOrder) { + FAIL(eWOFF_out_of_memory); + } + for (tableIndex = 0, numTables = 0; + tableIndex < numOrigTables; ++tableIndex) { + if ((status & eWOFF_warn_checksum_mismatch) != 0) { + /* check for DSIG table that we must drop if we're fixing checksums */ + tag = READ32BE(sfntDir[tableIndex].tag); + if (tag == TABLE_TAG_DSIG) { + status |= eWOFF_warn_removed_DSIG; + removedDsigSize = READ32BE(sfntDir[tableIndex].length); + continue; + } + } + tableOrder[numTables].offset = READ32BE(sfntDir[tableIndex].offset); + tableOrder[numTables].oldIndex = tableIndex; + tableOrder[numTables].newIndex = numTables; + ++numTables; + } + qsort(tableOrder, numTables, sizeof(tableOrderRec), compareOffsets); + + /* initially, allocate space for header and directory */ + tableOffset = sizeof(woffHeader) + numTables * sizeof(woffDirEntry); + woffData = (uint8_t *) malloc(tableOffset); + if (!woffData) { + FAIL(eWOFF_out_of_memory); + } + + /* accumulator for total expected size of decoded font */ + totalSfntSize = sizeof(sfntHeader) + numTables * sizeof(sfntDirEntry); + +/* + * We use a macro for this rather than creating a variable because woffData + * will get reallocated during encoding. The macro avoids the risk of using a + * stale pointer, and the compiler should optimize multiple successive uses. + */ +#define WOFFDIR ((woffDirEntry *) (woffData + sizeof(woffHeader))) + + for (order = 0; order < numTables; ++order) { + uLong sourceLen, destLen; + uint32_t sourceOffset; + + uint16_t oldIndex = tableOrder[order].oldIndex; + uint16_t newIndex = tableOrder[order].newIndex; + + WOFFDIR[newIndex].tag = sfntDir[oldIndex].tag; + if ((status & eWOFF_warn_checksum_mismatch) != 0) { + uint32_t csum = calcChecksum(&sfntDir[oldIndex], sfntData, sfntLen); + WOFFDIR[newIndex].checksum = READ32BE(csum); + } else { + WOFFDIR[newIndex].checksum = sfntDir[oldIndex].checksum; + } + WOFFDIR[newIndex].origLen = sfntDir[oldIndex].length; + WOFFDIR[newIndex].offset = READ32BE(tableOffset); + + /* allocate enough space for upper bound of compressed size */ + sourceOffset = READ32BE(sfntDir[oldIndex].offset); + if ((sourceOffset & 3) != 0) { + status |= eWOFF_warn_misaligned_table; + } + sourceLen = READ32BE(sfntDir[oldIndex].length); + if (sourceLen > sfntLen || sourceOffset > sfntLen - sourceLen) { + FAIL(eWOFF_invalid); + } + destLen = LONGALIGN(compressBound(sourceLen)); + woffData = (uint8_t *) realloc(woffData, tableOffset + destLen); + if (!woffData) { + FAIL(eWOFF_out_of_memory); + } + + /* do the compression directly into the WOFF data block */ + if (compress2((Bytef *) (woffData + tableOffset), &destLen, + (const Bytef *) (sfntData + sourceOffset), + sourceLen, 9) != Z_OK) { + FAIL(eWOFF_compression_failure); + } + if (destLen < sourceLen) { + /* compressed table was smaller */ + tableOffset += destLen; + WOFFDIR[newIndex].compLen = READ32BE(destLen); + } else { + /* compression didn't make it smaller, so store original data instead */ + destLen = sourceLen; + /* reallocate to ensure enough space for the table, + plus potential padding after it */ + woffData = (uint8_t *) realloc(woffData, + tableOffset + LONGALIGN(sourceLen)); + if (!woffData) { + FAIL(eWOFF_out_of_memory); + } + /* copy the original data into place */ + memcpy(woffData + tableOffset, + sfntData + READ32BE(sfntDir[oldIndex].offset), sourceLen); + tableOffset += sourceLen; + WOFFDIR[newIndex].compLen = WOFFDIR[newIndex].origLen; + } + + /* we always realloc woffData to a long-aligned size, so this is safe */ + while ((tableOffset & 3) != 0) { + woffData[tableOffset++] = 0; + } + + /* update total size of uncompressed OpenType with table size */ + totalSfntSize += sourceLen; + totalSfntSize = LONGALIGN(totalSfntSize); + } + + if (totalSfntSize > sfntLen) { + if (totalSfntSize > LONGALIGN(sfntLen)) { + FAIL(eWOFF_invalid); + } else { + status |= eWOFF_warn_unpadded_table; + } + } else if (totalSfntSize < sfntLen) { + /* check if the remaining data is a DSIG we're removing; + if so, we're already warning about that */ + if ((status & eWOFF_warn_removed_DSIG) != 0 || + sfntLen - totalSfntSize > + LONGALIGN(removedDsigSize) + sizeof(sfntDirEntry)) { + status |= eWOFF_warn_trailing_data; + } + } + + /* write the header */ + newHeader = (woffHeader *) (woffData); + newHeader->signature = WOFF_SIGNATURE; + newHeader->signature = READ32BE(newHeader->signature); + newHeader->flavor = header->version; + newHeader->length = READ32BE(tableOffset); + newHeader->numTables = READ16BE(numTables); + newHeader->reserved = 0; + newHeader->totalSfntSize = READ32BE(totalSfntSize); + newHeader->majorVersion = READ16BE(majorVersion); + newHeader->minorVersion = READ16BE(minorVersion); + newHeader->metaOffset = 0; + newHeader->metaCompLen = 0; + newHeader->metaOrigLen = 0; + newHeader->privOffset = 0; + newHeader->privLen = 0; + + free(tableOrder); + + if ((status & eWOFF_warn_checksum_mismatch) != 0) { + /* The original font had checksum errors, so we now decode our WOFF data + back to sfnt format (which fixes checkSumAdjustment), then re-encode + to get a clean copy. */ + const uint8_t * cleanSfnt = woffDecode(woffData, tableOffset, + &sfntLen, &status); + if (WOFF_FAILURE(status)) { + FAIL(status); + } + free(woffData); + woffData = (uint8_t *) woffEncode(cleanSfnt, sfntLen, + majorVersion, minorVersion, + &tableOffset, &status); + free((void *) cleanSfnt); + if (WOFF_FAILURE(status)) { + FAIL(status); + } + } + + if (woffLen) { + *woffLen = tableOffset; + } + if (pStatus) { + *pStatus |= status; + } + return woffData; + +failure: + if (tableOrder) { + free(tableOrder); + } + if (woffData) { + free(woffData); + } + if (pStatus) { + *pStatus = status; + } + return NULL; +} + +static const uint8_t * +rebuildWoff(const uint8_t * woffData, uint32_t * woffLen, + const uint8_t * metaData, uint32_t metaCompLen, uint32_t metaOrigLen, + const uint8_t * privData, uint32_t privLen, uint32_t * pStatus) +{ + const woffHeader * origHeader; + const woffDirEntry * woffDir; + uint8_t * newData = NULL; + uint8_t * tableData = NULL; + woffHeader * newHeader; + uint16_t numTables; + uint32_t tableLimit, totalSize, offset; + uint16_t i; + uint32_t status = eWOFF_ok; + + if (*woffLen < sizeof(woffHeader)) { + FAIL(eWOFF_invalid); + } + origHeader = (const woffHeader *) (woffData); + + if (READ32BE(origHeader->signature) != WOFF_SIGNATURE) { + FAIL(eWOFF_bad_signature); + } + + numTables = READ16BE(origHeader->numTables); + woffDir = (const woffDirEntry *) (woffData + sizeof(woffHeader)); + tableLimit = 0; + for (i = 0; i < numTables; ++i) { + uint32_t end = READ32BE(woffDir[i].offset) + READ32BE(woffDir[i].compLen); + if (end > tableLimit) { + tableLimit = end; + } + } + tableLimit = LONGALIGN(tableLimit); + + /* check for broken input (meta/priv data before sfnt tables) */ + offset = READ32BE(origHeader->metaOffset); + if (offset != 0 && offset < tableLimit) { + FAIL(eWOFF_illegal_order); + } + offset = READ32BE(origHeader->privOffset); + if (offset != 0 && offset < tableLimit) { + FAIL(eWOFF_illegal_order); + } + + totalSize = tableLimit; /* already long-aligned */ + if (metaCompLen) { + totalSize += metaCompLen; + } + if (privLen) { + totalSize = LONGALIGN(totalSize) + privLen; + } + newData = malloc(totalSize); + if (!newData) { + FAIL(eWOFF_out_of_memory); + } + + /* copy the header, directory, and sfnt tables */ + memcpy(newData, woffData, tableLimit); + + /* then overwrite the header fields that should be changed */ + newHeader = (woffHeader *) newData; + newHeader->length = READ32BE(totalSize); + newHeader->metaOffset = 0; + newHeader->metaCompLen = 0; + newHeader->metaOrigLen = 0; + newHeader->privOffset = 0; + newHeader->privLen = 0; + + offset = tableLimit; + if (metaData && metaCompLen > 0 && metaOrigLen > 0) { + newHeader->metaOffset = READ32BE(offset); + newHeader->metaCompLen = READ32BE(metaCompLen); + newHeader->metaOrigLen = READ32BE(metaOrigLen); + memcpy(newData + offset, metaData, metaCompLen); + offset += metaCompLen; + } + + if (privData && privLen > 0) { + while ((offset & 3) != 0) { + newData[offset++] = 0; + } + newHeader->privOffset = READ32BE(offset); + newHeader->privLen = READ32BE(privLen); + memcpy(newData + offset, privData, privLen); + offset += privLen; + } + + *woffLen = offset; + free((void *) woffData); + + if (pStatus) { + *pStatus |= status; + } + return newData; + +failure: + if (newData) { + free(newData); + } + if (pStatus) { + *pStatus = status; + } + return NULL; +} + +const uint8_t * +woffSetMetadata(const uint8_t * woffData, uint32_t * woffLen, + const uint8_t * metaData, uint32_t metaLen, + uint32_t * pStatus) +{ + const woffHeader * header; + uLong compLen = 0; + uint8_t * compData = NULL; + const uint8_t * privData = NULL; + uint32_t privLen = 0; + uint32_t status = eWOFF_ok; + + if (pStatus && WOFF_FAILURE(*pStatus)) { + return NULL; + } + + if (!woffData || !woffLen) { + FAIL(eWOFF_bad_parameter); + } + + if (*woffLen < sizeof(woffHeader)) { + FAIL(eWOFF_invalid); + } + header = (const woffHeader *) (woffData); + + if (READ32BE(header->signature) != WOFF_SIGNATURE) { + FAIL(eWOFF_bad_signature); + } + + if (header->privOffset != 0 && header->privLen != 0) { + privData = woffData + READ32BE(header->privOffset); + privLen = READ32BE(header->privLen); + if (privData + privLen > woffData + *woffLen) { + FAIL(eWOFF_invalid); + } + } + + if (metaData && metaLen > 0) { + compLen = compressBound(metaLen); + compData = malloc(compLen); + if (!compData) { + FAIL(eWOFF_out_of_memory); + } + + if (compress2((Bytef *) compData, &compLen, + (const Bytef *) metaData, metaLen, 9) != Z_OK) { + FAIL(eWOFF_compression_failure); + } + } + + woffData = rebuildWoff(woffData, woffLen, + compData, compLen, metaLen, + privData, privLen, pStatus); + free(compData); + return woffData; + +failure: + if (compData) { + free(compData); + } + if (pStatus) { + *pStatus = status; + } + return NULL; +} + +const uint8_t * +woffSetPrivateData(const uint8_t * woffData, uint32_t * woffLen, + const uint8_t * privData, uint32_t privLen, + uint32_t * pStatus) +{ + const woffHeader * header; + const uint8_t * metaData = NULL; + uint32_t metaLen = 0; + uint32_t status = eWOFF_ok; + + if (pStatus && WOFF_FAILURE(*pStatus)) { + return NULL; + } + + if (!woffData || !woffLen) { + FAIL(eWOFF_bad_parameter); + } + + if (*woffLen < sizeof(woffHeader)) { + FAIL(eWOFF_invalid); + } + header = (const woffHeader *) (woffData); + + if (READ32BE(header->signature) != WOFF_SIGNATURE) { + FAIL(eWOFF_bad_signature); + } + + if (header->metaOffset != 0 && header->metaCompLen != 0) { + metaData = woffData + READ32BE(header->metaOffset); + metaLen = READ32BE(header->metaCompLen); + if (metaData + metaLen > woffData + *woffLen) { + FAIL(eWOFF_invalid); + } + } + + woffData = rebuildWoff(woffData, woffLen, + metaData, metaLen, READ32BE(header->metaOrigLen), + privData, privLen, pStatus); + return woffData; + +failure: + if (pStatus) { + *pStatus = status; + } + return NULL; +} + +#endif /* WOFF_MOZILLA_CLIENT */ + +/******************************************************************/ +/* * * * * * * * * * * * * * DECODING * * * * * * * * * * * * * * */ +/******************************************************************/ + +static uint32_t +sanityCheck(const uint8_t * woffData, uint32_t woffLen) +{ + const woffHeader * header; + uint16_t numTables, i; + const woffDirEntry * dirEntry; + uint32_t tableTotal = 0; + + if (!woffData || !woffLen) { + return eWOFF_bad_parameter; + } + + if (woffLen < sizeof(woffHeader)) { + return eWOFF_invalid; + } + + header = (const woffHeader *) (woffData); + if (READ32BE(header->signature) != WOFF_SIGNATURE) { + return eWOFF_bad_signature; + } + + if (READ32BE(header->length) != woffLen || header->reserved != 0) { + return eWOFF_invalid; + } + + numTables = READ16BE(header->numTables); + if (woffLen < sizeof(woffHeader) + numTables * sizeof(woffDirEntry)) { + return eWOFF_invalid; + } + + dirEntry = (const woffDirEntry *) (woffData + sizeof(woffHeader)); + for (i = 0; i < numTables; ++i) { + uint32_t offs = READ32BE(dirEntry->offset); + uint32_t orig = READ32BE(dirEntry->origLen); + uint32_t comp = READ32BE(dirEntry->compLen); + if (comp > orig || comp > woffLen || offs > woffLen - comp) { + return eWOFF_invalid; + } + orig = (orig + 3) & ~3; + if (tableTotal > 0xffffffffU - orig) { + return eWOFF_invalid; + } + tableTotal += orig; + ++dirEntry; + } + + if (tableTotal > 0xffffffffU - sizeof(sfntHeader) - + numTables * sizeof(sfntDirEntry) || + READ32BE(header->totalSfntSize) != + tableTotal + sizeof(sfntHeader) + numTables * sizeof(sfntDirEntry)) { + return eWOFF_invalid; + } + + return eWOFF_ok; +} + +uint32_t +woffGetDecodedSize(const uint8_t * woffData, uint32_t woffLen, + uint32_t * pStatus) +{ + uint32_t status = eWOFF_ok; + uint32_t totalLen = 0; + + if (pStatus && WOFF_FAILURE(*pStatus)) { + return 0; + } + + status = sanityCheck(woffData, woffLen); + if (WOFF_FAILURE(status)) { + FAIL(status); + } + + totalLen = READ32BE(((const woffHeader *) (woffData))->totalSfntSize); + /* totalLen must be correctly rounded up to 4-byte alignment, otherwise + sanityCheck would have failed */ + +failure: + if (pStatus) { + *pStatus = status; + } + return totalLen; +} + +static void +woffDecodeToBufferInternal(const uint8_t * woffData, uint32_t woffLen, + uint8_t * sfntData, uint32_t bufferLen, + uint32_t * pActualSfntLen, uint32_t * pStatus) +{ + /* this is only called after sanityCheck has verified that + (a) basic header fields are ok + (b) all the WOFF table offset/length pairs are valid (within the data) + (c) the sum of original sizes + header/directory matches totalSfntSize + so we don't have to re-check those overflow conditions here */ + tableOrderRec * tableOrder = NULL; + const woffHeader * header; + uint16_t numTables; + uint16_t tableIndex; + uint16_t order; + const woffDirEntry * woffDir; + uint32_t totalLen; + sfntHeader * newHeader; + uint16_t searchRange, rangeShift, entrySelector; + uint32_t offset; + sfntDirEntry * sfntDir; + uint32_t headOffset = 0, headLength = 0; + sfntHeadTable * head; + uint32_t csum = 0; + const uint32_t * csumPtr; + uint32_t oldCheckSumAdjustment; + uint32_t status = eWOFF_ok; + + if (pStatus && WOFF_FAILURE(*pStatus)) { + return; + } + + /* check basic header fields */ + header = (const woffHeader *) (woffData); + if (READ32BE(header->flavor) != SFNT_VERSION_TT && + READ32BE(header->flavor) != SFNT_VERSION_CFF && + READ32BE(header->flavor) != SFNT_VERSION_true) { + status |= eWOFF_warn_unknown_version; + } + + numTables = READ16BE(header->numTables); + woffDir = (const woffDirEntry *) (woffData + sizeof(woffHeader)); + + totalLen = READ32BE(header->totalSfntSize); + + /* construct the sfnt header */ + newHeader = (sfntHeader *) (sfntData); + newHeader->version = header->flavor; + newHeader->numTables = READ16BE(numTables); + + /* calculate header fields for binary search */ + searchRange = numTables; + searchRange |= (searchRange >> 1); + searchRange |= (searchRange >> 2); + searchRange |= (searchRange >> 4); + searchRange |= (searchRange >> 8); + searchRange &= ~(searchRange >> 1); + searchRange *= 16; + newHeader->searchRange = READ16BE(searchRange); + rangeShift = numTables * 16 - searchRange; + newHeader->rangeShift = READ16BE(rangeShift); + entrySelector = 0; + while (searchRange > 16) { + ++entrySelector; + searchRange >>= 1; + } + newHeader->entrySelector = READ16BE(entrySelector); + + tableOrder = (tableOrderRec *) malloc(numTables * sizeof(tableOrderRec)); + if (!tableOrder) { + FAIL(eWOFF_out_of_memory); + } + for (tableIndex = 0; tableIndex < numTables; ++tableIndex) { + tableOrder[tableIndex].offset = READ32BE(woffDir[tableIndex].offset); + tableOrder[tableIndex].oldIndex = tableIndex; + } + qsort(tableOrder, numTables, sizeof(tableOrderRec), compareOffsets); + + /* process each table, filling in the sfnt directory */ + offset = sizeof(sfntHeader) + numTables * sizeof(sfntDirEntry); + sfntDir = (sfntDirEntry *) (sfntData + sizeof(sfntHeader)); + for (order = 0; order < numTables; ++order) { + uint32_t origLen, compLen, tag, sourceOffset; + tableIndex = tableOrder[order].oldIndex; + + /* validity of these was confirmed by sanityCheck */ + origLen = READ32BE(woffDir[tableIndex].origLen); + compLen = READ32BE(woffDir[tableIndex].compLen); + sourceOffset = READ32BE(woffDir[tableIndex].offset); + + sfntDir[tableIndex].tag = woffDir[tableIndex].tag; + sfntDir[tableIndex].offset = READ32BE(offset); + sfntDir[tableIndex].length = woffDir[tableIndex].origLen; + sfntDir[tableIndex].checksum = woffDir[tableIndex].checksum; + csum += READ32BE(sfntDir[tableIndex].checksum); + + if (compLen < origLen) { + uLongf destLen = origLen; + if (uncompress((Bytef *)(sfntData + offset), &destLen, + (const Bytef *)(woffData + sourceOffset), + compLen) != Z_OK || destLen != origLen) { + FAIL(eWOFF_compression_failure); + } + } else { + memcpy(sfntData + offset, woffData + sourceOffset, origLen); + } + + /* note that old Mac bitmap-only fonts have no 'head' table + (eg NISC18030.ttf) but a 'bhed' table instead */ + tag = READ32BE(sfntDir[tableIndex].tag); + if (tag == TABLE_TAG_head || tag == TABLE_TAG_bhed) { + headOffset = offset; + headLength = origLen; + } + + offset += origLen; + + while (offset < totalLen && (offset & 3) != 0) { + sfntData[offset++] = 0; + } + } + + if (headOffset > 0) { + /* the font checksum in the 'head' table depends on all the individual + table checksums (collected above), plus the header and directory + which are added in here */ + if (headLength < HEAD_TABLE_SIZE) { + FAIL(eWOFF_invalid); + } + head = (sfntHeadTable *)(sfntData + headOffset); + oldCheckSumAdjustment = READ32BE(head->checkSumAdjustment); + head->checkSumAdjustment = 0; + csumPtr = (const uint32_t *)sfntData; + while (csumPtr < (const uint32_t *)(sfntData + sizeof(sfntHeader) + + numTables * sizeof(sfntDirEntry))) { + csum += READ32BE(*csumPtr); + csumPtr++; + } + csum = SFNT_CHECKSUM_CALC_CONST - csum; + + if (oldCheckSumAdjustment != csum) { + /* if the checksum doesn't match, we fix it; but this will invalidate + any DSIG that may be present */ + status |= eWOFF_warn_checksum_mismatch; + } + head->checkSumAdjustment = READ32BE(csum); + } + + if (pActualSfntLen) { + *pActualSfntLen = totalLen; + } + if (pStatus) { + *pStatus |= status; + } + free(tableOrder); + return; + +failure: + if (tableOrder) { + free(tableOrder); + } + if (pActualSfntLen) { + *pActualSfntLen = 0; + } + if (pStatus) { + *pStatus = status; + } +} + +void +woffDecodeToBuffer(const uint8_t * woffData, uint32_t woffLen, + uint8_t * sfntData, uint32_t bufferLen, + uint32_t * pActualSfntLen, uint32_t * pStatus) +{ + uint32_t status = eWOFF_ok; + uint32_t totalLen; + + if (pStatus && WOFF_FAILURE(*pStatus)) { + return; + } + + status = sanityCheck(woffData, woffLen); + if (WOFF_FAILURE(status)) { + FAIL(status); + } + + if (!sfntData) { + FAIL(eWOFF_bad_parameter); + } + + totalLen = READ32BE(((const woffHeader *) (woffData))->totalSfntSize); + if (bufferLen < totalLen) { + FAIL(eWOFF_buffer_too_small); + } + + woffDecodeToBufferInternal(woffData, woffLen, sfntData, bufferLen, + pActualSfntLen, pStatus); + return; + +failure: + if (pActualSfntLen) { + *pActualSfntLen = 0; + } + if (pStatus) { + *pStatus = status; + } +} + +const uint8_t * +woffDecode(const uint8_t * woffData, uint32_t woffLen, + uint32_t * sfntLen, uint32_t * pStatus) +{ + uint32_t status = eWOFF_ok; + uint8_t * sfntData = NULL; + uint32_t bufLen; + + if (pStatus && WOFF_FAILURE(*pStatus)) { + return NULL; + } + + status = sanityCheck(woffData, woffLen); + if (WOFF_FAILURE(status)) { + FAIL(status); + } + + bufLen = READ32BE(((const woffHeader *) (woffData))->totalSfntSize); + sfntData = (uint8_t *) malloc(bufLen); + if (!sfntData) { + FAIL(eWOFF_out_of_memory); + } + + woffDecodeToBufferInternal(woffData, woffLen, sfntData, bufLen, + sfntLen, &status); + if (WOFF_FAILURE(status)) { + FAIL(status); + } + + if (pStatus) { + *pStatus |= status; + } + return sfntData; + +failure: + if (sfntData) { + free(sfntData); + } + if (pStatus) { + *pStatus = status; + } + return NULL; +} + +#ifndef WOFF_MOZILLA_CLIENT + +const uint8_t * +woffGetMetadata(const uint8_t * woffData, uint32_t woffLen, + uint32_t * metaLen, uint32_t * pStatus) +{ + const woffHeader * header; + uint32_t offset, compLen; + uLong origLen; + uint8_t * data = NULL; + uint32_t status = eWOFF_ok; + + if (pStatus && WOFF_FAILURE(*pStatus)) { + return NULL; + } + + status = sanityCheck(woffData, woffLen); + if (WOFF_FAILURE(status)) { + FAIL(status); + } + + header = (const woffHeader *) (woffData); + + offset = READ32BE(header->metaOffset); + compLen = READ32BE(header->metaCompLen); + origLen = READ32BE(header->metaOrigLen); + if (offset == 0 || compLen == 0 || origLen == 0) { + return NULL; + } + + if (compLen > woffLen || offset > woffLen - compLen) { + FAIL(eWOFF_invalid); + } + + data = malloc(origLen); + if (!data) { + FAIL(eWOFF_out_of_memory); + } + + if (uncompress((Bytef *)data, &origLen, + (const Bytef *)woffData + offset, compLen) != Z_OK || + origLen != READ32BE(header->metaOrigLen)) { + FAIL(eWOFF_compression_failure); + } + + if (metaLen) { + *metaLen = origLen; + } + if (pStatus) { + *pStatus |= status; + } + return data; + +failure: + if (data) { + free(data); + } + if (pStatus) { + *pStatus = status; + } + return NULL; +} + +const uint8_t * +woffGetPrivateData(const uint8_t * woffData, uint32_t woffLen, + uint32_t * privLen, uint32_t * pStatus) +{ + const woffHeader * header; + uint32_t offset, length; + uint8_t * data = NULL; + uint32_t status = eWOFF_ok; + + if (pStatus && WOFF_FAILURE(*pStatus)) { + return NULL; + } + + status = sanityCheck(woffData, woffLen); + if (WOFF_FAILURE(status)) { + FAIL(status); + } + + header = (const woffHeader *) (woffData); + + offset = READ32BE(header->privOffset); + length = READ32BE(header->privLen); + if (offset == 0 || length == 0) { + return NULL; + } + + if (length > woffLen || offset > woffLen - length) { + FAIL(eWOFF_invalid); + } + + data = malloc(length); + if (!data) { + FAIL(eWOFF_out_of_memory); + } + + memcpy(data, woffData + offset, length); + + if (privLen) { + *privLen = length; + } + if (pStatus) { + *pStatus |= status; + } + return data; + +failure: + if (data) { + free(data); + } + if (pStatus) { + *pStatus = status; + } + return NULL; +} + +void +woffGetFontVersion(const uint8_t * woffData, uint32_t woffLen, + uint16_t * major, uint16_t * minor, uint32_t * pStatus) +{ + const woffHeader * header; + uint32_t status = eWOFF_ok; + + if (pStatus && WOFF_FAILURE(*pStatus)) { + return; + } + + status = sanityCheck(woffData, woffLen); + if (WOFF_FAILURE(status)) { + FAIL(status); + } + + if (!major || !minor) { + FAIL(eWOFF_bad_parameter); + } + + *major = *minor = 0; + + header = (const woffHeader *) (woffData); + + *major = READ16BE(header->majorVersion); + *minor = READ16BE(header->minorVersion); + +failure: + if (pStatus) { + *pStatus = status; + } +} + +/* utility to print messages corresponding to WOFF encoder/decoder errors */ +void +woffPrintStatus(FILE * f, uint32_t status, const char * prefix) +{ + if (!prefix) { + prefix = ""; + } + if (WOFF_WARNING(status)) { + const char * template = "%sWOFF warning: %s\n"; + if (status & eWOFF_warn_unknown_version) { + fprintf(f, template, prefix, "unrecognized sfnt version"); + } + if (status & eWOFF_warn_checksum_mismatch) { + fprintf(f, template, prefix, "checksum mismatch (corrected)"); + } + if (status & eWOFF_warn_misaligned_table) { + fprintf(f, template, prefix, "misaligned font table"); + } + if (status & eWOFF_warn_trailing_data) { + fprintf(f, template, prefix, "extraneous input data discarded"); + } + if (status & eWOFF_warn_unpadded_table) { + fprintf(f, template, prefix, "final table not correctly padded"); + } + if (status & eWOFF_warn_removed_DSIG) { + fprintf(f, template, prefix, "digital signature (DSIG) table removed"); + } + } + if (WOFF_FAILURE(status)) { + const char * template = "%sWOFF error: %s\n"; + const char * msg; + switch (status & 0xff) { + case eWOFF_out_of_memory: + msg = "memory allocation failure"; + break; + case eWOFF_invalid: + msg = "invalid input font"; + break; + case eWOFF_compression_failure: + msg = "zlib compression/decompression failure"; + break; + case eWOFF_bad_signature: + msg = "incorrect WOFF file signature"; + break; + case eWOFF_buffer_too_small: + msg = "buffer too small"; + break; + case eWOFF_bad_parameter: + msg = "bad parameter to WOFF function"; + break; + case eWOFF_illegal_order: + msg = "incorrect table directory order"; + break; + default: + msg = "unknown internal error"; + break; + } + fprintf(f, template, prefix, msg); + } +} + +#endif /* not WOFF_MOZILLA_CLIENT */ diff --git a/src/calibre/utils/fonts/woff/woff.h b/src/calibre/utils/fonts/woff/woff.h new file mode 100644 index 0000000000..d8c6f55b08 --- /dev/null +++ b/src/calibre/utils/fonts/woff/woff.h @@ -0,0 +1,211 @@ +/* -*- Mode: C; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * ***** BEGIN LICENSE BLOCK ***** + * Version: MPL 1.1/GPL 2.0/LGPL 2.1 + * + * The contents of this file are subject to the Mozilla Public License Version + * 1.1 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * http://www.mozilla.org/MPL/ + * + * Software distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License + * for the specific language governing rights and limitations under the + * License. + * + * The Original Code is WOFF font packaging code. + * + * The Initial Developer of the Original Code is Mozilla Corporation. + * Portions created by the Initial Developer are Copyright (C) 2009 + * the Initial Developer. All Rights Reserved. + * + * Contributor(s): + * Jonathan Kew + * + * Alternatively, the contents of this file may be used under the terms of + * either the GNU General Public License Version 2 or later (the "GPL"), or + * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), + * in which case the provisions of the GPL or the LGPL are applicable instead + * of those above. If you wish to allow use of your version of this file only + * under the terms of either the GPL or the LGPL, and not to allow others to + * use your version of this file under the terms of the MPL, indicate your + * decision by deleting the provisions above and replace them with the notice + * and other provisions required by the GPL or the LGPL. If you do not delete + * the provisions above, a recipient may use your version of this file under + * the terms of any one of the MPL, the GPL or the LGPL. + * + * ***** END LICENSE BLOCK ***** */ + +#ifndef WOFF_H_ +#define WOFF_H_ + +/* API for the WOFF encoder and decoder */ + +#ifdef _MSC_VER /* MS VC lacks inttypes.h + but we can make do with a few definitons here */ +typedef char int8_t; +typedef short int16_t; +typedef int int32_t; +typedef unsigned char uint8_t; +typedef unsigned short uint16_t; +typedef unsigned int uint32_t; +#else +#include +#endif + +#include /* only for FILE, needed for woffPrintStatus */ + +/* error codes returned in the status parameter of WOFF functions */ +enum { + /* Success */ + eWOFF_ok = 0, + + /* Errors: no valid result returned */ + eWOFF_out_of_memory = 1, /* malloc or realloc failed */ + eWOFF_invalid = 2, /* invalid input file (e.g., bad offset) */ + eWOFF_compression_failure = 3, /* error in zlib call */ + eWOFF_bad_signature = 4, /* unrecognized file signature */ + eWOFF_buffer_too_small = 5, /* the provided buffer is too small */ + eWOFF_bad_parameter = 6, /* bad parameter (e.g., null source ptr) */ + eWOFF_illegal_order = 7, /* improperly ordered chunks in WOFF font */ + + /* Warnings: call succeeded but something odd was noticed. + Multiple warnings may be OR'd together. */ + eWOFF_warn_unknown_version = 0x0100, /* unrecognized version of sfnt, + not standard TrueType or CFF */ + eWOFF_warn_checksum_mismatch = 0x0200, /* bad checksum, use with caution; + any DSIG will be invalid */ + eWOFF_warn_misaligned_table = 0x0400, /* table not long-aligned; fixing, + but DSIG will be invalid */ + eWOFF_warn_trailing_data = 0x0800, /* trailing junk discarded, + any DSIG may be invalid */ + eWOFF_warn_unpadded_table = 0x1000, /* sfnt not correctly padded, + any DSIG may be invalid */ + eWOFF_warn_removed_DSIG = 0x2000 /* removed digital signature + while fixing checksum errors */ +}; + +/* Note: status parameters must be initialized to eWOFF_ok before calling + WOFF functions. If the status parameter contains an error code, + functions will return immediately. */ + +#define WOFF_SUCCESS(status) (((uint32_t)(status) & 0xff) == eWOFF_ok) +#define WOFF_FAILURE(status) (!WOFF_SUCCESS(status)) +#define WOFF_WARNING(status) ((uint32_t)(status) & ~0xff) + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef WOFF_DISABLE_ENCODING + +/***************************************************************************** + * Returns a new malloc() block containing the encoded data, or NULL on error; + * caller should free() this when finished with it. + * Returns length of the encoded data in woffLen. + * The new WOFF has no metadata or private block; + * see the following functions to update these elements. + */ +const uint8_t * woffEncode(const uint8_t * sfntData, uint32_t sfntLen, + uint16_t majorVersion, uint16_t minorVersion, + uint32_t * woffLen, uint32_t * status); + + +/***************************************************************************** + * Add the given metadata block to the WOFF font, replacing any existing + * metadata block. The block will be zlib-compressed. + * Metadata is required to be valid XML (use of UTF-8 is recommended), + * though this function does not currently check this. + * The woffData pointer must be a malloc() block (typically from woffEncode); + * it will be freed by this function and a new malloc() block will be returned. + * Returns NULL if an error occurs, in which case the original WOFF is NOT freed. + */ +const uint8_t * woffSetMetadata(const uint8_t * woffData, uint32_t * woffLen, + const uint8_t * metaData, uint32_t metaLen, + uint32_t * status); + + +/***************************************************************************** + * Add the given private data block to the WOFF font, replacing any existing + * private block. The block will NOT be zlib-compressed. + * Private data may be any arbitrary block of bytes; it may be externally + * compressed by the client if desired. + * The woffData pointer must be a malloc() block (typically from woffEncode); + * it will be freed by this function and a new malloc() block will be returned. + * Returns NULL if an error occurs, in which case the original WOFF is NOT freed. + */ +const uint8_t * woffSetPrivateData(const uint8_t * woffData, uint32_t * woffLen, + const uint8_t * privData, uint32_t privLen, + uint32_t * status); + +#endif /* WOFF_DISABLE_ENCODING */ + +/***************************************************************************** + * Returns the size of buffer needed to decode the font (or zero on error). + */ +uint32_t woffGetDecodedSize(const uint8_t * woffData, uint32_t woffLen, + uint32_t * pStatus); + + +/***************************************************************************** + * Decodes WOFF font to a caller-supplied buffer of size bufferLen. + * Returns the actual size of the decoded sfnt data in pActualSfntLen + * (must be <= bufferLen, otherwise an error will be returned). + */ +void woffDecodeToBuffer(const uint8_t * woffData, uint32_t woffLen, + uint8_t * sfntData, uint32_t bufferLen, + uint32_t * pActualSfntLen, uint32_t * pStatus); + + +/***************************************************************************** + * Returns a new malloc() block containing the decoded data, or NULL on error; + * caller should free() this when finished with it. + * Returns length of the decoded data in sfntLen. + */ +const uint8_t * woffDecode(const uint8_t * woffData, uint32_t woffLen, + uint32_t * sfntLen, uint32_t * status); + + +/***************************************************************************** + * Returns a new malloc() block containing the metadata from the WOFF font, + * or NULL if an error occurs or no metadata is present. + * Length of the metadata is returned in metaLen. + * The metadata is decompressed before returning. + */ +const uint8_t * woffGetMetadata(const uint8_t * woffData, uint32_t woffLen, + uint32_t * metaLen, uint32_t * status); + + +/***************************************************************************** + * Returns a new malloc() block containing the private data from the WOFF font, + * or NULL if an error occurs or no private data is present. + * Length of the private data is returned in privLen. + */ +const uint8_t * woffGetPrivateData(const uint8_t * woffData, uint32_t woffLen, + uint32_t * privLen, uint32_t * status); + + +/***************************************************************************** + * Returns the font version numbers from the WOFF font in the major and minor + * parameters. + * Check the status result to know if the function succeeded. + */ +void woffGetFontVersion(const uint8_t * woffData, uint32_t woffLen, + uint16_t * major, uint16_t * minor, + uint32_t * status); + + +/***************************************************************************** + * Utility to print warning and/or error status to the specified FILE*. + * The prefix string will be prepended to each line (ok to pass NULL if no + * prefix is wanted). + * (Provides terse English messages only, not intended for end-user display; + * user-friendly tools should map the status codes to their own messages.) + */ +void woffPrintStatus(FILE * f, uint32_t status, const char * prefix); + + +#ifdef __cplusplus +} +#endif + +#endif From c8358bb266b427209605ca9ca8628311a7ceb43c Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 21 Oct 2012 15:19:16 +0530 Subject: [PATCH 16/23] ... --- src/calibre/ebooks/metadata/fb2.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/calibre/ebooks/metadata/fb2.py b/src/calibre/ebooks/metadata/fb2.py index cecdfead0d..5a52ceb701 100644 --- a/src/calibre/ebooks/metadata/fb2.py +++ b/src/calibre/ebooks/metadata/fb2.py @@ -379,6 +379,10 @@ def set_metadata(stream, mi, apply_null=False, update_timestamp=False): stream.seek(0) stream.truncate() + # Apparently there exists FB2 reading software that chokes on the use of + # single quotes in xml declaration. Sigh. See + # http://www.mobileread.com/forums/showthread.php?p=2273184#post2273184 + stream.write(b'\n') stream.write(etree.tostring(root, method='xml', encoding='utf-8', - xml_declaration=True)) + xml_declaration=False)) From c85a932d4da17c7b9bdd50618f294461a953dbcd Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 21 Oct 2012 15:41:33 +0530 Subject: [PATCH 17/23] ... --- setup/build_environment.py | 6 ++++++ setup/extensions.py | 11 +++++------ src/calibre/test_build.py | 3 ++- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/setup/build_environment.py b/setup/build_environment.py index 69b1dc231b..6601578345 100644 --- a/setup/build_environment.py +++ b/setup/build_environment.py @@ -95,6 +95,9 @@ chmlib_inc_dirs = chmlib_lib_dirs = [] sqlite_inc_dirs = [] icu_inc_dirs = [] icu_lib_dirs = [] +zlib_inc_dirs = [] +zlib_lib_dirs = [] +zlib_libs = ['z'] if iswindows: prefix = r'C:\cygwin\home\kovid\sw' @@ -118,6 +121,9 @@ if iswindows: ft_lib_dirs = [sw_lib_dir] ft_libs = ['freetype'] ft_inc_dirs = [sw_inc_dir] + zlib_inc_dirs = [sw_inc_dir] + zlib_lib_dirs = [sw_lib_dir] + zlib_libs = ['zlib'] magick_inc_dirs = [os.path.join(prefix, 'build', 'ImageMagick-6.7.6')] magick_lib_dirs = [os.path.join(magick_inc_dirs[0], 'VisualMagick', 'lib')] diff --git a/setup/extensions.py b/setup/extensions.py index 8d49f904b7..989a9ddbe9 100644 --- a/setup/extensions.py +++ b/setup/extensions.py @@ -17,7 +17,8 @@ from setup.build_environment import (fc_inc, fc_lib, chmlib_inc_dirs, fc_error, podofo_inc, podofo_lib, podofo_error, pyqt, OSX_SDK, NMAKE, QMAKE, msvc, MT, win_inc, win_lib, win_ddk, magick_inc_dirs, magick_lib_dirs, magick_libs, chmlib_lib_dirs, sqlite_inc_dirs, icu_inc_dirs, - icu_lib_dirs, win_ddk_lib_dirs, ft_libs, ft_lib_dirs, ft_inc_dirs) + icu_lib_dirs, win_ddk_lib_dirs, ft_libs, ft_lib_dirs, ft_inc_dirs, + zlib_libs, zlib_lib_dirs, zlib_inc_dirs) MT isunix = islinux or isosx or isbsd @@ -50,10 +51,6 @@ class Extension(object): reflow_sources = glob.glob(os.path.join(SRC, 'calibre', 'ebooks', 'pdf', '*.cpp')) reflow_headers = glob.glob(os.path.join(SRC, 'calibre', 'ebooks', 'pdf', '*.h')) -woff_libs = ['z'] -if iswindows: - woff_libs = ['zlib'] - icu_libs = ['icudata', 'icui18n', 'icuuc', 'icuio'] icu_cflags = [] if iswindows: @@ -138,7 +135,9 @@ extensions = [ headers=[ 'calibre/utils/fonts/woff/woff.h', 'calibre/utils/fonts/woff/woff-private.h'], - libraries=woff_libs + libraries=zlib_libs, + lib_dirs=zlib_lib_dirs, + inc_dirs=zlib_inc_dirs, ), diff --git a/src/calibre/test_build.py b/src/calibre/test_build.py index 8bf3427f8c..8ca0a36528 100644 --- a/src/calibre/test_build.py +++ b/src/calibre/test_build.py @@ -33,7 +33,7 @@ def test_lxml(): raise RuntimeError('lxml failed') def test_freetype(): - from calibre.utils.fonts.freetype import test + from calibre.utils.fonts.free_type import test test() print ('FreeType OK!') @@ -117,6 +117,7 @@ def test_wpd(): def test_woff(): from calibre.utils.fonts.woff import test test() + print ('WOFF ok!') def test(): test_plugins() From d24835bff8f297330b81870c2576e34b909a6a0f Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 21 Oct 2012 18:39:13 +0530 Subject: [PATCH 18/23] ... --- src/calibre/gui2/convert/regex_builder.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/calibre/gui2/convert/regex_builder.py b/src/calibre/gui2/convert/regex_builder.py index 3ce8ac36d5..f3c1f0a826 100644 --- a/src/calibre/gui2/convert/regex_builder.py +++ b/src/calibre/gui2/convert/regex_builder.py @@ -211,6 +211,9 @@ class RegexEdit(QWidget, Ui_Edit): self.button.clicked.connect(self.builder) def builder(self): + if self.db is None: + self.doc_cache = _('Click the Open button below to open a ' + 'ebook to use for testing.') bld = RegexBuilder(self.db, self.book_id, self.edit.text(), self.doc_cache, self) if bld.cancelled: return From 1db24f51922f7c8975d1e3d3dbbed1f53283527c Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 21 Oct 2012 22:14:31 +0530 Subject: [PATCH 19/23] Windows: Check if any of the files of a book are in use before changing the title/author, this (usually) prevents the creation of duplicate files if one of the files is open in another program --- src/calibre/gui2/metadata/basic_widgets.py | 2 +- src/calibre/library/database2.py | 12 ++++++++-- src/calibre/utils/filenames.py | 28 ++++++++++++++++++++++ 3 files changed, 39 insertions(+), 3 deletions(-) diff --git a/src/calibre/gui2/metadata/basic_widgets.py b/src/calibre/gui2/metadata/basic_widgets.py index 36605c7584..0da9b1bcf4 100644 --- a/src/calibre/gui2/metadata/basic_widgets.py +++ b/src/calibre/gui2/metadata/basic_widgets.py @@ -270,7 +270,7 @@ class AuthorsEdit(EditWithComplete): import traceback fname = err.filename if err.filename else 'file' error_dialog(self, _('Permission denied'), - _('Could not open %s. Is it being used by another' + _('Could not open "%s". Is it being used by another' ' program?')%fname, det_msg=traceback.format_exc(), show=True) return False diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 5b4f7eec7e..5b2abd0c6b 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -7,7 +7,7 @@ __docformat__ = 'restructuredtext en' The database used to store ebook metadata ''' import os, sys, shutil, cStringIO, glob, time, functools, traceback, re, \ - json, uuid, hashlib, copy + json, uuid, hashlib, copy, errno from collections import defaultdict import threading, random from itertools import repeat @@ -30,7 +30,8 @@ from calibre.ptempfile import (PersistentTemporaryFile, base_dir, SpooledTemporaryFile) from calibre.customize.ui import run_plugins_on_import from calibre import isbytestring -from calibre.utils.filenames import ascii_filename, samefile +from calibre.utils.filenames import (ascii_filename, samefile, + windows_is_folder_in_use) from calibre.utils.date import (utcnow, now as nowf, utcfromtimestamp, parse_only_date, UNDEFINED_DATE) from calibre.utils.config import prefs, tweaks, from_json, to_json @@ -649,6 +650,13 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): spath = os.path.join(self.library_path, *current_path.split('/')) if current_path and os.path.exists(spath): # Migrate existing files + if iswindows: + uf = windows_is_folder_in_use(spath) + if uf is not None: + err = IOError(errno.EACCES, + _('File is open in another process')) + err.filename = uf + raise err cdata = self.cover(id, index_is_id=True) if cdata is not None: with lopen(os.path.join(tpath, 'cover.jpg'), 'wb') as f: diff --git a/src/calibre/utils/filenames.py b/src/calibre/utils/filenames.py index 65451dab9c..f6455831ba 100644 --- a/src/calibre/utils/filenames.py +++ b/src/calibre/utils/filenames.py @@ -249,4 +249,32 @@ def samefile(src, dst): os.path.normcase(os.path.abspath(dst))) return samestring +def windows_is_file_opened(path): + import win32file, winerror + from pywintypes import error + if isbytestring(path): path = path.decode(filesystem_encoding) + try: + h = win32file.CreateFile(path, win32file.GENERIC_READ, 0, None, + win32file.OPEN_EXISTING, 0, 0) + except error as e: + if getattr(e, 'winerror', 0) == winerror.ERROR_SHARING_VIOLATION: + return True + else: + win32file.CloseHandle(h) + return False + +def windows_is_folder_in_use(path): + ''' + Returns the path to a file that is used in another process in the specified + folder, or None if no such file exists. Note + that this function is not a guarantee. A file may well be opened in the + folder after this function returns. However, it is useful to handle the + common case of a sharing violation gracefully most of the time. + ''' + if isbytestring(path): path = path.decode(filesystem_encoding) + for x in os.listdir(path): + f = os.path.join(path, x) + if windows_is_file_opened(f): + return f + return None From d210af88606dfd879890b7bda51ef6d09807c440 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 22 Oct 2012 08:08:49 +0530 Subject: [PATCH 20/23] Fix Time Magazine --- recipes/time_magazine.recipe | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/recipes/time_magazine.recipe b/recipes/time_magazine.recipe index dfe897500e..9905a1df1d 100644 --- a/recipes/time_magazine.recipe +++ b/recipes/time_magazine.recipe @@ -23,16 +23,15 @@ class Time(BasicNewsRecipe): keep_only_tags = [ { - 'class':['tout1', 'entry-content', 'external-gallery-img', 'image-meta'] + 'class':['primary-col', 'tout1'] }, ] remove_tags = [ - {'class':['thumbnail', 'button']}, + {'class':['button', 'entry-sharing group', 'wp-paginate', + 'moving-markup', 'entry-comments']}, ] - - recursions = 10 - match_regexps = [r'/[0-9,]+-(2|3|4|5|6|7|8|9)(,\d+){0,1}.html',r'http://www.time.com/time/specials/packages/article/.*'] + extra_css = '.entry-date { padding-left: 2ex }' preprocess_regexps = [(re.compile( r''), lambda m:'')] @@ -45,7 +44,7 @@ class Time(BasicNewsRecipe): br.select_form(predicate=lambda f: 'action' in f.attrs and f.attrs['action'] == 'https://auth.time.com/login.php') br['username'] = self.username br['password'] = self.password - br['magcode'] = ['TD'] + # br['magcode'] = ['TD'] br.find_control('turl').readonly = False br['turl'] = 'http://www.time.com/time/magazine' br.find_control('rurl').readonly = False @@ -104,7 +103,14 @@ class Time(BasicNewsRecipe): method='text').strip() if not title: continue url = a[0].get('href') - url = re.sub('/magazine/article/0,9171','/subscriber/printout/0,8816', url) + if url.startswith('/'): + url = 'http://www.time.com'+url + if '/article/0,' in url: + soup = self.index_to_soup(url) + a = soup.find('a', href=lambda x:x and '/printout/' in x) + url = a['href'].replace('/printout', '/subscriber/printout') + else: + url += 'print/' if url.endswith('/') else '/print/' if url.startswith('/'): url = 'http://www.time.com'+url desc = '' @@ -112,10 +118,18 @@ class Time(BasicNewsRecipe): if p: desc = html.tostring(p[0], encoding=unicode, method='text') - self.log('\t', title, ':\n\t\t', desc) + self.log('\t', title, ':\n\t\t', url) yield { 'title' : title, 'url' : url, 'date' : '', 'description' : desc } + + def preprocess_html(self, soup): + for fig in soup.findAll('figure'): + img = fig.find('img') + if img is not None: + fig.replaceWith(img) + return soup + From 9070385bb779fe76a855219e482ba2562d864649 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 22 Oct 2012 08:17:01 +0530 Subject: [PATCH 21/23] ... --- src/calibre/gui2/library/models.py | 2 ++ src/calibre/web/fetch/simple.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py index 68031c8dd9..641ac23611 100644 --- a/src/calibre/gui2/library/models.py +++ b/src/calibre/gui2/library/models.py @@ -1368,6 +1368,8 @@ class DeviceBooksModel(BooksModel): # {{{ return QVariant(authors_to_string(au)) elif cname == 'size': size = self.db[self.map[row]].size + if not isinstance(size, (float, int)): + size = 0 return QVariant(human_readable(size)) elif cname == 'timestamp': dt = self.db[self.map[row]].datetime diff --git a/src/calibre/web/fetch/simple.py b/src/calibre/web/fetch/simple.py index 5480e3f683..bea45f1c8d 100644 --- a/src/calibre/web/fetch/simple.py +++ b/src/calibre/web/fetch/simple.py @@ -549,7 +549,7 @@ def option_parser(usage=_('%prog URL\n\nWhere URL is for example http://google.c def create_fetcher(options, image_map={}, log=None): if log is None: - log = Log() + log = Log(level=Log.DEBUG) if options.verbose else Log() return RecursiveFetcher(options, log, image_map={}) def main(args=sys.argv): From a69112b3e4bd66cc24a2116ee1f6ff108bf39989 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 22 Oct 2012 08:36:51 +0530 Subject: [PATCH 22/23] Ignore all exceptions in the font family chooser widget --- src/calibre/gui2/font_family_chooser.py | 30 +++++++++++++++++++------ src/calibre/gui2/main_window.py | 2 ++ 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/src/calibre/gui2/font_family_chooser.py b/src/calibre/gui2/font_family_chooser.py index 04d9dfdfb6..b6514e5011 100644 --- a/src/calibre/gui2/font_family_chooser.py +++ b/src/calibre/gui2/font_family_chooser.py @@ -55,6 +55,12 @@ def writing_system_for_font(font): class FontFamilyDelegate(QStyledItemDelegate): def sizeHint(self, option, index): + try: + return self.do_size_hint(option, index) + except: + return QSize(300, 50) + + def do_size_hint(self, option, index): text = index.data(Qt.DisplayRole).toString() font = QFont(option.font) font.setPointSize(QFontInfo(font).pointSize() * 1.5) @@ -62,6 +68,14 @@ class FontFamilyDelegate(QStyledItemDelegate): return QSize(m.width(text), m.height()) def paint(self, painter, option, index): + painter.save() + try: + self.do_paint(painter, option, index) + except: + pass + painter.restore() + + def do_paint(self, painter, option, index): text = unicode(index.data(Qt.DisplayRole).toString()) font = QFont(option.font) font.setPointSize(QFontInfo(font).pointSize() * 1.5) @@ -75,7 +89,6 @@ class FontFamilyDelegate(QStyledItemDelegate): r = option.rect if option.state & QStyle.State_Selected: - painter.save() painter.setBrush(option.palette.highlight()) painter.setPen(Qt.NoPen) painter.drawRect(option.rect) @@ -102,9 +115,6 @@ class FontFamilyDelegate(QStyledItemDelegate): painter.setFont(old) - if (option.state & QStyle.State_Selected): - painter.restore() - class FontFamilyChooser(QComboBox): def __init__(self, parent=None): @@ -138,7 +148,10 @@ class FontFamilyChooser(QComboBox): def sizeHint(self): ans = QComboBox.sizeHint(self) - ans.setWidth(QFontMetrics(self.font()).width('m'*14)) + try: + ans.setWidth(QFontMetrics(self.font()).width('m'*14)) + except: + pass return ans @dynamic_property @@ -157,12 +170,15 @@ class FontFamilyChooser(QComboBox): self.setCurrentIndex(idx) return property(fget=fget, fset=fset) - -if __name__ == '__main__': +def test(): app = QApplication([]) + app d = QDialog() d.setLayout(QVBoxLayout()) d.layout().addWidget(FontFamilyChooser(d)) d.layout().addWidget(QFontComboBox(d)) d.exec_() +if __name__ == '__main__': + test() + diff --git a/src/calibre/gui2/main_window.py b/src/calibre/gui2/main_window.py index 134aae3ad1..88827fe646 100644 --- a/src/calibre/gui2/main_window.py +++ b/src/calibre/gui2/main_window.py @@ -139,3 +139,5 @@ class MainWindow(QMainWindow): show=True) except BaseException: pass + except: + pass From 7c18f4f0d20d570f924357a680b8e57b51cea1c0 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 22 Oct 2012 10:56:42 +0530 Subject: [PATCH 23/23] Make the bundled fonts available on all platforms --- session.vim | 1 + src/calibre/__init__.py | 31 ++++++++----------- src/calibre/gui2/__init__.py | 3 +- src/calibre/gui2/viewer/documentview.py | 3 +- src/calibre/utils/fonts/__init__.py | 4 +-- src/calibre/utils/fonts/fc.py | 12 +++----- src/calibre/utils/fonts/fontconfig.c | 16 ++++++++++ src/calibre/utils/fonts/win_fonts.py | 41 ++++++++++++++++++++----- 8 files changed, 71 insertions(+), 40 deletions(-) diff --git a/session.vim b/session.vim index c60bcbdf4f..079975f8d4 100644 --- a/session.vim +++ b/session.vim @@ -9,6 +9,7 @@ let g:syntastic_cpp_include_dirs = [ \'/usr/include/qt4/QtGui', \'/usr/include/qt4', \'/usr/include/freetype2', + \'/usr/include/fontconfig', \'src/qtcurve/common', 'src/qtcurve', \'/usr/include/ImageMagick', \] diff --git a/src/calibre/__init__.py b/src/calibre/__init__.py index 31ce61328a..1fce4a9da6 100644 --- a/src/calibre/__init__.py +++ b/src/calibre/__init__.py @@ -329,6 +329,7 @@ def get_parsed_proxy(typ='http', debug=True): ans['port'] = int(ans['port']) except: if debug: + import traceback traceback.print_exc() else: if debug: @@ -689,26 +690,18 @@ def remove_bracketed_text(src, buf.append(char) return u''.join(buf) -if isosx: - import glob, shutil - fdir = os.path.expanduser('~/.fonts') - try: - if not os.path.exists(fdir): - os.makedirs(fdir) - if not os.path.exists(os.path.join(fdir, 'LiberationSans_Regular.ttf')): - base = P('fonts/liberation/*.ttf') - for f in glob.glob(base): - shutil.copy2(f, fdir) - except: - import traceback - traceback.print_exc() - def load_builtin_fonts(): - import glob - from PyQt4.Qt import QFontDatabase - base = P('fonts/liberation/*.ttf') - for f in glob.glob(base): - QFontDatabase.addApplicationFont(f) + if iswindows or isosx: + import glob + from PyQt4.Qt import QFontDatabase + for f in glob.glob(P('fonts/liberation/*.ttf')): + QFontDatabase.addApplicationFont(f) + else: + # On linux these are loaded by fontconfig which means that + # they are available to Qt as well, since Qt uses fontconfig + from calibre.utils.fonts import fontconfig + fontconfig + return 'Liberation Serif', 'Liberation Sans', 'Liberation Mono' diff --git a/src/calibre/gui2/__init__.py b/src/calibre/gui2/__init__.py index 0ac0783bd5..688b134f0e 100644 --- a/src/calibre/gui2/__init__.py +++ b/src/calibre/gui2/__init__.py @@ -12,7 +12,7 @@ from PyQt4.Qt import (QVariant, QFileInfo, QObject, SIGNAL, QBuffer, Qt, ORG_NAME = 'KovidsBrain' APP_UID = 'libprs500' -from calibre import prints +from calibre import prints, load_builtin_fonts from calibre.constants import (islinux, iswindows, isbsd, isfrozen, isosx, plugins, config_dir, filesystem_encoding, DEBUG) from calibre.utils.config import Config, ConfigProxy, dynamic, JSONConfig @@ -779,6 +779,7 @@ class Application(QApplication): qt_app = self self._file_open_paths = [] self._file_open_lock = RLock() + load_builtin_fonts() self.setup_styles(force_calibre_style) if DEBUG: diff --git a/src/calibre/gui2/viewer/documentview.py b/src/calibre/gui2/viewer/documentview.py index 8ea9c26d00..33b36cf6ec 100644 --- a/src/calibre/gui2/viewer/documentview.py +++ b/src/calibre/gui2/viewer/documentview.py @@ -16,7 +16,7 @@ from PyQt4.QtWebKit import QWebPage, QWebView, QWebSettings from calibre.gui2.viewer.flip import SlideFlip from calibre.gui2.shortcuts import Shortcuts -from calibre import prints, load_builtin_fonts +from calibre import prints from calibre.customize.ui import all_viewer_plugins from calibre.gui2.viewer.keys import SHORTCUTS from calibre.gui2.viewer.javascript import JavaScriptLoader @@ -86,7 +86,6 @@ class Document(QWebPage): # {{{ settings = self.settings() # Fonts - load_builtin_fonts() self.all_viewer_plugins = tuple(all_viewer_plugins()) for pl in self.all_viewer_plugins: pl.load_fonts() diff --git a/src/calibre/utils/fonts/__init__.py b/src/calibre/utils/fonts/__init__.py index e245b1536c..85bcd8eb39 100644 --- a/src/calibre/utils/fonts/__init__.py +++ b/src/calibre/utils/fonts/__init__.py @@ -111,9 +111,9 @@ fontconfig = Fonts() def test(): import os print(fontconfig.find_font_families()) - m = 'times new roman' if iswindows else 'liberation serif' + m = 'Liberation Serif' for ft, val in fontconfig.files_for_family(m).iteritems(): - print val[0], ft, val[1], os.path.getsize(val[1]) + print val[0], ft, val[1], os.path.getsize(val[0]) if __name__ == '__main__': test() diff --git a/src/calibre/utils/fonts/fc.py b/src/calibre/utils/fonts/fc.py index b6a4b1f906..793ee9b6b1 100644 --- a/src/calibre/utils/fonts/fc.py +++ b/src/calibre/utils/fonts/fc.py @@ -8,7 +8,7 @@ __docformat__ = 'restructuredtext en' import os, sys -from calibre.constants import plugins, iswindows, islinux, isbsd +from calibre.constants import plugins, islinux, isbsd _fc, _fc_err = plugins['fontconfig'] @@ -35,18 +35,14 @@ class FontConfig(Thread): if isinstance(config_dir, unicode): config_dir = config_dir.encode(sys.getfilesystemencoding()) config = os.path.join(config_dir, 'fonts.conf') - if iswindows and getattr(sys, 'frozen', False): - config_dir = os.path.join(os.path.dirname(sys.executable), - 'fontconfig') - if isinstance(config_dir, unicode): - config_dir = config_dir.encode(sys.getfilesystemencoding()) - config = os.path.join(config_dir, 'fonts.conf') try: _fc.initialize(config) except: import traceback traceback.print_exc() self.failed = True + if not self.failed and hasattr(_fc, 'add_font_dir'): + _fc.add_font_dir(P('fonts/liberation')) def wait(self): if not (islinux or isbsd): @@ -162,7 +158,7 @@ def test(): from pprint import pprint; pprint(fontconfig.find_font_families()) pprint(fontconfig.files_for_family('liberation serif')) - m = 'times new roman' if iswindows else 'liberation serif' + m = 'liberation serif' pprint(fontconfig.match(m+':slant=italic:weight=bold', verbose=True)) if __name__ == '__main__': diff --git a/src/calibre/utils/fonts/fontconfig.c b/src/calibre/utils/fonts/fontconfig.c index d8f7190798..be1cbdce13 100644 --- a/src/calibre/utils/fonts/fontconfig.c +++ b/src/calibre/utils/fonts/fontconfig.c @@ -44,6 +44,17 @@ fontconfig_initialize(PyObject *self, PyObject *args) { Py_RETURN_FALSE; } +static PyObject* +fontconfig_add_font_dir(PyObject *self, PyObject *args) { + FcChar8 *path; + + if (!PyArg_ParseTuple(args, "s", &path)) return NULL; + + if (FcConfigAppFontAddDir(NULL, path)) + Py_RETURN_TRUE; + Py_RETURN_FALSE; +} + static void fontconfig_cleanup_find(FcPattern *p, FcObjectSet *oset, FcFontSet *fs) { if (p != NULL) FcPatternDestroy(p); @@ -314,6 +325,11 @@ PyMethodDef fontconfig_methods[] = { }, + {"add_font_dir", fontconfig_add_font_dir, METH_VARARGS, + "add_font_dir(path_to_dir)\n\n" + "Add the fonts in the specified directory to the list of application specific fonts." + }, + {NULL, NULL, 0, NULL} }; diff --git a/src/calibre/utils/fonts/win_fonts.py b/src/calibre/utils/fonts/win_fonts.py index 747580d45e..9fcf5df226 100644 --- a/src/calibre/utils/fonts/win_fonts.py +++ b/src/calibre/utils/fonts/win_fonts.py @@ -19,6 +19,23 @@ class WinFonts(object): def __init__(self, winfonts): self.w = winfonts + # Windows thinks the Liberation font files are not valid, so we use + # this hack to make them available + self.app_font_families = {} + + for f in ('Serif', 'Sans', 'Mono'): + base = 'fonts/liberation/Liberation%s-%s.ttf' + self.app_font_families['Liberation %s'%f] = m = {} + for weight, is_italic in product( (self.w.FW_NORMAL, self.w.FW_BOLD), (False, True) ): + name = {(self.w.FW_NORMAL, False):'Regular', + (self.w.FW_NORMAL, True):'Italic', + (self.w.FW_BOLD, False):'Bold', + (self.w.FW_BOLD, True):'BoldItalic'}[(weight, + is_italic)] + m[(weight, is_italic)] = base%(f, name) + + # import pprint + # pprint.pprint(self.app_font_families) def font_families(self): names = set() @@ -30,7 +47,7 @@ class WinFonts(object): not font['name'].startswith('@') ): names.add(font['name']) - return sorted(names) + return sorted(names.union(frozenset(self.app_font_families))) def get_normalized_name(self, is_italic, weight): if is_italic: @@ -43,12 +60,18 @@ class WinFonts(object): family = type(u'')(family) ans = {} for weight, is_italic in product( (self.w.FW_NORMAL, self.w.FW_BOLD), (False, True) ): - try: - data = self.w.font_data(family, is_italic, weight) - except Exception as e: - prints('Failed to get font data for font: %s [%s] with error: %s'% - (family, self.get_normalized_name(is_italic, weight), e)) - continue + if family in self.app_font_families: + m = self.app_font_families[family] + path = m.get((weight, is_italic), None) + if path is None: continue + data = P(path, data=True) + else: + try: + data = self.w.font_data(family, is_italic, weight) + except Exception as e: + prints('Failed to get font data for font: %s [%s] with error: %s'% + (family, self.get_normalized_name(is_italic, weight), e)) + continue ok, sig = is_truetype_font(data) if not ok: @@ -122,7 +145,7 @@ def test_ttf_reading(): get_font_characteristics(raw) print() -if __name__ == '__main__': +def test(): base = os.path.abspath(__file__) d = os.path.dirname pluginsd = os.path.join(d(d(d(base))), 'plugins') @@ -143,3 +166,5 @@ if __name__ == '__main__': prints(' ', font, data[0], data[1], len(data[2])) print () +if __name__ == '__main__': + test()