Fix text rendering by using private Qt APIs

session.vim

@@ -12,6 +12,7 @@ let g:syntastic_cpp_include_dirs = [
             \'/usr/include/fontconfig',
             \'src/qtcurve/common', 'src/qtcurve',
             \'src/unrar',
+            \'src/qt-harfbuzz/src',
             \'/usr/include/ImageMagick',
             \]
 let g:syntastic_c_include_dirs = g:syntastic_cpp_include_dirs

setup/extensions.py

@@ -183,6 +183,13 @@ extensions = [
                 sip_files = ['calibre/gui2/progress_indicator/QProgressIndicator.sip']
                 ),
 
+    Extension('qt_hack',
+                ['calibre/ebooks/pdf/render/qt_hack.cpp'],
+                inc_dirs = ['calibre/ebooks/pdf/render', 'qt-harfbuzz/src'],
+                headers = ['calibre/ebooks/pdf/render/qt_hack.h'],
+                sip_files = ['calibre/ebooks/pdf/render/qt_hack.sip']
+                ),
+
     Extension('unrar',
               ['unrar/%s.cpp'%(x.partition('.')[0]) for x in '''
                rar.o strlist.o strfn.o pathfn.o savepos.o smallfn.o global.o file.o
@@ -545,6 +552,9 @@ class Build(Command):
                 VERSION  = 1.0.0
                 CONFIG   += %s
             ''')%(ext.name, ' '.join(ext.headers), ' '.join(ext.sources), archs)
+            if ext.inc_dirs:
+                idir = ' '.join(ext.inc_dirs)
+                pro += 'INCLUDEPATH = %s\n'%idir
             pro = pro.replace('\\', '\\\\')
             open(ext.name+'.pro', 'wb').write(pro)
             qmc = [QMAKE, '-o', 'Makefile']

src/calibre/constants.py

@@ -100,6 +100,7 @@ class Plugins(collections.Mapping):
                 'freetype',
                 'woff',
                 'unrar',
+                'qt_hack',
             ]
         if iswindows:
             plugins.extend(['winutil', 'wpd', 'winfonts'])

src/calibre/ebooks/pdf/render/engine.py

@@ -7,15 +7,17 @@ __license__   = 'GPL v3'
 __copyright__ = '2012, Kovid Goyal <kovid at kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import sys, traceback, unicodedata
+import sys, traceback
 from math import sqrt
 from collections import namedtuple
-from functools import wraps
+from functools import wraps, partial
 
+import sip
 from PyQt4.Qt import (QPaintEngine, QPaintDevice, Qt, QApplication, QPainter,
-                      QTransform, QPainterPath, QTextOption, QTextLayout,
+                      QTransform, QPainterPath, QImage, QByteArray, QBuffer,
-                      QImage, QByteArray, QBuffer, qRgba)
+                      qRgba)
 
+from calibre.constants import plugins
 from calibre.ebooks.pdf.render.serialize import (Color, PDFStream, Path)
 from calibre.ebooks.pdf.render.common import inch, A4
 from calibre.utils.fonts.sfnt.container import Sfnt
@@ -215,14 +217,15 @@ class PdfEngine(QPaintEngine):
         self.graphics_state = GraphicsState()
         self.errors_occurred = False
         self.errors, self.debug = errors, debug
-        self.text_option = QTextOption()
-        self.text_option.setWrapMode(QTextOption.NoWrap)
         self.fonts = {}
         i = QImage(1, 1, QImage.Format_ARGB32)
         i.fill(qRgba(0, 0, 0, 255))
         self.alpha_bit = i.constBits().asstring(4).find(b'\xff')
         self.current_page_num = 1
         self.current_page_inited = False
+        self.qt_hack, err = plugins['qt_hack']
+        if err:
+            raise RuntimeError('Failed to load qt_hack with err: %s'%err)
 
     def init_page(self):
         self.pdf.transform(self.pdf_system)
@@ -421,98 +424,48 @@ class PdfEngine(QPaintEngine):
             self.pdf.draw_rect(bl.x(), bl.y(), rect.width(), rect.height(),
                              stroke=self.do_stroke, fill=self.do_fill)
 
-    def get_text_layout(self, text_item, text):
+    def create_sfnt(self, text_item):
-        tl = QTextLayout(text, text_item.font(), self.paintDevice())
+        get_table = partial(self.qt_hack.get_sfnt_table, text_item)
-        self.text_option.setTextDirection(Qt.RightToLeft if
+        ans = Font(Sfnt(get_table))
-            text_item.renderFlags() & text_item.RightToLeft else Qt.LeftToRight)
+        glyph_map = self.qt_hack.get_glyph_map(text_item)
-        tl.setTextOption(self.text_option)
+        gm = {}
-        return tl
+        for uc, glyph_id in enumerate(glyph_map):
-
+            if glyph_id not in gm:
-    def update_glyph_map(self, text, indices, text_item, glyph_map):
+                gm[glyph_id] = unichr(uc)
-        '''
+        ans.full_glyph_map = gm
-        Map glyphs back to the unicode text they represent.
-        '''
-        pos = 0
-        tl = self.get_text_layout(text_item, '')
-        indices = list(indices)
-
-        def get_glyphs(string):
-            tl.setText(string)
-            tl.beginLayout()
-            line = tl.createLine()
-            if not line.isValid():
-                tl.endLayout()
-                return []
-            line.setLineWidth(int(1e12))
-            tl.endLayout()
-            ans = []
-            for run in tl.glyphRuns():
-                ans.extend(run.glyphIndexes())
         return ans
 
-        ipos = 0
-        while ipos < len(indices):
-            if indices[ipos] in glyph_map:
-                t = glyph_map[indices[ipos]]
-                if t == text[pos:pos+len(t)]:
-                    pos += len(t)
-                    ipos += 1
-                    continue
-
-            found = False
-            for l in xrange(1, 10):
-                string = text[pos:pos+l]
-                g = get_glyphs(string)
-                if g and g[0] == indices[ipos]:
-                    found = True
-                    glyph_map[g[0]] = string
-                    break
-            if not found:
-                self.debug(
-                    'Failed to find glyph->unicode mapping for text: %s'%text)
-                break
-            ipos += 1
-            pos += l
-
-        return text[pos:]
-
     @store_error
     def drawTextItem(self, point, text_item):
         # super(PdfEngine, self).drawTextItem(point, text_item)
         self.graphics_state(self)
-        text = type(u'')(text_item.text()).replace('\n', ' ')
+        gi = self.qt_hack.get_glyphs(point, text_item)
-        text = unicodedata.normalize('NFKC', text)
+        if not gi.indices:
-        tl = self.get_text_layout(text_item, text)
+            sip.delete(gi)
-        tl.setPosition(point)
-        tl.beginLayout()
-        line = tl.createLine()
-        if not line.isValid():
-            tl.endLayout()
             return
-        line.setLineWidth(int(1e12))
+        name = hash(bytes(gi.name))
-        tl.endLayout()
-        for run in tl.glyphRuns():
-            rf = run.rawFont()
-            name = hash(bytes(rf.fontTable('name')))
         if name not in self.fonts:
-                self.fonts[name] = Font(Sfnt(rf))
+            self.fonts[name] = self.create_sfnt(text_item)
         metrics = self.fonts[name]
-            indices = run.glyphIndexes()
+        for glyph_id in gi.indices:
-            text = self.update_glyph_map(text, indices, text_item, metrics.glyph_map)
+            try:
+                metrics.glyph_map[glyph_id] = metrics.full_glyph_map[glyph_id]
+            except (KeyError, ValueError):
+                pass
         glyphs = []
         pdf_pos = point
         first_baseline = None
-            for i, pos in enumerate(run.positions()):
+        for i, pos in enumerate(gi.positions):
             if first_baseline is None:
                 first_baseline = pos.y()
-                glyph_pos = point + pos
+            glyph_pos = pos
             delta = glyph_pos - pdf_pos
-                glyphs.append((delta.x(), pos.y()-first_baseline, indices[i]))
+            glyphs.append((delta.x(), pos.y()-first_baseline, gi.indices[i]))
             pdf_pos = glyph_pos
 
         self.pdf.draw_glyph_run([1, 0, 0, -1, point.x(),
-                point.y()], rf.pixelSize(), metrics, glyphs)
+            point.y()], gi.size, metrics, glyphs)
-
+        sip.delete(gi)
 
     @store_error
     def drawPolygon(self, points, mode):
@@ -645,12 +598,12 @@ if __name__ == '__main__':
             # f.setUnderline(True)
             # f.setOverline(True)
             # f.setStrikeOut(True)
-            f.setFamily('DejaVu Sans')
+            f.setFamily('Calibri')
             p.setFont(f)
             # p.setPen(QColor(0, 0, 255))
             # p.scale(2, 2)
             # p.rotate(45)
-            p.drawText(QPoint(0, 300), 'Some—text not By’s ū --- Д AV ff ff')
+            p.drawText(QPoint(300, 300), 'Some—text not By’s ū --- Д AV ff ff')
         finally:
             p.end()
         if dev.engine.errors_occurred:

src/calibre/ebooks/pdf/render/qt_hack.cpp

@@ -0,0 +1,66 @@
+/*
+ * qt_hack.cpp
+ * Copyright (C) 2012 Kovid Goyal <kovid at kovidgoyal.net>
+ *
+ * Distributed under terms of the GPL3 license.
+ */
+
+#include "qt_hack.h"
+
+#include <QtEndian>
+
+#include "private/qtextengine_p.h"
+#include "private/qfontengine_p.h"
+
+GlyphInfo* get_glyphs(QPointF &p, const QTextItem &text_item) {
+    QTextItemInt ti = static_cast<const QTextItemInt &>(text_item);
+    QFontEngine *fe = ti.fontEngine;
+    qreal size = ti.fontEngine->fontDef.pixelSize;
+#ifdef Q_WS_WIN
+    if (ti.fontEngine->type() == QFontEngine::Win) {
+        QFontEngineWin *fe = static_cast<QFontEngineWin *>(ti.fontEngine);
+        size = fe->tm.tmHeight;
+    }
+#endif
+    QVarLengthArray<glyph_t> glyphs;
+    QVarLengthArray<QFixedPoint> positions;
+    QTransform m = QTransform::fromTranslate(p.x(), p.y());
+    fe->getGlyphPositions(ti.glyphs, m, ti.flags, glyphs, positions);
+    QVector<QPointF> points = QVector<QPointF>(positions.count());
+    for (int i = 0; i < positions.count(); i++) {
+        points[i].setX(positions[i].x.toReal());
+        points[i].setY(positions[i].y.toReal());
+    }
+
+    QVector<unsigned int> indices = QVector<unsigned int>(glyphs.count());
+    for (int i = 0; i < glyphs.count(); i++)
+        indices[i] = (unsigned int)glyphs[i];
+
+    const quint32 *tag = reinterpret_cast<const quint32 *>("name");
+
+    return new GlyphInfo(fe->getSfntTable(qToBigEndian(*tag)), size, points, indices);
+}
+
+GlyphInfo::GlyphInfo(const QByteArray& name, qreal size, const QVector<QPointF> &positions, const QVector<unsigned int> &indices) :name(name), positions(positions), size(size), indices(indices) {
+}
+
+QByteArray get_sfnt_table(const QTextItem &text_item, const char* tag_name) {
+    QTextItemInt ti = static_cast<const QTextItemInt &>(text_item);
+    const quint32 *tag = reinterpret_cast<const quint32 *>(tag_name);
+    return ti.fontEngine->getSfntTable(qToBigEndian(*tag));
+}
+
+QVector<unsigned int>* get_glyph_map(const QTextItem &text_item) {
+    QTextItemInt ti = static_cast<const QTextItemInt &>(text_item);
+    QVector<unsigned int> *ans = new QVector<unsigned int>(0x10000);
+    QGlyphLayoutArray<10> glyphs;
+    int nglyphs = 10;
+
+    for (uint uc = 0; uc < 0x10000; ++uc) {
+        QChar ch(uc);
+        ti.fontEngine->stringToCMap(&ch, 1, &glyphs, &nglyphs, QTextEngine::GlyphIndicesOnly);
+        (*ans)[uc] = glyphs.glyphs[0];
+    }
+    return ans;
+}
+

src/calibre/ebooks/pdf/render/qt_hack.h

@@ -0,0 +1,34 @@
+/*
+ * qt_hack.h
+ * Copyright (C) 2012 Kovid Goyal <kovid at kovidgoyal.net>
+ *
+ * Distributed under terms of the GPL3 license.
+ */
+
+#pragma once
+
+#include <QGlyphRun>
+#include <QTextItem>
+#include <QPointF>
+
+
+class GlyphInfo {
+    public:
+        QByteArray name;
+        QVector<QPointF> positions;
+        qreal size;
+        QVector<unsigned int> indices;
+
+        GlyphInfo(const QByteArray &name, qreal size, const QVector<QPointF> &positions, const QVector<unsigned int> &indices);
+
+    private:
+        GlyphInfo(const GlyphInfo&);
+        GlyphInfo &operator=(const GlyphInfo&);
+};
+
+GlyphInfo* get_glyphs(QPointF &p, const QTextItem &text_item);
+
+QByteArray get_sfnt_table(const QTextItem &text_item, const char* tag_name);
+
+QVector<unsigned int>* get_glyph_map(const QTextItem &text_item);
+

src/calibre/ebooks/pdf/render/qt_hack.sip

@@ -0,0 +1,28 @@
+//Define the SIP wrapper to the qt_hack code
+//Author - Kovid Goyal <kovid@kovidgoyal.net>
+
+%Module(name=qt_hack, version=1)
+
+%Import QtCore/QtCoremod.sip
+%Import QtGui/QtGuimod.sip
+
+class GlyphInfo {
+%TypeHeaderCode
+#include <qt_hack.h>
+%End
+public:
+    QByteArray name;
+    qreal size;
+    QVector<QPointF> &positions;
+    QVector<unsigned int> indices;
+    GlyphInfo(const QByteArray &name, qreal size, const QVector<QPointF> &positions, const QVector<unsigned int> &indices);
+private:
+    GlyphInfo(const GlyphInfo& g);
+
+};
+
+GlyphInfo* get_glyphs(QPointF &p, const QTextItem &text_item);
+
+QByteArray get_sfnt_table(const QTextItem &text_item, const char* tag_name);
+
+QVector<unsigned int>* get_glyph_map(const QTextItem &text_item);

src/calibre/utils/fonts/sfnt/container.py

@@ -44,10 +44,10 @@ class Sfnt(object):
         b'post' : PostTable,
     }
 
-    def __init__(self, raw_or_qrawfont):
+    def __init__(self, raw_or_get_table):
         self.tables = {}
-        if isinstance(raw_or_qrawfont, bytes):
+        if isinstance(raw_or_get_table, bytes):
-            raw = raw_or_qrawfont
+            raw = raw_or_get_table
             self.sfnt_version = raw[:4]
             if self.sfnt_version not in {b'\x00\x01\x00\x00', b'OTTO', b'true',
                     b'type1'}:
@@ -62,7 +62,7 @@ class Sfnt(object):
                 b'VORG', b'EBDT', b'EBLC', b'EBSC', b'BASE', b'GSUB', b'GPOS',
                 b'GDEF', b'JSTF', b'gasp', b'hdmx', b'kern', b'LTSH', b'PCLT',
                 b'VDMX', b'vhea', b'vmtx', b'MATH'}:
-                table = bytes(raw_or_qrawfont.fontTable(table_tag))
+                table = bytes(raw_or_get_table(table_tag))
                 if table:
                     self.tables[table_tag] = self.TABLE_MAP.get(
                         table_tag, UnknownTable)(table)