Start work on a headless QPA plugin to allow Qt to be used without an X server in linux.

Useful for the various command line tools that require
Qt. For example conversion with PDF Output or MOBI output +
rasterization of SVG images. Or ebook-meta with rendering of first page
of EPUB as cover. Or downloading news using a WebKit browser as the
backend.

Makes it unnecessary to use xvfb and will hopefully end the pointless
bug reports from people that try to use calibre command line tools in
environments without X.

setup/extensions.py

@@ -25,7 +25,8 @@ py_lib_dir = os.path.join(sys.prefix, 'lib')
 
 class Extension(object):
 
-    def absolutize(self, paths):
+    @classmethod
+    def absolutize(cls, paths):
         return list(set([x if os.path.isabs(x) else os.path.join(SRC, x.replace('/',
             os.sep)) for x in paths]))
 
@@ -391,7 +392,7 @@ class Build(Command):
         ''')
 
     def add_options(self, parser):
-        choices = [e.name for e in extensions]+['all', 'style']
+        choices = [e.name for e in extensions]+['all', 'headless']
         parser.add_option('-1', '--only', choices=choices, default='all',
                 help=('Build only the named extension. Available: '+
                     ', '.join(choices)+'. Default:%default'))
@@ -419,10 +420,12 @@ class Build(Command):
                 os.makedirs(self.d(dest))
             self.info('\n####### Building extension', ext.name, '#'*7)
             self.build(ext, dest)
+        if opts.only in {'all', 'headless'}:
+            self.build_headless()
 
     def dest(self, ext):
         ex = '.pyd' if iswindows else '.so'
-        return os.path.join(SRC, 'calibre', 'plugins', ext.name)+ex
+        return os.path.join(SRC, 'calibre', 'plugins', getattr(ext, 'name', ext))+ex
 
     def inc_dirs_to_cflags(self, dirs):
         return ['-I'+x for x in dirs]
@@ -500,6 +503,46 @@ class Build(Command):
             print "Error while executing: %s\n" % (cmdline)
             raise
 
+    def build_headless(self):
+        if iswindows or isosx:
+            return  # Dont have headless operation on these platforms
+        self.info('\n####### Building headless QPA plugin', '#'*7)
+        a = Extension.absolutize
+        headers = a(['calibre/headless/headless_backingstore.h', 'calibre/headless/headless_integration.h'])
+        sources = a(['calibre/headless/main.cpp', 'calibre/headless/headless_backingstore.cpp', 'calibre/headless/headless_integration.cpp'])
+        others = a(['calibre/headless/headless.json'])
+        target = self.dest('headless')
+        if not self.newer(target, headers + sources + others):
+            return
+        pro = textwrap.dedent(
+        '''\
+            TARGET = headless
+            PLUGIN_TYPE = platforms
+            PLUGIN_CLASS_NAME = HeadlessIntegrationPlugin
+            load(qt_plugin)
+            QT += core-private gui-private platformsupport-private
+            HEADERS = {headers}
+            SOURCES = {sources}
+            OTHER_FILES = {others}
+            DESTDIR = {destdir}
+            CONFIG -= create_cmake  # Prevent qmake from generating a cmake build file which it puts in the calibre src directory
+            ''').format(
+                headers=' '.join(headers), sources=' '.join(sources), others=' '.join(others), destdir=self.d(target))
+        bdir = self.j(self.d(self.SRC), 'build', 'headless')
+        if not os.path.exists(bdir):
+            os.makedirs(bdir)
+        pf = self.j(bdir, 'headless.pro')
+        open(self.j(bdir, '.qmake.conf'), 'wb').close()
+        with open(pf, 'wb') as f:
+            f.write(pro.encode('utf-8'))
+        cwd = os.getcwd()
+        os.chdir(bdir)
+        try:
+            self.check_call([QMAKE] + [self.b(pf)])
+            self.check_call([make] + ['-j%d'%(cpu_count() or 1)])
+        finally:
+            os.chdir(cwd)
+
     def build_sip_files(self, ext, src_dir):
         sip_files = ext.sip_files
         ext.sip_files = []

src/calibre/gui2/__init__.py

@@ -875,11 +875,14 @@ def detach_gui():
 
 class Application(QApplication):
 
-    def __init__(self, args, force_calibre_style=False,
+    def __init__(self, args, force_calibre_style=False, override_program_name=None, headless=False):
-            override_program_name=None):
         self.file_event_hook = None
         if override_program_name:
             args = [override_program_name] + args[1:]
+        if headless:
+            if not args:
+                args = sys.argv[:1]
+            args.extend(['-platformpluginpath', sys.extensions_location, '-platform', 'headless'])
         qargs = [i.encode('utf-8') if isinstance(i, unicode) else i for i in args]
         self.pi = plugins['progress_indicator'][0]
         self.setup_styles(force_calibre_style)

src/calibre/headless/headless.json

@@ -0,0 +1,3 @@
+{
+    "Keys": [ "headless" ]
+}

src/calibre/headless/headless_backingstore.cpp

@@ -0,0 +1,51 @@
+#include "headless_backingstore.h"
+#include "headless_integration.h"
+#include "qscreen.h"
+#include <QtCore/qdebug.h>
+#include <qpa/qplatformscreen.h>
+#include <private/qguiapplication_p.h>
+
+QT_BEGIN_NAMESPACE
+
+HeadlessBackingStore::HeadlessBackingStore(QWindow *window)
+    : QPlatformBackingStore(window)
+    , mDebug(HeadlessIntegration::instance()->options() & HeadlessIntegration::DebugBackingStore)
+{
+    if (mDebug)
+        qDebug() << "HeadlessBackingStore::HeadlessBackingStore:" << (quintptr)this;
+}
+
+HeadlessBackingStore::~HeadlessBackingStore()
+{
+}
+
+QPaintDevice *HeadlessBackingStore::paintDevice()
+{
+    if (mDebug)
+        qDebug() << "HeadlessBackingStore::paintDevice";
+
+    return &mImage;
+}
+
+void HeadlessBackingStore::flush(QWindow *window, const QRegion &region, const QPoint &offset)
+{
+    Q_UNUSED(window);
+    Q_UNUSED(region);
+    Q_UNUSED(offset);
+
+    if (mDebug) {
+        static int c = 0;
+        QString filename = QString("output%1.png").arg(c++, 4, 10, QLatin1Char('0'));
+        qDebug() << "HeadlessBackingStore::flush() saving contents to" << filename.toLocal8Bit().constData();
+        mImage.save(filename);
+    }
+}
+
+void HeadlessBackingStore::resize(const QSize &size, const QRegion &)
+{
+    QImage::Format format = QGuiApplication::primaryScreen()->handle()->format();
+    if (mImage.size() != size)
+        mImage = QImage(size, format);
+}
+
+QT_END_NAMESPACE

src/calibre/headless/headless_backingstore.h

@@ -0,0 +1,25 @@
+#pragma once
+
+#include <qpa/qplatformbackingstore.h>
+#include <qpa/qplatformwindow.h>
+#include <QtGui/QImage>
+
+QT_BEGIN_NAMESPACE
+
+class HeadlessBackingStore : public QPlatformBackingStore
+{
+public:
+    HeadlessBackingStore(QWindow *window);
+    ~HeadlessBackingStore();
+
+    QPaintDevice *paintDevice();
+    void flush(QWindow *window, const QRegion &region, const QPoint &offset);
+    void resize(const QSize &size, const QRegion &staticContents);
+
+private:
+    QImage mImage;
+    const bool mDebug;
+};
+
+QT_END_NAMESPACE
+

src/calibre/headless/headless_integration.cpp

@@ -0,0 +1,105 @@
+#include "headless_integration.h"
+#include "headless_backingstore.h"
+#ifndef Q_OS_WIN
+#include <QtPlatformSupport/private/qgenericunixeventdispatcher_p.h>
+#else
+#include <QtCore/private/qeventdispatcher_win_p.h>
+#endif
+
+#include <QtGui/private/qpixmap_raster_p.h>
+#include <QtGui/private/qguiapplication_p.h>
+#include <qpa/qplatformwindow.h>
+#include <qpa/qplatformfontdatabase.h>
+
+QT_BEGIN_NAMESPACE
+
+static const char debugBackingStoreEnvironmentVariable[] = "QT_DEBUG_BACKINGSTORE";
+
+static inline unsigned parseOptions(const QStringList &paramList)
+{
+    unsigned options = 0;
+    foreach (const QString &param, paramList) {
+        if (param == QLatin1String("enable_fonts"))
+            options |= HeadlessIntegration::EnableFonts;
+    }
+    return options;
+}
+
+HeadlessIntegration::HeadlessIntegration(const QStringList &parameters)
+    : m_dummyFontDatabase(0)
+    , m_options(parseOptions(parameters))
+{
+    if (qEnvironmentVariableIsSet(debugBackingStoreEnvironmentVariable)
+        && qgetenv(debugBackingStoreEnvironmentVariable).toInt() > 0) {
+        m_options |= DebugBackingStore | EnableFonts;
+    }
+
+    HeadlessScreen *mPrimaryScreen = new HeadlessScreen();
+
+    mPrimaryScreen->mGeometry = QRect(0, 0, 240, 320);
+    mPrimaryScreen->mDepth = 32;
+    mPrimaryScreen->mFormat = QImage::Format_ARGB32_Premultiplied;
+
+    screenAdded(mPrimaryScreen);
+}
+
+HeadlessIntegration::~HeadlessIntegration()
+{
+    delete m_dummyFontDatabase;
+}
+
+bool HeadlessIntegration::hasCapability(QPlatformIntegration::Capability cap) const
+{
+    switch (cap) {
+    case ThreadedPixmaps: return true;
+    case MultipleWindows: return true;
+    default: return QPlatformIntegration::hasCapability(cap);
+    }
+}
+
+// Dummy font database that does not scan the fonts directory to be
+// used for command line tools like qmlplugindump that do not create windows
+// unless DebugBackingStore is activated.
+class DummyFontDatabase : public QPlatformFontDatabase
+{
+public:
+    virtual void populateFontDatabase() {}
+};
+
+QPlatformFontDatabase *HeadlessIntegration::fontDatabase() const
+{
+    if (m_options & EnableFonts)
+        return QPlatformIntegration::fontDatabase();
+    if (!m_dummyFontDatabase)
+        m_dummyFontDatabase = new DummyFontDatabase;
+    return m_dummyFontDatabase;
+}
+
+QPlatformWindow *HeadlessIntegration::createPlatformWindow(QWindow *window) const
+{
+    Q_UNUSED(window);
+    QPlatformWindow *w = new QPlatformWindow(window);
+    w->requestActivateWindow();
+    return w;
+}
+
+QPlatformBackingStore *HeadlessIntegration::createPlatformBackingStore(QWindow *window) const
+{
+    return new HeadlessBackingStore(window);
+}
+
+QAbstractEventDispatcher *HeadlessIntegration::createEventDispatcher() const
+{
+#ifdef Q_OS_WIN
+    return new QEventDispatcherWin32;
+#else
+    return createUnixEventDispatcher();
+#endif
+}
+
+HeadlessIntegration *HeadlessIntegration::instance()
+{
+    return static_cast<HeadlessIntegration *>(QGuiApplicationPrivate::platformIntegration());
+}
+
+QT_END_NAMESPACE

src/calibre/headless/headless_integration.h

@@ -0,0 +1,53 @@
+#pragma once
+
+#include <qpa/qplatformintegration.h>
+#include <qpa/qplatformscreen.h>
+
+QT_BEGIN_NAMESPACE
+
+class HeadlessScreen : public QPlatformScreen
+{
+public:
+    HeadlessScreen()
+        : mDepth(32), mFormat(QImage::Format_ARGB32_Premultiplied) {}
+
+    QRect geometry() const { return mGeometry; }
+    int depth() const { return mDepth; }
+    QImage::Format format() const { return mFormat; }
+
+public:
+    QRect mGeometry;
+    int mDepth;
+    QImage::Format mFormat;
+    QSize mPhysicalSize;
+};
+
+class HeadlessIntegration : public QPlatformIntegration
+{
+public:
+    enum Options { // Options to be passed on command line or determined from environment
+        DebugBackingStore = 0x1,
+        EnableFonts = 0x2
+    };
+
+    explicit HeadlessIntegration(const QStringList &parameters);
+    ~HeadlessIntegration();
+
+    bool hasCapability(QPlatformIntegration::Capability cap) const;
+    QPlatformFontDatabase *fontDatabase() const;
+
+    QPlatformWindow *createPlatformWindow(QWindow *window) const;
+    QPlatformBackingStore *createPlatformBackingStore(QWindow *window) const;
+    QAbstractEventDispatcher *createEventDispatcher() const;
+
+    unsigned options() const { return m_options; }
+
+    static HeadlessIntegration *instance();
+
+private:
+    mutable QPlatformFontDatabase *m_dummyFontDatabase;
+    unsigned m_options;
+};
+
+QT_END_NAMESPACE
+

src/calibre/headless/main.cpp

@@ -0,0 +1,24 @@
+#include <qpa/qplatformintegrationplugin.h>
+#include "headless_integration.h"
+
+QT_BEGIN_NAMESPACE
+
+class HeadlessIntegrationPlugin : public QPlatformIntegrationPlugin
+{
+    Q_OBJECT
+    Q_PLUGIN_METADATA(IID "org.qt-project.Qt.QPA.QPlatformIntegrationFactoryInterface.5.2" FILE "headless.json")
+public:
+    QPlatformIntegration *create(const QString&, const QStringList&);
+};
+
+QPlatformIntegration *HeadlessIntegrationPlugin::create(const QString& system, const QStringList& paramList)
+{
+    if (!system.compare(QLatin1String("headless"), Qt::CaseInsensitive))
+        return new HeadlessIntegration(paramList);
+
+    return 0;
+}
+
+QT_END_NAMESPACE
+
+#include "main.moc"

src/calibre/test_build.py

@@ -12,7 +12,7 @@ __docformat__ = 'restructuredtext en'
 Test a binary calibre build to ensure that all needed binary images/libraries have loaded.
 '''
 
-import cStringIO
+import cStringIO, os
 from calibre.constants import plugins, iswindows, islinux
 
 def test_dbus():
@@ -76,17 +76,23 @@ def test_apsw():
     print ('apsw OK!')
 
 def test_qt():
-    from PyQt5.Qt import (QDialog, QImageReader, QNetworkAccessManager)
+    from calibre.gui2 import Application
+    from PyQt5.Qt import (QImageReader, QNetworkAccessManager)
     from PyQt5.QtWebKitWidgets import QWebView
+    os.environ.pop('DISPLAY', None)
+    app = Application([], headless=islinux)
     fmts = set(map(unicode, QImageReader.supportedImageFormats()))
     testf = set(['jpg', 'png', 'mng', 'svg', 'ico', 'gif'])
     if testf.intersection(fmts) != testf:
         raise RuntimeError(
             "Qt doesn't seem to be able to load its image plugins")
-    QWebView, QDialog
+    QWebView()
+    del QWebView
     na = QNetworkAccessManager()
     if not hasattr(na, 'sslErrors'):
         raise RuntimeError('Qt not compiled with openssl')
+    del na
+    del app
     print ('Qt OK!')
 
 def test_imaging():