From 8da98e8ba40283be48fcfde7f1035854ccd150e0 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 26 Apr 2014 14:51:32 +0530 Subject: [PATCH] Start work on a headless QPA plugin to allow Qt to be used without an X server in linux. Useful for the various command line tools that require Qt. For example conversion with PDF Output or MOBI output + rasterization of SVG images. Or ebook-meta with rendering of first page of EPUB as cover. Or downloading news using a WebKit browser as the backend. Makes it unnecessary to use xvfb and will hopefully end the pointless bug reports from people that try to use calibre command line tools in environments without X. --- setup/extensions.py | 49 +++++++- src/calibre/gui2/__init__.py | 7 +- src/calibre/headless/.qmake.conf | 0 src/calibre/headless/headless.json | 3 + .../headless/headless_backingstore.cpp | 51 +++++++++ src/calibre/headless/headless_backingstore.h | 25 +++++ src/calibre/headless/headless_integration.cpp | 105 ++++++++++++++++++ src/calibre/headless/headless_integration.h | 53 +++++++++ src/calibre/headless/main.cpp | 24 ++++ src/calibre/test_build.py | 12 +- 10 files changed, 321 insertions(+), 8 deletions(-) create mode 100644 src/calibre/headless/.qmake.conf create mode 100644 src/calibre/headless/headless.json create mode 100644 src/calibre/headless/headless_backingstore.cpp create mode 100644 src/calibre/headless/headless_backingstore.h create mode 100644 src/calibre/headless/headless_integration.cpp create mode 100644 src/calibre/headless/headless_integration.h create mode 100644 src/calibre/headless/main.cpp diff --git a/setup/extensions.py b/setup/extensions.py index 73ad909ab5..02b7bdf327 100644 --- a/setup/extensions.py +++ b/setup/extensions.py @@ -25,7 +25,8 @@ py_lib_dir = os.path.join(sys.prefix, 'lib') class Extension(object): - def absolutize(self, paths): + @classmethod + def absolutize(cls, paths): return list(set([x if os.path.isabs(x) else os.path.join(SRC, x.replace('/', os.sep)) for x in paths])) @@ -391,7 +392,7 @@ class Build(Command): ''') def add_options(self, parser): - choices = [e.name for e in extensions]+['all', 'style'] + choices = [e.name for e in extensions]+['all', 'headless'] parser.add_option('-1', '--only', choices=choices, default='all', help=('Build only the named extension. Available: '+ ', '.join(choices)+'. Default:%default')) @@ -419,10 +420,12 @@ class Build(Command): os.makedirs(self.d(dest)) self.info('\n####### Building extension', ext.name, '#'*7) self.build(ext, dest) + if opts.only in {'all', 'headless'}: + self.build_headless() def dest(self, ext): ex = '.pyd' if iswindows else '.so' - return os.path.join(SRC, 'calibre', 'plugins', ext.name)+ex + return os.path.join(SRC, 'calibre', 'plugins', getattr(ext, 'name', ext))+ex def inc_dirs_to_cflags(self, dirs): return ['-I'+x for x in dirs] @@ -500,6 +503,46 @@ class Build(Command): print "Error while executing: %s\n" % (cmdline) raise + def build_headless(self): + if iswindows or isosx: + return # Dont have headless operation on these platforms + self.info('\n####### Building headless QPA plugin', '#'*7) + a = Extension.absolutize + headers = a(['calibre/headless/headless_backingstore.h', 'calibre/headless/headless_integration.h']) + sources = a(['calibre/headless/main.cpp', 'calibre/headless/headless_backingstore.cpp', 'calibre/headless/headless_integration.cpp']) + others = a(['calibre/headless/headless.json']) + target = self.dest('headless') + if not self.newer(target, headers + sources + others): + return + pro = textwrap.dedent( + '''\ + TARGET = headless + PLUGIN_TYPE = platforms + PLUGIN_CLASS_NAME = HeadlessIntegrationPlugin + load(qt_plugin) + QT += core-private gui-private platformsupport-private + HEADERS = {headers} + SOURCES = {sources} + OTHER_FILES = {others} + DESTDIR = {destdir} + CONFIG -= create_cmake # Prevent qmake from generating a cmake build file which it puts in the calibre src directory + ''').format( + headers=' '.join(headers), sources=' '.join(sources), others=' '.join(others), destdir=self.d(target)) + bdir = self.j(self.d(self.SRC), 'build', 'headless') + if not os.path.exists(bdir): + os.makedirs(bdir) + pf = self.j(bdir, 'headless.pro') + open(self.j(bdir, '.qmake.conf'), 'wb').close() + with open(pf, 'wb') as f: + f.write(pro.encode('utf-8')) + cwd = os.getcwd() + os.chdir(bdir) + try: + self.check_call([QMAKE] + [self.b(pf)]) + self.check_call([make] + ['-j%d'%(cpu_count() or 1)]) + finally: + os.chdir(cwd) + def build_sip_files(self, ext, src_dir): sip_files = ext.sip_files ext.sip_files = [] diff --git a/src/calibre/gui2/__init__.py b/src/calibre/gui2/__init__.py index 3da761fa19..99392a152a 100644 --- a/src/calibre/gui2/__init__.py +++ b/src/calibre/gui2/__init__.py @@ -875,11 +875,14 @@ def detach_gui(): class Application(QApplication): - def __init__(self, args, force_calibre_style=False, - override_program_name=None): + def __init__(self, args, force_calibre_style=False, override_program_name=None, headless=False): self.file_event_hook = None if override_program_name: args = [override_program_name] + args[1:] + if headless: + if not args: + args = sys.argv[:1] + args.extend(['-platformpluginpath', sys.extensions_location, '-platform', 'headless']) qargs = [i.encode('utf-8') if isinstance(i, unicode) else i for i in args] self.pi = plugins['progress_indicator'][0] self.setup_styles(force_calibre_style) diff --git a/src/calibre/headless/.qmake.conf b/src/calibre/headless/.qmake.conf new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/calibre/headless/headless.json b/src/calibre/headless/headless.json new file mode 100644 index 0000000000..d6e8663aa8 --- /dev/null +++ b/src/calibre/headless/headless.json @@ -0,0 +1,3 @@ +{ + "Keys": [ "headless" ] +} diff --git a/src/calibre/headless/headless_backingstore.cpp b/src/calibre/headless/headless_backingstore.cpp new file mode 100644 index 0000000000..d07e5681e3 --- /dev/null +++ b/src/calibre/headless/headless_backingstore.cpp @@ -0,0 +1,51 @@ +#include "headless_backingstore.h" +#include "headless_integration.h" +#include "qscreen.h" +#include +#include +#include + +QT_BEGIN_NAMESPACE + +HeadlessBackingStore::HeadlessBackingStore(QWindow *window) + : QPlatformBackingStore(window) + , mDebug(HeadlessIntegration::instance()->options() & HeadlessIntegration::DebugBackingStore) +{ + if (mDebug) + qDebug() << "HeadlessBackingStore::HeadlessBackingStore:" << (quintptr)this; +} + +HeadlessBackingStore::~HeadlessBackingStore() +{ +} + +QPaintDevice *HeadlessBackingStore::paintDevice() +{ + if (mDebug) + qDebug() << "HeadlessBackingStore::paintDevice"; + + return &mImage; +} + +void HeadlessBackingStore::flush(QWindow *window, const QRegion ®ion, const QPoint &offset) +{ + Q_UNUSED(window); + Q_UNUSED(region); + Q_UNUSED(offset); + + if (mDebug) { + static int c = 0; + QString filename = QString("output%1.png").arg(c++, 4, 10, QLatin1Char('0')); + qDebug() << "HeadlessBackingStore::flush() saving contents to" << filename.toLocal8Bit().constData(); + mImage.save(filename); + } +} + +void HeadlessBackingStore::resize(const QSize &size, const QRegion &) +{ + QImage::Format format = QGuiApplication::primaryScreen()->handle()->format(); + if (mImage.size() != size) + mImage = QImage(size, format); +} + +QT_END_NAMESPACE diff --git a/src/calibre/headless/headless_backingstore.h b/src/calibre/headless/headless_backingstore.h new file mode 100644 index 0000000000..68c37f2558 --- /dev/null +++ b/src/calibre/headless/headless_backingstore.h @@ -0,0 +1,25 @@ +#pragma once + +#include +#include +#include + +QT_BEGIN_NAMESPACE + +class HeadlessBackingStore : public QPlatformBackingStore +{ +public: + HeadlessBackingStore(QWindow *window); + ~HeadlessBackingStore(); + + QPaintDevice *paintDevice(); + void flush(QWindow *window, const QRegion ®ion, const QPoint &offset); + void resize(const QSize &size, const QRegion &staticContents); + +private: + QImage mImage; + const bool mDebug; +}; + +QT_END_NAMESPACE + diff --git a/src/calibre/headless/headless_integration.cpp b/src/calibre/headless/headless_integration.cpp new file mode 100644 index 0000000000..3fbed96d83 --- /dev/null +++ b/src/calibre/headless/headless_integration.cpp @@ -0,0 +1,105 @@ +#include "headless_integration.h" +#include "headless_backingstore.h" +#ifndef Q_OS_WIN +#include +#else +#include +#endif + +#include +#include +#include +#include + +QT_BEGIN_NAMESPACE + +static const char debugBackingStoreEnvironmentVariable[] = "QT_DEBUG_BACKINGSTORE"; + +static inline unsigned parseOptions(const QStringList ¶mList) +{ + unsigned options = 0; + foreach (const QString ¶m, paramList) { + if (param == QLatin1String("enable_fonts")) + options |= HeadlessIntegration::EnableFonts; + } + return options; +} + +HeadlessIntegration::HeadlessIntegration(const QStringList ¶meters) + : m_dummyFontDatabase(0) + , m_options(parseOptions(parameters)) +{ + if (qEnvironmentVariableIsSet(debugBackingStoreEnvironmentVariable) + && qgetenv(debugBackingStoreEnvironmentVariable).toInt() > 0) { + m_options |= DebugBackingStore | EnableFonts; + } + + HeadlessScreen *mPrimaryScreen = new HeadlessScreen(); + + mPrimaryScreen->mGeometry = QRect(0, 0, 240, 320); + mPrimaryScreen->mDepth = 32; + mPrimaryScreen->mFormat = QImage::Format_ARGB32_Premultiplied; + + screenAdded(mPrimaryScreen); +} + +HeadlessIntegration::~HeadlessIntegration() +{ + delete m_dummyFontDatabase; +} + +bool HeadlessIntegration::hasCapability(QPlatformIntegration::Capability cap) const +{ + switch (cap) { + case ThreadedPixmaps: return true; + case MultipleWindows: return true; + default: return QPlatformIntegration::hasCapability(cap); + } +} + +// Dummy font database that does not scan the fonts directory to be +// used for command line tools like qmlplugindump that do not create windows +// unless DebugBackingStore is activated. +class DummyFontDatabase : public QPlatformFontDatabase +{ +public: + virtual void populateFontDatabase() {} +}; + +QPlatformFontDatabase *HeadlessIntegration::fontDatabase() const +{ + if (m_options & EnableFonts) + return QPlatformIntegration::fontDatabase(); + if (!m_dummyFontDatabase) + m_dummyFontDatabase = new DummyFontDatabase; + return m_dummyFontDatabase; +} + +QPlatformWindow *HeadlessIntegration::createPlatformWindow(QWindow *window) const +{ + Q_UNUSED(window); + QPlatformWindow *w = new QPlatformWindow(window); + w->requestActivateWindow(); + return w; +} + +QPlatformBackingStore *HeadlessIntegration::createPlatformBackingStore(QWindow *window) const +{ + return new HeadlessBackingStore(window); +} + +QAbstractEventDispatcher *HeadlessIntegration::createEventDispatcher() const +{ +#ifdef Q_OS_WIN + return new QEventDispatcherWin32; +#else + return createUnixEventDispatcher(); +#endif +} + +HeadlessIntegration *HeadlessIntegration::instance() +{ + return static_cast(QGuiApplicationPrivate::platformIntegration()); +} + +QT_END_NAMESPACE diff --git a/src/calibre/headless/headless_integration.h b/src/calibre/headless/headless_integration.h new file mode 100644 index 0000000000..e504d41a5f --- /dev/null +++ b/src/calibre/headless/headless_integration.h @@ -0,0 +1,53 @@ +#pragma once + +#include +#include + +QT_BEGIN_NAMESPACE + +class HeadlessScreen : public QPlatformScreen +{ +public: + HeadlessScreen() + : mDepth(32), mFormat(QImage::Format_ARGB32_Premultiplied) {} + + QRect geometry() const { return mGeometry; } + int depth() const { return mDepth; } + QImage::Format format() const { return mFormat; } + +public: + QRect mGeometry; + int mDepth; + QImage::Format mFormat; + QSize mPhysicalSize; +}; + +class HeadlessIntegration : public QPlatformIntegration +{ +public: + enum Options { // Options to be passed on command line or determined from environment + DebugBackingStore = 0x1, + EnableFonts = 0x2 + }; + + explicit HeadlessIntegration(const QStringList ¶meters); + ~HeadlessIntegration(); + + bool hasCapability(QPlatformIntegration::Capability cap) const; + QPlatformFontDatabase *fontDatabase() const; + + QPlatformWindow *createPlatformWindow(QWindow *window) const; + QPlatformBackingStore *createPlatformBackingStore(QWindow *window) const; + QAbstractEventDispatcher *createEventDispatcher() const; + + unsigned options() const { return m_options; } + + static HeadlessIntegration *instance(); + +private: + mutable QPlatformFontDatabase *m_dummyFontDatabase; + unsigned m_options; +}; + +QT_END_NAMESPACE + diff --git a/src/calibre/headless/main.cpp b/src/calibre/headless/main.cpp new file mode 100644 index 0000000000..429dcba898 --- /dev/null +++ b/src/calibre/headless/main.cpp @@ -0,0 +1,24 @@ +#include +#include "headless_integration.h" + +QT_BEGIN_NAMESPACE + +class HeadlessIntegrationPlugin : public QPlatformIntegrationPlugin +{ + Q_OBJECT + Q_PLUGIN_METADATA(IID "org.qt-project.Qt.QPA.QPlatformIntegrationFactoryInterface.5.2" FILE "headless.json") +public: + QPlatformIntegration *create(const QString&, const QStringList&); +}; + +QPlatformIntegration *HeadlessIntegrationPlugin::create(const QString& system, const QStringList& paramList) +{ + if (!system.compare(QLatin1String("headless"), Qt::CaseInsensitive)) + return new HeadlessIntegration(paramList); + + return 0; +} + +QT_END_NAMESPACE + +#include "main.moc" diff --git a/src/calibre/test_build.py b/src/calibre/test_build.py index f55bfaf2d4..04679703d1 100644 --- a/src/calibre/test_build.py +++ b/src/calibre/test_build.py @@ -12,7 +12,7 @@ __docformat__ = 'restructuredtext en' Test a binary calibre build to ensure that all needed binary images/libraries have loaded. ''' -import cStringIO +import cStringIO, os from calibre.constants import plugins, iswindows, islinux def test_dbus(): @@ -76,17 +76,23 @@ def test_apsw(): print ('apsw OK!') def test_qt(): - from PyQt5.Qt import (QDialog, QImageReader, QNetworkAccessManager) + from calibre.gui2 import Application + from PyQt5.Qt import (QImageReader, QNetworkAccessManager) from PyQt5.QtWebKitWidgets import QWebView + os.environ.pop('DISPLAY', None) + app = Application([], headless=islinux) fmts = set(map(unicode, QImageReader.supportedImageFormats())) testf = set(['jpg', 'png', 'mng', 'svg', 'ico', 'gif']) if testf.intersection(fmts) != testf: raise RuntimeError( "Qt doesn't seem to be able to load its image plugins") - QWebView, QDialog + QWebView() + del QWebView na = QNetworkAccessManager() if not hasattr(na, 'sslErrors'): raise RuntimeError('Qt not compiled with openssl') + del na + del app print ('Qt OK!') def test_imaging():