Refactor application layout on windows

Now only the calibre executables are present in the top level directory
(which is added to PATH). No third party executables and no dlls.

This became necessary because the VS 2015 CRT is not manifest based, so
it would have to be placed in the top level dir in the old layout. Now
it can be put into the DLLs dir.

setup/installer/windows/freeze.py

@@ -72,11 +72,12 @@ class Win32Freeze(Command, WixMixIn):
         self.opts = opts
         self.src_root = self.d(self.SRC)
         self.base = self.j(self.d(self.SRC), 'build', 'winfrozen')
+        self.app_base = self.j(self.base, 'app')
         self.rc_template = self.j(self.d(self.a(__file__)), 'template.rc')
         self.py_ver = ''.join(map(str, sys.version_info[:2]))
-        self.lib_dir = self.j(self.base, 'Lib')
+        self.lib_dir = self.j(self.app_base, 'Lib')
-        self.pylib = self.j(self.base, 'pylib.zip')
+        self.pylib = self.j(self.app_base, 'pylib.zip')
-        self.dll_dir = self.j(self.base, 'DLLs')
+        self.dll_dir = self.j(self.app_base, 'DLLs')
         self.portable_base = self.j(self.d(self.base), 'Calibre Portable')
         self.obj_dir = self.j(self.src_root, 'build', 'launcher')
 
@@ -96,7 +97,8 @@ class Win32Freeze(Command, WixMixIn):
     def initbase(self):
         if self.e(self.base):
             shutil.rmtree(self.base)
-        os.makedirs(self.base)
+        os.makedirs(self.app_base)
+        os.mkdir(self.dll_dir)
 
     def add_plugins(self):
         self.info('Adding plugins...')
@@ -112,14 +114,14 @@ class Win32Freeze(Command, WixMixIn):
         # shutil.copytree(CRT, self.j(self.base, os.path.basename(CRT)))
 
         self.info('Adding resources...')
-        tgt = self.j(self.base, 'resources')
+        tgt = self.j(self.app_base, 'resources')
         if os.path.exists(tgt):
             shutil.rmtree(tgt)
         shutil.copytree(self.j(self.src_root, 'resources'), tgt)
 
         self.info('Adding Qt...')
-        shutil.copytree(os.path.join(self.python_base, 'DLLs') , self.dll_dir,
+        for x in glob.glob(os.path.join(self.python_base, 'DLLs', '*')):
-                ignore=shutil.ignore_patterns('msvc*.dll', 'Microsoft.*'))
+            shutil.copy2(x, self.dll_dir)
         for x in glob.glob(self.j(OPENSSL_DIR, 'bin', '*.dll')):
             shutil.copy2(x, self.dll_dir)
         for x in glob.glob(self.j(ICU_DIR, 'source', 'lib', '*.dll')):
@@ -208,7 +210,7 @@ class Win32Freeze(Command, WixMixIn):
         self.info('\nAdding Qt plugins...')
         qt_prefix = QT_DIR
         plugdir = self.j(qt_prefix, 'plugins')
-        tdir = self.j(self.base, 'qt_plugins')
+        tdir = self.j(self.app_base, 'qt_plugins')
         for d in QT_PLUGINS:
             self.info('\t', d)
             imfd = os.path.join(plugdir, d)
@@ -227,7 +229,7 @@ class Win32Freeze(Command, WixMixIn):
         self.info('\tAdding misc binary deps')
         bindir = os.path.join(SW, 'bin')
         for x in ('pdftohtml', 'pdfinfo', 'pdftoppm', 'jpegtran-calibre', 'cjpeg-calibre', 'optipng-calibre'):
-            shutil.copy2(os.path.join(bindir, x+'.exe'), self.base)
+            shutil.copy2(os.path.join(bindir, x+'.exe'), self.dll_dir)
         for pat in ('*.dll',):
             for f in glob.glob(os.path.join(bindir, pat)):
                 ok = True
@@ -545,7 +547,7 @@ class Win32Freeze(Command, WixMixIn):
                 ftype = '/T' + ('c' if src.endswith('.c') else 'p')
                 cmd = [msvc.cc] + cflags + ['/Fo'+obj, ftype + src]
                 self.run_builder(cmd, show_output=True)
-            exe = self.j(self.base, name)
+            exe = self.j(self.dll_dir, name)
             cmd = [msvc.linker] + ['/MACHINE:'+machine,
                     '/SUBSYSTEM:'+subsys, '/RELEASE',
                     '/OUT:'+exe] + [self.embed_resources(exe), obj] + libs
@@ -561,14 +563,13 @@ class Win32Freeze(Command, WixMixIn):
         dlflags = (['/DEBUG'] if debug else ['/INCREMENTAL:NO'])
         base = self.j(self.src_root, 'setup', 'installer', 'windows')
         sources = [self.j(base, x) for x in ['util.c',]]
-        headers = [self.j(base, x) for x in ['util.h',]]
         objects = [self.j(self.obj_dir, self.b(x)+'.obj') for x in sources]
-        cflags  = '/c /EHsc /MD /W3 /Ox /nologo /D_UNICODE'.split()
+        cflags  = '/c /EHsc /W3 /Ox /nologo /D_UNICODE'.split()
         cflags += ['/DPYDLL="python%s.dll"'%self.py_ver, '/I%s/include'%self.python_base]
         for src, obj in zip(sources, objects):
-            if not self.newer(obj, headers+[src]):
+            if not self.newer(obj, [src]):
                 continue
-            cmd = [msvc.cc] + cflags + dflags + ['/Fo'+obj, '/Tc'+src]
+            cmd = [msvc.cc] + cflags + dflags + ['/MD', '/Fo'+obj, '/Tc'+src]
             self.run_builder(cmd, show_output=True)
 
         dll = self.j(self.obj_dir, 'calibre-launcher.dll')
@@ -578,37 +579,39 @@ class Win32Freeze(Command, WixMixIn):
                    '/nologo', '/MACHINE:'+machine] + dlflags + objects + \
                 [self.embed_resources(dll),
                 '/LIBPATH:%s/libs'%self.python_base,
+                'delayimp.lib', 'user32.lib', 'shell32.lib',
                 'python%s.lib'%self.py_ver,
                 '/delayload:python%s.dll'%self.py_ver]
             self.info('Linking calibre-launcher.dll')
             self.run_builder(cmd, show_output=True)
 
         src = self.j(base, 'main.c')
-        shutil.copy2(dll, self.base)
+        shutil.copy2(dll, self.dll_dir)
         for typ in ('console', 'gui', ):
             self.info('Processing %s launchers'%typ)
             subsys = 'WINDOWS' if typ == 'gui' else 'CONSOLE'
             for mod, bname, func in zip(modules[typ], basenames[typ],
                     functions[typ]):
-                xflags = list(cflags)
+                xflags = list(cflags) + ['/MT']
                 if typ == 'gui':
                     xflags += ['/DGUI_APP=']
 
                 xflags += ['/DMODULE="%s"'%mod, '/DBASENAME="%s"'%bname,
                     '/DFUNCTION="%s"'%func]
                 dest = self.j(self.obj_dir, bname+'.obj')
-                if self.newer(dest, [src]+headers):
+                if self.newer(dest, [src]):
                     self.info('Compiling', bname)
                     cmd = [msvc.cc] + xflags + dflags + ['/Tc'+src, '/Fo'+dest]
                     self.run_builder(cmd)
                 exe = self.j(self.base, bname+'.exe')
                 lib = dll.replace('.dll', '.lib')
+                u32 = ['user32.lib'] if typ == 'gui' else []
                 if self.newer(exe, [dest, lib, self.rc_template, __file__]):
                     self.info('Linking', bname)
                     cmd = [msvc.linker] + ['/MACHINE:'+machine,
                             '/LIBPATH:'+self.obj_dir, '/SUBSYSTEM:'+subsys,
                             '/LIBPATH:%s/libs'%self.python_base, '/RELEASE',
-                            '/OUT:'+exe] + dlflags + [self.embed_resources(exe),
+                            '/OUT:'+exe] + u32 + dlflags + [self.embed_resources(exe),
                             dest, lib]
                     self.run_builder(cmd)
 

setup/installer/windows/main.c

@@ -2,33 +2,97 @@
  * Copyright 2009 Kovid Goyal
  */
 
-#include "util.h"
+#define UNICODE
+#define WINDOWS_LEAN_AND_MEAN
+#include<windows.h>
+#include<stdio.h>
+
+static int show_error(const wchar_t *preamble, const wchar_t *msg, const int code) {
+    wchar_t *buf;
+    buf = (wchar_t*)LocalAlloc(LMEM_ZEROINIT, sizeof(wchar_t)*
+            (wcslen(msg) + wcslen(preamble) + 80));
+
+    _snwprintf_s(buf, 
+        LocalSize(buf) / sizeof(wchar_t), _TRUNCATE,
+        L"%s\r\n  %s (Error Code: %d)\r\n", 
+        preamble, msg, code);
+
+#ifdef GUI_APP
+    MessageBeep(MB_ICONERROR);
+    MessageBox(NULL, buf, NULL, MB_OK|MB_ICONERROR);
+#else
+    wprintf_s(L"%s\n", buf);
+#endif
+    LocalFree(buf);
+    return code;
+}
+
+int show_last_error(wchar_t *preamble) {
+    wchar_t *msg = NULL;
+    DWORD dw = GetLastError(); 
+    int ret;
+
+    FormatMessage(
+        FORMAT_MESSAGE_ALLOCATE_BUFFER | 
+        FORMAT_MESSAGE_FROM_SYSTEM |
+        FORMAT_MESSAGE_IGNORE_INSERTS,
+        NULL,
+        dw,
+        MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),
+        (LPWSTR)&msg, 
+        0,
+        NULL );
+
+    ret = show_error(preamble, msg, (int)dw);
+    if (msg != NULL) LocalFree(msg);
+    return ret;
+}
+
+typedef int (__cdecl *ENTRYPROC)(const char*, const char*, const char*, int); 
+
+static ENTRYPROC load_launcher_dll() {
+    wchar_t buf[MAX_PATH] = {0};
+    wchar_t drive[4] = L"\0\0\0";
+    DWORD sz; 
+    HMODULE dll;
+    ENTRYPROC entrypoint;
+
+    if ((sz = GetModuleFileNameW(NULL, buf, MAX_PATH)) >= MAX_PATH - 30)
+        ExitProcess(show_error(L"Installation directory path too long", L"", 1));
+
+    while (sz > 0) {
+        if (buf[sz] == L'\\' || buf[sz] == L'/') break;
+        sz--;
+    }
+    if (sz <= 0)
+        ExitProcess(show_error(L"Executable path has no path separators", L"", 1));
+    buf[sz+1] = L'a'; buf[sz+2] = L'p'; buf[sz+3] = L'p'; buf[sz+4] = L'\\';
+    buf[sz+5] = L'D'; buf[sz+6] = L'L'; buf[sz+7] = L'L'; buf[sz+8] = L's';
+    buf[sz+9] = 0; buf[sz+10] = 0;
+    if (SetDllDirectoryW(buf) == 0) {
+        show_last_error(L"Failed to set DLL directory");
+        ExitProcess(1);
+    }
+    dll = LoadLibraryW(L"calibre-launcher.dll");
+    if (!dll) ExitProcess(show_last_error(L"Failed to get the calibre-launcher dll handle"));
+    entrypoint = (ENTRYPROC) GetProcAddress(dll, "execute_python_entrypoint");
+    if (!entrypoint) ExitProcess(show_last_error(L"Failed to get the calibre-launcher dll entry point"));
+    return entrypoint;
+}
 
 #ifdef GUI_APP
 
 int WINAPI                                                                                                      
 wWinMain(HINSTANCE Inst, HINSTANCE PrevInst, wchar_t *CmdLine, int CmdShow) {
-    set_gui_app((char)1);
+    load_launcher_dll()(BASENAME, MODULE, FUNCTION, 1);
-
-    // Redirect stdout and stderr to NUL so that python does not fail writing to them
-    redirect_out_stream(stdout);
-    redirect_out_stream(stderr);
-
-	execute_python_entrypoint(BASENAME, MODULE, FUNCTION);
-
 
     return 0; // This should really be returning the value set in the WM_QUIT message, but I cannot be bothered figuring out how to get that.
 }
 
 #else
 
-
 int wmain(int argc, wchar_t *argv) {
-    int ret = 0;
+    return load_launcher_dll()(BASENAME, MODULE, FUNCTION, 0);
-    set_gui_app((char)0);
-	ret = execute_python_entrypoint(BASENAME, MODULE, FUNCTION);
-
-    return ret;
 }
 
 #endif

setup/installer/windows/site.py

@@ -19,7 +19,7 @@ class PydImporter(object):
 
     def find_module(self, fullname, path=None):
         if self.items is None:
-            dlls_dir = os.path.join(sys.app_dir, 'DLLs')
+            dlls_dir = os.path.join(sys.app_dir, 'app', 'DLLs')
             items = self.items = {}
             for x in os.listdir(dlls_dir):
                 lx = x.lower()
@@ -63,8 +63,9 @@ def aliasmbcs():
             encodings.aliases.aliases[enc] = 'mbcs'
 
 def add_calibre_vars():
-    sys.resources_location = os.path.join(sys.app_dir, 'resources')
+    sys.new_app_layout = 1
-    sys.extensions_location = os.path.join(sys.app_dir, 'DLLs')
+    sys.resources_location = os.path.join(sys.app_dir, 'app', 'resources')
+    sys.extensions_location = os.path.join(sys.app_dir, 'app', 'DLLs')
 
     dv = os.environ.get('CALIBRE_DEVELOP_FROM', None)
     if dv and os.path.exists(dv):
@@ -95,6 +96,6 @@ def main():
     add_calibre_vars()
 
     # Needed for pywintypes to be able to load its DLL
-    sys.path.append(os.path.join(sys.app_dir, 'DLLs'))
+    sys.path.append(os.path.join(sys.app_dir, 'app', 'DLLs'))
 
     return run_entry_point()

setup/installer/windows/util.c

@@ -2,18 +2,25 @@
  * Copyright 2009 Kovid Goyal
  */
 
-#include "util.h"
+#define UNICODE
 
+#define _WIN32_WINNT 0x0502 
+#define WINDOWS_LEAN_AND_MEAN
+
+#include <windows.h>
+#include <Python.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <Shellapi.h>
 #include <delayimp.h>
 #include <io.h>
 #include <fcntl.h>
 
 
-static char GUI_APP = 0;
+static int GUI_APP = 0;
 static char python_dll[] = PYDLL;
 
-void set_gui_app(char yes) { GUI_APP = yes; }
+void set_gui_app(int yes) { GUI_APP = yes; }
-char is_gui_app() { return GUI_APP; }
 
 int calibre_show_python_error(const wchar_t *preamble, int code);
 
@@ -118,12 +125,12 @@ void load_python_dll() {
     size_t l;
 
     app_dir = get_app_dir();
-    l = strlen(app_dir)+20;
+    l = strlen(app_dir)+25;
     dll_dir = (char*) calloc(l, sizeof(char));
     qt_plugin_dir = (char*) calloc(l, sizeof(char));
     if (!dll_dir || !qt_plugin_dir) ExitProcess(_show_error(L"Out of memory", L"", 1));
-    _snprintf_s(dll_dir, l, _TRUNCATE, "%sDLLs", app_dir);
+    _snprintf_s(dll_dir, l, _TRUNCATE, "%s\\app\\DLLs", app_dir);
-    _snprintf_s(qt_plugin_dir, l, _TRUNCATE, "%sqt_plugins", app_dir);
+    _snprintf_s(qt_plugin_dir, l, _TRUNCATE, "%s\\app\\qt_plugins", app_dir);
     free(app_dir);
 
     _putenv_s("QT_PLUGIN_PATH", qt_plugin_dir);
@@ -196,7 +203,7 @@ void initialize_interpreter(const char *basename, const char *module, const char
     buf[strlen(buf)-1] = '\0';
 
     _snprintf_s(python_home, MAX_PATH, _TRUNCATE, "%s", buf);
-    _snprintf_s(path, MAX_PATH, _TRUNCATE, "%s\\pylib.zip", buf);
+    _snprintf_s(path, MAX_PATH, _TRUNCATE, "%s\\app\\pylib.zip", buf);
     free(buf);
 
 
@@ -337,10 +344,28 @@ int calibre_show_python_error(const wchar_t *preamble, int code) {
     return _show_error(preamble, L"", code);
 }
 
-int execute_python_entrypoint(const char *basename, const char *module, const char *function) {
+void redirect_out_stream(FILE *stream) {
+    FILE *f = NULL;
+    errno_t err;
+
+    err = freopen_s(&f, "NUL", "wt", stream);
+    if (err != 0) {
+        ExitProcess(show_last_error_crt(L"Failed to redirect stdout/stderr to NUL. This indicates a corrupted Windows install.\r\n You should contact Microsoft for assistance and/or follow the steps described here:\r\n http://bytes.com/topic/net/answers/264804-compile-error-null-device-missing"));
+    }
+}
+
+__declspec(dllexport) int __cdecl
+execute_python_entrypoint(const char *basename, const char *module, const char *function, int is_gui_app) {
     PyObject *site, *main, *res;
     int ret = 0;
 
+    if (is_gui_app) {
+        // Redirect stdout and stderr to NUL so that python does not fail writing to them
+        redirect_out_stream(stdout);
+        redirect_out_stream(stderr);
+    }
+    set_gui_app(is_gui_app);
+
     load_python_dll();
     initialize_interpreter(basename, module, function);
 
@@ -397,12 +422,4 @@ wchar_t* get_temp_filename(const wchar_t *prefix) {
      return szTempName;
 }
 
-void redirect_out_stream(FILE *stream) {
-    FILE *f = NULL;
-    errno_t err;
 
-    err = freopen_s(&f, "NUL", "wt", stream);
-    if (err != 0) {
-        ExitProcess(show_last_error_crt(L"Failed to redirect stdout/stderr to NUL. This indicates a corrupted Windows install.\r\n You should contact Microsoft for assistance and/or follow the steps described here:\r\n http://bytes.com/topic/net/answers/264804-compile-error-null-device-missing"));
-    }
-}

setup/installer/windows/util.h

@@ -1,45 +0,0 @@
-/*
- * Copyright 2009 Kovid Goyal
- */
-
-#pragma once
-
-#ifndef _UNICODE
-#define _UNICODE
-#endif
-
-#ifndef UNICODE
-#define UNICODE
-#endif
-
-#define _WIN32_WINNT 0x0502 
-
-#include <windows.h>
-#ifdef _DLL
-#   include <Python.h>
-#endif
-#include <stdio.h>
-#include <stdlib.h>
-#include <Shellapi.h>
-
-#define DllExport __declspec( dllexport )
-#define DllImport __declspec( dllimport )
-
-#ifdef _DLL
-#   define ExIm DllExport
-#   pragma comment(lib, "delayimp")
-#   pragma comment(lib, "user32")
-#   pragma comment(lib, "shell32")
-#else
-#   define ExIm DllImport
-#endif
-
-ExIm void set_gui_app(char yes);
-ExIm char is_gui_app();
-
-// Redirect output streams to NUL
-ExIm void redirect_out_stream(FILE *stream);
-
-// Execute python entry point defined by: module and function
-ExIm int execute_python_entrypoint(const char *basename, const char *module, const char *function);
-