Windows: Workaround for explorer shell extensions on windows that write to stdout

Use a pipe for reading output from the worker process instead of stdout.
Apparently there exist shell extensions that write to stdout. Le
bubbling sigh.

setup/installer/windows/file_dialogs.cpp

@@ -18,15 +18,15 @@
 
 #define PRINTERR(x) fprintf(stderr, "%s", x); fflush(stderr);
 
-bool write_bytes(size_t sz, const char* buf) {
+bool write_bytes(HANDLE pipe, DWORD sz, const char* buf) {
-    size_t num = 0;
+	DWORD written = 0;
-    while(sz > 0 && !feof(stdout) && !ferror(stdout)) {
+	if (!WriteFile(pipe, buf, sz, &written, NULL)) {
-        num = fwrite(buf, sizeof(char), sz, stdout);
+		fprintf(stderr, "Failed to write to pipe. GetLastError()=%d\n", GetLastError()); fflush(stderr); return false;
-        if (num == 0) break;
-        buf += num; sz -= num;
 	}
-    if (sz > 0) PRINTERR("Failed to write to stdout");
+	if (written != sz) {
-    return sz == 0;
+		fprintf(stderr, "Failed to write to pipe. Incomplete write, leftover bytes: %d", sz - written); fflush(stderr); return false;
+	}
+	return true;
 }
 
 bool read_bytes(size_t sz, char* buf, bool allow_incomplete=false) {
@@ -50,6 +50,7 @@ bool from_utf8(size_t sz, const char *src, LPWSTR* ans) {
 }
 
 char* to_utf8(LPCWSTR src, int *sz) {
+	// Convert to a null-terminated UTF-8 encoded bytearray, allocated on the heap
     char *ans = NULL;
     *sz = WideCharToMultiByte(CP_UTF8, WC_ERR_INVALID_CHARS, src, -1, NULL, 0, NULL, NULL);
 	if (!*sz) { PRINTERR("Failed to get size of UTF-16 string"); return NULL; }
@@ -101,7 +102,7 @@ static void print_com_error(HRESULT hr, const char *msg) {
 #define REPORTERR(hr, x) { print_com_error(hr, x); ret = 1; goto error; }
 #define CALLCOM(x, err) hr = x; if(FAILED(hr)) REPORTERR(hr, err)
 
-int show_dialog(HWND parent, bool save_dialog, LPWSTR title, LPWSTR folder, LPWSTR filename, LPWSTR save_path, bool multiselect, bool confirm_overwrite, bool only_dirs, bool no_symlinks, COMDLG_FILTERSPEC *file_types, UINT num_file_types) {
+int show_dialog(HANDLE pipe, HWND parent, bool save_dialog, LPWSTR title, LPWSTR folder, LPWSTR filename, LPWSTR save_path, bool multiselect, bool confirm_overwrite, bool only_dirs, bool no_symlinks, COMDLG_FILTERSPEC *file_types, UINT num_file_types) {
 	int ret = 0, name_sz = 0;
 	IFileDialog *pfd = NULL;
 	IShellItemArray *items = NULL;
@@ -154,7 +155,7 @@ int show_dialog(HWND parent, bool save_dialog, LPWSTR title, LPWSTR folder, LPWS
         path = to_utf8(name, &name_sz);
         CoTaskMemFree(name); name = NULL;
         if (path == NULL) return 1;
-        if (!write_bytes(name_sz, path)) return 1;
+        if (!write_bytes(pipe, name_sz, path)) return 1;
     } else {
         CALLCOM(((IFileOpenDialog*)pfd)->GetResults(&items), "Failed to get dialog results");
         CALLCOM(items->GetCount(&item_count), "Failed to get count of results");
@@ -165,7 +166,7 @@ int show_dialog(HWND parent, bool save_dialog, LPWSTR title, LPWSTR folder, LPWS
                     path = to_utf8(name, &name_sz);
                     CoTaskMemFree(name); name = NULL;
                     if (path == NULL) return 1;
-                    if (!write_bytes(name_sz, path)) return 1;
+                    if (!write_bytes(pipe, name_sz, path)) return 1;
                 }
             }
         }
@@ -182,15 +183,31 @@ error:
 #define SETBINARY(x) if(_setmode(_fileno(x), _O_BINARY) == -1) { PRINTERR("Failed to set binary mode"); return 1; }
 #define READBOOL(x)  READ(1, buf); x = !!buf[0]; 
 
+HANDLE open_named_pipe(LPWSTR pipename) {
+	HANDLE ans = INVALID_HANDLE_VALUE;
+	while(true) {
+		ans = CreateFileW(pipename, GENERIC_WRITE, 0, NULL, OPEN_EXISTING, 0, NULL);
+		if (ans != INVALID_HANDLE_VALUE) break;
+		if (GetLastError() != ERROR_PIPE_BUSY) {
+			fprintf(stderr, "Failed to open pipe. GetLastError()=%d\n", GetLastError()); fflush(stderr); return ans;
+		}
+		if (!WaitNamedPipeW(pipename, 20000)) {
+			fprintf(stderr, "Failed to open pipe. 20 second wait timed out.\n", GetLastError()); fflush(stderr); return ans;
+		}
+	}
+	return ans;
+}
+
 int WINAPI wWinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, PWSTR pCmdLine, int nCmdShow) {
 	char buf[257] = {0};
 	size_t key_size = 0;
 	HWND parent = NULL;
 	bool save_dialog = false, multiselect = false, confirm_overwrite = false, only_dirs = false, no_symlinks = false;
 	unsigned short len = 0;
-	LPWSTR title = NULL, folder = NULL, filename = NULL, save_path = NULL, echo = NULL;
+	LPWSTR title = NULL, folder = NULL, filename = NULL, save_path = NULL, echo = NULL, pipename = NULL;
     COMDLG_FILTERSPEC *file_types = NULL;
     UINT num_file_types = 0;
+	HANDLE pipe = INVALID_HANDLE_VALUE;
 
 	SETBINARY(stdout); SETBINARY(stdin); SETBINARY(stderr);
 	// The calibre executables call SetDllDirectory, we unset it here just in
@@ -211,6 +228,8 @@ int WINAPI wWinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, PWSTR pCmdLine
 			else { fprintf(stderr, "Unknown pointer size: %zd", sizeof(HWND)); fflush(stderr); return 1;}
 		}
 
+		else if CHECK_KEY("PIPENAME") { READSTR(pipename); pipe = open_named_pipe(pipename); if (pipe == INVALID_HANDLE_VALUE) return 1; }
+
 		else if CHECK_KEY("TITLE") { READSTR(title) }
 
 		else if CHECK_KEY("FOLDER") { READSTR(folder) }
@@ -239,12 +258,13 @@ int WINAPI wWinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, PWSTR pCmdLine
 		}
 	}
 
+	if (pipe == INVALID_HANDLE_VALUE) { PRINTERR("No pipename received"); return 1; }
+
     if (echo != NULL) { 
         int echo_sz = 0;
         char *echo_buf = to_utf8(echo, &echo_sz);
-        fprintf(stdout, "%s", echo_buf); 
+		return write_bytes(pipe, echo_sz, echo_buf) ? 0 : 1;
-        return 0; 
     }
 
-	return show_dialog(parent, save_dialog, title, folder, filename, save_path, multiselect, confirm_overwrite, only_dirs, no_symlinks, file_types, num_file_types);
+	return show_dialog(pipe, parent, save_dialog, title, folder, filename, save_path, multiselect, confirm_overwrite, only_dirs, no_symlinks, file_types, num_file_types);
 }

setup/installer/windows/freeze.py

@@ -41,7 +41,7 @@ DESCRIPTIONS = {
         'calibre-parallel': 'calibre worker process',
         'calibre-smtp' : 'Command line interface for sending books via email',
         'calibre-eject' : 'Helper program for ejecting connected reader devices',
-        'calibre-file-dialogs' : 'Helper program to show file open/save dialogs',
+        'calibre-file-dialog' : 'Helper program to show file open/save dialogs',
 }
 
 def walk(dir):
@@ -595,7 +595,7 @@ class Win32Freeze(Command, WixMixIn):
                     '/OUT:'+exe] + [self.embed_resources(exe), obj] + libs
             self.run_builder(cmd)
         base = self.j(self.src_root, 'setup', 'installer', 'windows')
-        build(self.j(base, 'file_dialogs.cpp'), 'calibre-file-dialogs.exe', 'WINDOWS', 'Ole32.lib Shell32.lib'.split())
+        build(self.j(base, 'file_dialogs.cpp'), 'calibre-file-dialog.exe', 'WINDOWS', 'Ole32.lib Shell32.lib'.split())
         build(self.j(base, 'eject.c'), 'calibre-eject.exe')
 
     def build_launchers(self, debug=False):

src/calibre/gui2/win_file_dialogs.py

@@ -6,12 +6,13 @@ from __future__ import (unicode_literals, division, absolute_import,
                         print_function)
 import sys, subprocess, struct, os
 from threading import Thread
+from uuid import uuid4
 
 from PyQt5.Qt import pyqtSignal, QEventLoop, Qt
 
 is64bit = sys.maxsize > (1 << 32)
 base = sys.extensions_location if hasattr(sys, 'new_app_layout') else os.path.dirname(sys.executable)
-HELPER = os.path.join(base, 'calibre-file-dialogs.exe')
+HELPER = os.path.join(base, 'calibre-file-dialog.exe')
 
 def is_ok():
     return os.path.exists(HELPER)
@@ -109,7 +110,8 @@ def run_file_dialog(
     from calibre.gui2 import sanitize_env_vars
     with sanitize_env_vars():
         env = os.environ.copy()
-    data = []
+    pipename = '\\\\.\\pipe\\%s' % uuid4()
+    data = [serialize_string('PIPENAME', pipename)]
     parent = parent or None
     if parent is not None:
         data.append(serialize_hwnd(get_hwnd(parent)))
@@ -150,16 +152,30 @@ def run_file_dialog(
     if file_types:
         data.append(serialize_file_types(file_types))
     loop = Loop()
+    server = PipeServer(pipename)
     h = Helper(subprocess.Popen(
         [HELPER], stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=subprocess.PIPE, env=env),
                data, loop.dialog_closed.emit)
     h.start()
     loop.exec_(QEventLoop.ExcludeUserInputEvents)
+    def decode(x):
+        x = x or b''
+        try:
+            x = x.decode('utf-8')
+        except Exception:
+            x = repr(x)
+        return x
+
     if h.rc != 0:
-        raise Exception('File dialog failed: ' + h.stderrdata.decode('utf-8'))
+        raise Exception('File dialog failed: ' + decode(h.stdoutdata) + ' ' + decode(h.stderrdata))
-    if not h.stdoutdata:
+    server.join(2)
+    if server.is_alive():
+        raise Exception('Timed out waiting for read from pipe to complete')
+    if server.err_msg:
+        raise Exception(server.err_msg)
+    if not server.data:
         return ()
-    ans = tuple((os.path.abspath(x.decode('utf-8')) for x in h.stdoutdata.split(b'\0') if x))
+    ans = tuple((os.path.abspath(x.decode('utf-8')) for x in server.data.split(b'\0') if x))
     return ans
 
 def get_initial_folder(name, title, default_dir='~', no_save_dir=False):
@@ -217,11 +233,74 @@ def choose_save_file(window, name, title, filters=[], all_files=True, initial_pa
             dynamic.set(name, ans)
         return ans
 
-def test():
+class PipeServer(Thread):
-    p = subprocess.Popen([HELPER], stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=subprocess.PIPE)
+
+    def __init__(self, pipename):
+        Thread.__init__(self, name='PipeServer')
+        self.daemon = True
+        import win32pipe, win32api, win32con
+        FILE_FLAG_FIRST_PIPE_INSTANCE = 0x00080000
+        PIPE_REJECT_REMOTE_CLIENTS = 0x00000008
+        self.pipe_handle = win32pipe.CreateNamedPipe(
+            pipename, win32pipe.PIPE_ACCESS_INBOUND | FILE_FLAG_FIRST_PIPE_INSTANCE,
+            win32pipe.PIPE_TYPE_BYTE | win32pipe.PIPE_READMODE_BYTE | win32pipe.PIPE_WAIT | PIPE_REJECT_REMOTE_CLIENTS,
+            1, 8192, 8192, 0, None)
+        win32api.SetHandleInformation(self.pipe_handle, win32con.HANDLE_FLAG_INHERIT, 0)
+        self.err_msg = None
+        self.data = b''
+        self.start()
+
+    def run(self):
+        import win32pipe, win32file, winerror, win32api
+        def as_unicode(err):
+            try:
+                self.err_msg = type('')(err)
+            except Exception:
+                self.err_msg = repr(err)
+        try:
+            try:
+                rc = win32pipe.ConnectNamedPipe(self.pipe_handle)
+            except Exception as err:
+                as_unicode(err)
+                return
+
+            if rc != 0:
+                self.err_msg = 'Failed to connect to client over named pipe: 0x%x' % rc
+                return
+
+            while True:
+                try:
+                    hr, data = win32file.ReadFile(self.pipe_handle, 1024 * 50, None)
+                except Exception as err:
+                    if getattr(err, 'winerror', None) == winerror.ERROR_BROKEN_PIPE:
+                        break  # pipe was closed at the other end
+                    as_unicode(err)
+                    break
+                if hr not in (winerror.ERROR_MORE_DATA, 0):
+                    self.err_msg = 'ReadFile on pipe failed with hr=%d' % hr
+                    break
+                if not data:
+                    break
+                self.data += data
+        finally:
+            win32api.CloseHandle(self.pipe_handle)
+            self.pipe_handle = None
+
+def test(helper=HELPER):
+    pipename = '\\\\.\\pipe\\%s' % uuid4()
     echo = '\U0001f431 Hello world!'
-    stdout, stderr = p.communicate(serialize_string('ECHO', echo))
+    data = serialize_string('PIPENAME', pipename) + serialize_string('ECHO', echo)
+    server = PipeServer(pipename)
+    p = subprocess.Popen([helper], stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=subprocess.PIPE)
+    stdout, stderr = p.communicate(data)
     if p.wait() != 0:
         raise Exception('File dialog failed: ' + stderr.decode('utf-8'))
-    if stdout.decode('utf-8') != echo:
+    if server.err_msg is not None:
-        raise RuntimeError('Unexpected response: %s' % stdout.decode('utf-8'))
+        raise RuntimeError(server.err_msg)
+    server.join(2)
+    q = server.data[:-1].decode('utf-8')
+    if q != echo:
+        raise RuntimeError('Unexpected response: %r' % server.data)
+
+if __name__ == '__main__':
+    test(sys.argv[-1])