Windows: Use a separate worker process to move files to the recycle bin. Fixes the problem of the recycle bin icon not being updated when deleting books into an empty recycle bin. Also avoids the overhead of launching a new, short-lived worker process for every delete.

setup/installer/windows/freeze.py

@@ -42,7 +42,6 @@ DESCRIPTIONS = {
         'calibre-server': 'Standalone calibre content server',
         'calibre-parallel': 'calibre worker process',
         'calibre-smtp' : 'Command line interface for sending books via email',
-        'calibre-recycle' : 'Helper program for deleting to recycle bin',
         'calibre-eject' : 'Helper program for ejecting connected reader devices',
 }
 
@@ -85,7 +84,6 @@ class Win32Freeze(Command, WixMixIn):
         self.initbase()
         self.build_launchers()
         self.build_eject()
-        self.build_recycle()
         self.add_plugins()
         self.freeze()
         self.embed_manifests()
@@ -546,21 +544,6 @@ class Win32Freeze(Command, WixMixIn):
         finally:
             os.chdir(cwd)
 
-    def build_recycle(self):
-        self.info('Building calibre-recycle.exe')
-        base = self.j(self.src_root, 'setup', 'installer', 'windows')
-        src = self.j(base, 'recycle.c')
-        obj = self.j(self.obj_dir, self.b(src)+'.obj')
-        cflags  = '/c /EHsc /MD /W3 /Ox /nologo /D_UNICODE'.split()
-        if self.newer(obj, src):
-            cmd = [msvc.cc] + cflags + ['/Fo'+obj, '/Tc'+src]
-            self.run_builder(cmd, show_output=True)
-        exe = self.j(self.base, 'calibre-recycle.exe')
-        cmd = [msvc.linker] + ['/MACHINE:'+machine,
-                '/SUBSYSTEM:CONSOLE', '/RELEASE',
-                '/OUT:'+exe] + [self.embed_resources(exe), obj, 'Shell32.lib']
-        self.run_builder(cmd)
-
     def build_eject(self):
         self.info('Building calibre-eject.exe')
         base = self.j(self.src_root, 'setup', 'installer', 'windows')

setup/installer/windows/recycle.c

@@ -1,28 +0,0 @@
-/*
- * recycle.c
- * Copyright (C) 2013 Kovid Goyal <kovid at kovidgoyal.net>
- *
- * Distributed under terms of the GPL3 license.
- */
-
-#include "Windows.h"
-#include "Shellapi.h"
-/* #include <wchar.h> */
-
-int wmain(int argc, wchar_t *argv[ ]) {
-    wchar_t buf[512] = {0};
-    SHFILEOPSTRUCTW op = {0};
-    if (argc != 2) return 1;
-    if (wcsnlen_s(argv[1], 512) > 510) return 1;
-    if (wcscpy_s(buf, 512, argv[1]) != 0) return 1; 
-
-    op.wFunc = FO_DELETE;
-    op.pFrom = buf;
-    op.pTo = NULL;
-    op.fFlags = FOF_ALLOWUNDO | FOF_NOCONFIRMATION | FOF_NOCONFIRMMKDIR | FOF_NOERRORUI | FOF_SILENT | FOF_RENAMEONCOLLISION;
-
-    /* wprintf(L"%ls\n", buf); */
-    return SHFileOperationW(&op);
-}
-
-

src/calibre/utils/ipc/simple_worker.py

@@ -136,6 +136,31 @@ def create_worker(env, priority='normal', cwd=None, func='main'):
     w(cwd=cwd, priority=priority)
     return listener, w
 
+def start_pipe_worker(command, env=None, priority='normal'):
+    import subprocess, atexit
+    from functools import partial
+    w = Worker(env or {})
+    args = {'stdout':subprocess.PIPE, 'stdin':subprocess.PIPE, 'env':w.env}
+    if iswindows:
+        import win32process
+        priority = {
+                'high'   : win32process.HIGH_PRIORITY_CLASS,
+                'normal' : win32process.NORMAL_PRIORITY_CLASS,
+                'low'    : win32process.IDLE_PRIORITY_CLASS}[priority]
+        args['creationflags'] = win32process.CREATE_NO_WINDOW|priority
+    else:
+        def renice(niceness):
+            try:
+                os.nice(niceness)
+            except:
+                pass
+        niceness = {'normal' : 0, 'low'    : 10, 'high'   : 20}[priority]
+        args['preexec_fn'] = partial(renice, niceness)
+
+    p = subprocess.Popen([w.executable, '--pipe-worker', command], **args)
+    atexit.register(w.kill)
+    return p
+
 def fork_job(mod_name, func_name, args=(), kwargs={}, timeout=300,  # seconds
         cwd=None, priority='normal', env={}, no_output=False, heartbeat=None,
         abort=None, module_is_source_code=False):

src/calibre/utils/ipc/worker.py

@@ -175,6 +175,9 @@ def main():
         func = getattr(mod, func)
         func()
         return
+    if '--pipe-worker' in sys.argv:
+        exec (sys.argv[-1])
+        return
     address = cPickle.loads(unhexlify(os.environ['CALIBRE_WORKER_ADDRESS']))
     key     = unhexlify(os.environ['CALIBRE_WORKER_KEY'])
     resultf = unhexlify(os.environ['CALIBRE_WORKER_RESULT']).decode('utf-8')

src/calibre/utils/recycle_bin.py

@@ -1,113 +1,86 @@
 #!/usr/bin/env python
 # vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import print_function
 
 __license__   = 'GPL v3'
 __copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import os, shutil, time
+import os, shutil, time, sys
 
-from calibre import isbytestring, force_unicode
+from calibre import isbytestring
 from calibre.constants import (iswindows, isosx, plugins, filesystem_encoding,
         islinux)
 
 recycle = None
 
 if iswindows:
-    import ctypes, subprocess, sys
+    from calibre.utils.ipc import eintr_retry_call
-    from ctypes import POINTER, Structure
+    from threading import Lock
-    from ctypes.wintypes import HANDLE, LPVOID, WORD, DWORD, BOOL, ULONG, LPCWSTR
+    recycler = None
-    RECYCLE = force_unicode(os.path.join(os.path.dirname(sys.executable), 'calibre-recycle.exe'), filesystem_encoding)
+    rlock = Lock()
-    LPDWORD = POINTER(DWORD)
+    def start_recycler():
-    LPHANDLE = POINTER(HANDLE)
+        global recycler
-    ULONG_PTR = POINTER(ULONG)
+        if recycler is None:
-    CREATE_NO_WINDOW = 0x08000000
+            from calibre.utils.ipc.simple_worker import start_pipe_worker
-    INFINITE = 0xFFFFFFFF
+            recycler = start_pipe_worker('from calibre.utils.recycle_bin import recycler_main; recycler_main()')
-    WAIT_FAILED = 0xFFFFFFFF
 
-    class SECURITY_ATTRIBUTES(Structure):
+    def recycle_path(path):
-        _fields_ = [("nLength", DWORD),
+        from win32com.shell import shell, shellcon
-                    ("lpSecurityDescriptor", LPVOID),
+        flags = (shellcon.FOF_ALLOWUNDO | shellcon.FOF_NOCONFIRMATION | shellcon.FOF_NOCONFIRMMKDIR | shellcon.FOF_NOERRORUI |
-                    ("bInheritHandle", BOOL)]
+                 shellcon.FOF_SILENT | shellcon.FOF_RENAMEONCOLLISION)
-    LPSECURITY_ATTRIBUTES = POINTER(SECURITY_ATTRIBUTES)
+        retcode, aborted = shell.SHFileOperation((0, shellcon.FO_DELETE, path, None, flags, None, None))
+        if retcode != 0 or aborted:
+            raise RuntimeError('Failed to delete: %r with error code: %d' % (path, retcode))
 
-    class STARTUPINFO(Structure):
+    def recycler_main():
-        _fields_ = [("cb", DWORD),
+        while True:
-                    ("lpReserved", LPCWSTR),
+            path = eintr_retry_call(sys.stdin.readline)
-                    ("lpDesktop", LPCWSTR),
+            if not path:
-                    ("lpTitle", LPCWSTR),
+                break
-                    ("dwX", DWORD),
+            try:
-                    ("dwY", DWORD),
+                path = path.decode('utf-8').rstrip()
-                    ("dwXSize", DWORD),
+            except (ValueError, TypeError):
-                    ("dwYSize", DWORD),
+                break
-                    ("dwXCountChars", DWORD),
+            try:
-                    ("dwYCountChars", DWORD),
+                recycle_path(path)
-                    ("dwFillAttribute", DWORD),
+            except:
-                    ("dwFlags", DWORD),
+                eintr_retry_call(print, b'KO', file=sys.stdout)
-                    ("wShowWindow", WORD),
+                sys.stdout.flush()
-                    ("cbReserved2", WORD),
+                import traceback
-                    ("lpReserved2", LPVOID),
+                traceback.print_exc()  # goes to stderr, which is the same as for parent process
-                    ("hStdInput", HANDLE),
+            else:
-                    ("hStdOutput", HANDLE),
+                eintr_retry_call(print, b'OK', file=sys.stdout)
-                    ("hStdError", HANDLE)]
+                sys.stdout.flush()
-    LPSTARTUPINFO = POINTER(STARTUPINFO)
 
-    class PROCESS_INFORMATION(Structure):
+    def delegate_recycle(path):
-        _fields_ = [("hProcess", HANDLE),
+        if '\n' in path:
-                    ("hThread", HANDLE),
+            raise ValueError('Cannot recycle paths that have newlines in them (%r)' % path)
-                    ("dwProcessId", DWORD),
+        with rlock:
-                    ("dwThreadId", DWORD)]
+            start_recycler()
-    LPPROCESS_INFORMATION = POINTER(PROCESS_INFORMATION)
+            eintr_retry_call(print, path.encode('utf-8'), file=recycler.stdin)
-
+            recycler.stdin.flush()
-    CreateProcess = ctypes.windll.kernel32.CreateProcessW
+            # Theoretically this could be made non-blocking using a
-    CreateProcess.argtypes = [LPCWSTR, LPCWSTR, LPSECURITY_ATTRIBUTES,
+            # thread+queue, however the original implementation was blocking,
-        LPSECURITY_ATTRIBUTES, BOOL, DWORD, LPVOID, LPCWSTR, LPSTARTUPINFO,
+            # so I am leaving it as blocking.
-        LPPROCESS_INFORMATION]
+            result = eintr_retry_call(recycler.stdout.readline)
-    CreateProcess.restype = BOOL
+            if result.rstrip() != b'OK':
-
+                raise RuntimeError('recycler failed to recycle: %r' % path)
-    WaitForSingleObject = ctypes.windll.kernel32.WaitForSingleObject
-    WaitForSingleObject.argtypes = [HANDLE, DWORD]
-    WaitForSingleObject.restype = DWORD
-
-    GetExitCodeProcess = ctypes.windll.kernel32.GetExitCodeProcess
-    GetExitCodeProcess.argtypes = [HANDLE, LPDWORD]
-    GetExitCodeProcess.restype = BOOL
-
-    CloseHandle = ctypes.windll.kernel32.CloseHandle
-    CloseHandle.argtypes = [HANDLE]
-    CloseHandle.restype = BOOL
 
     def recycle(path):
         # We have to run the delete to recycle bin in a separate process as the
         # morons who wrote SHFileOperation designed it to spin the event loop
         # even when no UI is created. And there is no other way to send files
-        # to the recycle bin on windows. Le Sigh. We dont use subprocess since
+        # to the recycle bin on windows. Le Sigh. So we do it in a worker
-        # there is no way to pass unicode arguments with subprocess in 2.7 and
+        # process. Unfortunately, if the worker process exits immediately after
-        # the twit that maintains subprocess believes that this is not an
+        # deleting to recycle bin, winblows does not update the recycle bin
-        # bug but a request for a new feature.
+        # icon. Le Double Sigh. So we use a long lived worker process, that is
+        # started on first recycle, and sticks around to handle subsequent
+        # recycles.
         if isinstance(path, bytes):
             path = path.decode(filesystem_encoding)
-        si = STARTUPINFO()
+        path = os.path.abspath(path)  # Windows does not like recycling relative paths
-        si.cb = ctypes.sizeof(si)
+        return delegate_recycle(path)
-        pi = PROCESS_INFORMATION()
-        exit_code = DWORD()
-        cmd = subprocess.list2cmdline([RECYCLE, path])
-        dwCreationFlags = CREATE_NO_WINDOW
-        if not CreateProcess(None, cmd, None, None, False, dwCreationFlags,
-                None, None, ctypes.byref(si), ctypes.byref(pi)):
-            raise ctypes.WinError()
-        try:
-            if WaitForSingleObject(pi.hProcess, INFINITE) == WAIT_FAILED:
-                raise ctypes.WinError()
-            if not GetExitCodeProcess(pi.hProcess, ctypes.byref(exit_code)):
-                raise ctypes.WinError()
-
-        finally:
-            CloseHandle(pi.hThread)
-            CloseHandle(pi.hProcess)
-        exit_code = exit_code.value
-        if exit_code != 0:
-            raise ctypes.WinError(exit_code)
 
 elif isosx:
     u = plugins['usbobserver'][0]