Improve implementation of lopen()

Greatly simplify window implementation. Reduce the average number of
system calls used per invocation on OS X and linux.

src/calibre/startup.py

@@ -6,7 +6,7 @@ __docformat__ = 'restructuredtext en'
 Perform various initialization tasks.
 '''
 
-import locale, sys, os, re
+import locale, sys
 
 # Default translation is NOOP
 import __builtin__
@@ -109,97 +109,40 @@ if not _run_once:
         except:
             pass
 
-    #
+    # local_open() opens a file that wont be inherited by child processes
-
+    if iswindows:
-    def local_open(name, mode='r', bufsize=-1):
+        def local_open(name, mode='r', bufsize=-1):
-        '''
+            mode += 'N'
-        Open a file that wont be inherited by child processes
+            return open(name, mode, bufsize)
-
+    elif isosx:
-        Only supports the following modes:
+        import fcntl
-            r, w, a, rb, wb, ab, r+, w+, a+, r+b, w+b, a+b
+        FIOCLEX = 0x20006601
-        '''
+        def local_open(name, mode='r', bufsize=-1):
-        if iswindows:
-            class fwrapper(object):
-
-                def __init__(self, name, fobject):
-                    object.__setattr__(self, 'fobject', fobject)
-                    object.__setattr__(self, 'name', name)
-
-                def __getattribute__(self, attr):
-                    if attr in ('name', '__enter__', '__str__', '__unicode__',
-                            '__repr__', '__exit__'):
-                        return object.__getattribute__(self, attr)
-                    fobject = object.__getattribute__(self, 'fobject')
-                    return getattr(fobject, attr)
-
-                def __setattr__(self, attr, val):
-                    fobject = object.__getattribute__(self, 'fobject')
-                    return setattr(fobject, attr, val)
-
-                def __repr__(self):
-                    fobject = object.__getattribute__(self, 'fobject')
-                    name = object.__getattribute__(self, 'name')
-                    return re.sub(r'''['"]<fdopen>['"]''', repr(name),
-                            repr(fobject))
-
-                def __str__(self):
-                    return repr(self)
-
-                def __unicode__(self):
-                    return repr(self).decode('utf-8')
-
-                def __enter__(self):
-                    fobject = object.__getattribute__(self, 'fobject')
-                    fobject.__enter__()
-                    return self
-
-                def __exit__(self, *args):
-                    fobject = object.__getattribute__(self, 'fobject')
-                    return fobject.__exit__(*args)
-
-            m = mode[0]
-            random = len(mode) > 1 and mode[1] == '+'
-            binary = mode[-1] == 'b'
-
-            if m == 'a':
-                flags = os.O_APPEND| os.O_RDWR
-                flags |= os.O_RANDOM if random else os.O_SEQUENTIAL
-            elif m == 'r':
-                if random:
-                    flags = os.O_RDWR | os.O_RANDOM
-                else:
-                    flags = os.O_RDONLY | os.O_SEQUENTIAL
-            elif m == 'w':
-                if random:
-                    flags = os.O_RDWR | os.O_RANDOM
-                else:
-                    flags = os.O_WRONLY | os.O_SEQUENTIAL
-                flags |= os.O_TRUNC | os.O_CREAT
-            if binary:
-                flags |= os.O_BINARY
-            else:
-                flags |= os.O_TEXT
-            flags |= os.O_NOINHERIT
-            fd = os.open(name, flags)
-            ans = os.fdopen(fd, mode, bufsize)
-            ans = fwrapper(name, ans)
-        else:
-            import fcntl
-            try:
-                cloexec_flag = fcntl.FD_CLOEXEC
-            except AttributeError:
-                cloexec_flag = 1
-            # Python 2.x uses fopen which on recent glibc/linux kernel at least
-            # respects the 'e' mode flag. On OS X the e is ignored. So to try
-            # to get atomicity where possible we pass 'e' and then only use
-            # fcntl only if CLOEXEC was not set.
-            if islinux:
-                mode += 'e'
             ans = open(name, mode, bufsize)
+            try:
+                fcntl.ioctl(ans.fileno(), FIOCLEX)
+            except EnvironmentError:
+                fcntl.fcntl(ans, fcntl.F_SETFD, fcntl.fcntl(ans, fcntl.F_GETFD) | fcntl.FD_CLOEXEC)
+            return ans
+    else:
+        import fcntl
+        try:
+            cloexec_flag = fcntl.FD_CLOEXEC
+        except AttributeError:
+            cloexec_flag = 1
+        supports_mode_e = False
+        def local_open(name, mode='r', bufsize=-1):
+            global supports_mode_e
+            mode += 'e'
+            ans = open(name, mode, bufsize)
+            if supports_mode_e:
+                return ans
             old = fcntl.fcntl(ans, fcntl.F_GETFD)
             if not (old & cloexec_flag):
                 fcntl.fcntl(ans, fcntl.F_SETFD, old | cloexec_flag)
-        return ans
+            else:
+                supports_mode_e = True
+            return ans
 
     __builtin__.__dict__['lopen'] = local_open
 
@@ -242,34 +185,47 @@ def test_lopen():
     from calibre.ptempfile import TemporaryDirectory
     from calibre import CurrentDir
     n = u'f\xe4llen'
+    print('testing lopen()')
 
-    with TemporaryDirectory() as tdir:
+    if iswindows:
-        with CurrentDir(tdir):
+        import msvcrt, win32api
-            with lopen(n, 'w') as f:
+        def assert_not_inheritable(f):
-                f.write('one')
+            if win32api.GetHandleInformation(msvcrt.get_osfhandle(f.fileno())) & 0b1:
-            print 'O_CREAT tested'
+                raise SystemExit('File handle is inheritable!')
-            with lopen(n, 'w+b') as f:
+    else:
-                f.write('two')
+        def assert_not_inheritable(f):
-            with lopen(n, 'r') as f:
+            if not fcntl.fcntl(f, fcntl.F_GETFD) & fcntl.FD_CLOEXEC:
-                if f.read() == 'two':
+                raise SystemExit('File handle is inheritable!')
-                    print 'O_TRUNC tested'
-                else:
-                    raise Exception('O_TRUNC failed')
-            with lopen(n, 'ab') as f:
-                f.write('three')
-            with lopen(n, 'r+') as f:
-                if f.read() == 'twothree':
-                    print 'O_APPEND tested'
-                else:
-                    raise Exception('O_APPEND failed')
-            with lopen(n, 'r+') as f:
-                f.seek(3)
-                f.write('xxxxx')
-                f.seek(0)
-                if f.read() == 'twoxxxxx':
-                    print 'O_RANDOM tested'
-                else:
-                    raise Exception('O_RANDOM failed')
 
+    def copen(*args):
+        ans = lopen(*args)
+        assert_not_inheritable(ans)
+        return ans
 
+    with TemporaryDirectory() as tdir, CurrentDir(tdir):
+        with copen(n, 'w') as f:
+            f.write('one')
 
+        print 'O_CREAT tested'
+        with copen(n, 'w+b') as f:
+            f.write('two')
+        with copen(n, 'r') as f:
+            if f.read() == 'two':
+                print 'O_TRUNC tested'
+            else:
+                raise Exception('O_TRUNC failed')
+        with copen(n, 'ab') as f:
+            f.write('three')
+        with copen(n, 'r+') as f:
+            if f.read() == 'twothree':
+                print 'O_APPEND tested'
+            else:
+                raise Exception('O_APPEND failed')
+        with copen(n, 'r+') as f:
+            f.seek(3)
+            f.write('xxxxx')
+            f.seek(0)
+            if f.read() == 'twoxxxxx':
+                print 'O_RANDOM tested'
+            else:
+                raise Exception('O_RANDOM failed')