sendfile() for OS X. Also make sending of files a little more robust against simultaneous file modification

src/calibre/srv/respond.py

@@ -177,9 +177,13 @@ class ReadableOutput(object):
     def write(self, dest):
         if self.use_sendfile:
             dest.flush()  # Ensure everything in the SocketFile buffer is sent before calling sendfile()
-            sendfile_to_socket(self.src_file, 0, self.content_length, dest)
+            sent = sendfile_to_socket(self.src_file, 0, self.content_length, dest)
         else:
-            copy_range(self.src_file, 0, self.content_length, dest)
+            sent = copy_range(self.src_file, 0, self.content_length, dest)
+        if sent != self.content_length:
+            raise IOError(
+                'Failed to send complete file (%r) (%s != %s bytes), perhaps the file was modified during send?' % (
+                    getattr(self.src_file, 'name', '<file>'), sent, self.content_length))
         self.src_file = None
 
     def write_compressed(self, dest):
@@ -201,10 +205,14 @@ class ReadableOutput(object):
         self.src_file = None
 
     def copy_range(self, start, size, dest):
-        func = sendfile_to_socket if self.use_sendfile else copy_range
         if self.use_sendfile:
             dest.flush()  # Ensure everything in the SocketFile buffer is sent before calling sendfile()
-        func(self.src_file, start, size, dest)
+            sent = sendfile_to_socket(self.src_file, start, size, dest)
+        else:
+            sent = copy_range(self.src_file, start, size, dest)
+        if sent != size:
+            raise IOError('Failed to send byte range from file (%r) (%s != %s bytes), perhaps the file was modified during send?' % (
+                    getattr(self.src_file, 'name', '<file>'), sent, size))
 
 class FileSystemOutputFile(ReadableOutput):
 

src/calibre/srv/sendfile.py

@@ -20,21 +20,57 @@ def file_metadata(fileobj):
         pass
 
 def copy_range(src_file, start, size, dest):
+    total_sent = 0
     src_file.seek(start)
     while size > 0:
         data = src_file.read(min(size, DEFAULT_BUFFER_SIZE))
+        if len(data) == 0:
+            break  # EOF
         dest.write(data)
         size -= len(data)
+        total_sent += len(data)
         del data
+    return total_sent
 
 
 if iswindows:
     sendfile_to_socket = None
 elif isosx:
-    sendfile_to_socket = None
+    libc = ctypes.CDLL(None, use_errno=True)
+    sendfile = ctypes.CFUNCTYPE(
+        ctypes.c_int, ctypes.c_int, ctypes.c_int, ctypes.c_int64, ctypes.POINTER(ctypes.c_int64), ctypes.c_void_p, ctypes.c_int, use_errno=True)(
+            ('sendfile', libc))
+    del libc
+
+    def sendfile_to_socket(fileobj, offset, size, socket_file):
+        timeout = socket_file.gettimeout()
+        if timeout == 0:
+            return copy_range(fileobj, offset, size, socket_file)
+        num_bytes = ctypes.c_int64(size)
+        total_sent = 0
+        while size > 0:
+            num_bytes.value = size
+            r, w, x = select([], [socket_file], [], timeout)
+            if not w:
+                raise socket.timeout('timed out in sendfile() waiting for socket to become writeable')
+            ret = sendfile(fileobj.fileno(), socket_file.fileno(), offset, ctypes.byref(num_bytes), None, 0)
+            if ret != 0:
+                err = ctypes.get_errno()
+                if err in (errno.EBADF, errno.ENOTSUP, errno.ENOTSOCK, errno.EOPNOTSUPP):
+                    return copy_range(fileobj, offset, size, socket_file)
+                if err not in (errno.EINTR, errno.EAGAIN):
+                    raise IOError((err, os.strerror(err)))
+            if num_bytes.value == 0:
+                break  # EOF
+            total_sent += num_bytes.value
+            size -= num_bytes.value
+            offset += num_bytes.value
+        return total_sent
+
 else:
     libc = ctypes.CDLL(None, use_errno=True)
-    sendfile = ctypes.CFUNCTYPE(ctypes.c_ssize_t, ctypes.c_int, ctypes.c_int, ctypes.POINTER(ctypes.c_int64), ctypes.c_size_t, use_errno=True)(('sendfile64', libc))
+    sendfile = ctypes.CFUNCTYPE(
+        ctypes.c_ssize_t, ctypes.c_int, ctypes.c_int, ctypes.POINTER(ctypes.c_int64), ctypes.c_size_t, use_errno=True)(('sendfile64', libc))
     del libc
 
     def sendfile_to_socket(fileobj, offset, size, socket_file):
@@ -52,10 +88,11 @@ else:
                 err = ctypes.get_errno()
                 if err in (errno.ENOSYS, errno.EINVAL):
                     return copy_range(fileobj, off.value, size, socket_file)
-                if err != errno.EAGAIN:
+                if err not in (errno.EINTR, errno.EAGAIN):
                     raise IOError((err, os.strerror(err)))
             elif sent == 0:
                 break  # EOF
             else:
                 size -= sent
                 total_sent += sent
+        return total_sent

src/calibre/srv/tests/http.py

@@ -216,6 +216,7 @@ class TestHTTP(BaseTest):
             r = conn.getresponse()
             self.ae(r.status, httplib.OK), self.ae(zlib.decompress(r.read(), 16+zlib.MAX_WBITS), b'an_etagged_path')
 
+            for i in '12':
                 # Test getting a filesystem file
                 server.change_handler(lambda conn: f)
                 conn = server.connect()
@@ -267,4 +268,7 @@ class TestHTTP(BaseTest):
                 conn.request('GET', '/test')
                 r = conn.getresponse()
                 self.ae(data, r.read())
+
+                server.loop.opts.use_sendfile ^= True
+                conn = server.connect()
     # }}}