diff --git a/src/calibre/srv/respond.py b/src/calibre/srv/respond.py index a2bbf355dc..a8ba4c191d 100644 --- a/src/calibre/srv/respond.py +++ b/src/calibre/srv/respond.py @@ -177,9 +177,13 @@ class ReadableOutput(object): def write(self, dest): if self.use_sendfile: dest.flush() # Ensure everything in the SocketFile buffer is sent before calling sendfile() - sendfile_to_socket(self.src_file, 0, self.content_length, dest) + sent = sendfile_to_socket(self.src_file, 0, self.content_length, dest) else: - copy_range(self.src_file, 0, self.content_length, dest) + sent = copy_range(self.src_file, 0, self.content_length, dest) + if sent != self.content_length: + raise IOError( + 'Failed to send complete file (%r) (%s != %s bytes), perhaps the file was modified during send?' % ( + getattr(self.src_file, 'name', ''), sent, self.content_length)) self.src_file = None def write_compressed(self, dest): @@ -201,10 +205,14 @@ class ReadableOutput(object): self.src_file = None def copy_range(self, start, size, dest): - func = sendfile_to_socket if self.use_sendfile else copy_range if self.use_sendfile: dest.flush() # Ensure everything in the SocketFile buffer is sent before calling sendfile() - func(self.src_file, start, size, dest) + sent = sendfile_to_socket(self.src_file, start, size, dest) + else: + sent = copy_range(self.src_file, start, size, dest) + if sent != size: + raise IOError('Failed to send byte range from file (%r) (%s != %s bytes), perhaps the file was modified during send?' % ( + getattr(self.src_file, 'name', ''), sent, size)) class FileSystemOutputFile(ReadableOutput): diff --git a/src/calibre/srv/sendfile.py b/src/calibre/srv/sendfile.py index ba86d6cadc..567a1b4bc1 100644 --- a/src/calibre/srv/sendfile.py +++ b/src/calibre/srv/sendfile.py @@ -20,21 +20,57 @@ def file_metadata(fileobj): pass def copy_range(src_file, start, size, dest): + total_sent = 0 src_file.seek(start) while size > 0: data = src_file.read(min(size, DEFAULT_BUFFER_SIZE)) + if len(data) == 0: + break # EOF dest.write(data) size -= len(data) + total_sent += len(data) del data + return total_sent if iswindows: sendfile_to_socket = None elif isosx: - sendfile_to_socket = None + libc = ctypes.CDLL(None, use_errno=True) + sendfile = ctypes.CFUNCTYPE( + ctypes.c_int, ctypes.c_int, ctypes.c_int, ctypes.c_int64, ctypes.POINTER(ctypes.c_int64), ctypes.c_void_p, ctypes.c_int, use_errno=True)( + ('sendfile', libc)) + del libc + + def sendfile_to_socket(fileobj, offset, size, socket_file): + timeout = socket_file.gettimeout() + if timeout == 0: + return copy_range(fileobj, offset, size, socket_file) + num_bytes = ctypes.c_int64(size) + total_sent = 0 + while size > 0: + num_bytes.value = size + r, w, x = select([], [socket_file], [], timeout) + if not w: + raise socket.timeout('timed out in sendfile() waiting for socket to become writeable') + ret = sendfile(fileobj.fileno(), socket_file.fileno(), offset, ctypes.byref(num_bytes), None, 0) + if ret != 0: + err = ctypes.get_errno() + if err in (errno.EBADF, errno.ENOTSUP, errno.ENOTSOCK, errno.EOPNOTSUPP): + return copy_range(fileobj, offset, size, socket_file) + if err not in (errno.EINTR, errno.EAGAIN): + raise IOError((err, os.strerror(err))) + if num_bytes.value == 0: + break # EOF + total_sent += num_bytes.value + size -= num_bytes.value + offset += num_bytes.value + return total_sent + else: libc = ctypes.CDLL(None, use_errno=True) - sendfile = ctypes.CFUNCTYPE(ctypes.c_ssize_t, ctypes.c_int, ctypes.c_int, ctypes.POINTER(ctypes.c_int64), ctypes.c_size_t, use_errno=True)(('sendfile64', libc)) + sendfile = ctypes.CFUNCTYPE( + ctypes.c_ssize_t, ctypes.c_int, ctypes.c_int, ctypes.POINTER(ctypes.c_int64), ctypes.c_size_t, use_errno=True)(('sendfile64', libc)) del libc def sendfile_to_socket(fileobj, offset, size, socket_file): @@ -52,10 +88,11 @@ else: err = ctypes.get_errno() if err in (errno.ENOSYS, errno.EINVAL): return copy_range(fileobj, off.value, size, socket_file) - if err != errno.EAGAIN: + if err not in (errno.EINTR, errno.EAGAIN): raise IOError((err, os.strerror(err))) elif sent == 0: break # EOF else: size -= sent total_sent += sent + return total_sent diff --git a/src/calibre/srv/tests/http.py b/src/calibre/srv/tests/http.py index 8ee4ea4cf0..1cb873bb38 100644 --- a/src/calibre/srv/tests/http.py +++ b/src/calibre/srv/tests/http.py @@ -216,55 +216,59 @@ class TestHTTP(BaseTest): r = conn.getresponse() self.ae(r.status, httplib.OK), self.ae(zlib.decompress(r.read(), 16+zlib.MAX_WBITS), b'an_etagged_path') - # Test getting a filesystem file - server.change_handler(lambda conn: f) - conn = server.connect() - conn.request('GET', '/test') - r = conn.getresponse() - etag = type('')(r.getheader('ETag')) - self.assertTrue(etag) - self.ae(r.getheader('Content-Type'), guess_type(f.name)[0]) - self.ae(type('')(r.getheader('Accept-Ranges')), 'bytes') - self.ae(int(r.getheader('Content-Length')), len(fdata)) - self.ae(r.status, httplib.OK), self.ae(r.read(), fdata) + for i in '12': + # Test getting a filesystem file + server.change_handler(lambda conn: f) + conn = server.connect() + conn.request('GET', '/test') + r = conn.getresponse() + etag = type('')(r.getheader('ETag')) + self.assertTrue(etag) + self.ae(r.getheader('Content-Type'), guess_type(f.name)[0]) + self.ae(type('')(r.getheader('Accept-Ranges')), 'bytes') + self.ae(int(r.getheader('Content-Length')), len(fdata)) + self.ae(r.status, httplib.OK), self.ae(r.read(), fdata) - conn.request('GET', '/test', headers={'Range':'bytes=0-25'}) - r = conn.getresponse() - self.ae(type('')(r.getheader('Accept-Ranges')), 'bytes') - self.ae(type('')(r.getheader('Content-Range')), 'bytes 0-25/%d' % len(fdata)) - self.ae(int(r.getheader('Content-Length')), 26) - self.ae(r.status, httplib.PARTIAL_CONTENT), self.ae(r.read(), fdata[0:26]) + conn.request('GET', '/test', headers={'Range':'bytes=0-25'}) + r = conn.getresponse() + self.ae(type('')(r.getheader('Accept-Ranges')), 'bytes') + self.ae(type('')(r.getheader('Content-Range')), 'bytes 0-25/%d' % len(fdata)) + self.ae(int(r.getheader('Content-Length')), 26) + self.ae(r.status, httplib.PARTIAL_CONTENT), self.ae(r.read(), fdata[0:26]) - conn.request('GET', '/test', headers={'Range':'bytes=100000-'}) - r = conn.getresponse() - self.ae(type('')(r.getheader('Content-Range')), 'bytes */%d' % len(fdata)) - self.ae(r.status, httplib.REQUESTED_RANGE_NOT_SATISFIABLE) + conn.request('GET', '/test', headers={'Range':'bytes=100000-'}) + r = conn.getresponse() + self.ae(type('')(r.getheader('Content-Range')), 'bytes */%d' % len(fdata)) + self.ae(r.status, httplib.REQUESTED_RANGE_NOT_SATISFIABLE) - conn.request('GET', '/test', headers={'Range':'bytes=25-50', 'If-Range':etag}) - r = conn.getresponse() - self.ae(int(r.getheader('Content-Length')), 26) - self.ae(r.status, httplib.PARTIAL_CONTENT), self.ae(r.read(), fdata[25:51]) + conn.request('GET', '/test', headers={'Range':'bytes=25-50', 'If-Range':etag}) + r = conn.getresponse() + self.ae(int(r.getheader('Content-Length')), 26) + self.ae(r.status, httplib.PARTIAL_CONTENT), self.ae(r.read(), fdata[25:51]) - conn.request('GET', '/test', headers={'Range':'bytes=25-50', 'If-Range':'"nomatch"'}) - r = conn.getresponse() - self.assertFalse(r.getheader('Content-Range')) - self.ae(int(r.getheader('Content-Length')), len(fdata)) - self.ae(r.status, httplib.OK), self.ae(r.read(), fdata) + conn.request('GET', '/test', headers={'Range':'bytes=25-50', 'If-Range':'"nomatch"'}) + r = conn.getresponse() + self.assertFalse(r.getheader('Content-Range')) + self.ae(int(r.getheader('Content-Length')), len(fdata)) + self.ae(r.status, httplib.OK), self.ae(r.read(), fdata) - conn.request('GET', '/test', headers={'Range':'bytes=0-25,26-50'}) - r = conn.getresponse() - clen = int(r.getheader('Content-Length')) - data = r.read() - self.ae(clen, len(data)) - buf = BytesIO(data) - self.ae(parse_multipart_byterange(buf, r.getheader('Content-Type')), [(0, fdata[:26]), (26, fdata[26:51])]) + conn.request('GET', '/test', headers={'Range':'bytes=0-25,26-50'}) + r = conn.getresponse() + clen = int(r.getheader('Content-Length')) + data = r.read() + self.ae(clen, len(data)) + buf = BytesIO(data) + self.ae(parse_multipart_byterange(buf, r.getheader('Content-Type')), [(0, fdata[:26]), (26, fdata[26:51])]) - # Test sending of larger file - lf.seek(0) - data = lf.read() - server.change_handler(lambda conn: lf) - conn = server.connect() - conn.request('GET', '/test') - r = conn.getresponse() - self.ae(data, r.read()) + # Test sending of larger file + lf.seek(0) + data = lf.read() + server.change_handler(lambda conn: lf) + conn = server.connect() + conn.request('GET', '/test') + r = conn.getresponse() + self.ae(data, r.read()) + + server.loop.opts.use_sendfile ^= True + conn = server.connect() # }}}