diff --git a/src/calibre/srv/http_response.py b/src/calibre/srv/http_response.py index 737858cf0e..4da0fc0328 100644 --- a/src/calibre/srv/http_response.py +++ b/src/calibre/srv/http_response.py @@ -5,28 +5,33 @@ __license__ = 'GPL v3' __copyright__ = '2015, Kovid Goyal ' -import os, hashlib, uuid, struct +import errno +import hashlib +import os +import struct +import uuid from collections import namedtuple -from io import BytesIO, DEFAULT_BUFFER_SIZE +from functools import wraps +from io import DEFAULT_BUFFER_SIZE, BytesIO from itertools import chain, repeat from operator import itemgetter -from functools import wraps -from polyglot.builtins import iteritems, itervalues, reraise, map, unicode_type, string_or_bytes - -from calibre import guess_type, force_unicode +from calibre import force_unicode, guess_type from calibre.constants import __version__ -from calibre.srv.loop import WRITE from calibre.srv.errors import HTTPSimpleResponse from calibre.srv.http_request import HTTPRequest, read_headers -from calibre.srv.sendfile import file_metadata, sendfile_to_socket_async, CannotSendfile, SendfileInterrupted +from calibre.srv.loop import WRITE from calibre.srv.utils import ( - MultiDict, http_date, HTTP1, HTTP11, socket_errors_socket_closed, - sort_q_values, get_translator_for_lang, Cookie, fast_now_strftime) -from calibre.utils.speedups import ReadOnlyFileBuffer + HTTP1, HTTP11, Cookie, MultiDict, fast_now_strftime, get_translator_for_lang, + http_date, socket_errors_socket_closed, sort_q_values +) from calibre.utils.monotonic import monotonic +from calibre.utils.speedups import ReadOnlyFileBuffer from polyglot import http_client, reprlib -from polyglot.builtins import error_message +from polyglot.builtins import ( + error_message, iteritems, itervalues, map, reraise, string_or_bytes, + unicode_type +) Range = namedtuple('Range', 'start stop size') MULTIPART_SEPARATOR = uuid.uuid4().hex @@ -37,6 +42,14 @@ import zlib from itertools import zip_longest +def file_metadata(fileobj): + try: + fd = fileobj.fileno() + return os.fstat(fd) + except Exception: + pass + + def header_list_to_file(buf): # {{{ buf.append('') return ReadOnlyFileBuffer(b''.join((x + '\r\n').encode('ascii') for x in buf)) @@ -374,17 +387,15 @@ class HTTPConnection(HTTPRequest): if limit <= 0: return True if self.use_sendfile and not isinstance(buf, (BytesIO, ReadOnlyFileBuffer)): + limit = min(limit, 2 ** 30) try: - sent = sendfile_to_socket_async(buf, pos, limit, self.socket) - except CannotSendfile: - self.use_sendfile = False - return False - except SendfileInterrupted: - return False - except IOError as e: + sent = os.sendfile(self.socket.fileno(), buf.fileno(), pos, limit) + except OSError as e: if e.errno in socket_errors_socket_closed: self.ready = self.use_sendfile = False return False + if e.errno in (errno.EAGAIN, errno.EINTR): + return False raise finally: self.last_activity = monotonic() @@ -561,7 +572,7 @@ class HTTPConnection(HTTPRequest): self.reset_state() return if isinstance(output, ReadableOutput): - self.use_sendfile = output.use_sendfile and self.opts.use_sendfile and sendfile_to_socket_async is not None and self.ssl_context is None + self.use_sendfile = output.use_sendfile and self.opts.use_sendfile and hasattr(os, 'sendfile') and self.ssl_context is None # sendfile() does not work with SSL sockets since encryption has to # be done in userspace if output.ranges is not None: diff --git a/src/calibre/srv/sendfile.py b/src/calibre/srv/sendfile.py deleted file mode 100644 index 49beff15ff..0000000000 --- a/src/calibre/srv/sendfile.py +++ /dev/null @@ -1,133 +0,0 @@ -#!/usr/bin/env python -# vim:fileencoding=utf-8 - - -__license__ = 'GPL v3' -__copyright__ = '2015, Kovid Goyal ' - -import os, ctypes, errno, socket -from io import DEFAULT_BUFFER_SIZE -from select import select - -from calibre.constants import islinux, ismacos -from calibre.srv.utils import eintr_retry_call - - -def file_metadata(fileobj): - try: - fd = fileobj.fileno() - return os.fstat(fd) - except Exception: - pass - - -def copy_range(src_file, start, size, dest): - total_sent = 0 - src_file.seek(start) - while size > 0: - data = eintr_retry_call(src_file.read, min(size, DEFAULT_BUFFER_SIZE)) - if len(data) == 0: - break # EOF - dest.write(data) - size -= len(data) - total_sent += len(data) - del data - return total_sent - - -class CannotSendfile(Exception): - pass - - -class SendfileInterrupted(Exception): - pass - - -sendfile_to_socket = sendfile_to_socket_async = None - -if ismacos: - libc = ctypes.CDLL(None, use_errno=True) - sendfile = ctypes.CFUNCTYPE( - ctypes.c_int, ctypes.c_int, ctypes.c_int, ctypes.c_int64, ctypes.POINTER(ctypes.c_int64), ctypes.c_void_p, ctypes.c_int, use_errno=True)( - ('sendfile', libc)) - del libc - - def sendfile_to_socket(fileobj, offset, size, socket_file): - timeout = socket_file.gettimeout() - if timeout == 0: - return copy_range(fileobj, offset, size, socket_file) - num_bytes = ctypes.c_int64(size) - total_sent = 0 - while size > 0: - num_bytes.value = size - r, w, x = select([], [socket_file], [], timeout) - if not w: - raise socket.timeout('timed out in sendfile() waiting for socket to become writeable') - ret = sendfile(fileobj.fileno(), socket_file.fileno(), offset, ctypes.byref(num_bytes), None, 0) - if ret != 0: - err = ctypes.get_errno() - if err in (errno.EBADF, errno.ENOTSUP, errno.ENOTSOCK, errno.EOPNOTSUPP): - return copy_range(fileobj, offset, size, socket_file) - if err not in (errno.EINTR, errno.EAGAIN): - raise IOError((err, os.strerror(err))) - if num_bytes.value == 0: - break # EOF - total_sent += num_bytes.value - size -= num_bytes.value - offset += num_bytes.value - return total_sent - - def sendfile_to_socket_async(fileobj, offset, size, socket_file): - num_bytes = ctypes.c_int64(size) - ret = sendfile(fileobj.fileno(), socket_file.fileno(), offset, ctypes.byref(num_bytes), None, 0) - if ret != 0: - err = ctypes.get_errno() - if err in (errno.EBADF, errno.ENOTSUP, errno.ENOTSOCK, errno.EOPNOTSUPP): - raise CannotSendfile() - if err == errno.EINTR: - raise SendfileInterrupted() - if err != errno.EAGAIN: - raise IOError((err, os.strerror(err))) - return num_bytes.value - -elif islinux: - libc = ctypes.CDLL(None, use_errno=True) - sendfile = ctypes.CFUNCTYPE( - ctypes.c_ssize_t, ctypes.c_int, ctypes.c_int, ctypes.POINTER(ctypes.c_int64), ctypes.c_size_t, use_errno=True)(('sendfile64', libc)) - del libc - - def sendfile_to_socket(fileobj, offset, size, socket_file): - off = ctypes.c_int64(offset) - timeout = socket_file.gettimeout() - if timeout == 0: - return copy_range(fileobj, off.value, size, socket_file) - total_sent = 0 - while size > 0: - r, w, x = select([], [socket_file], [], timeout) - if not w: - raise socket.timeout('timed out in sendfile() waiting for socket to become writeable') - sent = sendfile(socket_file.fileno(), fileobj.fileno(), ctypes.byref(off), size) - if sent < 0: - err = ctypes.get_errno() - if err in (errno.ENOSYS, errno.EINVAL): - return copy_range(fileobj, off.value, size, socket_file) - if err not in (errno.EINTR, errno.EAGAIN): - raise IOError((err, os.strerror(err))) - elif sent == 0: - break # EOF - else: - size -= sent - total_sent += sent - return total_sent - - def sendfile_to_socket_async(fileobj, offset, size, socket_file): - off = ctypes.c_int64(offset) - sent = sendfile(socket_file.fileno(), fileobj.fileno(), ctypes.byref(off), size) - if sent < 0: - err = ctypes.get_errno() - if err in (errno.ENOSYS, errno.EINVAL): - raise CannotSendfile() - if err in (errno.EINTR, errno.EAGAIN): - raise SendfileInterrupted() - raise IOError((err, os.strerror(err))) - return sent