Fix write_iter() not working if the iterator yields empty chunks

Caused gzip transfer encoding to break for data larger than the buffer
size
This commit is contained in:
Kovid Goyal 2015-06-14 12:13:08 +05:30
parent 2fcd2fd85e
commit dd5f24db21
2 changed files with 14 additions and 6 deletions

View File

@ -27,6 +27,7 @@ from calibre.utils.monotonic import monotonic
Range = namedtuple('Range', 'start stop size') Range = namedtuple('Range', 'start stop size')
MULTIPART_SEPARATOR = uuid.uuid4().hex.decode('ascii') MULTIPART_SEPARATOR = uuid.uuid4().hex.decode('ascii')
COMPRESSIBLE_TYPES = {'application/json', 'application/javascript', 'application/xml', 'application/oebps-package+xml'}
def header_list_to_file(buf): # {{{ def header_list_to_file(buf): # {{{
buf.append('') buf.append('')
@ -514,10 +515,14 @@ class HTTPConnection(HTTPRequest):
if chunk is None: if chunk is None:
self.set_state(WRITE, self.write_chunk, BytesIO(b'0\r\n\r\n'), output, last=True) self.set_state(WRITE, self.write_chunk, BytesIO(b'0\r\n\r\n'), output, last=True)
else: else:
if not isinstance(chunk, bytes): if chunk:
chunk = chunk.encode('utf-8') if not isinstance(chunk, bytes):
chunk = ('%X\r\n' % len(chunk)).encode('ascii') + chunk + b'\r\n' chunk = chunk.encode('utf-8')
self.set_state(WRITE, self.write_chunk, BytesIO(chunk), output) chunk = ('%X\r\n' % len(chunk)).encode('ascii') + chunk + b'\r\n'
self.set_state(WRITE, self.write_chunk, BytesIO(chunk), output)
else:
# Empty chunk, ignore it
self.write_iter(output, event)
def write_chunk(self, buf, output, event, last=False): def write_chunk(self, buf, output, event, last=False):
if self.write(buf): if self.write(buf):
@ -556,7 +561,7 @@ class HTTPConnection(HTTPRequest):
output = GeneratedOutput(output) output = GeneratedOutput(output)
ct = outheaders.get('Content-Type', '').partition(';')[0] ct = outheaders.get('Content-Type', '').partition(';')[0]
compressible = (not ct or ct.startswith('text/') or ct.startswith('image/svg') or compressible = (not ct or ct.startswith('text/') or ct.startswith('image/svg') or
ct in {'application/json', 'application/javascript', 'application/xml'}) ct.partition(';')[0] in COMPRESSIBLE_TYPES)
compressible = (compressible and request.status_code == httplib.OK and compressible = (compressible and request.status_code == httplib.OK and
(opts.compress_min_size > -1 and output.content_length >= opts.compress_min_size) and (opts.compress_min_size > -1 and output.content_length >= opts.compress_min_size) and
acceptable_encoding(request.inheaders.get('Accept-Encoding', '')) and not is_http1) acceptable_encoding(request.inheaders.get('Accept-Encoding', '')) and not is_http1)

View File

@ -308,9 +308,12 @@ class TestHTTP(BaseTest):
self.ae(r.read(), b'') self.ae(r.read(), b'')
# Test gzip # Test gzip
raw = b'a'*20000
server.change_handler(lambda conn: raw)
conn = server.connect()
conn.request('GET', '/an_etagged_path', headers={'Accept-Encoding':'gzip'}) conn.request('GET', '/an_etagged_path', headers={'Accept-Encoding':'gzip'})
r = conn.getresponse() r = conn.getresponse()
self.ae(r.status, httplib.OK), self.ae(zlib.decompress(r.read(), 16+zlib.MAX_WBITS), b'an_etagged_path') self.ae(r.status, httplib.OK), self.ae(zlib.decompress(r.read(), 16+zlib.MAX_WBITS), raw)
# Test getting a filesystem file # Test getting a filesystem file
for use_sendfile in (True, False): for use_sendfile in (True, False):