Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-09 03:04:10 -04:00
Code Issues Packages Projects Releases Wiki Activity

Do not use ImageMagick in the news download subsystem

Browse Source
This commit is contained in:
Kovid Goyal 2016-05-04 14:54:48 +05:30
parent 57d53a8a3b
commit db47669c10
3 changed files with 77 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
src/calibre/utils/img.py
View File

@ -23,23 +23,37 @@ def get_exe_path(name):
return os.path.join(base, name)
def image_from_data(data):
if isinstance(data, QImage):
return data
i = QImage()
if not i.loadFromData(data):
raise ValueError('Not a valid image')
return i
def image_to_data(img, compression_quality=95, fmt='JPEG'):
ba = QByteArray()
buf = QBuffer(ba)
buf.open(QBuffer.WriteOnly)
fmt = fmt.upper()
if img.hasAlphaChannel() and fmt in 'JPEG JPG'.split():
nimg = QImage(img.size(), QImage.Format_RGB32)
nimg.fill(Qt.white)
p = QPainter(nimg)
p.drawImage(0, 0, img)
p.end()
img = nimg
if not img.save(buf, fmt, quality=compression_quality):
raise ValueError('Failed to export image as ' + fmt)
return ba.data()
def scale_image(data, width=60, height=80, compression_quality=70, as_png=False, preserve_aspect_ratio=True):
''' Scale an image, returning it as either JPEG or PNG data (bytestring).
Transparency is alpha blended with white when converting to JPEG. Is thread
safe and does not require a QApplication. '''
# We use Qt instead of ImageMagick here because ImageMagick seems to use
# some kind of memory pool, causing memory consumption to sky rocket.
if isinstance(data, QImage):
img = data
else:
img = QImage()
if not img.loadFromData(data):
raise ValueError('Could not load image for thumbnail generation')
img = image_from_data(data)
if preserve_aspect_ratio:
scaled, nwidth, nheight = fit_image(img.width(), img.height(), width, height)
if scaled:
@ -47,20 +61,9 @@ def scale_image(data, width=60, height=80, compression_quality=70, as_png=False,
else:
if img.width() != width or img.height() != height:
img = img.scaled(width, height, Qt.IgnoreAspectRatio, Qt.SmoothTransformation)
if not as_png and img.hasAlphaChannel():
nimg = QImage(img.size(), QImage.Format_RGB32)
nimg.fill(Qt.white)
p = QPainter(nimg)
p.drawImage(0, 0, img)
p.end()
img = nimg
ba = QByteArray()
buf = QBuffer(ba)
buf.open(QBuffer.WriteOnly)
fmt = 'PNG' if as_png else 'JPEG'
if not img.save(buf, fmt, quality=compression_quality):
raise ValueError('Failed to export thumbnail image to: ' + fmt)
return img.width(), img.height(), ba.data()
w, h = img.width(), img.height()
return w, h, image_to_data(img, compression_quality=compression_quality, fmt=fmt)
def run_optimizer(file_path, cmd, as_filter=False, input_data=None):

59
src/calibre/web/fetch/simple.py
View File

@ -12,16 +12,16 @@ from urllib import url2pathname, quote
from httplib import responses
from base64 import b64decode
from calibre import browser, relpath, unicode_path, fit_image
from calibre import browser, relpath, unicode_path
from calibre.constants import filesystem_encoding, iswindows
from calibre.utils.filenames import ascii_filename
from calibre.ebooks.BeautifulSoup import BeautifulSoup, Tag
from calibre.ebooks.chardet import xml_to_unicode
from calibre.utils.config import OptionParser
from calibre.utils.logging import Log
from calibre.utils.magick import Image
from calibre.utils.magick.draw import identify_data, thumbnail
from calibre.utils.img import image_from_data, image_to_data
from calibre.utils.imghdr import what
from calibre.web.fetch.utils import rescale_image
class AbortArticle(Exception):
pass
@ -349,41 +349,7 @@ class RecursiveFetcher(object):
ns.replaceWith(src.replace(m.group(1), stylepath))
def rescale_image(self, data):
orig_w, orig_h, ifmt = identify_data(data)
orig_data = data # save it in case compression fails
if self.scale_news_images is not None:
wmax, hmax = self.scale_news_images
scale, new_w, new_h = fit_image(orig_w, orig_h, wmax, hmax)
if scale:
data = thumbnail(data, new_w, new_h, compression_quality=95)[-1]
orig_w = new_w
orig_h = new_h
if self.compress_news_images_max_size is None:
if self.compress_news_images_auto_size is None: # not compressing
return data
else:
maxsizeb = (orig_w * orig_h)/self.compress_news_images_auto_size
else:
maxsizeb = self.compress_news_images_max_size * 1024
scaled_data = data # save it in case compression fails
if len(scaled_data) <= maxsizeb: # no compression required
return scaled_data
img = Image()
quality = 95
img.load(data)
while len(data) >= maxsizeb and quality >= 5:
quality -= 5
img.set_compression_quality(quality)
data = img.export('jpg')
if len(data) >= len(scaled_data): # compression failed
return orig_data if len(orig_data) <= len(scaled_data) else scaled_data
if len(data) >= len(orig_data): # no improvement
return orig_data
return data
return rescale_image(data, self.scale_news_images, self.compress_news_images_max_size, self.compress_news_images_auto_size)
def process_images(self, soup, baseurl):
diskpath = unicode_path(os.path.join(self.current_dir, 'images'))
@ -430,19 +396,16 @@ class RecursiveFetcher(object):
tag['src'] = imgpath
else:
try:
# Ensure image is valid
img = image_from_data(data)
if itype not in {'png', 'jpg', 'jpeg'}:
itype = 'png' if itype == 'gif' else 'jpg'
im = Image()
im.load(data)
data = im.export(itype)
itype = 'png' if itype == 'gif' else 'jpeg'
data = image_to_data(img, fmt=itype)
if self.compress_news_images and itype in {'jpg','jpeg'}:
try:
data = self.rescale_image(data)
except:
data = self.rescale_image(img)
except Exception:
self.log.exception('failed to compress image '+iurl)
identify_data(data)
else:
identify_data(data)
# Moon+ apparently cannot handle .jpeg files
if itype == 'jpeg':
itype = 'jpg'
@ -452,7 +415,7 @@ class RecursiveFetcher(object):
with open(imgpath, 'wb') as x:
x.write(data)
tag['src'] = imgpath
except:
except Exception:
traceback.print_exc()
continue

44
src/calibre/web/fetch/utils.py Normal file
View File

@ -0,0 +1,44 @@
#!/usr/bin/env python2
# vim:fileencoding=utf-8
# License: GPLv3 Copyright: 2016, Kovid Goyal <kovid at kovidgoyal.net>
from __future__ import (unicode_literals, division, absolute_import,
print_function)
from calibre.utils.img import image_from_data, scale_image, image_to_data
def rescale_image(data, scale_news_images, compress_news_images_max_size, compress_news_images_auto_size):
orig_data = data # save it in case compression fails
img = image_from_data(data)
orig_w, orig_h = img.width(), img.height()
if scale_news_images is not None:
wmax, hmax = scale_news_images
if wmax < orig_w or hmax < orig_h:
orig_w, orig_h, data = scale_image(img, wmax, hmax, compression_quality=95)
if compress_news_images_max_size is None:
if compress_news_images_auto_size is None: # not compressing
return data
maxsizeb = (orig_w * orig_h)/compress_news_images_auto_size
else:
maxsizeb = compress_news_images_max_size * 1024
if len(data) <= maxsizeb: # no compression required
return data
scaled_data = data # save it in case compression fails
quality = 90
while len(data) >= maxsizeb and quality >= 5:
data = image_to_data(image_from_data(scaled_data), compression_quality=quality)
quality -= 5
if len(data) >= len(scaled_data): # compression failed
return orig_data if len(orig_data) <= len(scaled_data) else scaled_data
if len(data) >= len(orig_data): # no improvement
return orig_data
return data
if __name__ == '__main__':
import sys
data = sys.stdin.read()
sys.stdout.write(rescale_image(data, (768, 1024), None, 8))