mirror of
https://github.com/kovidgoyal/calibre.git
synced 2025-07-09 03:04:10 -04:00
Use only a single file for repeated data URIs
This commit is contained in:
parent
e8fdd7a83c
commit
b5644ec6b0
@ -103,9 +103,10 @@ def sanitize_file_name(x):
|
|||||||
return make_filename_safe(x)
|
return make_filename_safe(x)
|
||||||
|
|
||||||
|
|
||||||
def download_one(tdir, timeout, progress_report, url):
|
def download_one(tdir, timeout, progress_report, data_uri_map, url):
|
||||||
try:
|
try:
|
||||||
purl = urlparse(url)
|
purl = urlparse(url)
|
||||||
|
data_url_key = None
|
||||||
with NamedTemporaryFile(dir=tdir, delete=False) as df:
|
with NamedTemporaryFile(dir=tdir, delete=False) as df:
|
||||||
if purl.scheme == 'file':
|
if purl.scheme == 'file':
|
||||||
src = lopen(purl.path, 'rb')
|
src = lopen(purl.path, 'rb')
|
||||||
@ -117,6 +118,10 @@ def download_one(tdir, timeout, progress_report, url):
|
|||||||
if parts and parts[-1].lower() == 'base64':
|
if parts and parts[-1].lower() == 'base64':
|
||||||
payload = re.sub(r'\s+', '', payload)
|
payload = re.sub(r'\s+', '', payload)
|
||||||
payload = standard_b64decode(payload)
|
payload = standard_b64decode(payload)
|
||||||
|
seen_before = data_uri_map.get(payload)
|
||||||
|
if seen_before is not None:
|
||||||
|
return True, (url, filename, seen_before, guess_type(seen_before))
|
||||||
|
data_url_key = payload
|
||||||
else:
|
else:
|
||||||
payload = payload.encode('utf-8')
|
payload = payload.encode('utf-8')
|
||||||
src = BytesIO(payload)
|
src = BytesIO(payload)
|
||||||
@ -137,6 +142,8 @@ def download_one(tdir, timeout, progress_report, url):
|
|||||||
dest = ProgressTracker(df, url, sz, progress_report)
|
dest = ProgressTracker(df, url, sz, progress_report)
|
||||||
with closing(src):
|
with closing(src):
|
||||||
shutil.copyfileobj(src, dest)
|
shutil.copyfileobj(src, dest)
|
||||||
|
if data_url_key is not None:
|
||||||
|
data_uri_map[data_url_key] = dest.name
|
||||||
filename = sanitize_file_name(filename)
|
filename = sanitize_file_name(filename)
|
||||||
mt = guess_type(filename)
|
mt = guess_type(filename)
|
||||||
if mt in OEB_DOCS:
|
if mt in OEB_DOCS:
|
||||||
@ -151,10 +158,11 @@ def download_one(tdir, timeout, progress_report, url):
|
|||||||
def download_external_resources(container, urls, timeout=60, progress_report=lambda url, done, total: None):
|
def download_external_resources(container, urls, timeout=60, progress_report=lambda url, done, total: None):
|
||||||
failures = {}
|
failures = {}
|
||||||
replacements = {}
|
replacements = {}
|
||||||
|
data_uri_map = {}
|
||||||
with TemporaryDirectory('editor-download') as tdir:
|
with TemporaryDirectory('editor-download') as tdir:
|
||||||
pool = Pool(10)
|
pool = Pool(10)
|
||||||
with closing(pool):
|
with closing(pool):
|
||||||
for ok, result in pool.imap_unordered(partial(download_one, tdir, timeout, progress_report), urls):
|
for ok, result in pool.imap_unordered(partial(download_one, tdir, timeout, progress_report, data_uri_map), urls):
|
||||||
if ok:
|
if ok:
|
||||||
url, suggested_filename, downloaded_file, mt = result
|
url, suggested_filename, downloaded_file, mt = result
|
||||||
with lopen(downloaded_file, 'rb') as src:
|
with lopen(downloaded_file, 'rb') as src:
|
||||||
|
@ -49,11 +49,13 @@ class ChooseResources(QWidget):
|
|||||||
self.items.clear()
|
self.items.clear()
|
||||||
self.original_resources = resources
|
self.original_resources = resources
|
||||||
dc = 0
|
dc = 0
|
||||||
for url in resources:
|
for url, matches in resources.iteritems():
|
||||||
text = url
|
text = url
|
||||||
|
num = len(matches)
|
||||||
if text.startswith('data:'):
|
if text.startswith('data:'):
|
||||||
dc += 1
|
dc += 1
|
||||||
text = _('Data URL ({})').format(dc)
|
text = _('Data URL #{}').format(dc)
|
||||||
|
text += ' ({})'.format(ngettext('one instance', '{} instances', num).format(num))
|
||||||
i = QListWidgetItem(text, self.items)
|
i = QListWidgetItem(text, self.items)
|
||||||
i.setData(Qt.UserRole, url)
|
i.setData(Qt.UserRole, url)
|
||||||
i.setCheckState(Qt.Checked)
|
i.setCheckState(Qt.Checked)
|
||||||
|
Loading…
x
Reference in New Issue
Block a user