diff --git a/src/calibre/ebooks/oeb/polish/download.py b/src/calibre/ebooks/oeb/polish/download.py index 74cd7fc553..3e158c893f 100644 --- a/src/calibre/ebooks/oeb/polish/download.py +++ b/src/calibre/ebooks/oeb/polish/download.py @@ -43,8 +43,9 @@ def get_external_resources(container): for el, attr, link in iterhtmllinks(container, name): ans[link].append(name) elif media_type in OEB_STYLES: - for link in container.iterlinks(name): - ans[link].append(name) + for link in container.iterlinks(name, get_line_numbers=False): + if is_external(link): + ans[link].append(name) return dict(ans) @@ -99,8 +100,9 @@ def download_one(tdir, timeout, progress_report, url): src = urlopen(url, timeout=timeout) filename = get_filename(purl, src) sz = get_content_length(src) + progress_report(url, 0, sz) dest = ProgressTracker(df, url, sz, progress_report) - with src: + with closing(src): shutil.copyfileobj(src, dest) filename = sanitize_file_name2(filename) mt = guess_type(filename) @@ -110,7 +112,7 @@ def download_one(tdir, timeout, progress_report, url): raise ValueError('The external resource {} is not of a known type'.format(url)) return True, (url, sanitize_file_name2(filename), dest.name, mt) except Exception as err: - return False, url, as_unicode(err) + return False, (url, as_unicode(err)) def download_external_resources(container, urls, timeout=60, progress_report=lambda url, done, total: None): @@ -143,10 +145,11 @@ def replacer(url_map): def replace_resources(container, urls, replacements): url_maps = defaultdict(dict) changed = False - for url, name in urls.iteritems(): + for url, names in urls.iteritems(): replacement = replacements.get(url) if replacement is not None: - url_maps[name][url] = container.name_to_href(replacement, name) + for name in names: + url_maps[name][url] = container.name_to_href(replacement, name) for name, url_map in url_maps.iteritems(): r = replacer(url_map) container.replace_links(name, r) diff --git a/src/calibre/gui2/tweak_book/download.py b/src/calibre/gui2/tweak_book/download.py new file mode 100644 index 0000000000..8950fb41a0 --- /dev/null +++ b/src/calibre/gui2/tweak_book/download.py @@ -0,0 +1,251 @@ +#!/usr/bin/env python2 +# vim:fileencoding=utf-8 +# License: GPLv3 Copyright: 2016, Kovid Goyal + +from __future__ import (unicode_literals, division, absolute_import, + print_function) +from threading import Thread + +from PyQt5.Qt import ( + pyqtSignal, QWidget, QListWidget, QListWidgetItem, QLabel, Qt, + QVBoxLayout, QScrollArea, QProgressBar, QGridLayout, QSize) + +from calibre.gui2 import error_dialog, info_dialog, warning_dialog +from calibre.gui2.tweak_book import current_container +from calibre.gui2.tweak_book.widgets import Dialog +from calibre.gui2.progress_indicator import WaitStack +from calibre.ebooks.oeb.polish.download import get_external_resources, download_external_resources, replace_resources + + +class ChooseResources(QWidget): + + def __init__(self, parent=None): + QWidget.__init__(self, parent) + self.l = l = QVBoxLayout(self) + self.la = la = QLabel(_('Choose the external resources to download')) + la.setWordWrap(True) + l.addWidget(la) + self.items = i = QListWidget(self) + l.addWidget(i) + + def __iter__(self): + for i in xrange(self.items.count()): + yield self.items.item(i) + + def select_none(self): + for item in self: + item.setCheckState(Qt.Unchecked) + + def select_all(self): + for item in self: + item.setCheckState(Qt.Checked) + + @property + def resources(self): + return {i.text():self.original_resources[i.text()] for i in self if i.checkState() == Qt.Checked} + + @resources.setter + def resources(self, resources): + self.items.clear() + self.original_resources = resources + for url in resources: + i = QListWidgetItem(url, self.items) + i.setCheckState(Qt.Checked) + i.setFlags(Qt.ItemIsUserCheckable | Qt.ItemIsEnabled) + + +class DownloadStatus(QScrollArea): + + def __init__(self, parent=None): + QScrollArea.__init__(self, parent) + self.setWidgetResizable(True) + self.w = QWidget(self) + self.l = QGridLayout(self.w) + self.setWidget(self.w) + + def __call__(self, resources): + self.url_map = {} + self.labels = [] + for url in resources: + p = self.url_map[url] = QProgressBar(self.w) + p.setRange(0, 0) + self.l.addWidget(p, self.l.rowCount(), 0) + la = QLabel('\xa0' + url) + self.labels.append(la) + self.l.addWidget(la, self.l.rowCount()-1, 1) + self.l.addWidget(QLabel('')) + self.l.setRowStretch(self.l.rowCount()-1, 10) + + def progress(self, url, done, total): + p = self.url_map.get(url) + if p is not None: + if total > 0: + p.setRange(0, total) + p.setValue(done) + else: + p.setRange(0, 0) + + +class DownloadResources(Dialog): + + get_done = pyqtSignal(object, object) + progress = pyqtSignal(object, object, object) + download_done = pyqtSignal(object, object) + replace_done = pyqtSignal(object, object) + + def __init__(self, parent=None): + self.resources_replaced = False + Dialog.__init__(self, _('Download external resources'), 'download-external-resources', parent) + self.state = 0 + self.get_done.connect(self._get_done) + self.download_done.connect(self._download_done) + self.replace_done.connect(self._replace_done) + self.progress.connect(self.download_status.progress, type=Qt.QueuedConnection) + + def setup_ui(self): + self.choose_resources = cr = ChooseResources(self) + self.download_status = ds = DownloadStatus(self) + self.success = s = QLabel('') + s.setWordWrap(True) + self.l = l = QVBoxLayout(self) + self.wait = WaitStack(_('Searching for external resources...'), cr, self) + self.wait.addWidget(ds), self.wait.addWidget(s) + self.wait.start() + for t, f in ((_('Select &None'), cr.select_none), (_('Select &All'), cr.select_all)): + b = self.bb.addButton(t, self.bb.ActionRole) + b.clicked.connect(f), b.setAutoDefault(False) + self.bb.setVisible(False) + l.addWidget(self.wait), l.addWidget(self.bb) + t = Thread(name='GetResources', target=self.get_resources) + t.daemon = True + t.start() + + def get_resources(self): + tb = None + try: + ret = get_external_resources(current_container()) + except Exception as err: + import traceback + ret, tb = err, traceback.format_exc() + self.get_done.emit(ret, tb) + + def _get_done(self, x, tb): + if not self.isVisible(): + return self.reject() + if tb is not None: + error_dialog(self, _('Scan failed'), _( + 'Failed to scan for external resources, click "Show Details" for more information.'), + det_msg=tb, show=True) + self.reject() + else: + self.wait.stop() + self.state = 1 + resources = x + if not resources: + info_dialog(self, _('No external resources found'), _( + 'No external resources were found in this book.'), show=True) + self.reject() + return + self.choose_resources.resources = resources + self.bb.setVisible(True) + + def download_resources(self, resources): + tb = None + try: + ret = download_external_resources(current_container(), resources, progress_report=self.progress.emit) + except Exception as err: + import traceback + ret, tb = err, traceback.format_exc() + self.download_done.emit(ret, tb) + + def _download_done(self, ret, tb): + if not self.isVisible(): + return self.reject() + if tb is not None: + error_dialog(self, _('Download failed'), _( + 'Failed to download external resources, click "Show Details" for more information.'), + det_msg=tb, show=True) + self.reject() + else: + replacements, failures = ret + if failures: + tb = ['{}\n\t{}\n'.format(url, err) for url, err in failures.iteritems()] + if not replacements: + error_dialog(self, _('Download failed'), _( + 'Failed to download external resources, click "Show Details" for more information.'), + det_msg='\n'.join(tb), show=True) + self.reject() + return + else: + warning_dialog(self, _('Some downloads failed'), _( + 'Failed to download some external resources, click "Show Details" for more information.'), + det_msg='\n'.join(tb), show=True) + self.state = 2 + self.wait.msg = _('Updating resources in book...') + self.wait.start() + self.success.setText('

' + ngettext( + 'Successfully processed the external resource', 'Successfully processed {} external resources', len(replacements)).format(len(replacements))) + resources = self.choose_resources.resources + t = Thread(name='ReplaceResources', target=self.replace_resources, args=(resources, replacements)) + t.daemon = True + t.start() + + def replace_resources(self, resources, replacements): + tb = None + try: + ret = replace_resources(current_container(), resources, replacements) + except Exception as err: + import traceback + ret, tb = err, traceback.format_exc() + self.replace_done.emit(ret, tb) + + def _replace_done(self, ret, tb): + if tb is not None: + error_dialog(self, _('Replace failed'), _( + 'Failed to replace external resources, click "Show Details" for more information.'), + det_msg=tb, show=True) + Dialog.reject(self) + else: + self.wait.setCurrentIndex(3) + self.state = 3 + self.bb.clear() + self.resources_replaced = True + self.bb.setStandardButtons(self.bb.Close) + self.bb.setVisible(True) + + def accept(self): + if self.state == 0: + return self.reject() + if self.state == 1: + resources = self.choose_resources.resources + self.download_status(resources) + self.wait.setCurrentIndex(2) + self.bb.setVisible(False) + t = Thread(name='DownloadResources', target=self.download_resources, args=(resources,)) + t.daemon = True + t.start() + return + if self.state == 2: + return + self.wait.stop() + Dialog.accept(self) + + def reject(self): + if self.state == 2: + return + self.wait.stop() + return Dialog.reject(self) + + def sizeHint(self): + return QSize(800, 500) + +if __name__ == '__main__': + from calibre.gui2 import Application + import sys + app = Application([]) + from calibre.gui2.tweak_book import set_current_container + from calibre.gui2.tweak_book.boss import get_container + set_current_container(get_container(sys.argv[-1])) + d = DownloadResources() + d.exec_() + del d, app