E-book polishing: Add option to download external resources (images/stylesheets/etc.). Fixes #2067167 [Expose “Download External Resources” on command line](https://bugs.launchpad.net/calibre/+bug/2067167)

src/calibre/ebooks/oeb/polish/main.py

@@ -14,6 +14,7 @@ from functools import partial
 from calibre.ebooks.oeb.polish.container import get_container
 from calibre.ebooks.oeb.polish.cover import set_cover
 from calibre.ebooks.oeb.polish.css import remove_unused_css
+from calibre.ebooks.oeb.polish.download import download_external_resources, get_external_resources, replace_resources
 from calibre.ebooks.oeb.polish.embed import embed_all_fonts
 from calibre.ebooks.oeb.polish.hyphenation import add_soft_hyphens, remove_soft_hyphens
 from calibre.ebooks.oeb.polish.images import compress_images
@@ -22,6 +23,7 @@ from calibre.ebooks.oeb.polish.replace import smarten_punctuation
 from calibre.ebooks.oeb.polish.stats import StatsCollector
 from calibre.ebooks.oeb.polish.subset import iter_subsettable_fonts, subset_all_fonts
 from calibre.ebooks.oeb.polish.upgrade import upgrade_book
+from calibre.utils.localization import ngettext
 from calibre.utils.logging import Log
 from polyglot.builtins import iteritems
 
@@ -38,6 +40,7 @@ ALL_OPTS = {
     'upgrade_book': False,
     'add_soft_hyphens': False,
     'remove_soft_hyphens': False,
+    'download_external_resources': False,
 }
 
 CUSTOMIZATION = {
@@ -133,6 +136,12 @@ better when the text is justified, in readers that do not support hyphenation.</
 'remove_soft_hyphens': _('''\
 <p>Remove soft hyphens from all text in the book.</p>
 '''),
+
+'download_external_resources': _('''\
+<p>Download external resources such as images, stylesheets, etc. that point to URLs instead of files in the book.
+All such resources will be downloaded and added to the book so that the book no longer references any external resources.
+</p>
+'''),
 }
 
 
@@ -161,6 +170,30 @@ def update_metadata(ebook, new_opf):
         stream.write(opfbytes)
 
 
+def download_resources(ebook, report) -> bool:
+    changed = False
+    url_to_referrer_map = get_external_resources(ebook)
+    if url_to_referrer_map:
+        n = len(url_to_referrer_map)
+        report(ngettext('Downloading one external resource', 'Downloading {} external resources', n).format(n))
+        replacements, failures = download_external_resources(ebook, url_to_referrer_map)
+        if not failures:
+            report(_('Successfully downloaded all resources'))
+        else:
+            tb = [f'{url}\n\t{err}\n' for url, err in iteritems(failures)]
+            if replacements:
+                report(_('Failed to download some resources, see details below:'))
+            else:
+                report(_('Failed to download all resources, see details below:'))
+            report(tb)
+        if replacements:
+            if replace_resources(ebook, url_to_referrer_map, replacements):
+                changed = True
+    else:
+        report(_('No external resources found in book'))
+    return changed
+
+
 def polish_one(ebook, opts, report, customization=None):
     def rt(x):
         return report('\n### ' + x)
@@ -267,6 +300,16 @@ def polish_one(ebook, opts, report, customization=None):
         add_soft_hyphens(ebook, report)
         changed = True
 
+    if opts.download_external_resources:
+        rt(_('Downloading external resources'))
+        try:
+            download_resources(ebook, report)
+        except Exception:
+            import traceback
+            report(_('Failed to download resources with error:'))
+            report(traceback.format_exc())
+        report('')
+
     return changed
 
 
@@ -337,6 +380,7 @@ def option_parser():
     o('--add-soft-hyphens', '-H', help=CLI_HELP['add_soft_hyphens'])
     o('--remove-soft-hyphens', help=CLI_HELP['remove_soft_hyphens'])
     o('--upgrade-book', '-U', help=CLI_HELP['upgrade_book'])
+    o('--download-external-resources', '-d', help=CLI_HELP['download_external_resources'])
 
     o('--verbose', help=_('Produce more verbose output, useful for debugging.'))
 

src/calibre/gui2/actions/polish.py

@@ -87,6 +87,7 @@ class Polish(QDialog):  # {{{
             'remove_jacket':_('<h3>Remove book jacket</h3>%s')%HELP['remove_jacket'],
             'remove_unused_css':_('<h3>Remove unused CSS rules</h3>%s')%HELP['remove_unused_css'],
             'compress_images': _('<h3>Losslessly compress images</h3>%s') % HELP['compress_images'],
+            'download_external_resources': _('<h3>Download external resources</h3>%s') % HELP['download_external_resources'],
             'add_soft_hyphens': _('<h3>Add soft-hyphens</h3>%s') % HELP['add_soft_hyphens'],
             'remove_soft_hyphens': _('<h3>Remove soft-hyphens</h3>%s') % HELP['remove_soft_hyphens'],
             'upgrade_book': _('<h3>Upgrade book internals</h3>%s') % HELP['upgrade_book'],
@@ -109,6 +110,7 @@ class Polish(QDialog):  # {{{
             ('remove_jacket', _('&Remove a previously inserted book jacket')),
             ('remove_unused_css', _('Remove &unused CSS rules from the book')),
             ('compress_images', _('Losslessly &compress images')),
+            ('download_external_resources', _('&Download external resources')),
             ('add_soft_hyphens', _('Add s&oft hyphens')),
             ('remove_soft_hyphens', _('Remove so&ft hyphens')),
             ('upgrade_book', _('&Upgrade book internals')),