From d81c912b07e9c3d671f2a8e30046c064a27f9abd Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 9 Feb 2020 19:16:15 +0530 Subject: [PATCH 001/162] String changes --- Changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Changelog.yaml b/Changelog.yaml index ed0d4ba0a3..c1c60664bd 100644 --- a/Changelog.yaml +++ b/Changelog.yaml @@ -2083,7 +2083,7 @@ to appear as Unknown if metadata management was set to manual in calibre." - title: "Content server: Open links in the comments section from the book details page in new windows." tickets: [1737644] - - title: "Choose English as the User interface language when a locale related environment variable is set to the C locale" + - title: "Choose English as the user interface language when a locale related environment variable is set to the C locale" - title: "Linux installer: A nicer error message if the user tries to run the installer on an ARM machine" From 15c6ee80de0a69002c9ae135f48a48567002756c Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 9 Feb 2020 19:35:57 +0530 Subject: [PATCH 002/162] Fix clicking on author name in book details panel to search in goodreads not working if author has more than two parts in his name Fixes #1096 (Encode internet search URLs with spaces as %20 rather than +) --- src/calibre/ebooks/metadata/search_internet.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/calibre/ebooks/metadata/search_internet.py b/src/calibre/ebooks/metadata/search_internet.py index 2b45caba0d..bb962e83eb 100644 --- a/src/calibre/ebooks/metadata/search_internet.py +++ b/src/calibre/ebooks/metadata/search_internet.py @@ -5,7 +5,7 @@ from __future__ import absolute_import, division, print_function, unicode_literals from polyglot.builtins import iteritems -from polyglot.urllib import quote_plus +from polyglot.urllib import quote, quote_plus AUTHOR_SEARCHES = { 'goodreads': @@ -48,17 +48,21 @@ all_book_searches = BOOK_SEARCHES.__iter__ all_author_searches = AUTHOR_SEARCHES.__iter__ -def qquote(val): +def qquote(val, use_plus=True): if not isinstance(val, bytes): val = val.encode('utf-8') - ans = quote_plus(val) + ans = quote_plus(val) if use_plus else quote(val) if isinstance(ans, bytes): ans = ans.decode('utf-8') return ans +def specialised_quote(template, val): + return qquote(val, 'goodreads.com' not in template) + + def url_for(template, data): - return template.format(**{k: qquote(v) for k, v in iteritems(data)}) + return template.format(**{k: specialised_quote(template, v) for k, v in iteritems(data)}) def url_for_author_search(key, **kw): From 5bede4e36449a6d8df8d195d056da84e341fc03e Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 9 Feb 2020 22:10:13 +0530 Subject: [PATCH 003/162] Ensure shortcut action for context menus uses an updated icon actions from third party plugins set their icons in genesis() so the action will not have an icon --- src/calibre/gui2/bars.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/calibre/gui2/bars.py b/src/calibre/gui2/bars.py index 046004557b..61e358ac6f 100644 --- a/src/calibre/gui2/bars.py +++ b/src/calibre/gui2/bars.py @@ -413,6 +413,7 @@ if isosx: ia = iactions[what] ac = ia.qaction if not ac.menu() and hasattr(ia, 'shortcut_action_for_context_menu'): + ia.shortcut_action_for_context_menu.setIcon(ac.icon()) ac = ia.shortcut_action_for_context_menu m.addAction(CloneAction(ac, m)) @@ -506,6 +507,7 @@ else: ia = iactions[what] ac = ia.qaction if not ac.menu() and hasattr(ia, 'shortcut_action_for_context_menu'): + ia.shortcut_action_for_context_menu.setIcon(ac.icon()) ac = ia.shortcut_action_for_context_menu m.addAction(ac) From df58f715d824cdb1582d7f68c27a22e48029741f Mon Sep 17 00:00:00 2001 From: simonvg Date: Sun, 9 Feb 2020 22:17:42 +0100 Subject: [PATCH 004/162] Move the discarding of the loading flag later on the content loading process --- src/pyj/read_book/view.pyj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pyj/read_book/view.pyj b/src/pyj/read_book/view.pyj index 16cd2da6c6..a1fd39f8e8 100644 --- a/src/pyj/read_book/view.pyj +++ b/src/pyj/read_book/view.pyj @@ -1018,7 +1018,6 @@ class View: self.iframe_wrapper.init() def show_spine_item_stage2(self, resource_data): - self.currently_showing.loading = False # We cannot encrypt this message because the resource data contains # Blob objects which do not survive encryption self.processing_spine_item_display = True @@ -1032,6 +1031,7 @@ class View: def on_content_loaded(self, data): self.processing_spine_item_display = False + self.currently_showing.loading = False self.hide_loading() self.set_progress_frac(data.progress_frac, data.file_progress_frac) self.update_header_footer() From e10dc504cc9390391e4a1254f04e4223c51445c0 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 10 Feb 2020 07:19:06 +0530 Subject: [PATCH 005/162] py3: Fix error in Kobo driver when sorting collections Fixes #1098 (py3: Ignore TypeError when sorting device collections for kobo driver) --- src/calibre/devices/kobo/books.py | 26 ++-------------- src/calibre/devices/usbms/books.py | 49 +++++++++++++++--------------- 2 files changed, 28 insertions(+), 47 deletions(-) diff --git a/src/calibre/devices/kobo/books.py b/src/calibre/devices/kobo/books.py index f725751d00..89ff9c8411 100644 --- a/src/calibre/devices/kobo/books.py +++ b/src/calibre/devices/kobo/books.py @@ -7,16 +7,14 @@ import os, time, sys from functools import cmp_to_key from calibre.constants import preferred_encoding, DEBUG, ispy3 -from calibre import isbytestring, force_unicode -from calibre.utils.icu import sort_key +from calibre import isbytestring from calibre.ebooks.metadata.book.base import Metadata -from calibre.devices.usbms.books import Book as Book_ -from calibre.devices.usbms.books import CollectionsBookList +from calibre.devices.usbms.books import Book as Book_, CollectionsBookList, none_cmp from calibre.utils.config_base import prefs from calibre.devices.usbms.driver import debug_print from calibre.ebooks.metadata import author_to_author_sort -from polyglot.builtins import unicode_type, string_or_bytes, iteritems, itervalues, cmp +from polyglot.builtins import unicode_type, iteritems, itervalues class Book(Book_): @@ -292,24 +290,6 @@ class KTCollectionsBookList(CollectionsBookList): # Sort collections result = {} - def none_cmp(xx, yy): - x = xx[1] - y = yy[1] - if x is None and y is None: - # No sort_key needed here, because defaults are ascii - return cmp(xx[2], yy[2]) - if x is None: - return 1 - if y is None: - return -1 - if isinstance(x, string_or_bytes) and isinstance(y, string_or_bytes): - x, y = sort_key(force_unicode(x)), sort_key(force_unicode(y)) - c = cmp(x, y) - if c != 0: - return c - # same as above -- no sort_key needed here - return cmp(xx[2], yy[2]) - for category, lpaths in iteritems(collections): books = sorted(itervalues(lpaths), key=cmp_to_key(none_cmp)) result[category] = [x[0] for x in books] diff --git a/src/calibre/devices/usbms/books.py b/src/calibre/devices/usbms/books.py index 4777d86cf4..bbb89e316c 100644 --- a/src/calibre/devices/usbms/books.py +++ b/src/calibre/devices/usbms/books.py @@ -20,6 +20,31 @@ from calibre.utils.icu import sort_key from polyglot.builtins import string_or_bytes, iteritems, itervalues, cmp +def none_cmp(xx, yy): + x = xx[1] + y = yy[1] + if x is None and y is None: + # No sort_key needed here, because defaults are ascii + return cmp(xx[2], yy[2]) + if x is None: + return 1 + if y is None: + return -1 + if isinstance(x, string_or_bytes) and isinstance(y, string_or_bytes): + x, y = sort_key(force_unicode(x)), sort_key(force_unicode(y)) + try: + c = cmp(x, y) + except TypeError: + c = 0 + if c != 0: + return c + # same as above -- no sort_key needed here + try: + return cmp(xx[2], yy[2]) + except TypeError: + return 0 + + class Book(Metadata): def __init__(self, prefix, lpath, size=None, other=None): @@ -280,30 +305,6 @@ class CollectionsBookList(BookList): # Sort collections result = {} - def none_cmp(xx, yy): - x = xx[1] - y = yy[1] - if x is None and y is None: - # No sort_key needed here, because defaults are ascii - return cmp(xx[2], yy[2]) - if x is None: - return 1 - if y is None: - return -1 - if isinstance(x, string_or_bytes) and isinstance(y, string_or_bytes): - x, y = sort_key(force_unicode(x)), sort_key(force_unicode(y)) - try: - c = cmp(x, y) - except TypeError: - c = 0 - if c != 0: - return c - # same as above -- no sort_key needed here - try: - return cmp(xx[2], yy[2]) - except TypeError: - return 0 - for category, lpaths in iteritems(collections): books = sorted(itervalues(lpaths), key=cmp_to_key(none_cmp)) result[category] = [x[0] for x in books] From 00f921490b6ee3f94303faaf603eaef53ac6d8ab Mon Sep 17 00:00:00 2001 From: Sophist <3001893+Sophist-UK@users.noreply.github.com> Date: Mon, 10 Feb 2020 08:39:35 +0000 Subject: [PATCH 006/162] Fix series sort after using clear_series If you have books with series and series_index > 1, and use bulk metadata to clear_series, then sort by series puts these at the bottom of the list because the series_index has not been reset to 1. --- src/calibre/gui2/dialogs/metadata_bulk.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/calibre/gui2/dialogs/metadata_bulk.py b/src/calibre/gui2/dialogs/metadata_bulk.py index 24d73f465e..42a6b7f960 100644 --- a/src/calibre/gui2/dialogs/metadata_bulk.py +++ b/src/calibre/gui2/dialogs/metadata_bulk.py @@ -369,6 +369,7 @@ class MyBlockingBusy(QDialog): # {{{ if args.clear_series: self.progress_next_step_range.emit(0) cache.set_field('series', {bid: '' for bid in self.ids}) + cache.set_field('series_index', {bid:1.0 for bid in self.ids}) self.progress_finished_cur_step.emit() if args.pubdate is not None: From d05ac8093cdd6626e23441314e65db5dbcbf18a3 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 10 Feb 2020 15:14:02 +0530 Subject: [PATCH 007/162] Fix #1862591 [QResizeEvent error when changing libraries](https://bugs.launchpad.net/calibre/+bug/1862591) --- src/calibre/gui2/dbus_export/menu.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/gui2/dbus_export/menu.py b/src/calibre/gui2/dbus_export/menu.py index 96daa6fe28..d6b4164a9e 100644 --- a/src/calibre/gui2/dbus_export/menu.py +++ b/src/calibre/gui2/dbus_export/menu.py @@ -167,7 +167,7 @@ class DBusMenu(QObject): def eventFilter(self, obj, ev): ac = getattr(obj, 'menuAction', lambda : None)() ac_id = self.action_to_id(ac) - if ac_id is not None: + if ac_id is not None and hasattr(ev, 'action'): etype = ev.type() if etype == QEvent.ActionChanged: ac_id = self.action_to_id(ev.action()) From 449672bab1ccbe342bd0ce95076f18575c43bc24 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 10 Feb 2020 20:41:23 +0530 Subject: [PATCH 008/162] Make plugins_mirror.py more polyglot --- setup/plugins_mirror.py | 59 ++++++++++++++++++++++++++--------------- 1 file changed, 37 insertions(+), 22 deletions(-) diff --git a/setup/plugins_mirror.py b/setup/plugins_mirror.py index 3b5b6cc695..bd6c40b04a 100644 --- a/setup/plugins_mirror.py +++ b/setup/plugins_mirror.py @@ -10,7 +10,6 @@ import bz2 import errno import glob import gzip -import HTMLParser import io import json import os @@ -22,8 +21,6 @@ import subprocess import sys import tempfile import time -import urllib2 -import urlparse import zipfile import zlib from collections import namedtuple @@ -33,6 +30,24 @@ from email.utils import parsedate from functools import partial from multiprocessing.pool import ThreadPool from xml.sax.saxutils import escape, quoteattr + +try: + from html import unescape as u +except ImportError: + from HTMLParser import HTMLParser + u = HTMLParser().unescape + +try: + from urllib.parse import parse_qs, urlparse +except ImportError: + from urlparse import parse_qs, urlparse + + +try: + from urllib.error import URLError + from urllib.request import urlopen, Request, build_opener +except Exception: + from urllib2 import urlopen, Request, build_opener, URLError # }}} USER_AGENT = 'calibre mirror' @@ -44,15 +59,13 @@ INDEX = MR_URL + 'showpost.php?p=1362767&postcount=1' # INDEX = 'file:///t/raw.html' IndexEntry = namedtuple('IndexEntry', 'name url donate history uninstall deprecated thread_id') -u = HTMLParser.HTMLParser().unescape - socket.setdefaulttimeout(30) def read(url, get_info=False): # {{{ if url.startswith("file://"): - return urllib2.urlopen(url).read() - opener = urllib2.build_opener() + return urlopen(url).read() + opener = build_opener() opener.addheaders = [ ('User-Agent', USER_AGENT), ('Accept-Encoding', 'gzip,deflate'), @@ -62,7 +75,7 @@ def read(url, get_info=False): # {{{ try: res = opener.open(url) break - except urllib2.URLError as e: + except URLError as e: if not isinstance(e.reason, socket.timeout) or i == 9: raise time.sleep(random.randint(10, 45)) @@ -82,7 +95,7 @@ def read(url, get_info=False): # {{{ def url_to_plugin_id(url, deprecated): - query = urlparse.parse_qs(urlparse.urlparse(url).query) + query = parse_qs(urlparse(url).query) ans = (query['t'] if 't' in query else query['p'])[0] if deprecated: ans += '-deprecated' @@ -149,11 +162,13 @@ def convert_node(fields, x, names={}, import_data=None): return x.s.decode('utf-8') if isinstance(x.s, bytes) else x.s elif name == 'Num': return x.n + elif name == 'Constant': + return x.value elif name in {'Set', 'List', 'Tuple'}: func = {'Set':set, 'List':list, 'Tuple':tuple}[name] - return func(map(conv, x.elts)) + return func(list(map(conv, x.elts))) elif name == 'Dict': - keys, values = map(conv, x.keys), map(conv, x.values) + keys, values = list(map(conv, x.keys)), list(map(conv, x.values)) return dict(zip(keys, values)) elif name == 'Call': if len(x.args) != 1 and len(x.keywords) != 0: @@ -182,7 +197,7 @@ def get_import_data(name, mod, zf, names): if mod in names: raw = zf.open(names[mod]).read() module = ast.parse(raw, filename='__init__.py') - top_level_assigments = filter(lambda x:x.__class__.__name__ == 'Assign', ast.iter_child_nodes(module)) + top_level_assigments = [x for x in ast.iter_child_nodes(module) if x.__class__.__name__ == 'Assign'] for node in top_level_assigments: targets = {getattr(t, 'id', None) for t in node.targets} targets.discard(None) @@ -196,9 +211,9 @@ def get_import_data(name, mod, zf, names): def parse_metadata(raw, namelist, zf): module = ast.parse(raw, filename='__init__.py') - top_level_imports = filter(lambda x:x.__class__.__name__ == 'ImportFrom', ast.iter_child_nodes(module)) - top_level_classes = tuple(filter(lambda x:x.__class__.__name__ == 'ClassDef', ast.iter_child_nodes(module))) - top_level_assigments = filter(lambda x:x.__class__.__name__ == 'Assign', ast.iter_child_nodes(module)) + top_level_imports = [x for x in ast.iter_child_nodes(module) if x.__class__.__name__ == 'ImportFrom'] + top_level_classes = tuple(x for x in ast.iter_child_nodes(module) if x.__class__.__name__ == 'ClassDef') + top_level_assigments = [x for x in ast.iter_child_nodes(module) if x.__class__.__name__ == 'Assign'] defaults = { 'name':'', 'description':'', 'supported_platforms':['windows', 'osx', 'linux'], @@ -226,7 +241,7 @@ def parse_metadata(raw, namelist, zf): plugin_import_found |= inames else: all_imports.append((mod, [n.name for n in names])) - imported_names[n.asname or n.name] = mod + imported_names[names[-1].asname or names[-1].name] = mod if not plugin_import_found: return all_imports @@ -245,7 +260,7 @@ def parse_metadata(raw, namelist, zf): names[x] = val def parse_class(node): - class_assigments = filter(lambda x:x.__class__.__name__ == 'Assign', ast.iter_child_nodes(node)) + class_assigments = [x for x in ast.iter_child_nodes(node) if x.__class__.__name__ == 'Assign'] found = {} for node in class_assigments: targets = {getattr(t, 'id', None) for t in node.targets} @@ -337,7 +352,7 @@ def update_plugin_from_entry(plugin, entry): def fetch_plugin(old_index, entry): lm_map = {plugin['thread_id']:plugin for plugin in old_index.values()} - raw = read(entry.url) + raw = read(entry.url).decode('utf-8', 'replace') url, name = parse_plugin_zip_url(raw) if url is None: raise ValueError('Failed to find zip file URL for entry: %s' % repr(entry)) @@ -346,9 +361,9 @@ def fetch_plugin(old_index, entry): if plugin is not None: # Previously downloaded plugin lm = datetime(*tuple(map(int, re.split(r'\D', plugin['last_modified'])))[:6]) - request = urllib2.Request(url) + request = Request(url) request.get_method = lambda : 'HEAD' - with closing(urllib2.urlopen(request)) as response: + with closing(urlopen(request)) as response: info = response.info() slm = datetime(*parsedate(info.get('Last-Modified'))[:6]) if lm >= slm: @@ -413,7 +428,7 @@ def fetch_plugins(old_index): src = plugin['file'] plugin['file'] = src.partition('_')[-1] os.rename(src, plugin['file']) - raw = bz2.compress(json.dumps(ans, sort_keys=True, indent=4, separators=(',', ': '))) + raw = bz2.compress(json.dumps(ans, sort_keys=True, indent=4, separators=(',', ': ')).encode('utf-8')) atomic_write(raw, PLUGINS) # Cleanup any extra .zip files all_plugin_files = {p['file'] for p in ans.values()} @@ -503,7 +518,7 @@ h1 { text-align: center } name, count = x return '%s%s\n' % (escape(name), count) - pstats = map(plugin_stats, sorted(stats.items(), reverse=True, key=lambda x:x[1])) + pstats = list(map(plugin_stats, sorted(stats.items(), reverse=True, key=lambda x:x[1]))) stats = '''\ From ad222887849bb6e10f502f06b17f28df8a86cbdf Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 11 Feb 2020 10:16:29 +0530 Subject: [PATCH 009/162] py3: Fix an error when downloading books via Get Books Fixes #1862719 [TypeError: 'dict_keys' object does not support indexing](https://bugs.launchpad.net/calibre/+bug/1862719) --- src/calibre/gui2/dialogs/choose_format.py | 1 + src/calibre/gui2/store/search/search.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/calibre/gui2/dialogs/choose_format.py b/src/calibre/gui2/dialogs/choose_format.py index c9ef37118b..f8c3d54d1f 100644 --- a/src/calibre/gui2/dialogs/choose_format.py +++ b/src/calibre/gui2/dialogs/choose_format.py @@ -41,6 +41,7 @@ class ChooseFormatDialog(QDialog): bb.accepted.connect(self.accept), bb.rejected.connect(self.reject) h.addStretch(10), h.addWidget(self.buttonBox) + formats = list(formats) for format in formats: self.formats.addItem(QListWidgetItem(file_icon_provider().icon_from_ext(format.lower()), format.upper())) diff --git a/src/calibre/gui2/store/search/search.py b/src/calibre/gui2/store/search/search.py index b4a09d610a..09eb34c472 100644 --- a/src/calibre/gui2/store/search/search.py +++ b/src/calibre/gui2/store/search/search.py @@ -392,7 +392,7 @@ class SearchDialog(QDialog, Ui_Dialog): self.open_store(result) def download_book(self, result): - d = ChooseFormatDialog(self, _('Choose format to download to your library.'), result.downloads.keys()) + d = ChooseFormatDialog(self, _('Choose format to download to your library.'), list(result.downloads.keys())) if d.exec_() == d.Accepted: ext = d.format() fname = result.title[:60] + '.' + ext.lower() From 1e07b48116205ecc8d953cb463e6600cf8aa76a1 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 11 Feb 2020 11:24:40 +0530 Subject: [PATCH 010/162] Checking of changelog needs python 3 now --- setup/check.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/setup/check.py b/setup/check.py index dc4f56f598..fa99c33c76 100644 --- a/setup/check.py +++ b/setup/check.py @@ -6,7 +6,7 @@ __license__ = 'GPL v3' __copyright__ = '2009, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import sys, os, json, subprocess, errno, hashlib +import os, json, subprocess, errno, hashlib from setup import Command, build_cache_dir, edit_file, dump_json @@ -82,10 +82,8 @@ class Check(Command): p = subprocess.Popen(['rapydscript', 'lint', f]) return p.wait() != 0 if ext == '.yaml': - sys.path.insert(0, self.wn_path) - import whats_new - whats_new.render_changelog(self.j(self.d(self.SRC), 'Changelog.yaml')) - sys.path.remove(self.wn_path) + p = subprocess.Popen(['python', self.j(self.wn_path, 'whats_new.py'), f]) + return p.wait() != 0 def run(self, opts): self.fhash_cache = {} From a370e028a3c28de135c2ab47a3df63a0a34abfb6 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 11 Feb 2020 20:20:12 +0530 Subject: [PATCH 011/162] Viewer: Add a control to quit the viewer useful on touchscreens. Fixes #1862441 [Feature request: dedicated 'quit' option in viewer menu](https://bugs.launchpad.net/calibre/+bug/1862441) --- imgsrc/srv/window-restore.svg | 1 + src/calibre/gui2/viewer/ui.py | 1 + src/calibre/gui2/viewer/web_view.py | 3 +++ src/pyj/read_book/overlay.pyj | 4 +++- src/pyj/viewer-main.pyj | 2 ++ 5 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 imgsrc/srv/window-restore.svg diff --git a/imgsrc/srv/window-restore.svg b/imgsrc/srv/window-restore.svg new file mode 100644 index 0000000000..d0d3c7e076 --- /dev/null +++ b/imgsrc/srv/window-restore.svg @@ -0,0 +1 @@ + diff --git a/src/calibre/gui2/viewer/ui.py b/src/calibre/gui2/viewer/ui.py index 119d1777b8..4f486a736d 100644 --- a/src/calibre/gui2/viewer/ui.py +++ b/src/calibre/gui2/viewer/ui.py @@ -176,6 +176,7 @@ class EbookViewer(MainWindow): self.web_view.show_error.connect(self.show_error) self.web_view.print_book.connect(self.print_book, type=Qt.QueuedConnection) self.web_view.reset_interface.connect(self.reset_interface, type=Qt.QueuedConnection) + self.web_view.quit.connect(self.quit, type=Qt.QueuedConnection) self.web_view.shortcuts_changed.connect(self.shortcuts_changed) self.actions_toolbar.initialize(self.web_view, self.search_dock.toggleViewAction()) self.setCentralWidget(self.web_view) diff --git a/src/calibre/gui2/viewer/web_view.py b/src/calibre/gui2/viewer/web_view.py index 6dd4923ee8..9002c84287 100644 --- a/src/calibre/gui2/viewer/web_view.py +++ b/src/calibre/gui2/viewer/web_view.py @@ -270,6 +270,7 @@ class ViewerBridge(Bridge): print_book = from_js() clear_history = from_js() reset_interface = from_js() + quit = from_js() customize_toolbar = from_js() create_view = to_js() @@ -440,6 +441,7 @@ class WebView(RestartingWebEngineView): show_error = pyqtSignal(object, object, object) print_book = pyqtSignal() reset_interface = pyqtSignal() + quit = pyqtSignal() customize_toolbar = pyqtSignal() shortcuts_changed = pyqtSignal(object) paged_mode_changed = pyqtSignal() @@ -487,6 +489,7 @@ class WebView(RestartingWebEngineView): self.bridge.print_book.connect(self.print_book) self.bridge.clear_history.connect(self.clear_history) self.bridge.reset_interface.connect(self.reset_interface) + self.bridge.quit.connect(self.quit) self.bridge.customize_toolbar.connect(self.customize_toolbar) self.bridge.export_shortcut_map.connect(self.set_shortcut_map) self.shortcut_map = {} diff --git a/src/pyj/read_book/overlay.pyj b/src/pyj/read_book/overlay.pyj index e223d4c221..0de46604c5 100644 --- a/src/pyj/read_book/overlay.pyj +++ b/src/pyj/read_book/overlay.pyj @@ -339,7 +339,9 @@ class MainOverlay: # {{{ ac(_('Inspector'), _('Show the content inspector'), def(): self.overlay.hide(), ui_operations.toggle_inspector();, 'bug'), ac(_('Reset interface'), _('Reset viewer panels, toolbars and scrollbars to defaults'), - def(): self.overlay.hide(), ui_operations.reset_interface();, 'remove'), + def(): self.overlay.hide(), ui_operations.reset_interface();, 'window-restore'), + ac(_('Quit'), _('Close the viewer'), + def(): self.overlay.hide(), ui_operations.quit();, 'remove'), )) container.appendChild(set_css(E.div(class_=MAIN_OVERLAY_TS_CLASS, # top section onclick=def (evt):evt.stopPropagation();, diff --git a/src/pyj/viewer-main.pyj b/src/pyj/viewer-main.pyj index 4f47f8381d..0899da7a53 100644 --- a/src/pyj/viewer-main.pyj +++ b/src/pyj/viewer-main.pyj @@ -361,6 +361,8 @@ if window is window.top: sd.set('footer', defaults.footer) view.update_header_footer() to_python.reset_interface() + ui_operations.quit = def(): + to_python.quit() ui_operations.toggle_lookup = def(): to_python.toggle_lookup() ui_operations.selection_changed = def(selected_text): From 2862b4cdd24d065c7df71b34008ae7fd94ff2428 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 12 Feb 2020 07:27:44 +0530 Subject: [PATCH 012/162] Update New York Times --- recipes/nytimes.recipe | 13 +++++-------- recipes/nytimes_sub.recipe | 13 +++++-------- 2 files changed, 10 insertions(+), 16 deletions(-) diff --git a/recipes/nytimes.recipe b/recipes/nytimes.recipe index 87c09c62d1..55f77237f6 100644 --- a/recipes/nytimes.recipe +++ b/recipes/nytimes.recipe @@ -266,14 +266,11 @@ class NewYorkTimes(BasicNewsRecipe): if article.get('description'): self.log('\t\t', article['description']) - container = soup.find(itemtype='http://schema.org/CollectionPage') - container.find('header').extract() - div = container.find('div') - for section in div.findAll('section'): - for ol in section.findAll('ol'): - for article in self.parse_article_group(ol): - log(article) - yield article + container = soup.find(id='collection-{}'.format(slug)).find('section') + for ol in container.findAll('ol'): + for article in self.parse_article_group(ol): + log(article) + yield article def parse_web_sections(self): self.read_nyt_metadata() diff --git a/recipes/nytimes_sub.recipe b/recipes/nytimes_sub.recipe index ffdf42d62a..02ff62ba3d 100644 --- a/recipes/nytimes_sub.recipe +++ b/recipes/nytimes_sub.recipe @@ -266,14 +266,11 @@ class NewYorkTimes(BasicNewsRecipe): if article.get('description'): self.log('\t\t', article['description']) - container = soup.find(itemtype='http://schema.org/CollectionPage') - container.find('header').extract() - div = container.find('div') - for section in div.findAll('section'): - for ol in section.findAll('ol'): - for article in self.parse_article_group(ol): - log(article) - yield article + container = soup.find(id='collection-{}'.format(slug)).find('section') + for ol in container.findAll('ol'): + for article in self.parse_article_group(ol): + log(article) + yield article def parse_web_sections(self): self.read_nyt_metadata() From 9e0e846f257e76c74bf8a5bc60a3b615844d7a96 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 12 Feb 2020 08:04:33 +0530 Subject: [PATCH 013/162] Conversion pipeline: Fix styles applied via selectors to the element being ignored Fixes #1862401 [Class applied to html element is lost during conversion](https://bugs.launchpad.net/calibre/+bug/1862401) --- src/calibre/ebooks/oeb/transforms/flatcss.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/calibre/ebooks/oeb/transforms/flatcss.py b/src/calibre/ebooks/oeb/transforms/flatcss.py index 3c12a28080..f617a752f2 100644 --- a/src/calibre/ebooks/oeb/transforms/flatcss.py +++ b/src/calibre/ebooks/oeb/transforms/flatcss.py @@ -660,9 +660,8 @@ class CSSFlattener(object): stylizer = self.stylizers[item] if self.specializer is not None: self.specializer(item, stylizer) - body = html.find(XHTML('body')) fsize = self.context.dest.fbase - self.flatten_node(body, stylizer, names, styles, pseudo_styles, fsize, item.id) + self.flatten_node(html, stylizer, names, styles, pseudo_styles, fsize, item.id) items = sorted(((key, val) for (val, key) in iteritems(styles)), key=lambda x:numeric_sort_key(x[0])) # :hover must come after link and :active must come after :hover psels = sorted(pseudo_styles, key=lambda x : From a7a88de0f8d21318c533554600c0ac3b6c1a5229 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 12 Feb 2020 08:59:46 +0530 Subject: [PATCH 014/162] ... --- recipes/nytimes.recipe | 10 +++++++++- recipes/nytimes_sub.recipe | 10 +++++++++- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/recipes/nytimes.recipe b/recipes/nytimes.recipe index 55f77237f6..7aa9893ba8 100644 --- a/recipes/nytimes.recipe +++ b/recipes/nytimes.recipe @@ -266,7 +266,15 @@ class NewYorkTimes(BasicNewsRecipe): if article.get('description'): self.log('\t\t', article['description']) - container = soup.find(id='collection-{}'.format(slug)).find('section') + cid = slug.split('/')[-1] + if cid == 'dining': + cid = 'food' + try: + container = soup.find(id='collection-{}'.format(cid)).find('section') + except AttributeError: + container = None + if container is None: + raise ValueError('Failed to find articles container for slug: {}'.format(slug)) for ol in container.findAll('ol'): for article in self.parse_article_group(ol): log(article) diff --git a/recipes/nytimes_sub.recipe b/recipes/nytimes_sub.recipe index 02ff62ba3d..9663e10218 100644 --- a/recipes/nytimes_sub.recipe +++ b/recipes/nytimes_sub.recipe @@ -266,7 +266,15 @@ class NewYorkTimes(BasicNewsRecipe): if article.get('description'): self.log('\t\t', article['description']) - container = soup.find(id='collection-{}'.format(slug)).find('section') + cid = slug.split('/')[-1] + if cid == 'dining': + cid = 'food' + try: + container = soup.find(id='collection-{}'.format(cid)).find('section') + except AttributeError: + container = None + if container is None: + raise ValueError('Failed to find articles container for slug: {}'.format(slug)) for ol in container.findAll('ol'): for article in self.parse_article_group(ol): log(article) From 9c6444e3b89c6f4f54e1db726c046c59a172d7cb Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 12 Feb 2020 21:02:20 +0530 Subject: [PATCH 015/162] Avoid roundtripping image data when dragging and dropping image files Fixes #1862440 [Cover disappear when dragging new cover into editor and dropping it on the cover space](https://bugs.launchpad.net/calibre/+bug/1862440) --- src/calibre/gui2/dnd.py | 41 ++++++++++++++++++++++--------------- src/calibre/gui2/widgets.py | 6 +++++- 2 files changed, 30 insertions(+), 17 deletions(-) diff --git a/src/calibre/gui2/dnd.py b/src/calibre/gui2/dnd.py index 6ead535d45..fbeb64ec8c 100644 --- a/src/calibre/gui2/dnd.py +++ b/src/calibre/gui2/dnd.py @@ -198,14 +198,7 @@ def dnd_has_extension(md, extensions, allow_all_extensions=False): return bool(exts.intersection(frozenset(extensions))) -def dnd_get_image(md, image_exts=None): - ''' - Get the image in the QMimeData object md. - - :return: None, None if no image is found - QPixmap, None if an image is found, the pixmap is guaranteed not null - url, filename if a URL that points to an image is found - ''' +def dnd_get_local_image_and_pixmap(md, image_exts=None): if md.hasImage(): for x in md.formats(): x = unicode_type(x) @@ -214,14 +207,13 @@ def dnd_get_image(md, image_exts=None): pmap = QPixmap() pmap.loadFromData(cdata) if not pmap.isNull(): - return pmap, None - break + return pmap, cdata if md.hasFormat('application/octet-stream'): cdata = bytes(md.data('application/octet-stream')) pmap = QPixmap() pmap.loadFromData(cdata) if not pmap.isNull(): - return pmap, None + return pmap, cdata if image_exts is None: image_exts = image_extensions() @@ -229,23 +221,40 @@ def dnd_get_image(md, image_exts=None): # No image, look for an URL pointing to an image urls = urls_from_md(md) paths = [path_from_qurl(u) for u in urls] - # First look for a local file + # Look for a local file images = [xi for xi in paths if posixpath.splitext(unquote(xi))[1][1:].lower() in image_exts] images = [xi for xi in images if os.path.exists(xi)] - p = QPixmap() for path in images: try: with open(path, 'rb') as f: - p.loadFromData(f.read()) + cdata = f.read() except Exception: continue + p = QPixmap() + p.loadFromData(cdata) if not p.isNull(): - return p, None + return p, cdata - # No local images, look for remote ones + return None, None + +def dnd_get_image(md, image_exts=None): + ''' + Get the image in the QMimeData object md. + + :return: None, None if no image is found + QPixmap, None if an image is found, the pixmap is guaranteed not null + url, filename if a URL that points to an image is found + ''' + if image_exts is None: + image_exts = image_extensions() + pmap, data = dnd_get_local_image_and_pixmap(md, image_exts) + if pmap is not None: + return pmap, None + # Look for a remote image + urls = urls_from_md(md) # First, see if this is from Firefox rurl, fname = get_firefox_rurl(md, image_exts) diff --git a/src/calibre/gui2/widgets.py b/src/calibre/gui2/widgets.py index f2c55472c4..d5b7b66aa4 100644 --- a/src/calibre/gui2/widgets.py +++ b/src/calibre/gui2/widgets.py @@ -23,7 +23,7 @@ from calibre.ebooks import BOOK_EXTENSIONS from calibre.utils.config import prefs, XMLConfig from calibre.gui2.progress_indicator import ProgressIndicator as _ProgressIndicator from calibre.gui2.dnd import (dnd_has_image, dnd_get_image, dnd_get_files, - image_extensions, dnd_has_extension, DownloadDialog) + image_extensions, dnd_has_extension, dnd_get_local_image_and_pixmap, DownloadDialog) from calibre.utils.localization import localize_user_manual_link from polyglot.builtins import native_string_type, unicode_type, range @@ -238,6 +238,10 @@ class ImageDropMixin(object): # {{{ def dropEvent(self, event): event.setDropAction(Qt.CopyAction) md = event.mimeData() + pmap, data = dnd_get_local_image_and_pixmap(md) + if pmap is not None: + self.handle_image_drop(pmap, data) + return x, y = dnd_get_image(md) if x is not None: From 19d0c2ae1be2c070a678e56fc57a0045ee93adc8 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 13 Feb 2020 08:01:33 +0530 Subject: [PATCH 016/162] Edit book: Fix syntax highlighting for break-(before|after). Fixes #1863020 [[Editor] break-after and -before are unknown properties in CSS editor](https://bugs.launchpad.net/calibre/+bug/1863020) --- src/calibre/gui2/tweak_book/editor/syntax/css.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/calibre/gui2/tweak_book/editor/syntax/css.py b/src/calibre/gui2/tweak_book/editor/syntax/css.py index f641b9f66e..1f1d117221 100644 --- a/src/calibre/gui2/tweak_book/editor/syntax/css.py +++ b/src/calibre/gui2/tweak_book/editor/syntax/css.py @@ -53,6 +53,7 @@ content_tokens = [(re.compile(k), v, n) for k, v, n in [ r'outline-style|outline-width|overflow(?:-x|-y)?|padding-bottom|' r'padding-left|padding-right|padding-top|padding|' r'page-break-after|page-break-before|page-break-inside|' + r'break-before|break-after|' r'pause-after|pause-before|pause|pitch|pitch-range|' r'play-during|position|pre-wrap|pre-line|pre|quotes|richness|right|size|' r'speak-header|speak-numeral|speak-punctuation|speak|' From 305c97620e5bc931be9425ac585f4b7e73f5a834 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 15 Feb 2020 19:55:51 +0530 Subject: [PATCH 017/162] String changes --- manual/develop.rst | 2 +- manual/viewer.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/manual/develop.rst b/manual/develop.rst index b60453fc08..aca049091d 100644 --- a/manual/develop.rst +++ b/manual/develop.rst @@ -136,7 +136,7 @@ for inclusion into the main calibre repository: git clone git@github.com:/calibre.git git remote add upstream https://github.com/kovidgoyal/calibre.git - Replace above with your github username. That will get your fork checked out locally. + Replace above with your GitHub username. That will get your fork checked out locally. * You can make changes and commit them whenever you like. When you are ready to have your work merged, do a:: git push diff --git a/manual/viewer.rst b/manual/viewer.rst index ee90fe3067..31e0325f5b 100644 --- a/manual/viewer.rst +++ b/manual/viewer.rst @@ -143,7 +143,7 @@ Non re-flowable content Some books have very wide content that content be broken up at page boundaries. For example tables or :code:`
` tags. In such cases, you should switch the
 viewer to *flow mode* by pressing :kbd:`Ctrl+m` to read this content.
-Alternately, you can also add the following CSS to the Styling section of the
+Alternately, you can also add the following CSS to the :guilabel:`Styles` section of the
 viewer preferences to force the viewer to break up lines of text in
 :code:`
` tags::
 

From 6d4909978a24f431a6807b9864ce66ef1aa22bd2 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Sat, 15 Feb 2020 22:05:04 +0530
Subject: [PATCH 018/162] Fix #1863418 [Hovering buttons turns the text
 red](https://bugs.launchpad.net/calibre/+bug/1863418)

---
 src/pyj/read_book/prefs/font_size.pyj | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/pyj/read_book/prefs/font_size.pyj b/src/pyj/read_book/prefs/font_size.pyj
index 7ad7c24c7e..011e13b2f8 100644
--- a/src/pyj/read_book/prefs/font_size.pyj
+++ b/src/pyj/read_book/prefs/font_size.pyj
@@ -18,6 +18,7 @@ add_extra_css(def():
     style = rule(QUICK, 'li.current', background_color=get_color('window-background2'))
     style += rule(QUICK, 'li:hover', background_color=get_color('window-background2'))
     style += rule(CONTAINER, 'a:hover', color=get_color('window-hover-foreground'))
+    style += rule(CONTAINER, 'a.calibre-push-button:hover', color=get_color('button-text'))
     return style
 )
 

From b3b2317e98ce1df2051b40c6bf90fa89e4b5b163 Mon Sep 17 00:00:00 2001
From: simonvg 
Date: Sat, 15 Feb 2020 21:21:11 +0100
Subject: [PATCH 019/162] Remove first touch event not cancelable

---
 resources/content-server/reset.css | 1 +
 1 file changed, 1 insertion(+)

diff --git a/resources/content-server/reset.css b/resources/content-server/reset.css
index 57ace129ee..15fc5f748d 100644
--- a/resources/content-server/reset.css
+++ b/resources/content-server/reset.css
@@ -34,6 +34,7 @@ time, mark, audio, video {
 
 body {
     line-height:1.2;
+    touch-action:none;
 }
 
 article,aside,details,figcaption,figure,

From 01c64414a766b5f0aa9c7cd1a86d842b6b9f90b5 Mon Sep 17 00:00:00 2001
From: simonvg 
Date: Sat, 15 Feb 2020 21:29:01 +0100
Subject: [PATCH 020/162] Fix looping hold timer due to missing touch end event
 on content loaded

---
 src/pyj/read_book/iframe.pyj |  5 ++++-
 src/pyj/read_book/touch.pyj  | 13 ++++++++++---
 2 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/src/pyj/read_book/iframe.pyj b/src/pyj/read_book/iframe.pyj
index 1130130c88..c2bc8b3a17 100644
--- a/src/pyj/read_book/iframe.pyj
+++ b/src/pyj/read_book/iframe.pyj
@@ -44,7 +44,8 @@ from read_book.shortcuts import (
     create_shortcut_map, keyevent_as_shortcut, shortcut_for_key_event
 )
 from read_book.toc import update_visible_toc_anchors
-from read_book.touch import create_handlers as create_touch_handlers
+from read_book.touch import (create_handlers as create_touch_handlers,
+                             reset_handlers as reset_touch_handlers)
 from read_book.viewport import scroll_viewport
 from utils import (
     apply_cloned_selection, clone_selection, debounce, html_escape, is_ios
@@ -334,6 +335,8 @@ class IframeBoss:
         self.send_message('content_loaded', progress_frac=self.calculate_progress_frac(), file_progress_frac=progress_frac())
         self.last_cfi = None
         self.auto_scroll_action('resume')
+        reset_touch_handlers() # Needed to mitigate issue https://bugs.chromium.org/p/chromium/issues/detail?id=464579
+
         window.setTimeout(self.update_cfi, 0)
         window.setTimeout(self.update_toc_position, 0)
 
diff --git a/src/pyj/read_book/touch.pyj b/src/pyj/read_book/touch.pyj
index 8858350fd3..07e4bf5b28 100644
--- a/src/pyj/read_book/touch.pyj
+++ b/src/pyj/read_book/touch.pyj
@@ -140,6 +140,12 @@ class TouchHandler:
                 return True
         return False
 
+    def reset_handlers(self):
+        self.stop_hold_timer()
+        self.ongoing_touches = {}
+        self.gesture_id = None
+        self.handled_tap_hold = False
+
     def start_hold_timer(self):
         self.stop_hold_timer()
         self.hold_timer = window.setTimeout(self.check_for_hold, 100)
@@ -199,9 +205,7 @@ class TouchHandler:
         self.prune_expired_touches()
         if not self.has_active_touches:
             self.dispatch_gesture()
-            self.ongoing_touches = {}
-            self.gesture_id = None
-            self.handled_tap_hold = False
+            self.reset_handlers()
 
     def handle_touchcancel(self, ev):
         ev.preventDefault(), ev.stopPropagation()
@@ -279,6 +283,9 @@ def create_handlers():
     # on window instead of document
     install_handlers(document, main_touch_handler)
 
+def reset_handlers():
+    main_touch_handler.reset_handlers()
+
 def set_left_margin_handler(elem):
     install_handlers(elem, left_margin_handler)
 

From 14a803229f7174e587578bf14cd42d0ce6440f57 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Sun, 16 Feb 2020 12:05:46 +0530
Subject: [PATCH 021/162] py3: Fix #1863160 [Wrong links in the converted azw3
 file](https://bugs.launchpad.net/calibre/+bug/1863160)

---
 src/calibre/ebooks/mobi/writer8/skeleton.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/calibre/ebooks/mobi/writer8/skeleton.py b/src/calibre/ebooks/mobi/writer8/skeleton.py
index 1fab295273..286083331c 100644
--- a/src/calibre/ebooks/mobi/writer8/skeleton.py
+++ b/src/calibre/ebooks/mobi/writer8/skeleton.py
@@ -16,7 +16,7 @@ from lxml import etree
 from calibre import my_unichr
 from calibre.ebooks.oeb.base import XHTML_NS, extract
 from calibre.ebooks.mobi.utils import to_base, PolyglotDict
-from polyglot.builtins import iteritems, unicode_type
+from polyglot.builtins import iteritems, unicode_type, as_bytes
 
 CHUNK_SIZE = 8192
 
@@ -397,7 +397,7 @@ class Chunker(object):
             pos, fid = to_base(pos, min_num_digits=4), to_href(fid)
             return ':off:'.join((pos, fid)).encode('utf-8')
 
-        placeholder_map = {k:to_placeholder(v) for k, v in
+        placeholder_map = {as_bytes(k):to_placeholder(v) for k, v in
                 iteritems(self.placeholder_map)}
 
         # Now update the links

From 1d35b786015f02406e5e82260334ca5ef92fe0a1 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Sun, 16 Feb 2020 16:54:57 +0530
Subject: [PATCH 022/162] py3: Fix calibredb list

Fixes #1863470 [[python3] calibredb list fails](https://bugs.launchpad.net/calibre/+bug/1863470)
---
 src/calibre/db/cli/cmd_list.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/calibre/db/cli/cmd_list.py b/src/calibre/db/cli/cmd_list.py
index ecd0143af6..041c48fb6f 100644
--- a/src/calibre/db/cli/cmd_list.py
+++ b/src/calibre/db/cli/cmd_list.py
@@ -203,6 +203,7 @@ def do_list(
     )
     with ColoredStream(sys.stdout, fg='green'):
         prints(''.join(titles))
+    stdout = getattr(sys.stdout, 'buffer', sys.stdout)
 
     wrappers = [TextWrapper(x - 1).wrap if x > 1 else lambda y: y for x in widths]
 
@@ -214,10 +215,10 @@ def do_list(
         for l in range(lines):
             for i, field in enumerate(text):
                 ft = text[i][l] if l < len(text[i]) else u''
-                sys.stdout.write(ft.encode('utf-8'))
+                stdout.write(ft.encode('utf-8'))
                 if i < len(text) - 1:
                     filler = (u'%*s' % (widths[i] - str_width(ft) - 1, u''))
-                    sys.stdout.write((filler + separator).encode('utf-8'))
+                    stdout.write((filler + separator).encode('utf-8'))
             print()
 
 

From fe61a72b8339604d939ff2d148ce703d27168592 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Sun, 16 Feb 2020 18:20:06 +0530
Subject: [PATCH 023/162] Avoid runtime errors during shutdown of CLI plugins
 that use check_css

---
 src/calibre/ebooks/oeb/polish/check/css.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/calibre/ebooks/oeb/polish/check/css.py b/src/calibre/ebooks/oeb/polish/check/css.py
index f7d7ea3b68..efce567d32 100644
--- a/src/calibre/ebooks/oeb/polish/check/css.py
+++ b/src/calibre/ebooks/oeb/polish/check/css.py
@@ -222,7 +222,12 @@ class Pool(object):
             self.working = False
 
     def shutdown(self):
-        tuple(map(sip.delete, self.workers))
+
+        def safe_delete(x):
+            if not sip.isdeleted(x):
+                sip.delete(x)
+
+        tuple(map(safe_delete, self.workers))
         self.workers = []
 
 

From d72db01ff177e868ef4890279f914adb3e5b7b9b Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Sun, 16 Feb 2020 18:43:48 +0530
Subject: [PATCH 024/162] String changes

---
 manual/catalogs.rst | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/manual/catalogs.rst b/manual/catalogs.rst
index 9aed8ae4a4..3ef37efa7f 100644
--- a/manual/catalogs.rst
+++ b/manual/catalogs.rst
@@ -1,6 +1,6 @@
 .. _catalog_tut:
 
-Creating AZW3 • EPUB • MOBI catalogs 
+Creating AZW3 • EPUB • MOBI catalogs
 =====================================
 
 calibre's Create catalog feature enables you to create a catalog of your library in a variety of formats. This help file describes cataloging options when generating a catalog in AZW3, EPUB and MOBI formats.
@@ -19,7 +19,7 @@ If you want only *some* of your library cataloged, you have two options:
     * Create a multiple selection of the books you want cataloged. With more than one book selected in calibre's main window, only the selected books will be cataloged.
     * Use the Search field or the Tag browser to filter the displayed books. Only the displayed books will be cataloged.
 
-To begin catalog generation, select the menu item :guilabel:`Convert books > Create a catalog of the books in your calibre library`. You may also add a :guilabel:`Create Catalog` button to a toolbar in :guilabel:`Preferences > Interface > Toolbars` for easier access to the Generate catalog dialog.
+To begin catalog generation, select the menu item :guilabel:`Convert books > Create a catalog of the books in your calibre library`. You may also add a :guilabel:`Create Catalog` button to a toolbar in :guilabel:`Preferences > Interface > Toolbars & menus` for easier access to the Generate catalog dialog.
 
 .. image:: images/catalog_options.png
     :alt: Catalog options
@@ -125,7 +125,7 @@ Custom catalog covers
 
 .. |cc| image:: images/custom_cover.png
 
-|cc| With the `Generate Cover plugin `_ installed, you can create custom covers for your catalog. 
+|cc| With the `Generate Cover plugin `_ installed, you can create custom covers for your catalog.
 To install the plugin, go to :guilabel:`Preferences > Advanced > Plugins > Get new plugins`.
 
 Additional help resources
@@ -134,4 +134,3 @@ Additional help resources
 For more information on calibre's Catalog feature, see the MobileRead forum sticky `Creating Catalogs - Start here `_, where you can find information on how to customize the catalog templates, and how to submit a bug report.
 
 To ask questions or discuss calibre's Catalog feature with other users, visit the MobileRead forum `Calibre Catalogs `_.
-

From 48394ddc55d5edc044d6f13efd32100f8186ea8b Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Sun, 16 Feb 2020 19:51:42 +0530
Subject: [PATCH 025/162] String changes

---
 manual/viewer.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/manual/viewer.rst b/manual/viewer.rst
index 31e0325f5b..3d64623d89 100644
--- a/manual/viewer.rst
+++ b/manual/viewer.rst
@@ -125,8 +125,8 @@ Dictionary lookup
 -------------------
 
 You can look up the meaning of words in the current book by opening the
-:guilabel:`Lookup/search panel` via the viewer controls. Then simply double
-click on any word and its definition will be displayed in the lookup panel.
+:guilabel:`Lookup/search word panel` via the viewer controls. Then simply double
+click on any word and its definition will be displayed in the Lookup panel.
 
 
 Copying text and images

From 28051617578b191499512945b4f95233182b9d2d Mon Sep 17 00:00:00 2001
From: Norbert Preining 
Date: Mon, 17 Feb 2020 05:28:02 +0900
Subject: [PATCH 026/162] fix py3 version of calibredb show_metadata

---
 src/calibre/db/cli/cmd_show_metadata.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/calibre/db/cli/cmd_show_metadata.py b/src/calibre/db/cli/cmd_show_metadata.py
index ca76735070..08239cd9b8 100644
--- a/src/calibre/db/cli/cmd_show_metadata.py
+++ b/src/calibre/db/cli/cmd_show_metadata.py
@@ -49,8 +49,9 @@ def main(opts, args, dbctx):
     if mi is None:
         raise SystemExit('Id #%d is not present in database.' % id)
     if opts.as_opf:
+        stdout = getattr(sys.stdout, 'buffer', sys.stdout)
         mi = OPFCreator(getcwd(), mi)
-        mi.render(sys.stdout)
+        mi.render(stdout)
     else:
         prints(unicode_type(mi))
 

From 1ba8e6446867c130ffd3c6583afe90c687e5da9b Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Mon, 17 Feb 2020 03:15:53 +0530
Subject: [PATCH 027/162] py3: Fix clean_xml_text implementation

Fixes #1863517 [Characters are dropped from title](https://bugs.launchpad.net/calibre/+bug/1863517)
---
 src/calibre/utils/cleantext.py | 16 +++++++---------
 src/calibre/utils/speedup.c    |  7 +++++--
 2 files changed, 12 insertions(+), 11 deletions(-)

diff --git a/src/calibre/utils/cleantext.py b/src/calibre/utils/cleantext.py
index 2d62e06858..d7c9d74c20 100644
--- a/src/calibre/utils/cleantext.py
+++ b/src/calibre/utils/cleantext.py
@@ -8,15 +8,13 @@ from polyglot.builtins import codepoint_to_chr, map, range, filter
 from polyglot.html_entities import name2codepoint
 from calibre.constants import plugins, preferred_encoding
 
-try:
-    _ncxc = plugins['speedup'][0].clean_xml_chars
-except AttributeError:
-    native_clean_xml_chars = None
-else:
-    def native_clean_xml_chars(x):
-        if isinstance(x, bytes):
-            x = x.decode(preferred_encoding)
-        return _ncxc(x)
+_ncxc = plugins['speedup'][0].clean_xml_chars
+
+
+def native_clean_xml_chars(x):
+    if isinstance(x, bytes):
+        x = x.decode(preferred_encoding)
+    return _ncxc(x)
 
 
 def ascii_pat(for_binary=False):
diff --git a/src/calibre/utils/speedup.c b/src/calibre/utils/speedup.c
index ca990150d1..30b82638c7 100644
--- a/src/calibre/utils/speedup.c
+++ b/src/calibre/utils/speedup.c
@@ -394,8 +394,11 @@ clean_xml_chars(PyObject *self, PyObject *text) {
         // based on https://en.wikipedia.org/wiki/Valid_characters_in_XML#Non-restricted_characters
         // python 3.3+ unicode strings never contain surrogate pairs, since if
         // they did, they would be represented as UTF-32
-        if ((0x20 <= ch && ch <= 0xd7ff && ch != 0x7f) ||
-                ch == 9 || ch == 10 || ch == 13 ||
+        if ((0x20 <= ch && ch <= 0x7e) ||
+                ch == 0x9 || ch == 0xa || ch == 0xd || ch == 0x85 ||
+				(0x00A0 <= ch && ch <= 0xD7FF) ||
+				(0xE000 <= ch && ch <= 0xFDCF) ||
+				(0xFDF0 <= ch && ch <= 0xFFFD) ||
                 (0xffff < ch && ch <= 0x10ffff)) {
             PyUnicode_WRITE(text_kind, result_text, target_i, ch);
             target_i += 1;

From e9a899a9fab0d49d13f44f871fb42f49d650605b Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Mon, 17 Feb 2020 09:49:39 +0530
Subject: [PATCH 028/162] Fix for viewer window going off screen even when not
 restoring window geometry

---
 src/calibre/gui2/__init__.py  | 4 +++-
 src/calibre/gui2/viewer/ui.py | 2 ++
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/calibre/gui2/__init__.py b/src/calibre/gui2/__init__.py
index 87534b996d..cc2540d20f 100644
--- a/src/calibre/gui2/__init__.py
+++ b/src/calibre/gui2/__init__.py
@@ -978,13 +978,15 @@ class Application(QApplication):
         if not geom:
             return
         restored = widget.restoreGeometry(geom)
+        return restored
+
+    def ensure_window_on_screen(self, widget):
         screen_rect = self.desktop().availableGeometry(widget)
         if not widget.geometry().intersects(screen_rect):
             w = min(widget.width(), screen_rect.width() - 10)
             h = min(widget.height(), screen_rect.height() - 10)
             widget.resize(w, h)
             widget.move((screen_rect.width() - w) // 2, (screen_rect.height() - h) // 2)
-        return restored
 
     def setup_ui_font(self):
         f = QFont(QApplication.font())
diff --git a/src/calibre/gui2/viewer/ui.py b/src/calibre/gui2/viewer/ui.py
index 4f486a736d..5efbad2fa9 100644
--- a/src/calibre/gui2/viewer/ui.py
+++ b/src/calibre/gui2/viewer/ui.py
@@ -537,6 +537,8 @@ class EbookViewer(MainWindow):
         geom = vprefs['main_window_geometry']
         if geom and get_session_pref('remember_window_geometry', default=False):
             QApplication.instance().safe_restore_geometry(self, geom)
+        else:
+            QApplication.instance().ensure_window_on_screen(self)
         if state:
             self.restoreState(state, self.MAIN_WINDOW_STATE_VERSION)
             self.inspector_dock.setVisible(False)

From 351781e8699bff153a341d5099042edfaf8d5219 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Mon, 17 Feb 2020 10:12:10 +0530
Subject: [PATCH 029/162] oops

---
 src/calibre/gui2/__init__.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/calibre/gui2/__init__.py b/src/calibre/gui2/__init__.py
index cc2540d20f..a081926eba 100644
--- a/src/calibre/gui2/__init__.py
+++ b/src/calibre/gui2/__init__.py
@@ -978,6 +978,7 @@ class Application(QApplication):
         if not geom:
             return
         restored = widget.restoreGeometry(geom)
+        self.ensure_window_on_screen(widget)
         return restored
 
     def ensure_window_on_screen(self, widget):

From 936e8b63824d536b1f3dc74bcc69df813f187f94 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Mon, 17 Feb 2020 10:22:44 +0530
Subject: [PATCH 030/162] py3: Fix unicode spaces being replaced by normal
 space in the edit metadata dialog

Fixes #1863525 [IDEOGRAPHIC SPACE (U+3000) is not treated correctly](https://bugs.launchpad.net/calibre/+bug/1863525)
---
 src/calibre/gui2/metadata/basic_widgets.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/calibre/gui2/metadata/basic_widgets.py b/src/calibre/gui2/metadata/basic_widgets.py
index ef0555bcd6..93a61e15b6 100644
--- a/src/calibre/gui2/metadata/basic_widgets.py
+++ b/src/calibre/gui2/metadata/basic_widgets.py
@@ -30,6 +30,7 @@ from calibre.utils.date import (
     local_tz, qt_to_dt, as_local_time, UNDEFINED_DATE, is_date_undefined,
     utcfromtimestamp, parse_only_date, internal_iso_format_string)
 from calibre import strftime
+from calibre.constants import ispy3
 from calibre.ebooks import BOOK_EXTENSIONS
 from calibre.customize.ui import run_plugins_on_import
 from calibre.gui2.comments_editor import Editor
@@ -52,7 +53,7 @@ def save_dialog(parent, title, msg, det_msg=''):
 
 
 def clean_text(x):
-    return re.sub(r'\s', ' ', x.strip())
+    return re.sub(r'\s', ' ', x.strip(), flags=re.ASCII if ispy3 else 0)
 
 
 '''
@@ -221,7 +222,6 @@ class TitleEdit(EnLineEdit, ToMetadataMixin):
 
     @property
     def current_val(self):
-
         title = clean_text(unicode_type(self.text()))
         if not title:
             title = self.get_default()

From 3d93979fdba77b5cc86deaf6f3ab0b8ae890b00d Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Mon, 17 Feb 2020 16:04:14 +0530
Subject: [PATCH 031/162] ...

---
 manual/viewer.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/manual/viewer.rst b/manual/viewer.rst
index 3d64623d89..8697e31421 100644
--- a/manual/viewer.rst
+++ b/manual/viewer.rst
@@ -133,7 +133,7 @@ Copying text and images
 -------------------------
 
 You can select text and images by dragging the content with your mouse and then
-right clicking and selecting "Copy" to copy to the clipboard.  The copied
+right clicking and selecting :guilabel:`Copy` to copy to the clipboard.  The copied
 material can be pasted into another application as plain text and images.
 
 

From ea7ad6fc0a7d022db309bb27d9c6555ccad1b162 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Tue, 18 Feb 2020 08:15:09 +0530
Subject: [PATCH 032/162] Viewer: Calculate default column widths based on
 current font size

---
 src/pyj/read_book/paged_mode.pyj | 22 +++++++++++++++++++++-
 1 file changed, 21 insertions(+), 1 deletion(-)

diff --git a/src/pyj/read_book/paged_mode.pyj b/src/pyj/read_book/paged_mode.pyj
index a7135a5758..7acfa0c223 100644
--- a/src/pyj/read_book/paged_mode.pyj
+++ b/src/pyj/read_book/paged_mode.pyj
@@ -119,6 +119,26 @@ def fit_images():
         set_elem_data(img, 'height-limited', True)
 
 
+def cps_by_em_size():
+    ans = cps_by_em_size.ans
+    fs = window.getComputedStyle(document.body).fontSize
+    if not ans or cps_by_em_size.at_font_size is not fs:
+        d = document.createElement('span')
+        d.style.position = 'absolute'
+        d.style.visibility = 'hidden'
+        d.style.width = '1rem'
+        d.style.fontSize = '1rem'
+        d.style.paddingTop = d.style.paddingBottom = d.style.paddingLeft = d.style.paddingRight = '0'
+        d.style.marginTop = d.style.marginBottom = d.style.marginLeft = d.style.marginRight = '0'
+        d.style.borderStyle = 'none'
+        document.body.appendChild(d)
+        w = d.clientWidth
+        document.body.removeChild(d)
+        ans = cps_by_em_size.ans = max(2, w)
+        cps_by_em_size.at_font_size = fs
+    return ans
+
+
 def calc_columns_per_screen():
     cps = opts.columns_per_screen or {}
     cps = cps.landscape if scroll_viewport.width() > scroll_viewport.height() else cps.portrait
@@ -127,7 +147,7 @@ def calc_columns_per_screen():
     except:
         cps = 0
     if not cps:
-        cps = int(Math.floor(scroll_viewport.width() / 500.0))
+        cps = int(Math.floor(scroll_viewport.width() / (35 * cps_by_em_size())))
     cps = max(1, min(cps or 1, 20))
     return cps
 

From 1c9db874920ef6e7b3a1ee4c4fb6d6f3911b83ae Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Tue, 18 Feb 2020 08:29:22 +0530
Subject: [PATCH 033/162] Also add touch-action: none to the main document

---
 src/pyj/book_list/main.pyj | 4 +++-
 src/pyj/viewer-main.pyj    | 4 +++-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/pyj/book_list/main.pyj b/src/pyj/book_list/main.pyj
index b52de2eb9a..23a6d01152 100644
--- a/src/pyj/book_list/main.pyj
+++ b/src/pyj/book_list/main.pyj
@@ -67,7 +67,9 @@ def init_ui():
     install(translations)
     remove_initial_progress_bar()
     document.head.appendChild(E.style(get_widget_css()))
-    set_css(document.body, background_color=get_color('window-background'), color=get_color('window-foreground'))
+    # See https://github.com/kovidgoyal/calibre/pull/1101
+    # for why we need touch-action: none
+    set_css(document.body, background_color=get_color('window-background'), color=get_color('window-foreground'), touch_action='none')
     document.body.appendChild(E.div())
     document.body.lastChild.appendChild(E.div(id=book_list_container_id, style='display: none'))
     document.body.lastChild.appendChild(E.div(id=read_book_container_id, style='display: none'))
diff --git a/src/pyj/viewer-main.pyj b/src/pyj/viewer-main.pyj
index 0899da7a53..8da2bedf32 100644
--- a/src/pyj/viewer-main.pyj
+++ b/src/pyj/viewer-main.pyj
@@ -408,7 +408,9 @@ if window is window.top:
     window.onerror = onerror
     create_modal_container()
     document.head.appendChild(E.style(get_widget_css()))
-    set_css(document.body, background_color=get_color('window-background'), color=get_color('window-foreground'))
+    # See https://github.com/kovidgoyal/calibre/pull/1101
+    # for why we need touch-action: none
+    set_css(document.body, background_color=get_color('window-background'), color=get_color('window-foreground'), touch_action='none')
     setTimeout(def():
             window.onpopstate = on_pop_state
         , 0)  # We do this after event loop ticks over to avoid catching popstate events that some browsers send on page load

From 37d050f356deccfe0f83ea20da5294361a0af512 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Tue, 18 Feb 2020 09:13:50 +0530
Subject: [PATCH 034/162] Make Open with context menu creation code re-useable

---
 src/calibre/gui2/tweak_book/file_list.py | 55 +++++++++++++++---------
 1 file changed, 34 insertions(+), 21 deletions(-)

diff --git a/src/calibre/gui2/tweak_book/file_list.py b/src/calibre/gui2/tweak_book/file_list.py
index 3fffe3c3c4..94aeb22b04 100644
--- a/src/calibre/gui2/tweak_book/file_list.py
+++ b/src/calibre/gui2/tweak_book/file_list.py
@@ -188,7 +188,40 @@ class ItemDelegate(QStyledItemDelegate):  # {{{
 # }}}
 
 
-class FileList(QTreeWidget):
+class OpenWithHandler(object):  # {{{
+
+    def add_open_with_actions(self, menu, file_name):
+        from calibre.gui2.open_with import populate_menu, edit_programs
+        fmt = file_name.rpartition('.')[-1].lower()
+        if not fmt:
+            return
+        m = QMenu(_('Open %s with...') % file_name)
+
+        def connect_action(ac, entry):
+            connect_lambda(ac.triggered, self, lambda self: self.open_with(file_name, fmt, entry))
+
+        populate_menu(m, connect_action, fmt)
+        if len(m.actions()) == 0:
+            menu.addAction(_('Open %s with...') % file_name, partial(self.choose_open_with, file_name, fmt))
+        else:
+            m.addSeparator()
+            m.addAction(_('Add other application for %s files...') % fmt.upper(), partial(self.choose_open_with, file_name, fmt))
+            m.addAction(_('Edit Open With applications...'), partial(edit_programs, fmt, file_name))
+            menu.addMenu(m)
+            menu.ow = m
+
+    def choose_open_with(self, file_name, fmt):
+        from calibre.gui2.open_with import choose_program
+        entry = choose_program(fmt, self)
+        if entry is not None:
+            self.open_with(file_name, fmt, entry)
+
+    def open_with(self, file_name, fmt, entry):
+        raise NotImplementedError()
+# }}}
+
+
+class FileList(QTreeWidget, OpenWithHandler):
 
     delete_requested = pyqtSignal(object, object)
     reorder_spine = pyqtSignal(object)
@@ -586,26 +619,6 @@ class FileList(QTreeWidget):
         if len(list(m.actions())) > 0:
             m.popup(self.mapToGlobal(point))
 
-    def add_open_with_actions(self, menu, file_name):
-        from calibre.gui2.open_with import populate_menu, edit_programs
-        fmt = file_name.rpartition('.')[-1].lower()
-        if not fmt:
-            return
-        m = QMenu(_('Open %s with...') % file_name)
-
-        def connect_action(ac, entry):
-            connect_lambda(ac.triggered, self, lambda self: self.open_with(file_name, fmt, entry))
-
-        populate_menu(m, connect_action, fmt)
-        if len(m.actions()) == 0:
-            menu.addAction(_('Open %s with...') % file_name, partial(self.choose_open_with, file_name, fmt))
-        else:
-            m.addSeparator()
-            m.addAction(_('Add other application for %s files...') % fmt.upper(), partial(self.choose_open_with, file_name, fmt))
-            m.addAction(_('Edit Open With applications...'), partial(edit_programs, fmt, file_name))
-            menu.addMenu(m)
-            menu.ow = m
-
     def choose_open_with(self, file_name, fmt):
         from calibre.gui2.open_with import choose_program
         entry = choose_program(fmt, self)

From bb733041ccb5e272fde999123331ba384e797e0c Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Tue, 18 Feb 2020 09:43:40 +0530
Subject: [PATCH 035/162] Edit book: Preview panel: Allow right clicking on
 images to edit them

---
 src/calibre/gui2/tweak_book/boss.py    |  2 ++
 src/calibre/gui2/tweak_book/preview.py | 29 +++++++++++++++++++++-----
 2 files changed, 26 insertions(+), 5 deletions(-)

diff --git a/src/calibre/gui2/tweak_book/boss.py b/src/calibre/gui2/tweak_book/boss.py
index 9e9062409a..d707f8a78d 100644
--- a/src/calibre/gui2/tweak_book/boss.py
+++ b/src/calibre/gui2/tweak_book/boss.py
@@ -150,6 +150,8 @@ class Boss(QObject):
         self.gui.preview.split_requested.connect(self.split_requested)
         self.gui.preview.link_clicked.connect(self.link_clicked)
         self.gui.preview.render_process_restarted.connect(self.report_render_process_restart)
+        self.gui.preview.open_file_with.connect(self.open_file_with)
+        self.gui.preview.edit_file.connect(self.edit_file_requested)
         self.gui.check_book.item_activated.connect(self.check_item_activated)
         self.gui.check_book.check_requested.connect(self.check_requested)
         self.gui.check_book.fix_requested.connect(self.fix_requested)
diff --git a/src/calibre/gui2/tweak_book/preview.py b/src/calibre/gui2/tweak_book/preview.py
index 7839bad725..cce1b07951 100644
--- a/src/calibre/gui2/tweak_book/preview.py
+++ b/src/calibre/gui2/tweak_book/preview.py
@@ -3,10 +3,6 @@
 # License: GPLv3 Copyright: 2015, Kovid Goyal 
 from __future__ import absolute_import, division, print_function, unicode_literals
 
-# TODO:
-# live css
-# check that clicking on both internal and external links works
-
 import textwrap
 import time
 from collections import defaultdict
@@ -30,6 +26,7 @@ from calibre.ebooks.oeb.base import OEB_DOCS, XHTML_MIME, serialize
 from calibre.ebooks.oeb.polish.parsing import parse
 from calibre.gui2 import NO_URL_FORMATTING, error_dialog, open_url
 from calibre.gui2.tweak_book import TOP, actions, current_container, editors, tprefs
+from calibre.gui2.tweak_book.file_list import OpenWithHandler
 from calibre.gui2.viewer.web_view import send_reply
 from calibre.gui2.webengine import (
     Bridge, RestartingWebEngineView, create_script, from_js, insert_scripts,
@@ -337,7 +334,7 @@ class Inspector(QWidget):
         return QSize(1280, 600)
 
 
-class WebView(RestartingWebEngineView):
+class WebView(RestartingWebEngineView, OpenWithHandler):
 
     def __init__(self, parent=None):
         RestartingWebEngineView.__init__(self, parent)
@@ -398,8 +395,28 @@ class WebView(RestartingWebEngineView):
         menu.addAction(QIcon(I('debug.png')), _('Inspect element'), self.inspect)
         if url.partition(':')[0].lower() in {'http', 'https'}:
             menu.addAction(_('Open link'), partial(open_url, data.linkUrl()))
+        if data.MediaTypeImage <= data.mediaType() <= data.MediaTypeFile:
+            url = data.mediaUrl()
+            if url.scheme() == FAKE_PROTOCOL:
+                href = url.path().lstrip('/')
+                c = current_container()
+                current_name = self.parent().current_name
+                if current_name:
+                    resource_name = c.href_to_name(href, current_name)
+                    if resource_name and c.exists(resource_name) and resource_name not in c.names_that_must_not_be_changed:
+                        self.add_open_with_actions(menu, resource_name)
+                        if data.mediaType() == data.MediaTypeImage:
+                            mime = c.mime_map[resource_name]
+                            if mime.startswith('image/'):
+                                menu.addAction(_('Edit %s') % resource_name, partial(self.edit_image, resource_name))
         menu.exec_(ev.globalPos())
 
+    def open_with(self, file_name, fmt, entry):
+        self.parent().open_file_with.emit(file_name, fmt, entry)
+
+    def edit_image(self, resource_name):
+        self.parent().edit_file.emit(resource_name)
+
 
 class Preview(QWidget):
 
@@ -411,6 +428,8 @@ class Preview(QWidget):
     refreshed = pyqtSignal()
     live_css_data = pyqtSignal(object)
     render_process_restarted = pyqtSignal()
+    open_file_with = pyqtSignal(object, object, object)
+    edit_file = pyqtSignal(object)
 
     def __init__(self, parent=None):
         QWidget.__init__(self, parent)

From f0402832618a34d9fc1fbeb29aa8bee2fa2023f7 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Tue, 18 Feb 2020 21:24:55 +0530
Subject: [PATCH 036/162] Better error message for text less DJVU files

---
 src/calibre/ebooks/conversion/plugins/djvu_input.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/calibre/ebooks/conversion/plugins/djvu_input.py b/src/calibre/ebooks/conversion/plugins/djvu_input.py
index 8f25551af8..ed61ff04d3 100644
--- a/src/calibre/ebooks/conversion/plugins/djvu_input.py
+++ b/src/calibre/ebooks/conversion/plugins/djvu_input.py
@@ -28,8 +28,12 @@ class DJVUInput(InputFormatPlugin):
         from calibre.ebooks.djvu.djvu import DJVUFile
         x = DJVUFile(stream)
         x.get_text(stdout)
+        raw_text = stdout.getvalue()
+        if not raw_text:
+            raise ValueError('The DJVU file contains no text, only images, probably page scans.'
+                    ' calibre only supports conversion of DJVU files with actual text in them.')
 
-        html = convert_basic(stdout.getvalue().replace(b"\n", b' ').replace(
+        html = convert_basic(raw_text.replace(b"\n", b' ').replace(
             b'\037', b'\n\n'))
         # Run the HTMLized text through the html processing plugin.
         from calibre.customize.ui import plugin_for_input_format

From 9f222e809caaca0f495b86ef1b90b1a0ef324786 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Wed, 19 Feb 2020 09:59:06 +0530
Subject: [PATCH 037/162] Viewer: Handle error reporting for hidden text better

See #1863464 (Private bug)
---
 src/calibre/gui2/viewer/search.py | 27 ++++++++++++++++++++++-----
 1 file changed, 22 insertions(+), 5 deletions(-)

diff --git a/src/calibre/gui2/viewer/search.py b/src/calibre/gui2/viewer/search.py
index 1bca57b0e8..c8999e3b7a 100644
--- a/src/calibre/gui2/viewer/search.py
+++ b/src/calibre/gui2/viewer/search.py
@@ -20,6 +20,7 @@ from calibre.gui2 import warning_dialog
 from calibre.gui2.progress_indicator import ProgressIndicator
 from calibre.gui2.viewer.web_view import get_data, get_manifest, vprefs
 from calibre.gui2.widgets2 import HistoryComboBox
+from calibre.utils.monotonic import monotonic
 from polyglot.builtins import iteritems, unicode_type
 from polyglot.functools import lru_cache
 from polyglot.queue import Queue
@@ -74,6 +75,8 @@ class Search(object):
         self._regex = None
 
     def __eq__(self, other):
+        if not isinstance(other, Search):
+            return False
         return self.text == other.text and self.mode == other.mode and self.case_sensitive == other.case_sensitive
 
     @property
@@ -386,9 +389,6 @@ class Results(QListWidget):  # {{{
                 self.item_activated()
             for i in reversed(remove):
                 self.takeItem(i)
-            if self.count():
-                warning_dialog(self, _('Hidden text'), _(
-                    'Some search results were for hidden text, they have been removed.'), show=True)
 
 # }}}
 
@@ -401,6 +401,7 @@ class SearchPanel(QWidget):  # {{{
 
     def __init__(self, parent=None):
         QWidget.__init__(self, parent)
+        self.last_hidden_text_warning = None
         self.current_search = None
         self.l = l = QVBoxLayout(self)
         l.setContentsMargins(0, 0, 0, 0)
@@ -431,6 +432,7 @@ class SearchPanel(QWidget):  # {{{
         self.results.clear()
         self.spinner.start()
         self.current_search = search_query
+        self.last_hidden_text_warning = None
         self.search_tasks.put((search_query, current_name))
 
     def run_searches(self):
@@ -483,6 +485,7 @@ class SearchPanel(QWidget):  # {{{
 
     def clear_searches(self):
         self.current_search = None
+        self.last_hidden_text_warning = None
         searchable_text_for_name.cache_clear()
         self.spinner.stop()
         self.results.clear()
@@ -491,6 +494,7 @@ class SearchPanel(QWidget):  # {{{
         self.search_tasks.put(None)
         self.spinner.stop()
         self.current_search = None
+        self.last_hidden_text_warning = None
         self.searcher = None
 
     def find_next_requested(self, previous):
@@ -501,11 +505,24 @@ class SearchPanel(QWidget):  # {{{
 
     def search_result_not_found(self, sr):
         self.results.search_result_not_found(sr)
+        if self.results.count():
+            now = monotonic()
+            if self.last_hidden_text_warning is None or self.current_search != self.last_hidden_text_warning[1] or now - self.last_hidden_text_warning[0] > 5:
+                self.last_hidden_text_warning = now, self.current_search
+                warning_dialog(self, _('Hidden text'), _(
+                    'Some search results were for hidden or non-reflowable text, they will be removed.'), show=True)
+            elif self.last_hidden_text_warning is not None:
+                self.last_hidden_text_warning = now, self.last_hidden_text_warning[1]
+
         if not self.results.count() and not self.spinner.is_running:
             self.show_no_results_found()
 
     def show_no_results_found(self):
+        has_hidden_text = self.last_hidden_text_warning is not None and self.last_hidden_text_warning[1] == self.current_search
         if self.current_search:
-            warning_dialog(self, _('No matches found'), _(
-                'No matches were found for: {}').format(self.current_search.text), show=True)
+            if has_hidden_text:
+                msg = _('No displayable matches were found for:')
+            else:
+                msg = _('No matches were found for:')
+            warning_dialog(self, _('No matches found'), msg + '  {}'.format(self.current_search.text), show=True)
 # }}}

From 8655121e1690a61a7a7a7124c207b9d9b589f05c Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Wed, 19 Feb 2020 12:34:53 +0530
Subject: [PATCH 038/162] Viewer: Allow right clicking on the scrollbar to
 easily access commonly used scrolling shortcuts

---
 src/calibre/gui2/viewer/ui.py       | 48 +++++++++++++++++++++--------
 src/calibre/gui2/viewer/web_view.py |  3 ++
 src/pyj/read_book/iframe.pyj        |  5 +++
 src/pyj/read_book/scrollbar.pyj     | 14 +++++++--
 src/pyj/read_book/view.pyj          |  2 ++
 src/pyj/viewer-main.pyj             |  2 ++
 6 files changed, 59 insertions(+), 15 deletions(-)

diff --git a/src/calibre/gui2/viewer/ui.py b/src/calibre/gui2/viewer/ui.py
index 5efbad2fa9..19549f30ee 100644
--- a/src/calibre/gui2/viewer/ui.py
+++ b/src/calibre/gui2/viewer/ui.py
@@ -13,8 +13,8 @@ from hashlib import sha256
 from threading import Thread
 
 from PyQt5.Qt import (
-    QApplication, QDockWidget, QEvent, QMimeData, QModelIndex, QPixmap, QScrollBar,
-    Qt, QToolBar, QUrl, QVBoxLayout, QWidget, pyqtSignal
+    QApplication, QCursor, QDockWidget, QEvent, QMenu, QMimeData, QModelIndex,
+    QPixmap, Qt, QToolBar, QUrl, QVBoxLayout, QWidget, pyqtSignal
 )
 
 from calibre import prints
@@ -78,13 +78,6 @@ def path_key(path):
     return sha256(as_bytes(path)).hexdigest()
 
 
-class ScrollBar(QScrollBar):
-
-    def paintEvent(self, ev):
-        if self.isEnabled():
-            return QScrollBar.paintEvent(self, ev)
-
-
 class EbookViewer(MainWindow):
 
     msg_from_anotherinstance = pyqtSignal(object)
@@ -178,6 +171,7 @@ class EbookViewer(MainWindow):
         self.web_view.reset_interface.connect(self.reset_interface, type=Qt.QueuedConnection)
         self.web_view.quit.connect(self.quit, type=Qt.QueuedConnection)
         self.web_view.shortcuts_changed.connect(self.shortcuts_changed)
+        self.web_view.scrollbar_context_menu.connect(self.scrollbar_context_menu)
         self.actions_toolbar.initialize(self.web_view, self.search_dock.toggleViewAction())
         self.setCentralWidget(self.web_view)
         self.loading_overlay = LoadingOverlay(self)
@@ -193,14 +187,38 @@ class EbookViewer(MainWindow):
             rmap[v].append(k)
         self.actions_toolbar.set_tooltips(rmap)
 
-    def toggle_inspector(self):
-        visible = self.inspector_dock.toggleViewAction().isChecked()
-        self.inspector_dock.setVisible(not visible)
-
     def resizeEvent(self, ev):
         self.loading_overlay.resize(self.size())
         return MainWindow.resizeEvent(self, ev)
 
+    def scrollbar_context_menu(self, x, y, frac):
+        m = QMenu(self)
+        amap = {}
+
+        def a(text, name):
+            m.addAction(text)
+            amap[text] = name
+
+        a(_('Scroll here'), 'here')
+        m.addSeparator()
+        a(_('Start of book'), 'start_of_book')
+        a(_('End of book'), 'end_of_book')
+        m.addSeparator()
+        a(_('Previous section'), 'previous_section')
+        a(_('Next section'), 'next_section')
+        m.addSeparator()
+        a(_('Start of current file'), 'start_of_file')
+        a(_('End of current file'), 'end_of_file')
+
+        q = m.exec_(QCursor.pos())
+        if not q:
+            return
+        q = amap[q.text()]
+        if q == 'here':
+            self.web_view.goto_frac(frac)
+        else:
+            self.web_view.trigger_shortcut(q)
+
     # IPC {{{
     def handle_commandline_arg(self, arg):
         if arg:
@@ -246,6 +264,10 @@ class EbookViewer(MainWindow):
 
     # Docks (ToC, Bookmarks, Lookup, etc.) {{{
 
+    def toggle_inspector(self):
+        visible = self.inspector_dock.toggleViewAction().isChecked()
+        self.inspector_dock.setVisible(not visible)
+
     def toggle_toc(self):
         self.toc_dock.setVisible(not self.toc_dock.isVisible())
 
diff --git a/src/calibre/gui2/viewer/web_view.py b/src/calibre/gui2/viewer/web_view.py
index 9002c84287..62ddc7b0c7 100644
--- a/src/calibre/gui2/viewer/web_view.py
+++ b/src/calibre/gui2/viewer/web_view.py
@@ -272,6 +272,7 @@ class ViewerBridge(Bridge):
     reset_interface = from_js()
     quit = from_js()
     customize_toolbar = from_js()
+    scrollbar_context_menu = from_js(object, object, object)
 
     create_view = to_js()
     start_book_load = to_js()
@@ -443,6 +444,7 @@ class WebView(RestartingWebEngineView):
     reset_interface = pyqtSignal()
     quit = pyqtSignal()
     customize_toolbar = pyqtSignal()
+    scrollbar_context_menu = pyqtSignal(object, object, object)
     shortcuts_changed = pyqtSignal(object)
     paged_mode_changed = pyqtSignal()
     standalone_misc_settings_changed = pyqtSignal(object)
@@ -491,6 +493,7 @@ class WebView(RestartingWebEngineView):
         self.bridge.reset_interface.connect(self.reset_interface)
         self.bridge.quit.connect(self.quit)
         self.bridge.customize_toolbar.connect(self.customize_toolbar)
+        self.bridge.scrollbar_context_menu.connect(self.scrollbar_context_menu)
         self.bridge.export_shortcut_map.connect(self.set_shortcut_map)
         self.shortcut_map = {}
         self.bridge.report_cfi.connect(self.call_callback)
diff --git a/src/pyj/read_book/iframe.pyj b/src/pyj/read_book/iframe.pyj
index c2bc8b3a17..00facccc43 100644
--- a/src/pyj/read_book/iframe.pyj
+++ b/src/pyj/read_book/iframe.pyj
@@ -116,6 +116,7 @@ class IframeBoss:
             'window_size': self.received_window_size,
             'overlay_visibility_changed': self.on_overlay_visibility_changed,
             'show_search_result': self.show_search_result,
+            'handle_navigation_shortcut': self.on_handle_navigation_shortcut,
         }
         self.comm = IframeClient(handlers)
         self.last_window_ypos = 0
@@ -458,6 +459,10 @@ class IframeBoss:
                 else:
                     self.send_message('handle_shortcut', name=sc_name)
 
+    def on_handle_navigation_shortcut(self, data):
+        self.handle_navigation_shortcut(data.name, data.key or {
+            'key': '', 'altKey': False, 'ctrlKey': False, 'shiftKey': False, 'metaKey': False})
+
     def oncontextmenu(self, evt):
         if self.content_ready:
             evt.preventDefault()
diff --git a/src/pyj/read_book/scrollbar.pyj b/src/pyj/read_book/scrollbar.pyj
index cd7d89e835..639cf6dae6 100644
--- a/src/pyj/read_book/scrollbar.pyj
+++ b/src/pyj/read_book/scrollbar.pyj
@@ -7,7 +7,7 @@ from elementmaker import E
 from book_list.globals import get_session_data
 from book_list.theme import cached_color_to_rgba
 from dom import unique_id
-
+from read_book.globals import ui_operations
 
 SIZE = 10
 
@@ -31,7 +31,7 @@ class BookScrollbar:
         return E.div(
             id=self.container_id,
             style=f'height: 100vh; background-color: #aaa; width: {SIZE}px; border-radius: 5px',
-            onclick=self.bar_clicked,
+            onclick=self.bar_clicked, oncontextmenu=self.context_menu,
             E.div(
                 style=f'position: relative; width: 100%; height: {int(2.2*SIZE)}px; background-color: #444; border-radius: 5px',
                 onmousedown=self.on_bob_mousedown,
@@ -41,6 +41,16 @@ class BookScrollbar:
             )
         )
 
+    def context_menu(self, ev):
+        if ui_operations.scrollbar_context_menu:
+            ev.preventDefault(), ev.stopPropagation()
+            c = self.container
+            bob = c.firstChild
+            height = c.clientHeight - bob.clientHeight
+            top = max(0, min(ev.clientY - bob.clientHeight, height))
+            frac = max(0, min(top / height, 1))
+            ui_operations.scrollbar_context_menu(ev.screenX, ev.screenY, frac)
+
     def bar_clicked(self, evt):
         if evt.button is 0:
             c = self.container
diff --git a/src/pyj/read_book/view.pyj b/src/pyj/read_book/view.pyj
index a1fd39f8e8..a5ece1ebde 100644
--- a/src/pyj/read_book/view.pyj
+++ b/src/pyj/read_book/view.pyj
@@ -442,6 +442,8 @@ class View:
             self.toggle_autoscroll()
         elif data.name.startsWith('switch_color_scheme:'):
             self.switch_color_scheme(data.name.partition(':')[-1])
+        else:
+            self.iframe_wrapper.send_message('handle_navigation_shortcut', name=data.name)
 
     def on_selection_change(self, data):
         self.currently_showing.selected_text = data.text
diff --git a/src/pyj/viewer-main.pyj b/src/pyj/viewer-main.pyj
index 8da2bedf32..6d4119f1b0 100644
--- a/src/pyj/viewer-main.pyj
+++ b/src/pyj/viewer-main.pyj
@@ -403,6 +403,8 @@ if window is window.top:
         to_python.autoscroll_state_changed(active)
     ui_operations.search_result_not_found = def(sr):
         to_python.search_result_not_found(sr)
+    ui_operations.scrollbar_context_menu = def(x, y, frac):
+        to_python.scrollbar_context_menu(x, y, frac)
 
     document.body.appendChild(E.div(id='view'))
     window.onerror = onerror

From b99756d862add2533d02e628a4100544fd6f7d58 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Wed, 19 Feb 2020 13:06:21 +0530
Subject: [PATCH 039/162] Viewer: Fix stopping autoscroll at end of chapter not
 stopping next chapter jump. Fixes #1863487 [Clicking "Stop auto scroll"
 doesn't work when about to change
 chapter](https://bugs.launchpad.net/calibre/+bug/1863487)

---
 src/pyj/read_book/flow_mode.pyj | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/src/pyj/read_book/flow_mode.pyj b/src/pyj/read_book/flow_mode.pyj
index 4b810a2d8c..a020064eb6 100644
--- a/src/pyj/read_book/flow_mode.pyj
+++ b/src/pyj/read_book/flow_mode.pyj
@@ -236,6 +236,7 @@ class ScrollAnimator:
     def __init__(self):
         self.animation_id = None
         self.auto = False
+        self.auto_timer = None
 
     def is_running(self):
         return self.animation_id is not None
@@ -246,7 +247,7 @@ class ScrollAnimator:
 
         now = window.performance.now()
         self.end_time = now + self.DURATION
-        clearTimeout(self.auto_timer)
+        self.stop_auto_timer()
 
         if not self.is_running() or direction is not self.direction or auto is not self.auto:
             if self.auto and not auto:
@@ -299,6 +300,7 @@ class ScrollAnimator:
             self.pause()
             if opts.scroll_auto_boundary_delay >= 0:
                 self.auto_timer = setTimeout(def():
+                        self.auto_timer = None
                         get_boss().send_message('next_spine_item', previous=self.direction is DIRECTION.Up)
                     , opts.scroll_auto_boundary_delay * 1000)
         else:
@@ -324,6 +326,12 @@ class ScrollAnimator:
             window.cancelAnimationFrame(self.animation_id)
             self.animation_id = None
             self.report()
+        self.stop_auto_timer()
+
+    def stop_auto_timer(self):
+        if self.auto_timer is not None:
+            clearTimeout(self.auto_timer)
+            self.auto_timer = None
 
     def pause(self):
         if self.auto:
@@ -448,6 +456,7 @@ def auto_scroll_action(action):
     elif action is 'stop':
         if is_auto_scroll_active():
             toggle_autoscroll()
+        scroll_animator.stop_auto_timer()
     elif action is 'resume':
         auto_scroll_resume()
     return is_auto_scroll_active()

From 9730226f6ecd8f9b553d17fb1bed77cd77a3e40f Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Wed, 19 Feb 2020 20:45:20 +0530
Subject: [PATCH 040/162] Viewer: Fix current reading position not preserved
 when changing preferences and auto scroll is active. Fixes #1863438 [After
 adjusting the auto scroll speed I am jumped back in the
 book](https://bugs.launchpad.net/calibre/+bug/1863438)

CFI should be updated when jumping directly to preferences panel via
shortcut
---
 src/pyj/read_book/view.pyj | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/src/pyj/read_book/view.pyj b/src/pyj/read_book/view.pyj
index a5ece1ebde..90573e4c1d 100644
--- a/src/pyj/read_book/view.pyj
+++ b/src/pyj/read_book/view.pyj
@@ -425,7 +425,7 @@ class View:
         elif data.name is 'print':
             ui_operations.print_book()
         elif data.name is 'preferences':
-            self.overlay.show_prefs()
+            self.show_chrome({'initial_panel': 'show_prefs'})
         elif data.name is 'metadata':
             self.overlay.show_metadata()
         elif data.name is 'goto_location':
@@ -525,12 +525,16 @@ class View:
         elements = {}
         if data and data.elements:
             elements = data.elements
-        self.get_current_cfi('show-chrome-' + self.show_chrome_counter, self.do_show_chrome.bind(None, elements))
+        initial_panel = data?.initial_panel or None
+        self.get_current_cfi('show-chrome-' + self.show_chrome_counter, self.do_show_chrome.bind(None, elements, initial_panel))
 
-    def do_show_chrome(self, elements, request_id, cfi_data):
+    def do_show_chrome(self, elements, initial_panel, request_id, cfi_data):
         self.hide_overlays()
         self.update_cfi_data(cfi_data)
-        self.overlay.show(elements)
+        if initial_panel:
+            getattr(self.overlay, initial_panel)()
+        else:
+            self.overlay.show(elements)
 
     def show_search(self):
         self.hide_overlays()

From 2be73652a63d2f4016516a8d04c11c1416a90215 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Wed, 19 Feb 2020 21:31:53 +0530
Subject: [PATCH 041/162] Viewer: Ensure last read position is fully accurate

This is particularly important when quitting while autoscroll is active
as CFI is not updated during autoscroll.
---
 src/calibre/gui2/viewer/ui.py       | 22 ++++++++++++++++++++--
 src/calibre/gui2/viewer/web_view.py |  9 +++++++++
 src/pyj/read_book/view.pyj          | 16 ++++++++++++----
 src/pyj/viewer-main.pyj             |  8 ++++++++
 4 files changed, 49 insertions(+), 6 deletions(-)

diff --git a/src/calibre/gui2/viewer/ui.py b/src/calibre/gui2/viewer/ui.py
index 19549f30ee..e9879a736a 100644
--- a/src/calibre/gui2/viewer/ui.py
+++ b/src/calibre/gui2/viewer/ui.py
@@ -14,7 +14,7 @@ from threading import Thread
 
 from PyQt5.Qt import (
     QApplication, QCursor, QDockWidget, QEvent, QMenu, QMimeData, QModelIndex,
-    QPixmap, Qt, QToolBar, QUrl, QVBoxLayout, QWidget, pyqtSignal
+    QPixmap, Qt, QTimer, QToolBar, QUrl, QVBoxLayout, QWidget, pyqtSignal
 )
 
 from calibre import prints
@@ -87,7 +87,7 @@ class EbookViewer(MainWindow):
 
     def __init__(self, open_at=None, continue_reading=None, force_reload=False):
         MainWindow.__init__(self, None)
-        self.shutting_down = False
+        self.shutting_down = self.close_forced = False
         self.force_reload = force_reload
         connect_lambda(self.book_preparation_started, self, lambda self: self.loading_overlay(_(
             'Preparing book for first read, please wait')), type=Qt.QueuedConnection)
@@ -172,6 +172,7 @@ class EbookViewer(MainWindow):
         self.web_view.quit.connect(self.quit, type=Qt.QueuedConnection)
         self.web_view.shortcuts_changed.connect(self.shortcuts_changed)
         self.web_view.scrollbar_context_menu.connect(self.scrollbar_context_menu)
+        self.web_view.close_prep_finished.connect(self.close_prep_finished)
         self.actions_toolbar.initialize(self.web_view, self.search_dock.toggleViewAction())
         self.setCentralWidget(self.web_view)
         self.loading_overlay = LoadingOverlay(self)
@@ -568,7 +569,24 @@ class EbookViewer(MainWindow):
     def quit(self):
         self.close()
 
+    def force_close(self):
+        if not self.close_forced:
+            self.close_forced = True
+            self.quit()
+
+    def close_prep_finished(self, cfi):
+        if cfi:
+            self.cfi_changed(cfi)
+        self.force_close()
+
     def closeEvent(self, ev):
+        if self.current_book_data and self.web_view.view_is_ready and not self.close_forced:
+            ev.ignore()
+            if not self.shutting_down:
+                self.shutting_down = True
+                QTimer.singleShot(2000, self.force_close)
+                self.web_view.prepare_for_close()
+            return
         self.shutting_down = True
         self.search_widget.shutdown()
         try:
diff --git a/src/calibre/gui2/viewer/web_view.py b/src/calibre/gui2/viewer/web_view.py
index 62ddc7b0c7..77f64dfeb1 100644
--- a/src/calibre/gui2/viewer/web_view.py
+++ b/src/calibre/gui2/viewer/web_view.py
@@ -273,6 +273,7 @@ class ViewerBridge(Bridge):
     quit = from_js()
     customize_toolbar = from_js()
     scrollbar_context_menu = from_js(object, object, object)
+    close_prep_finished = from_js(object)
 
     create_view = to_js()
     start_book_load = to_js()
@@ -286,6 +287,7 @@ class ViewerBridge(Bridge):
     trigger_shortcut = to_js()
     set_system_palette = to_js()
     show_search_result = to_js()
+    prepare_for_close = to_js()
 
 
 def apply_font_settings(page_or_view):
@@ -445,6 +447,7 @@ class WebView(RestartingWebEngineView):
     quit = pyqtSignal()
     customize_toolbar = pyqtSignal()
     scrollbar_context_menu = pyqtSignal(object, object, object)
+    close_prep_finished = pyqtSignal(object)
     shortcuts_changed = pyqtSignal(object)
     paged_mode_changed = pyqtSignal()
     standalone_misc_settings_changed = pyqtSignal(object)
@@ -463,6 +466,7 @@ class WebView(RestartingWebEngineView):
         self.show_home_page_on_ready = True
         self._size_hint = QSize(int(w/3), int(w/2))
         self._page = WebPage(self)
+        self.view_is_ready = False
         self.bridge.bridge_ready.connect(self.on_bridge_ready)
         self.bridge.view_created.connect(self.on_view_created)
         self.bridge.content_file_changed.connect(self.on_content_file_changed)
@@ -494,6 +498,7 @@ class WebView(RestartingWebEngineView):
         self.bridge.quit.connect(self.quit)
         self.bridge.customize_toolbar.connect(self.customize_toolbar)
         self.bridge.scrollbar_context_menu.connect(self.scrollbar_context_menu)
+        self.bridge.close_prep_finished.connect(self.close_prep_finished)
         self.bridge.export_shortcut_map.connect(self.set_shortcut_map)
         self.shortcut_map = {}
         self.bridge.report_cfi.connect(self.call_callback)
@@ -578,6 +583,7 @@ class WebView(RestartingWebEngineView):
 
     def on_view_created(self, data):
         self.view_created.emit(data)
+        self.view_is_ready = True
 
     def on_content_file_changed(self, data):
         self.current_content_file = data
@@ -669,3 +675,6 @@ class WebView(RestartingWebEngineView):
 
     def palette_changed(self):
         self.execute_when_ready('set_system_palette', system_colors())
+
+    def prepare_for_close(self):
+        self.execute_when_ready('prepare_for_close')
diff --git a/src/pyj/read_book/view.pyj b/src/pyj/read_book/view.pyj
index 90573e4c1d..4dc7d98133 100644
--- a/src/pyj/read_book/view.pyj
+++ b/src/pyj/read_book/view.pyj
@@ -172,7 +172,7 @@ class View:
         self.current_progress_frac = self.current_file_progress_frac = 0
         self.current_toc_node = self.current_toc_toplevel_node = None
         self.report_cfi_callbacks = {}
-        self.show_chrome_counter = 0
+        self.get_cfi_counter = 0
         self.show_loading_callback_timer = None
         self.timer_ids = {'clock': 0}
         self.book_scrollbar = BookScrollbar(self)
@@ -521,12 +521,11 @@ class View:
         self.iframe.contentWindow.focus()
 
     def show_chrome(self, data):
-        self.show_chrome_counter += 1
         elements = {}
         if data and data.elements:
             elements = data.elements
         initial_panel = data?.initial_panel or None
-        self.get_current_cfi('show-chrome-' + self.show_chrome_counter, self.do_show_chrome.bind(None, elements, initial_panel))
+        self.get_current_cfi('show-chrome', self.do_show_chrome.bind(None, elements, initial_panel))
 
     def do_show_chrome(self, elements, initial_panel, request_id, cfi_data):
         self.hide_overlays()
@@ -536,6 +535,13 @@ class View:
         else:
             self.overlay.show(elements)
 
+    def prepare_for_close(self):
+
+        def close_prepared(request_id, cfi_data):
+            ui_operations.close_prep_finished(cfi_data.cfi)
+
+        self.get_current_cfi('prepare-close', close_prepared)
+
     def show_search(self):
         self.hide_overlays()
         if runtime.is_standalone_viewer:
@@ -903,6 +909,8 @@ class View:
             self.goto_named_destination(toc_node.dest, toc_node.frag)
 
     def get_current_cfi(self, request_id, callback):
+        self.get_cfi_counter += 1
+        request_id += ':' + self.get_cfi_counter
         self.report_cfi_callbacks[request_id] = callback
         self.iframe_wrapper.send_message('get_current_cfi', request_id=request_id)
 
@@ -923,7 +931,7 @@ class View:
     def on_report_cfi(self, data):
         cb = self.report_cfi_callbacks[data.request_id]
         if cb:
-            cb(data.request_id, {
+            cb(data.request_id.rpartition(':')[0], {
                 'cfi': data.cfi,
                 'progress_frac': data.progress_frac,
                 'file_progress_frac': data.file_progress_frac,
diff --git a/src/pyj/viewer-main.pyj b/src/pyj/viewer-main.pyj
index 6d4119f1b0..adab0bef94 100644
--- a/src/pyj/viewer-main.pyj
+++ b/src/pyj/viewer-main.pyj
@@ -293,6 +293,12 @@ def show_search_result(sr):
     if view:
         view.show_search_result(sr)
 
+@from_python
+def prepare_for_close():
+    if view:
+        view.prepare_for_close()
+    else:
+        ui_operations.close_prep_finished(None)
 
 def onerror(msg, script_url, line_number, column_number, error_object):
     if not error_object:
@@ -405,6 +411,8 @@ if window is window.top:
         to_python.search_result_not_found(sr)
     ui_operations.scrollbar_context_menu = def(x, y, frac):
         to_python.scrollbar_context_menu(x, y, frac)
+    ui_operations.close_prep_finished = def(cfi):
+        to_python.close_prep_finished(cfi)
 
     document.body.appendChild(E.div(id='view'))
     window.onerror = onerror

From 3196e631effe3eb3a40cd4307582910db58b12ad Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Thu, 20 Feb 2020 11:25:27 +0530
Subject: [PATCH 042/162] String changes

---
 src/calibre/gui2/tweak_book/ui.py | 2 +-
 src/pyj/read_book/overlay.pyj     | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/calibre/gui2/tweak_book/ui.py b/src/calibre/gui2/tweak_book/ui.py
index 0ce8f2f59c..149e58a537 100644
--- a/src/calibre/gui2/tweak_book/ui.py
+++ b/src/calibre/gui2/tweak_book/ui.py
@@ -512,7 +512,7 @@ class Main(MainWindow):
             'window-close.png', _('&Close current tab'), self.central.close_current_editor, 'close-current-tab', 'Ctrl+W', _(
                 'Close the currently open tab'))
         self.action_close_all_but_current_tab = reg(
-            'edit-clear.png', _('&Close other tabs'), self.central.close_all_but_current_editor, 'close-all-but-current-tab', 'Ctrl+Alt+W', _(
+            'edit-clear.png', _('C&lose other tabs'), self.central.close_all_but_current_editor, 'close-all-but-current-tab', 'Ctrl+Alt+W', _(
                 'Close all tabs except the current tab'))
         self.action_help = treg(
             'help.png', _('User &Manual'), lambda : open_url(QUrl(localize_user_manual_link(
diff --git a/src/pyj/read_book/overlay.pyj b/src/pyj/read_book/overlay.pyj
index 0de46604c5..8d9000715b 100644
--- a/src/pyj/read_book/overlay.pyj
+++ b/src/pyj/read_book/overlay.pyj
@@ -281,7 +281,7 @@ class MainOverlay:  # {{{
 
             E.ul(
                 ac(_('Font size'), _('Change text size'), self.overlay.show_font_size_chooser, 'Aa', True),
-                ac(_('Preferences'), _('Configure the book reader'), self.overlay.show_prefs, 'cogs'),
+                ac(_('Preferences'), _('Configure the book viewer'), self.overlay.show_prefs, 'cogs'),
             ),
 
             class_=MAIN_OVERLAY_ACTIONS_CLASS

From 2ef5d143c5f0da918472aed3714c3ad386ffec15 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Thu, 20 Feb 2020 14:00:45 +0530
Subject: [PATCH 043/162] Viewer: Fix a regression that broke detection of
 popup footnotes using epub3 markup

---
 src/pyj/read_book/footnotes.pyj | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/src/pyj/read_book/footnotes.pyj b/src/pyj/read_book/footnotes.pyj
index 459708b6db..e72a7fb758 100644
--- a/src/pyj/read_book/footnotes.pyj
+++ b/src/pyj/read_book/footnotes.pyj
@@ -14,6 +14,12 @@ def elem_roles(elem):
     return {k.toLowerCase(): True for k in (elem.getAttribute('role') or '').split(' ')}
 
 
+def epub_type(elem):
+    for a in elem.attributes:
+        if a.nodeName.toLowerCase() == 'epub:type' and a.nodeValue:
+            return a.nodeValue
+
+
 def get_containing_block(node):
     while node and node.tagName and block_names[node.tagName.toLowerCase()] is not True:
         node = node.parentNode
@@ -26,6 +32,8 @@ def is_footnote_link(a, dest_name, dest_frag, src_name, link_to_map):
         return True
     if roles['doc-link']:
         return False
+    if epub_type(a) is 'noteref':
+        return True
 
     # Check if node or any of its first few parents have vertical-align set
     x, num = a, 3
@@ -50,7 +58,7 @@ def is_footnote_link(a, dest_name, dest_frag, src_name, link_to_map):
     eid = a.getAttribute('id') or a.getAttribute('name')
     files_linking_to_self = link_to_map[src_name]
     if eid and files_linking_to_self:
-        files_linking_to_anchor = files_linking_to_self[eid]
+        files_linking_to_anchor = files_linking_to_self[eid] or v'[]'
         if files_linking_to_anchor.length > 1 or (files_linking_to_anchor.length == 1 and files_linking_to_anchor[0] is not src_name):
             # An  link that is linked back from some other
             # file in the spine, most likely an endnote. We exclude links that are
@@ -77,6 +85,11 @@ is_footnote_link.vert_aligns = {'sub': True, 'super': True, 'top': True, 'bottom
 
 
 def is_epub_footnote(node):
+    et = epub_type(node)
+    if et:
+        et = et.toLowerCase()
+        if et is 'note' or et is 'footnote' or et is 'rearnote':
+            return True
     roles = elem_roles(node)
     if roles['doc-note'] or roles['doc-footnote'] or roles['doc-rearnote']:
         return True

From fb1051476cfcb45f46cbdd5e35ac12b59d70fa36 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Thu, 20 Feb 2020 16:09:10 +0530
Subject: [PATCH 044/162] Add a new Quick select action to quickly select a
 virtual library with a few keystrokes. Activated by Ctrl+t or the Virtual
 library menu

---
 src/calibre/gui2/actions/virtual_library.py  |  9 +++++-
 src/calibre/gui2/search_restriction_mixin.py | 25 +++++++++++++++
 src/calibre/gui2/tweak_book/widgets.py       | 33 +++++++++++---------
 3 files changed, 51 insertions(+), 16 deletions(-)

diff --git a/src/calibre/gui2/actions/virtual_library.py b/src/calibre/gui2/actions/virtual_library.py
index 596b697528..496836257f 100644
--- a/src/calibre/gui2/actions/virtual_library.py
+++ b/src/calibre/gui2/actions/virtual_library.py
@@ -4,7 +4,7 @@
 
 from __future__ import absolute_import, division, print_function, unicode_literals
 
-from PyQt5.Qt import QToolButton
+from PyQt5.Qt import QToolButton, QAction
 
 from calibre.gui2.actions import InterfaceAction
 
@@ -24,6 +24,13 @@ class VirtualLibraryAction(InterfaceAction):
     def genesis(self):
         self.menu = m = self.qaction.menu()
         m.aboutToShow.connect(self.about_to_show_menu)
+        self.qs_action = QAction(self.gui)
+        self.gui.addAction(self.qs_action)
+        self.qs_action.triggered.connect(self.gui.choose_vl_triggerred)
+        self.gui.keyboard.register_shortcut(self.unique_name + ' - ' + 'quick-select-vl',
+            _('Quick select Virtual library'), default_keys=('Ctrl+T',),
+            action=self.qs_action, description=_('Quick select a Virtual library'),
+            group=self.action_spec[0])
 
     def about_to_show_menu(self):
         self.gui.build_virtual_library_menu(self.menu, add_tabs_action=False)
diff --git a/src/calibre/gui2/search_restriction_mixin.py b/src/calibre/gui2/search_restriction_mixin.py
index 23847ded5e..83aa172b90 100644
--- a/src/calibre/gui2/search_restriction_mixin.py
+++ b/src/calibre/gui2/search_restriction_mixin.py
@@ -380,6 +380,8 @@ class SearchRestrictionMixin(object):
         self.build_virtual_library_list(a, self.remove_vl_triggered)
         m.addMenu(a)
 
+        m.addAction(_('Quick select Virtual library'), self.choose_vl_triggerred)
+
         if add_tabs_action:
             if gprefs['show_vl_tabs']:
                 m.addAction(_('Hide virtual library tabs'), self.vl_tabs.disable_bar)
@@ -492,6 +494,29 @@ class SearchRestrictionMixin(object):
             return
         self._remove_vl(name, reapply=True)
 
+    def choose_vl_triggerred(self):
+        from calibre.gui2.tweak_book.widgets import QuickOpen, Results
+        db = self.library_view.model().db
+        virt_libs = db.prefs.get('virtual_libraries', {})
+        if not virt_libs:
+            return error_dialog(self, _('No virtual libraries'), _(
+                'No Virtual libraries present, create some first'), show=True)
+        example = '
{0}S{1}ome {0}B{1}ook {0}C{1}ollection
'.format(
+            '' % Results.EMPH, '')
+        chars = '
sbc
' % Results.EMPH
+        help_text = _('''
Quickly choose a Virtual library by typing in just a few characters from the file name into the field above.
+        For example, if want to choose the VL:
+        {example}
+        Simply type in the characters:
+        {chars}
+        and press Enter.''').format(example=example, chars=chars)
+
+        d = QuickOpen(
+                sorted(virt_libs.keys(), key=sort_key), parent=self, title=_('Choose Virtual library'),
+                name='vl-open', level1=' ', help_text=help_text)
+        if d.exec_() == d.Accepted and d.selected_result:
+            self.apply_virtual_library(library=d.selected_result)
+
     def _remove_vl(self, name, reapply=True):
         db = self.library_view.model().db
         virt_libs = db.prefs.get('virtual_libraries', {})
diff --git a/src/calibre/gui2/tweak_book/widgets.py b/src/calibre/gui2/tweak_book/widgets.py
index 1135135fff..ad0bcb2b5f 100644
--- a/src/calibre/gui2/tweak_book/widgets.py
+++ b/src/calibre/gui2/tweak_book/widgets.py
@@ -24,7 +24,7 @@ from calibre.gui2 import error_dialog, choose_files, choose_save_file, info_dial
 from calibre.gui2.tweak_book import tprefs, current_container
 from calibre.gui2.widgets2 import Dialog as BaseDialog, HistoryComboBox, to_plain_text, PARAGRAPH_SEPARATOR
 from calibre.utils.icu import primary_sort_key, sort_key, primary_contains, numeric_sort_key
-from calibre.utils.matcher import get_char, Matcher
+from calibre.utils.matcher import get_char, Matcher, DEFAULT_LEVEL1, DEFAULT_LEVEL2, DEFAULT_LEVEL3
 from calibre.gui2.complete2 import EditWithComplete
 from polyglot.builtins import iteritems, unicode_type, zip, getcwd, filter as ignore_me
 
@@ -408,11 +408,12 @@ class Results(QWidget):
 
 class QuickOpen(Dialog):
 
-    def __init__(self, items, parent=None):
-        self.matcher = Matcher(items)
+    def __init__(self, items, parent=None, title=None, name='quick-open', level1=DEFAULT_LEVEL1, level2=DEFAULT_LEVEL2, level3=DEFAULT_LEVEL3, help_text=None):
+        self.matcher = Matcher(items, level1=level1, level2=level2, level3=level3)
         self.matches = ()
         self.selected_result = None
-        Dialog.__init__(self, _('Choose file to edit'), 'quick-open', parent=parent)
+        self.help_text = help_text or self.default_help_text()
+        Dialog.__init__(self, title or _('Choose file to edit'), name, parent=parent)
 
     def sizeHint(self):
         ans = Dialog.sizeHint(self)
@@ -420,6 +421,18 @@ class QuickOpen(Dialog):
         ans.setHeight(max(600, ans.height()))
         return ans
 
+    def default_help_text(self):
+        example = '
{0}i{1}mages/{0}c{1}hapter1/{0}s{1}cene{0}3{1}.jpg
'.format(
+            '' % Results.EMPH, '')
+        chars = '
ics3
' % Results.EMPH
+
+        return _('''
Quickly choose a file by typing in just a few characters from the file name into the field above.
+        For example, if want to choose the file:
+        {example}
+        Simply type in the characters:
+        {chars}
+        and press Enter.''').format(example=example, chars=chars)
+
     def setup_ui(self):
         self.l = l = QVBoxLayout(self)
         self.setLayout(l)
@@ -428,17 +441,7 @@ class QuickOpen(Dialog):
         t.textEdited.connect(self.update_matches)
         l.addWidget(t, alignment=Qt.AlignTop)
 
-        example = '
{0}i{1}mages/{0}c{1}hapter1/{0}s{1}cene{0}3{1}.jpg
'.format(
-            '' % Results.EMPH, '')
-        chars = '
ics3
' % Results.EMPH
-
-        self.help_label = hl = QLabel(_(
-            '''
Quickly choose a file by typing in just a few characters from the file name into the field above.
-        For example, if want to choose the file:
-        {example}
-        Simply type in the characters:
-        {chars}
-        and press Enter.''').format(example=example, chars=chars))
+        self.help_label = hl = QLabel(self.help_text)
         hl.setContentsMargins(50, 50, 50, 50), hl.setAlignment(Qt.AlignTop | Qt.AlignHCenter)
         l.addWidget(hl)
         self.results = Results(self)

From dad962b176fd2f7330e1462926b2961fd3232f95 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Thu, 20 Feb 2020 20:19:43 +0530
Subject: [PATCH 045/162] Dont flatten head

---
 src/calibre/ebooks/oeb/transforms/flatcss.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/src/calibre/ebooks/oeb/transforms/flatcss.py b/src/calibre/ebooks/oeb/transforms/flatcss.py
index f617a752f2..43c2b191ca 100644
--- a/src/calibre/ebooks/oeb/transforms/flatcss.py
+++ b/src/calibre/ebooks/oeb/transforms/flatcss.py
@@ -351,7 +351,7 @@ class CSSFlattener(object):
                         value = 0.0
                     cssdict[property] = "%0.5fem" % (value / fsize)
 
-    def flatten_node(self, node, stylizer, names, styles, pseudo_styles, psize, item_id):
+    def flatten_node(self, node, stylizer, names, styles, pseudo_styles, psize, item_id, recurse=True):
         if not isinstance(node.tag, string_or_bytes) \
            or namespace(node.tag) != XHTML_NS:
             return
@@ -569,8 +569,9 @@ class CSSFlattener(object):
             del node.attrib['class']
         if 'style' in node.attrib:
             del node.attrib['style']
-        for child in node:
-            self.flatten_node(child, stylizer, names, styles, pseudo_styles, psize, item_id)
+        if recurse:
+            for child in node:
+                self.flatten_node(child, stylizer, names, styles, pseudo_styles, psize, item_id)
 
     def flatten_head(self, item, href, global_href):
         html = item.data
@@ -661,7 +662,8 @@ class CSSFlattener(object):
             if self.specializer is not None:
                 self.specializer(item, stylizer)
             fsize = self.context.dest.fbase
-            self.flatten_node(html, stylizer, names, styles, pseudo_styles, fsize, item.id)
+            self.flatten_node(html, stylizer, names, styles, pseudo_styles, fsize, item.id, recurse=False)
+            self.flatten_node(html.find(XHTML('body')), stylizer, names, styles, pseudo_styles, fsize, item.id)
         items = sorted(((key, val) for (val, key) in iteritems(styles)), key=lambda x:numeric_sort_key(x[0]))
         # :hover must come after link and :active must come after :hover
         psels = sorted(pseudo_styles, key=lambda x :

From e9fc598a494746312c8836a107996315fe428c84 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Thu, 20 Feb 2020 20:36:52 +0530
Subject: [PATCH 046/162] Avoid un-needed display:block class on 

---
 resources/templates/html.css | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/resources/templates/html.css b/resources/templates/html.css
index ef81fe5390..6122ca0c17 100644
--- a/resources/templates/html.css
+++ b/resources/templates/html.css
@@ -40,7 +40,7 @@
 
 /* blocks */
 
-html, div, map, dt, isindex, form {
+div, map, dt, isindex, form {
   display: block;
 }
 

From 16ec8de6afe1b3e5c9313b919058bc088230fe68 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Fri, 21 Feb 2020 08:57:12 +0530
Subject: [PATCH 047/162] version 4.11.0

---
 Changelog.yaml           | 46 ++++++++++++++++++++++++++++++++++++++++
 src/calibre/constants.py |  2 +-
 2 files changed, 47 insertions(+), 1 deletion(-)

diff --git a/Changelog.yaml b/Changelog.yaml
index c1c60664bd..d3fef71209 100644
--- a/Changelog.yaml
+++ b/Changelog.yaml
@@ -20,6 +20,52 @@
 #   new recipes:
 #     - title:
 
+- version: 4.11.0
+  date: 2020-02-21
+
+  new features:
+    - title: "Viewer: Allow right clicking on the scrollbar to easily access commonly used scrolling shortcuts"
+
+    - title: "Edit book: Preview panel: Allow right clicking on images to edit them"
+
+    - title: "Add a new Quick select action to quickly select a virtual library with a few keystrokes. Activated by Ctrl+t or the Virtual library menu"
+
+    - title: "Viewer: Calculate default column widths based on current font size"
+
+    - title: "Viewer: Add a control to quit the viewer useful on touchscreens."
+      tickets: [1862441]
+
+    - title: "Viewer: Add shortcut for showing metadata (Ctrl+n)"
+      tickets: [1862432]
+
+  bug fixes:
+    - title: "Viewer: Fix a regression that broke detection of pop-up footnotes using EPUB 3 markup"
+
+    - title: "Viewer: Fix current reading position not preserved when changing preferences and auto scroll is active."
+      tickets: [1863438]
+
+    - title: "Viewer: Fix stopping autoscroll at end of chapter not stopping next chapter jump."
+      tickets: [1863487]
+
+    - title: "Fix for viewer window going off screen even when not restoring window geometry"
+
+    - title: "Edit book: Fix syntax highlighting for break-(before|after)"
+      tickets: [1863020]
+
+    - title: "Fix drag and drop of some image files onto edit metadata dialog not working"
+      tickets: [1862440]
+
+    - title: "Conversion pipeline: Fix styles applied via selectors to the  element being ignored"
+      tickets: [1862401]
+
+    - title: "Bulk metadata edit: Fix clear series not resetting series index"
+
+    - title: "Fix clicking on author name in book details panel to search in Goodreads not working if author has more than two parts in his name"
+
+
+  improved recipes:
+    - New York Times
+
 - version: 4.10.0
   date: 2020-02-07
 
diff --git a/src/calibre/constants.py b/src/calibre/constants.py
index 1d16c85385..5d51b801e6 100644
--- a/src/calibre/constants.py
+++ b/src/calibre/constants.py
@@ -6,7 +6,7 @@ from polyglot.builtins import map, unicode_type, environ_item, hasenv, getenv, a
 import sys, locale, codecs, os, importlib, collections
 
 __appname__   = 'calibre'
-numeric_version = (4, 10, 1)
+numeric_version = (4, 11, 0)
 __version__   = '.'.join(map(unicode_type, numeric_version))
 git_version   = None
 __author__    = "Kovid Goyal "

From 2d61ed50d9afbcd3ec851b79e57fd07ecb7dace6 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Fri, 21 Feb 2020 10:48:02 +0530
Subject: [PATCH 048/162] Remove unused variable

---
 bypy/macos/util.c | 1 -
 1 file changed, 1 deletion(-)

diff --git a/bypy/macos/util.c b/bypy/macos/util.c
index ba46caefd7..0b3a40bbf9 100644
--- a/bypy/macos/util.c
+++ b/bypy/macos/util.c
@@ -163,7 +163,6 @@ run(const char **ENV_VARS, const char **ENV_VAR_VALS, char *PROGRAM,
     char *t = NULL;
     int ret = 0, i;
     PyObject *site, *mainf, *res;
-    uint32_t buf_size = PATH_MAX+1;
 
     for (i = 0; i < 3; i++) {
         t = rindex(full_exe_path, '/');

From 6e8a80341b8e97661d5490cc64b2eb79fce452d4 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Fri, 21 Feb 2020 11:37:42 +0530
Subject: [PATCH 049/162] Viewer: Fix a regression that broke the New bookmark
 button

Fixes #1864143 [Cannot create bookmark](https://bugs.launchpad.net/calibre/+bug/1864143)
---
 src/calibre/gui2/viewer/web_view.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/calibre/gui2/viewer/web_view.py b/src/calibre/gui2/viewer/web_view.py
index 77f64dfeb1..30171d21db 100644
--- a/src/calibre/gui2/viewer/web_view.py
+++ b/src/calibre/gui2/viewer/web_view.py
@@ -34,7 +34,7 @@ from calibre.srv.code import get_translations_data
 from calibre.utils.config import JSONConfig
 from calibre.utils.serialize import json_loads
 from calibre.utils.shared_file import share_open
-from polyglot.builtins import as_bytes, iteritems
+from polyglot.builtins import as_bytes, iteritems, unicode_type
 
 try:
     from PyQt5 import sip
@@ -633,7 +633,7 @@ class WebView(RestartingWebEngineView):
             vprefs['local_storage'] = sd
 
     def do_callback(self, func_name, callback):
-        cid = next(self.callback_id_counter)
+        cid = unicode_type(next(self.callback_id_counter))
         self.callback_map[cid] = callback
         self.execute_when_ready('get_current_cfi', cid)
 

From f77c3bf3a6c3949a95b96c06c7b0deeea61c4736 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Fri, 21 Feb 2020 11:47:40 +0530
Subject: [PATCH 050/162] Fix #1864152 [[Enhancement] Look of the Choose
 Virtual library](https://bugs.launchpad.net/calibre/+bug/1864152)

---
 src/calibre/gui2/search_restriction_mixin.py |  6 +++---
 src/calibre/gui2/tweak_book/widgets.py       | 16 +++++++++++-----
 2 files changed, 14 insertions(+), 8 deletions(-)

diff --git a/src/calibre/gui2/search_restriction_mixin.py b/src/calibre/gui2/search_restriction_mixin.py
index 83aa172b90..e2ab1c1594 100644
--- a/src/calibre/gui2/search_restriction_mixin.py
+++ b/src/calibre/gui2/search_restriction_mixin.py
@@ -495,15 +495,15 @@ class SearchRestrictionMixin(object):
         self._remove_vl(name, reapply=True)
 
     def choose_vl_triggerred(self):
-        from calibre.gui2.tweak_book.widgets import QuickOpen, Results
+        from calibre.gui2.tweak_book.widgets import QuickOpen, emphasis_style
         db = self.library_view.model().db
         virt_libs = db.prefs.get('virtual_libraries', {})
         if not virt_libs:
             return error_dialog(self, _('No virtual libraries'), _(
                 'No Virtual libraries present, create some first'), show=True)
         example = '
{0}S{1}ome {0}B{1}ook {0}C{1}ollection
'.format(
-            '' % Results.EMPH, '')
-        chars = '
sbc
' % Results.EMPH
+            '' % emphasis_style(), '')
+        chars = '
sbc
' % emphasis_style()
         help_text = _('''
Quickly choose a Virtual library by typing in just a few characters from the file name into the field above.
         For example, if want to choose the VL:
         {example}
diff --git a/src/calibre/gui2/tweak_book/widgets.py b/src/calibre/gui2/tweak_book/widgets.py
index ad0bcb2b5f..a0f302a4d2 100644
--- a/src/calibre/gui2/tweak_book/widgets.py
+++ b/src/calibre/gui2/tweak_book/widgets.py
@@ -275,9 +275,13 @@ def make_highlighted_text(emph, text, positions):
     return text
 
 
+def emphasis_style():
+    pal = QApplication.instance().palette()
+    return 'color: {}; font-weight: bold'.format(pal.color(pal.Link).name())
+
+
 class Results(QWidget):
 
-    EMPH = "color:magenta; font-weight:bold"
     MARGIN = 4
 
     item_selected = pyqtSignal()
@@ -355,7 +359,7 @@ class Results(QWidget):
         self.update()
 
     def make_text(self, text, positions):
-        text = QStaticText(make_highlighted_text(self.EMPH, text, positions))
+        text = QStaticText(make_highlighted_text(emphasis_style(), text, positions))
         text.setTextOption(self.text_option)
         text.setTextFormat(Qt.RichText)
         return text
@@ -423,8 +427,8 @@ class QuickOpen(Dialog):
 
     def default_help_text(self):
         example = '
{0}i{1}mages/{0}c{1}hapter1/{0}s{1}cene{0}3{1}.jpg
'.format(
-            '' % Results.EMPH, '')
-        chars = '
ics3
' % Results.EMPH
+            '' % emphasis_style(), '')
+        chars = '
ics3
' % emphasis_style()
 
         return _('''
Quickly choose a file by typing in just a few characters from the file name into the field above.
         For example, if want to choose the file:
@@ -439,6 +443,8 @@ class QuickOpen(Dialog):
 
         self.text = t = QLineEdit(self)
         t.textEdited.connect(self.update_matches)
+        t.setClearButtonEnabled(True)
+        t.setPlaceholderText(_('Search'))
         l.addWidget(t, alignment=Qt.AlignTop)
 
         self.help_label = hl = QLabel(self.help_text)
@@ -509,7 +515,7 @@ class NamesDelegate(QStyledItemDelegate):
             to.setWrapMode(to.NoWrap)
             to.setAlignment(Qt.AlignLeft | Qt.AlignVCenter)
             positions = sorted(set(positions) - {-1}, reverse=True)
-            text = '%s' % make_highlighted_text(Results.EMPH, text, positions)
+            text = '%s' % make_highlighted_text(emphasis_style(), text, positions)
             doc = QTextDocument()
             c = 'rgb(%d, %d, %d)'%c.getRgb()[:3]
             doc.setDefaultStyleSheet(' body { color: %s }'%c)

From 2a765dbe0506c7547a8ae40cb9c77e747fad6502 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Fri, 21 Feb 2020 11:49:26 +0530
Subject: [PATCH 051/162] ...

---
 src/calibre/gui2/search_restriction_mixin.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/calibre/gui2/search_restriction_mixin.py b/src/calibre/gui2/search_restriction_mixin.py
index e2ab1c1594..5c3c6ec6e7 100644
--- a/src/calibre/gui2/search_restriction_mixin.py
+++ b/src/calibre/gui2/search_restriction_mixin.py
@@ -504,7 +504,7 @@ class SearchRestrictionMixin(object):
         example = '
{0}S{1}ome {0}B{1}ook {0}C{1}ollection
'.format(
             '' % emphasis_style(), '')
         chars = '
sbc
' % emphasis_style()
-        help_text = _('''
Quickly choose a Virtual library by typing in just a few characters from the file name into the field above.
+        help_text = _('''
Quickly choose a Virtual library by typing in just a few characters from the library name into the field above.
         For example, if want to choose the VL:
         {example}
         Simply type in the characters:

From 679bd061e69348baec78a88d28fdde609cd8d0e2 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Fri, 21 Feb 2020 14:26:48 +0530
Subject: [PATCH 052/162] String changes

---
 src/calibre/gui2/tweak_book/boss.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/calibre/gui2/tweak_book/boss.py b/src/calibre/gui2/tweak_book/boss.py
index d707f8a78d..17f88e6723 100644
--- a/src/calibre/gui2/tweak_book/boss.py
+++ b/src/calibre/gui2/tweak_book/boss.py
@@ -1376,7 +1376,7 @@ class Boss(QObject):
         from calibre.gui2.open_with import run_program
         run_program(entry, dest.name, self)
         if question_dialog(self.gui, _('File opened'), _(
-            'When you are done editing {0} click "Update" to update'
+            'When you are done editing {0} click "Import" to update'
             ' the file in the book or "Discard" to lose any changes.').format(file_name),
             yes_text=_('Import'), no_text=_('Discard')
         ):

From 6b18100c703a6d510556bd5fd87508f16a6b96c5 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Fri, 21 Feb 2020 17:33:37 +0530
Subject: [PATCH 053/162] version 4.11.1

---
 Changelog.yaml           | 2 +-
 src/calibre/constants.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/Changelog.yaml b/Changelog.yaml
index d3fef71209..854349b4c3 100644
--- a/Changelog.yaml
+++ b/Changelog.yaml
@@ -20,7 +20,7 @@
 #   new recipes:
 #     - title:
 
-- version: 4.11.0
+- version: 4.11.1
   date: 2020-02-21
 
   new features:
diff --git a/src/calibre/constants.py b/src/calibre/constants.py
index 5d51b801e6..085d2fafe5 100644
--- a/src/calibre/constants.py
+++ b/src/calibre/constants.py
@@ -6,7 +6,7 @@ from polyglot.builtins import map, unicode_type, environ_item, hasenv, getenv, a
 import sys, locale, codecs, os, importlib, collections
 
 __appname__   = 'calibre'
-numeric_version = (4, 11, 0)
+numeric_version = (4, 11, 1)
 __version__   = '.'.join(map(unicode_type, numeric_version))
 git_version   = None
 __author__    = "Kovid Goyal "

From c7b8bd30e89c02689b0d023a314b7fc2ac113409 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Fri, 21 Feb 2020 19:26:09 +0530
Subject: [PATCH 054/162] String changes

---
 src/calibre/gui2/init.py                     |  2 +-
 src/calibre/gui2/search_restriction_mixin.py | 22 ++++++++++----------
 2 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/src/calibre/gui2/init.py b/src/calibre/gui2/init.py
index 9f3fa08602..4e7bbcba45 100644
--- a/src/calibre/gui2/init.py
+++ b/src/calibre/gui2/init.py
@@ -531,7 +531,7 @@ class VLTabs(QTabBar):  # {{{
             s = m._s = m.addMenu(_('Restore hidden tabs'))
             for x in hidden:
                 s.addAction(x, partial(self.restore, x))
-        m.addAction(_('Hide virtual library tabs'), self.disable_bar)
+        m.addAction(_('Hide Virtual library tabs'), self.disable_bar)
         if gprefs['vl_tabs_closable']:
             m.addAction(_('Lock virtual library tabs'), self.lock_tab)
         else:
diff --git a/src/calibre/gui2/search_restriction_mixin.py b/src/calibre/gui2/search_restriction_mixin.py
index 5c3c6ec6e7..43a45b1bb4 100644
--- a/src/calibre/gui2/search_restriction_mixin.py
+++ b/src/calibre/gui2/search_restriction_mixin.py
@@ -99,9 +99,9 @@ class CreateVirtualLibrary(QDialog):  # {{{
         self.existing_names = existing_names
 
         if editing:
-            self.setWindowTitle(_('Edit virtual library'))
+            self.setWindowTitle(_('Edit Virtual library'))
         else:
-            self.setWindowTitle(_('Create virtual library'))
+            self.setWindowTitle(_('Create Virtual library'))
         self.setWindowIcon(QIcon(I('lt.png')))
 
         gl = QGridLayout()
@@ -127,7 +127,7 @@ class CreateVirtualLibrary(QDialog):  # {{{
         gl.addWidget(self.vl_text, 1, 1)
         self.vl_text.setText(_build_full_search_string(self.gui))
 
-        self.sl = sl = QLabel('
'+_('Create a virtual library based on: ')+
+        self.sl = sl = QLabel('
'+_('Create a Virtual library based on: ')+
             ('{0}, '
             '{1}, '
             '{2}, '
@@ -143,11 +143,11 @@ class CreateVirtualLibrary(QDialog):  # {{{
         self.hl = hl = QLabel(_('''
             
Virtual libraries

 
-            
With virtual libraries, you can restrict calibre to only show
-            you books that match a search. When a virtual library is in effect, calibre
+            
With Virtual libraries, you can restrict calibre to only show
+            you books that match a search. When a Virtual library is in effect, calibre
             behaves as though the library contains only the matched books. The Tag browser
             display only the tags/authors/series/etc. that belong to the matched books and any searches
-            you do will only search within the books in the virtual library. This
+            you do will only search within the books in the Virtual library. This
             is a good way to partition your large library into smaller and easier to work with subsets.

 
             
For example you can use a Virtual library to only show you books with the Tag "Unread"
@@ -225,7 +225,7 @@ class CreateVirtualLibrary(QDialog):  # {{{
         if self.editing and (self.vl_text.text() != self.original_search or
                              self.new_name != self.editing):
             if not question_dialog(self.gui, _('Search text changed'),
-                         _('The virtual library name or the search text has changed. '
+                         _('The Virtual library name or the search text has changed. '
                            'Do you want to discard these changes?'),
                          default_yes=False):
                 self.vl_name.blockSignals(True)
@@ -263,13 +263,13 @@ class CreateVirtualLibrary(QDialog):  # {{{
         n = unicode_type(self.vl_name.currentText()).strip()
         if not n:
             error_dialog(self.gui, _('No name'),
-                         _('You must provide a name for the new virtual library'),
+                         _('You must provide a name for the new Virtual library'),
                          show=True)
             return
 
         if n.startswith('*'):
             error_dialog(self.gui, _('Invalid name'),
-                         _('A virtual library name cannot begin with "*"'),
+                         _('A Virtual library name cannot begin with "*"'),
                          show=True)
             return
 
@@ -283,7 +283,7 @@ class CreateVirtualLibrary(QDialog):  # {{{
         v = unicode_type(self.vl_text.text()).strip()
         if not v:
             error_dialog(self.gui, _('No search string'),
-                         _('You must provide a search to define the new virtual library'),
+                         _('You must provide a search to define the new Virtual library'),
                          show=True)
             return
 
@@ -298,7 +298,7 @@ class CreateVirtualLibrary(QDialog):  # {{{
 
         if not recs and not question_dialog(
                 self.gui, _('Search found no books'),
-                _('The search found no books, so the virtual library '
+                _('The search found no books, so the Virtual library '
                 'will be empty. Do you really want to use that search?'),
                 default_yes=False):
             return

From 07201fc28000c4759b5d7863623f531f47796244 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Sat, 22 Feb 2020 08:39:13 +0530
Subject: [PATCH 055/162] Content server: Fix a regression that broke scrolling
 in the server in 4.11. Fixes #1864263 [Content server new scrolling issues
 with 4.11.1](https://bugs.launchpad.net/calibre/+bug/1864263)

---
 src/pyj/book_list/main.pyj | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/src/pyj/book_list/main.pyj b/src/pyj/book_list/main.pyj
index 23a6d01152..b52de2eb9a 100644
--- a/src/pyj/book_list/main.pyj
+++ b/src/pyj/book_list/main.pyj
@@ -67,9 +67,7 @@ def init_ui():
     install(translations)
     remove_initial_progress_bar()
     document.head.appendChild(E.style(get_widget_css()))
-    # See https://github.com/kovidgoyal/calibre/pull/1101
-    # for why we need touch-action: none
-    set_css(document.body, background_color=get_color('window-background'), color=get_color('window-foreground'), touch_action='none')
+    set_css(document.body, background_color=get_color('window-background'), color=get_color('window-foreground'))
     document.body.appendChild(E.div())
     document.body.lastChild.appendChild(E.div(id=book_list_container_id, style='display: none'))
     document.body.lastChild.appendChild(E.div(id=read_book_container_id, style='display: none'))

From 20b2b0d5d45b27b904d97ba7425e41040ede610f Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Sat, 22 Feb 2020 08:53:54 +0530
Subject: [PATCH 056/162] Edit book: Fix right click to edit images in the
 preview panel not working in books with html files and images in separate top
 level folders

---
 src/calibre/gui2/tweak_book/preview.py | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)

diff --git a/src/calibre/gui2/tweak_book/preview.py b/src/calibre/gui2/tweak_book/preview.py
index cce1b07951..4f13b71e8c 100644
--- a/src/calibre/gui2/tweak_book/preview.py
+++ b/src/calibre/gui2/tweak_book/preview.py
@@ -400,15 +400,13 @@ class WebView(RestartingWebEngineView, OpenWithHandler):
             if url.scheme() == FAKE_PROTOCOL:
                 href = url.path().lstrip('/')
                 c = current_container()
-                current_name = self.parent().current_name
-                if current_name:
-                    resource_name = c.href_to_name(href, current_name)
-                    if resource_name and c.exists(resource_name) and resource_name not in c.names_that_must_not_be_changed:
-                        self.add_open_with_actions(menu, resource_name)
-                        if data.mediaType() == data.MediaTypeImage:
-                            mime = c.mime_map[resource_name]
-                            if mime.startswith('image/'):
-                                menu.addAction(_('Edit %s') % resource_name, partial(self.edit_image, resource_name))
+                resource_name = c.href_to_name(href)
+                if resource_name and c.exists(resource_name) and resource_name not in c.names_that_must_not_be_changed:
+                    self.add_open_with_actions(menu, resource_name)
+                    if data.mediaType() == data.MediaTypeImage:
+                        mime = c.mime_map[resource_name]
+                        if mime.startswith('image/'):
+                            menu.addAction(_('Edit %s') % resource_name, partial(self.edit_image, resource_name))
         menu.exec_(ev.globalPos())
 
     def open_with(self, file_name, fmt, entry):

From 7af7fcf561be415dbfa656533b4b7b9551686a17 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Sat, 22 Feb 2020 08:58:49 +0530
Subject: [PATCH 057/162] ...

---
 src/calibre/gui2/tweak_book/preview.py | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/src/calibre/gui2/tweak_book/preview.py b/src/calibre/gui2/tweak_book/preview.py
index 4f13b71e8c..01ba7ceaee 100644
--- a/src/calibre/gui2/tweak_book/preview.py
+++ b/src/calibre/gui2/tweak_book/preview.py
@@ -399,14 +399,15 @@ class WebView(RestartingWebEngineView, OpenWithHandler):
             url = data.mediaUrl()
             if url.scheme() == FAKE_PROTOCOL:
                 href = url.path().lstrip('/')
-                c = current_container()
-                resource_name = c.href_to_name(href)
-                if resource_name and c.exists(resource_name) and resource_name not in c.names_that_must_not_be_changed:
-                    self.add_open_with_actions(menu, resource_name)
-                    if data.mediaType() == data.MediaTypeImage:
-                        mime = c.mime_map[resource_name]
-                        if mime.startswith('image/'):
-                            menu.addAction(_('Edit %s') % resource_name, partial(self.edit_image, resource_name))
+                if href:
+                    c = current_container()
+                    resource_name = c.href_to_name(href)
+                    if resource_name and c.exists(resource_name) and resource_name not in c.names_that_must_not_be_changed:
+                        self.add_open_with_actions(menu, resource_name)
+                        if data.mediaType() == data.MediaTypeImage:
+                            mime = c.mime_map[resource_name]
+                            if mime.startswith('image/'):
+                                menu.addAction(_('Edit %s') % resource_name, partial(self.edit_image, resource_name))
         menu.exec_(ev.globalPos())
 
     def open_with(self, file_name, fmt, entry):

From 56c0102d20adcaf9ff278126536f4275ef35aac4 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Sat, 22 Feb 2020 09:25:45 +0530
Subject: [PATCH 058/162] When running Edit book on a book with no formats ask
 if an empty book should be created. Fixes #1864234 [[Enhancement] Add option
 to open Edit book even if the selected book is not an EPUB or
 AZW3](https://bugs.launchpad.net/calibre/+bug/1864234)

---
 src/calibre/gui2/actions/add.py        | 24 ++++++++++++++----------
 src/calibre/gui2/actions/tweak_epub.py | 22 ++++++++++++++++------
 2 files changed, 30 insertions(+), 16 deletions(-)

diff --git a/src/calibre/gui2/actions/add.py b/src/calibre/gui2/actions/add.py
index 97f23c3e5f..c22d5d2bf5 100644
--- a/src/calibre/gui2/actions/add.py
+++ b/src/calibre/gui2/actions/add.py
@@ -232,21 +232,25 @@ class AddAction(InterfaceAction):
                         return
 
         for id_ in ids:
-            from calibre.ebooks.oeb.polish.create import create_book
-            pt = PersistentTemporaryFile(suffix='.' + format_)
-            pt.close()
-            try:
-                mi = db.new_api.get_metadata(id_, get_cover=False,
-                                    get_user_categories=False, cover_as_data=False)
-                create_book(mi, pt.name, fmt=format_)
-                db.add_format_with_hooks(id_, format_, pt.name, index_is_id=True, notify=True)
-            finally:
-                os.remove(pt.name)
+            self.add_empty_format_to_book(id_, format_)
 
         current_idx = self.gui.library_view.currentIndex()
         if current_idx.isValid():
             view.model().current_changed(current_idx, current_idx)
 
+    def add_empty_format_to_book(self, book_id, fmt):
+        from calibre.ebooks.oeb.polish.create import create_book
+        db = self.gui.current_db
+        pt = PersistentTemporaryFile(suffix='.' + fmt.lower())
+        pt.close()
+        try:
+            mi = db.new_api.get_metadata(book_id, get_cover=False,
+                                get_user_categories=False, cover_as_data=False)
+            create_book(mi, pt.name, fmt=fmt.lower())
+            db.add_format_with_hooks(book_id, fmt, pt.name, index_is_id=True, notify=True)
+        finally:
+            os.remove(pt.name)
+
     def add_archive(self, single):
         paths = choose_files(
             self.gui, 'recursive-archive-add', _('Choose archive file'),
diff --git a/src/calibre/gui2/actions/tweak_epub.py b/src/calibre/gui2/actions/tweak_epub.py
index b8cfd8b1ce..d4c343cf6f 100644
--- a/src/calibre/gui2/actions/tweak_epub.py
+++ b/src/calibre/gui2/actions/tweak_epub.py
@@ -10,7 +10,7 @@ import time
 
 from PyQt5.Qt import QTimer, QDialog, QDialogButtonBox, QCheckBox, QVBoxLayout, QLabel, Qt
 
-from calibre.gui2 import error_dialog
+from calibre.gui2 import error_dialog, question_dialog
 from calibre.gui2.actions import InterfaceAction
 
 
@@ -105,13 +105,23 @@ class TweakEpubAction(InterfaceAction):
         from calibre.ebooks.oeb.polish.main import SUPPORTED
         db = self.gui.library_view.model().db
         fmts = db.formats(book_id, index_is_id=True) or ''
-        fmts = [x.upper().strip() for x in fmts.split(',')]
+        fmts = [x.upper().strip() for x in fmts.split(',') if x]
         tweakable_fmts = set(fmts).intersection(SUPPORTED)
         if not tweakable_fmts:
-            return error_dialog(self.gui, _('Cannot edit book'),
-                    _('The book must be in the %s formats to edit.'
-                        '\n\nFirst convert the book to one of these formats.') % (_(' or ').join(SUPPORTED)),
-                    show=True)
+            if not fmts:
+                if not question_dialog(self.gui, _('No editable formats'),
+                    _('Do you want to create an empty EPUB file to edit?')):
+                    return
+                tweakable_fmts = {'EPUB'}
+                self.gui.iactions['Add Books'].add_empty_format_to_book(book_id, 'EPUB')
+                current_idx = self.gui.library_view.currentIndex()
+                if current_idx.isValid():
+                    self.gui.library_view.model().current_changed(current_idx, current_idx)
+            else:
+                return error_dialog(self.gui, _('Cannot edit book'), _(
+                    'The book must be in the %s formats to edit.'
+                    '\n\nFirst convert the book to one of these formats.'
+                ) % (_(' or ').join(SUPPORTED)), show=True)
         from calibre.gui2.tweak_book import tprefs
         tprefs.refresh()  # In case they were changed in a Tweak Book process
         if len(tweakable_fmts) > 1:

From 08e6d01928dc7199d65651666abcd243c174b0ad Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Sat, 22 Feb 2020 09:30:12 +0530
Subject: [PATCH 059/162] Slightly increase tap threshold

---
 src/pyj/read_book/touch.pyj | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/pyj/read_book/touch.pyj b/src/pyj/read_book/touch.pyj
index 58fab95b38..fe1b12821b 100644
--- a/src/pyj/read_book/touch.pyj
+++ b/src/pyj/read_book/touch.pyj
@@ -6,7 +6,7 @@ from read_book.globals import get_boss, ui_operations
 from read_book.viewport import scroll_viewport
 
 HOLD_THRESHOLD = 750  # milliseconds
-TAP_THRESHOLD = 7  # pixels
+TAP_THRESHOLD = 8  # pixels
 TAP_LINK_THRESHOLD = 5  # pixels
 PINCH_THRESHOLD = 10  # pixels
 LONG_TAP_THRESHOLD = 500  # milliseconds

From 42d5db549d43162326225c10c5c2012c8d70c422 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Sat, 22 Feb 2020 10:05:57 +0530
Subject: [PATCH 060/162] Viewer: Fix number of columns not auto-adjusting when
 font size is changed until a subsequent chapter change occurs

---
 src/calibre/gui2/viewer/web_view.py | 11 +++++++--
 src/pyj/read_book/iframe.pyj        | 35 +++++++++++++++++++++++------
 src/pyj/read_book/paged_mode.pyj    |  4 ++++
 src/pyj/read_book/view.pyj          |  3 +++
 src/pyj/viewer-main.pyj             |  7 ++++++
 5 files changed, 51 insertions(+), 9 deletions(-)

diff --git a/src/calibre/gui2/viewer/web_view.py b/src/calibre/gui2/viewer/web_view.py
index 30171d21db..e5830a4159 100644
--- a/src/calibre/gui2/viewer/web_view.py
+++ b/src/calibre/gui2/viewer/web_view.py
@@ -288,6 +288,7 @@ class ViewerBridge(Bridge):
     set_system_palette = to_js()
     show_search_result = to_js()
     prepare_for_close = to_js()
+    viewer_font_size_changed = to_js()
 
 
 def apply_font_settings(page_or_view):
@@ -309,6 +310,8 @@ def apply_font_settings(page_or_view):
     sf = fs.get('standard_font') or 'serif'
     sf = getattr(s, {'serif': 'SerifFont', 'sans': 'SansSerifFont', 'mono': 'FixedFont'}[sf])
     s.setFontFamily(s.StandardFont, s.fontFamily(sf))
+    old_minimum = s.fontSize(s.MinimumFontSize)
+    old_base = s.fontSize(s.DefaultFontSize)
     mfs = fs.get('minimum_font_size')
     if mfs is None:
         s.resetFontSize(s.MinimumFontSize)
@@ -318,6 +321,10 @@ def apply_font_settings(page_or_view):
     if bfs is not None:
         s.setFontSize(s.DefaultFontSize, bfs)
 
+    font_size_changed = old_minimum != s.fontSize(s.MinimumFontSize) or old_base != s.fontSize(s.DefaultFontSize)
+    if font_size_changed and hasattr(page_or_view, 'execute_when_ready'):
+        page_or_view.execute_when_ready('viewer_font_size_changed')
+
     return s
 
 
@@ -610,7 +617,7 @@ class WebView(RestartingWebEngineView):
     def set_session_data(self, key, val):
         if key == '*' and val is None:
             vprefs['session_data'] = {}
-            apply_font_settings(self._page)
+            apply_font_settings(self)
             self.paged_mode_changed.emit()
             self.standalone_misc_settings_changed.emit()
         elif key != '*':
@@ -618,7 +625,7 @@ class WebView(RestartingWebEngineView):
             sd[key] = val
             vprefs['session_data'] = sd
             if key in ('standalone_font_settings', 'base_font_size'):
-                apply_font_settings(self._page)
+                apply_font_settings(self)
             elif key == 'read_mode':
                 self.paged_mode_changed.emit()
             elif key == 'standalone_misc_settings':
diff --git a/src/pyj/read_book/iframe.pyj b/src/pyj/read_book/iframe.pyj
index 00facccc43..08b809f2e6 100644
--- a/src/pyj/read_book/iframe.pyj
+++ b/src/pyj/read_book/iframe.pyj
@@ -24,11 +24,11 @@ from read_book.mathjax import apply_mathjax
 from read_book.paged_mode import (
     anchor_funcs as paged_anchor_funcs,
     auto_scroll_action as paged_auto_scroll_action, calc_columns_per_screen,
-    current_cfi, handle_gesture as paged_handle_gesture,
-    handle_shortcut as paged_handle_shortcut, jump_to_cfi as paged_jump_to_cfi,
-    layout as paged_layout, onwheel as paged_onwheel,
-    prepare_for_resize as paged_prepare_for_resize, progress_frac,
-    reset_paged_mode_globals, resize_done as paged_resize_done,
+    will_columns_per_screen_change, current_cfi,
+    handle_gesture as paged_handle_gesture, handle_shortcut as paged_handle_shortcut,
+    jump_to_cfi as paged_jump_to_cfi, layout as paged_layout,
+    onwheel as paged_onwheel, prepare_for_resize as paged_prepare_for_resize,
+    progress_frac, reset_paged_mode_globals, resize_done as paged_resize_done,
     scroll_by_page as paged_scroll_by_page, scroll_to_elem,
     scroll_to_fraction as paged_scroll_to_fraction, snap_to_selection
 )
@@ -44,8 +44,9 @@ from read_book.shortcuts import (
     create_shortcut_map, keyevent_as_shortcut, shortcut_for_key_event
 )
 from read_book.toc import update_visible_toc_anchors
-from read_book.touch import (create_handlers as create_touch_handlers,
-                             reset_handlers as reset_touch_handlers)
+from read_book.touch import (
+    create_handlers as create_touch_handlers, reset_handlers as reset_touch_handlers
+)
 from read_book.viewport import scroll_viewport
 from utils import (
     apply_cloned_selection, clone_selection, debounce, html_escape, is_ios
@@ -99,6 +100,7 @@ class IframeBoss:
         handlers = {
             'change_color_scheme': self.change_color_scheme,
             'change_font_size': self.change_font_size,
+            'viewer_font_size_changed': self.viewer_font_size_changed,
             'change_scroll_speed': self.change_scroll_speed,
             'display': self.display,
             'find': self.find,
@@ -264,10 +266,29 @@ class IframeBoss:
         else:
             paged_scroll_by_page(backwards, data.all_pages_on_screen)
 
+
     def change_font_size(self, data):
         if data.base_font_size? and data.base_font_size != opts.base_font_size:
             opts.base_font_size = data.base_font_size
             apply_font_size()
+            if not runtime.is_standalone_viewer:
+                # in the standalone viewer this is a separate event as
+                # apply_font_size() is a no-op
+                self.relayout_on_font_size_change()
+
+    def viewer_font_size_changed(self, data):
+        opts.base_font_size = data.base_font_size
+        self.relayout_on_font_size_change()
+
+    def relayout_on_font_size_change(self):
+        if current_layout_mode() is not 'flow' and will_columns_per_screen_change():
+            self.do_layout(self.is_titlepage)
+            if self.last_cfi:
+                cfi = self.last_cfi[len('epubcfi(/'):-1].partition('/')[2]
+                if cfi:
+                    paged_jump_to_cfi('/' + cfi)
+            self.update_cfi()
+            self.update_toc_position()
 
     def change_scroll_speed(self, data):
         if data.lines_per_sec_auto?:
diff --git a/src/pyj/read_book/paged_mode.pyj b/src/pyj/read_book/paged_mode.pyj
index 7acfa0c223..5c6f2bbab4 100644
--- a/src/pyj/read_book/paged_mode.pyj
+++ b/src/pyj/read_book/paged_mode.pyj
@@ -152,6 +152,10 @@ def calc_columns_per_screen():
     return cps
 
 
+def will_columns_per_screen_change():
+    return calc_columns_per_screen() != cols_per_screen
+
+
 def layout(is_single_page, on_resize):
     nonlocal _in_paged_mode, col_width, col_and_gap, screen_height, gap, screen_width, is_full_screen_layout, cols_per_screen, number_of_cols
     body_style = window.getComputedStyle(document.body)
diff --git a/src/pyj/read_book/view.pyj b/src/pyj/read_book/view.pyj
index 4dc7d98133..087defc5ac 100644
--- a/src/pyj/read_book/view.pyj
+++ b/src/pyj/read_book/view.pyj
@@ -1066,6 +1066,9 @@ class View:
     def update_font_size(self):
         self.iframe_wrapper.send_message('change_font_size', base_font_size=get_session_data().get('base_font_size'))
 
+    def viewer_font_size_changed(self):
+        self.iframe_wrapper.send_message('viewer_font_size_changed', base_font_size=get_session_data().get('base_font_size'))
+
     def update_scroll_speed(self, amt):
         self.iframe_wrapper.send_message('change_scroll_speed', lines_per_sec_auto=change_scroll_speed(amt))
 
diff --git a/src/pyj/viewer-main.pyj b/src/pyj/viewer-main.pyj
index adab0bef94..bab68c0464 100644
--- a/src/pyj/viewer-main.pyj
+++ b/src/pyj/viewer-main.pyj
@@ -300,6 +300,13 @@ def prepare_for_close():
     else:
         ui_operations.close_prep_finished(None)
 
+
+@from_python
+def viewer_font_size_changed():
+    if view:
+        view.viewer_font_size_changed()
+
+
 def onerror(msg, script_url, line_number, column_number, error_object):
     if not error_object:
         # cross domain error

From 9191034495954a1d7f4e4ecd966e6acb42aff588 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Sat, 22 Feb 2020 11:04:12 +0530
Subject: [PATCH 061/162] Also revert touch-actions: none in standalone viewer

Breaks using touch to scroll the preferences screen for example
---
 src/pyj/viewer-main.pyj | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/src/pyj/viewer-main.pyj b/src/pyj/viewer-main.pyj
index bab68c0464..bc916008cf 100644
--- a/src/pyj/viewer-main.pyj
+++ b/src/pyj/viewer-main.pyj
@@ -425,9 +425,7 @@ if window is window.top:
     window.onerror = onerror
     create_modal_container()
     document.head.appendChild(E.style(get_widget_css()))
-    # See https://github.com/kovidgoyal/calibre/pull/1101
-    # for why we need touch-action: none
-    set_css(document.body, background_color=get_color('window-background'), color=get_color('window-foreground'), touch_action='none')
+    set_css(document.body, background_color=get_color('window-background'), color=get_color('window-foreground'))
     setTimeout(def():
             window.onpopstate = on_pop_state
         , 0)  # We do this after event loop ticks over to avoid catching popstate events that some browsers send on page load

From c2d492fdcbf18773912ba5e0b5945753458aa950 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Sat, 22 Feb 2020 15:56:53 +0530
Subject: [PATCH 062/162] String changes

---
 manual/catalogs.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/manual/catalogs.rst b/manual/catalogs.rst
index 3ef37efa7f..d356bd8cfa 100644
--- a/manual/catalogs.rst
+++ b/manual/catalogs.rst
@@ -19,7 +19,7 @@ If you want only *some* of your library cataloged, you have two options:
     * Create a multiple selection of the books you want cataloged. With more than one book selected in calibre's main window, only the selected books will be cataloged.
     * Use the Search field or the Tag browser to filter the displayed books. Only the displayed books will be cataloged.
 
-To begin catalog generation, select the menu item :guilabel:`Convert books > Create a catalog of the books in your calibre library`. You may also add a :guilabel:`Create Catalog` button to a toolbar in :guilabel:`Preferences > Interface > Toolbars & menus` for easier access to the Generate catalog dialog.
+To begin catalog generation, select the menu item :guilabel:`Convert books > Create a catalog of the books in your calibre library`. You may also add a :guilabel:`Create catalog` button to a toolbar in :guilabel:`Preferences > Interface > Toolbars & menus` for easier access to the Generate catalog dialog.
 
 .. image:: images/catalog_options.png
     :alt: Catalog options

From 0e9b59a6e2d05bdc2f03e4f6b17239c373e809d3 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Sat, 22 Feb 2020 16:01:25 +0530
Subject: [PATCH 063/162] version 4.11.2

---
 Changelog.yaml           | 4 +++-
 src/calibre/constants.py | 2 +-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/Changelog.yaml b/Changelog.yaml
index 854349b4c3..7288f5d2d5 100644
--- a/Changelog.yaml
+++ b/Changelog.yaml
@@ -20,7 +20,7 @@
 #   new recipes:
 #     - title:
 
-- version: 4.11.1
+- version: 4.11.2
   date: 2020-02-21
 
   new features:
@@ -39,6 +39,8 @@
       tickets: [1862432]
 
   bug fixes:
+    - title: "4.11.2 fixes a couple of regressions that broke the New bookmark button in the viewer and scrolling in the content server library view. Also fixes calculation of default column widths in viewer not changing when font size is changed."
+
     - title: "Viewer: Fix a regression that broke detection of pop-up footnotes using EPUB 3 markup"
 
     - title: "Viewer: Fix current reading position not preserved when changing preferences and auto scroll is active."
diff --git a/src/calibre/constants.py b/src/calibre/constants.py
index 085d2fafe5..067a473a94 100644
--- a/src/calibre/constants.py
+++ b/src/calibre/constants.py
@@ -6,7 +6,7 @@ from polyglot.builtins import map, unicode_type, environ_item, hasenv, getenv, a
 import sys, locale, codecs, os, importlib, collections
 
 __appname__   = 'calibre'
-numeric_version = (4, 11, 1)
+numeric_version = (4, 11, 2)
 __version__   = '.'.join(map(unicode_type, numeric_version))
 git_version   = None
 __author__    = "Kovid Goyal "

From 9112f967a0334222790d7e8a3e9faa556e266c14 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Sun, 23 Feb 2020 07:32:02 +0530
Subject: [PATCH 064/162] py3 compat

---
 src/calibre/ebooks/mobi/debug/headers.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/calibre/ebooks/mobi/debug/headers.py b/src/calibre/ebooks/mobi/debug/headers.py
index 308d7c262b..ef4fe8f6eb 100644
--- a/src/calibre/ebooks/mobi/debug/headers.py
+++ b/src/calibre/ebooks/mobi/debug/headers.py
@@ -6,14 +6,14 @@ __license__   = 'GPL v3'
 __copyright__ = '2012, Kovid Goyal '
 __docformat__ = 'restructuredtext en'
 
-import struct, datetime, os, numbers
+import struct, datetime, os, numbers, binascii
 
 from calibre.utils.date import utc_tz
 from calibre.ebooks.mobi.reader.headers import NULL_INDEX
 from calibre.ebooks.mobi.langcodes import main_language, sub_language
 from calibre.ebooks.mobi.debug import format_bytes
 from calibre.ebooks.mobi.utils import get_trailing_data
-from polyglot.builtins import as_bytes, iteritems, range, unicode_type
+from polyglot.builtins import iteritems, range, unicode_type
 
 # PalmDB {{{
 
@@ -210,7 +210,7 @@ class EXTHRecord(object):
             else:
                 self.data, = struct.unpack(b'>L', self.data)
         elif self.type in {209, 300}:
-            self.data = as_bytes(self.data.encode('hex'))
+            self.data = binascii.hexlify(self.data)
 
     def __str__(self):
         return '%s (%d): %r'%(self.name, self.type, self.data)

From 18071ad9f5f896c934767850e7eabce39816059f Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Sun, 23 Feb 2020 07:37:38 +0530
Subject: [PATCH 065/162] String changes

---
 src/calibre/gui2/wizard/library.ui | 2 +-
 src/pyj/read_book/overlay.pyj      | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/calibre/gui2/wizard/library.ui b/src/calibre/gui2/wizard/library.ui
index 74fccf0dd2..276849f073 100644
--- a/src/calibre/gui2/wizard/library.ui
+++ b/src/calibre/gui2/wizard/library.ui
@@ -53,7 +53,7 @@
    
     
      
-      If a calibre library already exists at the new location, calibre will use it automatically.
+      If a calibre library already exists at the newly selected location, calibre will use it automatically.
      
      
       true
diff --git a/src/pyj/read_book/overlay.pyj b/src/pyj/read_book/overlay.pyj
index 8d9000715b..7d36e29f78 100644
--- a/src/pyj/read_book/overlay.pyj
+++ b/src/pyj/read_book/overlay.pyj
@@ -247,7 +247,7 @@ class MainOverlay:  # {{{
         sync_action = ac(_('Sync'), _('Get last read position and annotations from the server'), self.overlay.sync_book, 'cloud-download')
         delete_action = ac(_('Delete'), _('Delete this book from the device'), self.overlay.delete_book, 'trash')
         reload_action = ac(_('Reload'), _('Reload this book from the {}').format( _('computer') if runtime.is_standalone_viewer else _('server')), self.overlay.reload_book, 'refresh')
-        home_action = ac(_('Home'), _('Return to list of books'), def(): home();, 'home')
+        home_action = ac(_('Home'), _('Return to the home page'), def(): home();, 'home')
         back_action = ac(_('Back'), None, self.back, 'arrow-left')
         forward_action = ac(_('Forward'), None, self.forward, 'arrow-right')
         if runtime.is_standalone_viewer:

From 6290903cbfe6d46eb20eb8538936ef6fe0d9bcc1 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Sun, 23 Feb 2020 08:12:49 +0530
Subject: [PATCH 066/162] Fix #1864311 [[Enhancement] Make the symbols better
 aligned with the search field on the Content
 server](https://bugs.launchpad.net/calibre/+bug/1864311)

---
 src/pyj/read_book/search.pyj | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/pyj/read_book/search.pyj b/src/pyj/read_book/search.pyj
index a3ce46f81d..94f5ac2d04 100644
--- a/src/pyj/read_book/search.pyj
+++ b/src/pyj/read_book/search.pyj
@@ -18,7 +18,7 @@ add_extra_css(def():
     sel = '.' + CLASS_NAME
     style = build_rule(sel, text_align='right', user_select='none')
     sel += ' > div '
-    style += build_rule(sel, display='inline-flex', pointer_events='auto', background_color=get_color('window-background'), padding='1ex')
+    style += build_rule(sel, display='inline-flex', align_items='center', pointer_events='auto', background_color=get_color('window-background'), padding='1ex')
     return style
 )
 

From a41fcfd3db7e20f596ba34b018914e704025ab6e Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Sun, 23 Feb 2020 08:40:11 +0530
Subject: [PATCH 067/162] Fix #1864340 [calibredb list output to file is
 broken](https://bugs.launchpad.net/calibre/+bug/1864340)

py3 apparently cant print newlines correctly when stdout is redirected
to a file. Sigh.
---
 src/calibre/db/cli/cmd_list.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/calibre/db/cli/cmd_list.py b/src/calibre/db/cli/cmd_list.py
index 041c48fb6f..a1afc96000 100644
--- a/src/calibre/db/cli/cmd_list.py
+++ b/src/calibre/db/cli/cmd_list.py
@@ -214,12 +214,12 @@ def do_list(
         lines = max(map(len, text))
         for l in range(lines):
             for i, field in enumerate(text):
-                ft = text[i][l] if l < len(text[i]) else u''
+                ft = text[i][l] if l < len(text[i]) else ''
                 stdout.write(ft.encode('utf-8'))
                 if i < len(text) - 1:
-                    filler = (u'%*s' % (widths[i] - str_width(ft) - 1, u''))
+                    filler = ('%*s' % (widths[i] - str_width(ft) - 1, ''))
                     stdout.write((filler + separator).encode('utf-8'))
-            print()
+            stdout.write(b'\n')
 
 
 def option_parser(get_parser, args):

From d3eb47e86f88e79dd2a1c78fe890f04a9594fbd2 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Sun, 23 Feb 2020 08:44:17 +0530
Subject: [PATCH 068/162] Use platform dependant line separator for calibredb
 list

---
 src/calibre/db/cli/cmd_list.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/calibre/db/cli/cmd_list.py b/src/calibre/db/cli/cmd_list.py
index a1afc96000..edd5712555 100644
--- a/src/calibre/db/cli/cmd_list.py
+++ b/src/calibre/db/cli/cmd_list.py
@@ -13,7 +13,7 @@ from calibre import prints
 from calibre.db.cli.utils import str_width
 from calibre.ebooks.metadata import authors_to_string
 from calibre.utils.date import isoformat
-from polyglot.builtins import iteritems, unicode_type, map
+from polyglot.builtins import as_bytes, iteritems, map, unicode_type
 
 readonly = True
 version = 0  # change this if you change signature of implementation()
@@ -204,6 +204,7 @@ def do_list(
     with ColoredStream(sys.stdout, fg='green'):
         prints(''.join(titles))
     stdout = getattr(sys.stdout, 'buffer', sys.stdout)
+    linesep = as_bytes(os.linesep)
 
     wrappers = [TextWrapper(x - 1).wrap if x > 1 else lambda y: y for x in widths]
 
@@ -219,7 +220,7 @@ def do_list(
                 if i < len(text) - 1:
                     filler = ('%*s' % (widths[i] - str_width(ft) - 1, ''))
                     stdout.write((filler + separator).encode('utf-8'))
-            stdout.write(b'\n')
+            stdout.write(linesep)
 
 
 def option_parser(get_parser, args):

From 8439a6b09c682b500191e78c8d0f7144dc6e3825 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Sun, 23 Feb 2020 11:01:28 +0530
Subject: [PATCH 069/162] ...

---
 src/pyj/read_book/toc.pyj | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/pyj/read_book/toc.pyj b/src/pyj/read_book/toc.pyj
index e1981b20d5..3b99d4445b 100644
--- a/src/pyj/read_book/toc.pyj
+++ b/src/pyj/read_book/toc.pyj
@@ -159,7 +159,7 @@ def create_toc_panel(book, container, onclick):
     t = _('Search Table of Contents')
     search_bar = create_search_bar(do_search.bind(toc_panel_id), 'search-book-toc', button=search_button, placeholder=t)
     set_css(search_bar, flex_grow='10', margin_right='1em')
-    container.appendChild(E.div(style='margin: 1ex 1em; display: flex;', search_bar, search_button))
+    container.appendChild(E.div(style='margin: 1ex 1em; display: flex; align-items: center', search_bar, search_button))
 
 
 def current_toc_anchor_map(tam, anchor_funcs):

From df0dfd04deb0e11d01b96bed74114f56153b335f Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Sun, 23 Feb 2020 11:42:32 +0530
Subject: [PATCH 070/162] Fix #1864341 [Python error  when trying to select
 several books](https://bugs.launchpad.net/calibre/+bug/1864341)

---
 src/calibre/gui2/library/alternate_views.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/src/calibre/gui2/library/alternate_views.py b/src/calibre/gui2/library/alternate_views.py
index bf78e5791e..9b16c65f93 100644
--- a/src/calibre/gui2/library/alternate_views.py
+++ b/src/calibre/gui2/library/alternate_views.py
@@ -1049,12 +1049,15 @@ class GridView(QListView):
     def number_of_columns(self):
         # Number of columns currently visible in the grid
         if self._ncols is None:
+            dpr = self.device_pixel_ratio
+            width = int(dpr * self.delegate.cover_size.width())
+            height = int(dpr * self.delegate.cover_size.height())
             step = max(10, self.spacing())
-            for y in range(step, 500, step):
-                for x in range(step, 500, step):
+            for y in range(step, 2 * height, step):
+                for x in range(step, 2 * width, step):
                     i = self.indexAt(QPoint(x, y))
                     if i.isValid():
-                        for x in range(self.viewport().width() - step, self.viewport().width() - 300, -step):
+                        for x in range(self.viewport().width() - step, self.viewport().width() - width, -step):
                             j = self.indexAt(QPoint(x, y))
                             if j.isValid():
                                 self._ncols = j.row() - i.row() + 1
@@ -1070,7 +1073,8 @@ class GridView(QListView):
             if not ci.isValid():
                 return
             c = ci.row()
-            delta = {Qt.Key_Left: -1, Qt.Key_Right: 1, Qt.Key_Up: -self.number_of_columns(), Qt.Key_Down: self.number_of_columns()}[k]
+            ncols = self.number_of_columns() or 1
+            delta = {Qt.Key_Left: -1, Qt.Key_Right: 1, Qt.Key_Up: -ncols, Qt.Key_Down: ncols}[k]
             n = max(0, min(c + delta, self.model().rowCount(None) - 1))
             if n == c:
                 return

From 80bd26a25411a63427fbce8011aef749706dd45a Mon Sep 17 00:00:00 2001
From: James Cridland 
Date: Sun, 23 Feb 2020 19:08:30 +1000
Subject: [PATCH 071/162] Update guardian.png

Updated icon to reflect new logo, first introduced in January 2018
---
 recipes/icons/guardian.png | Bin 222 -> 479 bytes
 1 file changed, 0 insertions(+), 0 deletions(-)

diff --git a/recipes/icons/guardian.png b/recipes/icons/guardian.png
index 86b42fd4b6b3d1176992e640e01434f817fc958e..2904b71dfe8a292ae870d6c7f88cd851ea7433bc 100644
GIT binary patch
delta 453
zcmcb|c%ONKO8vS3pAc6DFkn_sWYI_jk!tZw$`L>|lX@bP8dMN0o4~4>#G;YJ0wmSq
zS=3`$fQmJf{{R2aq@KW|pT@48B5Iy<{QMm$>wJF0jD5$iE#7d{Euq^ixbfrXZ)#^Qqx1?fT!AQ@q0S
zms9I*BgOIhOVecYcd(eIn(07!+5Yu6{1-oD!M
Date: Sun, 23 Feb 2020 19:15:16 +1000
Subject: [PATCH 072/162] 1843: added an icon and description.

The description and icon is from the 1843 website; the language change is to reflect its publication in London, UK.
---
 recipes/1843.recipe    |   3 ++-
 recipes/icons/1843.png | Bin 0 -> 288 bytes
 2 files changed, 2 insertions(+), 1 deletion(-)
 create mode 100644 recipes/icons/1843.png

diff --git a/recipes/1843.recipe b/recipes/1843.recipe
index 8805c35415..188f5ab4c9 100644
--- a/recipes/1843.recipe
+++ b/recipes/1843.recipe
@@ -15,7 +15,8 @@ def classes(classes):
 class E1843(BasicNewsRecipe):
     title = '1843'
     __author__ = 'Kovid Goyal'
-    language = 'en'
+    description = 'The ideas, culture and lifestyle magazine from The Economist'
+    language = 'en_GB'
     no_stylesheets = True
     remove_javascript = True
     oldest_article = 365
diff --git a/recipes/icons/1843.png b/recipes/icons/1843.png
new file mode 100644
index 0000000000000000000000000000000000000000..bc32d1e180fa5d816c19320628b1ddeab51ff21b
GIT binary patch
literal 288
zcmV+*0pI?KP){(;$P+RFnQRXf><|Z@nd4lX+X6j8?<|;SkA1&k;
zB;yhx;|?A5r>ydlnec&$?s0kSVr%M3Rp>fK;Q7G**)#M1XO2ZK<69a8qEv{U-i0wXf{Zxa!PWreJYuW*xJ^ubErkGX
ze3jZQX>zD8ayD+lR~P}i(}BP&?ag<%X3ur*pvgHhp@2(-Q}xfq0C;EZv2d2w_SB*g
mA63uz*U$@(dF)T>i~Im?Dg?M}%5oe40000
Date: Sun, 23 Feb 2020 19:38:16 +0530
Subject: [PATCH 073/162] Viewer: Add a keyboard shortcut (Ctrl+w) to toggle
 the scrollbar. Fixes #1864356 [[Enhancement] Hide scrollbar by right clicking
 on it](https://bugs.launchpad.net/calibre/+bug/1864356)

---
 src/calibre/gui2/viewer/ui.py   |  2 ++
 src/pyj/read_book/shortcuts.pyj |  6 ++++++
 src/pyj/read_book/view.pyj      | 11 +++++++++++
 3 files changed, 19 insertions(+)

diff --git a/src/calibre/gui2/viewer/ui.py b/src/calibre/gui2/viewer/ui.py
index e9879a736a..14198bf5a7 100644
--- a/src/calibre/gui2/viewer/ui.py
+++ b/src/calibre/gui2/viewer/ui.py
@@ -210,6 +210,8 @@ class EbookViewer(MainWindow):
         m.addSeparator()
         a(_('Start of current file'), 'start_of_file')
         a(_('End of current file'), 'end_of_file')
+        m.addSeparator()
+        a(_('Hide this scrollbar'), 'toggle_scrollbar')
 
         q = m.exec_(QCursor.pos())
         if not q:
diff --git a/src/pyj/read_book/shortcuts.pyj b/src/pyj/read_book/shortcuts.pyj
index 7345b359f9..8c69a0aece 100644
--- a/src/pyj/read_book/shortcuts.pyj
+++ b/src/pyj/read_book/shortcuts.pyj
@@ -228,6 +228,12 @@ def shortcuts_definition():
         _('Toggle between Paged mode and Flow mode for text layout')
     ),
 
+    'toggle_scrollbar': desc(
+        'Ctrl+w',
+        'ui',
+        _('Toggle the scrollbar')
+    ),
+
     'toggle_reference_mode': desc(
         'Ctrl+x',
         'ui',
diff --git a/src/pyj/read_book/view.pyj b/src/pyj/read_book/view.pyj
index 087defc5ac..01918d668e 100644
--- a/src/pyj/read_book/view.pyj
+++ b/src/pyj/read_book/view.pyj
@@ -269,6 +269,15 @@ class View:
     def reference_mode_overlay(self):
         return document.getElementById('reference-mode-overlay')
 
+    def set_scrollbar_visibility(self, visible):
+        sd = get_session_data()
+        sd.set('book_scrollbar', bool(visible))
+        self.book_scrollbar.apply_visibility()
+
+    def toggle_scrollbar(self):
+        sd = get_session_data()
+        self.set_scrollbar_visibility(not sd.get('book_scrollbar'))
+
     def on_lookup_word(self, data):
         if runtime.is_standalone_viewer:
             ui_operations.selection_changed(data.word)
@@ -388,6 +397,8 @@ class View:
             self.toggle_paged_mode()
         elif data.name is 'toggle_toolbar':
             self.toggle_toolbar()
+        elif data.name is 'toggle_scrollbar':
+            self.toggle_scrollbar()
         elif data.name is 'quit':
             ui_operations.quit()
         elif data.name is 'start_search':

From 4b00679b2eb99d013c45e0d9983515ff4002f9b0 Mon Sep 17 00:00:00 2001
From: James Cridland 
Date: Mon, 24 Feb 2020 10:51:40 +1000
Subject: [PATCH 074/162] Update ABC News Australia

Significant update to restore this recipe to work. Replaced logo to one being used from 2018. Rebuilt the feed list.
---
 recipes/abc_au.recipe    |  61 +++++++++++++++++++++------------------
 recipes/icons/abc_au.png | Bin 332 -> 717 bytes
 2 files changed, 33 insertions(+), 28 deletions(-)

diff --git a/recipes/abc_au.recipe b/recipes/abc_au.recipe
index ac21e6d730..c737371820 100644
--- a/recipes/abc_au.recipe
+++ b/recipes/abc_au.recipe
@@ -6,50 +6,55 @@ abc.net.au/news
 import re
 from calibre.web.feeds.recipes import BasicNewsRecipe
 
-
 class ABCNews(BasicNewsRecipe):
     title = 'ABC News'
-    __author__ = 'Pat Stapleton, Dean Cording'
-    description = 'News from Australia'
-    masthead_url = 'http://www.abc.net.au/news/assets/v5/images/common/logo-news.png'
-    cover_url = 'http://www.abc.net.au/news/assets/v5/images/common/logo-news.png'
-
+    __author__ = 'Pat Stapleton, Dean Cording, James Cridland'
+    description = 'From the Australian Broadcasting Corporation. The ABC is owned and funded by the Australian Government, but is editorially independent.'
+    masthead_url = 'https://www.abc.net.au/cm/lb/8212706/data/news-logo-2017---desktop-print-data.png'
+    cover_url = 'https://www.abc.net.au/news/linkableblob/8413676/data/abc-news-og-data.jpg'
+    cover_margins = (0,20,'#000000')
     oldest_article = 2
-    max_articles_per_feed = 100
-    no_stylesheets = False
+    handle_gzip = True
+    no_stylesheets = True
     use_embedded_content = False
+    scale_news_images_to_device = True
     encoding = 'utf8'
     publisher = 'ABC News'
-    category = 'News, Australia, World'
+    category = 'Australia,News'
     language = 'en_AU'
-    publication_type = 'newsportal'
-#    preprocess_regexps     = [(re.compile(r'', re.DOTALL), lambda m: '')]
-# Remove annoying map links (inline-caption class is also used for some
-# image captions! hence regex to match maps.google)
+    publication_type = 'newspaper'
+    extra_css = '.byline{font-size:smaller;margin-bottom:10px;}.inline-caption{display:block;font-size:smaller;text-decoration: none;}'
     preprocess_regexps = [(re.compile(
-        r';`08lmz(&GyJan{`T|U9o(h%VjqMaUMtGa
zPV{wuK6_HXEBodpHC9sRP8^=moRJ!N<
z{q3m_wMG&xZpybOiwGu*&nUkoe(%liw&mtRsT;$EXB`!rvvieEv!mWQk2u-lS*MT7
zNcvuOPd;pHwCpO|q?Ul6Y
zz|Un)_fCIxZ^@Ht4y>7=Xnku>O^U}q)=a&Nw|I5myj{?I(b(1g^Jdwo-3G6G=NWX(
zl;h1hfA*A}{f<^$UdA(1_a;VM%T`?Qj>*$geM9uM4SDwAdRaPgg&ebxsLQ0GOlraR2}S

literal 332
zcmV-S0ki&zP)
z!AcVV00!XyZF&zoMJLaB6*`nR(L>;fE|F0gmDL92XiAs^Bb-VIQewL1R;E}Xn&dJM
zbrd1KYQBQsm(ww)B^_@qNXMR*6)kPaxG`($l67}{(Ujzqf_{6xnKGc)j9-=%
z+^}lfS9xK=B}Kmtd*rFd+Flzmq~g1(JXA7j)CViBO16zTP&I4PXCrb~Nm)}-!-D+t
z%sX|v#x)GdZDr5x8?fYugc-xu9h&mSHJMOSGwqf^Eq@$(>89&iYF6BnhJtDH>dv}s
z$9wxKx?J(mfvOzo)1&E$Ip=koHEB`V8G~NgkYw38y*4FH7xgGgj+I^TKu#>%l8!BF
e($Tgmr{XW9*$)Bdt8q^N0000
Date: Mon, 24 Feb 2020 11:30:54 +1000
Subject: [PATCH 075/162] Updated recipe for The Courier-Mail

Added new RSS feeds, and cover image/masthead. Added a new icon.
---
 recipes/courier_mail.recipe    |  35 +++++++++++++++++----------------
 recipes/icons/courier_mail.png | Bin 512 -> 835 bytes
 2 files changed, 18 insertions(+), 17 deletions(-)

diff --git a/recipes/courier_mail.recipe b/recipes/courier_mail.recipe
index b6a6d32c39..ae423fd1fc 100644
--- a/recipes/courier_mail.recipe
+++ b/recipes/courier_mail.recipe
@@ -1,30 +1,31 @@
 from calibre.web.feeds.news import BasicNewsRecipe
-
+import datetime
 
 class Politics(BasicNewsRecipe):
-    title = u'Courier Mail'
+    title = u'The Courier-Mail'
+    description = 'Breaking news headlines for Brisbane and Queensland, Australia. The Courier-Mail is owned by News Corp Australia.'
     language = 'en_AU'
-    __author__ = 'Krittika Goyal'
+    __author__ = 'Krittika Goyal, James Cridland'
     oldest_article = 3  # days
     max_articles_per_feed = 20
     use_embedded_content = False
 
+    d = datetime.datetime.today()
+    cover_url='http://mfeeds.news.com.au/smedia/NCCOURIER/NCCM_1_' + d.strftime('%Y_%m_%d') + '_thumb_big.jpg'
+    masthead_url='https://couriermail.digitaleditions.com.au/images/couriermail-logo.jpg'
+
     no_stylesheets = True
     auto_cleanup = True
+    handle_gzip = True
 
     feeds = [
-        ('Top Stories',
-         'http://feeds.news.com.au/public/rss/2.0/bcm_top_stories_257.xml'),
-        ('Breaking News',
-         'http://feeds.news.com.au/public/rss/2.0/bcm_breaking_news_67.xml'),
-        ('Queensland News',
-         'http://feeds.news.com.au/public/rss/2.0/bcm_queensland_news_70.xml'),
-        ('Technology News',
-         'http://feeds.news.com.au/public/rss/2.0/bcm_technology_news_66.xml'),
-        ('Entertainment News',
-         'http://feeds.news.com.au/public/rss/2.0/bcm_entertainment_news_256.xml'),
-        ('Business News',
-         'http://feeds.news.com.au/public/rss/2.0/bcm_business_news_64.xml'),
-        ('Sport News',
-         'http://feeds.news.com.au/public/rss/2.0/bcm_sports_news_65.xml'),
+        ('Top Stories', 'http://www.couriermail.com.au/rss'),
+        ('Breaking', 'https://www.couriermail.com.au/news/breaking-news/rss'),
+        ('Queensland', 'https://www.couriermail.com.au/news/queensland/rss'),
+        ('Technology', 'https://www.couriermail.com.au/technology/rss'),
+        ('Entertainment', 'https://www.couriermail.com.au/entertainment/rss'),
+        ('Finance','https://www.couriermail.com.au/business/rss'),
+        ('Sport', 'https://www.couriermail.com.au/sport/rss'),
     ]
+
+# This isn't perfect, but works rather better than it once did. To do - remove links to subscription content.
\ No newline at end of file
diff --git a/recipes/icons/courier_mail.png b/recipes/icons/courier_mail.png
index 9ccd38d2f3734514241eaf07f05cec3aabaca18f..6478f64f4962a372ff71cc8d40f6fc353c93666f 100644
GIT binary patch
delta 824
zcmV-81IPS;1j7c97=H)?0002|8116~00Rn1L_t(|0b+P3qe@c*Xs;ZgsVxoV+2LYIl
z7N4ZdNK$8Z#aM|+qEs01W~0U@iPMl288PCnj2e$5Oo^3D_?KO1;YoxaeSU4#(_gN&x
zn5N@6*3)O|qJO2g8m>&73K%w!Wa+exTj(Ld{zHV)m0%DR_08M(f+f_s3mjdCXkNbRx{d+h
zKDk3TM>pJ^eFp)c@7a%cW!fyEWChQ^?ma$`^hCL9uYZ^SMwgnlb}S2`wq`xn@cE0`
zFF6m8>J0!~tgTq>?d;N%iB&Y@M)=zW<;yesUm30UYgGfStJu^To>gEa=k9
z4_NO5T)IM1X7Z&eO3XNRf-#IkN01?WzG8BRuEAlW)%lA9ISW7!2EZVA4507YOOm8v
zF0$_2<#QEc0MEZd*(xA21~M9@U`sN%4
zD(M1c(ljo1lJKp9&tDv=X$=7mea5n}>b@<4ab&}?~w!$+H!E(Zq;CGk^0
zYpGQSXrL`$708eS1tivLfc~NH+QSzp?z4a~a1Uh6iP)m^&_;%zZ^3JS;w-$-8bmx%
zyOy)vE=l^>h{?2`R@7s|Ns96vF;fnC{pR1^8RaqJSb?E7lRML(t}+j73@A#LN3fl#)w2C~$T@C07g
zJ?b=vUwHN9h<{*@MpFjJ3<_V0=n9gSK!Fz4i3sE>#7nwlAW2i_RD_CY8UrONGi^MG
zh;TDOQy@bY)2!#H(AtuhGu>gC@_kHT!1F+yWS*8sh
z2{-wxgM5L?(g80G6``}tLz=9(1(pruOxSbu667SyI%^;1OIfJUc~#OZVuRcMJ
zZcKESmniV#Ja|QlI_mg04EgzNkd>e>b-a^Q1}O_AO~B$-%e7FM-0FeFYb%pg40e?r
owAV4XL4EuUY9q?cu>buI-xa^~D(={
Date: Mon, 24 Feb 2020 11:36:37 +1000
Subject: [PATCH 076/162] Delete business_spectator.recipe

This publication is no longer available. It's now part of The Australian, which has a separate recipe here.
---
 recipes/business_spectator.recipe | 40 -------------------------------
 1 file changed, 40 deletions(-)
 delete mode 100644 recipes/business_spectator.recipe

diff --git a/recipes/business_spectator.recipe b/recipes/business_spectator.recipe
deleted file mode 100644
index 31b03570f6..0000000000
--- a/recipes/business_spectator.recipe
+++ /dev/null
@@ -1,40 +0,0 @@
-__license__ = 'GPL v3'
-__copyright__ = '2010, Dean Cording'
-'''
-abc.net.au/news
-'''
-import re
-from calibre.web.feeds.recipes import BasicNewsRecipe
-
-
-class BusinessSpectator(BasicNewsRecipe):
-    title = 'Business Spectator'
-    __author__ = 'Dean Cording'
-    description = 'Australian Business News & commentary delivered the way you want it.'
-    masthead_url = 'http://www.businessspectator.com.au/bs.nsf/logo-business-spectator.gif'
-    cover_url = masthead_url
-
-    oldest_article = 2
-    max_articles_per_feed = 100
-    no_stylesheets = True
-    auto_cleanup = True
-    use_embedded_content = False
-    encoding = 'utf8'
-    publisher = 'Business Spectator'
-    category = 'News, Australia, Business'
-    language = 'en_AU'
-    publication_type = 'newsportal'
-    preprocess_regexps = [(re.compile(r'', re.DOTALL), lambda m: '')]
-    conversion_options = {
-        'comments': description, 'tags': category, 'language': language, 'publisher': publisher, 'linearize_tables': False
-    }
-
-    feeds = [
-        ('Top Stories', 'http://www.businessspectator.com.au/top-stories.rss'),
-        ('Alan Kohler', 'http://www.businessspectator.com.au/bs.nsf/RSS?readform&type=spectators&cat=Alan%20Kohler'),
-        ('Robert Gottliebsen', 'http://www.businessspectator.com.au/bs.nsf/RSS?readform&type=spectators&cat=Robert%20Gottliebsen'),
-        ('Stephen Bartholomeusz',
-         'http://www.businessspectator.com.au/bs.nsf/RSS?readform&type=spectators&cat=Stephen%20Bartholomeusz'),
-        ('Daily Dossier', 'http://www.businessspectator.com.au/bs.nsf/RSS?readform&type=kgb&cat=dossier'),
-        ('Australia', 'http://www.businessspectator.com.au/bs.nsf/RSS?readform&type=region&cat=australia'),
-    ]

From 1088d699b789ac8ba7b5e629affdf24ba7683e70 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Mon, 24 Feb 2020 07:14:00 +0530
Subject: [PATCH 077/162] pep8

---
 recipes/abc_au.recipe       | 11 ++++++-----
 recipes/courier_mail.recipe |  3 ++-
 2 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/recipes/abc_au.recipe b/recipes/abc_au.recipe
index c737371820..eae6cee270 100644
--- a/recipes/abc_au.recipe
+++ b/recipes/abc_au.recipe
@@ -6,6 +6,7 @@ abc.net.au/news
 import re
 from calibre.web.feeds.recipes import BasicNewsRecipe
 
+
 class ABCNews(BasicNewsRecipe):
     title = 'ABC News'
     __author__ = 'Pat Stapleton, Dean Cording, James Cridland'
@@ -25,7 +26,7 @@ class ABCNews(BasicNewsRecipe):
     publication_type = 'newspaper'
     extra_css = '.byline{font-size:smaller;margin-bottom:10px;}.inline-caption{display:block;font-size:smaller;text-decoration: none;}'
     preprocess_regexps = [(re.compile(
-        r'
Date: Mon, 24 Feb 2020 07:37:11 +0530
Subject: [PATCH 078/162] Edit Book: Allow selecting the contents of a tag with
 Ctrl+Alt+t

---
 manual/edit.rst                                   |  3 ++-
 src/calibre/gui2/tweak_book/editor/smarts/html.py | 14 ++++++++++++++
 2 files changed, 16 insertions(+), 1 deletion(-)

diff --git a/manual/edit.rst b/manual/edit.rst
index 45bb93202b..3b0e71d079 100644
--- a/manual/edit.rst
+++ b/manual/edit.rst
@@ -802,7 +802,8 @@ The HTML editor has very sophisticated syntax highlighting. Features include:
     * The text inside bold, italic and heading tags is made bold/italic
     * As you move your cursor through the HTML, the matching HTML tags are
       highlighted, and you can jump to the opening or closing tag with the
-      keyboard shortcuts :kbd:`Ctrl+{` and :kbd:`Ctrl+}`
+      keyboard shortcuts :kbd:`Ctrl+{` and :kbd:`Ctrl+}`. Similarly, you
+      can select the contents of a tag with :kbd:`Ctrl+Alt+T`.
     * Invalid HTML is highlighted with a red underline
     * Spelling errors in the text inside HTML tags and attributes such as title
       are highlighted. The spell checking is language aware, based on the value
diff --git a/src/calibre/gui2/tweak_book/editor/smarts/html.py b/src/calibre/gui2/tweak_book/editor/smarts/html.py
index 91f031cd29..45c4af2915 100644
--- a/src/calibre/gui2/tweak_book/editor/smarts/html.py
+++ b/src/calibre/gui2/tweak_book/editor/smarts/html.py
@@ -352,6 +352,18 @@ class Smarts(NullSmarts):
         editor.setTextCursor(c)
         return True
 
+    def select_tag_contents(self, editor):
+        editor.highlighter.join()
+        start = self.last_matched_tag
+        end = self.last_matched_closing_tag
+        if start is None or end is None:
+            return False
+        c = editor.textCursor()
+        c.setPosition(start.start_block.position() + start.end_offset + 1)
+        c.setPosition(end.start_block.position() + end.start_offset, c.KeepAnchor)
+        editor.setTextCursor(c)
+        return True
+
     def remove_tag(self, editor):
         editor.highlighter.join()
         if not self.last_matched_closing_tag and not self.last_matched_tag:
@@ -662,6 +674,8 @@ class Smarts(NullSmarts):
             if int(mods & Qt.ControlModifier):
                 if self.jump_to_enclosing_tag(editor, key == Qt.Key_BraceLeft):
                     return True
+        if key == Qt.Key_T and int(ev.modifiers() & (Qt.ControlModifier | Qt.AltModifier)):
+            return self.select_tag_contents(editor)
 
         return False
 

From ffe4e65ef208686a333078b95a3fb77fe373c58e Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Mon, 24 Feb 2020 07:55:49 +0530
Subject: [PATCH 079/162] Better __str__ for search objects

---
 src/calibre/gui2/viewer/search.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/src/calibre/gui2/viewer/search.py b/src/calibre/gui2/viewer/search.py
index c8999e3b7a..b712372968 100644
--- a/src/calibre/gui2/viewer/search.py
+++ b/src/calibre/gui2/viewer/search.py
@@ -97,6 +97,11 @@ class Search(object):
             self._regex = regex.compile(expr, flags)
         return self._regex
 
+    def __str__(self):
+        from collections import namedtuple
+        s = ('text', 'mode', 'case_sensitive', 'backwards')
+        return str(namedtuple('Search', s)(*tuple(getattr(self, x) for x in s)))
+
 
 class SearchFinished(object):
 
@@ -143,6 +148,11 @@ class SearchResult(object):
     def is_or_is_after(self, result_from_js):
         return result_from_js['spine_idx'] == self.spine_idx and self.index >= result_from_js['index'] and result_from_js['text'] == self.text
 
+    def __str__(self):
+        from collections import namedtuple
+        s = self.__slots__[:-1]
+        return str(namedtuple('SearchResult', s)(*tuple(getattr(self, x) for x in s)))
+
 
 @lru_cache(maxsize=None)
 def searchable_text_for_name(name):

From 43094dc2e4c13b5e8ea2307b109e48b55f665f81 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Tue, 25 Feb 2020 07:27:56 +0530
Subject: [PATCH 080/162] Tweak settings limits

---
 src/pyj/read_book/prefs/font_size.pyj | 2 +-
 src/pyj/read_book/prefs/scrolling.pyj | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/pyj/read_book/prefs/font_size.pyj b/src/pyj/read_book/prefs/font_size.pyj
index 011e13b2f8..5970979aa7 100644
--- a/src/pyj/read_book/prefs/font_size.pyj
+++ b/src/pyj/read_book/prefs/font_size.pyj
@@ -41,7 +41,7 @@ def change_font_size_by(amt):
     sd = get_session_data()
     sz = sd.get('base_font_size')
     nsz = sz + amt
-    nsz = max(8, min(nsz, 40))
+    nsz = max(8, min(nsz, 80))
     change_font_size(nsz)
 
 def show_custom_size(ev):
diff --git a/src/pyj/read_book/prefs/scrolling.pyj b/src/pyj/read_book/prefs/scrolling.pyj
index 3ae6947910..d61c597ef9 100644
--- a/src/pyj/read_book/prefs/scrolling.pyj
+++ b/src/pyj/read_book/prefs/scrolling.pyj
@@ -13,7 +13,7 @@ from session import defaults
 CONTAINER = unique_id('standalone-scrolling-settings')
 
 # Scroll speeds in lines/sec
-MIN_SCROLL_SPEED_AUTO = 0.25
+MIN_SCROLL_SPEED_AUTO = 0.05
 MAX_SCROLL_SPEED_AUTO = 5
 
 MIN_SCROLL_AUTO_DELAY = -1

From 95f67127f128cd370ab32b2ff67d6dba077437c4 Mon Sep 17 00:00:00 2001
From: James Cridland 
Date: Tue, 25 Feb 2020 21:29:50 +1000
Subject: [PATCH 081/162] Create macrobusiness.png

A favicon for this recipe.
---
 recipes/icons/macrobusiness.png | Bin 0 -> 921 bytes
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 recipes/icons/macrobusiness.png

diff --git a/recipes/icons/macrobusiness.png b/recipes/icons/macrobusiness.png
new file mode 100644
index 0000000000000000000000000000000000000000..f5f03126381e2c6251bb5d66420e0ee882ac8148
GIT binary patch
literal 921
zcmV;K17`e*P)CD5i<`>~AYk|2_}6clRI2F7X(c6*H*N*&-iL5}N8P$ZCzBfh
zDpOMIh72(w2=3daU;na@P|*Ro<`x$1J0Q@MXn}91Nt39Bg^d8I_CrRP5-n}}46ta|
z5q_~xo{73Pbb8RCBgkvrqqqIgk){OUJZW0<066(>*z7uE7XG+QonD=jBMN{N9&S5e
zFhcycZ1Y~d&O|UE_}Iy~SFbS@^VaP$G&Ds4#Jzk4e~Y#q;P1C(yAVLg@sp&;D7!&J
z*|XWal~GkCI6Eaj-*v`pIJ?fAQ&E$m=%+WM@(+{yTRg%5LCbQbZ(+!4MTdCy~5Xtb*t`
zW*kiX044qbHog0?qmE+U3?`K(8i2*tr2dsNYtaVpl)wM@0iy3ez=5{x&^gc23L@^N
zhyWH_la}T*b^|N9`q}jA17+x`Gb{$Pes)G>rO$>e{2w#T5$40hEFQx7l-{
zcb~U_T2#byPEK(gH5N*rbsHEJ6;c5(;@`ewmzs8+a!gFQn&e-<&04fJZ`n3YNg1NF
z0CIG+-QZy`1synqVn8b+c`jYS&N5+gRYs-^04M-BMKIa+AA~ws;ONAyBGLEm!zu#+
z2JV4n=WgsS^z230xeIW!>D3o?LS_IoSg%!UU_}|X>C+#7P~&ijWd^`Vl~XWl*~W}$
zg*Of^jxUtr0qlkhGsaFHIod3sDm~q4Ji9b;ddqA)g)|HB*|5==4VcmxP@S3OwS1N1
z$g$CP@AGesuN}vXcN#Mx>Ff9A+XIajcJ{o($T5DKx8Xsi0RTq@OiErJIsrcb>JJ
Date: Tue, 25 Feb 2020 21:36:43 +1000
Subject: [PATCH 082/162] Added Herald Sun and The Age icons

Corrected Herald Sun icon. Added The Age icon.
---
 recipes/icons/melbourne_herald_sun.png | Bin 331 -> 1147 bytes
 recipes/icons/the_age.png              | Bin 0 -> 788 bytes
 2 files changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 recipes/icons/the_age.png

diff --git a/recipes/icons/melbourne_herald_sun.png b/recipes/icons/melbourne_herald_sun.png
index 8fe1a324659314dc4609f141f30f50ec34767ab4..c8e51bbc71de8bc907adb2b80d590f04298010e8 100644
GIT binary patch
delta 1137
zcmV-%1djX50{aM%8Gix*008_L?V|ty1VTwfK~#7F?3Dww^H>yxQ;%)iwr$(CpBwzP
zZQHhOvxrmMNd4y2&*R0ps}1(Wcd^e2azZ=qul`g7tHK#2Mk`TTp{5jdq`X!SKUs}V
zR;`s*X{41JNyTx=Z^w{dkN$N4(F&mE-vYGzAMoD*RLy@a>wo#tTL8f5tpZNzz@OJu
zIJepFln&fko%tr>d0qL_I&gV_0;Q#Oq*6_uMo_2Wq`XFlRB9#p(MtdY&fPoAyy$p%
zr$O9pjOBm346*sg!gmZd;j28`1#wSNhllla^!*uj1;)VWClmB1Gi@^t1~$@>byRrT
ze*Z^P&1Vt%-+va^9v#9Vzvck2YM3T4ChQB=7t4%8q5FoJ&*;Qo)yIIY^h6VxybjZ_S%s`EdquFW#5=;
zPoxS1{kX^l3RKB)?&`iq-kZ$p-8XopD67O-Im&!K+kY-5GORmsb6*~*(!me*=fT0e
z#ukH?U(9F#$PL84HrW~(UguT72@~uI1T*7ccE#(c$jLY*
zE4m1&(EKphnoMQBYymLE7EiSRxTYBZR&Z7)ZW|W`#T8DyGsOZff)z-sb&@LW1s%w}
zhUho5EPrhQzF+7<)GUZnjJ=|_p*i5}PTUSb%!{^Qa{O-pTa(P^cc7$IT1kcG%bAwa
zA-4<|yC?YWbo&Q09q-R@{Jhv@52YFcAP!C>n5hTKrhpJ!IQYYI|b+bcU&aLn1@H0k6L554Gk)6p#G;cL5I#H~FIJUl%zfwPY#TL7?5xi#cS@x+@e*KL$qnxs^csu>@$mMV;1p
z+}Tdv7dWpYgH%JH6@ONsv&T~D%#U*{g*bpY-a5!MWV`>kz3e%i
zBQbEB|Hl5tZ{|Cg_bE5=RwFYVU!BG*}t!VxOU4b{P0{bdp_i#p$`<%(@
zC`k>eQj@0>9{S$kQaq(Q5s_DuPXg6^ZBX=Tv&+_%;?;cy!L;M~C8
zTRRq!tMgpav;iKtK;F{d3%;eNLf-t7mDaj!cE)U#TI+{_A-)hSr-Myd{Q{`v&Hl*O
z0$?sYBIZ>Ch%|yc9@2yW4=|8-@-t5=_>1T(3aVJB0xT%=G{}E=og45U3vkQseC7ZE
N002ovPDHLkV1mT2k_G?(

diff --git a/recipes/icons/the_age.png b/recipes/icons/the_age.png
new file mode 100644
index 0000000000000000000000000000000000000000..7e80906f77bc744b2d5ae81b8e28186d24319d62
GIT binary patch
literal 788
zcmV+v1MB>WP)$sC6`%%!i`RQ>8m01N*pfnOxQys3#}oP!S~maI-@8lwnG_aLUVd$N^V
zvR5d_H-hIK#>M9o&hw8Hz}LouP+I~kx1_m?*3O#0+R`$Sx
z-n#Pu(JfpOO&ntJq0EKLHYCt=F8=fjm#)uUxE7B18rsDujIH9STI_4=K%{PB%Wu3o?6{KadYA#BqU&~T6Rz{E9>asJ}y6WL=T7-DvhmjoA(3(KH*#>SiX8I
zT=HA=PRr|z1hARvVQX(+*5ezBu?s}
z5!0Hq>4*9%cK|fwD4NiCX;A`4UurZ_6+==(7-$@
Date: Tue, 25 Feb 2020 21:47:17 +1000
Subject: [PATCH 083/162] Update list_apart.recipe

Corrected this publication, which is from the US, to be English (US). Corrected cover_url which was a 404 error.
---
 recipes/list_apart.recipe | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/recipes/list_apart.recipe b/recipes/list_apart.recipe
index f78e87879e..8105b04b71 100644
--- a/recipes/list_apart.recipe
+++ b/recipes/list_apart.recipe
@@ -12,12 +12,12 @@ class AListApart (BasicNewsRecipe):
     __copyright__ = '2012, Marc Busqué '
     title = u'A List Apart'
     description = u'A List Apart Magazine (ISSN: 1534-0295) explores the design, development, and meaning of web content, with a special focus on web standards and best practices. This recipe retrieve articles and columns.'  # noqa
-    language = 'en'
+    language = 'en_US'
     tags = 'web development, software'
     oldest_article = 120
     remove_empty_feeds = True
     encoding = 'utf8'
-    cover_url = u'http://alistapart.com/pix/alalogo.gif'
+    cover_url = u'https://alistapart.com/wp-content/uploads/2019/03/cropped-icon_navigation-laurel-512.jpg'
 
     def get_extra_css(self):
         if not self.extra_css:

From b2a0b89685738a1b0d2693639d3cfcd58bc42847 Mon Sep 17 00:00:00 2001
From: James Cridland 
Date: Tue, 25 Feb 2020 21:50:45 +1000
Subject: [PATCH 084/162] Update queueacmorg.recipe

Published in New York, this publication is in US English; amended.
---
 recipes/queueacmorg.recipe | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/recipes/queueacmorg.recipe b/recipes/queueacmorg.recipe
index e326b2500e..2e0fcf82f7 100644
--- a/recipes/queueacmorg.recipe
+++ b/recipes/queueacmorg.recipe
@@ -23,7 +23,7 @@ class QueueAcmOrg(BasicNewsRecipe):
     oldest_article        = 45
     max_articles_per_feed = 10
     auto_cleanup          = True
-    language              = 'en'
+    language              = 'en_US'
 
     # Feed are found here: http://queue.acm.org/rssfeeds.cfm
     feeds = [

From c7a59fc92149146d11172ea33ad04f9daefbe9f5 Mon Sep 17 00:00:00 2001
From: James Cridland 
Date: Tue, 25 Feb 2020 21:55:42 +1000
Subject: [PATCH 085/162] Removed AdsoftheWorld

This recipe only returns blank pages. Its RSS feed returns mainly video ads and collections, which are unparsable.
---
 recipes/ads_of_the_world.recipe    |  26 --------------------------
 recipes/icons/ads_of_the_world.png | Bin 1321 -> 0 bytes
 2 files changed, 26 deletions(-)
 delete mode 100644 recipes/ads_of_the_world.recipe
 delete mode 100644 recipes/icons/ads_of_the_world.png

diff --git a/recipes/ads_of_the_world.recipe b/recipes/ads_of_the_world.recipe
deleted file mode 100644
index d62766da98..0000000000
--- a/recipes/ads_of_the_world.recipe
+++ /dev/null
@@ -1,26 +0,0 @@
-from calibre.web.feeds.news import BasicNewsRecipe
-
-
-class AdvancedUserRecipe1336986047(BasicNewsRecipe):
-    title = u'Ads of the World'
-    oldest_article = 7
-    max_articles_per_feed = 100
-    auto_cleanup = False
-    description = 'The best international advertising campaigns'
-    language = 'en'
-    __author__ = 'faber1971'
-
-    no_stylesheets = True
-    keep_only_tags = [
-        dict(name='div', attrs={'id': 'primary'})
-    ]
-
-    remove_tags = [
-        dict(name='ul', attrs={'class': 'links inline'}), dict(name='div', attrs={'class': 'form-item'}), dict(
-            name='div', attrs={'id': ['options', 'comments']}), dict(name='ul', attrs={'id': 'nodePager'})
-    ]
-
-    reverse_article_order = True
-    masthead_url = 'http://bigcatgroup.co.uk/files/2011/01/05-ads-of-the-world.png'
-    feeds = [
-        (u'Ads of the world', u'http://feeds.feedburner.com/adsoftheworld-latest')]
diff --git a/recipes/icons/ads_of_the_world.png b/recipes/icons/ads_of_the_world.png
deleted file mode 100644
index d801e0e50b415bb5342017e9a9e57f572d3270b6..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 1321
zcmV+^1=jkBP)7%Q6vzK>X4kRTUu!$As|FGpQ8)cifvO5fB|s^t2#~l?apJ%M4iyIw9DqwDBrbpe
zRh7yK0e21{DvFR=6{!~rh*FwHfYS6MCTZf@aqT$v?!3og{Smh=Qc<(o!_2&y`R&{H
zX+?iH&LoOR6SN{top<%LSEO+BHmjdVDYBFh0EPer29%)n5McXz7WW9h7uK%w6$>H=T6gthNMHjZKzbOyh!9B8tr!A~41`C*;d!rF
zXg2!mA<_l}HkRPKkhDX4*QJQ$yJGRG*X+wMB+q8DZ`?in*<9hn%NP3x0h3VI$SsC1
zvM22x+Md_ItLOgwqEJwM^#C-Wh&0HEs9XL!hz$H%zV5S(rxSZK
z=|3tp5-_kW<{Z3tbWbYfXk9aAg*ni`V;)bq2RDw6Mj|W_;G5#rPi8NfAkbF1DaEzg
zn{xpioh{ax$J2=?v*|A{|FdNLtZjdg&p)0?ow!yyR$Ls5MqkL~0u5|ZFK-yy7>>Mu
z@#6GKeP=vw5g8VuAkf;$&Isj`&i$!asp)H_D@&)A$}vj|B=D@Rku_*yT{0Q6YQA}E>o7wc
ztz(vcVlaKGxU?l2wTRACs)c$ZOlsMf)0N8pTuy)`&;Pzye0_NMaDHsER{gPD{<^rZ
z=rt|X3rL6TiS&GsYJYoZ3
zB@d)xFOKG(%4Oc0nmN8u+Luco*)evqSQ?E+_6?>1uppw<$N9}A&pTVKZgQNM*0YU9
z)i*3KfTctUI1#c&;^Dbk^WhqcVjdhA^q;TnI+GEEbG29JXG?$
zGnML4c%bb0hQ(ukE}MNmmlKf-EA=B+uFN(XieTC{Pz{ei`gzunEH*qFibVpoep{@)
zbZUwTwI(l%dFSl(1}78`=>^Zb<~O6ZE*pQ@=L6@b?h1!p+hWK<)4S$-L5Bgr(jXG}
z1`9QxNtmF!MH+@>e%WIZR$>$RMrNujK7^GpNy+n>#3EYu`FyoT1g*#pvISbrK+6Ui
zASeSdtLi~J{*-|jVE`6^pcKT27zjZOxK(lq41s_FBfVWk?|nTrr~mp*_2WNkl%<7w
z58Po!I~T1PV^!m|CJb0w6m=S}JJ0B5Q*{G_{y(FqPi%RS00NTE0RVtVFDqE=4ym)!
f^2Dx;q&@o={zU=aK4D)x00000NkvXXu0mjfLaSfJ


From ba928291b93b13aa7cdf6beed5abf0de0734c617 Mon Sep 17 00:00:00 2001
From: James Cridland 
Date: Tue, 25 Feb 2020 22:51:41 +1000
Subject: [PATCH 086/162] Added Spectator Australia

A new recipe for Spectator Australia.
---
 recipes/icons/spectator-au.png | Bin 0 -> 1058 bytes
 recipes/spectator-au.recipe    |  36 +++++++++++++++++++++++++++++++++
 2 files changed, 36 insertions(+)
 create mode 100644 recipes/icons/spectator-au.png
 create mode 100644 recipes/spectator-au.recipe

diff --git a/recipes/icons/spectator-au.png b/recipes/icons/spectator-au.png
new file mode 100644
index 0000000000000000000000000000000000000000..c5508d9675e45b622cb0d84c50bca558fd1902ac
GIT binary patch
literal 1058
zcmV+-1l{|IP)!~|0@0{y9(>MVz1F!;c
zGVls829yBZ{3-!s=4&ULXJ=5~?2LFl;6A_)XsDr?XYZ3|CoKf%0&W13Odd(|j;^UC
zUkCUS(6NH9(=L{1hGR5*L7I^mty}@U*|a=u-nEWX@-2Wd%bV(T%AbEj@`k&JU3?v}
z3$G<|#w7&yKNk0hL(Hy-hb-U{j!xv=z!z9RTK@J
zFYZk`3c%#({y+J2X0li!cFFY&cdtVA$KSAnpgL5q@ArS$_S1+k>e1I*%E0Q?P?7>G>vYVxaKw^M^-jH(It~n`J8Yo%dq;}
z5UD#JL;x)G^b
zapn78aILZ#^^q~0Ap7KtMn3dU2>9xIbt_g4qskAzGTgN?x#wTQ35p+lhBT_trPrh3
z4XB5aY8+R7{1ve#a>|7`LGIaCkQ@jcbb{=NigB!5NK63>EQf2g&1r_Cs46COoTLuVm(U6%P!|+DGcv`>}&GsFB{)q$JHoq+H3=OQ}q@z`-XnwBYg#_pD0x
z@#p`$Z0I^#Dr1!3Q;cu0Bgq@@wy1x_QmR_Z+qD{!(b4m+#Jk;oMjJkb_!T#kyyZUP
zSKUhZ
Date: Tue, 25 Feb 2020 19:20:48 +0530
Subject: [PATCH 087/162] pep8

---
 recipes/spectator-au.recipe | 23 +++++++++++++++++++----
 1 file changed, 19 insertions(+), 4 deletions(-)

diff --git a/recipes/spectator-au.recipe b/recipes/spectator-au.recipe
index 3e8bd16cab..735d9460a7 100644
--- a/recipes/spectator-au.recipe
+++ b/recipes/spectator-au.recipe
@@ -3,9 +3,9 @@ __copyright__ = '2011, Pat Stapleton '
 '''
 https://www.spectator.com.au/
 '''
-import re
 from calibre.web.feeds.recipes import BasicNewsRecipe
 
+
 class SpectatorAU(BasicNewsRecipe):
     title = 'Spectator Australia'
     __author__ = 'Pat Stapleton, Dean Cording, James Cridland'
@@ -24,13 +24,28 @@ class SpectatorAU(BasicNewsRecipe):
     publication_type = 'newspaper'
     extra_css = '.article-header__author{margin-bottom:20px;}'
     conversion_options = {
-        'comments': description, 'tags': category, 'language': language, 'publisher': publisher, 'linearize_tables': False
+        'comments': description,
+        'tags': category,
+        'language': language,
+        'publisher': publisher,
+        'linearize_tables': False
     }
 
     keep_only_tags = [dict(attrs={'class': ['article']})]
-    remove_tags = [dict(attrs={'class': ['big-author','article-header__category','margin-menu','related-stories','disqus_thread','middle-promo','show-comments','article-tags']}), dict(name=['h4','hr'])]
+    remove_tags = [
+        dict(
+            attrs={
+                'class': [
+                    'big-author', 'article-header__category', 'margin-menu',
+                    'related-stories', 'disqus_thread', 'middle-promo',
+                    'show-comments', 'article-tags'
+                ]
+            }
+        ),
+        dict(name=['h4', 'hr'])
+    ]
     remove_attributes = ['width', 'height']
 
     feeds = [
         ('Spectator Australia', 'https://www.spectator.com.au/feed/'),
-        ]
+    ]

From d53a981dd16fd7a26175722fe8dc690ed475b6bf Mon Sep 17 00:00:00 2001
From: Steve Gilberd 
Date: Wed, 26 Feb 2020 02:52:12 +1300
Subject: [PATCH 088/162] Should be byte string

---
 src/calibre/devices/smart_device_app/driver.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/calibre/devices/smart_device_app/driver.py b/src/calibre/devices/smart_device_app/driver.py
index d32a1afe52..088a84fbf6 100644
--- a/src/calibre/devices/smart_device_app/driver.py
+++ b/src/calibre/devices/smart_device_app/driver.py
@@ -636,7 +636,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
             s = self._json_encode(self.opcodes[op], arg)
             if print_debug_info and extra_debug:
                 self._debug('send string', s)
-            self._send_byte_string(self.device_socket, (b'%d' % len(s)) + s)
+            self._send_byte_string(self.device_socket, (b'%d' % len(s)) + s.encode())
             if not wait_for_response:
                 return None, None
             return self._receive_from_client(print_debug_info=print_debug_info)

From ae67bb3517831b60a7ce69d4dd9360aa0adc7114 Mon Sep 17 00:00:00 2001
From: Steve Gilberd 
Date: Wed, 26 Feb 2020 03:01:36 +1300
Subject: [PATCH 089/162] More encoding fixes

---
 src/calibre/devices/smart_device_app/driver.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/calibre/devices/smart_device_app/driver.py b/src/calibre/devices/smart_device_app/driver.py
index 26c3da9115..e3c3a52e8b 100644
--- a/src/calibre/devices/smart_device_app/driver.py
+++ b/src/calibre/devices/smart_device_app/driver.py
@@ -842,9 +842,9 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
                     json_metadata[key]['book'] = self.json_codec.encode_book_metadata(book['book'])
                     json_metadata[key]['last_used'] = book['last_used']
                     result = json.dumps(json_metadata, indent=2, default=to_json)
-                    fd.write("%0.7d\n"%(len(result)+1))
-                    fd.write(result)
-                    fd.write('\n')
+                    fd.write(("%0.7d\n"%(len(result)+1)).encode())
+                    fd.write(result.encode())
+                    fd.write(b'\n')
                     count += 1
             self._debug('wrote', count, 'entries, purged', purged, 'entries')
 

From 17b09210eeea2dd72f601bbfc03ebba1c03da4dc Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Tue, 25 Feb 2020 19:42:58 +0530
Subject: [PATCH 090/162] String changes

---
 src/calibre/gui2/tweak_book/ui.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/calibre/gui2/tweak_book/ui.py b/src/calibre/gui2/tweak_book/ui.py
index 149e58a537..7046924952 100644
--- a/src/calibre/gui2/tweak_book/ui.py
+++ b/src/calibre/gui2/tweak_book/ui.py
@@ -474,7 +474,7 @@ class Main(MainWindow):
                                          'find', {'direction':'up'}, ('Shift+F3', 'Shift+Ctrl+G'), _('Find previous match'))
         self.action_replace = sreg('replace', _('&Replace'),
                                    'replace', keys=('Ctrl+R'), description=_('Replace current match'))
-        self.action_replace_next = sreg('replace-next', _('&Replace and find next'),
+        self.action_replace_next = sreg('replace-next', _('Replace and find ne&xt'),
                                         'replace-find', {'direction':'down'}, ('Ctrl+]'), _('Replace current match and find next'))
         self.action_replace_previous = sreg('replace-previous', _('R&eplace and find previous'),
                                         'replace-find', {'direction':'up'}, ('Ctrl+['), _('Replace current match and find previous'))

From afcb54df1e053c858026db4d99226a58e62c34d2 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Wed, 26 Feb 2020 07:04:59 +0530
Subject: [PATCH 091/162] RTF Input: Fix handling of RTF files with invalid
 encoded text. Fixes #1864719 [RTF conversion error: argument 1 must be
 unicode, not str](https://bugs.launchpad.net/calibre/+bug/1864719)

---
 src/calibre/ebooks/rtf2xml/convert_to_tags.py | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/src/calibre/ebooks/rtf2xml/convert_to_tags.py b/src/calibre/ebooks/rtf2xml/convert_to_tags.py
index 5ea516bb6b..b9c11754da 100644
--- a/src/calibre/ebooks/rtf2xml/convert_to_tags.py
+++ b/src/calibre/ebooks/rtf2xml/convert_to_tags.py
@@ -1,6 +1,5 @@
 from __future__ import unicode_literals, absolute_import, print_function, division
 import os, sys
-from codecs import EncodedFile
 
 from calibre.ebooks.rtf2xml import copy, check_encoding
 from calibre.ptempfile import better_mktemp
@@ -274,15 +273,10 @@ class ConvertToTags:
         if self.__convert_utf or self.__bad_encoding:
             copy_obj = copy.Copy(bug_handler=self.__bug_handler)
             copy_obj.rename(self.__write_to, self.__file)
-            file_encoding = "utf-8"
-            if self.__bad_encoding:
-                file_encoding = "us-ascii"
             with open_for_read(self.__file) as read_obj:
                 with open_for_write(self.__write_to) as write_obj:
-                    write_objenc = EncodedFile(write_obj, self.__encoding,
-                                    file_encoding, 'replace')
                     for line in read_obj:
-                        write_objenc.write(line)
+                        write_obj.write(line)
         copy_obj = copy.Copy(bug_handler=self.__bug_handler)
         if self.__copy:
             copy_obj.copy_file(self.__write_to, "convert_to_tags.data")

From edcdafe53683fdfa8fab6a7a43f8a15e8bea1b44 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Wed, 26 Feb 2020 09:08:45 +0530
Subject: [PATCH 092/162] Viewer: Keyboard shortcuts to change number of
 columns (Ctrl+[ and Ctrl+])

---
 src/pyj/read_book/iframe.pyj     | 33 +++++++++++++++++++++++---------
 src/pyj/read_book/paged_mode.pyj |  5 +++++
 src/pyj/read_book/shortcuts.pyj  | 18 +++++++++++++++++
 src/pyj/read_book/view.pyj       | 13 +++++++++++++
 4 files changed, 60 insertions(+), 9 deletions(-)

diff --git a/src/pyj/read_book/iframe.pyj b/src/pyj/read_book/iframe.pyj
index 08b809f2e6..695dc2dab4 100644
--- a/src/pyj/read_book/iframe.pyj
+++ b/src/pyj/read_book/iframe.pyj
@@ -24,13 +24,14 @@ from read_book.mathjax import apply_mathjax
 from read_book.paged_mode import (
     anchor_funcs as paged_anchor_funcs,
     auto_scroll_action as paged_auto_scroll_action, calc_columns_per_screen,
-    will_columns_per_screen_change, current_cfi,
-    handle_gesture as paged_handle_gesture, handle_shortcut as paged_handle_shortcut,
-    jump_to_cfi as paged_jump_to_cfi, layout as paged_layout,
-    onwheel as paged_onwheel, prepare_for_resize as paged_prepare_for_resize,
-    progress_frac, reset_paged_mode_globals, resize_done as paged_resize_done,
+    current_cfi, get_columns_per_screen_data, handle_gesture as paged_handle_gesture,
+    handle_shortcut as paged_handle_shortcut, jump_to_cfi as paged_jump_to_cfi,
+    layout as paged_layout, onwheel as paged_onwheel,
+    prepare_for_resize as paged_prepare_for_resize, progress_frac,
+    reset_paged_mode_globals, resize_done as paged_resize_done,
     scroll_by_page as paged_scroll_by_page, scroll_to_elem,
-    scroll_to_fraction as paged_scroll_to_fraction, snap_to_selection
+    scroll_to_fraction as paged_scroll_to_fraction, snap_to_selection,
+    will_columns_per_screen_change
 )
 from read_book.referencing import (
     elem_for_ref, end_reference_mode, start_reference_mode
@@ -100,7 +101,8 @@ class IframeBoss:
         handlers = {
             'change_color_scheme': self.change_color_scheme,
             'change_font_size': self.change_font_size,
-            'viewer_font_size_changed': self.viewer_font_size_changed,
+            'change_number_of_columns': self.change_number_of_columns,
+            'number_of_columns_changed': self.number_of_columns_changed,
             'change_scroll_speed': self.change_scroll_speed,
             'display': self.display,
             'find': self.find,
@@ -276,9 +278,18 @@ class IframeBoss:
                 # apply_font_size() is a no-op
                 self.relayout_on_font_size_change()
 
-    def viewer_font_size_changed(self, data):
-        opts.base_font_size = data.base_font_size
+    def change_number_of_columns(self, data):
+        if current_layout_mode() is 'flow':
+            return
+        cdata = get_columns_per_screen_data()
+        delta = int(data.delta)
+        if delta is 0:
+            new_val = 0
+        else:
+            new_val = max(1, cdata.cps + delta)
+        opts.columns_per_screen[cdata.which] = new_val
         self.relayout_on_font_size_change()
+        self.send_message('columns_per_screen_changed', which=cdata.which, cps=new_val)
 
     def relayout_on_font_size_change(self):
         if current_layout_mode() is not 'flow' and will_columns_per_screen_change():
@@ -290,6 +301,10 @@ class IframeBoss:
             self.update_cfi()
             self.update_toc_position()
 
+    def number_of_columns_changed(self, data):
+        opts.columns_per_screen = data.columns_per_screen
+        self.relayout_on_font_size_change()
+
     def change_scroll_speed(self, data):
         if data.lines_per_sec_auto?:
             opts.lines_per_sec_auto = data.lines_per_sec_auto
diff --git a/src/pyj/read_book/paged_mode.pyj b/src/pyj/read_book/paged_mode.pyj
index 5c6f2bbab4..e2bb712378 100644
--- a/src/pyj/read_book/paged_mode.pyj
+++ b/src/pyj/read_book/paged_mode.pyj
@@ -152,6 +152,11 @@ def calc_columns_per_screen():
     return cps
 
 
+def get_columns_per_screen_data():
+    which = 'landscape' if scroll_viewport.width() > scroll_viewport.height() else 'portrait'
+    return {'which': which, 'cps': calc_columns_per_screen()}
+
+
 def will_columns_per_screen_change():
     return calc_columns_per_screen() != cols_per_screen
 
diff --git a/src/pyj/read_book/shortcuts.pyj b/src/pyj/read_book/shortcuts.pyj
index 8c69a0aece..437ca8d6af 100644
--- a/src/pyj/read_book/shortcuts.pyj
+++ b/src/pyj/read_book/shortcuts.pyj
@@ -216,6 +216,24 @@ def shortcuts_definition():
         _('Decrease font size'),
     ),
 
+    'increase_number_of_columns': desc(
+        v"['Ctrl+]']",
+        'ui',
+        _('Increase number of columns'),
+    ),
+
+    'decrease_number_of_columns': desc(
+        v"['Ctrl+[']",
+        'ui',
+        _('Decrease number of columns'),
+    ),
+
+    'reset_number_of_columns': desc(
+        v"['Ctrl+Alt+c']",
+        'ui',
+        _('Make number of columns automatic'),
+    ),
+
     'toggle_full_screen': desc(
         v"['F11', 'Ctrl+Shift+F']",
         'ui',
diff --git a/src/pyj/read_book/view.pyj b/src/pyj/read_book/view.pyj
index 01918d668e..62d06c9b5c 100644
--- a/src/pyj/read_book/view.pyj
+++ b/src/pyj/read_book/view.pyj
@@ -237,6 +237,7 @@ class View:
             'request_size': self.on_request_size,
             'scroll_to_anchor': self.on_scroll_to_anchor,
             'selectionchange': self.on_selection_change,
+            'columns_per_screen_changed': self.on_columns_per_screen_changed,
             'show_chrome': self.show_chrome,
             'show_footnote': self.on_show_footnote,
             'update_cfi': self.on_update_cfi,
@@ -453,6 +454,12 @@ class View:
             self.toggle_autoscroll()
         elif data.name.startsWith('switch_color_scheme:'):
             self.switch_color_scheme(data.name.partition(':')[-1])
+        elif data.name is 'increase_number_of_columns':
+            self.iframe_wrapper.send_message('change_number_of_columns', delta=1)
+        elif data.name is 'decrease_number_of_columns':
+            self.iframe_wrapper.send_message('change_number_of_columns', delta=-1)
+        elif data.name is 'reset_number_of_columns':
+            self.iframe_wrapper.send_message('change_number_of_columns', delta=0)
         else:
             self.iframe_wrapper.send_message('handle_navigation_shortcut', name=data.name)
 
@@ -461,6 +468,12 @@ class View:
         if ui_operations.selection_changed:
             ui_operations.selection_changed(data.text)
 
+    def on_columns_per_screen_changed(self, data):
+        sd = get_session_data()
+        cps = sd.get('columns_per_screen') or {}
+        cps[data.which] = int(data.cps)
+        sd.set('columns_per_screen', cps)
+
     def switch_color_scheme(self, name):
         get_session_data().set('current_color_scheme', name)
         ui_operations.redisplay_book()

From 6ac6071167c648d6db0149d73609eab7637e338b Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Wed, 26 Feb 2020 09:10:54 +0530
Subject: [PATCH 093/162] Error message when trying to change number of columns
 in flow mode

---
 src/pyj/read_book/iframe.pyj    | 2 ++
 src/pyj/read_book/shortcuts.pyj | 6 +++---
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/src/pyj/read_book/iframe.pyj b/src/pyj/read_book/iframe.pyj
index 695dc2dab4..11a7e5ff90 100644
--- a/src/pyj/read_book/iframe.pyj
+++ b/src/pyj/read_book/iframe.pyj
@@ -280,6 +280,8 @@ class IframeBoss:
 
     def change_number_of_columns(self, data):
         if current_layout_mode() is 'flow':
+            self.send_message('error', title=_('In flow mode'), msg=_(
+                'Cannot change number of pages per screen in flow mode, switch to paged mode first.'))
             return
         cdata = get_columns_per_screen_data()
         delta = int(data.delta)
diff --git a/src/pyj/read_book/shortcuts.pyj b/src/pyj/read_book/shortcuts.pyj
index 437ca8d6af..8316be03a6 100644
--- a/src/pyj/read_book/shortcuts.pyj
+++ b/src/pyj/read_book/shortcuts.pyj
@@ -219,19 +219,19 @@ def shortcuts_definition():
     'increase_number_of_columns': desc(
         v"['Ctrl+]']",
         'ui',
-        _('Increase number of columns'),
+        _('Increase number of pages per screen'),
     ),
 
     'decrease_number_of_columns': desc(
         v"['Ctrl+[']",
         'ui',
-        _('Decrease number of columns'),
+        _('Decrease number of pages per screen'),
     ),
 
     'reset_number_of_columns': desc(
         v"['Ctrl+Alt+c']",
         'ui',
-        _('Make number of columns automatic'),
+        _('Make number of pages per screen automatic'),
     ),
 
     'toggle_full_screen': desc(

From cc18d6dab4db166c205135529cb0858c2c432475 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Wed, 26 Feb 2020 11:09:44 +0530
Subject: [PATCH 094/162] Widen more limits

Fixes #1864789 [[Enhancement] Increase pause time before switching chapter with auto-scrolling](https://bugs.launchpad.net/calibre/+bug/1864789)
---
 src/pyj/read_book/prefs/scrolling.pyj | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/pyj/read_book/prefs/scrolling.pyj b/src/pyj/read_book/prefs/scrolling.pyj
index d61c597ef9..71a6511eb6 100644
--- a/src/pyj/read_book/prefs/scrolling.pyj
+++ b/src/pyj/read_book/prefs/scrolling.pyj
@@ -17,10 +17,10 @@ MIN_SCROLL_SPEED_AUTO = 0.05
 MAX_SCROLL_SPEED_AUTO = 5
 
 MIN_SCROLL_AUTO_DELAY = -1
-MAX_SCROLL_AUTO_DELAY = 10
+MAX_SCROLL_AUTO_DELAY = 50
 
-MIN_SCROLL_SPEED_SMOOTH = 10
-MAX_SCROLL_SPEED_SMOOTH = 50
+MIN_SCROLL_SPEED_SMOOTH = 5
+MAX_SCROLL_SPEED_SMOOTH = 80
 
 def restore_defaults():
     container = get_container()

From 22cb38d65798217c09f8b81da35735212995ceb5 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Wed, 26 Feb 2020 14:43:52 +0530
Subject: [PATCH 095/162] ...

---
 recipes/the_baffler.recipe | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/recipes/the_baffler.recipe b/recipes/the_baffler.recipe
index 3b84cf54ff..8fb9c96ebb 100644
--- a/recipes/the_baffler.recipe
+++ b/recipes/the_baffler.recipe
@@ -15,7 +15,7 @@ class TheBaffler(BasicNewsRecipe):
     __author__ = 'Jose Ortiz'
     description = ('This magazine contains left-wing criticism, cultural analysis, shorts'
                    ' stories, poems and art.  They publish six print issues annually.')
-    language = 'en_US'
+    language = 'en'
     encoding = 'UTF-8'
     no_javascript = True
     no_stylesheets = True

From 45418c3ebe98def68a270e36590257b6341e64ca Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Wed, 26 Feb 2020 15:32:35 +0530
Subject: [PATCH 096/162] ...

---
 src/calibre/gui2/widgets.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/calibre/gui2/widgets.py b/src/calibre/gui2/widgets.py
index d5b7b66aa4..5ef65a5d1c 100644
--- a/src/calibre/gui2/widgets.py
+++ b/src/calibre/gui2/widgets.py
@@ -524,7 +524,7 @@ class EnLineEdit(LineEditECM, QLineEdit):  # {{{
     def event(self, ev):
         # See https://bugreports.qt.io/browse/QTBUG-46911
         if ev.type() == ev.ShortcutOverride and (
-                ev.key() in (Qt.Key_Left, Qt.Key_Right) and (ev.modifiers() & ~Qt.KeypadModifier) == Qt.ControlModifier):
+                hasattr(ev, 'key') and ev.key() in (Qt.Key_Left, Qt.Key_Right) and (ev.modifiers() & ~Qt.KeypadModifier) == Qt.ControlModifier):
             ev.accept()
         return QLineEdit.event(self, ev)
 

From 9864af035af39f054611beba99a900f203a0886d Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Wed, 26 Feb 2020 22:17:15 +0530
Subject: [PATCH 097/162] Use inverted name for ISO 639 names in preference

---
 setup/translations.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/setup/translations.py b/setup/translations.py
index 78e1e94407..19e7641ce2 100644
--- a/setup/translations.py
+++ b/setup/translations.py
@@ -100,8 +100,12 @@ class POT(Command):  # {{{
             root = json.load(f)
         entries = root['639-3']
         ans = []
-        for x in sorted(entries, key=lambda x:(x.get('name') or '').lower()):
-            name = x.get('name')
+
+        def name_getter(x):
+            return x.get('inverted_name') or x.get('name')
+
+        for x in sorted(entries, key=lambda x:name_getter(x).lower()):
+            name = name_getter(x)
             if name:
                 ans.append(u'msgid "{}"'.format(name))
                 ans.append('msgstr ""')
@@ -849,7 +853,7 @@ class ISO639(Command):  # {{{
                 threeb = unicode_type(threeb)
             if threeb is None:
                 continue
-            name = x.get('name')
+            name = x.get('inverted_name') or x.get('name')
             if name:
                 name = unicode_type(name)
             if not name or name[0] in '!~=/\'"':

From da1b823e07dbdaa951f4141be26ac238c8f32fa5 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Thu, 27 Feb 2020 07:22:03 +0530
Subject: [PATCH 098/162] String changes

---
 src/calibre/gui2/tweak_book/ui.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/calibre/gui2/tweak_book/ui.py b/src/calibre/gui2/tweak_book/ui.py
index 7046924952..00062e385b 100644
--- a/src/calibre/gui2/tweak_book/ui.py
+++ b/src/calibre/gui2/tweak_book/ui.py
@@ -370,7 +370,7 @@ class Main(MainWindow):
         self.action_save_copy = treg('save.png', _('Save a ©'), self.boss.save_copy, 'save-copy', 'Ctrl+Alt+S', _('Save a copy of the book'))
         self.action_quit = treg('window-close.png', _('&Quit'), self.boss.quit, 'quit', 'Ctrl+Q', _('Quit'))
         self.action_preferences = treg('config.png', _('&Preferences'), self.boss.preferences, 'preferences', 'Ctrl+P', _('Preferences'))
-        self.action_new_book = treg('plus.png', _('Create &new, empty book'), self.boss.new_book, 'new-book', (), _('Create a new, empty book'))
+        self.action_new_book = treg('plus.png', _('Create new, &empty book'), self.boss.new_book, 'new-book', (), _('Create a new, empty book'))
         self.action_import_book = treg('add_book.png', _('&Import an HTML or DOCX file as a new book'),
                                       self.boss.import_book, 'import-book', (), _('Import an HTML or DOCX file as a new book'))
         self.action_quick_edit = treg('modified.png', _('&Quick open a file to edit'), self.boss.quick_open, 'quick-open', ('Ctrl+T'), _(
@@ -493,7 +493,7 @@ class Main(MainWindow):
 
         # Check Book actions
         group = _('Check book')
-        self.action_check_book = treg('debug.png', _('&Check book'), self.boss.check_requested, 'check-book', ('F7'), _('Check book for errors'))
+        self.action_check_book = treg('debug.png', _('C&heck book'), self.boss.check_requested, 'check-book', ('F7'), _('Check book for errors'))
         self.action_spell_check_book = treg('spell-check.png', _('Check &spelling'), self.boss.spell_check_requested, 'spell-check-book', ('Alt+F7'), _(
             'Check book for spelling errors'))
         self.action_check_book_next = reg('forward.png', _('&Next error'), partial(

From 6a71dd647b8b80c9c79706b7a9ee89c615fbfed9 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Thu, 27 Feb 2020 07:38:03 +0530
Subject: [PATCH 099/162] ...

---
 src/calibre/gui2/tweak_book/ui.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/calibre/gui2/tweak_book/ui.py b/src/calibre/gui2/tweak_book/ui.py
index 00062e385b..008fa5186c 100644
--- a/src/calibre/gui2/tweak_book/ui.py
+++ b/src/calibre/gui2/tweak_book/ui.py
@@ -359,7 +359,6 @@ class Main(MainWindow):
         self.action_edit_previous_file = treg('arrow-up.png', _('Edit &previous file'), partial(self.boss.edit_next_file, backwards=True),
                 'edit-previous-file', 'Ctrl+Alt+Up', _('Edit the previous file in the spine'))
         # Qt does not generate shortcut overrides for cmd+arrow on os x which
-        # Qt does not generate shortcut overrides for cmd+arrow on os x which
         # means these shortcuts interfere with editing
         self.action_global_undo = treg('back.png', _('&Revert to before'), self.boss.do_global_undo, 'global-undo', () if isosx else 'Ctrl+Left',
                                       _('Revert book to before the last action (Undo)'))

From af3be6ddd97870cebfaddc120396bb9c184c7c99 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Thu, 27 Feb 2020 15:28:15 +0530
Subject: [PATCH 100/162] Viewer: Save current position after 3 seconds of last
 position change

Useful if the viewer crashes on resume from sleep.
---
 src/calibre/gui2/viewer/ui.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/calibre/gui2/viewer/ui.py b/src/calibre/gui2/viewer/ui.py
index 14198bf5a7..f4a12dcafd 100644
--- a/src/calibre/gui2/viewer/ui.py
+++ b/src/calibre/gui2/viewer/ui.py
@@ -92,6 +92,9 @@ class EbookViewer(MainWindow):
         connect_lambda(self.book_preparation_started, self, lambda self: self.loading_overlay(_(
             'Preparing book for first read, please wait')), type=Qt.QueuedConnection)
         self.maximized_at_last_fullscreen = False
+        self.save_pos_timer = t = QTimer(self)
+        t.setSingleShot(True), t.setInterval(3000), t.setTimerType(Qt.VeryCoarseTimer)
+        connect_lambda(t.timeout, self, lambda self: self.save_annotations(in_book_file=False))
         self.pending_open_at = open_at
         self.base_window_title = _('E-book viewer')
         self.setWindowTitle(self.base_window_title)
@@ -534,10 +537,12 @@ class EbookViewer(MainWindow):
             return
         self.current_book_data['annotations_map']['last-read'] = [{
             'pos': cfi, 'pos_type': 'epubcfi', 'timestamp': utcnow()}]
+        self.save_pos_timer.start()
     # }}}
 
     # State serialization {{{
     def save_annotations(self, in_book_file=True):
+        print(11111111111)
         if not self.current_book_data:
             return
         amap = self.current_book_data['annotations_map']

From 90aba42b2a01c8ce1d3d5cc63c5fba2a519ca3dd Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Thu, 27 Feb 2020 15:30:02 +0530
Subject: [PATCH 101/162] ...

---
 src/calibre/gui2/viewer/ui.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/calibre/gui2/viewer/ui.py b/src/calibre/gui2/viewer/ui.py
index f4a12dcafd..92fcf3b7bd 100644
--- a/src/calibre/gui2/viewer/ui.py
+++ b/src/calibre/gui2/viewer/ui.py
@@ -542,7 +542,6 @@ class EbookViewer(MainWindow):
 
     # State serialization {{{
     def save_annotations(self, in_book_file=True):
-        print(11111111111)
         if not self.current_book_data:
             return
         amap = self.current_book_data['annotations_map']

From 2f701318d253942039aefa53dc60bce759b7187a Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Fri, 28 Feb 2020 14:45:07 +0530
Subject: [PATCH 102/162] Viewer: Fix searching in Regex and Whole words mode
 not working well.

Viewer: Fix searching for multiple words in fixed layout books not
working. Fixes #1863464 [Private bug](https://bugs.launchpad.net/calibre/+bug/1863464)
---
 src/calibre/gui2/viewer/search.py |  61 +++++++-------
 src/pyj/read_book/find.pyj        | 128 ++++++++++++++++++++++++++++++
 src/pyj/read_book/iframe.pyj      |  42 ++--------
 src/pyj/utils.pyj                 |  15 ----
 4 files changed, 167 insertions(+), 79 deletions(-)
 create mode 100644 src/pyj/read_book/find.pyj

diff --git a/src/calibre/gui2/viewer/search.py b/src/calibre/gui2/viewer/search.py
index b712372968..50dcac5df3 100644
--- a/src/calibre/gui2/viewer/search.py
+++ b/src/calibre/gui2/viewer/search.py
@@ -53,16 +53,21 @@ class BusySpinner(QWidget):  # {{{
 
 quote_map= {'"':'"“”', "'": "'‘’"}
 qpat = regex.compile(r'''(['"])''')
+spat = regex.compile(r'(\s+)')
 
 
 def text_to_regex(text):
     ans = []
-    for part in qpat.split(text):
-        r = quote_map.get(part)
-        if r is not None:
-            ans.append('[' + r + ']')
+    for wpart in spat.split(text):
+        if not wpart.strip():
+            ans.append(r'\s+')
         else:
-            ans.append(regex.escape(part))
+            for part in qpat.split(wpart):
+                r = quote_map.get(part)
+                if r is not None:
+                    ans.append('[' + r + ']')
+                else:
+                    ans.append(regex.escape(part))
     return ''.join(ans)
 
 
@@ -111,10 +116,11 @@ class SearchFinished(object):
 
 class SearchResult(object):
 
-    __slots__ = ('search_query', 'before', 'text', 'after', 'spine_idx', 'index', 'file_name', '_static_text')
+    __slots__ = ('search_query', 'before', 'text', 'after', 'q', 'spine_idx', 'index', 'file_name', '_static_text')
 
-    def __init__(self, search_query, before, text, after, name, spine_idx, index):
+    def __init__(self, search_query, before, text, after, q, name, spine_idx, index):
         self.search_query = search_query
+        self.q = q
         self.before, self.text, self.after = before, text, after
         self.spine_idx, self.index = spine_idx, index
         self.file_name = name
@@ -145,8 +151,8 @@ class SearchResult(object):
             'before': self.before, 'after': self.after, 'mode': self.search_query.mode
         }
 
-    def is_or_is_after(self, result_from_js):
-        return result_from_js['spine_idx'] == self.spine_idx and self.index >= result_from_js['index'] and result_from_js['text'] == self.text
+    def is_result(self, result_from_js):
+        return result_from_js['spine_idx'] == self.spine_idx and self.index == result_from_js['index'] and result_from_js['text'] == self.text
 
     def __str__(self):
         from collections import namedtuple
@@ -179,10 +185,7 @@ def searchable_text_for_name(name):
             stack.append(tail)
         if children:
             stack.extend(reversed(children))
-    # Normalize whitespace to a single space, this will cause failures
-    # when searching over spaces in pre nodes, but that is a lesser evil
-    # since the DOM converts \n, \t etc to a single space
-    return regex.sub(r'\s+', ' ', ''.join(ans))
+    return ''.join(ans)
 
 
 def search_in_name(name, search_query, ctx_size=50):
@@ -383,23 +386,24 @@ class Results(QListWidget):  # {{{
         self.item_activated()
 
     def search_result_not_found(self, sr):
-        remove = []
+        remove = None
         for i in range(self.count()):
             item = self.item(i)
             r = item.data(Qt.UserRole)
-            if r.is_or_is_after(sr):
-                remove.append(i)
-        if remove:
-            last_i = remove[-1]
-            if last_i < self.count() - 1:
-                self.setCurrentRow(last_i + 1)
+            if r.is_result(sr):
+                remove = i
+        if remove is not None:
+            q = sr['spine_idx']
+            for i in range(remove + 1, self.count()):
+                item = self.item(i)
+                r = item.data(Qt.UserRole)
+                if r.spine_index != q:
+                    break
+                r.index -= 1
+            self.takeItem(remove)
+            if remove < self.count():
+                self.setCurrentRow(remove)
                 self.item_activated()
-            elif remove[0] > 0:
-                self.setCurrentRow(remove[0] - 1)
-                self.item_activated()
-            for i in reversed(remove):
-                self.takeItem(i)
-
 # }}}
 
 
@@ -469,8 +473,9 @@ class SearchPanel(QWidget):  # {{{
                 try:
                     for i, result in enumerate(search_in_name(name, search_query)):
                         before, text, after = result
-                        self.results_found.emit(SearchResult(search_query, before, text, after, name, spine_idx, counter[text]))
-                        counter[text] += 1
+                        q = (before or '')[-5:] + text + (after or '')[:5]
+                        self.results_found.emit(SearchResult(search_query, before, text, after, q, name, spine_idx, counter[q]))
+                        counter[q] += 1
                 except Exception:
                     import traceback
                     traceback.print_exc()
diff --git a/src/pyj/read_book/find.pyj b/src/pyj/read_book/find.pyj
new file mode 100644
index 0000000000..03ea74f69d
--- /dev/null
+++ b/src/pyj/read_book/find.pyj
@@ -0,0 +1,128 @@
+# vim:fileencoding=utf-8
+# License: GPL v3 Copyright: 2020, Kovid Goyal 
+from __python__ import bound_methods, hash_literals
+
+
+def build_text_map():
+    node_list = v'[]'
+    flat_text = ''
+    ignored_tags = {
+        'style': True, 'script': True, 'noscript': True, 'title': True, 'meta': True, 'head': True, 'link': True, 'html': True,
+        'img': True
+    }
+
+    def process_node(node):
+        nonlocal flat_text
+        if node.nodeType is Node.TEXT_NODE:
+            text = node.nodeValue
+            if text and text.length:
+                node_list.push({'node': node, 'offset': flat_text.length, 'length': text.length})
+                flat_text += text
+        elif node.nodeType is Node.ELEMENT_NODE:
+            if not node.hasChildNodes():
+                return
+            tag = node.tagName.toLowerCase()
+            if ignored_tags[tag]:
+                return
+            style = window.getComputedStyle(node)
+            if style.display is 'none' or style.visibility is 'hidden':
+                return
+            children = node.childNodes
+            for i in range(children.length):
+                process_node(v'children[i]')
+
+    process_node(document.body)
+    return {'timestamp': window.performance.now(), 'flat_text': flat_text, 'node_list': node_list}
+
+
+def find_node_for_index_binary(node_list, idx_in_flat_text, start):
+    # Do a binary search for idx
+    start = start or 0
+    end = node_list.length - 1
+    while start <= end:
+        mid = Math.floor((start + end)/2)
+        q = node_list[mid]
+        limit = q.offset + q.length
+        if q.offset <= idx_in_flat_text and limit > idx_in_flat_text:
+            start_node = q.node
+            start_offset = idx_in_flat_text - q.offset
+            return start_node, start_offset, mid
+        if limit <= idx_in_flat_text:
+            start = mid + 1
+        else:
+            end = mid - 1
+    return None, None, None
+
+
+def find_node_for_index_linear(node_list, idx_in_flat_text, start):
+    start = start or 0
+    for i in range(start, node_list.length):
+        q = node_list[i]
+        limit = q.offset + q.length
+        if q.offset <= idx_in_flat_text and limit > idx_in_flat_text:
+            start_node = q.node
+            start_offset = idx_in_flat_text - q.offset
+            return start_node, start_offset, i
+    return None, None, None
+
+
+def find_specific_occurrence(q, num, before_len, after_len, text_map):
+    if not q or not q.length:
+        return
+    from_idx = 0
+    flat_text = text_map.flat_text
+    pos = 0
+    match_num = -1
+    while True:
+        idx = flat_text.indexOf(q, from_idx)
+        if idx < 0:
+            break
+        match_num += 1
+        from_idx = idx + 1
+        if num < match_num:
+            continue
+        start_node, start_offset, node_pos = find_node_for_index_binary(text_map.node_list, idx + before_len, pos)
+        if start_node is not None:
+            pos = node_pos
+            end_node, end_offset, node_pos = find_node_for_index_linear(text_map.node_list, idx + q.length - after_len, pos)
+            if end_node is not None:
+                return {
+                    'start_node': start_node, 'start_offset': start_offset, 'start_pos': pos,
+                    'end_node': end_node, 'end_offset': end_offset, 'end_pos': node_pos,
+                    'idx_in_flat_text': idx
+                }
+        break
+
+
+cache = {}
+
+
+def reset_find_caches():
+    nonlocal cache
+    cache = {}
+
+
+def select_find_result(match):
+    sel = window.getSelection()
+    sel.setBaseAndExtent(match.start_node, match.start_offset, match.end_node, match.end_offset)
+
+
+def select_search_result(sr):
+    window.getSelection().removeAllRanges()
+    if not cache.text_map:
+        cache.text_map = build_text_map()
+    q = ''
+    before_len = after_len = 0
+    if sr.before:
+        q = sr.before[-5:]
+        before_len = q.length
+    q += sr.text
+    if sr.after:
+        after = sr.after[:5]
+        after_len = after.length
+        q += after
+    match = find_specific_occurrence(q, int(sr.index), before_len, after_len, cache.text_map)
+    if not match:
+        return False
+    select_find_result(match)
+    return True
diff --git a/src/pyj/read_book/iframe.pyj b/src/pyj/read_book/iframe.pyj
index 11a7e5ff90..40f55f0bcf 100644
--- a/src/pyj/read_book/iframe.pyj
+++ b/src/pyj/read_book/iframe.pyj
@@ -9,6 +9,7 @@ from fs_images import fix_fullscreen_svg_images
 from iframe_comm import IframeClient
 from read_book.cfi import scroll_to as scroll_to_cfi
 from read_book.extract import get_elements
+from read_book.find import reset_find_caches, select_search_result
 from read_book.flow_mode import (
     anchor_funcs as flow_anchor_funcs, auto_scroll_action as flow_auto_scroll_action,
     flow_onwheel, flow_to_scroll_fraction, handle_gesture as flow_handle_gesture,
@@ -49,9 +50,7 @@ from read_book.touch import (
     create_handlers as create_touch_handlers, reset_handlers as reset_touch_handlers
 )
 from read_book.viewport import scroll_viewport
-from utils import (
-    apply_cloned_selection, clone_selection, debounce, html_escape, is_ios
-)
+from utils import debounce, html_escape, is_ios
 
 FORCE_FLOW_MODE = False
 CALIBRE_VERSION = '__CALIBRE_VERSION__'
@@ -339,6 +338,7 @@ class IframeBoss:
         self.content_loaded_stage2()
 
     def content_loaded_stage2(self):
+        reset_find_caches()
         self.connect_links()
         self.content_ready = True
         # this is the loading styles used to suppress scrollbars during load
@@ -580,39 +580,9 @@ class IframeBoss:
                 self.send_message('find_in_spine', text=data.text, backwards=data.backwards, searched_in_spine=data.searched_in_spine)
 
     def show_search_result(self, data, from_load):
-        sr = data.search_result
-        idx = -1
-        window.getSelection().removeAllRanges()
-        while idx < sr.index:
-            if not window.find(sr.text, True, False, False, False, False):
-                self.send_message('search_result_not_found', search_result=sr)
-                break
-            if sr.mode is not 'normal':
-                # verify we have the correct match since regexes can have
-                # boundary conditions
-                sel = window.getSelection()
-                ranges = clone_selection(sel)
-                r = ranges[0]
-                if sr.before:
-                    p = r.cloneRange()
-                    p.collapse(True)
-                    sel = apply_cloned_selection(v'[p]')
-                    sel.modify('extend', 'left', 'character')
-                    if sel.toString() is not sr.before[-1]:
-                        apply_cloned_selection(ranges)
-                        continue
-                if sr.after:
-                    p = r.cloneRange()
-                    p.collapse(False)
-                    sel = apply_cloned_selection(v'[p]')
-                    sel.modify('extend', 'right', 'character')
-                    if sel.toString() is not sr.after[0]:
-                        apply_cloned_selection(ranges)
-                        continue
-                apply_cloned_selection(ranges)
-            idx += 1
-        if idx > -1 and current_layout_mode() is not 'flow':
-            snap_to_selection()
+        if select_search_result(data.search_result):
+            if current_layout_mode() is not 'flow':
+                snap_to_selection()
 
     def reference_item_changed(self, ref_num_or_none):
         self.send_message('reference_item_changed', refnum=ref_num_or_none, index=current_spine_item().index)
diff --git a/src/pyj/utils.pyj b/src/pyj/utils.pyj
index 625f5524e3..93e1747e10 100644
--- a/src/pyj/utils.pyj
+++ b/src/pyj/utils.pyj
@@ -252,21 +252,6 @@ def sandboxed_html(html, style, sandbox):
     return ans
 
 
-def clone_selection(sel):
-    ans = v'[]'
-    for i in range(sel.rangeCount):
-        ans.push(sel.getRangeAt(i).cloneRange())
-    return ans
-
-
-def apply_cloned_selection(ranges):
-    sel = window.getSelection()
-    sel.removeAllRanges()
-    for r in ranges:
-        sel.addRange(r)
-    return sel
-
-
 if __name__ is '__main__':
     from pythonize import strings
     strings()

From 9924018740c423f41673bf9a36dc077314c52fad Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Fri, 28 Feb 2020 18:19:46 +0530
Subject: [PATCH 103/162] Fix #1865126 [[Enhancement] Remove empty
 space](https://bugs.launchpad.net/calibre/+bug/1865126)

---
 src/calibre/gui2/convert/page_setup.ui | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/calibre/gui2/convert/page_setup.ui b/src/calibre/gui2/convert/page_setup.ui
index a2cb9cfc10..e3b759a81a 100644
--- a/src/calibre/gui2/convert/page_setup.ui
+++ b/src/calibre/gui2/convert/page_setup.ui
@@ -96,7 +96,10 @@
        
         Margins
        
-       
+       
+        
+         QFormLayout::FieldsStayAtSizeHint
+        
         
          
           

From 2c794839db8906659f17b2708cd42e2c5be568aa Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Fri, 28 Feb 2020 18:27:56 +0530
Subject: [PATCH 104/162] Forgot to report hidden search results

---
 src/calibre/gui2/viewer/search.py | 4 ++--
 src/pyj/read_book/iframe.pyj      | 2 ++
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/calibre/gui2/viewer/search.py b/src/calibre/gui2/viewer/search.py
index 50dcac5df3..c33a1d3646 100644
--- a/src/calibre/gui2/viewer/search.py
+++ b/src/calibre/gui2/viewer/search.py
@@ -401,8 +401,8 @@ class Results(QListWidget):  # {{{
                     break
                 r.index -= 1
             self.takeItem(remove)
-            if remove < self.count():
-                self.setCurrentRow(remove)
+            if self.count():
+                self.setCurrentRow(min(remove, self.count()-1))
                 self.item_activated()
 # }}}
 
diff --git a/src/pyj/read_book/iframe.pyj b/src/pyj/read_book/iframe.pyj
index 40f55f0bcf..b5a2d7c883 100644
--- a/src/pyj/read_book/iframe.pyj
+++ b/src/pyj/read_book/iframe.pyj
@@ -583,6 +583,8 @@ class IframeBoss:
         if select_search_result(data.search_result):
             if current_layout_mode() is not 'flow':
                 snap_to_selection()
+        else:
+            self.send_message('search_result_not_found', search_result=data.search_result)
 
     def reference_item_changed(self, ref_num_or_none):
         self.send_message('reference_item_changed', refnum=ref_num_or_none, index=current_spine_item().index)

From f5e0c975e564216d7383888e876b86f55cc8711f Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Fri, 28 Feb 2020 20:30:07 +0530
Subject: [PATCH 105/162] Update Wired

---
 recipes/wired.recipe       | 15 +++++++++++++++
 recipes/wired_daily.recipe | 15 +++++++++++++++
 2 files changed, 30 insertions(+)

diff --git a/recipes/wired.recipe b/recipes/wired.recipe
index 1504fd6d99..e9a9a4454c 100644
--- a/recipes/wired.recipe
+++ b/recipes/wired.recipe
@@ -4,6 +4,7 @@ __copyright__ = '2014, Darko Miletic '
 www.wired.com
 '''
 
+from calibre import browser
 from calibre.web.feeds.news import BasicNewsRecipe
 
 
@@ -80,3 +81,17 @@ class WiredDailyNews(BasicNewsRecipe):
             articles.extend(self.parse_wired_index_page(baseurl.format(pagenum), seen))
 
         return [('Magazine Articles', articles)]
+
+    # Wired changes the content it delivers based on cookies, so the
+    # following ensures that we send no cookies
+    def get_browser(self, *args, **kwargs):
+        return self
+
+    def clone_browser(self, *args, **kwargs):
+        return self.get_browser()
+
+    def open_novisit(self, *args, **kwargs):
+        br = browser()
+        return br.open_novisit(*args, **kwargs)
+
+    open = open_novisit
diff --git a/recipes/wired_daily.recipe b/recipes/wired_daily.recipe
index 62d7df23cd..42d0357a98 100644
--- a/recipes/wired_daily.recipe
+++ b/recipes/wired_daily.recipe
@@ -4,6 +4,7 @@ __copyright__ = '2014, Darko Miletic '
 www.wired.com
 '''
 
+from calibre import browser
 from calibre.web.feeds.news import BasicNewsRecipe
 
 
@@ -66,3 +67,17 @@ class WiredDailyNews(BasicNewsRecipe):
 
     def get_article_url(self, article):
         return article.get('link',  None)
+
+    # Wired changes the content it delivers based on cookies, so the
+    # following ensures that we send no cookies
+    def get_browser(self, *args, **kwargs):
+        return self
+
+    def clone_browser(self, *args, **kwargs):
+        return self.get_browser()
+
+    def open_novisit(self, *args, **kwargs):
+        br = browser()
+        return br.open_novisit(*args, **kwargs)
+
+    open = open_novisit

From 0f8c8d71a100fc892213a8718bf4350c6850b566 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Sat, 29 Feb 2020 09:05:47 +0530
Subject: [PATCH 106/162] String changes

---
 manual/news.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/manual/news.rst b/manual/news.rst
index 630d2eb171..a6f3f999ef 100644
--- a/manual/news.rst
+++ b/manual/news.rst
@@ -40,7 +40,7 @@ and then the :guilabel:`Add a custom news source` menu item and then the
 .. image:: images/custom_news.png
     :align: center
 
-First enter ``calibre Blog`` into the :guilabel:`Recipe title` field. This will be the title of the e-book that will be created from the articles in the above feeds.
+First enter ``Calibre Blog`` into the :guilabel:`Recipe title` field. This will be the title of the e-book that will be created from the articles in the above feeds.
 
 The next two fields (:guilabel:`Oldest article` and :guilabel:`Max. number of articles`) allow you some control over how many articles should be downloaded from each feed, and they are pretty self explanatory.
 

From 3453380fe0cfdd896bb0c29ec23628308afc58c3 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Sat, 29 Feb 2020 11:10:27 +0530
Subject: [PATCH 107/162] Fix incorrect titles on book list prefs panels

---
 src/pyj/book_list/prefs.pyj | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/pyj/book_list/prefs.pyj b/src/pyj/book_list/prefs.pyj
index 750dfb5425..1c4a7ff0d5 100644
--- a/src/pyj/book_list/prefs.pyj
+++ b/src/pyj/book_list/prefs.pyj
@@ -232,8 +232,10 @@ def prefs_panel_handler(title, get_prefs_data, on_close=None, icon='close'):
             on_close()
         back()
 
-    return def init_prefs_panel(container_id):  # noqa:unused-local
+    def init_prefs_panel(container_id):
         container = document.getElementById(container_id)
-        create_top_bar(container, title, action=close_action, icon=icon)
+        create_top_bar(container, title=title, action=close_action, icon=icon)
         container.appendChild(E.div())
         create_prefs_widget(container.lastChild, get_prefs_data())
+
+    return init_prefs_panel

From eddc3254bdcb8e1d82907bf1cf6433aa187b442f Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Sat, 29 Feb 2020 11:12:01 +0530
Subject: [PATCH 108/162] Fix #1865213 [[Enhancement] Configure Tag browser on
 Content server](https://bugs.launchpad.net/calibre/+bug/1865213)

---
 src/pyj/book_list/search.pyj | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/pyj/book_list/search.pyj b/src/pyj/book_list/search.pyj
index 250c015c51..071d9bc04f 100644
--- a/src/pyj/book_list/search.pyj
+++ b/src/pyj/book_list/search.pyj
@@ -466,7 +466,7 @@ def get_prefs():
 
         {
             'name':'partition_method',
-            'text':_('Tag browser category partitioning method'),
+            'text':_('Category partitioning method'),
             'choices':[('first letter', _('First Letter')), ('disable', _('Disable')), ('partition', _('Partition'))],
             'tooltip':_('Choose how Tag browser subcategories are displayed when'
                         ' there are more items than the limit. Select by first'

From a533694b65c66e0daf0e270c6b3a3bdf8de0af5d Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Sat, 29 Feb 2020 11:48:25 +0530
Subject: [PATCH 109/162] Viewer: When starting without a book allowing
 quitting the viewer by clicking the close button on the "Open book" page

Fixes #1864343 [Can't close window when E-book viewer is opened in full screen mode](https://bugs.launchpad.net/calibre/+bug/1864343)
---
 src/pyj/read_book/overlay.pyj | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/src/pyj/read_book/overlay.pyj b/src/pyj/read_book/overlay.pyj
index 7d36e29f78..84d6a91f8e 100644
--- a/src/pyj/read_book/overlay.pyj
+++ b/src/pyj/read_book/overlay.pyj
@@ -481,20 +481,30 @@ class OpenBook: # {{{
     def __init__(self, overlay, closeable):
         self.overlay = overlay
         self.closeable = closeable
-        self.is_not_escapable = not closeable  # prevent Esc key from closing
+
+    def handle_escape(self):
+        if self.closeable:
+            self.overlay.hide_current_panel()
+        else:
+            ui_operations.quit()
 
     def on_container_click(self, evt):
         pass  # Dont allow panel to be closed by a click
 
     def show(self, container):
         container.style.backgroundColor = get_color('window-background')
-        close_button_style = '' if self.closeable else 'display: none'
         container.appendChild(E.div(
             style='padding: 1ex 1em; border-bottom: solid 1px currentColor; display:flex; justify-content: space-between',
                 E.h2(_('Open a new book')),
                 E.div(
-                    svgicon('close'), style=f'cursor:pointer; {close_button_style}',
-                    onclick=def(event):event.preventDefault(), event.stopPropagation(), self.overlay.hide_current_panel(event);,
+                    svgicon('close'), style=f'cursor:pointer',
+                    onclick=def(event):
+                        event.preventDefault(), event.stopPropagation()
+                        if self.closeable:
+                            self.overlay.hide_current_panel(event)
+                        else:
+                            ui_operations.quit()
+                    ,
                     class_='simple-link'),
             ))
         create_open_book(container, self.overlay.view?.book)

From 48fb202737d3a1efe01dc614c2ec4228322faa8e Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Sun, 1 Mar 2020 17:16:23 +0530
Subject: [PATCH 110/162] Update link to CI workflow

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index c72a26ff6e..d25f07e4b0 100644
--- a/README.md
+++ b/README.md
@@ -10,7 +10,7 @@ reading. It is cross platform, running on Linux, Windows and macOS.
 
 For more information, see the [calibre About page](https://calibre-ebook.com/about)
 
-[![Build Status](https://github.com/kovidgoyal/calibre/workflows/Continuous%20Integration/badge.svg)](https://github.com/kovidgoyal/calibre/actions?workflow=Continuous+Integration)
+[![Build Status](https://github.com/kovidgoyal/calibre/workflows/Continuous%20Integration/badge.svg)](https://github.com/kovidgoyal/calibre/actions?query=workflow%3ACI)
 
 ## Screenshots  
 

From c3f906ea84c49234fe786e3960ccebf3880c6111 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Mon, 2 Mar 2020 07:46:29 +0530
Subject: [PATCH 111/162] Fix #1864350 [Copy all in the Book details window
 doesn't copy all](https://bugs.launchpad.net/calibre/+bug/1864350)

---
 src/calibre/gui2/book_details.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/calibre/gui2/book_details.py b/src/calibre/gui2/book_details.py
index 30c6471eed..1e89b26da9 100644
--- a/src/calibre/gui2/book_details.py
+++ b/src/calibre/gui2/book_details.py
@@ -279,7 +279,7 @@ def add_item_specific_entries(menu, data, book_info):
 def details_context_menu_event(view, ev, book_info):
     url = view.anchorAt(ev.pos())
     menu = view.createStandardContextMenu()
-    menu.addAction(QIcon(I('edit-copy.png')), _('Copy &all'), partial(copy_all, book_info))
+    menu.addAction(QIcon(I('edit-copy.png')), _('Copy &all'), partial(copy_all, view))
     search_internet_added = False
     if url and url.startswith('action:'):
         data = json_loads(from_hex_bytes(url.split(':', 1)[1]))

From 2c19714c4057942e8857579e979d87ddd6c35c69 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Mon, 2 Mar 2020 08:21:55 +0530
Subject: [PATCH 112/162] Automatically extract the source DOCX file from
 Kindle Create KPF files when adding them to calibre. If you prefer to
 preserve the KPF file you can disable the KPF Extract plugin in
 Preferences->Plugins

---
 src/calibre/customize/builtins.py      |  4 ++--
 src/calibre/ebooks/__init__.py         |  2 +-
 src/calibre/ebooks/metadata/archive.py | 23 +++++++++++++++++++++++
 3 files changed, 26 insertions(+), 3 deletions(-)

diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py
index a855be92fa..44880cbc0e 100644
--- a/src/calibre/customize/builtins.py
+++ b/src/calibre/customize/builtins.py
@@ -9,7 +9,7 @@ from calibre import guess_type
 from calibre.customize import (FileTypePlugin, MetadataReaderPlugin,
     MetadataWriterPlugin, PreferencesPlugin, InterfaceActionBase, StoreBase)
 from calibre.constants import numeric_version
-from calibre.ebooks.metadata.archive import ArchiveExtract, get_comic_metadata
+from calibre.ebooks.metadata.archive import ArchiveExtract, KPFExtract, get_comic_metadata
 from calibre.ebooks.html.to_zip import HTML2ZIP
 
 plugins = []
@@ -124,7 +124,7 @@ class TXT2TXTZ(FileTypePlugin):
             return path_to_ebook
 
 
-plugins += [HTML2ZIP, PML2PMLZ, TXT2TXTZ, ArchiveExtract,]
+plugins += [HTML2ZIP, PML2PMLZ, TXT2TXTZ, ArchiveExtract, KPFExtract]
 # }}}
 
 # Metadata reader plugins {{{
diff --git a/src/calibre/ebooks/__init__.py b/src/calibre/ebooks/__init__.py
index c240ded56c..a88f7519f6 100644
--- a/src/calibre/ebooks/__init__.py
+++ b/src/calibre/ebooks/__init__.py
@@ -38,7 +38,7 @@ BOOK_EXTENSIONS = ['lrf', 'rar', 'zip', 'rtf', 'lit', 'txt', 'txtz', 'text', 'ht
                    'epub', 'fb2', 'fbz', 'djv', 'djvu', 'lrx', 'cbr', 'cbz', 'cbc', 'oebzip',
                    'rb', 'imp', 'odt', 'chm', 'tpz', 'azw1', 'pml', 'pmlz', 'mbp', 'tan', 'snb',
                    'xps', 'oxps', 'azw4', 'book', 'zbf', 'pobi', 'docx', 'docm', 'md',
-                   'textile', 'markdown', 'ibook', 'ibooks', 'iba', 'azw3', 'ps', 'kepub', 'kfx']
+                   'textile', 'markdown', 'ibook', 'ibooks', 'iba', 'azw3', 'ps', 'kepub', 'kfx', 'kpf']
 
 
 def return_raster_image(path):
diff --git a/src/calibre/ebooks/metadata/archive.py b/src/calibre/ebooks/metadata/archive.py
index 2bdc35e76d..23be91decc 100644
--- a/src/calibre/ebooks/metadata/archive.py
+++ b/src/calibre/ebooks/metadata/archive.py
@@ -40,6 +40,29 @@ def archive_type(stream):
     return ans
 
 
+class KPFExtract(FileTypePlugin):
+
+    name = 'KPF Extract'
+    author = 'Kovid Goyal'
+    description = _('Extract the source DOCX file from Amazon Kindle Create KPF files.'
+            ' Note this will not contain any edits made in the Kindle Create program itself.')
+    file_types = {'kpf'}
+    supported_platforms = ['windows', 'osx', 'linux']
+    on_import = True
+
+    def run(self, archive):
+        from calibre.utils.zipfile import ZipFile
+        with ZipFile(archive, 'r') as zf:
+            fnames = zf.namelist()
+            candidates = [x for x in fnames if x.lower().endswith('.docx')]
+            if not candidates:
+                return archive
+            of = self.temporary_file('_kpf_extract.docx')
+            with closing(of):
+                of.write(zf.read(candidates[0]))
+        return of.name
+
+
 class ArchiveExtract(FileTypePlugin):
     name = 'Archive Extract'
     author = 'Kovid Goyal'

From 59efc39317306132da080220e06499e0934d359d Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Mon, 2 Mar 2020 09:16:34 +0530
Subject: [PATCH 113/162] Content server: Add a button to delete all locally
 cached books. Fixes #1864305 [[Enhancement] Add a remove all books option in
 the downloaded books page on the
 Server](https://bugs.launchpad.net/calibre/+bug/1864305)

---
 src/pyj/book_list/local_books.pyj | 57 +++++++++++++++++++++++++++++--
 1 file changed, 54 insertions(+), 3 deletions(-)

diff --git a/src/pyj/book_list/local_books.pyj b/src/pyj/book_list/local_books.pyj
index c35dc0d852..35a6521449 100644
--- a/src/pyj/book_list/local_books.pyj
+++ b/src/pyj/book_list/local_books.pyj
@@ -7,12 +7,12 @@ from gettext import gettext as _
 
 from book_list.globals import get_db
 from book_list.router import home, open_book
-from book_list.top_bar import create_top_bar
+from book_list.top_bar import add_button, create_top_bar
 from book_list.ui import set_panel_handler
-from book_list.views import DEFAULT_MODE, setup_view_mode, get_view_mode
+from book_list.views import DEFAULT_MODE, get_view_mode, setup_view_mode
 from dom import clear, ensure_id
 from modals import create_custom_dialog, error_dialog
-from utils import conditional_timeout
+from utils import conditional_timeout, safe_set_inner_html
 from widgets import create_button
 
 CLASS_NAME = 'local-books-list'
@@ -49,6 +49,56 @@ def delete_book(book, book_idx):
     )
 
 
+def confirm_delete_all():
+    num_of_books = book_list_data.books.length
+    create_custom_dialog(_('Are you sure?'), def(parent, close_modal):
+
+        def action(doit):
+            if doit:
+                clear(parent)
+                delete_all(parent, close_modal)
+            else:
+                close_modal()
+
+        msg = _('This will remove all {} downloaded books from local storage. Are you sure?').format(num_of_books)
+        m = E.div()
+        safe_set_inner_html(m, msg)
+        parent.appendChild(E.div(
+            m,
+            E.div(class_='button-box',
+                create_button(_('OK'), None, action.bind(None, True)),
+                '\xa0',
+                create_button(_('Cancel'), None, action.bind(None, False), highlight=True),
+            )
+        ))
+    )
+
+
+
+def delete_all(msg_parent, close_modal):
+    db = get_db()
+    books = list(book_list_data.books)
+
+    def delete_one():
+        if not books.length:
+            close_modal()
+            show_recent_stage2.call(book_list_data.container_id, books)
+            return
+        clear(msg_parent)
+        safe_set_inner_html(msg_parent, _('Deleting {} books, please wait...').format(books.length))
+        book_to_delete = books.pop()
+        db.delete_book(book_list_data.book_data[book_to_delete], def(book, err_string):
+            if err_string:
+                close_modal()
+                show_recent_stage2.call(book_list_data.container_id, books)
+                error_dialog(_('Failed to delete book'), err_string)
+            else:
+                delete_one()
+    )
+    delete_one()
+
+
+
 def on_select(book, book_idx):
     title = this
 
@@ -158,6 +208,7 @@ def show_recent():
 def init(container_id):
     container = document.getElementById(container_id)
     create_top_bar(container, title=_('Downloaded books'), action=home, icon='home')
+    add_button(container, 'trash', confirm_delete_all, _('Delete all downloaded books'))
     # book list
     recent = E.div(class_=CLASS_NAME)
     recent_container_id = ensure_id(recent)

From ae109d8f91f2de6bc4ea6756eee746cb97aca37c Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Mon, 2 Mar 2020 09:29:39 +0530
Subject: [PATCH 114/162] Fix delete all handling

---
 src/pyj/book_list/local_books.pyj | 29 ++++++++++++++++++-----------
 1 file changed, 18 insertions(+), 11 deletions(-)

diff --git a/src/pyj/book_list/local_books.pyj b/src/pyj/book_list/local_books.pyj
index 35a6521449..c815b22483 100644
--- a/src/pyj/book_list/local_books.pyj
+++ b/src/pyj/book_list/local_books.pyj
@@ -50,7 +50,9 @@ def delete_book(book, book_idx):
 
 
 def confirm_delete_all():
-    num_of_books = book_list_data.books.length
+    num_of_books = book_list_data.books?.length
+    if not num_of_books:
+        return
     create_custom_dialog(_('Are you sure?'), def(parent, close_modal):
 
         def action(doit):
@@ -79,10 +81,13 @@ def delete_all(msg_parent, close_modal):
     db = get_db()
     books = list(book_list_data.books)
 
+    def refresh():
+        show_recent_stage2.call(book_list_data.container_id, [book_list_data.book_data[i] for i in books])
+
     def delete_one():
         if not books.length:
             close_modal()
-            show_recent_stage2.call(book_list_data.container_id, books)
+            refresh()
             return
         clear(msg_parent)
         safe_set_inner_html(msg_parent, _('Deleting {} books, please wait...').format(books.length))
@@ -90,7 +95,7 @@ def delete_all(msg_parent, close_modal):
         db.delete_book(book_list_data.book_data[book_to_delete], def(book, err_string):
             if err_string:
                 close_modal()
-                show_recent_stage2.call(book_list_data.container_id, books)
+                refresh()
                 error_dialog(_('Failed to delete book'), err_string)
             else:
                 delete_one()
@@ -171,27 +176,29 @@ def apply_view_mode(mode):
 
 
 def create_books_list(container, books):
+    clear(container)
     book_list_data.container_id = ensure_id(container)
     book_list_data.book_data = {i:book for i, book in enumerate(books)}
     book_list_data.books = list(range(books.length))
     book_list_data.mode = None
     book_list_data.thumbnail_cache = {}
-    container.appendChild(E.div(data_component='book_list'))
-    apply_view_mode(get_view_mode())
-
-def show_recent_stage2(books):
-    container = document.getElementById(this)
-    if not container:
-        return
-    clear(container)
     if not books.length:
         container.appendChild(E.div(
             style='margin: 1rem 1rem',
             _('No downloaded books present')
         ))
+    else:
+        container.appendChild(E.div(data_component='book_list'))
+        apply_view_mode(get_view_mode())
+
+
+def show_recent_stage2(books):
+    container = document.getElementById(this)
+    if not container:
         return
     create_books_list(container, books)
 
+
 def show_recent():
     container = this
     db = get_db()

From 6c29207c878ee85b540bdbeaec3120caa5557956 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Mon, 2 Mar 2020 10:21:03 +0530
Subject: [PATCH 115/162] Rather than remove matches for hidden text, simply
 mark them as not visible

---
 src/calibre/gui2/viewer/search.py | 66 +++++++++++--------------------
 1 file changed, 24 insertions(+), 42 deletions(-)

diff --git a/src/calibre/gui2/viewer/search.py b/src/calibre/gui2/viewer/search.py
index c33a1d3646..05d3f92150 100644
--- a/src/calibre/gui2/viewer/search.py
+++ b/src/calibre/gui2/viewer/search.py
@@ -10,9 +10,9 @@ from threading import Thread
 
 import regex
 from PyQt5.Qt import (
-    QCheckBox, QComboBox, QHBoxLayout, QIcon, QLabel, QListWidget, QListWidgetItem,
-    QStaticText, QStyle, QStyledItemDelegate, Qt, QToolButton, QVBoxLayout, QWidget,
-    pyqtSignal
+    QCheckBox, QComboBox, QHBoxLayout, QIcon, QLabel, QListWidget,
+    QListWidgetItem, QStaticText, QStyle, QStyledItemDelegate, Qt, QToolButton,
+    QVBoxLayout, QWidget, pyqtSignal
 )
 
 from calibre.ebooks.conversion.search_replace import REGEX_FLAGS
@@ -20,7 +20,6 @@ from calibre.gui2 import warning_dialog
 from calibre.gui2.progress_indicator import ProgressIndicator
 from calibre.gui2.viewer.web_view import get_data, get_manifest, vprefs
 from calibre.gui2.widgets2 import HistoryComboBox
-from calibre.utils.monotonic import monotonic
 from polyglot.builtins import iteritems, unicode_type
 from polyglot.functools import lru_cache
 from polyglot.queue import Queue
@@ -116,7 +115,7 @@ class SearchFinished(object):
 
 class SearchResult(object):
 
-    __slots__ = ('search_query', 'before', 'text', 'after', 'q', 'spine_idx', 'index', 'file_name', '_static_text')
+    __slots__ = ('search_query', 'before', 'text', 'after', 'q', 'spine_idx', 'index', 'file_name', '_static_text', 'is_hidden')
 
     def __init__(self, search_query, before, text, after, q, name, spine_idx, index):
         self.search_query = search_query
@@ -125,6 +124,7 @@ class SearchResult(object):
         self.spine_idx, self.index = spine_idx, index
         self.file_name = name
         self._static_text = None
+        self.is_hidden = False
 
     @property
     def static_text(self):
@@ -148,11 +148,11 @@ class SearchResult(object):
     def for_js(self):
         return {
             'file_name': self.file_name, 'spine_idx': self.spine_idx, 'index': self.index, 'text': self.text,
-            'before': self.before, 'after': self.after, 'mode': self.search_query.mode
+            'before': self.before, 'after': self.after, 'mode': self.search_query.mode, 'q': self.q
         }
 
     def is_result(self, result_from_js):
-        return result_from_js['spine_idx'] == self.spine_idx and self.index == result_from_js['index'] and result_from_js['text'] == self.text
+        return result_from_js['spine_idx'] == self.spine_idx and self.index == result_from_js['index'] and result_from_js['q'] == self.q
 
     def __str__(self):
         from collections import namedtuple
@@ -344,8 +344,14 @@ class ResultsDelegate(QStyledItemDelegate):  # {{{
         c = p.color(group, c)
         painter.setClipRect(option.rect)
         painter.setPen(c)
+        height = result.static_text.size().height()
+        tl = option.rect.topLeft()
+        x, y = tl.x(), tl.y()
+        y += (option.rect.height() - height) // 2
+        if result.is_hidden:
+            x += option.decorationSize.width() + 4
         try:
-            painter.drawStaticText(option.rect.topLeft(), result.static_text)
+            painter.drawStaticText(x, y, result.static_text)
         except Exception:
             import traceback
             traceback.print_exc()
@@ -360,21 +366,23 @@ class Results(QListWidget):  # {{{
     def __init__(self, parent=None):
         QListWidget.__init__(self, parent)
         self.setFocusPolicy(Qt.NoFocus)
-        self.setStyleSheet('QListWidget::item { padding: 3px; }')
         self.delegate = ResultsDelegate(self)
         self.setItemDelegate(self.delegate)
         self.itemClicked.connect(self.item_activated)
+        self.blank_icon = QIcon(I('blank.png'))
 
     def add_result(self, result):
         i = QListWidgetItem(' ', self)
         i.setData(Qt.UserRole, result)
+        i.setIcon(self.blank_icon)
         return self.count()
 
     def item_activated(self):
         i = self.currentItem()
         if i:
             sr = i.data(Qt.UserRole)
-            self.show_search_result.emit(sr)
+            if not sr.is_hidden:
+                self.show_search_result.emit(sr)
 
     def find_next(self, previous):
         if self.count() < 1:
@@ -386,24 +394,14 @@ class Results(QListWidget):  # {{{
         self.item_activated()
 
     def search_result_not_found(self, sr):
-        remove = None
         for i in range(self.count()):
             item = self.item(i)
             r = item.data(Qt.UserRole)
             if r.is_result(sr):
-                remove = i
-        if remove is not None:
-            q = sr['spine_idx']
-            for i in range(remove + 1, self.count()):
-                item = self.item(i)
-                r = item.data(Qt.UserRole)
-                if r.spine_index != q:
-                    break
-                r.index -= 1
-            self.takeItem(remove)
-            if self.count():
-                self.setCurrentRow(min(remove, self.count()-1))
-                self.item_activated()
+                r.is_hidden = True
+                item.setToolTip(_('This text is hidden in the book, so cannot be displayed'))
+                item.setIcon(QIcon(I('dialog_warning.png')))
+                break
 # }}}
 
 
@@ -520,24 +518,8 @@ class SearchPanel(QWidget):  # {{{
 
     def search_result_not_found(self, sr):
         self.results.search_result_not_found(sr)
-        if self.results.count():
-            now = monotonic()
-            if self.last_hidden_text_warning is None or self.current_search != self.last_hidden_text_warning[1] or now - self.last_hidden_text_warning[0] > 5:
-                self.last_hidden_text_warning = now, self.current_search
-                warning_dialog(self, _('Hidden text'), _(
-                    'Some search results were for hidden or non-reflowable text, they will be removed.'), show=True)
-            elif self.last_hidden_text_warning is not None:
-                self.last_hidden_text_warning = now, self.last_hidden_text_warning[1]
-
-        if not self.results.count() and not self.spinner.is_running:
-            self.show_no_results_found()
 
     def show_no_results_found(self):
-        has_hidden_text = self.last_hidden_text_warning is not None and self.last_hidden_text_warning[1] == self.current_search
-        if self.current_search:
-            if has_hidden_text:
-                msg = _('No displayable matches were found for:')
-            else:
-                msg = _('No matches were found for:')
-            warning_dialog(self, _('No matches found'), msg + '  {}'.format(self.current_search.text), show=True)
+        msg = _('No matches were found for:')
+        warning_dialog(self, _('No matches found'), msg + '  {}'.format(self.current_search.text), show=True)
 # }}}

From b41b0a99af195435e754d553eb959a973f952d8a Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Mon, 2 Mar 2020 10:36:12 +0530
Subject: [PATCH 116/162] Use a label rather than a tooltip for hidden text

---
 src/calibre/gui2/viewer/search.py | 20 +++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

diff --git a/src/calibre/gui2/viewer/search.py b/src/calibre/gui2/viewer/search.py
index 05d3f92150..957c991846 100644
--- a/src/calibre/gui2/viewer/search.py
+++ b/src/calibre/gui2/viewer/search.py
@@ -399,9 +399,15 @@ class Results(QListWidget):  # {{{
             r = item.data(Qt.UserRole)
             if r.is_result(sr):
                 r.is_hidden = True
-                item.setToolTip(_('This text is hidden in the book, so cannot be displayed'))
                 item.setIcon(QIcon(I('dialog_warning.png')))
                 break
+
+    @property
+    def current_result_is_hidden(self):
+        item = self.currentItem()
+        if item and item.data(Qt.UserRole) and item.data(Qt.UserRole).is_hidden:
+            return True
+        return False
 # }}}
 
 
@@ -425,10 +431,19 @@ class SearchPanel(QWidget):  # {{{
         l.addWidget(si)
         self.results = r = Results(self)
         r.show_search_result.connect(self.do_show_search_result, type=Qt.QueuedConnection)
+        r.currentRowChanged.connect(self.update_hidden_message)
         l.addWidget(r, 100)
         self.spinner = s = BusySpinner(self)
         s.setVisible(False)
         l.addWidget(s)
+        self.hidden_message = la = QLabel(_('This text is hidden in the book and cannot be displayed'))
+        la.setStyleSheet('QLabel { margin-left: 1ex }')
+        la.setWordWrap(True)
+        la.setVisible(False)
+        l.addWidget(la)
+
+    def update_hidden_message(self):
+        self.hidden_message.setVisible(self.results.current_result_is_hidden)
 
     def focus_input(self):
         self.search_input.focus_input()
@@ -442,6 +457,7 @@ class SearchPanel(QWidget):  # {{{
             self.searcher.daemon = True
             self.searcher.start()
         self.results.clear()
+        self.hidden_message.setVisible(False)
         self.spinner.start()
         self.current_search = search_query
         self.last_hidden_text_warning = None
@@ -491,6 +507,7 @@ class SearchPanel(QWidget):  # {{{
             # first result
             self.results.setCurrentRow(0)
             self.results.item_activated()
+        self.update_hidden_message()
 
     def visibility_changed(self, visible):
         if visible:
@@ -518,6 +535,7 @@ class SearchPanel(QWidget):  # {{{
 
     def search_result_not_found(self, sr):
         self.results.search_result_not_found(sr)
+        self.update_hidden_message()
 
     def show_no_results_found(self):
         msg = _('No matches were found for:')

From 6bdbc6f07d081be192db926f056c6512bc49ee4a Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Mon, 2 Mar 2020 11:17:39 +0530
Subject: [PATCH 117/162] Viewer: Fix Ctrl+i shortcut not working on Windows

---
 src/pyj/read_book/shortcuts.pyj | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/pyj/read_book/shortcuts.pyj b/src/pyj/read_book/shortcuts.pyj
index 8316be03a6..fdc6c93c28 100644
--- a/src/pyj/read_book/shortcuts.pyj
+++ b/src/pyj/read_book/shortcuts.pyj
@@ -51,10 +51,13 @@ def get_key_text(evt):
         cc = key.charCodeAt(0)
         # on windows in webengine pressing ctrl+ascii char gives us an ascii
         # control code
-        if (0 < cc < 32 or key is 'Enter') and evt.ctrlKey and not evt.metaKey and not evt.altKey:
+        if (0 < cc < 32 or key is 'Enter' or key is 'Tab') and evt.ctrlKey and not evt.metaKey and not evt.altKey:
             if key is 'Enter':
                 if evt.code and evt.code is not 'Enter':
                     key = 'm'
+            elif key is 'Tab':
+                if evt.code and evt.code is not 'Tab':
+                    key = 'i'
             else:
                 key = chr(96 + cc)
     return key

From 78f858a87596a883afe6c0610fce3426c76ed86d Mon Sep 17 00:00:00 2001
From: xcffl <--list>
Date: Mon, 2 Mar 2020 22:08:38 +0800
Subject: [PATCH 118/162] Fix & update Douban API

---
 src/calibre/ebooks/metadata/sources/douban.py | 300 +++++++++---------
 1 file changed, 152 insertions(+), 148 deletions(-)

diff --git a/src/calibre/ebooks/metadata/sources/douban.py b/src/calibre/ebooks/metadata/sources/douban.py
index bb044f7cca..67ec87ee49 100644
--- a/src/calibre/ebooks/metadata/sources/douban.py
+++ b/src/calibre/ebooks/metadata/sources/douban.py
@@ -3,7 +3,7 @@
 
 from __future__ import absolute_import, division, print_function, unicode_literals
 
-__license__   = 'GPL v3'
+__license__ = 'GPL v3'
 __copyright__ = '2011, Kovid Goyal ; 2011, Li Fanxi '
 __docformat__ = 'restructuredtext en'
 
@@ -14,27 +14,26 @@ try:
 except ImportError:
     from Queue import Empty, Queue
 
-
 from calibre.ebooks.metadata import check_isbn
 from calibre.ebooks.metadata.sources.base import Option, Source
 from calibre.ebooks.metadata.book.base import Metadata
 from calibre import as_unicode
 
 NAMESPACES = {
-              'openSearch':'http://a9.com/-/spec/opensearchrss/1.0/',
-              'atom' : 'http://www.w3.org/2005/Atom',
-              'db': 'https://www.douban.com/xmlns/',
-              'gd': 'http://schemas.google.com/g/2005'
-            }
+    'openSearch': 'http://a9.com/-/spec/opensearchrss/1.0/',
+    'atom': 'http://www.w3.org/2005/Atom',
+    'db': 'https://www.douban.com/xmlns/',
+    'gd': 'http://schemas.google.com/g/2005'
+}
 
 
 def get_details(browser, url, timeout):  # {{{
     try:
-        if Douban.DOUBAN_API_KEY and Douban.DOUBAN_API_KEY != '':
+        if Douban.DOUBAN_API_KEY:
             url = url + "?apikey=" + Douban.DOUBAN_API_KEY
         raw = browser.open_novisit(url, timeout=timeout).read()
     except Exception as e:
-        gc = getattr(e, 'getcode', lambda : -1)
+        gc = getattr(e, 'getcode', lambda: -1)
         if gc() != 403:
             raise
         # Douban is throttling us, wait a little
@@ -42,97 +41,73 @@ def get_details(browser, url, timeout):  # {{{
         raw = browser.open_novisit(url, timeout=timeout).read()
 
     return raw
+
+
 # }}}
 
 
 class Douban(Source):
 
     name = 'Douban Books'
-    author = 'Li Fanxi'
-    version = (2, 1, 2)
+    author = 'Li Fanxi, xcffl'
+    version = (3, 0, 0)
     minimum_calibre_version = (2, 80, 0)
 
-    description = _('Downloads metadata and covers from Douban.com. '
-            'Useful only for Chinese language books.')
+    description = _(
+        'Downloads metadata and covers from Douban.com. '
+        'Useful only for Chinese language books.'
+    )
 
     capabilities = frozenset(['identify', 'cover'])
-    touched_fields = frozenset(['title', 'authors', 'tags',
-        'pubdate', 'comments', 'publisher', 'identifier:isbn', 'rating',
-        'identifier:douban'])  # language currently disabled
+    touched_fields = frozenset([
+        'title', 'authors', 'tags', 'pubdate', 'comments', 'publisher',
+        'identifier:isbn', 'rating', 'identifier:douban'
+    ])  # language currently disabled
     supports_gzip_transfer_encoding = True
     cached_cover_url_is_reliable = True
 
-    DOUBAN_API_KEY = '0bd1672394eb1ebf2374356abec15c3d'
+    DOUBAN_API_KEY = '0df993c66c0c636e29ecbb5344252a4a'
+    DOUBAN_API_URL = 'https://api.douban.com/v2/book/search'
     DOUBAN_BOOK_URL = 'https://book.douban.com/subject/%s/'
 
     options = (
-        Option('include_subtitle_in_title', 'bool', True, _('Include subtitle in book title:'),
-               _('Whether to append subtitle in the book title.')),
+        Option(
+            'include_subtitle_in_title', 'bool', True,
+            _('Include subtitle in book title:'),
+            _('Whether to append subtitle in the book title.')
+        ),
     )
 
     def to_metadata(self, browser, log, entry_, timeout):  # {{{
-        from lxml import etree
-        from calibre.ebooks.chardet import xml_to_unicode
         from calibre.utils.date import parse_date, utcnow
-        from calibre.utils.cleantext import clean_ascii_chars
 
-        XPath = partial(etree.XPath, namespaces=NAMESPACES)
-        entry          = XPath('//atom:entry')
-        entry_id       = XPath('descendant::atom:id')
-        title          = XPath('descendant::atom:title')
-        description    = XPath('descendant::atom:summary')
-        subtitle       = XPath("descendant::db:attribute[@name='subtitle']")
-        publisher      = XPath("descendant::db:attribute[@name='publisher']")
-        isbn           = XPath("descendant::db:attribute[@name='isbn13']")
-        date           = XPath("descendant::db:attribute[@name='pubdate']")
-        creator        = XPath("descendant::db:attribute[@name='author']")
-        booktag        = XPath("descendant::db:tag/attribute::name")
-        rating         = XPath("descendant::gd:rating/attribute::average")
-        cover_url      = XPath("descendant::atom:link[@rel='image']/attribute::href")
+        douban_id = entry_.get('id')
+        title = entry_.get('title')
+        description = entry_.get('summary')
+        # subtitle = entry_.get('subtitle')  # TODO: std metada doesn't have this field
+        publisher = entry_.get('publisher')
+        isbns = entry_.get('isbn13')  # ISBN11 is obsolute, use ISBN13
+        pubdate = entry_.get('pubdate')
+        authors = entry_.get('author')
+        book_tags = entry_.get('tags')
+        rating = entry_.get('rating')
+        cover_url = entry_.get('image')
+        series = entry_.get('series')
 
-        def get_text(extra, x):
-            try:
-                ans = x(extra)
-                if ans:
-                    ans = ans[0].text
-                    if ans and ans.strip():
-                        return ans.strip()
-            except:
-                log.exception('Programming error:')
-            return None
-
-        id_url = entry_id(entry_)[0].text.replace('http://', 'https://')
-        douban_id = id_url.split('/')[-1]
-        title_ = ': '.join([x.text for x in title(entry_)]).strip()
-        subtitle = ': '.join([x.text for x in subtitle(entry_)]).strip()
-        if self.prefs['include_subtitle_in_title'] and len(subtitle) > 0:
-            title_ = title_ + ' - ' + subtitle
-        authors = [x.text.strip() for x in creator(entry_) if x.text]
         if not authors:
             authors = [_('Unknown')]
-        if not id_url or not title:
+        if not douban_id or not title:
             # Silently discard this entry
             return None
 
-        mi = Metadata(title_, authors)
-        mi.identifiers = {'douban':douban_id}
-        try:
-            log.info(id_url)
-            raw = get_details(browser, id_url, timeout)
-            feed = etree.fromstring(
-                xml_to_unicode(clean_ascii_chars(raw), strip_encoding_pats=True)[0],
-                parser=etree.XMLParser(recover=True, no_network=True, resolve_entities=False)
-            )
-            extra = entry(feed)[0]
-        except:
-            log.exception('Failed to get additional details for', mi.title)
-            return mi
-        mi.comments = get_text(extra, description)
-        mi.publisher = get_text(extra, publisher)
+        mi = Metadata(title, authors)
+        mi.identifiers = {'douban': douban_id}
+        mi.publisher = publisher
+        mi.comments = description
+        # mi.subtitle = subtitle
 
         # ISBN
-        isbns = []
-        for x in [t.text for t in isbn(extra)]:
+        for x in isbns:
             if check_isbn(x):
                 isbns.append(x)
         if isbns:
@@ -140,52 +115,45 @@ class Douban(Source):
         mi.all_isbns = isbns
 
         # Tags
-        try:
-            btags = [x for x in booktag(extra) if x]
-            tags = []
-            for t in btags:
-                atags = [y.strip() for y in t.split('/')]
-                for tag in atags:
-                    if tag not in tags:
-                        tags.append(tag)
-        except:
-            log.exception('Failed to parse tags:')
-            tags = []
-        if tags:
-            mi.tags = [x.replace(',', ';') for x in tags]
+        mi.tags = [tag['name'] for tag in book_tags]
 
         # pubdate
-        pubdate = get_text(extra, date)
         if pubdate:
             try:
                 default = utcnow().replace(day=15)
                 mi.pubdate = parse_date(pubdate, assume_utc=True, default=default)
             except:
-                log.error('Failed to parse pubdate %r'%pubdate)
+                log.error('Failed to parse pubdate %r' % pubdate)
 
         # Ratings
-        if rating(extra):
+        if rating:
             try:
-                mi.rating = float(rating(extra)[0]) / 2.0
+                mi.rating = float(rating['average']) / 2.0
             except:
                 log.exception('Failed to parse rating')
                 mi.rating = 0
 
         # Cover
         mi.has_douban_cover = None
-        u = cover_url(extra)
+        u = cover_url
         if u:
-            u = u[0].replace('/spic/', '/lpic/')
             # If URL contains "book-default", the book doesn't have a cover
             if u.find('book-default') == -1:
                 mi.has_douban_cover = u
+
+        # Series
+        if series:
+            mi.series = series['title']
+
         return mi
+
     # }}}
 
     def get_book_url(self, identifiers):  # {{{
         db = identifiers.get('douban', None)
         if db is not None:
-            return ('douban', db, self.DOUBAN_BOOK_URL%db)
+            return ('douban', db, self.DOUBAN_BOOK_URL % db)
+
     # }}}
 
     def create_query(self, log, title=None, authors=None, identifiers={}):  # {{{
@@ -193,9 +161,9 @@ class Douban(Source):
             from urllib.parse import urlencode
         except ImportError:
             from urllib import urlencode
-        SEARCH_URL = 'https://api.douban.com/book/subjects?'
-        ISBN_URL = 'https://api.douban.com/book/subject/isbn/'
-        SUBJECT_URL = 'https://api.douban.com/book/subject/'
+        SEARCH_URL = 'https://api.douban.com/v2/book/search?count=10&'
+        ISBN_URL = 'https://api.douban.com/v2/book/isbn/'
+        SUBJECT_URL = 'https://api.douban.com/v2/book/'
 
         q = ''
         t = None
@@ -208,16 +176,18 @@ class Douban(Source):
             q = subject
             t = 'subject'
         elif title or authors:
+
             def build_term(prefix, parts):
                 return ' '.join(x for x in parts)
+
             title_tokens = list(self.get_title_tokens(title))
             if title_tokens:
                 q += build_term('title', title_tokens)
-            author_tokens = list(self.get_author_tokens(authors,
-                    only_first_author=True))
+            author_tokens = list(
+                self.get_author_tokens(authors, only_first_author=True)
+            )
             if author_tokens:
-                q += ((' ' if q != '' else '') +
-                    build_term('author', author_tokens))
+                q += ((' ' if q != '' else '') + build_term('author', author_tokens))
             t = 'search'
         q = q.strip()
         if isinstance(q, type(u'')):
@@ -231,24 +201,40 @@ class Douban(Source):
             url = SUBJECT_URL + q
         else:
             url = SEARCH_URL + urlencode({
-                    'q': q,
-                    })
+                'q': q,
+            })
         if self.DOUBAN_API_KEY and self.DOUBAN_API_KEY != '':
             if t == "isbn" or t == "subject":
                 url = url + "?apikey=" + self.DOUBAN_API_KEY
             else:
                 url = url + "&apikey=" + self.DOUBAN_API_KEY
         return url
+
     # }}}
 
-    def download_cover(self, log, result_queue, abort,  # {{{
-            title=None, authors=None, identifiers={}, timeout=30, get_best_cover=False):
+    def download_cover(
+        self,
+        log,
+        result_queue,
+        abort,  # {{{
+        title=None,
+        authors=None,
+        identifiers={},
+        timeout=30,
+        get_best_cover=False
+    ):
         cached_url = self.get_cached_cover_url(identifiers)
         if cached_url is None:
             log.info('No cached cover found, running identify')
             rq = Queue()
-            self.identify(log, rq, abort, title=title, authors=authors,
-                    identifiers=identifiers)
+            self.identify(
+                log,
+                rq,
+                abort,
+                title=title,
+                authors=authors,
+                identifiers=identifiers
+            )
             if abort.is_set():
                 return
             results = []
@@ -257,8 +243,11 @@ class Douban(Source):
                     results.append(rq.get_nowait())
                 except Empty:
                     break
-            results.sort(key=self.identify_results_keygen(
-                title=title, authors=authors, identifiers=identifiers))
+            results.sort(
+                key=self.identify_results_keygen(
+                    title=title, authors=authors, identifiers=identifiers
+                )
+            )
             for mi in results:
                 cached_url = self.get_cached_cover_url(mi.identifiers)
                 if cached_url is not None:
@@ -291,11 +280,18 @@ class Douban(Source):
             url = self.cached_identifier_to_cover_url(db)
 
         return url
+
     # }}}
 
-    def get_all_details(self, br, log, entries, abort,  # {{{
-            result_queue, timeout):
-        from lxml import etree
+    def get_all_details(
+        self,
+        br,
+        log,
+        entries,
+        abort,  # {{{
+        result_queue,
+        timeout
+    ):
         for relevance, i in enumerate(entries):
             try:
                 ans = self.to_metadata(br, log, i, timeout)
@@ -305,29 +301,31 @@ class Douban(Source):
                     for isbn in getattr(ans, 'all_isbns', []):
                         self.cache_isbn_to_identifier(isbn, db)
                     if ans.has_douban_cover:
-                        self.cache_identifier_to_cover_url(db,
-                                ans.has_douban_cover)
+                        self.cache_identifier_to_cover_url(db, ans.has_douban_cover)
                     self.clean_downloaded_metadata(ans)
                     result_queue.put(ans)
             except:
-                log.exception(
-                    'Failed to get metadata for identify entry:',
-                    etree.tostring(i))
+                log.exception('Failed to get metadata for identify entry:', i)
             if abort.is_set():
                 break
+
     # }}}
 
-    def identify(self, log, result_queue, abort, title=None, authors=None,  # {{{
-            identifiers={}, timeout=30):
-        from lxml import etree
-        from calibre.ebooks.chardet import xml_to_unicode
-        from calibre.utils.cleantext import clean_ascii_chars
+    def identify(
+        self,
+        log,
+        result_queue,
+        abort,
+        title=None,
+        authors=None,  # {{{
+        identifiers={},
+        timeout=30
+    ):
+        import json
 
-        XPath = partial(etree.XPath, namespaces=NAMESPACES)
-        entry          = XPath('//atom:entry')
-
-        query = self.create_query(log, title=title, authors=authors,
-                identifiers=identifiers)
+        query = self.create_query(
+            log, title=title, authors=authors, identifiers=identifiers
+        )
         if not query:
             log.error('Insufficient metadata to construct query')
             return
@@ -335,45 +333,51 @@ class Douban(Source):
         try:
             raw = br.open_novisit(query, timeout=timeout).read()
         except Exception as e:
-            log.exception('Failed to make identify query: %r'%query)
+            log.exception('Failed to make identify query: %r' % query)
             return as_unicode(e)
         try:
-            parser = etree.XMLParser(recover=True, no_network=True)
-            feed = etree.fromstring(xml_to_unicode(clean_ascii_chars(raw),
-                strip_encoding_pats=True)[0], parser=parser)
-            entries = entry(feed)
+            entries = json.loads(raw)['books']
         except Exception as e:
             log.exception('Failed to parse identify results')
             return as_unicode(e)
         if not entries and identifiers and title and authors and \
                 not abort.is_set():
-            return self.identify(log, result_queue, abort, title=title,
-                    authors=authors, timeout=timeout)
-
+            return self.identify(
+                log,
+                result_queue,
+                abort,
+                title=title,
+                authors=authors,
+                timeout=timeout
+            )
         # There is no point running these queries in threads as douban
         # throttles requests returning 403 Forbidden errors
         self.get_all_details(br, log, entries, abort, result_queue, timeout)
 
         return None
+
     # }}}
 
 
 if __name__ == '__main__':  # tests {{{
     # To run these test use: calibre-debug -e src/calibre/ebooks/metadata/sources/douban.py
-    from calibre.ebooks.metadata.sources.test import (test_identify_plugin,
-            title_test, authors_test)
-    test_identify_plugin(Douban.name,
-        [
-            (
-                {'identifiers':{'isbn': '9787536692930'}, 'title':'三体',
-                    'authors':['刘慈欣']},
-                [title_test('三体', exact=True),
-                    authors_test(['刘慈欣'])]
-            ),
-
-            (
-                {'title': 'Linux内核修炼之道', 'authors':['任桥伟']},
-                [title_test('Linux内核修炼之道', exact=False)]
-            ),
-    ])
+    from calibre.ebooks.metadata.sources.test import (
+        test_identify_plugin, title_test, authors_test
+    )
+    test_identify_plugin(
+        Douban.name, [
+            ({
+                'identifiers': {
+                    'isbn': '9787536692930'
+                },
+                'title': '三体',
+                'authors': ['刘慈欣']
+            }, [title_test('三体', exact=True),
+                authors_test(['刘慈欣'])]),
+            ({
+                'title': 'Linux内核修炼之道',
+                'authors': ['任桥伟']
+            }, [title_test('Linux内核修炼之道', exact=False)]),
+        ]
+    )
 # }}}

From 348338f75cf74c077c77f04b785d0f16a8bdf892 Mon Sep 17 00:00:00 2001
From: xcffl <--list>
Date: Mon, 2 Mar 2020 22:48:17 +0800
Subject: [PATCH 119/162] Remove Chinese special chars in titles

---
 src/calibre/ebooks/metadata/sources/base.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/calibre/ebooks/metadata/sources/base.py b/src/calibre/ebooks/metadata/sources/base.py
index db52433514..bd7ed3d9a0 100644
--- a/src/calibre/ebooks/metadata/sources/base.py
+++ b/src/calibre/ebooks/metadata/sources/base.py
@@ -384,7 +384,7 @@ class Source(Plugin):
                 # Remove hyphens only if they have whitespace before them
                 (r'(\s-)', ' '),
                 # Replace other special chars with a space
-                (r'''[:,;!@$%^&*(){}.`~"\s\[\]/]''', ' '),
+                (r'''[:,;!@$%^&*(){}.`~"\s\[\]/]《》''', ' '),
             ]]
 
             for pat, repl in title_patterns:

From 3b4f584a91913c574d9e6effde9db209b92a4834 Mon Sep 17 00:00:00 2001
From: xcffl <--list>
Date: Tue, 3 Mar 2020 09:51:36 +0800
Subject: [PATCH 120/162] Improve tokenization of CJK author names

---
 src/calibre/ebooks/metadata/sources/base.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/calibre/ebooks/metadata/sources/base.py b/src/calibre/ebooks/metadata/sources/base.py
index bd7ed3d9a0..2d05d17814 100644
--- a/src/calibre/ebooks/metadata/sources/base.py
+++ b/src/calibre/ebooks/metadata/sources/base.py
@@ -343,8 +343,8 @@ class Source(Plugin):
 
         if authors:
             # Leave ' in there for Irish names
-            remove_pat = re.compile(r'[!@#$%^&*(){}`~"\s\[\]/]')
-            replace_pat = re.compile(r'[-+.:;,]')
+            remove_pat = re.compile(r'[!@#$%^&*()（）「」{}`~"\s\[\]/]')
+            replace_pat = re.compile(r'[-+.:;,，。；：]')
             if only_first_author:
                 authors = authors[:1]
             for au in authors:
@@ -384,7 +384,7 @@ class Source(Plugin):
                 # Remove hyphens only if they have whitespace before them
                 (r'(\s-)', ' '),
                 # Replace other special chars with a space
-                (r'''[:,;!@$%^&*(){}.`~"\s\[\]/]《》''', ' '),
+                (r'''[:,;!@$%^&*(){}.`~"\s\[\]/]《》「」“”''', ' '),
             ]]
 
             for pat, repl in title_patterns:

From 74fa1b7504d6a693a0f4d8bcbee2e9a683407435 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Tue, 3 Mar 2020 09:14:48 +0530
Subject: [PATCH 121/162] PDF Output: Add a hangcheck for loading HTML if there
 is no progress for sixty seconds abort

See #1865380 (Private bug)
---
 src/calibre/ebooks/pdf/html_writer.py | 20 +++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

diff --git a/src/calibre/ebooks/pdf/html_writer.py b/src/calibre/ebooks/pdf/html_writer.py
index 17db4bc616..79bb32750b 100644
--- a/src/calibre/ebooks/pdf/html_writer.py
+++ b/src/calibre/ebooks/pdf/html_writer.py
@@ -18,7 +18,7 @@ from operator import attrgetter, itemgetter
 
 from html5_parser import parse
 from PyQt5.Qt import (
-    QApplication, QMarginsF, QObject, QPageLayout, QTimer, QUrl, pyqtSignal
+    QApplication, QMarginsF, QObject, QPageLayout, Qt, QTimer, QUrl, pyqtSignal
 )
 from PyQt5.QtWebEngineCore import QWebEngineUrlRequestInterceptor
 from PyQt5.QtWebEngineWidgets import QWebEnginePage, QWebEngineProfile
@@ -39,6 +39,7 @@ from calibre.srv.render_book import check_for_maths
 from calibre.utils.fonts.sfnt.container import Sfnt, UnsupportedFont
 from calibre.utils.fonts.sfnt.merge import merge_truetype_fonts_for_pdf
 from calibre.utils.logging import default_log
+from calibre.utils.monotonic import monotonic
 from calibre.utils.podofo import (
     dedup_type3_fonts, get_podofo, remove_unused_fonts, set_metadata_implementation
 )
@@ -172,10 +173,26 @@ class Renderer(QWebEnginePage):
 
         self.titleChanged.connect(self.title_changed)
         self.loadStarted.connect(self.load_started)
+        self.loadProgress.connect(self.load_progress)
         self.loadFinished.connect(self.load_finished)
+        self.load_hang_check_timer = t = QTimer(self)
+        self.load_started_at = 0
+        t.setTimerType(Qt.VeryCoarseTimer)
+        t.setInterval(60 * 1000)
+        t.setSingleShot(True)
+        t.timeout.connect(self.on_load_hang)
 
     def load_started(self):
+        self.load_started_at = monotonic()
         self.load_complete = False
+        self.load_hang_check_timer.start()
+
+    def load_progress(self, amt):
+        self.load_hang_check_timer.start()
+
+    def on_load_hang(self):
+        self.log(self.log_prefix, 'Loading not complete after {} seconds, aborting.'.format(int(monotonic() - self.load_started_at)))
+        self.load_finished(False)
 
     def title_changed(self, title):
         if self.wait_for_title and title == self.wait_for_title and self.load_complete:
@@ -187,6 +204,7 @@ class Renderer(QWebEnginePage):
 
     def load_finished(self, ok):
         self.load_complete = True
+        self.load_hang_check_timer.stop()
         if not ok:
             self.working = False
             self.work_done.emit(self, 'Load of {} failed'.format(self.url().toString()))

From c2f3bc2dbb080fce24f51722afe82021e36a7707 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Tue, 3 Mar 2020 09:27:55 +0530
Subject: [PATCH 122/162] ...

---
 src/calibre/ebooks/pdf/html_writer.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/calibre/ebooks/pdf/html_writer.py b/src/calibre/ebooks/pdf/html_writer.py
index 79bb32750b..10411a9955 100644
--- a/src/calibre/ebooks/pdf/html_writer.py
+++ b/src/calibre/ebooks/pdf/html_writer.py
@@ -50,6 +50,7 @@ from polyglot.builtins import (
 from polyglot.urllib import urlparse
 
 OK, KILL_SIGNAL = range(0, 2)
+HANG_TIME = 60  # seconds
 # }}}
 
 
@@ -178,7 +179,7 @@ class Renderer(QWebEnginePage):
         self.load_hang_check_timer = t = QTimer(self)
         self.load_started_at = 0
         t.setTimerType(Qt.VeryCoarseTimer)
-        t.setInterval(60 * 1000)
+        t.setInterval(HANG_TIME * 1000)
         t.setSingleShot(True)
         t.timeout.connect(self.on_load_hang)
 

From 9e57c0a5cd6ddd856cf3da9732c855fe12143e57 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Tue, 3 Mar 2020 10:55:57 +0530
Subject: [PATCH 123/162] Fix #1863487 [Clicking "Stop auto scroll" doesn't
 work when about to change
 chapter](https://bugs.launchpad.net/calibre/+bug/1863487)

---
 src/pyj/read_book/flow_mode.pyj | 27 ++++++++++++++++-----------
 1 file changed, 16 insertions(+), 11 deletions(-)

diff --git a/src/pyj/read_book/flow_mode.pyj b/src/pyj/read_book/flow_mode.pyj
index a020064eb6..a7c1e8513f 100644
--- a/src/pyj/read_book/flow_mode.pyj
+++ b/src/pyj/read_book/flow_mode.pyj
@@ -142,7 +142,7 @@ def scroll_by_page(direction):
 
 
 def is_auto_scroll_active():
-    return scroll_animator.auto and scroll_animator.is_running()
+    return scroll_animator.is_active()
 
 
 def start_autoscroll():
@@ -150,9 +150,9 @@ def start_autoscroll():
 
 
 def toggle_autoscroll():
-    running = False
     if is_auto_scroll_active():
         cancel_scroll()
+        running = False
     else:
         start_autoscroll()
         running = True
@@ -237,9 +237,13 @@ class ScrollAnimator:
         self.animation_id = None
         self.auto = False
         self.auto_timer = None
+        self.paused = False
 
     def is_running(self):
-        return self.animation_id is not None
+        return self.animation_id is not None or self.auto_timer is not None
+
+    def is_active(self):
+        return self.is_running() and (self.auto or self.auto_timer is not None)
 
     def start(self, direction, auto):
         if self.wait:
@@ -247,7 +251,7 @@ class ScrollAnimator:
 
         now = window.performance.now()
         self.end_time = now + self.DURATION
-        self.stop_auto_timer()
+        self.stop_auto_spine_transition()
 
         if not self.is_running() or direction is not self.direction or auto is not self.auto:
             if self.auto and not auto:
@@ -299,13 +303,14 @@ class ScrollAnimator:
         if scroll_finished:
             self.pause()
             if opts.scroll_auto_boundary_delay >= 0:
-                self.auto_timer = setTimeout(def():
-                        self.auto_timer = None
-                        get_boss().send_message('next_spine_item', previous=self.direction is DIRECTION.Up)
-                    , opts.scroll_auto_boundary_delay * 1000)
+                self.auto_timer = setTimeout(self.request_next_spine_item, opts.scroll_auto_boundary_delay * 1000)
         else:
             self.animation_id = window.requestAnimationFrame(self.auto_scroll)
 
+    def request_next_spine_item(self):
+        self.auto_timer = None
+        get_boss().send_message('next_spine_item', previous=self.direction is DIRECTION.Up)
+
     def report(self):
         amt = window.pageYOffset - self.start_offset
         if abs(amt) > 0 and self.csi_idx is current_spine_item().index:
@@ -326,12 +331,13 @@ class ScrollAnimator:
             window.cancelAnimationFrame(self.animation_id)
             self.animation_id = None
             self.report()
-        self.stop_auto_timer()
+        self.stop_auto_spine_transition()
 
-    def stop_auto_timer(self):
+    def stop_auto_spine_transition(self):
         if self.auto_timer is not None:
             clearTimeout(self.auto_timer)
             self.auto_timer = None
+            self.paused = False
 
     def pause(self):
         if self.auto:
@@ -456,7 +462,6 @@ def auto_scroll_action(action):
     elif action is 'stop':
         if is_auto_scroll_active():
             toggle_autoscroll()
-        scroll_animator.stop_auto_timer()
     elif action is 'resume':
         auto_scroll_resume()
     return is_auto_scroll_active()

From 449182d7bd7c5128dc6b74796022612f6acdf33d Mon Sep 17 00:00:00 2001
From: David 
Date: Tue, 3 Mar 2020 21:35:41 +1100
Subject: [PATCH 124/162] Update for Kobo firmware and the Series Tab

The new firmware has a Series Tab which uses the seriesId in the database to group the books.
---
 src/calibre/devices/kobo/books.py  |  7 ++++
 src/calibre/devices/kobo/driver.py | 62 +++++++++++++++++++++++-------
 2 files changed, 55 insertions(+), 14 deletions(-)

diff --git a/src/calibre/devices/kobo/books.py b/src/calibre/devices/kobo/books.py
index 89ff9c8411..f3561807fe 100644
--- a/src/calibre/devices/kobo/books.py
+++ b/src/calibre/devices/kobo/books.py
@@ -70,6 +70,7 @@ class Book(Book_):
         self.can_put_on_shelves = True
         self.kobo_series        = None
         self.kobo_series_number = None  # Kobo stores the series number as string. And it can have a leading "#".
+        self.kobo_series_id     = None
         self.kobo_subtitle      = None
 
         if thumbnail_name is not None:
@@ -84,6 +85,10 @@ class Book(Book_):
         # If we don't have a content Id, we don't know what type it is.
         return self.contentID and self.contentID.startswith("file")
 
+    @property
+    def has_kobo_series(self):
+        return self.kobo_series is not None
+
     @property
     def is_purchased_kepub(self):
         return self.contentID and not self.contentID.startswith("file")
@@ -102,6 +107,8 @@ class Book(Book_):
             fmt('Content ID', self.contentID)
         if self.kobo_series:
             fmt('Kobo Series', self.kobo_series + ' #%s'%self.kobo_series_number)
+        if self.kobo_series_id:
+            fmt('Kobo Series ID', self.kobo_series_id)
         if self.kobo_subtitle:
             fmt('Subtitle', self.kobo_subtitle)
         if self.mime:
diff --git a/src/calibre/devices/kobo/driver.py b/src/calibre/devices/kobo/driver.py
index 50f6cdcaf4..c370074ae9 100644
--- a/src/calibre/devices/kobo/driver.py
+++ b/src/calibre/devices/kobo/driver.py
@@ -83,7 +83,7 @@ class KOBO(USBMS):
 
     dbversion = 0
     fwversion = (0,0,0)
-    supported_dbversion = 156
+    supported_dbversion = 158
     has_kepubs = False
 
     supported_platforms = ['windows', 'osx', 'linux']
@@ -1349,7 +1349,7 @@ class KOBOTOUCH(KOBO):
         ' Based on the existing Kobo driver by %s.') % KOBO.author
 #    icon        = I('devices/kobotouch.jpg')
 
-    supported_dbversion             = 157
+    supported_dbversion             = 158
     min_supported_dbversion         = 53
     min_dbversion_series            = 65
     min_dbversion_externalid        = 65
@@ -1357,11 +1357,12 @@ class KOBOTOUCH(KOBO):
     min_dbversion_images_on_sdcard  = 77
     min_dbversion_activity          = 77
     min_dbversion_keywords          = 82
+    min_dbversion_seriesid          = 136
 
     # Starting with firmware version 3.19.x, the last number appears to be is a
     # build number. A number will be recorded here but it can be safely ignored
     # when testing the firmware version.
-    max_supported_fwversion         = (4, 19, 14114)
+    max_supported_fwversion         = (4, 20, 14601)
     # The following document firwmare versions where new function or devices were added.
     # Not all are used, but this feels a good place to record it.
     min_fwversion_shelves           = (2, 0, 0)
@@ -1377,11 +1378,13 @@ class KOBOTOUCH(KOBO):
     min_librah20_fwversion          = (4, 16, 13337)  # "Reviewers" release.
     min_fwversion_epub_location     = (4, 17, 13651)  # ePub reading location without full contentid.
     min_fwversion_dropbox           = (4, 18, 13737)  # The Forma only at this point.
+    min_fwversion_serieslist        = (4, 20, 14601)  # Series list needs the SeriesID to be set.
 
     has_kepubs = True
 
     booklist_class = KTCollectionsBookList
     book_class = Book
+    kobo_series_dict = {}
 
     MAX_PATH_LEN = 185  # 250 - (len(" - N3_LIBRARY_SHELF.parsed") + len("F:\.kobo\images\"))
     KOBO_EXTRA_CSSFILE = 'kobo_extra.css'
@@ -1610,7 +1613,8 @@ class KOBOTOUCH(KOBO):
             bl_cache[b.lpath] = idx
 
         def update_booklist(prefix, path, ContentID, ContentType, MimeType, ImageID,
-                            title, authors, DateCreated, Description, Publisher, series, seriesnumber,
+                            title, authors, DateCreated, Description, Publisher,
+                            series, seriesnumber, SeriesID, SeriesNumberFloat,
                             ISBN, Language, Subtitle,
                             readstatus, expired, favouritesindex, accessibility, isdownloaded,
                             userid, bookshelves
@@ -1747,10 +1751,16 @@ class KOBOTOUCH(KOBO):
                     bl[idx].kobo_metadata       = kobo_metadata
                     bl[idx].kobo_series         = series
                     bl[idx].kobo_series_number  = seriesnumber
+                    bl[idx].kobo_series_id      = SeriesID
                     bl[idx].kobo_subtitle       = Subtitle
                     bl[idx].can_put_on_shelves  = allow_shelves
                     bl[idx].mime                = MimeType
 
+                    if not bl[idx].is_sideloaded and bl[idx].has_kobo_series and SeriesID is not None:
+                        if show_debug:
+                            debug_print('KoboTouch:update_booklist - Have purchased kepub with series, saving SeriesID=', SeriesID)
+                        self.kobo_series_dict[series] = SeriesID
+
                     if lpath in playlist_map:
                         bl[idx].device_collections  = playlist_map.get(lpath,[])
                         bl[idx].current_shelves     = bookshelves
@@ -1800,10 +1810,16 @@ class KOBOTOUCH(KOBO):
                     book.kobo_metadata      = kobo_metadata
                     book.kobo_series        = series
                     book.kobo_series_number = seriesnumber
+                    book.kobo_series_id     = SeriesID
                     book.kobo_subtitle      = Subtitle
                     book.can_put_on_shelves = allow_shelves
 #                    debug_print('KoboTouch:update_booklist - title=', title, 'book.device_collections', book.device_collections)
 
+                    if not book.is_sideloaded and book.has_kobo_series and SeriesID is not None:
+                        if show_debug:
+                            debug_print('KoboTouch:update_booklist - Have purchased kepub with series, saving SeriesID=', SeriesID)
+                        self.kobo_series_dict[series] = SeriesID
+
                     if bl.add_book(book, replace_metadata=False):
                         changed = True
                     if show_debug:
@@ -1863,6 +1879,10 @@ class KOBOTOUCH(KOBO):
                 columns += ", Series, SeriesNumber, ___UserID, ExternalId, Subtitle"
             else:
                 columns += ', null as Series, null as SeriesNumber, ___UserID, null as ExternalId, null as Subtitle'
+            if self.supports_series_list:
+                columns += ", SeriesID, SeriesNumberFloat"
+            else:
+                columns += ', null as SeriesID, null as SeriesNumberFloat'
 
             where_clause = ''
             if self.supports_kobo_archive() or self.supports_overdrive():
@@ -1957,7 +1977,8 @@ class KOBOTOUCH(KOBO):
                 prefix = self._card_a_prefix if oncard == 'carda' else self._main_prefix
                 changed = update_booklist(prefix, path, row['ContentID'], row['ContentType'], row['MimeType'], row['ImageId'],
                                           row['Title'], row['Attribution'], row['DateCreated'], row['Description'], row['Publisher'],
-                                          row['Series'], row['SeriesNumber'], row['ISBN'], row['Language'], row['Subtitle'],
+                                          row['Series'], row['SeriesNumber'], row['SeriesID'], row['SeriesNumberFloat'],
+                                          row['ISBN'], row['Language'], row['Subtitle'],
                                           row['ReadStatus'], row['___ExpirationStatus'],
                                           int(row['FavouritesIndex']), row['Accessibility'], row['IsDownloaded'],
                                           row['___UserID'], bookshelves
@@ -1972,6 +1993,7 @@ class KOBOTOUCH(KOBO):
                 self.dump_bookshelves(connection)
             else:
                 debug_print("KoboTouch:books - automatically managing metadata")
+            debug_print("KoboTouch:books - self.kobo_series_dict=", self.kobo_series_dict)
         # Remove books that are no longer in the filesystem. Cache contains
         # indices into the booklist if book not in filesystem, None otherwise
         # Do the operation in reverse order so indices remain valid
@@ -3127,21 +3149,29 @@ class KOBOTOUCH(KOBO):
                 kobo_series_number = None
             series_number_changed = not (kobo_series_number == newmi.series_index)
 
-        if series_changed or series_number_changed:
-            if newmi.series is not None:
-                new_series = newmi.series
-                try:
-                    new_series_number = "%g" % newmi.series_index
-                except:
-                    new_series_number = None
-            else:
-                new_series = None
+        if newmi.series is not None:
+            new_series = newmi.series
+            try:
+                new_series_number = "%g" % newmi.series_index
+            except:
                 new_series_number = None
+        else:
+            new_series = None
+            new_series_number = None
 
+        if series_changed or series_number_changed:
             update_values.append(new_series)
             set_clause += ', Series = ? '
             update_values.append(new_series_number)
             set_clause += ', SeriesNumber = ? '
+        if self.supports_series_list and book.is_sideloaded:
+            series_id = self.kobo_series_dict.get(new_series, new_series)
+            if not book.kobo_series_id == series_id or series_changed or series_number_changed: 
+                update_values.append(series_id)
+                set_clause += ', SeriesID = ? '
+                update_values.append(new_series_number)
+                set_clause += ', SeriesNumberFloat = ? '
+                debug_print("KoboTouch:set_core_metadata Setting SeriesID - new_series='%s', series_id='%s'" % (new_series, series_id))
 
         if not series_only:
             if not (newmi.title == kobo_metadata.title):
@@ -3537,6 +3567,10 @@ class KOBOTOUCH(KOBO):
     def supports_series(self):
         return self.dbversion >= self.min_dbversion_series
 
+    @property
+    def supports_series_list(self):
+        return self.dbversion >= self.min_dbversion_seriesid and self.fwversion >= self.min_fwversion_serieslist
+
     def supports_kobo_archive(self):
         return self.dbversion >= self.min_dbversion_archive
 

From 6c9d97f2bf7866814094519ee3b5ec5023914ba5 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Tue, 3 Mar 2020 17:47:26 +0530
Subject: [PATCH 125/162] pep8

---
 src/calibre/devices/kobo/driver.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/calibre/devices/kobo/driver.py b/src/calibre/devices/kobo/driver.py
index c370074ae9..dbe3b89c4b 100644
--- a/src/calibre/devices/kobo/driver.py
+++ b/src/calibre/devices/kobo/driver.py
@@ -3166,7 +3166,7 @@ class KOBOTOUCH(KOBO):
             set_clause += ', SeriesNumber = ? '
         if self.supports_series_list and book.is_sideloaded:
             series_id = self.kobo_series_dict.get(new_series, new_series)
-            if not book.kobo_series_id == series_id or series_changed or series_number_changed: 
+            if not book.kobo_series_id == series_id or series_changed or series_number_changed:
                 update_values.append(series_id)
                 set_clause += ', SeriesID = ? '
                 update_values.append(new_series_number)

From b94b60ac06ddea05cf5fe10b674928664cacb5bf Mon Sep 17 00:00:00 2001
From: Lewix Liu 
Date: Tue, 3 Mar 2020 05:31:18 -0800
Subject: [PATCH 126/162] Fix the Douban metadata download plugin

---
 src/calibre/ebooks/metadata/sources/douban.py | 20 ++++++++++++++-----
 1 file changed, 15 insertions(+), 5 deletions(-)

diff --git a/src/calibre/ebooks/metadata/sources/douban.py b/src/calibre/ebooks/metadata/sources/douban.py
index 28e9598366..a910074077 100644
--- a/src/calibre/ebooks/metadata/sources/douban.py
+++ b/src/calibre/ebooks/metadata/sources/douban.py
@@ -85,7 +85,7 @@ class Douban(Source):
         description = entry_.get('summary')
         # subtitle = entry_.get('subtitle')  # TODO: std metada doesn't have this field
         publisher = entry_.get('publisher')
-        isbns = entry_.get('isbn13')  # ISBN11 is obsolute, use ISBN13
+        isbn = entry_.get('isbn13')  # ISBN11 is obsolute, use ISBN13
         pubdate = entry_.get('pubdate')
         authors = entry_.get('author')
         book_tags = entry_.get('tags')
@@ -106,9 +106,14 @@ class Douban(Source):
         # mi.subtitle = subtitle
 
         # ISBN
-        for x in isbns:
-            if check_isbn(x):
-                isbns.append(x)
+        isbns = []
+        if isinstance(isbn, basestring):
+            if check_isbn(isbn):
+                isbns.append(isbn)
+        else:
+            for x in isbn:
+                if check_isbn(x):
+                    isbns.append(x)
         if isbns:
             mi.isbn = sorted(isbns, key=len)[-1]
         mi.all_isbns = isbns
@@ -335,10 +340,15 @@ class Douban(Source):
             log.exception('Failed to make identify query: %r' % query)
             return as_unicode(e)
         try:
-            entries = json.loads(raw)['books']
+            j = json.loads(raw)
         except Exception as e:
             log.exception('Failed to parse identify results')
             return as_unicode(e)
+        if j.has_key('books'):
+            entries = j['books']
+        else:
+            entries = []
+            entries.append(j)
         if not entries and identifiers and title and authors and \
                 not abort.is_set():
             return self.identify(

From fcc0624b742b5517f9057b61c6b979c8681eaea9 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Thu, 5 Mar 2020 07:13:19 +0530
Subject: [PATCH 127/162] Fix #1863438 [After adjusting the auto scroll speed I
 am jumped back in the book](https://bugs.launchpad.net/calibre/+bug/1863438)

---
 src/pyj/read_book/overlay.pyj | 2 +-
 src/pyj/read_book/view.pyj    | 4 ++++
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/pyj/read_book/overlay.pyj b/src/pyj/read_book/overlay.pyj
index 84d6a91f8e..17b7d25aa6 100644
--- a/src/pyj/read_book/overlay.pyj
+++ b/src/pyj/read_book/overlay.pyj
@@ -463,7 +463,7 @@ class PrefsOverlay: # {{{
     def on_hide(self):
         if self.changes_occurred:
             self.changes_occurred = False
-            ui_operations.redisplay_book()
+            self.overlay.view.preferences_changed()
 
 # }}}
 
diff --git a/src/pyj/read_book/view.pyj b/src/pyj/read_book/view.pyj
index 62d06c9b5c..b42f9269d9 100644
--- a/src/pyj/read_book/view.pyj
+++ b/src/pyj/read_book/view.pyj
@@ -757,6 +757,10 @@ class View:
             show_controls_help()
             sd.set('controls_help_shown_count', c + 1)
 
+    def preferences_changed(self):
+        ui_operations.update_url_state(True)
+        ui_operations.redisplay_book()
+
     def redisplay_book(self):
         # redisplay_book() is called when settings are changed
         sd = get_session_data()

From 20cba609fcda03aa1c21278869a816eaf843e2ee Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Thu, 5 Mar 2020 08:32:13 +0530
Subject: [PATCH 128/162] ...

---
 recipes/nytimes.recipe     | 1 +
 recipes/nytimes_sub.recipe | 1 +
 2 files changed, 2 insertions(+)

diff --git a/recipes/nytimes.recipe b/recipes/nytimes.recipe
index 7aa9893ba8..d2526678ce 100644
--- a/recipes/nytimes.recipe
+++ b/recipes/nytimes.recipe
@@ -90,6 +90,7 @@ class NewYorkTimes(BasicNewsRecipe):
     compress_news_images = True
     compress_news_images_auto_size = 5
     remove_attributes = ['style']
+    conversion_options = {'flow_size': 0}
 
     remove_tags = [
         dict(attrs={'aria-label':'tools'.split()}),
diff --git a/recipes/nytimes_sub.recipe b/recipes/nytimes_sub.recipe
index 9663e10218..c4647a5b69 100644
--- a/recipes/nytimes_sub.recipe
+++ b/recipes/nytimes_sub.recipe
@@ -90,6 +90,7 @@ class NewYorkTimes(BasicNewsRecipe):
     compress_news_images = True
     compress_news_images_auto_size = 5
     remove_attributes = ['style']
+    conversion_options = {'flow_size': 0}
 
     remove_tags = [
         dict(attrs={'aria-label':'tools'.split()}),

From 42d055e511f9d296f2f68cd508795f0024faf8f8 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Thu, 5 Mar 2020 08:58:47 +0530
Subject: [PATCH 129/162] Clicking on the current Virtual library in the
 dropdown menu now closes the virtual library. Fixes #1864229 [[Enhancement]
 Close VL when clicking on the VL in the VL
 menu](https://bugs.launchpad.net/calibre/+bug/1864229)

---
 src/calibre/gui2/search_restriction_mixin.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/calibre/gui2/search_restriction_mixin.py b/src/calibre/gui2/search_restriction_mixin.py
index 43a45b1bb4..ae8c95a261 100644
--- a/src/calibre/gui2/search_restriction_mixin.py
+++ b/src/calibre/gui2/search_restriction_mixin.py
@@ -421,8 +421,12 @@ class SearchRestrictionMixin(object):
 
         virt_libs = db.prefs.get('virtual_libraries', {})
         for vl in sorted(virt_libs.keys(), key=sort_key):
-            a = m.addAction(self.checked if vl == current_lib else self.empty, vl.replace('&', '&&'))
-            a.triggered.connect(partial(self.apply_virtual_library, library=vl))
+            is_current = vl == current_lib
+            a = m.addAction(self.checked if is_current else self.empty, vl.replace('&', '&&'))
+            if is_current:
+                a.triggered.connect(self.clear_vl.click)
+            else:
+                a.triggered.connect(partial(self.apply_virtual_library, library=vl))
 
     def virtual_library_menu_about_to_show(self):
         self.build_virtual_library_menu(self.virtual_library_menu)

From ed232acc8a9937c7a7e4390198464cbb6c6bbe1e Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Thu, 5 Mar 2020 09:00:41 +0530
Subject: [PATCH 130/162] Dont clear the additional restriction

---
 src/calibre/gui2/search_restriction_mixin.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/calibre/gui2/search_restriction_mixin.py b/src/calibre/gui2/search_restriction_mixin.py
index ae8c95a261..f73afb5b13 100644
--- a/src/calibre/gui2/search_restriction_mixin.py
+++ b/src/calibre/gui2/search_restriction_mixin.py
@@ -424,7 +424,7 @@ class SearchRestrictionMixin(object):
             is_current = vl == current_lib
             a = m.addAction(self.checked if is_current else self.empty, vl.replace('&', '&&'))
             if is_current:
-                a.triggered.connect(self.clear_vl.click)
+                a.triggered.connect(self.apply_virtual_library)
             else:
                 a.triggered.connect(partial(self.apply_virtual_library, library=vl))
 

From c4d6dd7dfe4db4adeb6891e89e2cb02098cac313 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Thu, 5 Mar 2020 09:38:08 +0530
Subject: [PATCH 131/162] Fix #1864273 [In the Customize Get books search make
 some columns centered](https://bugs.launchpad.net/calibre/+bug/1864273)

---
 .../gui2/store/config/chooser/models.py       | 46 +++++++++++++++----
 .../gui2/store/config/chooser/results_view.py |  4 +-
 2 files changed, 39 insertions(+), 11 deletions(-)

diff --git a/src/calibre/gui2/store/config/chooser/models.py b/src/calibre/gui2/store/config/chooser/models.py
index 365565f0a7..fe836e12ce 100644
--- a/src/calibre/gui2/store/config/chooser/models.py
+++ b/src/calibre/gui2/store/config/chooser/models.py
@@ -6,27 +6,51 @@ __license__ = 'GPL 3'
 __copyright__ = '2011, John Schember '
 __docformat__ = 'restructuredtext en'
 
-from PyQt5.Qt import (Qt, QAbstractItemModel, QIcon, QModelIndex, QSize)
 
-from calibre.customize.ui import is_disabled, disable_plugin, enable_plugin
-from calibre.db.search import _match, CONTAINS_MATCH, EQUALS_MATCH, REGEXP_MATCH
+from PyQt5.Qt import (
+    QAbstractItemModel, QIcon, QModelIndex, QStyledItemDelegate, Qt
+)
+
+from calibre import fit_image
+from calibre.customize.ui import disable_plugin, enable_plugin, is_disabled
+from calibre.db.search import CONTAINS_MATCH, EQUALS_MATCH, REGEXP_MATCH, _match
 from calibre.utils.config_base import prefs
 from calibre.utils.icu import sort_key
 from calibre.utils.search_query_parser import SearchQueryParser
-from polyglot.builtins import unicode_type, range
+from polyglot.builtins import range, unicode_type
+
+
+class Delegate(QStyledItemDelegate):
+
+    def paint(self, painter, option, index):
+        icon = index.data(Qt.DecorationRole)
+        if icon and not icon.isNull():
+            QStyledItemDelegate.paint(self, painter, option, QModelIndex())
+            pw, ph = option.rect.width(), option.rect.height()
+            scaled, w, h = fit_image(option.decorationSize.width(), option.decorationSize.height(), pw, ph)
+            r = option.rect
+            if pw > w:
+                x = (pw - w) // 2
+                r = r.adjusted(x, 0, -x, 0)
+            if ph > h:
+                y = (ph - h) // 2
+                r = r.adjusted(0, y, 0, -y)
+            painter.drawPixmap(r, icon.pixmap(w, h))
+        else:
+            QStyledItemDelegate.paint(self, painter, option, index)
 
 
 class Matches(QAbstractItemModel):
 
     HEADERS = [_('Enabled'), _('Name'), _('No DRM'), _('Headquarters'), _('Affiliate'), _('Formats')]
-    HTML_COLS = [1]
+    HTML_COLS = (1,)
+    CENTERED_COLUMNS = (2, 3, 4)
 
     def __init__(self, plugins):
         QAbstractItemModel.__init__(self)
 
         self.NO_DRM_ICON = QIcon(I('ok.png'))
-        self.DONATE_ICON = QIcon()
-        self.DONATE_ICON.addFile(I('donate.png'), QSize(16, 16))
+        self.DONATE_ICON = QIcon(I('donate.png'))
 
         self.all_matches = plugins
         self.matches = plugins
@@ -123,6 +147,10 @@ class Matches(QAbstractItemModel):
                 if is_disabled(result):
                     return Qt.Unchecked
                 return Qt.Checked
+        elif role == Qt.TextAlignmentRole:
+            if col in self.CENTERED_COLUMNS:
+                return Qt.AlignHCenter
+            return Qt.AlignLeft
         elif role == Qt.ToolTipRole:
             if col == 0:
                 if is_disabled(result):
@@ -182,9 +210,7 @@ class Matches(QAbstractItemModel):
         if not self.matches:
             return
         descending = order == Qt.DescendingOrder
-        self.matches.sort(None,
-            lambda x: sort_key(unicode_type(self.data_as_text(x, col))),
-            descending)
+        self.matches.sort(key=lambda x: sort_key(unicode_type(self.data_as_text(x, col))), reverse=descending)
         if reset:
             self.beginResetModel(), self.endResetModel()
 
diff --git a/src/calibre/gui2/store/config/chooser/results_view.py b/src/calibre/gui2/store/config/chooser/results_view.py
index 432a6b6448..208d23e64d 100644
--- a/src/calibre/gui2/store/config/chooser/results_view.py
+++ b/src/calibre/gui2/store/config/chooser/results_view.py
@@ -12,7 +12,7 @@ from PyQt5.Qt import (Qt, QTreeView, QSize, QMenu)
 
 from calibre.customize.ui import store_plugins
 from calibre.gui2.metadata.single_download import RichTextDelegate
-from calibre.gui2.store.config.chooser.models import Matches
+from calibre.gui2.store.config.chooser.models import Matches, Delegate
 from polyglot.builtins import range
 
 
@@ -27,6 +27,8 @@ class ResultsView(QTreeView):
         self.setIconSize(QSize(24, 24))
 
         self.rt_delegate = RichTextDelegate(self)
+        self.delegate = Delegate()
+        self.setItemDelegate(self.delegate)
 
         for i in self._model.HTML_COLS:
             self.setItemDelegateForColumn(i, self.rt_delegate)

From 2742fa1f96a8df3ba2ea81c29312396808cf0cb0 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Thu, 5 Mar 2020 09:40:29 +0530
Subject: [PATCH 132/162] Fix #1860597 [[Enhancement] Make the list of tweaks
 better so that you don't need to scroll to read
 it](https://bugs.launchpad.net/calibre/+bug/1860597)

---
 src/calibre/gui2/preferences/tweaks.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/calibre/gui2/preferences/tweaks.py b/src/calibre/gui2/preferences/tweaks.py
index 3ff445f6ee..a63c88b667 100644
--- a/src/calibre/gui2/preferences/tweaks.py
+++ b/src/calibre/gui2/preferences/tweaks.py
@@ -372,6 +372,7 @@ class TweaksView(QListView):
         self.setAlternatingRowColors(True)
         self.setSpacing(5)
         self.setVerticalScrollMode(self.ScrollPerPixel)
+        self.setMinimumWidth(300)
 
     def currentChanged(self, cur, prev):
         QListView.currentChanged(self, cur, prev)

From 14025af9e62c66dd18ff406a49f6223f56544210 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Thu, 5 Mar 2020 09:56:52 +0530
Subject: [PATCH 133/162] Fix #1865915 [[Enhancement] Change list order by drag
 and drop](https://bugs.launchpad.net/calibre/+bug/1865915)

---
 src/calibre/gui2/preferences/behavior.py | 15 ++++++++++++---
 src/calibre/gui2/preferences/behavior.ui |  9 +++++++++
 2 files changed, 21 insertions(+), 3 deletions(-)

diff --git a/src/calibre/gui2/preferences/behavior.py b/src/calibre/gui2/preferences/behavior.py
index d390d7a216..e93a449952 100644
--- a/src/calibre/gui2/preferences/behavior.py
+++ b/src/calibre/gui2/preferences/behavior.py
@@ -7,6 +7,7 @@ __copyright__ = '2010, Kovid Goyal '
 __docformat__ = 'restructuredtext en'
 
 import re
+from functools import partial
 
 from PyQt5.Qt import Qt, QListWidgetItem
 
@@ -22,6 +23,13 @@ from calibre.utils.icu import sort_key
 from polyglot.builtins import unicode_type, range
 
 
+def input_order_drop_event(self, ev):
+    ret = self.opt_input_order.__class__.dropEvent(self.opt_input_order, ev)
+    if ev.isAccepted():
+        self.changed_signal.emit()
+    return ret
+
+
 class OutputFormatSetting(Setting):
 
     CHOICES_SEARCH_FLAGS = Qt.MatchFixedString
@@ -62,6 +70,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
 
         self.input_up_button.clicked.connect(self.up_input)
         self.input_down_button.clicked.connect(self.down_input)
+        self.opt_input_order.dropEvent = partial(input_order_drop_event, self)
         for signal in ('Activated', 'Changed', 'DoubleClicked', 'Clicked'):
             signal = getattr(self.opt_internally_viewed_formats, 'item'+signal)
             signal.connect(self.internally_viewed_formats_changed)
@@ -147,7 +156,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
         for format in input_map + list(all_formats.difference(input_map)):
             item = QListWidgetItem(format, self.opt_input_order)
             item.setData(Qt.UserRole, (format))
-            item.setFlags(Qt.ItemIsEnabled|Qt.ItemIsSelectable)
+            item.setFlags(Qt.ItemIsEnabled|Qt.ItemIsSelectable|Qt.ItemIsDragEnabled)
 
     def up_input(self, *args):
         idx = self.opt_input_order.currentRow()
@@ -175,6 +184,6 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
 
 
 if __name__ == '__main__':
-    from PyQt5.Qt import QApplication
-    app = QApplication([])
+    from calibre.gui2 import Application
+    app = Application([])
     test_widget('Interface', 'Behavior')
diff --git a/src/calibre/gui2/preferences/behavior.ui b/src/calibre/gui2/preferences/behavior.ui
index 72bc8958e2..d700aa33c9 100644
--- a/src/calibre/gui2/preferences/behavior.ui
+++ b/src/calibre/gui2/preferences/behavior.ui
@@ -104,6 +104,15 @@
               0
              
             
+            
+             true
+            
+            
+             QAbstractItemView::InternalMove
+            
+            
+             Qt::MoveAction
+            
             
              true
             

From 3b3bd92129049c1d1727aa07f9ddce732f709d67 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Thu, 5 Mar 2020 20:10:48 +0530
Subject: [PATCH 134/162] pep8

---
 src/calibre/ebooks/metadata/sources/douban.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/calibre/ebooks/metadata/sources/douban.py b/src/calibre/ebooks/metadata/sources/douban.py
index a910074077..387f03fabc 100644
--- a/src/calibre/ebooks/metadata/sources/douban.py
+++ b/src/calibre/ebooks/metadata/sources/douban.py
@@ -107,7 +107,7 @@ class Douban(Source):
 
         # ISBN
         isbns = []
-        if isinstance(isbn, basestring):
+        if isinstance(isbn, (type(''), bytes)):
             if check_isbn(isbn):
                 isbns.append(isbn)
         else:
@@ -344,7 +344,7 @@ class Douban(Source):
         except Exception as e:
             log.exception('Failed to parse identify results')
             return as_unicode(e)
-        if j.has_key('books'):
+        if 'books' in j:
             entries = j['books']
         else:
             entries = []

From 5bf84f7688a19e64d415d585dd0cb0bbd90447de Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Fri, 6 Mar 2020 07:22:01 +0530
Subject: [PATCH 135/162] version 4.12.0

---
 Changelog.yaml           | 46 ++++++++++++++++++++++++++++++++++++++++
 src/calibre/constants.py |  2 +-
 2 files changed, 47 insertions(+), 1 deletion(-)

diff --git a/Changelog.yaml b/Changelog.yaml
index 7288f5d2d5..486f415e4b 100644
--- a/Changelog.yaml
+++ b/Changelog.yaml
@@ -20,6 +20,52 @@
 #   new recipes:
 #     - title:
 
+- version: 4.12.0
+  date: 2020-03-06
+
+  new features:
+    - title: "Kobo driver: Add support for new firmware with the series list on the device"
+
+    - title: "Automatically extract the source DOCX file from Kindle Create KPF files when adding them to calibre. If you prefer to preserve the KPF file you can disable the KPF Extract plugin in Preferences->Plugins"
+
+    - title: "Content server: Add a button to delete all locally cached books."
+      tickets: [1864305]
+
+    - title: "Edit Book: Allow selecting the contents of a tag with Ctrl+Alt+t"
+
+    - title: "Viewer: Save current position after 3 seconds of last position change. Useful if the viewer crashes on resume from sleep."
+
+    - title: "Viewer: Add a keyboard shortcut (Ctrl+w) to toggle the scrollbar."
+      tickets: [1864356]
+
+    - title: "Viewer: Keyboard shortcuts to change number of columns (Ctrl+[ and Ctrl+])"
+
+  bug fixes:
+    - title: "Fix the Douban metadata download plugin"
+      tickets: [1853091]
+
+    - title: "Viewer: Fix searching in Regex and Whole words mode not working well."
+
+    - title: "Viewer: Fix searching for multiple words in fixed layout books not working."
+      tickets: [1863464]
+
+    - title: "RTF Input: Fix handling of RTF files with invalid encoded text."
+      tickets: [1864719]
+
+    - title: "PDF Output: Add a hangcheck for loading HTML if there is no progress for sixty seconds abort"
+      tickets: [1865380]
+
+    - title: 'Viewer: When starting without a book allowing quitting the viewer by clicking the close button on the "Open book" page'
+      tickets: [1864343]
+
+  improved recipes:
+    - Wired
+    - ABC News Australia
+
+  new recipes:
+    - title: Spectator Australia
+      author: James Cridland
+
 - version: 4.11.2
   date: 2020-02-21
 
diff --git a/src/calibre/constants.py b/src/calibre/constants.py
index 067a473a94..83b89fd8c3 100644
--- a/src/calibre/constants.py
+++ b/src/calibre/constants.py
@@ -6,7 +6,7 @@ from polyglot.builtins import map, unicode_type, environ_item, hasenv, getenv, a
 import sys, locale, codecs, os, importlib, collections
 
 __appname__   = 'calibre'
-numeric_version = (4, 11, 2)
+numeric_version = (4, 12, 0)
 __version__   = '.'.join(map(unicode_type, numeric_version))
 git_version   = None
 __author__    = "Kovid Goyal "

From 1809676dbad79d912ba5343b333703352a5ac7b7 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Fri, 6 Mar 2020 08:27:46 +0530
Subject: [PATCH 136/162] Use -zz for rsync

---
 setup/hosting.py | 2 +-
 setup/upload.py  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/setup/hosting.py b/setup/hosting.py
index 4e8851ebec..2d37dbebcd 100644
--- a/setup/hosting.py
+++ b/setup/hosting.py
@@ -102,7 +102,7 @@ class SourceForge(Base):  # {{{
             for i in range(5):
                 try:
                     check_call([
-                        'rsync', '-h', '-z', '--progress', '-e', 'ssh -x', x,
+                        'rsync', '-h', '-zz', '--progress', '-e', 'ssh -x', x,
                         '%s,%s@frs.sourceforge.net:%s' %
                         (self.username, self.project, self.rdir + '/')
                     ])
diff --git a/setup/upload.py b/setup/upload.py
index bfe5a99cd5..4ec3a21ab7 100644
--- a/setup/upload.py
+++ b/setup/upload.py
@@ -123,7 +123,7 @@ def get_fosshub_data():
 
 def send_data(loc):
     subprocess.check_call([
-        'rsync', '--inplace', '--delete', '-r', '-z', '-h', '--progress', '-e',
+        'rsync', '--inplace', '--delete', '-r', '-zz', '-h', '--progress', '-e',
         'ssh -x', loc + '/', '%s@%s:%s' % (STAGING_USER, STAGING_HOST, STAGING_DIR)
     ])
 

From 5e773bbd3dc169f09040edde9f5eda9222eddb93 Mon Sep 17 00:00:00 2001
From: jnozsc 
Date: Thu, 5 Mar 2020 23:29:39 -0800
Subject: [PATCH 137/162] douban.py: download hi-res cover

---
 src/calibre/ebooks/metadata/sources/douban.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/calibre/ebooks/metadata/sources/douban.py b/src/calibre/ebooks/metadata/sources/douban.py
index 387f03fabc..c77bdb8dc7 100644
--- a/src/calibre/ebooks/metadata/sources/douban.py
+++ b/src/calibre/ebooks/metadata/sources/douban.py
@@ -48,8 +48,8 @@ def get_details(browser, url, timeout):  # {{{
 class Douban(Source):
 
     name = 'Douban Books'
-    author = 'Li Fanxi, xcffl'
-    version = (3, 0, 0)
+    author = 'Li Fanxi, xcffl, jnozsc'
+    version = (3, 1, 0)
     minimum_calibre_version = (2, 80, 0)
 
     description = _(
@@ -90,7 +90,7 @@ class Douban(Source):
         authors = entry_.get('author')
         book_tags = entry_.get('tags')
         rating = entry_.get('rating')
-        cover_url = entry_.get('image')
+        cover_url = entry_.get('images', {}).get('large')
         series = entry_.get('series')
 
         if not authors:

From 166a242ee36d5b41968931f2470380c8921d5c22 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Sat, 7 Mar 2020 08:14:31 +0530
Subject: [PATCH 138/162] PDF Output: Ignore glyph size mismatches when merging
 fonts for TTF.  Fixes #1866364 [PDF Conversion and Unicode
 Error](https://bugs.launchpad.net/calibre/+bug/1866364)

There appear to be a few mysterious windows systems where there are
fonts installed that cause this behavior in chromiums PDF system.
---
 src/calibre/ebooks/pdf/html_writer.py | 10 +++++-----
 src/calibre/utils/fonts/sfnt/merge.py |  5 +++--
 2 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/src/calibre/ebooks/pdf/html_writer.py b/src/calibre/ebooks/pdf/html_writer.py
index 10411a9955..c3abe969b4 100644
--- a/src/calibre/ebooks/pdf/html_writer.py
+++ b/src/calibre/ebooks/pdf/html_writer.py
@@ -919,7 +919,7 @@ def fonts_are_identical(fonts):
     return True
 
 
-def merge_font(fonts):
+def merge_font(fonts, log):
     # choose the largest font as the base font
     fonts.sort(key=lambda f: len(f['Data'] or b''), reverse=True)
     base_font = fonts[0]
@@ -932,7 +932,7 @@ def merge_font(fonts):
     cmaps = list(filter(None, (f['ToUnicode'] for f in t0_fonts)))
     if cmaps:
         t0_font['ToUnicode'] = as_bytes(merge_cmaps(cmaps))
-    base_font['sfnt'], width_for_glyph_id, height_for_glyph_id = merge_truetype_fonts_for_pdf(*(f['sfnt'] for f in descendant_fonts))
+    base_font['sfnt'], width_for_glyph_id, height_for_glyph_id = merge_truetype_fonts_for_pdf(tuple(f['sfnt'] for f in descendant_fonts), log)
     widths = []
     arrays = tuple(filter(None, (f['W'] for f in descendant_fonts)))
     if arrays:
@@ -947,7 +947,7 @@ def merge_font(fonts):
     return t0_font, base_font, references_to_drop
 
 
-def merge_fonts(pdf_doc):
+def merge_fonts(pdf_doc, log):
     all_fonts = pdf_doc.list_fonts(True)
     base_font_map = {}
 
@@ -976,7 +976,7 @@ def merge_fonts(pdf_doc):
     items = []
     for name, fonts in iteritems(base_font_map):
         if mergeable(fonts):
-            t0_font, base_font, references_to_drop = merge_font(fonts)
+            t0_font, base_font, references_to_drop = merge_font(fonts, log)
             for ref in references_to_drop:
                 replacements[ref] = t0_font['Reference']
             data = base_font['sfnt']()[0]
@@ -1246,7 +1246,7 @@ def convert(opf_path, opts, metadata=None, output_path=None, log=default_log, co
         page_number_display_map, page_layout, page_margins_map,
         pdf_metadata, report_progress, toc if has_toc else None)
 
-    merge_fonts(pdf_doc)
+    merge_fonts(pdf_doc, log)
     num_removed = dedup_type3_fonts(pdf_doc)
     if num_removed:
         log('Removed', num_removed, 'duplicated Type3 glyphs')
diff --git a/src/calibre/utils/fonts/sfnt/merge.py b/src/calibre/utils/fonts/sfnt/merge.py
index 1da10537fd..d86dc61cf0 100644
--- a/src/calibre/utils/fonts/sfnt/merge.py
+++ b/src/calibre/utils/fonts/sfnt/merge.py
@@ -12,7 +12,7 @@ class GlyphSizeMismatch(ValueError):
     pass
 
 
-def merge_truetype_fonts_for_pdf(*fonts):
+def merge_truetype_fonts_for_pdf(fonts, log=None):
     # only merges the glyf and loca tables, ignoring all other tables
     all_glyphs = {}
     ans = fonts[0]
@@ -28,7 +28,8 @@ def merge_truetype_fonts_for_pdf(*fonts):
                     all_glyphs[glyph_id] = glyf.glyph_data(offset, sz, as_raw=True)
                 else:
                     if abs(sz - len(prev_glyph_data)) > 8:
-                        raise GlyphSizeMismatch('Size mismatch for glyph id: {} prev_sz: {} sz: {}'.format(glyph_id, len(prev_glyph_data), sz))
+                        if log is not None:
+                            log('Size mismatch for glyph id: {} prev_sz: {} sz: {}'.format(glyph_id, len(prev_glyph_data), sz))
 
     glyf = ans[b'glyf']
     head = ans[b'head']

From f4836770cf7682f67d3245f3942da8dc0ac512e0 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Sat, 7 Mar 2020 08:36:47 +0530
Subject: [PATCH 139/162] Fix #1866298 [[Enhancement] Remove space in Welcome
 wizard](https://bugs.launchpad.net/calibre/+bug/1866298)

---
 src/calibre/gui2/wizard/library.ui | 97 ++++++++++++++++++------------
 1 file changed, 59 insertions(+), 38 deletions(-)

diff --git a/src/calibre/gui2/wizard/library.ui b/src/calibre/gui2/wizard/library.ui
index 276849f073..55b037320d 100644
--- a/src/calibre/gui2/wizard/library.ui
+++ b/src/calibre/gui2/wizard/library.ui
@@ -6,7 +6,7 @@
    
     0
     0
-    481
+    614
     300
    
   
@@ -19,18 +19,38 @@
   
    The one stop solution to all your e-book needs.
   
-  
-   
-    
-     
-      Choose your &language:
-     
-     
-      language
-     
-    
+  
+   
+    
+     
+      
+       
+        Choose your &language:
+       
+       
+        language
+       
+      
+     
+     
+      
+     
+     
+      
+       
+        Qt::Horizontal
+       
+       
+        
+         40
+         20
+        
+       
+      
+     
+    
    
-   
+   
     
      
       Qt::Vertical
@@ -43,14 +63,35 @@
      
     
    
-   
-    
+   
+    
      
-      &Change
+      <p>Choose a location for your books. When you add books to calibre, they will be copied here. Use an <b>empty folder</b> for a new calibre library:
+     
+     
+      true
      
     
    
-   
+   
+    
+     
+      
+       
+        true
+       
+      
+     
+     
+      
+       
+        &Change
+       
+      
+     
+    
+   
+   
     
      
       If a calibre library already exists at the newly selected location, calibre will use it automatically.
@@ -60,14 +101,7 @@
      
     
    
-   
-    
-     
-      true
-     
-    
-   
-   
+   
     
      
       Qt::Vertical
@@ -80,20 +114,7 @@
      
     
    
-   
-    
-   
-   
-    
-     
-      <p>Choose a location for your books. When you add books to calibre, they will be copied here. Use an <b>empty folder</b> for a new calibre library:
-     
-     
-      true
-     
-    
-   
-   
+   
     
      
       

From cec69cad8a78379b134e75457d1bdb4674e81734 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Sat, 7 Mar 2020 08:45:45 +0530
Subject: [PATCH 140/162] Fix #1866301 [[Enhancement] Move Restore default
 settings button](https://bugs.launchpad.net/calibre/+bug/1866301)

---
 src/pyj/book_list/prefs.pyj | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/pyj/book_list/prefs.pyj b/src/pyj/book_list/prefs.pyj
index 1c4a7ff0d5..aa8c9d8989 100644
--- a/src/pyj/book_list/prefs.pyj
+++ b/src/pyj/book_list/prefs.pyj
@@ -214,7 +214,7 @@ def create_prefs_widget(container, prefs_data):
     if state.widgets.length:
         container.appendChild(
             E.div(
-                style='margin:1ex 1em; padding: 1em; text-align:center',
+                style='margin:1rem;',
                 create_button(_('Restore default settings'), 'refresh', reset_to_defaults)
             )
         )

From 8a86cc17345de2ec5e2a676f369146cdfa93716e Mon Sep 17 00:00:00 2001
From: Charles Haley 
Date: Sat, 7 Mar 2020 15:24:17 +0000
Subject: [PATCH 141/162] Enhancement requests #1866405 (Filter available
 items) and #1866400 (Add languages to categories)

---
 src/calibre/gui2/dialogs/tag_categories.py | 16 ++++++---
 src/calibre/gui2/dialogs/tag_categories.ui | 39 ++++++++++++++++++----
 2 files changed, 45 insertions(+), 10 deletions(-)

diff --git a/src/calibre/gui2/dialogs/tag_categories.py b/src/calibre/gui2/dialogs/tag_categories.py
index af3caeafdb..6436626713 100644
--- a/src/calibre/gui2/dialogs/tag_categories.py
+++ b/src/calibre/gui2/dialogs/tag_categories.py
@@ -10,7 +10,7 @@ from calibre.gui2.dialogs.tag_categories_ui import Ui_TagCategories
 from calibre.gui2.dialogs.confirm_delete import confirm
 from calibre.gui2 import error_dialog
 from calibre.constants import islinux
-from calibre.utils.icu import sort_key, strcmp
+from calibre.utils.icu import sort_key, strcmp, primary_contains
 from polyglot.builtins import iteritems, unicode_type
 
 
@@ -72,9 +72,11 @@ class TagCategories(QDialog, Ui_TagCategories):
                            lambda: [t.original_name.replace('|', ',') for t in self.db_categories['authors']],
                            lambda: [t.original_name for t in self.db_categories['series']],
                            lambda: [t.original_name for t in self.db_categories['publisher']],
-                           lambda: [t.original_name for t in self.db_categories['tags']]
+                           lambda: [t.original_name for t in self.db_categories['tags']],
+                           lambda: [t.original_name for t in self.db_categories['languages']]
                           ]
-        category_names  = ['', _('Authors'), ngettext('Series', 'Series', 2), _('Publishers'), _('Tags')]
+        category_names  = ['', _('Authors'), ngettext('Series', 'Series', 2),
+                            _('Publishers'), _('Tags'), _('Languages')]
 
         for key,cc in iteritems(self.db.custom_field_metadata()):
             if cc['datatype'] in ['text', 'series', 'enumeration']:
@@ -106,6 +108,7 @@ class TagCategories(QDialog, Ui_TagCategories):
         self.category_box.currentIndexChanged[int].connect(self.select_category)
         self.category_filter_box.currentIndexChanged[int].connect(
                                                 self.display_filtered_categories)
+        self.item_filter_box.textEdited.connect(self.display_filtered_items)
         self.delete_category_button.clicked.connect(self.del_category)
         if islinux:
             self.available_items_box.itemDoubleClicked.connect(self.apply_tags)
@@ -168,14 +171,19 @@ class TagCategories(QDialog, Ui_TagCategories):
         w.setToolTip(_('Category lookup name: ') + item.label)
         return w
 
+    def display_filtered_items(self, text):
+        self.display_filtered_categories(None)
+
     def display_filtered_categories(self, idx):
         idx = idx if idx is not None else self.category_filter_box.currentIndex()
         self.available_items_box.clear()
         self.applied_items_box.clear()
+        item_filter = self.item_filter_box.text()
         for item in self.all_items_sorted:
             if idx == 0 or item.label == self.category_labels[idx]:
                 if item.index not in self.applied_items and item.exists:
-                    self.available_items_box.addItem(self.make_list_widget(item))
+                    if primary_contains(item_filter, item.name):
+                        self.available_items_box.addItem(self.make_list_widget(item))
         for index in self.applied_items:
             self.applied_items_box.addItem(self.make_list_widget(self.all_items[index]))
 
diff --git a/src/calibre/gui2/dialogs/tag_categories.ui b/src/calibre/gui2/dialogs/tag_categories.ui
index 198a99d4b2..7274a9bf88 100644
--- a/src/calibre/gui2/dialogs/tag_categories.ui
+++ b/src/calibre/gui2/dialogs/tag_categories.ui
@@ -33,7 +33,7 @@
           category_box
          
         
-     
+       
        
         
          
@@ -64,6 +64,26 @@
          
         
        
+       
+        
+         
+          Item &filter: 
+         
+         
+          Qt::AlignLeading|Qt::AlignLeft|Qt::AlignVCenter
+         
+         
+          item_filter_box
+         
+        
+       
+       
+        
+         
+          Enter text to filter the available items. Case and accents are ignored.
+         
+        
+       
       
      
      
@@ -136,6 +156,13 @@
          
         
        
+       
+        
+         
+           
+         
+        
+       
       
      
     
@@ -152,7 +179,7 @@
        
       
      
-     
+     
       
        
         true
@@ -165,7 +192,7 @@
        
       
      
-     
+     
       
        
         Apply tags to current tag category
@@ -189,7 +216,7 @@
        
       
      
-     
+     
       
        
         true
@@ -199,7 +226,7 @@
        
       
      
-     
+     
       
        
         Unapply (remove) tag from current tag category
@@ -213,7 +240,7 @@
        
       
      
-     
+     
       
        
         Qt::Horizontal

From adf95e7a22bc4c97a13d4bb490b4b9e931594a7f Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Mon, 9 Mar 2020 07:49:07 +0530
Subject: [PATCH 142/162] Viewer: Fix searching in flow mode not scrolling to
 display the search results. Fixes #1866519 [ebook-viewer 4.12 searching
 doesn't jump to results](https://bugs.launchpad.net/calibre/+bug/1866519)

---
 src/pyj/read_book/flow_mode.pyj | 12 ++++++++++++
 src/pyj/read_book/iframe.pyj    |  6 ++++--
 2 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/src/pyj/read_book/flow_mode.pyj b/src/pyj/read_book/flow_mode.pyj
index a7c1e8513f..80bc81d235 100644
--- a/src/pyj/read_book/flow_mode.pyj
+++ b/src/pyj/read_book/flow_mode.pyj
@@ -465,3 +465,15 @@ def auto_scroll_action(action):
     elif action is 'resume':
         auto_scroll_resume()
     return is_auto_scroll_active()
+
+
+def ensure_selection_visible():
+    s = window.getSelection()
+    if not s.anchorNode:
+        return
+    p = s.anchorNode
+    while p:
+        if p.scrollIntoView:
+            p.scrollIntoView()
+            return
+        p = p.parentNode
diff --git a/src/pyj/read_book/iframe.pyj b/src/pyj/read_book/iframe.pyj
index b5a2d7c883..cf4d256249 100644
--- a/src/pyj/read_book/iframe.pyj
+++ b/src/pyj/read_book/iframe.pyj
@@ -14,7 +14,7 @@ from read_book.flow_mode import (
     anchor_funcs as flow_anchor_funcs, auto_scroll_action as flow_auto_scroll_action,
     flow_onwheel, flow_to_scroll_fraction, handle_gesture as flow_handle_gesture,
     handle_shortcut as flow_handle_shortcut, layout as flow_layout,
-    scroll_by_page as flow_scroll_by_page
+    scroll_by_page as flow_scroll_by_page, ensure_selection_visible
 )
 from read_book.footnotes import is_footnote_link
 from read_book.globals import (
@@ -581,7 +581,9 @@ class IframeBoss:
 
     def show_search_result(self, data, from_load):
         if select_search_result(data.search_result):
-            if current_layout_mode() is not 'flow':
+            if current_layout_mode() is 'flow':
+                ensure_selection_visible()
+            else:
                 snap_to_selection()
         else:
             self.send_message('search_result_not_found', search_result=data.search_result)

From b3934874466fa6e397f45eb4ba8b100ca47e7e51 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Mon, 9 Mar 2020 09:31:40 +0530
Subject: [PATCH 143/162] Fix test failing when run alone

---
 src/calibre/db/tests/writing.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/calibre/db/tests/writing.py b/src/calibre/db/tests/writing.py
index 0dff91bfd0..74c43aee4c 100644
--- a/src/calibre/db/tests/writing.py
+++ b/src/calibre/db/tests/writing.py
@@ -421,13 +421,13 @@ class WritingTest(BaseTest):
         cache.set_metadata(2, mi)
         nmi = cache.get_metadata(2, get_cover=True, cover_as_data=True)
         ae(oldmi.cover_data, nmi.cover_data)
-        self.compare_metadata(nmi, oldmi, exclude={'last_modified', 'format_metadata'})
+        self.compare_metadata(nmi, oldmi, exclude={'last_modified', 'format_metadata', 'formats'})
         cache.set_metadata(1, mi2, force_changes=True)
         nmi2 = cache.get_metadata(1, get_cover=True, cover_as_data=True)
         # The new code does not allow setting of #series_index to None, instead
         # it is reset to 1.0
         ae(nmi2.get_extra('#series'), 1.0)
-        self.compare_metadata(nmi2, oldmi2, exclude={'last_modified', 'format_metadata', '#series_index'})
+        self.compare_metadata(nmi2, oldmi2, exclude={'last_modified', 'format_metadata', '#series_index', 'formats'})
 
         cache = self.init_cache(self.cloned_library)
         mi = cache.get_metadata(1)

From 0ada2ad42bf3f05fd780d989ed640bc76163e99f Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Mon, 9 Mar 2020 09:36:48 +0530
Subject: [PATCH 144/162] DB: Ensure that set_metadata() sets author_sort to a
 correct value if the Metadata object has authors but no author_sort. Fixes
 #1116 (Generate an author_sort in fetch-ebook-metadata output)

---
 src/calibre/db/cache.py         | 12 ++++++++++--
 src/calibre/db/tests/writing.py |  7 +++++++
 2 files changed, 17 insertions(+), 2 deletions(-)

diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py
index 11e02d083b..f2b01b68b5 100644
--- a/src/calibre/db/cache.py
+++ b/src/calibre/db/cache.py
@@ -26,7 +26,7 @@ from calibre.db.tables import VirtualTable
 from calibre.db.write import get_series_values, uniq
 from calibre.db.lazy import FormatMetadata, FormatsList, ProxyMetadata
 from calibre.ebooks import check_ebook_format
-from calibre.ebooks.metadata import string_to_authors, author_to_author_sort
+from calibre.ebooks.metadata import string_to_authors, author_to_author_sort, authors_to_sort_string
 from calibre.ebooks.metadata.book.base import Metadata
 from calibre.ebooks.metadata.opf2 import metadata_to_opf
 from calibre.ptempfile import (base_dir, PersistentTemporaryFile,
@@ -1297,6 +1297,7 @@ class Cache(object):
         if set_title and mi.title:
             path_changed = True
             set_field('title', mi.title)
+        authors_changed = False
         if set_authors:
             path_changed = True
             if not mi.authors:
@@ -1305,6 +1306,7 @@ class Cache(object):
             for a in mi.authors:
                 authors += string_to_authors(a)
             set_field('authors', authors)
+            authors_changed = True
 
         if path_changed:
             self._update_path({book_id})
@@ -1339,7 +1341,13 @@ class Cache(object):
                     if val is not None:
                         protected_set_field(field, val)
 
-                for field in ('author_sort', 'publisher', 'series', 'tags', 'comments',
+                val = mi.get('author_sort', None)
+                if authors_changed and (not val or mi.is_null('author_sort')):
+                    val = authors_to_sort_string(mi.authors)
+                if authors_changed or (force_changes and val is not None) or not mi.is_null('author_sort'):
+                    protected_set_field('author_sort', val)
+
+                for field in ('publisher', 'series', 'tags', 'comments',
                     'languages', 'pubdate'):
                     val = mi.get(field, None)
                     if (force_changes and val is not None) or not mi.is_null(field):
diff --git a/src/calibre/db/tests/writing.py b/src/calibre/db/tests/writing.py
index 74c43aee4c..8d858c12d9 100644
--- a/src/calibre/db/tests/writing.py
+++ b/src/calibre/db/tests/writing.py
@@ -11,6 +11,7 @@ from functools import partial
 from io import BytesIO
 
 from calibre.ebooks.metadata import author_to_author_sort, title_sort
+from calibre.ebooks.metadata.book.base import Metadata
 from calibre.utils.date import UNDEFINED_DATE
 from calibre.db.tests.base import BaseTest, IMG
 from polyglot.builtins import iteritems, itervalues, unicode_type
@@ -436,6 +437,12 @@ class WritingTest(BaseTest):
         cache.set_metadata(3, mi)
         self.assertEqual(set(otags), set(cache.field_for('tags', 3)), 'case changes should not be allowed in set_metadata')
 
+        # test that setting authors without author sort results in an
+        # auto-generated authors sort
+        mi = Metadata('empty', ['a1', 'a2'])
+        cache.set_metadata(1, mi)
+        self.assertEqual('a1 & a2', cache.field_for('author_sort', 1))
+
     # }}}
 
     def test_conversion_options(self):  # {{{

From 948a15965e5ab0507e4bb6d75f5a3e8c5685fc7e Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Mon, 9 Mar 2020 15:40:36 +0530
Subject: [PATCH 145/162] Maximum font size for margin text should be the body
 font size not hardcoded to 12px

---
 src/pyj/read_book/view.pyj | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/src/pyj/read_book/view.pyj b/src/pyj/read_book/view.pyj
index b42f9269d9..fe306e30ef 100644
--- a/src/pyj/read_book/view.pyj
+++ b/src/pyj/read_book/view.pyj
@@ -127,9 +127,23 @@ def show_controls_help():
 
 # }}}
 
+
+def maximum_font_size():
+    ans = maximum_font_size.ans
+    if not ans:
+        q = window.getComputedStyle(document.body).fontSize
+        if q and q.endsWith('px'):
+            q = parseInt(q)
+            if q and not isNaN(q):
+                ans = maximum_font_size.ans = q
+                return ans
+        ans = maximum_font_size.ans = 12
+    return ans
+
+
 def margin_elem(sd, which, id, onclick, oncontextmenu):
     sz = sd.get(which, 20)
-    fsz = min(max(0, sz - 6), 12)
+    fsz = min(max(0, sz - 6), maximum_font_size())
     s = '; text-overflow: ellipsis; white-space: nowrap; overflow: hidden'
     ans = E.div(
         style=f'height:{sz}px; overflow: hidden; font-size:{fsz}px; width:100%; padding: 0; display: flex; justify-content: space-between; align-items: center',

From 6e4ed94a6b525fbc05deb4799481eb2089a12d7a Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Mon, 9 Mar 2020 22:00:00 +0530
Subject: [PATCH 146/162] Update Newsweek

Fixes #1866636 [newsweek won't download](https://bugs.launchpad.net/calibre/+bug/1866636)
---
 recipes/newsweek.recipe | 74 +++++++++++++++++------------------------
 1 file changed, 31 insertions(+), 43 deletions(-)

diff --git a/recipes/newsweek.recipe b/recipes/newsweek.recipe
index a8dc8d91e6..fc55dac112 100644
--- a/recipes/newsweek.recipe
+++ b/recipes/newsweek.recipe
@@ -1,3 +1,8 @@
+#!/usr/bin/env python
+# vim:fileencoding=utf-8
+# License: GPLv3 Copyright: 2015, Kovid Goyal 
+
+import json
 from calibre.web.feeds.news import BasicNewsRecipe
 from collections import defaultdict
 
@@ -49,28 +54,23 @@ class Newsweek(BasicNewsRecipe):
         a = li.xpath('descendant::a[@href]')[0]
         url = href_to_url(a, add_piano=True)
         self.timefmt = self.tag_to_string(a)
-        img = li.xpath('descendant::a[@href]//img[@data-src]')[0]
-        self.cover_url = img.get('data-src').partition('?')[0]
+        img = li.xpath('descendant::a[@href]//source[@type="image/jpeg"]/@srcset')[0]
+        self.cover_url = img.partition('?')[0]
+        self.log('Found cover url:', self.cover_url)
         root = self.index_to_soup(url, as_tree=True)
         features = []
-        try:
-            div = root.xpath('//div[@class="magazine-features"]')[0]
-        except IndexError:
-            pass
-        else:
-            for a in div.xpath('descendant::div[@class="h1"]//a[@href]'):
-                title = self.tag_to_string(a)
-                article = a.xpath('ancestor::article')[0]
-                desc = ''
-                s = article.xpath('descendant::div[@class="summary"]')
-                if s:
-                    desc = self.tag_to_string(s[0])
-                features.append({'title': title, 'url': href_to_url(a), 'description': desc})
-                self.log(title, href_to_url(a))
+        for article in root.xpath('//div[@class="magazine-features"]//article'):
+            a = article.xpath('descendant::a[@class="article-link"]')[0]
+            title = self.tag_to_string(a)
+            url = href_to_url(a)
+            desc = ''
+            s = article.xpath('descendant::div[@class="summary"]')
+            if s:
+                desc = self.tag_to_string(s[0])
+            features.append({'title': title, 'url': href_to_url(a), 'description': desc})
+            self.log(title, url)
 
-        index = []
-        if features:
-            index.append(('Features', features))
+        index = [('Features', features)]
         sections = defaultdict(list)
         for widget in ('editor-pick',):
             self.parse_widget(widget, sections)
@@ -79,30 +79,18 @@ class Newsweek(BasicNewsRecipe):
         return index
 
     def parse_widget(self, widget, sections):
-        root = self.index_to_soup('https://d.newsweek.com/widget/' + widget, as_tree=True)
-        div = root.xpath('//div')[0]
-        href_xpath = 'descendant::*[local-name()="h1" or local-name()="h2" or local-name()="h3" or local-name()="h4"]/a[@href]'
-        for a in div.xpath(href_xpath):
-            title = self.tag_to_string(a)
-            article = a.xpath('ancestor::article')[0]
-            desc = ''
-            s = article.xpath('descendant::div[@class="summary"]')
-            if s:
-                desc = self.tag_to_string(s[0])
-            sec = article.xpath('descendant::div[@class="category"]')
-            if sec:
-                sec = self.tag_to_string(sec[0])
-            else:
-                sec = 'Articles'
-            sections[sec].append(
-                {'title': title, 'url': href_to_url(a), 'description': desc})
-            self.log(title, href_to_url(a))
-            if desc:
-                self.log('\t' + desc)
-            self.log('')
-
-    def print_version(self, url):
-        return url + '?piano_d=1'
+        raw = self.index_to_soup('https://d.newsweek.com/json/' + widget, raw=True)
+        data = json.loads(raw)['items']
+        for item in data:
+            title = item['title']
+            url = BASE + item['link']
+            self.log(title, url)
+            sections[item['label']].append(
+                {
+                    'title': title,
+                    'url': url,
+                    'description': item['description'],
+                })
 
     def preprocess_html(self, soup):
         # Parallax images in the articles are loaded as background images

From 0c6729937448396d28907977becfc84e8c153733 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Tue, 10 Mar 2020 08:16:24 +0530
Subject: [PATCH 147/162] Check Book: Do not fail if non-UTF-8 stylesheets are
 present in the book. Fixes #1866701 [Private
 bug](https://bugs.launchpad.net/calibre/+bug/1866701)

---
 src/calibre/ebooks/oeb/polish/check/main.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/calibre/ebooks/oeb/polish/check/main.py b/src/calibre/ebooks/oeb/polish/check/main.py
index b9c43ff6f2..f4715f50dc 100644
--- a/src/calibre/ebooks/oeb/polish/check/main.py
+++ b/src/calibre/ebooks/oeb/polish/check/main.py
@@ -48,16 +48,18 @@ def run_checks(container):
     xml_items, html_items, raster_images, stylesheets = [], [], [], []
     for name, mt in iteritems(container.mime_map):
         items = None
+        decode = False
         if mt in XML_TYPES:
             items = xml_items
         elif mt in OEB_DOCS:
             items = html_items
         elif mt in OEB_STYLES:
+            decode = True
             items = stylesheets
         elif is_raster_image(mt):
             items = raster_images
         if items is not None:
-            items.append((name, mt, container.open(name, 'rb').read()))
+            items.append((name, mt, container.raw_data(name, decode=decode)))
     errors.extend(run_checkers(check_html_size, html_items))
     errors.extend(run_checkers(check_xml_parsing, xml_items))
     errors.extend(run_checkers(check_xml_parsing, html_items))

From 0e6e5afe8dcbe76bd09c2bb6d23dc8f305726c91 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Tue, 10 Mar 2020 10:00:35 +0530
Subject: [PATCH 148/162] MOBI Output: Improve conversion of PNG images with
 transparency to GIF

---
 src/calibre/ebooks/mobi/utils.py |  9 ++-------
 src/calibre/utils/img.py         | 32 +++++++++++++++++++++++++++-----
 2 files changed, 29 insertions(+), 12 deletions(-)

diff --git a/src/calibre/ebooks/mobi/utils.py b/src/calibre/ebooks/mobi/utils.py
index faf7e5a6b5..0121777702 100644
--- a/src/calibre/ebooks/mobi/utils.py
+++ b/src/calibre/ebooks/mobi/utils.py
@@ -10,7 +10,7 @@ import struct, string, zlib, os
 from collections import OrderedDict
 from io import BytesIO
 
-from calibre.utils.img import save_cover_data_to, scale_image, image_to_data, image_from_data, resize_image
+from calibre.utils.img import save_cover_data_to, scale_image, image_to_data, image_from_data, resize_image, png_data_to_gif_data
 from calibre.utils.imghdr import what
 from calibre.ebooks import normalize
 from polyglot.builtins import unicode_type, range, as_bytes, map
@@ -417,13 +417,8 @@ def to_base(num, base=32, min_num_digits=None):
 def mobify_image(data):
     'Convert PNG images to GIF as the idiotic Kindle cannot display some PNG'
     fmt = what(None, data)
-
     if fmt == 'png':
-        from PIL import Image
-        im = Image.open(BytesIO(data))
-        buf = BytesIO()
-        im.save(buf, 'gif')
-        data = buf.getvalue()
+        data = png_data_to_gif_data(data)
     return data
 
 # Font records {{{
diff --git a/src/calibre/utils/img.py b/src/calibre/utils/img.py
index bf10b2b7af..71d1966138 100644
--- a/src/calibre/utils/img.py
+++ b/src/calibre/utils/img.py
@@ -67,6 +67,32 @@ def load_jxr_data(data):
 
 # }}}
 
+# png to gif {{{
+
+
+def png_data_to_gif_data(data):
+    from PIL import Image
+    img = Image.open(BytesIO(data))
+    buf = BytesIO()
+    if img.mode in ('p', 'P'):
+        transparency = img.info.get('transparency')
+        if transparency is not None:
+            img.save(buf, 'gif', transparency=transparency)
+        else:
+            img.save(buf, 'gif')
+    elif img.mode in ('rgba', 'RGBA'):
+        alpha = img.split()[3]
+        mask = Image.eval(alpha, lambda a: 255 if a <=128 else 0)
+        img = img.convert('RGB').convert('P', palette=Image.ADAPTIVE, colors=255)
+        img.paste(255, mask)
+        img.save(buf, 'gif', transparency=255)
+    else:
+        img = img.convert('P', palette=Image.ADAPTIVE)
+        img.save(buf, 'gif')
+    return buf.getvalue()
+
+# }}}
+
 # Loading images {{{
 
 
@@ -140,11 +166,7 @@ def image_to_data(img, compression_quality=95, fmt='JPEG', png_compression_level
         w.setQuality(90)
         if not w.write(img):
             raise ValueError('Failed to export image as ' + fmt + ' with error: ' + w.errorString())
-        from PIL import Image
-        im = Image.open(BytesIO(ba.data()))
-        buf = BytesIO()
-        im.save(buf, 'gif')
-        return buf.getvalue()
+        return png_data_to_gif_data(ba.data())
     is_jpeg = fmt in ('JPG', 'JPEG')
     w = QImageWriter(buf, fmt.encode('ascii'))
     if is_jpeg:

From 3df15e222a951e683b10f1fe37f12c0a1d6a4cb8 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Tue, 10 Mar 2020 13:37:21 +0530
Subject: [PATCH 149/162] MOBI Input: Dont auto-convert images in PNG/GIF
 formats to JPEG

---
 src/calibre/ebooks/mobi/reader/mobi6.py | 41 +++++++++++++++++--------
 1 file changed, 28 insertions(+), 13 deletions(-)

diff --git a/src/calibre/ebooks/mobi/reader/mobi6.py b/src/calibre/ebooks/mobi/reader/mobi6.py
index 6400295a39..85988dbb04 100644
--- a/src/calibre/ebooks/mobi/reader/mobi6.py
+++ b/src/calibre/ebooks/mobi/reader/mobi6.py
@@ -10,7 +10,7 @@ import shutil, os, re, struct, textwrap, io
 
 from lxml import html, etree
 
-from calibre import (xml_entity_to_unicode, entity_to_unicode)
+from calibre import xml_entity_to_unicode, entity_to_unicode, guess_type
 from calibre.utils.cleantext import clean_ascii_chars, clean_xml_chars
 from calibre.ebooks import DRMError, unit_convert
 from calibre.ebooks.chardet import strip_encoding_declarations
@@ -178,7 +178,7 @@ class MobiReader(object):
         self.processed_html = strip_encoding_declarations(self.processed_html)
         self.processed_html = re.sub(r'&(\S+?);', xml_entity_to_unicode,
             self.processed_html)
-        self.extract_images(processed_records, output_dir)
+        image_name_map = self.extract_images(processed_records, output_dir)
         self.replace_page_breaks()
         self.cleanup_html()
 
@@ -272,7 +272,7 @@ class MobiReader(object):
             head.insert(0, title)
             head.text = '\n\t'
 
-        self.upshift_markup(root)
+        self.upshift_markup(root, image_name_map)
         guides = root.xpath('//guide')
         guide = guides[0] if guides else None
         metadata_elems = root.xpath('//metadata')
@@ -389,8 +389,9 @@ class MobiReader(object):
             raw += unit
         return raw
 
-    def upshift_markup(self, root):
+    def upshift_markup(self, root, image_name_map=None):
         self.log.debug('Converting style information to CSS...')
+        image_name_map = image_name_map or {}
         size_map = {
             'xx-small': '0.5',
             'x-small': '1',
@@ -510,10 +511,11 @@ class MobiReader(object):
                     recindex = attrib.pop(attr, None) or recindex
                 if recindex is not None:
                     try:
-                        recindex = '%05d'%int(recindex)
-                    except:
+                        recindex = int(recindex)
+                    except Exception:
                         pass
-                    attrib['src'] = 'images/%s.jpg' % recindex
+                    else:
+                        attrib['src'] = 'images/' + image_name_map.get(recindex, '%05d.jpg' % recindex)
                 for attr in ('width', 'height'):
                     if attr in attrib:
                         val = attrib[attr]
@@ -674,7 +676,7 @@ class MobiReader(object):
         for i in getattr(self, 'image_names', []):
             path = os.path.join(bp, 'images', i)
             added.add(path)
-            manifest.append((path, 'image/jpeg'))
+            manifest.append((path, guess_type(path)[0] or 'image/jpeg'))
         if cover_copied is not None:
             manifest.append((cover_copied, 'image/jpeg'))
 
@@ -870,6 +872,7 @@ class MobiReader(object):
             os.makedirs(output_dir)
         image_index = 0
         self.image_names = []
+        image_name_map = {}
         start = getattr(self.book_header, 'first_image_index', -1)
         if start > self.num_sections or start < 0:
             # BAEN PRC files have bad headers
@@ -882,18 +885,30 @@ class MobiReader(object):
             image_index += 1
             if data[:4] in {b'FLIS', b'FCIS', b'SRCS', b'\xe9\x8e\r\n',
                     b'RESC', b'BOUN', b'FDST', b'DATP', b'AUDI', b'VIDE'}:
-                # This record is a known non image type, not need to try to
+                # This record is a known non image type, no need to try to
                 # load the image
                 continue
 
-            path = os.path.join(output_dir, '%05d.jpg' % image_index)
             try:
-                if what(None, data) not in {'jpg', 'jpeg', 'gif', 'png', 'bmp'}:
-                    continue
-                save_cover_data_to(data, path, minify_to=(10000, 10000))
+                imgfmt = what(None, data)
             except Exception:
                 continue
+            if imgfmt not in {'jpg', 'jpeg', 'gif', 'png', 'bmp'}:
+                continue
+            if imgfmt == 'jpeg':
+                imgfmt = 'jpg'
+            path = os.path.join(output_dir, '%05d.%s' % (image_index, imgfmt))
+            image_name_map[image_index] = os.path.basename(path)
+            if imgfmt in ('gif', 'png'):
+                with open(path, 'wb') as f:
+                    f.write(data)
+            else:
+                try:
+                    save_cover_data_to(data, path, minify_to=(10000, 10000))
+                except Exception:
+                    continue
             self.image_names.append(os.path.basename(path))
+        return image_name_map
 
 
 def test_mbp_regex():

From 4a661f91382e30476ef315caa8eaa06601617cda Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Tue, 10 Mar 2020 13:50:28 +0530
Subject: [PATCH 150/162] MOBI Input: Upshift non-animated GIF to PNG as it is
 a more widely supported format

---
 src/calibre/ebooks/mobi/reader/mobi6.py | 10 ++++++++--
 src/calibre/utils/img.py                | 16 +++++++++++++++-
 2 files changed, 23 insertions(+), 3 deletions(-)

diff --git a/src/calibre/ebooks/mobi/reader/mobi6.py b/src/calibre/ebooks/mobi/reader/mobi6.py
index 85988dbb04..ebd54e326e 100644
--- a/src/calibre/ebooks/mobi/reader/mobi6.py
+++ b/src/calibre/ebooks/mobi/reader/mobi6.py
@@ -21,7 +21,7 @@ from calibre.ebooks.metadata import MetaInformation
 from calibre.ebooks.metadata.opf2 import OPFCreator, OPF
 from calibre.ebooks.metadata.toc import TOC
 from calibre.ebooks.mobi.reader.headers import BookHeader
-from calibre.utils.img import save_cover_data_to
+from calibre.utils.img import save_cover_data_to, gif_data_to_png_data, AnimatedGIF
 from calibre.utils.imghdr import what
 from polyglot.builtins import iteritems, unicode_type, range, map
 
@@ -897,9 +897,15 @@ class MobiReader(object):
                 continue
             if imgfmt == 'jpeg':
                 imgfmt = 'jpg'
+            if imgfmt == 'gif':
+                try:
+                    data = gif_data_to_png_data(data)
+                    imgfmt = 'png'
+                except AnimatedGIF:
+                    pass
             path = os.path.join(output_dir, '%05d.%s' % (image_index, imgfmt))
             image_name_map[image_index] = os.path.basename(path)
-            if imgfmt in ('gif', 'png'):
+            if imgfmt == 'png':
                 with open(path, 'wb') as f:
                     f.write(data)
             else:
diff --git a/src/calibre/utils/img.py b/src/calibre/utils/img.py
index 71d1966138..e6f14e3a62 100644
--- a/src/calibre/utils/img.py
+++ b/src/calibre/utils/img.py
@@ -67,7 +67,7 @@ def load_jxr_data(data):
 
 # }}}
 
-# png to gif {{{
+# png <-> gif {{{
 
 
 def png_data_to_gif_data(data):
@@ -91,6 +91,20 @@ def png_data_to_gif_data(data):
         img.save(buf, 'gif')
     return buf.getvalue()
 
+
+class AnimatedGIF(ValueError):
+    pass
+
+
+def gif_data_to_png_data(data, discard_animation=False):
+    from PIL import Image
+    img = Image.open(BytesIO(data))
+    if img.is_animated and not discard_animation:
+        raise AnimatedGIF()
+    buf = BytesIO()
+    img.save(buf, 'png')
+    return buf.getvalue()
+
 # }}}
 
 # Loading images {{{

From 2d38fb1b22a5a86d1f5b85f69b383371dbd4ef57 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Wed, 11 Mar 2020 17:21:39 +0530
Subject: [PATCH 151/162] Remove references to drmfree.calibre-ebook.com

That site is being retired. Text about DRM has been migrated
to the calibre manual.
---
 manual/drm.rst                                | 107 ++++++++++++++++++
 manual/faq.rst                                |   2 +-
 manual/gui.rst                                |   2 +-
 manual/simple_index.rst                       |   1 +
 src/calibre/customize/builtins.py             |  10 --
 src/calibre/gui2/actions/store.py             |   6 +-
 src/calibre/gui2/dialogs/drm_error.ui         |   3 +-
 .../gui2/store/stores/open_books_plugin.py    |  80 -------------
 8 files changed, 115 insertions(+), 96 deletions(-)
 create mode 100644 manual/drm.rst
 delete mode 100644 src/calibre/gui2/store/stores/open_books_plugin.py

diff --git a/manual/drm.rst b/manual/drm.rst
new file mode 100644
index 0000000000..551ed63e6c
--- /dev/null
+++ b/manual/drm.rst
@@ -0,0 +1,107 @@
+
+.. _dmr:
+
+Digital Rights Management (DRM)
+===============================================
+
+Digital rights management (DRM) is a generic term for access control
+technologies that can be used by hardware manufacturers, publishers, copyright
+holders and individuals to try to impose limitations on the usage of digital
+content and devices. It is also, sometimes, disparagingly described as Digital
+Restrictions Management. The term is used to describe any technology which
+inhibits uses (legitimate or otherwise) of digital content that were not
+desired or foreseen by the content provider. The term generally doesn't refer
+to other forms of copy protection which can be circumvented without modifying
+the file or device, such as serial numbers or key-files. It can also refer to
+restrictions associated with specific instances of digital works or devices.
+DRM technologies attempt to control use of digital media by preventing access,
+copying or conversion to other formats by end users. See `wikipedia
+`_.
+
+
+What does DRM imply for me personally?
+------------------------------------------
+
+When you buy an e-book with DRM you don't really own it but have purchased the
+permission to use it in a manner dictated to you by the seller. DRM limits what
+you can do with e-books you have "bought". Often people who buy books with DRM
+are unaware of the extent of these restrictions. These restrictions prevent you
+from reformatting the e-book to your liking, including making stylistic changes
+like adjusting the font sizes, although there is software that empowers you to
+do such things for non DRM books. People are often surprised that an e-book
+they have bought in a particular format cannot be converted to another format
+if the e-book has DRM. So if you have an Amazon Kindle and buy a book sold by
+Barnes and Nobles, you should know that if that e-book has DRM you will not be
+able to read it on your Kindle. Notice that I am talking about a book you buy,
+not steal or pirate but BUY.
+
+
+What does DRM do for authors?
+----------------------------------
+
+Publishers of DRMed e-books argue that the DRM is all for the sake of authors
+and to protect their artistic integrity and prevent piracy. But DRM does NOT
+prevent piracy. People who want to pirate content or use pirated content still
+do it and succeed. The three major DRM schemes for e-books today are run by
+Amazon, Adobe and Barnes and Noble and all three DRM schemes have been cracked.
+All DRM does is inconvenience legitimate users. It can be argued that it
+actually harms authors as people who would have bought the book choose to find
+a pirated version as they are not willing to put up with DRM. Those that would
+pirate in the absence of DRM do so in its presence as well. To reiterate, the
+key point is that DRM *does not prevent piracy*. So DRM is not only pointless
+and harmful to buyers of e-books but also a waste of money.
+
+
+DRM and freedom
+-------------------
+
+Although digital content can be used to make information as well as creative
+works easily available to everyone and empower humanity, this is not in the
+interests of some publishers who want to steer people away from this
+possibility of freedom simply to maintain their relevance in world developing
+so fast that they cant keep up.
+
+
+Why does calibre not support DRM?
+-------------------------------------
+
+calibre is open source software while DRM by its very nature is closed. If
+calibre were to support opening or viewing DRM files it could be trivially
+modified to be used as a tool for DRM removal which is illegal under today's
+laws. Open source software and DRM are a clash of principles. While DRM is all
+about controlling the user open source software is about empowering the user.
+The two simply can not coexist.
+
+
+What is calibre's view on content providers?
+------------------------------------------------
+
+We firmly believe that authors and other content providers should be
+compensated for their efforts, but DRM is not the way to go about it. We are
+developing this database of DRM-free e-books from various sources to help you
+find DRM-free alternatives and to help independent authors and publishers of
+DRM-free e-books publicize their content. We hope you will find this useful and
+we request that you do not pirate the content made available to you here.
+
+
+How can I help fight DRM?
+-----------------------------
+
+As somebody who reads and buys e-books you can help fight DRM. Do not buy
+e-books with DRM. There are some publishers who publish DRM-free e-books. Make
+an effort to see if they carry the e-book you are looking for. If you like
+books by certain independent authors that sell DRM-free e-books and you can
+afford it make donations to them. This is money well spent as their e-books
+tend to be cheaper (there may be exceptions) than the ones you would buy from
+publishers of DRMed books and would probably work on all devices you own in the
+future saving you the cost of buying the e-book again. Do not discourage
+publishers and authors of DRM-free e-books by pirating their content. Content
+providers deserve compensation for their efforts. Do not punish them for trying
+to make your reading experience better by making available DRM-free e-books. In
+the long run this is detrimental to you.  If you have bought books from sellers
+that carry both DRMed as well as DRM-free books, not knowing if they carry DRM
+or not make it a point to leave a comment or review on the website informing
+future buyers of its DRM status. Many sellers do not think it important to
+clearly indicate to their buyers if an e-book carries DRM or not.  `Here
+` you will find a Guide to
+DRM-free living.
diff --git a/manual/faq.rst b/manual/faq.rst
index f9b0862422..6cb991ef41 100644
--- a/manual/faq.rst
+++ b/manual/faq.rst
@@ -983,7 +983,7 @@ If you want to backup the calibre configuration/plugins, you have to backup the
 
 How do I use purchased EPUB books with calibre (or what do I do with .acsm files)?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-Most purchased EPUB books have `DRM `_. This prevents calibre from opening them. You can still use calibre to store and transfer them to your e-book reader. First, you must authorize your reader on a windows machine with Adobe Digital Editions. Once this is done, EPUB books transferred with calibre will work fine on your reader. When you purchase an epub book from a website, you will get an ".acsm" file. This file should be opened with Adobe Digital Editions, which will then download the actual ".epub" e-book. The e-book file will be stored in the folder "My Digital Editions", from where you can add it to calibre.
+Most purchased EPUB books have :doc:`DRM `. This prevents calibre from opening them. You can still use calibre to store and transfer them to your e-book reader. First, you must authorize your reader on a windows machine with Adobe Digital Editions. Once this is done, EPUB books transferred with calibre will work fine on your reader. When you purchase an epub book from a website, you will get an ".acsm" file. This file should be opened with Adobe Digital Editions, which will then download the actual ".epub" e-book. The e-book file will be stored in the folder "My Digital Editions", from where you can add it to calibre.
 
 I am getting a "Permission Denied" error?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
diff --git a/manual/gui.rst b/manual/gui.rst
index ab48192309..f5e67d51a6 100644
--- a/manual/gui.rst
+++ b/manual/gui.rst
@@ -94,7 +94,7 @@ Convert books
     :class: float-right-img
 
 |cei| E-books can be converted from a number of formats into whatever format your e-book reader prefers.
-Many e-books available for purchase will be protected by `Digital Rights Management `_ *(DRM)* technology.
+Many e-books available for purchase will be protected by :doc:`Digital Rights Management ` *(DRM)* technology.
 calibre will not convert these e-books. It is easy to remove the DRM from many formats, but as this may be illegal,
 you will have to find tools to liberate your books yourself and then use calibre to convert them.
 
diff --git a/manual/simple_index.rst b/manual/simple_index.rst
index 18a62cbd93..bc1607d34d 100644
--- a/manual/simple_index.rst
+++ b/manual/simple_index.rst
@@ -41,4 +41,5 @@ available `_.
    customize
    generated/en/cli-index
    develop
+   drm
    glossary
diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py
index 44880cbc0e..4a3af02002 100644
--- a/src/calibre/customize/builtins.py
+++ b/src/calibre/customize/builtins.py
@@ -1738,15 +1738,6 @@ class StoreNextoStore(StoreBase):
     affiliate = True
 
 
-class StoreOpenBooksStore(StoreBase):
-    name = 'Open Books'
-    description = 'Comprehensive listing of DRM free e-books from a variety of sources provided by users of calibre.'
-    actual_plugin = 'calibre.gui2.store.stores.open_books_plugin:OpenBooksStore'
-
-    drm_free_only = True
-    headquarters = 'US'
-
-
 class StoreOzonRUStore(StoreBase):
     name = 'OZON.ru'
     description = 'e-books from OZON.ru'
@@ -1910,7 +1901,6 @@ plugins += [
     StoreMillsBoonUKStore,
     StoreMobileReadStore,
     StoreNextoStore,
-    StoreOpenBooksStore,
     StoreOzonRUStore,
     StorePragmaticBookshelfStore,
     StorePublioStore,
diff --git a/src/calibre/gui2/actions/store.py b/src/calibre/gui2/actions/store.py
index ba948ac738..7b889ec51b 100644
--- a/src/calibre/gui2/actions/store.py
+++ b/src/calibre/gui2/actions/store.py
@@ -13,6 +13,7 @@ from PyQt5.Qt import QIcon, QSize
 from calibre.gui2 import error_dialog
 from calibre.gui2.actions import InterfaceAction
 from calibre.gui2.dialogs.confirm_delete import confirm
+from calibre.utils.localization import localize_user_manual_link
 
 
 class StoreAction(InterfaceAction):
@@ -146,8 +147,9 @@ class StoreAction(InterfaceAction):
             'buying from. Be sure to double check that any books you get '
             'will work with your e-book reader, especially if the book you '
             'are buying has '
-            'DRM.'
-            )), 'about_get_books_msg',
+            'DRM.'
+            ).format(localize_user_manual_link(
+                'https://manual.calibre-ebook.com/drm.html'))), 'about_get_books_msg',
             parent=self.gui, show_cancel_button=False,
             confirm_msg=_('Show this message again'),
             pixmap='dialog_information.png', title=_('About Get books'))
diff --git a/src/calibre/gui2/dialogs/drm_error.ui b/src/calibre/gui2/dialogs/drm_error.ui
index 08d3f3d60a..6d3a39e519 100644
--- a/src/calibre/gui2/dialogs/drm_error.ui
+++ b/src/calibre/gui2/dialogs/drm_error.ui
@@ -44,8 +44,7 @@
     
      
       <p>This book is locked by <b>DRM</b>. To learn more about DRM and why you cannot read or convert this book in calibre, 
-          <a href="https://drmfree.calibre-ebook.com/about#drm">click here</a>.<p>A large number of recent, DRM free releases are 
-          available at <a href="https://drmfree.calibre-ebook.com">Open Books</a>.
+          <a href="https://manual.calibre-ebook.com/drm.html">click here</a>.<p>
      
      
       true
diff --git a/src/calibre/gui2/store/stores/open_books_plugin.py b/src/calibre/gui2/store/stores/open_books_plugin.py
deleted file mode 100644
index 8f8c5dd305..0000000000
--- a/src/calibre/gui2/store/stores/open_books_plugin.py
+++ /dev/null
@@ -1,80 +0,0 @@
-# -*- coding: utf-8 -*-
-from __future__ import absolute_import, division, print_function, unicode_literals
-
-store_version = 1  # Needed for dynamic plugin loading
-
-__license__ = 'GPL 3'
-__copyright__ = '2011, John Schember '
-__docformat__ = 'restructuredtext en'
-
-from contextlib import closing
-try:
-    from urllib.parse import quote_plus
-except ImportError:
-    from urllib import quote_plus
-
-from lxml import html
-
-from PyQt5.Qt import QUrl
-
-from calibre import browser, url_slash_cleaner
-from calibre.gui2 import open_url
-from calibre.gui2.store import StorePlugin
-from calibre.gui2.store.basic_config import BasicStoreConfig
-from calibre.gui2.store.search_result import SearchResult
-from calibre.gui2.store.web_store_dialog import WebStoreDialog
-
-
-class OpenBooksStore(BasicStoreConfig, StorePlugin):
-
-    def open(self, parent=None, detail_item=None, external=False):
-        url = 'https://drmfree.calibre-ebook.com/'
-
-        if external or self.config.get('open_external', False):
-            open_url(QUrl(url_slash_cleaner(detail_item if detail_item else url)))
-        else:
-            d = WebStoreDialog(self.gui, url, parent, detail_item)
-            d.setWindowTitle(self.name)
-            d.set_tags(self.config.get('tags', ''))
-            d.exec_()
-
-    def search(self, query, max_results=10, timeout=60):
-        url = 'https://drmfree.calibre-ebook.com/search/?q=' + quote_plus(query)
-
-        br = browser()
-
-        counter = max_results
-        with closing(br.open(url, timeout=timeout)) as f:
-            doc = html.fromstring(f.read())
-            for data in doc.xpath('//ul[@id="object_list"]//li'):
-                if counter <= 0:
-                    break
-
-                id = ''.join(data.xpath('.//div[@class="links"]/a[1]/@href'))
-                id = id.strip()
-                if not id:
-                    continue
-
-                cover_url = ''.join(data.xpath('.//div[@class="cover"]/img/@src'))
-
-                price = ''.join(data.xpath('.//div[@class="price"]/text()'))
-                a, b, price = price.partition('Price:')
-                price = price.strip()
-                if not price:
-                    continue
-
-                title = ''.join(data.xpath('.//div/strong/text()'))
-                author = ''.join(data.xpath('.//div[@class="author"]//text()'))
-                author = author.partition('by')[-1]
-
-                counter -= 1
-
-                s = SearchResult()
-                s.cover_url = cover_url
-                s.title = title.strip()
-                s.author = author.strip()
-                s.price = price.strip()
-                s.detail_item = id.strip()
-                s.drm = SearchResult.DRM_UNLOCKED
-
-                yield s

From d7657cd34cdd5cbdbb849a5fed8cc120ea1a918b Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Wed, 11 Mar 2020 17:37:19 +0530
Subject: [PATCH 152/162] String changes

---
 manual/virtual_libraries.rst       | 23 +++++++++++------------
 src/calibre/gui2/tag_browser/ui.py |  2 +-
 src/pyj/book_list/local_books.pyj  | 13 ++++++++++---
 3 files changed, 22 insertions(+), 16 deletions(-)

diff --git a/manual/virtual_libraries.rst b/manual/virtual_libraries.rst
index d3138d6ced..8f471dd446 100644
--- a/manual/virtual_libraries.rst
+++ b/manual/virtual_libraries.rst
@@ -44,15 +44,15 @@ selected author.
 You can switch back to the full library at any time by once again clicking the
 :guilabel:`Virtual library` and selecting the entry named :guilabel:``.
 
-Virtual libraries are based on *searches*. You can use any search as the 
-basis of a virtual library. The virtual library will contain only the 
-books matched by that search. First, type in the search you want to use 
-in the Search bar or build a search using the :guilabel:`Tag browser`. 
-When you are happy with the returned results, click the Virtual library 
-button, choose :guilabel:`Create library` and enter a name for the new virtual 
-library. The virtual library will then be created based on the search 
-you just typed in. Searches are very powerful, for examples of the kinds 
-of things you can do with them, see :ref:`search_interface`. 
+Virtual libraries are based on *searches*. You can use any search as the
+basis of a Virtual library. The Virtual library will contain only the
+books matched by that search. First, type in the search you want to use
+in the Search bar or build a search using the :guilabel:`Tag browser`.
+When you are happy with the returned results, click the :guilabel:`Virtual library`
+button, choose :guilabel:`Create library` and enter a name for the new Virtual
+library. The Virtual library will then be created based on the search
+you just typed in. Searches are very powerful, for examples of the kinds
+of things you can do with them, see :ref:`search_interface`.
 
 Examples of useful Virtual libraries
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -82,7 +82,7 @@ You can edit a previously created virtual library or remove it, by clicking the
 
 You can tell calibre that you always want to apply a particular virtual library
 when the current library is opened, by going to
-:guilabel:`Preferences->Interface->Behavior`. 
+:guilabel:`Preferences->Interface->Behavior`.
 
 You can quickly use the current search as a temporary virtual library by
 clicking the :guilabel:`Virtual library` button and choosing the
@@ -103,7 +103,7 @@ example, ``vl:Read`` will find all the books in the *Read* virtual library. The
 ``vl:Read and vl:"Science Fiction"`` will find all the books that are in both the *Read* and
 *Science Fiction* virtual libraries.
 
-The value following ``vl:`` must be the name of a virtual library. If the virtual library name 
+The value following ``vl:`` must be the name of a virtual library. If the virtual library name
 contains spaces then surround it with quotes.
 
 One use for a virtual library search is in the content server. In
@@ -124,4 +124,3 @@ saved search that shows you unread books, you can click the :guilabel:`Virtual
 Library` button and choose the :guilabel:`Additional restriction` option to
 show only unread Historical Fiction books. To learn about saved searches, see
 :ref:`saved_searches`.
-
diff --git a/src/calibre/gui2/tag_browser/ui.py b/src/calibre/gui2/tag_browser/ui.py
index 86f37160eb..33cc4d9a55 100644
--- a/src/calibre/gui2/tag_browser/ui.py
+++ b/src/calibre/gui2/tag_browser/ui.py
@@ -114,7 +114,7 @@ class TagBrowserMixin(object):  # {{{
         if new_category_name is not None:
             new_name = new_category_name.replace('.', '')
         else:
-            new_name = _('New Category').replace('.', '')
+            new_name = _('New category').replace('.', '')
         n = new_name
         while True:
             new_cat = on_category_key[1:] + '.' + n
diff --git a/src/pyj/book_list/local_books.pyj b/src/pyj/book_list/local_books.pyj
index c815b22483..9b36bc08a3 100644
--- a/src/pyj/book_list/local_books.pyj
+++ b/src/pyj/book_list/local_books.pyj
@@ -3,7 +3,7 @@
 from __python__ import bound_methods, hash_literals
 
 from elementmaker import E
-from gettext import gettext as _
+from gettext import gettext as _, ngettext
 
 from book_list.globals import get_db
 from book_list.router import home, open_book
@@ -62,7 +62,10 @@ def confirm_delete_all():
             else:
                 close_modal()
 
-        msg = _('This will remove all {} downloaded books from local storage. Are you sure?').format(num_of_books)
+        msg = ngettext(
+            'This will remove the downloaded book from local storage. Are you sure?',
+            'This will remove all {} downloaded books from local storage. Are you sure?',
+            num_of_books).format(num_of_books)
         m = E.div()
         safe_set_inner_html(m, msg)
         parent.appendChild(E.div(
@@ -90,7 +93,11 @@ def delete_all(msg_parent, close_modal):
             refresh()
             return
         clear(msg_parent)
-        safe_set_inner_html(msg_parent, _('Deleting {} books, please wait...').format(books.length))
+        safe_set_inner_html(msg_parent, ngettext(
+            'Deleting one book, please wait...',
+            'Deleting {} books, please wait...',
+            books.length or 0).format(books.length)
+        )
         book_to_delete = books.pop()
         db.delete_book(book_list_data.book_data[book_to_delete], def(book, err_string):
             if err_string:

From 08049cbbbe571eab3b476e287d1e9e7ddfa7bb5a Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Thu, 12 Mar 2020 06:06:33 +0530
Subject: [PATCH 153/162] Metadata download dialog: Make the space occupied by
 the right panel adjustable

---
 src/calibre/gui2/metadata/single_download.py | 34 +++++++++++---------
 1 file changed, 18 insertions(+), 16 deletions(-)

diff --git a/src/calibre/gui2/metadata/single_download.py b/src/calibre/gui2/metadata/single_download.py
index 998cd25d31..1b157e8a20 100644
--- a/src/calibre/gui2/metadata/single_download.py
+++ b/src/calibre/gui2/metadata/single_download.py
@@ -20,7 +20,7 @@ from PyQt5.Qt import (
     QWidget, QTableView, QGridLayout, QPalette, QTimer, pyqtSignal,
     QAbstractTableModel, QSize, QListView, QPixmap, QModelIndex,
     QAbstractListModel, QRect, QTextBrowser, QStringListModel, QMenu,
-    QCursor, QHBoxLayout, QPushButton, QSizePolicy)
+    QCursor, QHBoxLayout, QPushButton, QSizePolicy, QSplitter)
 
 from calibre.customize.ui import metadata_plugins
 from calibre.ebooks.metadata import authors_to_string, rating_to_stars
@@ -317,8 +317,6 @@ class Comments(HTMLDisplay):  # {{{
     def __init__(self, parent=None):
         HTMLDisplay.__init__(self, parent)
         self.setAcceptDrops(False)
-        self.setMaximumWidth(300)
-        self.setMinimumWidth(300)
         self.wait_timer = QTimer(self)
         self.wait_timer.timeout.connect(self.update_wait)
         self.wait_timer.setInterval(800)
@@ -374,13 +372,6 @@ class Comments(HTMLDisplay):  # {{{
         
         '''%(c,)
         self.setHtml(templ%html)
-
-    def sizeHint(self):
-        # This is needed, because on windows the dialog cannot be resized to
-        # so that this widgets height become < sizeHint().height(). Qt sets the
-        # sizeHint to (800, 600), which makes the dialog unusable on smaller
-        # screens.
-        return QSize(800, 300)
 # }}}
 
 
@@ -454,31 +445,41 @@ class IdentifyWidget(QWidget):  # {{{
         self.abort = Event()
         self.caches = {}
 
-        self.l = l = QGridLayout()
-        self.setLayout(l)
+        self.l = l = QVBoxLayout(self)
 
         names = [''+p.name+'' for p in metadata_plugins(['identify']) if
                 p.is_configured()]
         self.top = QLabel('
'+_('calibre is downloading metadata from: ') +
             ', '.join(names))
         self.top.setWordWrap(True)
-        l.addWidget(self.top, 0, 0)
+        l.addWidget(self.top)
 
+        self.splitter = s = QSplitter(self)
+        s.setChildrenCollapsible(False)
+        l.addWidget(s, 100)
         self.results_view = ResultsView(self)
         self.results_view.book_selected.connect(self.emit_book_selected)
         self.get_result = self.results_view.get_result
-        l.addWidget(self.results_view, 1, 0)
+        s.addWidget(self.results_view)
 
         self.comments_view = Comments(self)
-        l.addWidget(self.comments_view, 1, 1)
+        s.addWidget(self.comments_view)
+        s.setStretchFactor(0, 2)
+        s.setStretchFactor(1, 1)
 
         self.results_view.show_details_signal.connect(self.comments_view.show_data)
 
         self.query = QLabel('download starting...')
         self.query.setWordWrap(True)
-        l.addWidget(self.query, 2, 0, 1, 2)
+        l.addWidget(self.query)
 
         self.comments_view.show_wait()
+        state = gprefs.get('metadata-download-identify-widget-splitter-state')
+        if state is not None:
+            s.restoreState(state)
+
+    def save_state(self):
+        gprefs['metadata-download-identify-widget-splitter-state'] = bytearray(self.splitter.saveState())
 
     def emit_book_selected(self, book):
         self.book_selected.emit(book, self.caches)
@@ -1091,6 +1092,7 @@ class FullFetch(QDialog):  # {{{
     def accept(self):
         # Prevent the usual dialog accept mechanisms from working
         gprefs['metadata_single_gui_geom'] = bytearray(self.saveGeometry())
+        self.identify_widget.save_state()
         if DEBUG_DIALOG:
             if self.stack.currentIndex() == 2:
                 return QDialog.accept(self)

From 8b473b8d9ccc3893d610b30a3be62f0bfe8a3204 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Fri, 13 Mar 2020 15:46:07 +0530
Subject: [PATCH 154/162] Fix edit open with applications not working from
 files browser in editor

---
 src/calibre/gui2/tweak_book/file_list.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/calibre/gui2/tweak_book/file_list.py b/src/calibre/gui2/tweak_book/file_list.py
index 94aeb22b04..cacd4bd094 100644
--- a/src/calibre/gui2/tweak_book/file_list.py
+++ b/src/calibre/gui2/tweak_book/file_list.py
@@ -206,7 +206,7 @@ class OpenWithHandler(object):  # {{{
         else:
             m.addSeparator()
             m.addAction(_('Add other application for %s files...') % fmt.upper(), partial(self.choose_open_with, file_name, fmt))
-            m.addAction(_('Edit Open With applications...'), partial(edit_programs, fmt, file_name))
+            m.addAction(_('Edit Open With applications...'), partial(edit_programs, fmt, self))
             menu.addMenu(m)
             menu.ow = m
 

From acbddd78457d1608f458c788e02b4e17c199aed9 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Fri, 13 Mar 2020 22:56:59 +0530
Subject: [PATCH 155/162] String changes

---
 Changelog.yaml                            | 2 +-
 manual/drm.rst                            | 2 +-
 src/calibre/gui2/actions/edit_metadata.py | 2 +-
 src/calibre/gui2/book_details.py          | 2 +-
 src/calibre/gui2/open_with.py             | 2 +-
 src/calibre/gui2/tweak_book/file_list.py  | 2 +-
 6 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/Changelog.yaml b/Changelog.yaml
index 486f415e4b..a73f1c5ed2 100644
--- a/Changelog.yaml
+++ b/Changelog.yaml
@@ -1123,7 +1123,7 @@
     - title: "Content server: Fix editing metadata that affects multiple books causing all the metadata for all the books to become the same."
       tickets: [1812781]
 
-    - title: "Open With: Fix using .bat files as the program not working."
+    - title: "Open with: Fix using .bat files as the program not working."
       tickets: [1811045]
 
     - title: "ZIP Output: Fix an error when building the ToC on macOS for some books with non-ASCII ToC entries"
diff --git a/manual/drm.rst b/manual/drm.rst
index 551ed63e6c..67ecd13f7c 100644
--- a/manual/drm.rst
+++ b/manual/drm.rst
@@ -16,7 +16,7 @@ the file or device, such as serial numbers or key-files. It can also refer to
 restrictions associated with specific instances of digital works or devices.
 DRM technologies attempt to control use of digital media by preventing access,
 copying or conversion to other formats by end users. See `wikipedia
-`_.
+`_.
 
 
 What does DRM imply for me personally?
diff --git a/src/calibre/gui2/actions/edit_metadata.py b/src/calibre/gui2/actions/edit_metadata.py
index 657d7cc1fc..f385defff9 100644
--- a/src/calibre/gui2/actions/edit_metadata.py
+++ b/src/calibre/gui2/actions/edit_metadata.py
@@ -312,7 +312,7 @@ class EditMetadataAction(InterfaceAction):
                 intro_msg=_('The downloaded metadata is on the left and the original metadata'
                             ' is on the right. If a downloaded value is blank or unknown,'
                             ' the original value is used.'),
-                action_button=(_('&View Book'), I('view.png'), self.gui.iactions['View'].view_historical),
+                action_button=(_('&View book'), I('view.png'), self.gui.iactions['View'].view_historical),
                 db=db
             )
             if d.exec_() == d.Accepted:
diff --git a/src/calibre/gui2/book_details.py b/src/calibre/gui2/book_details.py
index 1e89b26da9..5b77323a8f 100644
--- a/src/calibre/gui2/book_details.py
+++ b/src/calibre/gui2/book_details.py
@@ -212,7 +212,7 @@ def add_format_entries(menu, data, book_info):
         else:
             m.addSeparator()
             m.addAction(_('Add other application for %s files...') % fmt.upper(), partial(book_info.choose_open_with, book_id, fmt))
-            m.addAction(_('Edit Open With applications...'), partial(edit_programs, fmt, book_info))
+            m.addAction(_('Edit Open with applications...'), partial(edit_programs, fmt, book_info))
             menu.addMenu(m)
             menu.ow = m
         if fmt.upper() in SUPPORTED:
diff --git a/src/calibre/gui2/open_with.py b/src/calibre/gui2/open_with.py
index 8eed3f1757..c829fe8972 100644
--- a/src/calibre/gui2/open_with.py
+++ b/src/calibre/gui2/open_with.py
@@ -444,7 +444,7 @@ def register_keyboard_shortcuts(gui=None, finalize=False):
             unique_name = application['uuid']
             func = partial(gui.open_with_action_triggerred, filetype, application)
             ac.triggered.connect(func)
-            gui.keyboard.register_shortcut(unique_name, name, action=ac, group=_('Open With'))
+            gui.keyboard.register_shortcut(unique_name, name, action=ac, group=_('Open with'))
             gui.addAction(ac)
             registered_shortcuts[unique_name] = ac
     if finalize:
diff --git a/src/calibre/gui2/tweak_book/file_list.py b/src/calibre/gui2/tweak_book/file_list.py
index cacd4bd094..d533947a73 100644
--- a/src/calibre/gui2/tweak_book/file_list.py
+++ b/src/calibre/gui2/tweak_book/file_list.py
@@ -206,7 +206,7 @@ class OpenWithHandler(object):  # {{{
         else:
             m.addSeparator()
             m.addAction(_('Add other application for %s files...') % fmt.upper(), partial(self.choose_open_with, file_name, fmt))
-            m.addAction(_('Edit Open With applications...'), partial(edit_programs, fmt, self))
+            m.addAction(_('Edit Open with applications...'), partial(edit_programs, fmt, self))
             menu.addMenu(m)
             menu.ow = m
 

From 47a711a87199b61816ca30769982f9a70b1a0290 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Sun, 15 Mar 2020 16:59:05 +0530
Subject: [PATCH 156/162] Update Spectator Magazine

---
 recipes/spectator_magazine.recipe | 155 ++++++++++++++++++++----------
 1 file changed, 106 insertions(+), 49 deletions(-)

diff --git a/recipes/spectator_magazine.recipe b/recipes/spectator_magazine.recipe
index f066f17936..fca46b08f8 100644
--- a/recipes/spectator_magazine.recipe
+++ b/recipes/spectator_magazine.recipe
@@ -1,10 +1,19 @@
+#!/usr/bin/env python2
+# vim:fileencoding=utf-8
+# License: GPLv3 Copyright: 2015, Kovid Goyal 
+
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+import json
+import re
+
+from mechanize import Request
+
 from calibre.web.feeds.recipes import BasicNewsRecipe
 
 
-def class_sel(cls):
-    def f(x):
-        return x and cls in x.split()
-    return f
+def absolutize(url):
+    return 'https://spectator.co.uk' + url
 
 
 class Spectator(BasicNewsRecipe):
@@ -15,52 +24,100 @@ class Spectator(BasicNewsRecipe):
     language = 'en'
 
     no_stylesheets = True
-
-    keep_only_tags = dict(name='div', attrs={
-                          'class': ['article-header__text', 'featured-image', 'article-content']})
-    remove_tags = [
-        dict(name='div', attrs={'id': ['disqus_thread']}),
-        dict(attrs={'class': ['middle-promo',
-                              'sharing', 'mejs-player-holder']}),
-        dict(name='a', onclick=lambda x: x and '__gaTracker' in x and 'outbound-article' in x),
-    ]
-    remove_tags_after = [
-        dict(name='hr', attrs={'class': 'sticky-clear'}),
-    ]
-
-    def parse_spec_section(self, div):
-        h2 = div.find('h2')
-        sectitle = self.tag_to_string(h2)
-        self.log('Section:', sectitle)
-        articles = []
-        for div in div.findAll('div', id=lambda x: x and x.startswith('post-')):
-            h2 = div.find('h2', attrs={'class': class_sel('term-item__title')})
-            if h2 is None:
-                h2 = div.find(attrs={'class': class_sel('news-listing__title')})
-            title = self.tag_to_string(h2)
-            a = h2.find('a')
-            url = a['href']
-            desc = ''
-            self.log('\tArticle:', title)
-            p = div.find(attrs={'class': class_sel('term-item__excerpt')})
-            if p is not None:
-                desc = self.tag_to_string(p)
-            articles.append({'title': title, 'url': url, 'description': desc})
-        return sectitle, articles
+    use_embedded_content = True
 
     def parse_index(self):
-        soup = self.index_to_soup('https://www.spectator.co.uk/magazine/')
-        a = soup.find('a', attrs={'class': 'issue-details__cover-link'})
-        self.timefmt = ' [%s]' % a['title']
-        self.cover_url = a['href']
-        if self.cover_url.startswith('//'):
-            self.cover_url = 'http:' + self.cover_url
+        br = self.get_browser()
+        main_js = br.open_novisit('https://spectator.co.uk/main.js').read().decode('utf-8')
+        data = {}
+        fields = ('apiKey', 'apiSecret', 'contentEnvironment', 'siteUrl', 'magazineIssueContentUrl', 'contentUrl')
+        pat = r'this.({})\s*=\s*"(.+?)"'.format('|'.join(fields))
+        for m in re.finditer(pat, main_js):
+            data[m.group(1)] = m.group(2)
+        self.log('Got Spectator data:', data)
+        headers = {
+            'api_key': data['apiKey'],
+            'origin': data['siteUrl'],
+            'access_token': data['apiSecret'],
+            'Accept-language': 'en-GB,en-US;q=0.9,en;q=0.8',
+            'Accept-encoding': 'gzip, deflate',
+            'Accept': '*/*',
+        }
 
-        feeds = []
+        def make_url(utype, query, includes=(), limit=None):
+            ans = data[utype] + '/entries?environment=' + data['contentEnvironment']
+            if limit is not None:
+                ans += '&limit={}'.format(limit)
+            for inc in includes:
+                ans += '&include[]=' + inc
+            ans += '&query=' + json.dumps(query)
+            return ans
 
-        div = soup.find(attrs={'class': class_sel('content-area')})
-        for x in div.findAll(attrs={'class': class_sel('magazine-section-holder')}):
-            title, articles = self.parse_spec_section(x)
-            if articles:
-                feeds.append((title, articles))
-        return feeds
+        def get_result(url):
+            self.log('Fetching:', url)
+            req = Request(url, headers=headers)
+            raw = br.open_novisit(req).read().decode('utf-8')
+            return json.loads(raw)['entries']
+
+        # Get current issue
+        url = data['magazineIssueContentUrl'] + '/entries?environment=' + data['contentEnvironment'] + "&desc=issue_date&limit=1&only[BASE][]=url"
+        result = get_result(url)
+        slug = result[0]['url']
+        uid = result[0]['uid']  # noqa
+        date = slug.split('/')[-1]
+        self.log('Downloading issue:', date)
+
+        # Cover information
+        url = make_url(
+            'magazineIssueContentUrl',
+            {'url': slug},
+            limit=1
+        )
+        self.cover_url = get_result(url)[0]['magazine_cover']['url']
+        self.log('Found cover:', self.cover_url)
+
+        # List of articles
+        url = make_url(
+            'contentUrl',
+            {
+                "magazine_content_production_only.magazine_issue": {
+                    "$in_query": {"url": slug},
+                    "_content_type_uid": "magazine_issue"
+                },
+                "_content_type_uid": "article"
+            },
+            includes=(
+                'topic', 'magazine_content_production_only.magazine_issue',
+                'magazine_content_production_only.magazine_subsection', 'author'
+            )
+        )
+        result = get_result(url)
+        articles = {}
+        for entry in result:
+            title = entry['title']
+            url = absolutize(entry['url'])
+            blocks = []
+            a = blocks.append
+            byline = entry.get('byline') or ''
+            if byline:
+                a('
{}
'.format(byline))
+            if entry.get('author'):
+                for au in reversed(entry['author']):
+                    au = entry['author'][0]
+                    cac = ''
+                    if au.get('caricature'):
+                        cac = ''.format(au['caricature']['url'])
+                    a('
{} 
'.format(hi['url']))
+                if hi.get('description'):
+                    a('
{}
'.format(hi['description']))
+            a(entry['text_body'])
+            section = 'Unknown'
+            if entry.get('topic'):
+                topic = entry['topic'][0]
+                section = topic['title']
+            articles.setdefault(section, []).append({
+                'title': title, 'url': url, 'description': byline, 'content': '\n\n'.join(blocks)})
+        return [(sec, articles[sec]) for sec in sorted(articles)]

From d860c2a51f5d2044c0ad15632954a9b26b141783 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Sun, 15 Mar 2020 17:20:44 +0530
Subject: [PATCH 157/162] Fix #1867512 [Recipe that doesn't
 work](https://bugs.launchpad.net/calibre/+bug/1867512)

---
 recipes/20_minutos.recipe   |   2 +-
 recipes/20minutos.recipe    |  65 ------------------------------------
 recipes/icons/20minutos.png | Bin 661 -> 0 bytes
 3 files changed, 1 insertion(+), 66 deletions(-)
 delete mode 100644 recipes/20minutos.recipe
 delete mode 100644 recipes/icons/20minutos.png

diff --git a/recipes/20_minutos.recipe b/recipes/20_minutos.recipe
index 208fbc7401..c74cae4440 100644
--- a/recipes/20_minutos.recipe
+++ b/recipes/20_minutos.recipe
@@ -13,7 +13,7 @@ from calibre.web.feeds.news import BasicNewsRecipe
 
 class AdvancedUserRecipe1294946868(BasicNewsRecipe):
 
-    title = u'20 Minutos new'
+    title = u'20 Minutos'
     publisher = u'Grupo 20 Minutos'
 
     __author__ = 'Luis Hernandez'
diff --git a/recipes/20minutos.recipe b/recipes/20minutos.recipe
deleted file mode 100644
index e40fc174fa..0000000000
--- a/recipes/20minutos.recipe
+++ /dev/null
@@ -1,65 +0,0 @@
-__license__ = 'GPL v3'
-__copyright__ = '2011, Darko Miletic '
-'''
-www.20minutos.es
-'''
-
-from calibre.web.feeds.news import BasicNewsRecipe
-
-
-class t20Minutos(BasicNewsRecipe):
-    title = '20 Minutos'
-    __author__ = 'Darko Miletic'
-    description = 'Diario de informacion general y local mas leido de Espania, noticias de ultima hora de Espania, el mundo, local, deportes, noticias curiosas y mas'  # noqa
-    publisher = '20 Minutos Online SL'
-    category = 'news, politics, Spain'
-    oldest_article = 2
-    max_articles_per_feed = 200
-    no_stylesheets = True
-    encoding = 'utf8'
-    use_embedded_content = True
-    language = 'es'
-    remove_empty_feeds = True
-    publication_type = 'newspaper'
-    masthead_url = 'http://estaticos.20minutos.es/css4/img/ui/logo-301x54.png'
-    extra_css             = """
-                               body{font-family: Arial,Helvetica,sans-serif }
-                               img{margin-bottom: 0.4em; display:block}
-                            """
-
-    conversion_options = {
-        'comment': description, 'tags': category, 'publisher': publisher, 'language': language
-    }
-
-    remove_tags = [dict(attrs={'class': 'mf-viral'})]
-    remove_attributes = ['border']
-
-    feeds = [
-
-    (u'Principal', u'http://20minutos.feedsportal.com/c/32489/f/478284/index.rss'),
-    (u'Cine', u'http://20minutos.feedsportal.com/c/32489/f/478285/index.rss'),
-    (u'Internacional', u'http://20minutos.feedsportal.com/c/32489/f/492689/index.rss'),
-    (u'Deportes', u'http://20minutos.feedsportal.com/c/32489/f/478286/index.rss'),
-    (u'Nacional', u'http://20minutos.feedsportal.com/c/32489/f/492688/index.rss'),
-    (u'Economia', u'http://20minutos.feedsportal.com/c/32489/f/492690/index.rss'),
-    (u'Tecnologia', u'http://20minutos.feedsportal.com/c/32489/f/478292/index.rss')
-    ]
-
-    def preprocess_html(self, soup):
-        for item in soup.findAll(style=True):
-            del item['style']
-        for item in soup.findAll('a'):
-            limg = item.find('img')
-            if item.string is not None:
-                str = item.string
-                item.replaceWith(str)
-            else:
-                if limg:
-                    item.name = 'div'
-                    item.attrs = []
-                else:
-                    str = self.tag_to_string(item)
-                    item.replaceWith(str)
-        for item in soup.findAll('img', alt=False):
-            item['alt'] = 'image'
-        return soup
diff --git a/recipes/icons/20minutos.png b/recipes/icons/20minutos.png
deleted file mode 100644
index 8d3df68ca28f0daef6d1d585d472182ef29f6dcb..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 661
zcmV;G0&4wT7tb}ioI)+#&ed&dz!|Cp3a1#&XlUo
zoU+!Xw$`P()~&wQt;626zRtbF&c?&Wt+vLkuDz|UwWX%5l$xcCp0#_1rE`LuYkibj
zX@ogplrdeDDOsE;S*0mer7>NlF=3@SWu-Y~tvP9}5J8LxJd6lDga9~%05F6AEQ|m&
zi~u-{05p^UEPDtvd&bk}&eP}K-{;=j-sj)m=j-?H@Avoj_y7C%|NsB@`~Uy*?rqHau|D%|sGNb?W
z>vVc%zV2pD;7<(U2PSqsh2J$WjiE1-=eN-W#3^Y`2f)mEk{b*mHS`uj2zqZ;P4}b}
z3rYR9Dkp$FHvri&X^xzJ+8QJ`fYu9`
zUWJ6>dc@5x*MehcK#_ub+qSoIV~7IsnG#rwTd+^@8dm^BhcxRg4_6|i7if|6C@uy3
z7P`3>5tAfhtUB!`qHcMv>$Mp~Wx8s)u56_)F+6WBNU{hy^Db{lF*&$c2kcFB4?61u
vTb(DlF+wo0Q`nXiHqk)iS!PKFP53uopK(rSVU8G^00000NkvXXu0mjfKT#~-


From eba29b14d0694a5233694d964e34538e83c2827b Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Sun, 15 Mar 2020 17:22:14 +0530
Subject: [PATCH 158/162] Remove non-working recipe

---
 recipes/ZIVE.sk.recipe | 43 ------------------------------------------
 1 file changed, 43 deletions(-)
 delete mode 100644 recipes/ZIVE.sk.recipe

diff --git a/recipes/ZIVE.sk.recipe b/recipes/ZIVE.sk.recipe
deleted file mode 100644
index f1d5c2febb..0000000000
--- a/recipes/ZIVE.sk.recipe
+++ /dev/null
@@ -1,43 +0,0 @@
-from calibre.web.feeds.news import BasicNewsRecipe
-import re
-
-
-class ZiveRecipe(BasicNewsRecipe):
-    __license__ = 'GPL v3'
-    __author__ = 'Abelturd'
-    language = 'sk'
-    version = 1
-
-    title = u'ZIVE.sk'
-    publisher = u''
-    category = u'News, Newspaper'
-    description = u'Naj\u010d\xedtanej\u0161\xed denn\xedk opo\u010d\xedta\u010doch, IT a internete. '
-    encoding = 'UTF-8'
-
-    oldest_article = 7
-    max_articles_per_feed = 100
-    use_embedded_content = False
-    remove_empty_feeds = True
-
-    no_stylesheets = True
-    remove_javascript = True
-    cover_url = 'http://www.zive.sk/Client.Images/Logos/logo-zive-sk.gif'
-
-    feeds = []
-    feeds.append((u'V\u0161etky \u010dl\xe1nky',
-                  u'http://www.zive.sk/rss/sc-47/default.aspx'))
-
-    preprocess_regexps = [
-        (re.compile(r'
Pokra.*ie
', re.DOTALL | re.IGNORECASE),
-         lambda match: ''),
-
-    ]
-
-    remove_tags = []
-
-    keep_only_tags = [dict(name='h1'), dict(name='span', attrs={
-        'class': 'arlist-data-info-author'}), dict(name='div', attrs={'class': 'bbtext font-resizer-area'}), ]
-    extra_css = '''
-                h1 {font-size:140%;font-family:georgia,serif; font-weight:bold}
-                h3 {font-size:115%;font-family:georgia,serif; font-weight:bold}
-                '''

From 199ed3daf5d4baa129e6162758f279eb550a743b Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Sun, 15 Mar 2020 17:26:11 +0530
Subject: [PATCH 159/162] String changes

---
 Changelog.yaml                                   |  6 +++---
 manual/simple_index.rst                          |  2 +-
 manual/sub_groups.rst                            | 10 +++++-----
 src/calibre/gui2/dialogs/saved_search_editor.py  |  2 +-
 src/calibre/gui2/preferences/metadata_sources.ui |  2 +-
 5 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/Changelog.yaml b/Changelog.yaml
index a73f1c5ed2..f1587abc14 100644
--- a/Changelog.yaml
+++ b/Changelog.yaml
@@ -1049,7 +1049,7 @@
     - title: "Allow adding files to selected book records from the clipboard. To use copy a file from windows explorer, right click the Add books button and choose: Add files to selected books from clipboard"
       tickets: [1815419]
 
-    - title: "Tag browser: When right clicking on a saved search add a menu option to search using the raw search expression."
+    - title: "Tag browser: When right clicking on a Saved search add a menu option to search using the raw search expression."
       tickets: [1816274]
 
     - title: "Tag browser: Have pressing the Enter key find the next match."
@@ -2209,7 +2209,7 @@ to appear as Unknown if metadata management was set to manual in calibre."
 
     - title: "Edit book: Pre-select existing cover image (if any) in add cover dialog"
 
-    - title: "Make the Manage saved searches dialog a little easier for new users."
+    - title: "Make the Manage Saved searches dialog a little easier for new users."
       tickets: [1733163]
 
     - title: "Add a tweak to control behavior of Enter on the book list"
@@ -2226,7 +2226,7 @@ to appear as Unknown if metadata management was set to manual in calibre."
     - title: "Content server: Improve rendering of tags/categories with long words on small screens."
       tickets: [1734119]
 
-    - title: "Fix first added saved search not appearing in Tag browser until calibre restart."
+    - title: "Fix first added Saved search not appearing in Tag browser until calibre restart."
       tickets: [1733151]
 
     - title: "When checking added books for duplicates, also check on the language field. So books with the same title/authors but different languages are not considered duplicates."
diff --git a/manual/simple_index.rst b/manual/simple_index.rst
index bc1607d34d..60eec123d2 100644
--- a/manual/simple_index.rst
+++ b/manual/simple_index.rst
@@ -21,7 +21,7 @@ available `_.
 
 .. only:: online
 
-    **An e-book version of this user manual is available in** `EPUB format `_,  `AZW3 (Kindle Fire) format `_ and `PDF format `_.
+    **An e-book version of this User Manual is available in** `EPUB format `_,  `AZW3 (Kindle Fire) format `_ and `PDF format `_.
 
 .. rubric:: Sections
 
diff --git a/manual/sub_groups.rst b/manual/sub_groups.rst
index 086640e954..d318c37150 100644
--- a/manual/sub_groups.rst
+++ b/manual/sub_groups.rst
@@ -55,7 +55,7 @@ Setup
 
 By now, your question might be "How was all of this setup?" There are three steps: 1) create the custom column, 2) tell calibre that the new column is to be treated as a hierarchy, and 3) add genres.
 
-You create the custom column in the usual way, using Preferences -> Add your own columns. This example uses "#genre" as the lookup name and "Genre" as the column heading. The column type is "Comma-separated text, like tags, shown in the Tag browser." 
+You create the custom column in the usual way, using Preferences -> Add your own columns. This example uses "#genre" as the lookup name and "Genre" as the column heading. The column type is "Comma-separated text, like tags, shown in the Tag browser."
 
 .. image:: images/sg_cc.jpg
     :align: center
@@ -98,7 +98,7 @@ The Tag browser search mechanism knows if an item has children. If it does, clic
 Restrictions
 ---------------
 
-If you search for a genre then create a saved search for it, you can use the 'restrict to' box to create a virtual library of books with that genre. This is useful if you want to do other searches within the genre or to manage/update metadata for books in the genre. Continuing our example, you can create a saved search named 'History.Japanese' by first clicking on the genre Japanese in the Tag browser to get a search into the search box, entering History.Japanese into the saved search box, then pushing the "save search" button (the green box with the white plus, on the right-hand side).
+If you search for a genre then create a saved search for it, you can use the 'restrict to' box to create a Virtual library of books with that genre. This is useful if you want to do other searches within the genre or to manage/update metadata for books in the genre. Continuing our example, you can create a Saved search named 'History.Japanese' by first clicking on the genre Japanese in the Tag browser to get a search into the search box, entering History.Japanese into the saved search box, then pushing the "save search" button (the green box with the white plus, on the right-hand side).
 
 .. image:: images/sg_restrict.jpg
     :align: center
@@ -110,11 +110,11 @@ After creating the saved search, you can use it as a restriction.
 
 Useful template functions
 -------------------------
- 
+
  You might want to use the genre information in a template, such as with save to disk or send to device. The question might then be "How do I get the outermost genre name or names?" A calibre template function, subitems, is provided to make doing this easier.
- 
+
  For example, assume you want to add the outermost genre level to the save-to-disk template to make genre folders, as in "History/The Gathering Storm - Churchill, Winston". To do this, you must extract the first level of the hierarchy and add it to the front along with a slash to indicate that it should make a folder. The template below accomplishes this::
- 
+
     {#genre:subitems(0,1)||/}{title} - {authors}
 
 See :ref:`The template language ` for more information about templates and the :func:`subitems` function.
diff --git a/src/calibre/gui2/dialogs/saved_search_editor.py b/src/calibre/gui2/dialogs/saved_search_editor.py
index ec100c7f8f..1b51f8060c 100644
--- a/src/calibre/gui2/dialogs/saved_search_editor.py
+++ b/src/calibre/gui2/dialogs/saved_search_editor.py
@@ -94,7 +94,7 @@ class SavedSearchEditor(Dialog):
     def __init__(self, parent, initial_search=None):
         self.initial_search = initial_search
         Dialog.__init__(
-            self, _('Manage saved searches'), 'manage-saved-searches', parent)
+            self, _('Manage Saved searches'), 'manage-saved-searches', parent)
 
     def setup_ui(self):
         from calibre.gui2.ui import get_gui
diff --git a/src/calibre/gui2/preferences/metadata_sources.ui b/src/calibre/gui2/preferences/metadata_sources.ui
index 5165482c2a..3d93a19491 100644
--- a/src/calibre/gui2/preferences/metadata_sources.ui
+++ b/src/calibre/gui2/preferences/metadata_sources.ui
@@ -119,7 +119,7 @@
                Restore your own subset of checked fields that you define using the 'Set as default' button
               
               
-               &Select default
+               Select &default
               
              
             

From a7d1f4c88f283b68972fd89a0c78b60906d3d416 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Sun, 15 Mar 2020 17:31:29 +0530
Subject: [PATCH 160/162] Update Glasgow Herald

---
 recipes/glasgow_herald.recipe | 22 +++++++++++++++-------
 1 file changed, 15 insertions(+), 7 deletions(-)

diff --git a/recipes/glasgow_herald.recipe b/recipes/glasgow_herald.recipe
index 7f83b141d4..388dff9783 100644
--- a/recipes/glasgow_herald.recipe
+++ b/recipes/glasgow_herald.recipe
@@ -1,3 +1,9 @@
+#!/usr/bin/env python
+# vim:fileencoding=utf-8
+# License: GPLv3 Copyright: 2015, Kovid Goyal 
+
+from __future__ import absolute_import, division, print_function, unicode_literals
+
 from calibre.web.feeds.news import BasicNewsRecipe
 
 
@@ -15,10 +21,12 @@ class GlasgowHerald(BasicNewsRecipe):
     auto_cleanup = True
 
     feeds = [
-        (u'News', u'http://www.heraldscotland.com/cmlink/1.758'),
-        (u'Sport', u'http://www.heraldscotland.com/cmlink/1.761'),
-        (u'Business', u'http://www.heraldscotland.com/cmlink/1.763'),
-        (u'Life & Style', u'http://www.heraldscotland.com/cmlink/1.770'),
-        (u'Arts & Entertainment',
-         u'http://www.heraldscotland.com/cmlink/1.768',),
-        (u'Columnists', u'http://www.heraldscotland.com/cmlink/1.658574')]
+        (u'News', u'https://www.heraldscotland.com/news/rss/'),
+        (u'Sport', u'https://www.heraldscotland.com/sport/rss/'),
+        (u'Business', u'https://www.heraldscotland.com/business_hq/rss/'),
+        (u'Lifestyle', u'https://www.heraldscotland.com/life_style/rss/'),
+        (u'Arts & Entertainment', u'https://www.heraldscotland.com/arts_ents/rss/',),
+        (u'Politics', u'https://www.heraldscotland.com/politics/rss/'),
+        (u'Columnists', u'https://www.heraldscotland.com/opinion/columnists/rss/')
+
+        ]

From ac0d67ee6fe11e05736babd9a45285ba9b84437b Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Sun, 15 Mar 2020 17:49:32 +0530
Subject: [PATCH 161/162] Remove non-working recipes

---
 recipes/icons/rossijkaja_gazeta.png | Bin 620 -> 0 bytes
 recipes/icons/vedomosti.png         | Bin 430 -> 0 bytes
 recipes/rossijkaja_gazeta.recipe    |  72 ----------
 recipes/vedomosti.recipe            | 207 ----------------------------
 4 files changed, 279 deletions(-)
 delete mode 100644 recipes/icons/rossijkaja_gazeta.png
 delete mode 100644 recipes/icons/vedomosti.png
 delete mode 100644 recipes/rossijkaja_gazeta.recipe
 delete mode 100644 recipes/vedomosti.recipe

diff --git a/recipes/icons/rossijkaja_gazeta.png b/recipes/icons/rossijkaja_gazeta.png
deleted file mode 100644
index 3b883db5b4e1deb3ffb4f24a4771df7694a482c9..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 620
zcmV-y0+aoTP)3voVtf?AHi-C$(%3@jLP_pT!M_;rXmHE2Vm0$Lxc&wE948Wv>plL}@T6#9
z)07T#A^4fYzm;ozLDi4tl`BQDEm*esBYKp#>nc9+GQ11QUF!W*?kuPd!&l>qV1fPkO@ZR%-7~eoD=;}
zhq=7L_yqn)pEjOMI9cN}`6Mt2HUNGve*g#nf^mQ!GmL-w!g6O<`xkrw0000K>Jy%)zg
zaqgn@o-j5LgznBRw4+zogMi~A9|q)DRG0|rAg6DFcAn4tcdf^Ea1)Dxp4
zO0dMQ0%d2t9-?^)nPe0wXIn~+q%AIp-g)76*i|>7Lo6ItZsmeX?uF0@=0r1vtC0|rZH)`c+2>XQ3YSR^wpW>eMijnFk46x<{907*qoM6N<$g8nGJR{#J2

diff --git a/recipes/rossijkaja_gazeta.recipe b/recipes/rossijkaja_gazeta.recipe
deleted file mode 100644
index 9929eb7350..0000000000
--- a/recipes/rossijkaja_gazeta.recipe
+++ /dev/null
@@ -1,72 +0,0 @@
-# vim:fileencoding=utf-8
-from calibre.web.feeds.news import BasicNewsRecipe
-
-
-class AdjectiveSpecies(BasicNewsRecipe):
-    title = u'Российская Газета'
-    __author__ = 'bug_me_not'
-    cover_url = u'http://img.rg.ru/img/d/logo2012.png'
-    description = 'Российская Газета'
-    publisher = 'Правительство Российской Федерации'
-    category = 'news'
-    language = 'ru'
-    no_stylesheets = True
-    remove_javascript = True
-    oldest_article = 300
-    max_articles_per_feed = 100
-
-    remove_tags_before = dict(name='h1')
-    remove_tags_after = dict(name='div', attrs={'class': 'ar-citate'})
-    remove_tags = [dict(name='div', attrs={'class': 'insert_left'}),
-                   dict(name='a', attrs={'href': '#comments'}),
-                   dict(name='div', attrs={'class': 'clear'}),
-                   dict(name='div', attrs={'class': 'ar-citate'}),
-                   dict(name='div', attrs={'class': 'ar-social red'}),
-                   dict(name='div', attrs={'class': 'clear clear-head'}), ]
-
-    feeds = [
-        (u'Все материалы', u'http://www.rg.ru/tema/rss.xml'),
-        (u'Еженедельный выпуск',
-         u'http://www.rg.ru/tema/izd-subbota/rss.xml'),
-        (u'Государство',
-         u'http://www.rg.ru/tema/gos/rss.xml'),
-        (u'Экономика',
-         u'http://www.rg.ru/tema/ekonomika/rss.xml'),
-        (u'Бизнес',
-         u'http://www.rg.ru/tema/izd-biznes/rss.xml'),
-        (u'В мире', u'http://www.rg.ru/tema/mir/rss.xml'),
-        (u'Происшествия',
-         u'http://www.rg.ru/tema/bezopasnost/rss.xml'),
-        (u'Общество',
-         u'http://www.rg.ru/tema/obshestvo/rss.xml'),
-        (u'Культура',
-         u'http://www.rg.ru/tema/kultura/rss.xml'),
-        (u'Спорт', u'http://www.rg.ru/tema/sport/rss.xml'),
-        (u'Документы', u'http://rg.ru/tema/doc-any/rss.xml'),
-        (u'РГ: Башкортостан',
-         u'http://www.rg.ru/org/filial/bashkortostan/rss.xml'),
-        (u'РГ: Волга-Кама',
-         u'http://www.rg.ru/org/filial/volga-kama/rss.xml'),
-        (u'РГ: Восточная Сибирь',
-         u'http://www.rg.ru/org/filial/enisey/rss.xml'),
-        (u'РГ: Дальний Восток',
-         u'http://www.rg.ru/org/filial/dvostok/rss.xml'),
-        (u'РГ: Кубань. Северный Кавказ',
-         u'http://www.rg.ru/org/filial/kuban/rss.xml'),
-        (u'РГ: Пермский край',
-         u'http://www.rg.ru/org/filial/permkray/rss.xml'),
-        (u'РГ: Приволжье',
-         u'http://www.rg.ru/org/filial/privolzhe/rss.xml'),
-        (u'РГ: Северо-Запад',
-         u'http://www.rg.ru/org/filial/szapad/rss.xml'),
-        (u'РГ: Сибирь',
-         u'http://www.rg.ru/org/filial/sibir/rss.xml'),
-        (u'РГ: Средняя Волга',
-         u'http://www.rg.ru/org/filial/svolga/rss.xml'),
-        (u'РГ: Урал и Западная Сибирь',
-         u'http://www.rg.ru/org/filial/ural/rss.xml'),
-        (u'РГ: Центральная Россия',
-         u'http://www.rg.ru/org/filial/roscentr/rss.xml'),
-        (u'РГ: Юг России',
-         u'http://www.rg.ru/org/filial/jugrossii/rss.xml'),
-    ]
diff --git a/recipes/vedomosti.recipe b/recipes/vedomosti.recipe
deleted file mode 100644
index 0270e221b1..0000000000
--- a/recipes/vedomosti.recipe
+++ /dev/null
@@ -1,207 +0,0 @@
-#!/usr/bin/env  python2
-
-u'''
-Ведомости
-'''
-
-from calibre.web.feeds.feedparser import parse
-from calibre.ebooks.BeautifulSoup import Tag
-from calibre.web.feeds.news import BasicNewsRecipe
-
-
-def new_tag(soup, name, attrs=()):
-    impl = getattr(soup, 'new_tag', None)
-    if impl is not None:
-        return impl(name, attrs=dict(attrs))
-    return Tag(soup, name, attrs=attrs or None)
-
-
-class VedomostiRecipe(BasicNewsRecipe):
-    title = u'Ведомости'
-    __author__ = 'Nikolai Kotchetkov'
-    publisher = 'vedomosti.ru'
-    category = 'press, Russia'
-    description = u'Ежедневная деловая газета'
-    oldest_article = 3
-    max_articles_per_feed = 100
-
-    masthead_url = u'http://motorro.com/imgdir/logos/ved_logo_black2_cropped.gif'
-    cover_url = u'http://motorro.com/imgdir/logos/ved_logo_black2_cropped.gif'
-
-    # Add feed names if you want them to be sorted (feeds of this list appear
-    # first)
-    sortOrder = [u'_default', u'Первая полоса', u'Власть и деньги']
-
-    encoding = 'cp1251'
-    language = 'ru'
-    no_stylesheets = True
-    remove_javascript = True
-    recursions = 0
-
-    conversion_options = {
-        'comment': description, 'tags': category, 'publisher': publisher, 'language': language
-    }
-
-    keep_only_tags = [dict(name='td', attrs={'class': ['second_content']})]
-
-    remove_tags_after = [dict(name='div', attrs={'class': 'article_text'})]
-
-    remove_tags = [
-        dict(name='div', attrs={'class': ['sep', 'choice', 'articleRightTbl']})]
-
-    feeds = [u'http://www.vedomosti.ru/newspaper/out/rss.xml']
-
-    # base URL for relative links
-    base_url = u'http://www.vedomosti.ru'
-
-    extra_css = 'h1 {font-size: 1.5em; margin: 0em 0em 0em 0em; text-align: center;}'\
-                'h2 {font-size: 1.0em; margin: 0em 0em 0em 0em;}'\
-                'h3 {font-size: 0.8em; margin: 0em 0em 0em 0em;}'\
-                '.article_date {font-size: 0.5em; color: gray; font-family: monospace; text-align:right;}'\
-                '.article_authors {font-size: 0.5em; color: gray; font-family: monospace; text-align:right;}'\
-                '.article_img {width:100%; text-align: center; padding: 3px 3px 3px 3px;}'\
-                '.article_img_desc {width:100%; text-align: center; font-size: 0.5em; color: gray; font-family: monospace;}'\
-                '.article_desc {font-size: 1em; font-style:italic;}'
-
-    def parse_index(self):
-        try:
-            feedData = parse(self.feeds[0])
-            if not feedData:
-                raise NotImplementedError
-            self.log("parse_index: Feed loaded successfully.")
-            try:
-                if feedData.feed.title:
-                    self.title = feedData.feed.title
-                    self.log("parse_index: Title updated to: ", self.title)
-            except Exception:
-                pass
-            try:
-                if feedData.feed.description:
-                    self.description = feedData.feed.description
-                    self.log("parse_index: Description updated to: ",
-                            self.description)
-            except Exception:
-                pass
-
-            def get_virtual_feed_articles(feed):
-                if feed in feeds:
-                    return feeds[feed][1]
-                self.log("Adding new feed: ", feed)
-                articles = []
-                feeds[feed] = (feed, articles)
-                return articles
-
-            feeds = {}
-
-            # Iterate feed items and distribute articles using tags
-            for item in feedData.entries:
-                link = item.get('link', '')
-                title = item.get('title', '')
-                if '' == link or '' == title:
-                    continue
-                article = {'title': title, 'url': link, 'description': item.get(
-                    'description', ''), 'date': item.get('date', ''), 'content': ''}
-                if not item.get('tags'):  # noqa
-                    get_virtual_feed_articles('_default').append(article)
-                    continue
-                for tag in item.tags:
-                    addedToDefault = False
-                    term = tag.get('term', '')
-                    if '' == term:
-                        if (not addedToDefault):
-                            get_virtual_feed_articles(
-                                '_default').append(article)
-                        continue
-                    get_virtual_feed_articles(term).append(article)
-
-            # Get feed list
-            # Select sorted feeds first of all
-            result = []
-            for feedName in self.sortOrder:
-                if (not feeds.get(feedName)):
-                    continue
-                result.append(feeds[feedName])
-                del feeds[feedName]
-            result = result + feeds.values()
-
-            return result
-
-        except Exception as err:
-            self.log(err)
-            raise NotImplementedError
-
-    def preprocess_html(self, soup):
-        return self.adeify_images(soup)
-
-    def postprocess_html(self, soup, first_fetch):
-
-        # Find article
-        contents = soup.find('div', {'class': ['article_text']})
-        if not contents:
-            self.log('postprocess_html: article div not found!')
-            return soup
-        contents.extract()
-
-        # Find title
-        title = soup.find('h1')
-        if title:
-            contents.insert(0, title)
-
-        # Find article image
-        newstop = soup.find('div', {'class': ['newstop']})
-        if newstop:
-            img = newstop.find('img')
-            if img:
-                imgDiv = new_tag(soup, 'div')
-                imgDiv['class'] = 'article_img'
-
-                if img.get('width'):
-                    del(img['width'])
-                if img.get('height'):
-                    del(img['height'])
-
-                # find description
-                element = img.parent.nextSibling
-
-                img.extract()
-                imgDiv.insert(0, img)
-
-                while element:
-                    if not isinstance(element, Tag):
-                        continue
-                    nextElement = element.nextSibling
-                    if 'p' == element.name:
-                        element.extract()
-                        element['class'] = 'article_img_desc'
-                        imgDiv.insert(len(imgDiv.contents), element)
-                    element = nextElement
-
-                contents.insert(1, imgDiv)
-
-        # find article abstract
-        abstract = soup.find('p', {'class': ['subhead']})
-        if abstract:
-            abstract['class'] = 'article_desc'
-            contents.insert(2, abstract)
-
-        # Find article authors
-        authorsDiv = soup.find('div', {'class': ['autors']})
-        if authorsDiv:
-            authorsP = authorsDiv.find('p')
-            if authorsP:
-                authorsP['class'] = 'article_authors'
-                contents.insert(len(contents.contents), authorsP)
-
-        # Fix urls that use relative path
-        urls = contents.findAll('a', href=True)
-        if urls:
-            for url in urls:
-                if '/' == url['href'][0]:
-                    url['href'] = self.base_url + url['href']
-
-        body = soup.find('td', {'class': ['second_content']})
-        if body:
-            body.replaceWith(contents)
-
-        self.log('Result: ', soup.prettify())
-        return soup

From a15acae96d310cea9395c64290d07640af518753 Mon Sep 17 00:00:00 2001
From: Kovid Goyal 
Date: Sun, 15 Mar 2020 18:29:26 +0530
Subject: [PATCH 162/162] Update LA Times

---
 recipes/latimes.recipe | 126 +++++++----------------------------------
 1 file changed, 19 insertions(+), 107 deletions(-)

diff --git a/recipes/latimes.recipe b/recipes/latimes.recipe
index 0fd773ec22..da7d2afc1d 100644
--- a/recipes/latimes.recipe
+++ b/recipes/latimes.recipe
@@ -2,13 +2,9 @@
 
 import re
 from collections import defaultdict
-from pprint import pformat
 
-from calibre.utils.date import strptime, utcnow
 from calibre.web.feeds.news import BasicNewsRecipe
 
-DT_EPOCH = strptime('1970-01-01', '%Y-%m-%d', assume_utc=True)
-
 DIR_COLLECTIONS = [['world'],
                    ['nation'],
                    ['politics'],
@@ -29,84 +25,22 @@ DIR_COLLECTIONS = [['world'],
                    ['travel'],
                    ['fashion']]
 
-SECTIONS=['THE WORLD',
-          'THE NATION',
-          'POLITICS',
-          'OPINION',
-          'CALIFORNIA',
-          'OBITUARIES',
-          'BUSINESS',
-          'HOLLYWOOD',
-          'SPORTS',
-          'ENTERTAINMENT',
-          'MOVIES',
-          'TELEVISION',
-          'BOOKS',
-          'FOOD',
-          'HEALTH',
-          'SCIENCE AND TECHNOLOGY',
-          'HOME',
-          'TRAVEL',
-          'FASHION',
-          'NEWSLETTERS'
-          'OTHER']
+
+def classes(classes):
+    q = frozenset(classes.split(' '))
+    return dict(attrs={
+        'class': lambda x: x and frozenset(x.split()).intersection(q)})
 
 
 def absurl(url):
     if url.startswith('/'):
-        url = 'http://www.latimes.com' + url
+        url = 'https://www.latimes.com' + url
     return url
 
 
-def check_words(words):
-    return lambda x: x and frozenset(words.split()).intersection(x.split())
-
-
 def what_section(url):
-    if re.compile(r'^https?://www[.]latimes[.]com/local/obituaries').search(url):
-        return 'OBITUARIES'
-    elif re.compile(r'^https?://www[.]latimes[.]com/business/hollywood').search(url):
-        return 'HOLLYWOOD'
-    elif re.compile(r'^https?://www[.]latimes[.]com/entertainment/movies').search(url):
-        return 'MOVIES'
-    elif re.compile(r'^https?://www[.]latimes[.]com/entertainment/tv').search(url):
-        return 'TELEVISION'
-    elif re.compile(r'^https?://www[.]latimes[.]com/business/technology').search(url):
-        return 'SCIENCE AND TECHNOLOGY'
-    elif re.compile(r'^https?://www[.]latimes[.]com/world').search(url):
-        return 'THE WORLD'
-    elif re.compile(r'^https?://www[.]latimes[.]com/nation').search(url):
-        return 'THE NATION'
-    elif re.compile(r'^https?://www[.]latimes[.]com/politics').search(url):
-        return 'POLITICS'
-    elif re.compile(r'^https?://www[.]latimes[.]com/opinion').search(url):
-        return 'OPINION'
-    elif re.compile(r'^https?://www[.]latimes[.]com/(?:local|style)').search(url):
-        return 'CALIFORNIA'
-    elif re.compile(r'^https?://www[.]latimes[.]com/business').search(url):
-        return 'BUSINESS'
-    elif re.compile(r'^https?://www[.]latimes[.]com/sports').search(url):
-        return 'SPORTS'
-    elif re.compile(r'^https?://www[.]latimes[.]com/entertainment').search(url):
-        return 'ENTERTAINMENT'
-    elif re.compile(r'^https?://www[.]latimes[.]com/books').search(url):
-        return 'BOOKS'
-    elif re.compile(r'^https?://www[.]latimes[.]com/food').search(url):
-        return 'FOOD'
-    elif re.compile(r'^https?://www[.]latimes[.]com/health').search(url):
-        return 'HEALTH'
-    elif re.compile(r'^https?://www[.]latimes[.]com/science').search(url):
-        return 'SCIENCE AND TECHNOLOGY'
-    elif re.compile(r'^https?://www[.]latimes[.]com/home').search(url):
-        return 'HOME'
-    elif re.compile(r'^https?://www[.]latimes[.]com/travel').search(url):
-        return 'TRAVEL'
-    elif re.compile(r'^https?://www[.]latimes[.]com/fashion').search(url):
-        return 'FASHION'
-    elif re.compile(r'^https?://www[.]latimes[.]com/newsletter').search(url):
-        return 'NEWSLETTERS'
-    else:
-        return 'OTHER'
+    parts = url.split('/')
+    return parts[-4].capitalize()
 
 
 class LATimes(BasicNewsRecipe):
@@ -126,32 +60,25 @@ class LATimes(BasicNewsRecipe):
     cover_url = 'http://www.latimes.com/includes/sectionfronts/A1.pdf'
 
     keep_only_tags = [
-        dict(name='header', attrs={'id': 'top'}),
-        dict(name='article'),
-        dict(name='div', attrs={'id': 'liveblog-story-wrapper'})
+        classes('ArticlePage-breadcrumbs ArticlePage-headline ArticlePage-mainContent'),
     ]
 
     remove_tags= [
-        dict(name='div', attrs={'class': check_words(
-            'hidden-tablet hidden-mobile hidden-desktop pb-f-ads-dfp')})
-    ]
-
-    remove_tags_after = [
-        dict(name='div', attrs={'class': check_words('pb-f-article-body')})
+        classes('ArticlePage-actions Enhancement hidden-tablet hidden-mobile hidden-desktop pb-f-ads-dfp')
     ]
 
     def parse_index(self):
-        index = 'http://www.latimes.com/'
-        pat = r'^(?:https?://www[.]latimes[.]com)?/[^#]+20[0-9]{6}-(?:html)?story[.]html'
+        index = 'https://www.latimes.com/'
+        pat = r'^https://www\.latimes\.com/[^/]+?/story/20\d{2}-\d{2}-\d{2}/\S+'
         articles = self.find_articles(index, pat)
         for collection in DIR_COLLECTIONS:
+            if self.test:
+                continue
             topdir = collection.pop(0)
-            index = 'http://www.latimes.com/' + topdir + '/'
-            pat = r'^(?:https?://www[.]latimes[.]com)?/' + \
-                      topdir + '/[^#]+20[0-9]{6}-(?:html)?story[.]html'
-            articles += self.find_articles(index, pat)
+            collection_index = index + topdir + '/'
+            articles += self.find_articles(collection_index, pat)
             for subdir in collection:
-                sub_index = index + subdir + '/'
+                sub_index = collection_index + subdir + '/'
                 articles += self.find_articles(sub_index, pat)
 
         feeds = defaultdict(list)
@@ -159,12 +86,7 @@ class LATimes(BasicNewsRecipe):
             section = what_section(article['url'])
             feeds[section].append(article)
 
-        keys = []
-        for key in SECTIONS:
-            if key in feeds.keys():
-                keys.append(key)
-        self.log(pformat(dict(feeds)))
-        return [(k, feeds[k]) for k in keys]
+        return [(k, feeds[k]) for k in sorted(feeds)]
 
     def preprocess_html(self, soup):
         for img in soup.findAll('img', attrs={'data-src': True}):
@@ -190,16 +112,6 @@ class LATimes(BasicNewsRecipe):
         alinks = [a for a in alinks if len(
             a.contents) == 1 and a.find(text=True, recursive=False)]
         articles = [
-            {'title': a.find(text=True), 'url': absurl(a['href'])} for a in alinks]
-        date_rx = re.compile(
-            r'^https?://www[.]latimes[.]com/[^#]+-(?P20[0-9]{6})-(?:html)?story[.]html')
-        for article in articles:
-            mdate = date_rx.match(article['url'])
-            if mdate is not None:
-                try:
-                    article['timestamp'] = (strptime(mdate.group('date'),'%Y%m%d') - DT_EPOCH).total_seconds()
-                except Exception:
-                    article['timestamp'] = (utcnow() - DT_EPOCH).total_seconds()
-                article['url'] = mdate.group(0)
+            {'title': self.tag_to_string(a), 'url': absurl(a['href'])} for a in alinks]
         self.log('Found: ', len(articles), ' articles.\n')
         return articles