diff --git a/Changelog.yaml b/Changelog.yaml index ed0d4ba0a3..f1587abc14 100644 --- a/Changelog.yaml +++ b/Changelog.yaml @@ -20,6 +20,100 @@ # new recipes: # - title: +- version: 4.12.0 + date: 2020-03-06 + + new features: + - title: "Kobo driver: Add support for new firmware with the series list on the device" + + - title: "Automatically extract the source DOCX file from Kindle Create KPF files when adding them to calibre. If you prefer to preserve the KPF file you can disable the KPF Extract plugin in Preferences->Plugins" + + - title: "Content server: Add a button to delete all locally cached books." + tickets: [1864305] + + - title: "Edit Book: Allow selecting the contents of a tag with Ctrl+Alt+t" + + - title: "Viewer: Save current position after 3 seconds of last position change. Useful if the viewer crashes on resume from sleep." + + - title: "Viewer: Add a keyboard shortcut (Ctrl+w) to toggle the scrollbar." + tickets: [1864356] + + - title: "Viewer: Keyboard shortcuts to change number of columns (Ctrl+[ and Ctrl+])" + + bug fixes: + - title: "Fix the Douban metadata download plugin" + tickets: [1853091] + + - title: "Viewer: Fix searching in Regex and Whole words mode not working well." + + - title: "Viewer: Fix searching for multiple words in fixed layout books not working." + tickets: [1863464] + + - title: "RTF Input: Fix handling of RTF files with invalid encoded text." + tickets: [1864719] + + - title: "PDF Output: Add a hangcheck for loading HTML if there is no progress for sixty seconds abort" + tickets: [1865380] + + - title: 'Viewer: When starting without a book allowing quitting the viewer by clicking the close button on the "Open book" page' + tickets: [1864343] + + improved recipes: + - Wired + - ABC News Australia + + new recipes: + - title: Spectator Australia + author: James Cridland + +- version: 4.11.2 + date: 2020-02-21 + + new features: + - title: "Viewer: Allow right clicking on the scrollbar to easily access commonly used scrolling shortcuts" + + - title: "Edit book: Preview panel: Allow right clicking on images to edit them" + + - title: "Add a new Quick select action to quickly select a virtual library with a few keystrokes. Activated by Ctrl+t or the Virtual library menu" + + - title: "Viewer: Calculate default column widths based on current font size" + + - title: "Viewer: Add a control to quit the viewer useful on touchscreens." + tickets: [1862441] + + - title: "Viewer: Add shortcut for showing metadata (Ctrl+n)" + tickets: [1862432] + + bug fixes: + - title: "4.11.2 fixes a couple of regressions that broke the New bookmark button in the viewer and scrolling in the content server library view. Also fixes calculation of default column widths in viewer not changing when font size is changed." + + - title: "Viewer: Fix a regression that broke detection of pop-up footnotes using EPUB 3 markup" + + - title: "Viewer: Fix current reading position not preserved when changing preferences and auto scroll is active." + tickets: [1863438] + + - title: "Viewer: Fix stopping autoscroll at end of chapter not stopping next chapter jump." + tickets: [1863487] + + - title: "Fix for viewer window going off screen even when not restoring window geometry" + + - title: "Edit book: Fix syntax highlighting for break-(before|after)" + tickets: [1863020] + + - title: "Fix drag and drop of some image files onto edit metadata dialog not working" + tickets: [1862440] + + - title: "Conversion pipeline: Fix styles applied via selectors to the element being ignored" + tickets: [1862401] + + - title: "Bulk metadata edit: Fix clear series not resetting series index" + + - title: "Fix clicking on author name in book details panel to search in Goodreads not working if author has more than two parts in his name" + + + improved recipes: + - New York Times + - version: 4.10.0 date: 2020-02-07 @@ -955,7 +1049,7 @@ - title: "Allow adding files to selected book records from the clipboard. To use copy a file from windows explorer, right click the Add books button and choose: Add files to selected books from clipboard" tickets: [1815419] - - title: "Tag browser: When right clicking on a saved search add a menu option to search using the raw search expression." + - title: "Tag browser: When right clicking on a Saved search add a menu option to search using the raw search expression." tickets: [1816274] - title: "Tag browser: Have pressing the Enter key find the next match." @@ -1029,7 +1123,7 @@ - title: "Content server: Fix editing metadata that affects multiple books causing all the metadata for all the books to become the same." tickets: [1812781] - - title: "Open With: Fix using .bat files as the program not working." + - title: "Open with: Fix using .bat files as the program not working." tickets: [1811045] - title: "ZIP Output: Fix an error when building the ToC on macOS for some books with non-ASCII ToC entries" @@ -2083,7 +2177,7 @@ to appear as Unknown if metadata management was set to manual in calibre." - title: "Content server: Open links in the comments section from the book details page in new windows." tickets: [1737644] - - title: "Choose English as the User interface language when a locale related environment variable is set to the C locale" + - title: "Choose English as the user interface language when a locale related environment variable is set to the C locale" - title: "Linux installer: A nicer error message if the user tries to run the installer on an ARM machine" @@ -2115,7 +2209,7 @@ to appear as Unknown if metadata management was set to manual in calibre." - title: "Edit book: Pre-select existing cover image (if any) in add cover dialog" - - title: "Make the Manage saved searches dialog a little easier for new users." + - title: "Make the Manage Saved searches dialog a little easier for new users." tickets: [1733163] - title: "Add a tweak to control behavior of Enter on the book list" @@ -2132,7 +2226,7 @@ to appear as Unknown if metadata management was set to manual in calibre." - title: "Content server: Improve rendering of tags/categories with long words on small screens." tickets: [1734119] - - title: "Fix first added saved search not appearing in Tag browser until calibre restart." + - title: "Fix first added Saved search not appearing in Tag browser until calibre restart." tickets: [1733151] - title: "When checking added books for duplicates, also check on the language field. So books with the same title/authors but different languages are not considered duplicates." diff --git a/README.md b/README.md index c72a26ff6e..d25f07e4b0 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ reading. It is cross platform, running on Linux, Windows and macOS. For more information, see the [calibre About page](https://calibre-ebook.com/about) -[![Build Status](https://github.com/kovidgoyal/calibre/workflows/Continuous%20Integration/badge.svg)](https://github.com/kovidgoyal/calibre/actions?workflow=Continuous+Integration) +[![Build Status](https://github.com/kovidgoyal/calibre/workflows/Continuous%20Integration/badge.svg)](https://github.com/kovidgoyal/calibre/actions?query=workflow%3ACI) ## Screenshots diff --git a/bypy/macos/util.c b/bypy/macos/util.c index ba46caefd7..0b3a40bbf9 100644 --- a/bypy/macos/util.c +++ b/bypy/macos/util.c @@ -163,7 +163,6 @@ run(const char **ENV_VARS, const char **ENV_VAR_VALS, char *PROGRAM, char *t = NULL; int ret = 0, i; PyObject *site, *mainf, *res; - uint32_t buf_size = PATH_MAX+1; for (i = 0; i < 3; i++) { t = rindex(full_exe_path, '/'); diff --git a/imgsrc/srv/window-restore.svg b/imgsrc/srv/window-restore.svg new file mode 100644 index 0000000000..d0d3c7e076 --- /dev/null +++ b/imgsrc/srv/window-restore.svg @@ -0,0 +1 @@ + diff --git a/manual/catalogs.rst b/manual/catalogs.rst index 9aed8ae4a4..d356bd8cfa 100644 --- a/manual/catalogs.rst +++ b/manual/catalogs.rst @@ -1,6 +1,6 @@ .. _catalog_tut: -Creating AZW3 • EPUB • MOBI catalogs +Creating AZW3 • EPUB • MOBI catalogs ===================================== calibre's Create catalog feature enables you to create a catalog of your library in a variety of formats. This help file describes cataloging options when generating a catalog in AZW3, EPUB and MOBI formats. @@ -19,7 +19,7 @@ If you want only *some* of your library cataloged, you have two options: * Create a multiple selection of the books you want cataloged. With more than one book selected in calibre's main window, only the selected books will be cataloged. * Use the Search field or the Tag browser to filter the displayed books. Only the displayed books will be cataloged. -To begin catalog generation, select the menu item :guilabel:`Convert books > Create a catalog of the books in your calibre library`. You may also add a :guilabel:`Create Catalog` button to a toolbar in :guilabel:`Preferences > Interface > Toolbars` for easier access to the Generate catalog dialog. +To begin catalog generation, select the menu item :guilabel:`Convert books > Create a catalog of the books in your calibre library`. You may also add a :guilabel:`Create catalog` button to a toolbar in :guilabel:`Preferences > Interface > Toolbars & menus` for easier access to the Generate catalog dialog. .. image:: images/catalog_options.png :alt: Catalog options @@ -125,7 +125,7 @@ Custom catalog covers .. |cc| image:: images/custom_cover.png -|cc| With the `Generate Cover plugin `_ installed, you can create custom covers for your catalog. +|cc| With the `Generate Cover plugin `_ installed, you can create custom covers for your catalog. To install the plugin, go to :guilabel:`Preferences > Advanced > Plugins > Get new plugins`. Additional help resources @@ -134,4 +134,3 @@ Additional help resources For more information on calibre's Catalog feature, see the MobileRead forum sticky `Creating Catalogs - Start here `_, where you can find information on how to customize the catalog templates, and how to submit a bug report. To ask questions or discuss calibre's Catalog feature with other users, visit the MobileRead forum `Calibre Catalogs `_. - diff --git a/manual/develop.rst b/manual/develop.rst index b60453fc08..aca049091d 100644 --- a/manual/develop.rst +++ b/manual/develop.rst @@ -136,7 +136,7 @@ for inclusion into the main calibre repository: git clone git@github.com:/calibre.git git remote add upstream https://github.com/kovidgoyal/calibre.git - Replace above with your github username. That will get your fork checked out locally. + Replace above with your GitHub username. That will get your fork checked out locally. * You can make changes and commit them whenever you like. When you are ready to have your work merged, do a:: git push diff --git a/manual/drm.rst b/manual/drm.rst new file mode 100644 index 0000000000..67ecd13f7c --- /dev/null +++ b/manual/drm.rst @@ -0,0 +1,107 @@ + +.. _dmr: + +Digital Rights Management (DRM) +=============================================== + +Digital rights management (DRM) is a generic term for access control +technologies that can be used by hardware manufacturers, publishers, copyright +holders and individuals to try to impose limitations on the usage of digital +content and devices. It is also, sometimes, disparagingly described as Digital +Restrictions Management. The term is used to describe any technology which +inhibits uses (legitimate or otherwise) of digital content that were not +desired or foreseen by the content provider. The term generally doesn't refer +to other forms of copy protection which can be circumvented without modifying +the file or device, such as serial numbers or key-files. It can also refer to +restrictions associated with specific instances of digital works or devices. +DRM technologies attempt to control use of digital media by preventing access, +copying or conversion to other formats by end users. See `wikipedia +`_. + + +What does DRM imply for me personally? +------------------------------------------ + +When you buy an e-book with DRM you don't really own it but have purchased the +permission to use it in a manner dictated to you by the seller. DRM limits what +you can do with e-books you have "bought". Often people who buy books with DRM +are unaware of the extent of these restrictions. These restrictions prevent you +from reformatting the e-book to your liking, including making stylistic changes +like adjusting the font sizes, although there is software that empowers you to +do such things for non DRM books. People are often surprised that an e-book +they have bought in a particular format cannot be converted to another format +if the e-book has DRM. So if you have an Amazon Kindle and buy a book sold by +Barnes and Nobles, you should know that if that e-book has DRM you will not be +able to read it on your Kindle. Notice that I am talking about a book you buy, +not steal or pirate but BUY. + + +What does DRM do for authors? +---------------------------------- + +Publishers of DRMed e-books argue that the DRM is all for the sake of authors +and to protect their artistic integrity and prevent piracy. But DRM does NOT +prevent piracy. People who want to pirate content or use pirated content still +do it and succeed. The three major DRM schemes for e-books today are run by +Amazon, Adobe and Barnes and Noble and all three DRM schemes have been cracked. +All DRM does is inconvenience legitimate users. It can be argued that it +actually harms authors as people who would have bought the book choose to find +a pirated version as they are not willing to put up with DRM. Those that would +pirate in the absence of DRM do so in its presence as well. To reiterate, the +key point is that DRM *does not prevent piracy*. So DRM is not only pointless +and harmful to buyers of e-books but also a waste of money. + + +DRM and freedom +------------------- + +Although digital content can be used to make information as well as creative +works easily available to everyone and empower humanity, this is not in the +interests of some publishers who want to steer people away from this +possibility of freedom simply to maintain their relevance in world developing +so fast that they cant keep up. + + +Why does calibre not support DRM? +------------------------------------- + +calibre is open source software while DRM by its very nature is closed. If +calibre were to support opening or viewing DRM files it could be trivially +modified to be used as a tool for DRM removal which is illegal under today's +laws. Open source software and DRM are a clash of principles. While DRM is all +about controlling the user open source software is about empowering the user. +The two simply can not coexist. + + +What is calibre's view on content providers? +------------------------------------------------ + +We firmly believe that authors and other content providers should be +compensated for their efforts, but DRM is not the way to go about it. We are +developing this database of DRM-free e-books from various sources to help you +find DRM-free alternatives and to help independent authors and publishers of +DRM-free e-books publicize their content. We hope you will find this useful and +we request that you do not pirate the content made available to you here. + + +How can I help fight DRM? +----------------------------- + +As somebody who reads and buys e-books you can help fight DRM. Do not buy +e-books with DRM. There are some publishers who publish DRM-free e-books. Make +an effort to see if they carry the e-book you are looking for. If you like +books by certain independent authors that sell DRM-free e-books and you can +afford it make donations to them. This is money well spent as their e-books +tend to be cheaper (there may be exceptions) than the ones you would buy from +publishers of DRMed books and would probably work on all devices you own in the +future saving you the cost of buying the e-book again. Do not discourage +publishers and authors of DRM-free e-books by pirating their content. Content +providers deserve compensation for their efforts. Do not punish them for trying +to make your reading experience better by making available DRM-free e-books. In +the long run this is detrimental to you. If you have bought books from sellers +that carry both DRMed as well as DRM-free books, not knowing if they carry DRM +or not make it a point to leave a comment or review on the website informing +future buyers of its DRM status. Many sellers do not think it important to +clearly indicate to their buyers if an e-book carries DRM or not. `Here +` you will find a Guide to +DRM-free living. diff --git a/manual/edit.rst b/manual/edit.rst index 45bb93202b..3b0e71d079 100644 --- a/manual/edit.rst +++ b/manual/edit.rst @@ -802,7 +802,8 @@ The HTML editor has very sophisticated syntax highlighting. Features include: * The text inside bold, italic and heading tags is made bold/italic * As you move your cursor through the HTML, the matching HTML tags are highlighted, and you can jump to the opening or closing tag with the - keyboard shortcuts :kbd:`Ctrl+{` and :kbd:`Ctrl+}` + keyboard shortcuts :kbd:`Ctrl+{` and :kbd:`Ctrl+}`. Similarly, you + can select the contents of a tag with :kbd:`Ctrl+Alt+T`. * Invalid HTML is highlighted with a red underline * Spelling errors in the text inside HTML tags and attributes such as title are highlighted. The spell checking is language aware, based on the value diff --git a/manual/faq.rst b/manual/faq.rst index f9b0862422..6cb991ef41 100644 --- a/manual/faq.rst +++ b/manual/faq.rst @@ -983,7 +983,7 @@ If you want to backup the calibre configuration/plugins, you have to backup the How do I use purchased EPUB books with calibre (or what do I do with .acsm files)? ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Most purchased EPUB books have `DRM `_. This prevents calibre from opening them. You can still use calibre to store and transfer them to your e-book reader. First, you must authorize your reader on a windows machine with Adobe Digital Editions. Once this is done, EPUB books transferred with calibre will work fine on your reader. When you purchase an epub book from a website, you will get an ".acsm" file. This file should be opened with Adobe Digital Editions, which will then download the actual ".epub" e-book. The e-book file will be stored in the folder "My Digital Editions", from where you can add it to calibre. +Most purchased EPUB books have :doc:`DRM `. This prevents calibre from opening them. You can still use calibre to store and transfer them to your e-book reader. First, you must authorize your reader on a windows machine with Adobe Digital Editions. Once this is done, EPUB books transferred with calibre will work fine on your reader. When you purchase an epub book from a website, you will get an ".acsm" file. This file should be opened with Adobe Digital Editions, which will then download the actual ".epub" e-book. The e-book file will be stored in the folder "My Digital Editions", from where you can add it to calibre. I am getting a "Permission Denied" error? ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/manual/gui.rst b/manual/gui.rst index ab48192309..f5e67d51a6 100644 --- a/manual/gui.rst +++ b/manual/gui.rst @@ -94,7 +94,7 @@ Convert books :class: float-right-img |cei| E-books can be converted from a number of formats into whatever format your e-book reader prefers. -Many e-books available for purchase will be protected by `Digital Rights Management `_ *(DRM)* technology. +Many e-books available for purchase will be protected by :doc:`Digital Rights Management ` *(DRM)* technology. calibre will not convert these e-books. It is easy to remove the DRM from many formats, but as this may be illegal, you will have to find tools to liberate your books yourself and then use calibre to convert them. diff --git a/manual/news.rst b/manual/news.rst index 630d2eb171..a6f3f999ef 100644 --- a/manual/news.rst +++ b/manual/news.rst @@ -40,7 +40,7 @@ and then the :guilabel:`Add a custom news source` menu item and then the .. image:: images/custom_news.png :align: center -First enter ``calibre Blog`` into the :guilabel:`Recipe title` field. This will be the title of the e-book that will be created from the articles in the above feeds. +First enter ``Calibre Blog`` into the :guilabel:`Recipe title` field. This will be the title of the e-book that will be created from the articles in the above feeds. The next two fields (:guilabel:`Oldest article` and :guilabel:`Max. number of articles`) allow you some control over how many articles should be downloaded from each feed, and they are pretty self explanatory. diff --git a/manual/simple_index.rst b/manual/simple_index.rst index 18a62cbd93..60eec123d2 100644 --- a/manual/simple_index.rst +++ b/manual/simple_index.rst @@ -21,7 +21,7 @@ available `_. .. only:: online - **An e-book version of this user manual is available in** `EPUB format `_, `AZW3 (Kindle Fire) format `_ and `PDF format `_. + **An e-book version of this User Manual is available in** `EPUB format `_, `AZW3 (Kindle Fire) format `_ and `PDF format `_. .. rubric:: Sections @@ -41,4 +41,5 @@ available `_. customize generated/en/cli-index develop + drm glossary diff --git a/manual/sub_groups.rst b/manual/sub_groups.rst index 086640e954..d318c37150 100644 --- a/manual/sub_groups.rst +++ b/manual/sub_groups.rst @@ -55,7 +55,7 @@ Setup By now, your question might be "How was all of this setup?" There are three steps: 1) create the custom column, 2) tell calibre that the new column is to be treated as a hierarchy, and 3) add genres. -You create the custom column in the usual way, using Preferences -> Add your own columns. This example uses "#genre" as the lookup name and "Genre" as the column heading. The column type is "Comma-separated text, like tags, shown in the Tag browser." +You create the custom column in the usual way, using Preferences -> Add your own columns. This example uses "#genre" as the lookup name and "Genre" as the column heading. The column type is "Comma-separated text, like tags, shown in the Tag browser." .. image:: images/sg_cc.jpg :align: center @@ -98,7 +98,7 @@ The Tag browser search mechanism knows if an item has children. If it does, clic Restrictions --------------- -If you search for a genre then create a saved search for it, you can use the 'restrict to' box to create a virtual library of books with that genre. This is useful if you want to do other searches within the genre or to manage/update metadata for books in the genre. Continuing our example, you can create a saved search named 'History.Japanese' by first clicking on the genre Japanese in the Tag browser to get a search into the search box, entering History.Japanese into the saved search box, then pushing the "save search" button (the green box with the white plus, on the right-hand side). +If you search for a genre then create a saved search for it, you can use the 'restrict to' box to create a Virtual library of books with that genre. This is useful if you want to do other searches within the genre or to manage/update metadata for books in the genre. Continuing our example, you can create a Saved search named 'History.Japanese' by first clicking on the genre Japanese in the Tag browser to get a search into the search box, entering History.Japanese into the saved search box, then pushing the "save search" button (the green box with the white plus, on the right-hand side). .. image:: images/sg_restrict.jpg :align: center @@ -110,11 +110,11 @@ After creating the saved search, you can use it as a restriction. Useful template functions ------------------------- - + You might want to use the genre information in a template, such as with save to disk or send to device. The question might then be "How do I get the outermost genre name or names?" A calibre template function, subitems, is provided to make doing this easier. - + For example, assume you want to add the outermost genre level to the save-to-disk template to make genre folders, as in "History/The Gathering Storm - Churchill, Winston". To do this, you must extract the first level of the hierarchy and add it to the front along with a slash to indicate that it should make a folder. The template below accomplishes this:: - + {#genre:subitems(0,1)||/}{title} - {authors} See :ref:`The template language ` for more information about templates and the :func:`subitems` function. diff --git a/manual/viewer.rst b/manual/viewer.rst index ee90fe3067..8697e31421 100644 --- a/manual/viewer.rst +++ b/manual/viewer.rst @@ -125,15 +125,15 @@ Dictionary lookup ------------------- You can look up the meaning of words in the current book by opening the -:guilabel:`Lookup/search panel` via the viewer controls. Then simply double -click on any word and its definition will be displayed in the lookup panel. +:guilabel:`Lookup/search word panel` via the viewer controls. Then simply double +click on any word and its definition will be displayed in the Lookup panel. Copying text and images ------------------------- You can select text and images by dragging the content with your mouse and then -right clicking and selecting "Copy" to copy to the clipboard. The copied +right clicking and selecting :guilabel:`Copy` to copy to the clipboard. The copied material can be pasted into another application as plain text and images. @@ -143,7 +143,7 @@ Non re-flowable content Some books have very wide content that content be broken up at page boundaries. For example tables or :code:`
` tags. In such cases, you should switch the
 viewer to *flow mode* by pressing :kbd:`Ctrl+m` to read this content.
-Alternately, you can also add the following CSS to the Styling section of the
+Alternately, you can also add the following CSS to the :guilabel:`Styles` section of the
 viewer preferences to force the viewer to break up lines of text in
 :code:`
` tags::
 
diff --git a/manual/virtual_libraries.rst b/manual/virtual_libraries.rst
index d3138d6ced..8f471dd446 100644
--- a/manual/virtual_libraries.rst
+++ b/manual/virtual_libraries.rst
@@ -44,15 +44,15 @@ selected author.
 You can switch back to the full library at any time by once again clicking the
 :guilabel:`Virtual library` and selecting the entry named :guilabel:``.
 
-Virtual libraries are based on *searches*. You can use any search as the 
-basis of a virtual library. The virtual library will contain only the 
-books matched by that search. First, type in the search you want to use 
-in the Search bar or build a search using the :guilabel:`Tag browser`. 
-When you are happy with the returned results, click the Virtual library 
-button, choose :guilabel:`Create library` and enter a name for the new virtual 
-library. The virtual library will then be created based on the search 
-you just typed in. Searches are very powerful, for examples of the kinds 
-of things you can do with them, see :ref:`search_interface`. 
+Virtual libraries are based on *searches*. You can use any search as the
+basis of a Virtual library. The Virtual library will contain only the
+books matched by that search. First, type in the search you want to use
+in the Search bar or build a search using the :guilabel:`Tag browser`.
+When you are happy with the returned results, click the :guilabel:`Virtual library`
+button, choose :guilabel:`Create library` and enter a name for the new Virtual
+library. The Virtual library will then be created based on the search
+you just typed in. Searches are very powerful, for examples of the kinds
+of things you can do with them, see :ref:`search_interface`.
 
 Examples of useful Virtual libraries
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -82,7 +82,7 @@ You can edit a previously created virtual library or remove it, by clicking the
 
 You can tell calibre that you always want to apply a particular virtual library
 when the current library is opened, by going to
-:guilabel:`Preferences->Interface->Behavior`. 
+:guilabel:`Preferences->Interface->Behavior`.
 
 You can quickly use the current search as a temporary virtual library by
 clicking the :guilabel:`Virtual library` button and choosing the
@@ -103,7 +103,7 @@ example, ``vl:Read`` will find all the books in the *Read* virtual library. The
 ``vl:Read and vl:"Science Fiction"`` will find all the books that are in both the *Read* and
 *Science Fiction* virtual libraries.
 
-The value following ``vl:`` must be the name of a virtual library. If the virtual library name 
+The value following ``vl:`` must be the name of a virtual library. If the virtual library name
 contains spaces then surround it with quotes.
 
 One use for a virtual library search is in the content server. In
@@ -124,4 +124,3 @@ saved search that shows you unread books, you can click the :guilabel:`Virtual
 Library` button and choose the :guilabel:`Additional restriction` option to
 show only unread Historical Fiction books. To learn about saved searches, see
 :ref:`saved_searches`.
-
diff --git a/recipes/1843.recipe b/recipes/1843.recipe
index 8805c35415..188f5ab4c9 100644
--- a/recipes/1843.recipe
+++ b/recipes/1843.recipe
@@ -15,7 +15,8 @@ def classes(classes):
 class E1843(BasicNewsRecipe):
     title = '1843'
     __author__ = 'Kovid Goyal'
-    language = 'en'
+    description = 'The ideas, culture and lifestyle magazine from The Economist'
+    language = 'en_GB'
     no_stylesheets = True
     remove_javascript = True
     oldest_article = 365
diff --git a/recipes/20_minutos.recipe b/recipes/20_minutos.recipe
index 208fbc7401..c74cae4440 100644
--- a/recipes/20_minutos.recipe
+++ b/recipes/20_minutos.recipe
@@ -13,7 +13,7 @@ from calibre.web.feeds.news import BasicNewsRecipe
 
 class AdvancedUserRecipe1294946868(BasicNewsRecipe):
 
-    title = u'20 Minutos new'
+    title = u'20 Minutos'
     publisher = u'Grupo 20 Minutos'
 
     __author__ = 'Luis Hernandez'
diff --git a/recipes/20minutos.recipe b/recipes/20minutos.recipe
deleted file mode 100644
index e40fc174fa..0000000000
--- a/recipes/20minutos.recipe
+++ /dev/null
@@ -1,65 +0,0 @@
-__license__ = 'GPL v3'
-__copyright__ = '2011, Darko Miletic '
-'''
-www.20minutos.es
-'''
-
-from calibre.web.feeds.news import BasicNewsRecipe
-
-
-class t20Minutos(BasicNewsRecipe):
-    title = '20 Minutos'
-    __author__ = 'Darko Miletic'
-    description = 'Diario de informacion general y local mas leido de Espania, noticias de ultima hora de Espania, el mundo, local, deportes, noticias curiosas y mas'  # noqa
-    publisher = '20 Minutos Online SL'
-    category = 'news, politics, Spain'
-    oldest_article = 2
-    max_articles_per_feed = 200
-    no_stylesheets = True
-    encoding = 'utf8'
-    use_embedded_content = True
-    language = 'es'
-    remove_empty_feeds = True
-    publication_type = 'newspaper'
-    masthead_url = 'http://estaticos.20minutos.es/css4/img/ui/logo-301x54.png'
-    extra_css             = """
-                               body{font-family: Arial,Helvetica,sans-serif }
-                               img{margin-bottom: 0.4em; display:block}
-                            """
-
-    conversion_options = {
-        'comment': description, 'tags': category, 'publisher': publisher, 'language': language
-    }
-
-    remove_tags = [dict(attrs={'class': 'mf-viral'})]
-    remove_attributes = ['border']
-
-    feeds = [
-
-    (u'Principal', u'http://20minutos.feedsportal.com/c/32489/f/478284/index.rss'),
-    (u'Cine', u'http://20minutos.feedsportal.com/c/32489/f/478285/index.rss'),
-    (u'Internacional', u'http://20minutos.feedsportal.com/c/32489/f/492689/index.rss'),
-    (u'Deportes', u'http://20minutos.feedsportal.com/c/32489/f/478286/index.rss'),
-    (u'Nacional', u'http://20minutos.feedsportal.com/c/32489/f/492688/index.rss'),
-    (u'Economia', u'http://20minutos.feedsportal.com/c/32489/f/492690/index.rss'),
-    (u'Tecnologia', u'http://20minutos.feedsportal.com/c/32489/f/478292/index.rss')
-    ]
-
-    def preprocess_html(self, soup):
-        for item in soup.findAll(style=True):
-            del item['style']
-        for item in soup.findAll('a'):
-            limg = item.find('img')
-            if item.string is not None:
-                str = item.string
-                item.replaceWith(str)
-            else:
-                if limg:
-                    item.name = 'div'
-                    item.attrs = []
-                else:
-                    str = self.tag_to_string(item)
-                    item.replaceWith(str)
-        for item in soup.findAll('img', alt=False):
-            item['alt'] = 'image'
-        return soup
diff --git a/recipes/ZIVE.sk.recipe b/recipes/ZIVE.sk.recipe
deleted file mode 100644
index f1d5c2febb..0000000000
--- a/recipes/ZIVE.sk.recipe
+++ /dev/null
@@ -1,43 +0,0 @@
-from calibre.web.feeds.news import BasicNewsRecipe
-import re
-
-
-class ZiveRecipe(BasicNewsRecipe):
-    __license__ = 'GPL v3'
-    __author__ = 'Abelturd'
-    language = 'sk'
-    version = 1
-
-    title = u'ZIVE.sk'
-    publisher = u''
-    category = u'News, Newspaper'
-    description = u'Naj\u010d\xedtanej\u0161\xed denn\xedk opo\u010d\xedta\u010doch, IT a internete. '
-    encoding = 'UTF-8'
-
-    oldest_article = 7
-    max_articles_per_feed = 100
-    use_embedded_content = False
-    remove_empty_feeds = True
-
-    no_stylesheets = True
-    remove_javascript = True
-    cover_url = 'http://www.zive.sk/Client.Images/Logos/logo-zive-sk.gif'
-
-    feeds = []
-    feeds.append((u'V\u0161etky \u010dl\xe1nky',
-                  u'http://www.zive.sk/rss/sc-47/default.aspx'))
-
-    preprocess_regexps = [
-        (re.compile(r'
Pokra.*ie
', re.DOTALL | re.IGNORECASE),
-         lambda match: ''),
-
-    ]
-
-    remove_tags = []
-
-    keep_only_tags = [dict(name='h1'), dict(name='span', attrs={
-        'class': 'arlist-data-info-author'}), dict(name='div', attrs={'class': 'bbtext font-resizer-area'}), ]
-    extra_css = '''
-                h1 {font-size:140%;font-family:georgia,serif; font-weight:bold}
-                h3 {font-size:115%;font-family:georgia,serif; font-weight:bold}
-                '''
diff --git a/recipes/abc_au.recipe b/recipes/abc_au.recipe
index ac21e6d730..eae6cee270 100644
--- a/recipes/abc_au.recipe
+++ b/recipes/abc_au.recipe
@@ -9,47 +9,53 @@ from calibre.web.feeds.recipes import BasicNewsRecipe
 
 class ABCNews(BasicNewsRecipe):
     title = 'ABC News'
-    __author__ = 'Pat Stapleton, Dean Cording'
-    description = 'News from Australia'
-    masthead_url = 'http://www.abc.net.au/news/assets/v5/images/common/logo-news.png'
-    cover_url = 'http://www.abc.net.au/news/assets/v5/images/common/logo-news.png'
-
+    __author__ = 'Pat Stapleton, Dean Cording, James Cridland'
+    description = 'From the Australian Broadcasting Corporation. The ABC is owned and funded by the Australian Government, but is editorially independent.'
+    masthead_url = 'https://www.abc.net.au/cm/lb/8212706/data/news-logo-2017---desktop-print-data.png'
+    cover_url = 'https://www.abc.net.au/news/linkableblob/8413676/data/abc-news-og-data.jpg'
+    cover_margins = (0,20,'#000000')
     oldest_article = 2
-    max_articles_per_feed = 100
-    no_stylesheets = False
+    handle_gzip = True
+    no_stylesheets = True
     use_embedded_content = False
+    scale_news_images_to_device = True
     encoding = 'utf8'
     publisher = 'ABC News'
-    category = 'News, Australia, World'
+    category = 'Australia,News'
     language = 'en_AU'
-    publication_type = 'newsportal'
-#    preprocess_regexps     = [(re.compile(r'', re.DOTALL), lambda m: '')]
-# Remove annoying map links (inline-caption class is also used for some
-# image captions! hence regex to match maps.google)
+    publication_type = 'newspaper'
+    extra_css = '.byline{font-size:smaller;margin-bottom:10px;}.inline-caption{display:block;font-size:smaller;text-decoration: none;}'
     preprocess_regexps = [(re.compile(
-        r'', re.DOTALL), lambda m: '')]
-    conversion_options = {
-        'comments': description, 'tags': category, 'language': language, 'publisher': publisher, 'linearize_tables': False
-    }
-
-    feeds = [
-        ('Top Stories', 'http://www.businessspectator.com.au/top-stories.rss'),
-        ('Alan Kohler', 'http://www.businessspectator.com.au/bs.nsf/RSS?readform&type=spectators&cat=Alan%20Kohler'),
-        ('Robert Gottliebsen', 'http://www.businessspectator.com.au/bs.nsf/RSS?readform&type=spectators&cat=Robert%20Gottliebsen'),
-        ('Stephen Bartholomeusz',
-         'http://www.businessspectator.com.au/bs.nsf/RSS?readform&type=spectators&cat=Stephen%20Bartholomeusz'),
-        ('Daily Dossier', 'http://www.businessspectator.com.au/bs.nsf/RSS?readform&type=kgb&cat=dossier'),
-        ('Australia', 'http://www.businessspectator.com.au/bs.nsf/RSS?readform&type=region&cat=australia'),
-    ]
diff --git a/recipes/courier_mail.recipe b/recipes/courier_mail.recipe
index b6a6d32c39..1dc90d3b5f 100644
--- a/recipes/courier_mail.recipe
+++ b/recipes/courier_mail.recipe
@@ -1,30 +1,32 @@
 from calibre.web.feeds.news import BasicNewsRecipe
+import datetime
 
 
 class Politics(BasicNewsRecipe):
-    title = u'Courier Mail'
+    title = u'The Courier-Mail'
+    description = 'Breaking news headlines for Brisbane and Queensland, Australia. The Courier-Mail is owned by News Corp Australia.'
     language = 'en_AU'
-    __author__ = 'Krittika Goyal'
+    __author__ = 'Krittika Goyal, James Cridland'
     oldest_article = 3  # days
     max_articles_per_feed = 20
     use_embedded_content = False
 
+    d = datetime.datetime.today()
+    cover_url='http://mfeeds.news.com.au/smedia/NCCOURIER/NCCM_1_' + d.strftime('%Y_%m_%d') + '_thumb_big.jpg'
+    masthead_url='https://couriermail.digitaleditions.com.au/images/couriermail-logo.jpg'
+
     no_stylesheets = True
     auto_cleanup = True
+    handle_gzip = True
 
     feeds = [
-        ('Top Stories',
-         'http://feeds.news.com.au/public/rss/2.0/bcm_top_stories_257.xml'),
-        ('Breaking News',
-         'http://feeds.news.com.au/public/rss/2.0/bcm_breaking_news_67.xml'),
-        ('Queensland News',
-         'http://feeds.news.com.au/public/rss/2.0/bcm_queensland_news_70.xml'),
-        ('Technology News',
-         'http://feeds.news.com.au/public/rss/2.0/bcm_technology_news_66.xml'),
-        ('Entertainment News',
-         'http://feeds.news.com.au/public/rss/2.0/bcm_entertainment_news_256.xml'),
-        ('Business News',
-         'http://feeds.news.com.au/public/rss/2.0/bcm_business_news_64.xml'),
-        ('Sport News',
-         'http://feeds.news.com.au/public/rss/2.0/bcm_sports_news_65.xml'),
+        ('Top Stories', 'http://www.couriermail.com.au/rss'),
+        ('Breaking', 'https://www.couriermail.com.au/news/breaking-news/rss'),
+        ('Queensland', 'https://www.couriermail.com.au/news/queensland/rss'),
+        ('Technology', 'https://www.couriermail.com.au/technology/rss'),
+        ('Entertainment', 'https://www.couriermail.com.au/entertainment/rss'),
+        ('Finance','https://www.couriermail.com.au/business/rss'),
+        ('Sport', 'https://www.couriermail.com.au/sport/rss'),
     ]
+
+# This isn't perfect, but works rather better than it once did. To do - remove links to subscription content.
diff --git a/recipes/glasgow_herald.recipe b/recipes/glasgow_herald.recipe
index 7f83b141d4..388dff9783 100644
--- a/recipes/glasgow_herald.recipe
+++ b/recipes/glasgow_herald.recipe
@@ -1,3 +1,9 @@
+#!/usr/bin/env python
+# vim:fileencoding=utf-8
+# License: GPLv3 Copyright: 2015, Kovid Goyal 
+
+from __future__ import absolute_import, division, print_function, unicode_literals
+
 from calibre.web.feeds.news import BasicNewsRecipe
 
 
@@ -15,10 +21,12 @@ class GlasgowHerald(BasicNewsRecipe):
     auto_cleanup = True
 
     feeds = [
-        (u'News', u'http://www.heraldscotland.com/cmlink/1.758'),
-        (u'Sport', u'http://www.heraldscotland.com/cmlink/1.761'),
-        (u'Business', u'http://www.heraldscotland.com/cmlink/1.763'),
-        (u'Life & Style', u'http://www.heraldscotland.com/cmlink/1.770'),
-        (u'Arts & Entertainment',
-         u'http://www.heraldscotland.com/cmlink/1.768',),
-        (u'Columnists', u'http://www.heraldscotland.com/cmlink/1.658574')]
+        (u'News', u'https://www.heraldscotland.com/news/rss/'),
+        (u'Sport', u'https://www.heraldscotland.com/sport/rss/'),
+        (u'Business', u'https://www.heraldscotland.com/business_hq/rss/'),
+        (u'Lifestyle', u'https://www.heraldscotland.com/life_style/rss/'),
+        (u'Arts & Entertainment', u'https://www.heraldscotland.com/arts_ents/rss/',),
+        (u'Politics', u'https://www.heraldscotland.com/politics/rss/'),
+        (u'Columnists', u'https://www.heraldscotland.com/opinion/columnists/rss/')
+
+        ]
diff --git a/recipes/icons/1843.png b/recipes/icons/1843.png
new file mode 100644
index 0000000000..bc32d1e180
Binary files /dev/null and b/recipes/icons/1843.png differ
diff --git a/recipes/icons/20minutos.png b/recipes/icons/20minutos.png
deleted file mode 100644
index 8d3df68ca2..0000000000
Binary files a/recipes/icons/20minutos.png and /dev/null differ
diff --git a/recipes/icons/abc_au.png b/recipes/icons/abc_au.png
index 438f7c0b45..e1b14c72c9 100644
Binary files a/recipes/icons/abc_au.png and b/recipes/icons/abc_au.png differ
diff --git a/recipes/icons/ads_of_the_world.png b/recipes/icons/ads_of_the_world.png
deleted file mode 100644
index d801e0e50b..0000000000
Binary files a/recipes/icons/ads_of_the_world.png and /dev/null differ
diff --git a/recipes/icons/courier_mail.png b/recipes/icons/courier_mail.png
index 9ccd38d2f3..6478f64f49 100644
Binary files a/recipes/icons/courier_mail.png and b/recipes/icons/courier_mail.png differ
diff --git a/recipes/icons/guardian.png b/recipes/icons/guardian.png
index 86b42fd4b6..2904b71dfe 100644
Binary files a/recipes/icons/guardian.png and b/recipes/icons/guardian.png differ
diff --git a/recipes/icons/macrobusiness.png b/recipes/icons/macrobusiness.png
new file mode 100644
index 0000000000..f5f0312638
Binary files /dev/null and b/recipes/icons/macrobusiness.png differ
diff --git a/recipes/icons/melbourne_herald_sun.png b/recipes/icons/melbourne_herald_sun.png
index 8fe1a32465..c8e51bbc71 100644
Binary files a/recipes/icons/melbourne_herald_sun.png and b/recipes/icons/melbourne_herald_sun.png differ
diff --git a/recipes/icons/rossijkaja_gazeta.png b/recipes/icons/rossijkaja_gazeta.png
deleted file mode 100644
index 3b883db5b4..0000000000
Binary files a/recipes/icons/rossijkaja_gazeta.png and /dev/null differ
diff --git a/recipes/icons/spectator-au.png b/recipes/icons/spectator-au.png
new file mode 100644
index 0000000000..c5508d9675
Binary files /dev/null and b/recipes/icons/spectator-au.png differ
diff --git a/recipes/icons/the_age.png b/recipes/icons/the_age.png
new file mode 100644
index 0000000000..7e80906f77
Binary files /dev/null and b/recipes/icons/the_age.png differ
diff --git a/recipes/icons/vedomosti.png b/recipes/icons/vedomosti.png
deleted file mode 100644
index 0e4ddf4798..0000000000
Binary files a/recipes/icons/vedomosti.png and /dev/null differ
diff --git a/recipes/latimes.recipe b/recipes/latimes.recipe
index 0fd773ec22..da7d2afc1d 100644
--- a/recipes/latimes.recipe
+++ b/recipes/latimes.recipe
@@ -2,13 +2,9 @@
 
 import re
 from collections import defaultdict
-from pprint import pformat
 
-from calibre.utils.date import strptime, utcnow
 from calibre.web.feeds.news import BasicNewsRecipe
 
-DT_EPOCH = strptime('1970-01-01', '%Y-%m-%d', assume_utc=True)
-
 DIR_COLLECTIONS = [['world'],
                    ['nation'],
                    ['politics'],
@@ -29,84 +25,22 @@ DIR_COLLECTIONS = [['world'],
                    ['travel'],
                    ['fashion']]
 
-SECTIONS=['THE WORLD',
-          'THE NATION',
-          'POLITICS',
-          'OPINION',
-          'CALIFORNIA',
-          'OBITUARIES',
-          'BUSINESS',
-          'HOLLYWOOD',
-          'SPORTS',
-          'ENTERTAINMENT',
-          'MOVIES',
-          'TELEVISION',
-          'BOOKS',
-          'FOOD',
-          'HEALTH',
-          'SCIENCE AND TECHNOLOGY',
-          'HOME',
-          'TRAVEL',
-          'FASHION',
-          'NEWSLETTERS'
-          'OTHER']
+
+def classes(classes):
+    q = frozenset(classes.split(' '))
+    return dict(attrs={
+        'class': lambda x: x and frozenset(x.split()).intersection(q)})
 
 
 def absurl(url):
     if url.startswith('/'):
-        url = 'http://www.latimes.com' + url
+        url = 'https://www.latimes.com' + url
     return url
 
 
-def check_words(words):
-    return lambda x: x and frozenset(words.split()).intersection(x.split())
-
-
 def what_section(url):
-    if re.compile(r'^https?://www[.]latimes[.]com/local/obituaries').search(url):
-        return 'OBITUARIES'
-    elif re.compile(r'^https?://www[.]latimes[.]com/business/hollywood').search(url):
-        return 'HOLLYWOOD'
-    elif re.compile(r'^https?://www[.]latimes[.]com/entertainment/movies').search(url):
-        return 'MOVIES'
-    elif re.compile(r'^https?://www[.]latimes[.]com/entertainment/tv').search(url):
-        return 'TELEVISION'
-    elif re.compile(r'^https?://www[.]latimes[.]com/business/technology').search(url):
-        return 'SCIENCE AND TECHNOLOGY'
-    elif re.compile(r'^https?://www[.]latimes[.]com/world').search(url):
-        return 'THE WORLD'
-    elif re.compile(r'^https?://www[.]latimes[.]com/nation').search(url):
-        return 'THE NATION'
-    elif re.compile(r'^https?://www[.]latimes[.]com/politics').search(url):
-        return 'POLITICS'
-    elif re.compile(r'^https?://www[.]latimes[.]com/opinion').search(url):
-        return 'OPINION'
-    elif re.compile(r'^https?://www[.]latimes[.]com/(?:local|style)').search(url):
-        return 'CALIFORNIA'
-    elif re.compile(r'^https?://www[.]latimes[.]com/business').search(url):
-        return 'BUSINESS'
-    elif re.compile(r'^https?://www[.]latimes[.]com/sports').search(url):
-        return 'SPORTS'
-    elif re.compile(r'^https?://www[.]latimes[.]com/entertainment').search(url):
-        return 'ENTERTAINMENT'
-    elif re.compile(r'^https?://www[.]latimes[.]com/books').search(url):
-        return 'BOOKS'
-    elif re.compile(r'^https?://www[.]latimes[.]com/food').search(url):
-        return 'FOOD'
-    elif re.compile(r'^https?://www[.]latimes[.]com/health').search(url):
-        return 'HEALTH'
-    elif re.compile(r'^https?://www[.]latimes[.]com/science').search(url):
-        return 'SCIENCE AND TECHNOLOGY'
-    elif re.compile(r'^https?://www[.]latimes[.]com/home').search(url):
-        return 'HOME'
-    elif re.compile(r'^https?://www[.]latimes[.]com/travel').search(url):
-        return 'TRAVEL'
-    elif re.compile(r'^https?://www[.]latimes[.]com/fashion').search(url):
-        return 'FASHION'
-    elif re.compile(r'^https?://www[.]latimes[.]com/newsletter').search(url):
-        return 'NEWSLETTERS'
-    else:
-        return 'OTHER'
+    parts = url.split('/')
+    return parts[-4].capitalize()
 
 
 class LATimes(BasicNewsRecipe):
@@ -126,32 +60,25 @@ class LATimes(BasicNewsRecipe):
     cover_url = 'http://www.latimes.com/includes/sectionfronts/A1.pdf'
 
     keep_only_tags = [
-        dict(name='header', attrs={'id': 'top'}),
-        dict(name='article'),
-        dict(name='div', attrs={'id': 'liveblog-story-wrapper'})
+        classes('ArticlePage-breadcrumbs ArticlePage-headline ArticlePage-mainContent'),
     ]
 
     remove_tags= [
-        dict(name='div', attrs={'class': check_words(
-            'hidden-tablet hidden-mobile hidden-desktop pb-f-ads-dfp')})
-    ]
-
-    remove_tags_after = [
-        dict(name='div', attrs={'class': check_words('pb-f-article-body')})
+        classes('ArticlePage-actions Enhancement hidden-tablet hidden-mobile hidden-desktop pb-f-ads-dfp')
     ]
 
     def parse_index(self):
-        index = 'http://www.latimes.com/'
-        pat = r'^(?:https?://www[.]latimes[.]com)?/[^#]+20[0-9]{6}-(?:html)?story[.]html'
+        index = 'https://www.latimes.com/'
+        pat = r'^https://www\.latimes\.com/[^/]+?/story/20\d{2}-\d{2}-\d{2}/\S+'
         articles = self.find_articles(index, pat)
         for collection in DIR_COLLECTIONS:
+            if self.test:
+                continue
             topdir = collection.pop(0)
-            index = 'http://www.latimes.com/' + topdir + '/'
-            pat = r'^(?:https?://www[.]latimes[.]com)?/' + \
-                      topdir + '/[^#]+20[0-9]{6}-(?:html)?story[.]html'
-            articles += self.find_articles(index, pat)
+            collection_index = index + topdir + '/'
+            articles += self.find_articles(collection_index, pat)
             for subdir in collection:
-                sub_index = index + subdir + '/'
+                sub_index = collection_index + subdir + '/'
                 articles += self.find_articles(sub_index, pat)
 
         feeds = defaultdict(list)
@@ -159,12 +86,7 @@ class LATimes(BasicNewsRecipe):
             section = what_section(article['url'])
             feeds[section].append(article)
 
-        keys = []
-        for key in SECTIONS:
-            if key in feeds.keys():
-                keys.append(key)
-        self.log(pformat(dict(feeds)))
-        return [(k, feeds[k]) for k in keys]
+        return [(k, feeds[k]) for k in sorted(feeds)]
 
     def preprocess_html(self, soup):
         for img in soup.findAll('img', attrs={'data-src': True}):
@@ -190,16 +112,6 @@ class LATimes(BasicNewsRecipe):
         alinks = [a for a in alinks if len(
             a.contents) == 1 and a.find(text=True, recursive=False)]
         articles = [
-            {'title': a.find(text=True), 'url': absurl(a['href'])} for a in alinks]
-        date_rx = re.compile(
-            r'^https?://www[.]latimes[.]com/[^#]+-(?P20[0-9]{6})-(?:html)?story[.]html')
-        for article in articles:
-            mdate = date_rx.match(article['url'])
-            if mdate is not None:
-                try:
-                    article['timestamp'] = (strptime(mdate.group('date'),'%Y%m%d') - DT_EPOCH).total_seconds()
-                except Exception:
-                    article['timestamp'] = (utcnow() - DT_EPOCH).total_seconds()
-                article['url'] = mdate.group(0)
+            {'title': self.tag_to_string(a), 'url': absurl(a['href'])} for a in alinks]
         self.log('Found: ', len(articles), ' articles.\n')
         return articles
diff --git a/recipes/list_apart.recipe b/recipes/list_apart.recipe
index f78e87879e..497889ae1d 100644
--- a/recipes/list_apart.recipe
+++ b/recipes/list_apart.recipe
@@ -17,7 +17,7 @@ class AListApart (BasicNewsRecipe):
     oldest_article = 120
     remove_empty_feeds = True
     encoding = 'utf8'
-    cover_url = u'http://alistapart.com/pix/alalogo.gif'
+    cover_url = u'https://alistapart.com/wp-content/uploads/2019/03/cropped-icon_navigation-laurel-512.jpg'
 
     def get_extra_css(self):
         if not self.extra_css:
diff --git a/recipes/newsweek.recipe b/recipes/newsweek.recipe
index a8dc8d91e6..fc55dac112 100644
--- a/recipes/newsweek.recipe
+++ b/recipes/newsweek.recipe
@@ -1,3 +1,8 @@
+#!/usr/bin/env python
+# vim:fileencoding=utf-8
+# License: GPLv3 Copyright: 2015, Kovid Goyal 
+
+import json
 from calibre.web.feeds.news import BasicNewsRecipe
 from collections import defaultdict
 
@@ -49,28 +54,23 @@ class Newsweek(BasicNewsRecipe):
         a = li.xpath('descendant::a[@href]')[0]
         url = href_to_url(a, add_piano=True)
         self.timefmt = self.tag_to_string(a)
-        img = li.xpath('descendant::a[@href]//img[@data-src]')[0]
-        self.cover_url = img.get('data-src').partition('?')[0]
+        img = li.xpath('descendant::a[@href]//source[@type="image/jpeg"]/@srcset')[0]
+        self.cover_url = img.partition('?')[0]
+        self.log('Found cover url:', self.cover_url)
         root = self.index_to_soup(url, as_tree=True)
         features = []
-        try:
-            div = root.xpath('//div[@class="magazine-features"]')[0]
-        except IndexError:
-            pass
-        else:
-            for a in div.xpath('descendant::div[@class="h1"]//a[@href]'):
-                title = self.tag_to_string(a)
-                article = a.xpath('ancestor::article')[0]
-                desc = ''
-                s = article.xpath('descendant::div[@class="summary"]')
-                if s:
-                    desc = self.tag_to_string(s[0])
-                features.append({'title': title, 'url': href_to_url(a), 'description': desc})
-                self.log(title, href_to_url(a))
+        for article in root.xpath('//div[@class="magazine-features"]//article'):
+            a = article.xpath('descendant::a[@class="article-link"]')[0]
+            title = self.tag_to_string(a)
+            url = href_to_url(a)
+            desc = ''
+            s = article.xpath('descendant::div[@class="summary"]')
+            if s:
+                desc = self.tag_to_string(s[0])
+            features.append({'title': title, 'url': href_to_url(a), 'description': desc})
+            self.log(title, url)
 
-        index = []
-        if features:
-            index.append(('Features', features))
+        index = [('Features', features)]
         sections = defaultdict(list)
         for widget in ('editor-pick',):
             self.parse_widget(widget, sections)
@@ -79,30 +79,18 @@ class Newsweek(BasicNewsRecipe):
         return index
 
     def parse_widget(self, widget, sections):
-        root = self.index_to_soup('https://d.newsweek.com/widget/' + widget, as_tree=True)
-        div = root.xpath('//div')[0]
-        href_xpath = 'descendant::*[local-name()="h1" or local-name()="h2" or local-name()="h3" or local-name()="h4"]/a[@href]'
-        for a in div.xpath(href_xpath):
-            title = self.tag_to_string(a)
-            article = a.xpath('ancestor::article')[0]
-            desc = ''
-            s = article.xpath('descendant::div[@class="summary"]')
-            if s:
-                desc = self.tag_to_string(s[0])
-            sec = article.xpath('descendant::div[@class="category"]')
-            if sec:
-                sec = self.tag_to_string(sec[0])
-            else:
-                sec = 'Articles'
-            sections[sec].append(
-                {'title': title, 'url': href_to_url(a), 'description': desc})
-            self.log(title, href_to_url(a))
-            if desc:
-                self.log('\t' + desc)
-            self.log('')
-
-    def print_version(self, url):
-        return url + '?piano_d=1'
+        raw = self.index_to_soup('https://d.newsweek.com/json/' + widget, raw=True)
+        data = json.loads(raw)['items']
+        for item in data:
+            title = item['title']
+            url = BASE + item['link']
+            self.log(title, url)
+            sections[item['label']].append(
+                {
+                    'title': title,
+                    'url': url,
+                    'description': item['description'],
+                })
 
     def preprocess_html(self, soup):
         # Parallax images in the articles are loaded as background images
diff --git a/recipes/nytimes.recipe b/recipes/nytimes.recipe
index 87c09c62d1..d2526678ce 100644
--- a/recipes/nytimes.recipe
+++ b/recipes/nytimes.recipe
@@ -90,6 +90,7 @@ class NewYorkTimes(BasicNewsRecipe):
     compress_news_images = True
     compress_news_images_auto_size = 5
     remove_attributes = ['style']
+    conversion_options = {'flow_size': 0}
 
     remove_tags = [
         dict(attrs={'aria-label':'tools'.split()}),
@@ -266,14 +267,19 @@ class NewYorkTimes(BasicNewsRecipe):
             if article.get('description'):
                 self.log('\t\t', article['description'])
 
-        container = soup.find(itemtype='http://schema.org/CollectionPage')
-        container.find('header').extract()
-        div = container.find('div')
-        for section in div.findAll('section'):
-            for ol in section.findAll('ol'):
-                for article in self.parse_article_group(ol):
-                    log(article)
-                    yield article
+        cid = slug.split('/')[-1]
+        if cid == 'dining':
+            cid = 'food'
+        try:
+            container = soup.find(id='collection-{}'.format(cid)).find('section')
+        except AttributeError:
+            container = None
+        if container is None:
+            raise ValueError('Failed to find articles container for slug: {}'.format(slug))
+        for ol in container.findAll('ol'):
+            for article in self.parse_article_group(ol):
+                log(article)
+                yield article
 
     def parse_web_sections(self):
         self.read_nyt_metadata()
diff --git a/recipes/nytimes_sub.recipe b/recipes/nytimes_sub.recipe
index ffdf42d62a..c4647a5b69 100644
--- a/recipes/nytimes_sub.recipe
+++ b/recipes/nytimes_sub.recipe
@@ -90,6 +90,7 @@ class NewYorkTimes(BasicNewsRecipe):
     compress_news_images = True
     compress_news_images_auto_size = 5
     remove_attributes = ['style']
+    conversion_options = {'flow_size': 0}
 
     remove_tags = [
         dict(attrs={'aria-label':'tools'.split()}),
@@ -266,14 +267,19 @@ class NewYorkTimes(BasicNewsRecipe):
             if article.get('description'):
                 self.log('\t\t', article['description'])
 
-        container = soup.find(itemtype='http://schema.org/CollectionPage')
-        container.find('header').extract()
-        div = container.find('div')
-        for section in div.findAll('section'):
-            for ol in section.findAll('ol'):
-                for article in self.parse_article_group(ol):
-                    log(article)
-                    yield article
+        cid = slug.split('/')[-1]
+        if cid == 'dining':
+            cid = 'food'
+        try:
+            container = soup.find(id='collection-{}'.format(cid)).find('section')
+        except AttributeError:
+            container = None
+        if container is None:
+            raise ValueError('Failed to find articles container for slug: {}'.format(slug))
+        for ol in container.findAll('ol'):
+            for article in self.parse_article_group(ol):
+                log(article)
+                yield article
 
     def parse_web_sections(self):
         self.read_nyt_metadata()
diff --git a/recipes/rossijkaja_gazeta.recipe b/recipes/rossijkaja_gazeta.recipe
deleted file mode 100644
index 9929eb7350..0000000000
--- a/recipes/rossijkaja_gazeta.recipe
+++ /dev/null
@@ -1,72 +0,0 @@
-# vim:fileencoding=utf-8
-from calibre.web.feeds.news import BasicNewsRecipe
-
-
-class AdjectiveSpecies(BasicNewsRecipe):
-    title = u'Российская Газета'
-    __author__ = 'bug_me_not'
-    cover_url = u'http://img.rg.ru/img/d/logo2012.png'
-    description = 'Российская Газета'
-    publisher = 'Правительство Российской Федерации'
-    category = 'news'
-    language = 'ru'
-    no_stylesheets = True
-    remove_javascript = True
-    oldest_article = 300
-    max_articles_per_feed = 100
-
-    remove_tags_before = dict(name='h1')
-    remove_tags_after = dict(name='div', attrs={'class': 'ar-citate'})
-    remove_tags = [dict(name='div', attrs={'class': 'insert_left'}),
-                   dict(name='a', attrs={'href': '#comments'}),
-                   dict(name='div', attrs={'class': 'clear'}),
-                   dict(name='div', attrs={'class': 'ar-citate'}),
-                   dict(name='div', attrs={'class': 'ar-social red'}),
-                   dict(name='div', attrs={'class': 'clear clear-head'}), ]
-
-    feeds = [
-        (u'Все материалы', u'http://www.rg.ru/tema/rss.xml'),
-        (u'Еженедельный выпуск',
-         u'http://www.rg.ru/tema/izd-subbota/rss.xml'),
-        (u'Государство',
-         u'http://www.rg.ru/tema/gos/rss.xml'),
-        (u'Экономика',
-         u'http://www.rg.ru/tema/ekonomika/rss.xml'),
-        (u'Бизнес',
-         u'http://www.rg.ru/tema/izd-biznes/rss.xml'),
-        (u'В мире', u'http://www.rg.ru/tema/mir/rss.xml'),
-        (u'Происшествия',
-         u'http://www.rg.ru/tema/bezopasnost/rss.xml'),
-        (u'Общество',
-         u'http://www.rg.ru/tema/obshestvo/rss.xml'),
-        (u'Культура',
-         u'http://www.rg.ru/tema/kultura/rss.xml'),
-        (u'Спорт', u'http://www.rg.ru/tema/sport/rss.xml'),
-        (u'Документы', u'http://rg.ru/tema/doc-any/rss.xml'),
-        (u'РГ: Башкортостан',
-         u'http://www.rg.ru/org/filial/bashkortostan/rss.xml'),
-        (u'РГ: Волга-Кама',
-         u'http://www.rg.ru/org/filial/volga-kama/rss.xml'),
-        (u'РГ: Восточная Сибирь',
-         u'http://www.rg.ru/org/filial/enisey/rss.xml'),
-        (u'РГ: Дальний Восток',
-         u'http://www.rg.ru/org/filial/dvostok/rss.xml'),
-        (u'РГ: Кубань. Северный Кавказ',
-         u'http://www.rg.ru/org/filial/kuban/rss.xml'),
-        (u'РГ: Пермский край',
-         u'http://www.rg.ru/org/filial/permkray/rss.xml'),
-        (u'РГ: Приволжье',
-         u'http://www.rg.ru/org/filial/privolzhe/rss.xml'),
-        (u'РГ: Северо-Запад',
-         u'http://www.rg.ru/org/filial/szapad/rss.xml'),
-        (u'РГ: Сибирь',
-         u'http://www.rg.ru/org/filial/sibir/rss.xml'),
-        (u'РГ: Средняя Волга',
-         u'http://www.rg.ru/org/filial/svolga/rss.xml'),
-        (u'РГ: Урал и Западная Сибирь',
-         u'http://www.rg.ru/org/filial/ural/rss.xml'),
-        (u'РГ: Центральная Россия',
-         u'http://www.rg.ru/org/filial/roscentr/rss.xml'),
-        (u'РГ: Юг России',
-         u'http://www.rg.ru/org/filial/jugrossii/rss.xml'),
-    ]
diff --git a/recipes/spectator-au.recipe b/recipes/spectator-au.recipe
new file mode 100644
index 0000000000..735d9460a7
--- /dev/null
+++ b/recipes/spectator-au.recipe
@@ -0,0 +1,51 @@
+__license__ = 'GPL v3'
+__copyright__ = '2011, Pat Stapleton '
+'''
+https://www.spectator.com.au/
+'''
+from calibre.web.feeds.recipes import BasicNewsRecipe
+
+
+class SpectatorAU(BasicNewsRecipe):
+    title = 'Spectator Australia'
+    __author__ = 'Pat Stapleton, Dean Cording, James Cridland'
+    description = 'Spectator Australia is an Australian edition of The Spectator, first published in the UK in July 1828.'
+    masthead_url = 'https://www.spectator.com.au/content/themes/spectator-australia/assets/images/spec-aus-logo.png'
+    cover_url = 'https://spectator.imgix.net/content/uploads/2015/10/Spectator-Australia-Logo.jpg'
+    oldest_article = 7
+    handle_gzip = True
+    no_stylesheets = True
+    use_embedded_content = False
+    scale_news_images_to_device = True
+    encoding = 'utf8'
+    publisher = 'Spectator Australia'
+    category = 'Australia,News'
+    language = 'en_AU'
+    publication_type = 'newspaper'
+    extra_css = '.article-header__author{margin-bottom:20px;}'
+    conversion_options = {
+        'comments': description,
+        'tags': category,
+        'language': language,
+        'publisher': publisher,
+        'linearize_tables': False
+    }
+
+    keep_only_tags = [dict(attrs={'class': ['article']})]
+    remove_tags = [
+        dict(
+            attrs={
+                'class': [
+                    'big-author', 'article-header__category', 'margin-menu',
+                    'related-stories', 'disqus_thread', 'middle-promo',
+                    'show-comments', 'article-tags'
+                ]
+            }
+        ),
+        dict(name=['h4', 'hr'])
+    ]
+    remove_attributes = ['width', 'height']
+
+    feeds = [
+        ('Spectator Australia', 'https://www.spectator.com.au/feed/'),
+    ]
diff --git a/recipes/spectator_magazine.recipe b/recipes/spectator_magazine.recipe
index f066f17936..fca46b08f8 100644
--- a/recipes/spectator_magazine.recipe
+++ b/recipes/spectator_magazine.recipe
@@ -1,10 +1,19 @@
+#!/usr/bin/env python2
+# vim:fileencoding=utf-8
+# License: GPLv3 Copyright: 2015, Kovid Goyal 
+
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+import json
+import re
+
+from mechanize import Request
+
 from calibre.web.feeds.recipes import BasicNewsRecipe
 
 
-def class_sel(cls):
-    def f(x):
-        return x and cls in x.split()
-    return f
+def absolutize(url):
+    return 'https://spectator.co.uk' + url
 
 
 class Spectator(BasicNewsRecipe):
@@ -15,52 +24,100 @@ class Spectator(BasicNewsRecipe):
     language = 'en'
 
     no_stylesheets = True
-
-    keep_only_tags = dict(name='div', attrs={
-                          'class': ['article-header__text', 'featured-image', 'article-content']})
-    remove_tags = [
-        dict(name='div', attrs={'id': ['disqus_thread']}),
-        dict(attrs={'class': ['middle-promo',
-                              'sharing', 'mejs-player-holder']}),
-        dict(name='a', onclick=lambda x: x and '__gaTracker' in x and 'outbound-article' in x),
-    ]
-    remove_tags_after = [
-        dict(name='hr', attrs={'class': 'sticky-clear'}),
-    ]
-
-    def parse_spec_section(self, div):
-        h2 = div.find('h2')
-        sectitle = self.tag_to_string(h2)
-        self.log('Section:', sectitle)
-        articles = []
-        for div in div.findAll('div', id=lambda x: x and x.startswith('post-')):
-            h2 = div.find('h2', attrs={'class': class_sel('term-item__title')})
-            if h2 is None:
-                h2 = div.find(attrs={'class': class_sel('news-listing__title')})
-            title = self.tag_to_string(h2)
-            a = h2.find('a')
-            url = a['href']
-            desc = ''
-            self.log('\tArticle:', title)
-            p = div.find(attrs={'class': class_sel('term-item__excerpt')})
-            if p is not None:
-                desc = self.tag_to_string(p)
-            articles.append({'title': title, 'url': url, 'description': desc})
-        return sectitle, articles
+    use_embedded_content = True
 
     def parse_index(self):
-        soup = self.index_to_soup('https://www.spectator.co.uk/magazine/')
-        a = soup.find('a', attrs={'class': 'issue-details__cover-link'})
-        self.timefmt = ' [%s]' % a['title']
-        self.cover_url = a['href']
-        if self.cover_url.startswith('//'):
-            self.cover_url = 'http:' + self.cover_url
+        br = self.get_browser()
+        main_js = br.open_novisit('https://spectator.co.uk/main.js').read().decode('utf-8')
+        data = {}
+        fields = ('apiKey', 'apiSecret', 'contentEnvironment', 'siteUrl', 'magazineIssueContentUrl', 'contentUrl')
+        pat = r'this.({})\s*=\s*"(.+?)"'.format('|'.join(fields))
+        for m in re.finditer(pat, main_js):
+            data[m.group(1)] = m.group(2)
+        self.log('Got Spectator data:', data)
+        headers = {
+            'api_key': data['apiKey'],
+            'origin': data['siteUrl'],
+            'access_token': data['apiSecret'],
+            'Accept-language': 'en-GB,en-US;q=0.9,en;q=0.8',
+            'Accept-encoding': 'gzip, deflate',
+            'Accept': '*/*',
+        }
 
-        feeds = []
+        def make_url(utype, query, includes=(), limit=None):
+            ans = data[utype] + '/entries?environment=' + data['contentEnvironment']
+            if limit is not None:
+                ans += '&limit={}'.format(limit)
+            for inc in includes:
+                ans += '&include[]=' + inc
+            ans += '&query=' + json.dumps(query)
+            return ans
 
-        div = soup.find(attrs={'class': class_sel('content-area')})
-        for x in div.findAll(attrs={'class': class_sel('magazine-section-holder')}):
-            title, articles = self.parse_spec_section(x)
-            if articles:
-                feeds.append((title, articles))
-        return feeds
+        def get_result(url):
+            self.log('Fetching:', url)
+            req = Request(url, headers=headers)
+            raw = br.open_novisit(req).read().decode('utf-8')
+            return json.loads(raw)['entries']
+
+        # Get current issue
+        url = data['magazineIssueContentUrl'] + '/entries?environment=' + data['contentEnvironment'] + "&desc=issue_date&limit=1&only[BASE][]=url"
+        result = get_result(url)
+        slug = result[0]['url']
+        uid = result[0]['uid']  # noqa
+        date = slug.split('/')[-1]
+        self.log('Downloading issue:', date)
+
+        # Cover information
+        url = make_url(
+            'magazineIssueContentUrl',
+            {'url': slug},
+            limit=1
+        )
+        self.cover_url = get_result(url)[0]['magazine_cover']['url']
+        self.log('Found cover:', self.cover_url)
+
+        # List of articles
+        url = make_url(
+            'contentUrl',
+            {
+                "magazine_content_production_only.magazine_issue": {
+                    "$in_query": {"url": slug},
+                    "_content_type_uid": "magazine_issue"
+                },
+                "_content_type_uid": "article"
+            },
+            includes=(
+                'topic', 'magazine_content_production_only.magazine_issue',
+                'magazine_content_production_only.magazine_subsection', 'author'
+            )
+        )
+        result = get_result(url)
+        articles = {}
+        for entry in result:
+            title = entry['title']
+            url = absolutize(entry['url'])
+            blocks = []
+            a = blocks.append
+            byline = entry.get('byline') or ''
+            if byline:
+                a('
{}
'.format(byline))
+            if entry.get('author'):
+                for au in reversed(entry['author']):
+                    au = entry['author'][0]
+                    cac = ''
+                    if au.get('caricature'):
+                        cac = ''.format(au['caricature']['url'])
+                    a(''.format(hi['url']))
+                if hi.get('description'):
+                    a('
{}
'.format(hi['description']))
+            a(entry['text_body'])
+            section = 'Unknown'
+            if entry.get('topic'):
+                topic = entry['topic'][0]
+                section = topic['title']
+            articles.setdefault(section, []).append({
+                'title': title, 'url': url, 'description': byline, 'content': '\n\n'.join(blocks)})
+        return [(sec, articles[sec]) for sec in sorted(articles)]
diff --git a/recipes/the_baffler.recipe b/recipes/the_baffler.recipe
index 3b84cf54ff..8fb9c96ebb 100644
--- a/recipes/the_baffler.recipe
+++ b/recipes/the_baffler.recipe
@@ -15,7 +15,7 @@ class TheBaffler(BasicNewsRecipe):
     __author__ = 'Jose Ortiz'
     description = ('This magazine contains left-wing criticism, cultural analysis, shorts'
                    ' stories, poems and art.  They publish six print issues annually.')
-    language = 'en_US'
+    language = 'en'
     encoding = 'UTF-8'
     no_javascript = True
     no_stylesheets = True
diff --git a/recipes/vedomosti.recipe b/recipes/vedomosti.recipe
deleted file mode 100644
index 0270e221b1..0000000000
--- a/recipes/vedomosti.recipe
+++ /dev/null
@@ -1,207 +0,0 @@
-#!/usr/bin/env  python2
-
-u'''
-Ведомости
-'''
-
-from calibre.web.feeds.feedparser import parse
-from calibre.ebooks.BeautifulSoup import Tag
-from calibre.web.feeds.news import BasicNewsRecipe
-
-
-def new_tag(soup, name, attrs=()):
-    impl = getattr(soup, 'new_tag', None)
-    if impl is not None:
-        return impl(name, attrs=dict(attrs))
-    return Tag(soup, name, attrs=attrs or None)
-
-
-class VedomostiRecipe(BasicNewsRecipe):
-    title = u'Ведомости'
-    __author__ = 'Nikolai Kotchetkov'
-    publisher = 'vedomosti.ru'
-    category = 'press, Russia'
-    description = u'Ежедневная деловая газета'
-    oldest_article = 3
-    max_articles_per_feed = 100
-
-    masthead_url = u'http://motorro.com/imgdir/logos/ved_logo_black2_cropped.gif'
-    cover_url = u'http://motorro.com/imgdir/logos/ved_logo_black2_cropped.gif'
-
-    # Add feed names if you want them to be sorted (feeds of this list appear
-    # first)
-    sortOrder = [u'_default', u'Первая полоса', u'Власть и деньги']
-
-    encoding = 'cp1251'
-    language = 'ru'
-    no_stylesheets = True
-    remove_javascript = True
-    recursions = 0
-
-    conversion_options = {
-        'comment': description, 'tags': category, 'publisher': publisher, 'language': language
-    }
-
-    keep_only_tags = [dict(name='td', attrs={'class': ['second_content']})]
-
-    remove_tags_after = [dict(name='div', attrs={'class': 'article_text'})]
-
-    remove_tags = [
-        dict(name='div', attrs={'class': ['sep', 'choice', 'articleRightTbl']})]
-
-    feeds = [u'http://www.vedomosti.ru/newspaper/out/rss.xml']
-
-    # base URL for relative links
-    base_url = u'http://www.vedomosti.ru'
-
-    extra_css = 'h1 {font-size: 1.5em; margin: 0em 0em 0em 0em; text-align: center;}'\
-                'h2 {font-size: 1.0em; margin: 0em 0em 0em 0em;}'\
-                'h3 {font-size: 0.8em; margin: 0em 0em 0em 0em;}'\
-                '.article_date {font-size: 0.5em; color: gray; font-family: monospace; text-align:right;}'\
-                '.article_authors {font-size: 0.5em; color: gray; font-family: monospace; text-align:right;}'\
-                '.article_img {width:100%; text-align: center; padding: 3px 3px 3px 3px;}'\
-                '.article_img_desc {width:100%; text-align: center; font-size: 0.5em; color: gray; font-family: monospace;}'\
-                '.article_desc {font-size: 1em; font-style:italic;}'
-
-    def parse_index(self):
-        try:
-            feedData = parse(self.feeds[0])
-            if not feedData:
-                raise NotImplementedError
-            self.log("parse_index: Feed loaded successfully.")
-            try:
-                if feedData.feed.title:
-                    self.title = feedData.feed.title
-                    self.log("parse_index: Title updated to: ", self.title)
-            except Exception:
-                pass
-            try:
-                if feedData.feed.description:
-                    self.description = feedData.feed.description
-                    self.log("parse_index: Description updated to: ",
-                            self.description)
-            except Exception:
-                pass
-
-            def get_virtual_feed_articles(feed):
-                if feed in feeds:
-                    return feeds[feed][1]
-                self.log("Adding new feed: ", feed)
-                articles = []
-                feeds[feed] = (feed, articles)
-                return articles
-
-            feeds = {}
-
-            # Iterate feed items and distribute articles using tags
-            for item in feedData.entries:
-                link = item.get('link', '')
-                title = item.get('title', '')
-                if '' == link or '' == title:
-                    continue
-                article = {'title': title, 'url': link, 'description': item.get(
-                    'description', ''), 'date': item.get('date', ''), 'content': ''}
-                if not item.get('tags'):  # noqa
-                    get_virtual_feed_articles('_default').append(article)
-                    continue
-                for tag in item.tags:
-                    addedToDefault = False
-                    term = tag.get('term', '')
-                    if '' == term:
-                        if (not addedToDefault):
-                            get_virtual_feed_articles(
-                                '_default').append(article)
-                        continue
-                    get_virtual_feed_articles(term).append(article)
-
-            # Get feed list
-            # Select sorted feeds first of all
-            result = []
-            for feedName in self.sortOrder:
-                if (not feeds.get(feedName)):
-                    continue
-                result.append(feeds[feedName])
-                del feeds[feedName]
-            result = result + feeds.values()
-
-            return result
-
-        except Exception as err:
-            self.log(err)
-            raise NotImplementedError
-
-    def preprocess_html(self, soup):
-        return self.adeify_images(soup)
-
-    def postprocess_html(self, soup, first_fetch):
-
-        # Find article
-        contents = soup.find('div', {'class': ['article_text']})
-        if not contents:
-            self.log('postprocess_html: article div not found!')
-            return soup
-        contents.extract()
-
-        # Find title
-        title = soup.find('h1')
-        if title:
-            contents.insert(0, title)
-
-        # Find article image
-        newstop = soup.find('div', {'class': ['newstop']})
-        if newstop:
-            img = newstop.find('img')
-            if img:
-                imgDiv = new_tag(soup, 'div')
-                imgDiv['class'] = 'article_img'
-
-                if img.get('width'):
-                    del(img['width'])
-                if img.get('height'):
-                    del(img['height'])
-
-                # find description
-                element = img.parent.nextSibling
-
-                img.extract()
-                imgDiv.insert(0, img)
-
-                while element:
-                    if not isinstance(element, Tag):
-                        continue
-                    nextElement = element.nextSibling
-                    if 'p' == element.name:
-                        element.extract()
-                        element['class'] = 'article_img_desc'
-                        imgDiv.insert(len(imgDiv.contents), element)
-                    element = nextElement
-
-                contents.insert(1, imgDiv)
-
-        # find article abstract
-        abstract = soup.find('p', {'class': ['subhead']})
-        if abstract:
-            abstract['class'] = 'article_desc'
-            contents.insert(2, abstract)
-
-        # Find article authors
-        authorsDiv = soup.find('div', {'class': ['autors']})
-        if authorsDiv:
-            authorsP = authorsDiv.find('p')
-            if authorsP:
-                authorsP['class'] = 'article_authors'
-                contents.insert(len(contents.contents), authorsP)
-
-        # Fix urls that use relative path
-        urls = contents.findAll('a', href=True)
-        if urls:
-            for url in urls:
-                if '/' == url['href'][0]:
-                    url['href'] = self.base_url + url['href']
-
-        body = soup.find('td', {'class': ['second_content']})
-        if body:
-            body.replaceWith(contents)
-
-        self.log('Result: ', soup.prettify())
-        return soup
diff --git a/recipes/wired.recipe b/recipes/wired.recipe
index 1504fd6d99..e9a9a4454c 100644
--- a/recipes/wired.recipe
+++ b/recipes/wired.recipe
@@ -4,6 +4,7 @@ __copyright__ = '2014, Darko Miletic '
 www.wired.com
 '''
 
+from calibre import browser
 from calibre.web.feeds.news import BasicNewsRecipe
 
 
@@ -80,3 +81,17 @@ class WiredDailyNews(BasicNewsRecipe):
             articles.extend(self.parse_wired_index_page(baseurl.format(pagenum), seen))
 
         return [('Magazine Articles', articles)]
+
+    # Wired changes the content it delivers based on cookies, so the
+    # following ensures that we send no cookies
+    def get_browser(self, *args, **kwargs):
+        return self
+
+    def clone_browser(self, *args, **kwargs):
+        return self.get_browser()
+
+    def open_novisit(self, *args, **kwargs):
+        br = browser()
+        return br.open_novisit(*args, **kwargs)
+
+    open = open_novisit
diff --git a/recipes/wired_daily.recipe b/recipes/wired_daily.recipe
index 62d7df23cd..42d0357a98 100644
--- a/recipes/wired_daily.recipe
+++ b/recipes/wired_daily.recipe
@@ -4,6 +4,7 @@ __copyright__ = '2014, Darko Miletic '
 www.wired.com
 '''
 
+from calibre import browser
 from calibre.web.feeds.news import BasicNewsRecipe
 
 
@@ -66,3 +67,17 @@ class WiredDailyNews(BasicNewsRecipe):
 
     def get_article_url(self, article):
         return article.get('link',  None)
+
+    # Wired changes the content it delivers based on cookies, so the
+    # following ensures that we send no cookies
+    def get_browser(self, *args, **kwargs):
+        return self
+
+    def clone_browser(self, *args, **kwargs):
+        return self.get_browser()
+
+    def open_novisit(self, *args, **kwargs):
+        br = browser()
+        return br.open_novisit(*args, **kwargs)
+
+    open = open_novisit
diff --git a/resources/templates/html.css b/resources/templates/html.css
index ef81fe5390..6122ca0c17 100644
--- a/resources/templates/html.css
+++ b/resources/templates/html.css
@@ -40,7 +40,7 @@
 
 /* blocks */
 
-html, div, map, dt, isindex, form {
+div, map, dt, isindex, form {
   display: block;
 }
 
diff --git a/setup/check.py b/setup/check.py
index dc4f56f598..fa99c33c76 100644
--- a/setup/check.py
+++ b/setup/check.py
@@ -6,7 +6,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2009, Kovid Goyal '
 __docformat__ = 'restructuredtext en'
 
-import sys, os, json, subprocess, errno, hashlib
+import os, json, subprocess, errno, hashlib
 from setup import Command, build_cache_dir, edit_file, dump_json
 
 
@@ -82,10 +82,8 @@ class Check(Command):
             p = subprocess.Popen(['rapydscript', 'lint', f])
             return p.wait() != 0
         if ext == '.yaml':
-            sys.path.insert(0, self.wn_path)
-            import whats_new
-            whats_new.render_changelog(self.j(self.d(self.SRC), 'Changelog.yaml'))
-            sys.path.remove(self.wn_path)
+            p = subprocess.Popen(['python', self.j(self.wn_path, 'whats_new.py'), f])
+            return p.wait() != 0
 
     def run(self, opts):
         self.fhash_cache = {}
diff --git a/setup/hosting.py b/setup/hosting.py
index 4e8851ebec..2d37dbebcd 100644
--- a/setup/hosting.py
+++ b/setup/hosting.py
@@ -102,7 +102,7 @@ class SourceForge(Base):  # {{{
             for i in range(5):
                 try:
                     check_call([
-                        'rsync', '-h', '-z', '--progress', '-e', 'ssh -x', x,
+                        'rsync', '-h', '-zz', '--progress', '-e', 'ssh -x', x,
                         '%s,%s@frs.sourceforge.net:%s' %
                         (self.username, self.project, self.rdir + '/')
                     ])
diff --git a/setup/plugins_mirror.py b/setup/plugins_mirror.py
index 3b5b6cc695..bd6c40b04a 100644
--- a/setup/plugins_mirror.py
+++ b/setup/plugins_mirror.py
@@ -10,7 +10,6 @@ import bz2
 import errno
 import glob
 import gzip
-import HTMLParser
 import io
 import json
 import os
@@ -22,8 +21,6 @@ import subprocess
 import sys
 import tempfile
 import time
-import urllib2
-import urlparse
 import zipfile
 import zlib
 from collections import namedtuple
@@ -33,6 +30,24 @@ from email.utils import parsedate
 from functools import partial
 from multiprocessing.pool import ThreadPool
 from xml.sax.saxutils import escape, quoteattr
+
+try:
+    from html import unescape as u
+except ImportError:
+    from HTMLParser import HTMLParser
+    u = HTMLParser().unescape
+
+try:
+    from urllib.parse import parse_qs, urlparse
+except ImportError:
+    from urlparse import parse_qs, urlparse
+
+
+try:
+    from urllib.error import URLError
+    from urllib.request import urlopen, Request, build_opener
+except Exception:
+    from urllib2 import urlopen, Request, build_opener, URLError
 # }}}
 
 USER_AGENT = 'calibre mirror'
@@ -44,15 +59,13 @@ INDEX = MR_URL + 'showpost.php?p=1362767&postcount=1'
 # INDEX = 'file:///t/raw.html'
 
 IndexEntry = namedtuple('IndexEntry', 'name url donate history uninstall deprecated thread_id')
-u = HTMLParser.HTMLParser().unescape
-
 socket.setdefaulttimeout(30)
 
 
 def read(url, get_info=False):  # {{{
     if url.startswith("file://"):
-        return urllib2.urlopen(url).read()
-    opener = urllib2.build_opener()
+        return urlopen(url).read()
+    opener = build_opener()
     opener.addheaders = [
         ('User-Agent', USER_AGENT),
         ('Accept-Encoding', 'gzip,deflate'),
@@ -62,7 +75,7 @@ def read(url, get_info=False):  # {{{
         try:
             res = opener.open(url)
             break
-        except urllib2.URLError as e:
+        except URLError as e:
             if not isinstance(e.reason, socket.timeout) or i == 9:
                 raise
             time.sleep(random.randint(10, 45))
@@ -82,7 +95,7 @@ def read(url, get_info=False):  # {{{
 
 
 def url_to_plugin_id(url, deprecated):
-    query = urlparse.parse_qs(urlparse.urlparse(url).query)
+    query = parse_qs(urlparse(url).query)
     ans = (query['t'] if 't' in query else query['p'])[0]
     if deprecated:
         ans += '-deprecated'
@@ -149,11 +162,13 @@ def convert_node(fields, x, names={}, import_data=None):
         return x.s.decode('utf-8') if isinstance(x.s, bytes) else x.s
     elif name == 'Num':
         return x.n
+    elif name == 'Constant':
+        return x.value
     elif name in {'Set', 'List', 'Tuple'}:
         func = {'Set':set, 'List':list, 'Tuple':tuple}[name]
-        return func(map(conv, x.elts))
+        return func(list(map(conv, x.elts)))
     elif name == 'Dict':
-        keys, values = map(conv, x.keys), map(conv, x.values)
+        keys, values = list(map(conv, x.keys)), list(map(conv, x.values))
         return dict(zip(keys, values))
     elif name == 'Call':
         if len(x.args) != 1 and len(x.keywords) != 0:
@@ -182,7 +197,7 @@ def get_import_data(name, mod, zf, names):
     if mod in names:
         raw = zf.open(names[mod]).read()
         module = ast.parse(raw, filename='__init__.py')
-        top_level_assigments = filter(lambda x:x.__class__.__name__ == 'Assign', ast.iter_child_nodes(module))
+        top_level_assigments = [x for x in ast.iter_child_nodes(module) if x.__class__.__name__ == 'Assign']
         for node in top_level_assigments:
             targets = {getattr(t, 'id', None) for t in node.targets}
             targets.discard(None)
@@ -196,9 +211,9 @@ def get_import_data(name, mod, zf, names):
 
 def parse_metadata(raw, namelist, zf):
     module = ast.parse(raw, filename='__init__.py')
-    top_level_imports = filter(lambda x:x.__class__.__name__ == 'ImportFrom', ast.iter_child_nodes(module))
-    top_level_classes = tuple(filter(lambda x:x.__class__.__name__ == 'ClassDef', ast.iter_child_nodes(module)))
-    top_level_assigments = filter(lambda x:x.__class__.__name__ == 'Assign', ast.iter_child_nodes(module))
+    top_level_imports = [x for x in ast.iter_child_nodes(module) if x.__class__.__name__ == 'ImportFrom']
+    top_level_classes = tuple(x for x in ast.iter_child_nodes(module) if x.__class__.__name__ == 'ClassDef')
+    top_level_assigments = [x for x in ast.iter_child_nodes(module) if x.__class__.__name__ == 'Assign']
     defaults = {
         'name':'', 'description':'',
         'supported_platforms':['windows', 'osx', 'linux'],
@@ -226,7 +241,7 @@ def parse_metadata(raw, namelist, zf):
                 plugin_import_found |= inames
             else:
                 all_imports.append((mod, [n.name for n in names]))
-                imported_names[n.asname or n.name] = mod
+                imported_names[names[-1].asname or names[-1].name] = mod
     if not plugin_import_found:
         return all_imports
 
@@ -245,7 +260,7 @@ def parse_metadata(raw, namelist, zf):
                 names[x] = val
 
     def parse_class(node):
-        class_assigments = filter(lambda x:x.__class__.__name__ == 'Assign', ast.iter_child_nodes(node))
+        class_assigments = [x for x in ast.iter_child_nodes(node) if x.__class__.__name__ == 'Assign']
         found = {}
         for node in class_assigments:
             targets = {getattr(t, 'id', None) for t in node.targets}
@@ -337,7 +352,7 @@ def update_plugin_from_entry(plugin, entry):
 
 def fetch_plugin(old_index, entry):
     lm_map = {plugin['thread_id']:plugin for plugin in old_index.values()}
-    raw = read(entry.url)
+    raw = read(entry.url).decode('utf-8', 'replace')
     url, name = parse_plugin_zip_url(raw)
     if url is None:
         raise ValueError('Failed to find zip file URL for entry: %s' % repr(entry))
@@ -346,9 +361,9 @@ def fetch_plugin(old_index, entry):
     if plugin is not None:
         # Previously downloaded plugin
         lm = datetime(*tuple(map(int, re.split(r'\D', plugin['last_modified'])))[:6])
-        request = urllib2.Request(url)
+        request = Request(url)
         request.get_method = lambda : 'HEAD'
-        with closing(urllib2.urlopen(request)) as response:
+        with closing(urlopen(request)) as response:
             info = response.info()
         slm = datetime(*parsedate(info.get('Last-Modified'))[:6])
         if lm >= slm:
@@ -413,7 +428,7 @@ def fetch_plugins(old_index):
             src = plugin['file']
             plugin['file'] = src.partition('_')[-1]
             os.rename(src, plugin['file'])
-    raw = bz2.compress(json.dumps(ans, sort_keys=True, indent=4, separators=(',', ': ')))
+    raw = bz2.compress(json.dumps(ans, sort_keys=True, indent=4, separators=(',', ': ')).encode('utf-8'))
     atomic_write(raw, PLUGINS)
     # Cleanup any extra .zip files
     all_plugin_files = {p['file'] for p in ans.values()}
@@ -503,7 +518,7 @@ h1 { text-align: center }
         name, count = x
         return '%s%s\n' % (escape(name), count)
 
-    pstats = map(plugin_stats, sorted(stats.items(), reverse=True, key=lambda x:x[1]))
+    pstats = list(map(plugin_stats, sorted(stats.items(), reverse=True, key=lambda x:x[1])))
     stats = '''\
 
 
diff --git a/setup/translations.py b/setup/translations.py
index 78e1e94407..19e7641ce2 100644
--- a/setup/translations.py
+++ b/setup/translations.py
@@ -100,8 +100,12 @@ class POT(Command):  # {{{
             root = json.load(f)
         entries = root['639-3']
         ans = []
-        for x in sorted(entries, key=lambda x:(x.get('name') or '').lower()):
-            name = x.get('name')
+
+        def name_getter(x):
+            return x.get('inverted_name') or x.get('name')
+
+        for x in sorted(entries, key=lambda x:name_getter(x).lower()):
+            name = name_getter(x)
             if name:
                 ans.append(u'msgid "{}"'.format(name))
                 ans.append('msgstr ""')
@@ -849,7 +853,7 @@ class ISO639(Command):  # {{{
                 threeb = unicode_type(threeb)
             if threeb is None:
                 continue
-            name = x.get('name')
+            name = x.get('inverted_name') or x.get('name')
             if name:
                 name = unicode_type(name)
             if not name or name[0] in '!~=/\'"':
diff --git a/setup/upload.py b/setup/upload.py
index bfe5a99cd5..4ec3a21ab7 100644
--- a/setup/upload.py
+++ b/setup/upload.py
@@ -123,7 +123,7 @@ def get_fosshub_data():
 
 def send_data(loc):
     subprocess.check_call([
-        'rsync', '--inplace', '--delete', '-r', '-z', '-h', '--progress', '-e',
+        'rsync', '--inplace', '--delete', '-r', '-zz', '-h', '--progress', '-e',
         'ssh -x', loc + '/', '%s@%s:%s' % (STAGING_USER, STAGING_HOST, STAGING_DIR)
     ])
 
diff --git a/src/calibre/constants.py b/src/calibre/constants.py
index 1d16c85385..83b89fd8c3 100644
--- a/src/calibre/constants.py
+++ b/src/calibre/constants.py
@@ -6,7 +6,7 @@ from polyglot.builtins import map, unicode_type, environ_item, hasenv, getenv, a
 import sys, locale, codecs, os, importlib, collections
 
 __appname__   = 'calibre'
-numeric_version = (4, 10, 1)
+numeric_version = (4, 12, 0)
 __version__   = '.'.join(map(unicode_type, numeric_version))
 git_version   = None
 __author__    = "Kovid Goyal "
diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py
index a855be92fa..4a3af02002 100644
--- a/src/calibre/customize/builtins.py
+++ b/src/calibre/customize/builtins.py
@@ -9,7 +9,7 @@ from calibre import guess_type
 from calibre.customize import (FileTypePlugin, MetadataReaderPlugin,
     MetadataWriterPlugin, PreferencesPlugin, InterfaceActionBase, StoreBase)
 from calibre.constants import numeric_version
-from calibre.ebooks.metadata.archive import ArchiveExtract, get_comic_metadata
+from calibre.ebooks.metadata.archive import ArchiveExtract, KPFExtract, get_comic_metadata
 from calibre.ebooks.html.to_zip import HTML2ZIP
 
 plugins = []
@@ -124,7 +124,7 @@ class TXT2TXTZ(FileTypePlugin):
             return path_to_ebook
 
 
-plugins += [HTML2ZIP, PML2PMLZ, TXT2TXTZ, ArchiveExtract,]
+plugins += [HTML2ZIP, PML2PMLZ, TXT2TXTZ, ArchiveExtract, KPFExtract]
 # }}}
 
 # Metadata reader plugins {{{
@@ -1738,15 +1738,6 @@ class StoreNextoStore(StoreBase):
     affiliate = True
 
 
-class StoreOpenBooksStore(StoreBase):
-    name = 'Open Books'
-    description = 'Comprehensive listing of DRM free e-books from a variety of sources provided by users of calibre.'
-    actual_plugin = 'calibre.gui2.store.stores.open_books_plugin:OpenBooksStore'
-
-    drm_free_only = True
-    headquarters = 'US'
-
-
 class StoreOzonRUStore(StoreBase):
     name = 'OZON.ru'
     description = 'e-books from OZON.ru'
@@ -1910,7 +1901,6 @@ plugins += [
     StoreMillsBoonUKStore,
     StoreMobileReadStore,
     StoreNextoStore,
-    StoreOpenBooksStore,
     StoreOzonRUStore,
     StorePragmaticBookshelfStore,
     StorePublioStore,
diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py
index 11e02d083b..f2b01b68b5 100644
--- a/src/calibre/db/cache.py
+++ b/src/calibre/db/cache.py
@@ -26,7 +26,7 @@ from calibre.db.tables import VirtualTable
 from calibre.db.write import get_series_values, uniq
 from calibre.db.lazy import FormatMetadata, FormatsList, ProxyMetadata
 from calibre.ebooks import check_ebook_format
-from calibre.ebooks.metadata import string_to_authors, author_to_author_sort
+from calibre.ebooks.metadata import string_to_authors, author_to_author_sort, authors_to_sort_string
 from calibre.ebooks.metadata.book.base import Metadata
 from calibre.ebooks.metadata.opf2 import metadata_to_opf
 from calibre.ptempfile import (base_dir, PersistentTemporaryFile,
@@ -1297,6 +1297,7 @@ class Cache(object):
         if set_title and mi.title:
             path_changed = True
             set_field('title', mi.title)
+        authors_changed = False
         if set_authors:
             path_changed = True
             if not mi.authors:
@@ -1305,6 +1306,7 @@ class Cache(object):
             for a in mi.authors:
                 authors += string_to_authors(a)
             set_field('authors', authors)
+            authors_changed = True
 
         if path_changed:
             self._update_path({book_id})
@@ -1339,7 +1341,13 @@ class Cache(object):
                     if val is not None:
                         protected_set_field(field, val)
 
-                for field in ('author_sort', 'publisher', 'series', 'tags', 'comments',
+                val = mi.get('author_sort', None)
+                if authors_changed and (not val or mi.is_null('author_sort')):
+                    val = authors_to_sort_string(mi.authors)
+                if authors_changed or (force_changes and val is not None) or not mi.is_null('author_sort'):
+                    protected_set_field('author_sort', val)
+
+                for field in ('publisher', 'series', 'tags', 'comments',
                     'languages', 'pubdate'):
                     val = mi.get(field, None)
                     if (force_changes and val is not None) or not mi.is_null(field):
diff --git a/src/calibre/db/cli/cmd_list.py b/src/calibre/db/cli/cmd_list.py
index ecd0143af6..edd5712555 100644
--- a/src/calibre/db/cli/cmd_list.py
+++ b/src/calibre/db/cli/cmd_list.py
@@ -13,7 +13,7 @@ from calibre import prints
 from calibre.db.cli.utils import str_width
 from calibre.ebooks.metadata import authors_to_string
 from calibre.utils.date import isoformat
-from polyglot.builtins import iteritems, unicode_type, map
+from polyglot.builtins import as_bytes, iteritems, map, unicode_type
 
 readonly = True
 version = 0  # change this if you change signature of implementation()
@@ -203,6 +203,8 @@ def do_list(
     )
     with ColoredStream(sys.stdout, fg='green'):
         prints(''.join(titles))
+    stdout = getattr(sys.stdout, 'buffer', sys.stdout)
+    linesep = as_bytes(os.linesep)
 
     wrappers = [TextWrapper(x - 1).wrap if x > 1 else lambda y: y for x in widths]
 
@@ -213,12 +215,12 @@ def do_list(
         lines = max(map(len, text))
         for l in range(lines):
             for i, field in enumerate(text):
-                ft = text[i][l] if l < len(text[i]) else u''
-                sys.stdout.write(ft.encode('utf-8'))
+                ft = text[i][l] if l < len(text[i]) else ''
+                stdout.write(ft.encode('utf-8'))
                 if i < len(text) - 1:
-                    filler = (u'%*s' % (widths[i] - str_width(ft) - 1, u''))
-                    sys.stdout.write((filler + separator).encode('utf-8'))
-            print()
+                    filler = ('%*s' % (widths[i] - str_width(ft) - 1, ''))
+                    stdout.write((filler + separator).encode('utf-8'))
+            stdout.write(linesep)
 
 
 def option_parser(get_parser, args):
diff --git a/src/calibre/db/cli/cmd_show_metadata.py b/src/calibre/db/cli/cmd_show_metadata.py
index ca76735070..08239cd9b8 100644
--- a/src/calibre/db/cli/cmd_show_metadata.py
+++ b/src/calibre/db/cli/cmd_show_metadata.py
@@ -49,8 +49,9 @@ def main(opts, args, dbctx):
     if mi is None:
         raise SystemExit('Id #%d is not present in database.' % id)
     if opts.as_opf:
+        stdout = getattr(sys.stdout, 'buffer', sys.stdout)
         mi = OPFCreator(getcwd(), mi)
-        mi.render(sys.stdout)
+        mi.render(stdout)
     else:
         prints(unicode_type(mi))
 
diff --git a/src/calibre/db/tests/writing.py b/src/calibre/db/tests/writing.py
index 0dff91bfd0..8d858c12d9 100644
--- a/src/calibre/db/tests/writing.py
+++ b/src/calibre/db/tests/writing.py
@@ -11,6 +11,7 @@ from functools import partial
 from io import BytesIO
 
 from calibre.ebooks.metadata import author_to_author_sort, title_sort
+from calibre.ebooks.metadata.book.base import Metadata
 from calibre.utils.date import UNDEFINED_DATE
 from calibre.db.tests.base import BaseTest, IMG
 from polyglot.builtins import iteritems, itervalues, unicode_type
@@ -421,13 +422,13 @@ class WritingTest(BaseTest):
         cache.set_metadata(2, mi)
         nmi = cache.get_metadata(2, get_cover=True, cover_as_data=True)
         ae(oldmi.cover_data, nmi.cover_data)
-        self.compare_metadata(nmi, oldmi, exclude={'last_modified', 'format_metadata'})
+        self.compare_metadata(nmi, oldmi, exclude={'last_modified', 'format_metadata', 'formats'})
         cache.set_metadata(1, mi2, force_changes=True)
         nmi2 = cache.get_metadata(1, get_cover=True, cover_as_data=True)
         # The new code does not allow setting of #series_index to None, instead
         # it is reset to 1.0
         ae(nmi2.get_extra('#series'), 1.0)
-        self.compare_metadata(nmi2, oldmi2, exclude={'last_modified', 'format_metadata', '#series_index'})
+        self.compare_metadata(nmi2, oldmi2, exclude={'last_modified', 'format_metadata', '#series_index', 'formats'})
 
         cache = self.init_cache(self.cloned_library)
         mi = cache.get_metadata(1)
@@ -436,6 +437,12 @@ class WritingTest(BaseTest):
         cache.set_metadata(3, mi)
         self.assertEqual(set(otags), set(cache.field_for('tags', 3)), 'case changes should not be allowed in set_metadata')
 
+        # test that setting authors without author sort results in an
+        # auto-generated authors sort
+        mi = Metadata('empty', ['a1', 'a2'])
+        cache.set_metadata(1, mi)
+        self.assertEqual('a1 & a2', cache.field_for('author_sort', 1))
+
     # }}}
 
     def test_conversion_options(self):  # {{{
diff --git a/src/calibre/devices/kobo/books.py b/src/calibre/devices/kobo/books.py
index f725751d00..f3561807fe 100644
--- a/src/calibre/devices/kobo/books.py
+++ b/src/calibre/devices/kobo/books.py
@@ -7,16 +7,14 @@ import os, time, sys
 from functools import cmp_to_key
 
 from calibre.constants import preferred_encoding, DEBUG, ispy3
-from calibre import isbytestring, force_unicode
-from calibre.utils.icu import sort_key
+from calibre import isbytestring
 
 from calibre.ebooks.metadata.book.base import Metadata
-from calibre.devices.usbms.books import Book as Book_
-from calibre.devices.usbms.books import CollectionsBookList
+from calibre.devices.usbms.books import Book as Book_, CollectionsBookList, none_cmp
 from calibre.utils.config_base import prefs
 from calibre.devices.usbms.driver import debug_print
 from calibre.ebooks.metadata import author_to_author_sort
-from polyglot.builtins import unicode_type, string_or_bytes, iteritems, itervalues, cmp
+from polyglot.builtins import unicode_type, iteritems, itervalues
 
 
 class Book(Book_):
@@ -72,6 +70,7 @@ class Book(Book_):
         self.can_put_on_shelves = True
         self.kobo_series        = None
         self.kobo_series_number = None  # Kobo stores the series number as string. And it can have a leading "#".
+        self.kobo_series_id     = None
         self.kobo_subtitle      = None
 
         if thumbnail_name is not None:
@@ -86,6 +85,10 @@ class Book(Book_):
         # If we don't have a content Id, we don't know what type it is.
         return self.contentID and self.contentID.startswith("file")
 
+    @property
+    def has_kobo_series(self):
+        return self.kobo_series is not None
+
     @property
     def is_purchased_kepub(self):
         return self.contentID and not self.contentID.startswith("file")
@@ -104,6 +107,8 @@ class Book(Book_):
             fmt('Content ID', self.contentID)
         if self.kobo_series:
             fmt('Kobo Series', self.kobo_series + ' #%s'%self.kobo_series_number)
+        if self.kobo_series_id:
+            fmt('Kobo Series ID', self.kobo_series_id)
         if self.kobo_subtitle:
             fmt('Subtitle', self.kobo_subtitle)
         if self.mime:
@@ -292,24 +297,6 @@ class KTCollectionsBookList(CollectionsBookList):
         # Sort collections
         result = {}
 
-        def none_cmp(xx, yy):
-            x = xx[1]
-            y = yy[1]
-            if x is None and y is None:
-                # No sort_key needed here, because defaults are ascii
-                return cmp(xx[2], yy[2])
-            if x is None:
-                return 1
-            if y is None:
-                return -1
-            if isinstance(x, string_or_bytes) and isinstance(y, string_or_bytes):
-                x, y = sort_key(force_unicode(x)), sort_key(force_unicode(y))
-            c = cmp(x, y)
-            if c != 0:
-                return c
-            # same as above -- no sort_key needed here
-            return cmp(xx[2], yy[2])
-
         for category, lpaths in iteritems(collections):
             books = sorted(itervalues(lpaths), key=cmp_to_key(none_cmp))
             result[category] = [x[0] for x in books]
diff --git a/src/calibre/devices/kobo/driver.py b/src/calibre/devices/kobo/driver.py
index 50f6cdcaf4..dbe3b89c4b 100644
--- a/src/calibre/devices/kobo/driver.py
+++ b/src/calibre/devices/kobo/driver.py
@@ -83,7 +83,7 @@ class KOBO(USBMS):
 
     dbversion = 0
     fwversion = (0,0,0)
-    supported_dbversion = 156
+    supported_dbversion = 158
     has_kepubs = False
 
     supported_platforms = ['windows', 'osx', 'linux']
@@ -1349,7 +1349,7 @@ class KOBOTOUCH(KOBO):
         ' Based on the existing Kobo driver by %s.') % KOBO.author
 #    icon        = I('devices/kobotouch.jpg')
 
-    supported_dbversion             = 157
+    supported_dbversion             = 158
     min_supported_dbversion         = 53
     min_dbversion_series            = 65
     min_dbversion_externalid        = 65
@@ -1357,11 +1357,12 @@ class KOBOTOUCH(KOBO):
     min_dbversion_images_on_sdcard  = 77
     min_dbversion_activity          = 77
     min_dbversion_keywords          = 82
+    min_dbversion_seriesid          = 136
 
     # Starting with firmware version 3.19.x, the last number appears to be is a
     # build number. A number will be recorded here but it can be safely ignored
     # when testing the firmware version.
-    max_supported_fwversion         = (4, 19, 14114)
+    max_supported_fwversion         = (4, 20, 14601)
     # The following document firwmare versions where new function or devices were added.
     # Not all are used, but this feels a good place to record it.
     min_fwversion_shelves           = (2, 0, 0)
@@ -1377,11 +1378,13 @@ class KOBOTOUCH(KOBO):
     min_librah20_fwversion          = (4, 16, 13337)  # "Reviewers" release.
     min_fwversion_epub_location     = (4, 17, 13651)  # ePub reading location without full contentid.
     min_fwversion_dropbox           = (4, 18, 13737)  # The Forma only at this point.
+    min_fwversion_serieslist        = (4, 20, 14601)  # Series list needs the SeriesID to be set.
 
     has_kepubs = True
 
     booklist_class = KTCollectionsBookList
     book_class = Book
+    kobo_series_dict = {}
 
     MAX_PATH_LEN = 185  # 250 - (len(" - N3_LIBRARY_SHELF.parsed") + len("F:\.kobo\images\"))
     KOBO_EXTRA_CSSFILE = 'kobo_extra.css'
@@ -1610,7 +1613,8 @@ class KOBOTOUCH(KOBO):
             bl_cache[b.lpath] = idx
 
         def update_booklist(prefix, path, ContentID, ContentType, MimeType, ImageID,
-                            title, authors, DateCreated, Description, Publisher, series, seriesnumber,
+                            title, authors, DateCreated, Description, Publisher,
+                            series, seriesnumber, SeriesID, SeriesNumberFloat,
                             ISBN, Language, Subtitle,
                             readstatus, expired, favouritesindex, accessibility, isdownloaded,
                             userid, bookshelves
@@ -1747,10 +1751,16 @@ class KOBOTOUCH(KOBO):
                     bl[idx].kobo_metadata       = kobo_metadata
                     bl[idx].kobo_series         = series
                     bl[idx].kobo_series_number  = seriesnumber
+                    bl[idx].kobo_series_id      = SeriesID
                     bl[idx].kobo_subtitle       = Subtitle
                     bl[idx].can_put_on_shelves  = allow_shelves
                     bl[idx].mime                = MimeType
 
+                    if not bl[idx].is_sideloaded and bl[idx].has_kobo_series and SeriesID is not None:
+                        if show_debug:
+                            debug_print('KoboTouch:update_booklist - Have purchased kepub with series, saving SeriesID=', SeriesID)
+                        self.kobo_series_dict[series] = SeriesID
+
                     if lpath in playlist_map:
                         bl[idx].device_collections  = playlist_map.get(lpath,[])
                         bl[idx].current_shelves     = bookshelves
@@ -1800,10 +1810,16 @@ class KOBOTOUCH(KOBO):
                     book.kobo_metadata      = kobo_metadata
                     book.kobo_series        = series
                     book.kobo_series_number = seriesnumber
+                    book.kobo_series_id     = SeriesID
                     book.kobo_subtitle      = Subtitle
                     book.can_put_on_shelves = allow_shelves
 #                    debug_print('KoboTouch:update_booklist - title=', title, 'book.device_collections', book.device_collections)
 
+                    if not book.is_sideloaded and book.has_kobo_series and SeriesID is not None:
+                        if show_debug:
+                            debug_print('KoboTouch:update_booklist - Have purchased kepub with series, saving SeriesID=', SeriesID)
+                        self.kobo_series_dict[series] = SeriesID
+
                     if bl.add_book(book, replace_metadata=False):
                         changed = True
                     if show_debug:
@@ -1863,6 +1879,10 @@ class KOBOTOUCH(KOBO):
                 columns += ", Series, SeriesNumber, ___UserID, ExternalId, Subtitle"
             else:
                 columns += ', null as Series, null as SeriesNumber, ___UserID, null as ExternalId, null as Subtitle'
+            if self.supports_series_list:
+                columns += ", SeriesID, SeriesNumberFloat"
+            else:
+                columns += ', null as SeriesID, null as SeriesNumberFloat'
 
             where_clause = ''
             if self.supports_kobo_archive() or self.supports_overdrive():
@@ -1957,7 +1977,8 @@ class KOBOTOUCH(KOBO):
                 prefix = self._card_a_prefix if oncard == 'carda' else self._main_prefix
                 changed = update_booklist(prefix, path, row['ContentID'], row['ContentType'], row['MimeType'], row['ImageId'],
                                           row['Title'], row['Attribution'], row['DateCreated'], row['Description'], row['Publisher'],
-                                          row['Series'], row['SeriesNumber'], row['ISBN'], row['Language'], row['Subtitle'],
+                                          row['Series'], row['SeriesNumber'], row['SeriesID'], row['SeriesNumberFloat'],
+                                          row['ISBN'], row['Language'], row['Subtitle'],
                                           row['ReadStatus'], row['___ExpirationStatus'],
                                           int(row['FavouritesIndex']), row['Accessibility'], row['IsDownloaded'],
                                           row['___UserID'], bookshelves
@@ -1972,6 +1993,7 @@ class KOBOTOUCH(KOBO):
                 self.dump_bookshelves(connection)
             else:
                 debug_print("KoboTouch:books - automatically managing metadata")
+            debug_print("KoboTouch:books - self.kobo_series_dict=", self.kobo_series_dict)
         # Remove books that are no longer in the filesystem. Cache contains
         # indices into the booklist if book not in filesystem, None otherwise
         # Do the operation in reverse order so indices remain valid
@@ -3127,21 +3149,29 @@ class KOBOTOUCH(KOBO):
                 kobo_series_number = None
             series_number_changed = not (kobo_series_number == newmi.series_index)
 
-        if series_changed or series_number_changed:
-            if newmi.series is not None:
-                new_series = newmi.series
-                try:
-                    new_series_number = "%g" % newmi.series_index
-                except:
-                    new_series_number = None
-            else:
-                new_series = None
+        if newmi.series is not None:
+            new_series = newmi.series
+            try:
+                new_series_number = "%g" % newmi.series_index
+            except:
                 new_series_number = None
+        else:
+            new_series = None
+            new_series_number = None
 
+        if series_changed or series_number_changed:
             update_values.append(new_series)
             set_clause += ', Series = ? '
             update_values.append(new_series_number)
             set_clause += ', SeriesNumber = ? '
+        if self.supports_series_list and book.is_sideloaded:
+            series_id = self.kobo_series_dict.get(new_series, new_series)
+            if not book.kobo_series_id == series_id or series_changed or series_number_changed:
+                update_values.append(series_id)
+                set_clause += ', SeriesID = ? '
+                update_values.append(new_series_number)
+                set_clause += ', SeriesNumberFloat = ? '
+                debug_print("KoboTouch:set_core_metadata Setting SeriesID - new_series='%s', series_id='%s'" % (new_series, series_id))
 
         if not series_only:
             if not (newmi.title == kobo_metadata.title):
@@ -3537,6 +3567,10 @@ class KOBOTOUCH(KOBO):
     def supports_series(self):
         return self.dbversion >= self.min_dbversion_series
 
+    @property
+    def supports_series_list(self):
+        return self.dbversion >= self.min_dbversion_seriesid and self.fwversion >= self.min_fwversion_serieslist
+
     def supports_kobo_archive(self):
         return self.dbversion >= self.min_dbversion_archive
 
diff --git a/src/calibre/devices/smart_device_app/driver.py b/src/calibre/devices/smart_device_app/driver.py
index d32a1afe52..0011edc32a 100644
--- a/src/calibre/devices/smart_device_app/driver.py
+++ b/src/calibre/devices/smart_device_app/driver.py
@@ -36,7 +36,7 @@ from calibre.utils.filenames import ascii_filename as sanitize, shorten_componen
 from calibre.utils.mdns import (publish as publish_zeroconf, unpublish as
         unpublish_zeroconf, get_all_ips)
 from calibre.utils.socket_inheritance import set_socket_inherit
-from polyglot.builtins import unicode_type, iteritems, itervalues
+from polyglot.builtins import as_bytes, unicode_type, iteritems, itervalues
 from polyglot import queue
 
 
@@ -100,7 +100,7 @@ class ConnectionListener(Thread):
                         s = self.driver._json_encode(
                                         self.driver.opcodes['CALIBRE_BUSY'],
                                         {'otherDevice': d.get_gui_name()})
-                        self.driver._send_byte_string(device_socket, (b'%d' % len(s)) + s)
+                        self.driver._send_byte_string(device_socket, (b'%d' % len(s)) + as_bytes(s))
                         sock.close()
                     except queue.Empty:
                         pass
@@ -636,7 +636,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
             s = self._json_encode(self.opcodes[op], arg)
             if print_debug_info and extra_debug:
                 self._debug('send string', s)
-            self._send_byte_string(self.device_socket, (b'%d' % len(s)) + s)
+            self._send_byte_string(self.device_socket, (b'%d' % len(s)) + as_bytes(s))
             if not wait_for_response:
                 return None, None
             return self._receive_from_client(print_debug_info=print_debug_info)
@@ -841,10 +841,10 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
                     json_metadata = defaultdict(dict)
                     json_metadata[key]['book'] = self.json_codec.encode_book_metadata(book['book'])
                     json_metadata[key]['last_used'] = book['last_used']
-                    result = json.dumps(json_metadata, indent=2, default=to_json)
-                    fd.write("%0.7d\n"%(len(result)+1))
+                    result = as_bytes(json.dumps(json_metadata, indent=2, default=to_json))
+                    fd.write(("%0.7d\n"%(len(result)+1)).encode('ascii'))
                     fd.write(result)
-                    fd.write('\n')
+                    fd.write(b'\n')
                     count += 1
             self._debug('wrote', count, 'entries, purged', purged, 'entries')
 
diff --git a/src/calibre/devices/usbms/books.py b/src/calibre/devices/usbms/books.py
index 4777d86cf4..bbb89e316c 100644
--- a/src/calibre/devices/usbms/books.py
+++ b/src/calibre/devices/usbms/books.py
@@ -20,6 +20,31 @@ from calibre.utils.icu import sort_key
 from polyglot.builtins import string_or_bytes, iteritems, itervalues, cmp
 
 
+def none_cmp(xx, yy):
+    x = xx[1]
+    y = yy[1]
+    if x is None and y is None:
+        # No sort_key needed here, because defaults are ascii
+        return cmp(xx[2], yy[2])
+    if x is None:
+        return 1
+    if y is None:
+        return -1
+    if isinstance(x, string_or_bytes) and isinstance(y, string_or_bytes):
+        x, y = sort_key(force_unicode(x)), sort_key(force_unicode(y))
+    try:
+        c = cmp(x, y)
+    except TypeError:
+        c = 0
+    if c != 0:
+        return c
+    # same as above -- no sort_key needed here
+    try:
+        return cmp(xx[2], yy[2])
+    except TypeError:
+        return 0
+
+
 class Book(Metadata):
 
     def __init__(self, prefix, lpath, size=None, other=None):
@@ -280,30 +305,6 @@ class CollectionsBookList(BookList):
         # Sort collections
         result = {}
 
-        def none_cmp(xx, yy):
-            x = xx[1]
-            y = yy[1]
-            if x is None and y is None:
-                # No sort_key needed here, because defaults are ascii
-                return cmp(xx[2], yy[2])
-            if x is None:
-                return 1
-            if y is None:
-                return -1
-            if isinstance(x, string_or_bytes) and isinstance(y, string_or_bytes):
-                x, y = sort_key(force_unicode(x)), sort_key(force_unicode(y))
-            try:
-                c = cmp(x, y)
-            except TypeError:
-                c = 0
-            if c != 0:
-                return c
-            # same as above -- no sort_key needed here
-            try:
-                return cmp(xx[2], yy[2])
-            except TypeError:
-                return 0
-
         for category, lpaths in iteritems(collections):
             books = sorted(itervalues(lpaths), key=cmp_to_key(none_cmp))
             result[category] = [x[0] for x in books]
diff --git a/src/calibre/ebooks/__init__.py b/src/calibre/ebooks/__init__.py
index c240ded56c..a88f7519f6 100644
--- a/src/calibre/ebooks/__init__.py
+++ b/src/calibre/ebooks/__init__.py
@@ -38,7 +38,7 @@ BOOK_EXTENSIONS = ['lrf', 'rar', 'zip', 'rtf', 'lit', 'txt', 'txtz', 'text', 'ht
                    'epub', 'fb2', 'fbz', 'djv', 'djvu', 'lrx', 'cbr', 'cbz', 'cbc', 'oebzip',
                    'rb', 'imp', 'odt', 'chm', 'tpz', 'azw1', 'pml', 'pmlz', 'mbp', 'tan', 'snb',
                    'xps', 'oxps', 'azw4', 'book', 'zbf', 'pobi', 'docx', 'docm', 'md',
-                   'textile', 'markdown', 'ibook', 'ibooks', 'iba', 'azw3', 'ps', 'kepub', 'kfx']
+                   'textile', 'markdown', 'ibook', 'ibooks', 'iba', 'azw3', 'ps', 'kepub', 'kfx', 'kpf']
 
 
 def return_raster_image(path):
diff --git a/src/calibre/ebooks/conversion/plugins/djvu_input.py b/src/calibre/ebooks/conversion/plugins/djvu_input.py
index 8f25551af8..ed61ff04d3 100644
--- a/src/calibre/ebooks/conversion/plugins/djvu_input.py
+++ b/src/calibre/ebooks/conversion/plugins/djvu_input.py
@@ -28,8 +28,12 @@ class DJVUInput(InputFormatPlugin):
         from calibre.ebooks.djvu.djvu import DJVUFile
         x = DJVUFile(stream)
         x.get_text(stdout)
+        raw_text = stdout.getvalue()
+        if not raw_text:
+            raise ValueError('The DJVU file contains no text, only images, probably page scans.'
+                    ' calibre only supports conversion of DJVU files with actual text in them.')
 
-        html = convert_basic(stdout.getvalue().replace(b"\n", b' ').replace(
+        html = convert_basic(raw_text.replace(b"\n", b' ').replace(
             b'\037', b'\n\n'))
         # Run the HTMLized text through the html processing plugin.
         from calibre.customize.ui import plugin_for_input_format
diff --git a/src/calibre/ebooks/metadata/archive.py b/src/calibre/ebooks/metadata/archive.py
index 2bdc35e76d..23be91decc 100644
--- a/src/calibre/ebooks/metadata/archive.py
+++ b/src/calibre/ebooks/metadata/archive.py
@@ -40,6 +40,29 @@ def archive_type(stream):
     return ans
 
 
+class KPFExtract(FileTypePlugin):
+
+    name = 'KPF Extract'
+    author = 'Kovid Goyal'
+    description = _('Extract the source DOCX file from Amazon Kindle Create KPF files.'
+            ' Note this will not contain any edits made in the Kindle Create program itself.')
+    file_types = {'kpf'}
+    supported_platforms = ['windows', 'osx', 'linux']
+    on_import = True
+
+    def run(self, archive):
+        from calibre.utils.zipfile import ZipFile
+        with ZipFile(archive, 'r') as zf:
+            fnames = zf.namelist()
+            candidates = [x for x in fnames if x.lower().endswith('.docx')]
+            if not candidates:
+                return archive
+            of = self.temporary_file('_kpf_extract.docx')
+            with closing(of):
+                of.write(zf.read(candidates[0]))
+        return of.name
+
+
 class ArchiveExtract(FileTypePlugin):
     name = 'Archive Extract'
     author = 'Kovid Goyal'
diff --git a/src/calibre/ebooks/metadata/search_internet.py b/src/calibre/ebooks/metadata/search_internet.py
index 2b45caba0d..bb962e83eb 100644
--- a/src/calibre/ebooks/metadata/search_internet.py
+++ b/src/calibre/ebooks/metadata/search_internet.py
@@ -5,7 +5,7 @@
 from __future__ import absolute_import, division, print_function, unicode_literals
 
 from polyglot.builtins import iteritems
-from polyglot.urllib import quote_plus
+from polyglot.urllib import quote, quote_plus
 
 AUTHOR_SEARCHES = {
     'goodreads':
@@ -48,17 +48,21 @@ all_book_searches = BOOK_SEARCHES.__iter__
 all_author_searches = AUTHOR_SEARCHES.__iter__
 
 
-def qquote(val):
+def qquote(val, use_plus=True):
     if not isinstance(val, bytes):
         val = val.encode('utf-8')
-    ans = quote_plus(val)
+    ans = quote_plus(val) if use_plus else quote(val)
     if isinstance(ans, bytes):
         ans = ans.decode('utf-8')
     return ans
 
 
+def specialised_quote(template, val):
+    return qquote(val, 'goodreads.com' not in template)
+
+
 def url_for(template, data):
-    return template.format(**{k: qquote(v) for k, v in iteritems(data)})
+    return template.format(**{k: specialised_quote(template, v) for k, v in iteritems(data)})
 
 
 def url_for_author_search(key, **kw):
diff --git a/src/calibre/ebooks/metadata/sources/base.py b/src/calibre/ebooks/metadata/sources/base.py
index db52433514..2d05d17814 100644
--- a/src/calibre/ebooks/metadata/sources/base.py
+++ b/src/calibre/ebooks/metadata/sources/base.py
@@ -343,8 +343,8 @@ class Source(Plugin):
 
         if authors:
             # Leave ' in there for Irish names
-            remove_pat = re.compile(r'[!@#$%^&*(){}`~"\s\[\]/]')
-            replace_pat = re.compile(r'[-+.:;,]')
+            remove_pat = re.compile(r'[!@#$%^&*()（）「」{}`~"\s\[\]/]')
+            replace_pat = re.compile(r'[-+.:;,，。；：]')
             if only_first_author:
                 authors = authors[:1]
             for au in authors:
@@ -384,7 +384,7 @@ class Source(Plugin):
                 # Remove hyphens only if they have whitespace before them
                 (r'(\s-)', ' '),
                 # Replace other special chars with a space
-                (r'''[:,;!@$%^&*(){}.`~"\s\[\]/]''', ' '),
+                (r'''[:,;!@$%^&*(){}.`~"\s\[\]/]《》「」“”''', ' '),
             ]]
 
             for pat, repl in title_patterns:
diff --git a/src/calibre/ebooks/metadata/sources/douban.py b/src/calibre/ebooks/metadata/sources/douban.py
index bb044f7cca..c77bdb8dc7 100644
--- a/src/calibre/ebooks/metadata/sources/douban.py
+++ b/src/calibre/ebooks/metadata/sources/douban.py
@@ -3,38 +3,36 @@
 
 from __future__ import absolute_import, division, print_function, unicode_literals
 
-__license__   = 'GPL v3'
+__license__ = 'GPL v3'
 __copyright__ = '2011, Kovid Goyal ; 2011, Li Fanxi '
 __docformat__ = 'restructuredtext en'
 
 import time
-from functools import partial
 try:
     from queue import Empty, Queue
 except ImportError:
     from Queue import Empty, Queue
 
-
 from calibre.ebooks.metadata import check_isbn
 from calibre.ebooks.metadata.sources.base import Option, Source
 from calibre.ebooks.metadata.book.base import Metadata
 from calibre import as_unicode
 
 NAMESPACES = {
-              'openSearch':'http://a9.com/-/spec/opensearchrss/1.0/',
-              'atom' : 'http://www.w3.org/2005/Atom',
-              'db': 'https://www.douban.com/xmlns/',
-              'gd': 'http://schemas.google.com/g/2005'
-            }
+    'openSearch': 'http://a9.com/-/spec/opensearchrss/1.0/',
+    'atom': 'http://www.w3.org/2005/Atom',
+    'db': 'https://www.douban.com/xmlns/',
+    'gd': 'http://schemas.google.com/g/2005'
+}
 
 
 def get_details(browser, url, timeout):  # {{{
     try:
-        if Douban.DOUBAN_API_KEY and Douban.DOUBAN_API_KEY != '':
+        if Douban.DOUBAN_API_KEY:
             url = url + "?apikey=" + Douban.DOUBAN_API_KEY
         raw = browser.open_novisit(url, timeout=timeout).read()
     except Exception as e:
-        gc = getattr(e, 'getcode', lambda : -1)
+        gc = getattr(e, 'getcode', lambda: -1)
         if gc() != 403:
             raise
         # Douban is throttling us, wait a little
@@ -42,150 +40,124 @@ def get_details(browser, url, timeout):  # {{{
         raw = browser.open_novisit(url, timeout=timeout).read()
 
     return raw
+
+
 # }}}
 
 
 class Douban(Source):
 
     name = 'Douban Books'
-    author = 'Li Fanxi'
-    version = (2, 1, 2)
+    author = 'Li Fanxi, xcffl, jnozsc'
+    version = (3, 1, 0)
     minimum_calibre_version = (2, 80, 0)
 
-    description = _('Downloads metadata and covers from Douban.com. '
-            'Useful only for Chinese language books.')
+    description = _(
+        'Downloads metadata and covers from Douban.com. '
+        'Useful only for Chinese language books.'
+    )
 
     capabilities = frozenset(['identify', 'cover'])
-    touched_fields = frozenset(['title', 'authors', 'tags',
-        'pubdate', 'comments', 'publisher', 'identifier:isbn', 'rating',
-        'identifier:douban'])  # language currently disabled
+    touched_fields = frozenset([
+        'title', 'authors', 'tags', 'pubdate', 'comments', 'publisher',
+        'identifier:isbn', 'rating', 'identifier:douban'
+    ])  # language currently disabled
     supports_gzip_transfer_encoding = True
     cached_cover_url_is_reliable = True
 
-    DOUBAN_API_KEY = '0bd1672394eb1ebf2374356abec15c3d'
+    DOUBAN_API_KEY = '0df993c66c0c636e29ecbb5344252a4a'
+    DOUBAN_API_URL = 'https://api.douban.com/v2/book/search'
     DOUBAN_BOOK_URL = 'https://book.douban.com/subject/%s/'
 
     options = (
-        Option('include_subtitle_in_title', 'bool', True, _('Include subtitle in book title:'),
-               _('Whether to append subtitle in the book title.')),
+        Option(
+            'include_subtitle_in_title', 'bool', True,
+            _('Include subtitle in book title:'),
+            _('Whether to append subtitle in the book title.')
+        ),
     )
 
     def to_metadata(self, browser, log, entry_, timeout):  # {{{
-        from lxml import etree
-        from calibre.ebooks.chardet import xml_to_unicode
         from calibre.utils.date import parse_date, utcnow
-        from calibre.utils.cleantext import clean_ascii_chars
 
-        XPath = partial(etree.XPath, namespaces=NAMESPACES)
-        entry          = XPath('//atom:entry')
-        entry_id       = XPath('descendant::atom:id')
-        title          = XPath('descendant::atom:title')
-        description    = XPath('descendant::atom:summary')
-        subtitle       = XPath("descendant::db:attribute[@name='subtitle']")
-        publisher      = XPath("descendant::db:attribute[@name='publisher']")
-        isbn           = XPath("descendant::db:attribute[@name='isbn13']")
-        date           = XPath("descendant::db:attribute[@name='pubdate']")
-        creator        = XPath("descendant::db:attribute[@name='author']")
-        booktag        = XPath("descendant::db:tag/attribute::name")
-        rating         = XPath("descendant::gd:rating/attribute::average")
-        cover_url      = XPath("descendant::atom:link[@rel='image']/attribute::href")
+        douban_id = entry_.get('id')
+        title = entry_.get('title')
+        description = entry_.get('summary')
+        # subtitle = entry_.get('subtitle')  # TODO: std metada doesn't have this field
+        publisher = entry_.get('publisher')
+        isbn = entry_.get('isbn13')  # ISBN11 is obsolute, use ISBN13
+        pubdate = entry_.get('pubdate')
+        authors = entry_.get('author')
+        book_tags = entry_.get('tags')
+        rating = entry_.get('rating')
+        cover_url = entry_.get('images', {}).get('large')
+        series = entry_.get('series')
 
-        def get_text(extra, x):
-            try:
-                ans = x(extra)
-                if ans:
-                    ans = ans[0].text
-                    if ans and ans.strip():
-                        return ans.strip()
-            except:
-                log.exception('Programming error:')
-            return None
-
-        id_url = entry_id(entry_)[0].text.replace('http://', 'https://')
-        douban_id = id_url.split('/')[-1]
-        title_ = ': '.join([x.text for x in title(entry_)]).strip()
-        subtitle = ': '.join([x.text for x in subtitle(entry_)]).strip()
-        if self.prefs['include_subtitle_in_title'] and len(subtitle) > 0:
-            title_ = title_ + ' - ' + subtitle
-        authors = [x.text.strip() for x in creator(entry_) if x.text]
         if not authors:
             authors = [_('Unknown')]
-        if not id_url or not title:
+        if not douban_id or not title:
             # Silently discard this entry
             return None
 
-        mi = Metadata(title_, authors)
-        mi.identifiers = {'douban':douban_id}
-        try:
-            log.info(id_url)
-            raw = get_details(browser, id_url, timeout)
-            feed = etree.fromstring(
-                xml_to_unicode(clean_ascii_chars(raw), strip_encoding_pats=True)[0],
-                parser=etree.XMLParser(recover=True, no_network=True, resolve_entities=False)
-            )
-            extra = entry(feed)[0]
-        except:
-            log.exception('Failed to get additional details for', mi.title)
-            return mi
-        mi.comments = get_text(extra, description)
-        mi.publisher = get_text(extra, publisher)
+        mi = Metadata(title, authors)
+        mi.identifiers = {'douban': douban_id}
+        mi.publisher = publisher
+        mi.comments = description
+        # mi.subtitle = subtitle
 
         # ISBN
         isbns = []
-        for x in [t.text for t in isbn(extra)]:
-            if check_isbn(x):
-                isbns.append(x)
+        if isinstance(isbn, (type(''), bytes)):
+            if check_isbn(isbn):
+                isbns.append(isbn)
+        else:
+            for x in isbn:
+                if check_isbn(x):
+                    isbns.append(x)
         if isbns:
             mi.isbn = sorted(isbns, key=len)[-1]
         mi.all_isbns = isbns
 
         # Tags
-        try:
-            btags = [x for x in booktag(extra) if x]
-            tags = []
-            for t in btags:
-                atags = [y.strip() for y in t.split('/')]
-                for tag in atags:
-                    if tag not in tags:
-                        tags.append(tag)
-        except:
-            log.exception('Failed to parse tags:')
-            tags = []
-        if tags:
-            mi.tags = [x.replace(',', ';') for x in tags]
+        mi.tags = [tag['name'] for tag in book_tags]
 
         # pubdate
-        pubdate = get_text(extra, date)
         if pubdate:
             try:
                 default = utcnow().replace(day=15)
                 mi.pubdate = parse_date(pubdate, assume_utc=True, default=default)
             except:
-                log.error('Failed to parse pubdate %r'%pubdate)
+                log.error('Failed to parse pubdate %r' % pubdate)
 
         # Ratings
-        if rating(extra):
+        if rating:
             try:
-                mi.rating = float(rating(extra)[0]) / 2.0
+                mi.rating = float(rating['average']) / 2.0
             except:
                 log.exception('Failed to parse rating')
                 mi.rating = 0
 
         # Cover
         mi.has_douban_cover = None
-        u = cover_url(extra)
+        u = cover_url
         if u:
-            u = u[0].replace('/spic/', '/lpic/')
             # If URL contains "book-default", the book doesn't have a cover
             if u.find('book-default') == -1:
                 mi.has_douban_cover = u
+
+        # Series
+        if series:
+            mi.series = series['title']
+
         return mi
+
     # }}}
 
     def get_book_url(self, identifiers):  # {{{
         db = identifiers.get('douban', None)
         if db is not None:
-            return ('douban', db, self.DOUBAN_BOOK_URL%db)
+            return ('douban', db, self.DOUBAN_BOOK_URL % db)
+
     # }}}
 
     def create_query(self, log, title=None, authors=None, identifiers={}):  # {{{
@@ -193,9 +165,9 @@ class Douban(Source):
             from urllib.parse import urlencode
         except ImportError:
             from urllib import urlencode
-        SEARCH_URL = 'https://api.douban.com/book/subjects?'
-        ISBN_URL = 'https://api.douban.com/book/subject/isbn/'
-        SUBJECT_URL = 'https://api.douban.com/book/subject/'
+        SEARCH_URL = 'https://api.douban.com/v2/book/search?count=10&'
+        ISBN_URL = 'https://api.douban.com/v2/book/isbn/'
+        SUBJECT_URL = 'https://api.douban.com/v2/book/'
 
         q = ''
         t = None
@@ -208,16 +180,18 @@ class Douban(Source):
             q = subject
             t = 'subject'
         elif title or authors:
+
             def build_term(prefix, parts):
                 return ' '.join(x for x in parts)
+
             title_tokens = list(self.get_title_tokens(title))
             if title_tokens:
                 q += build_term('title', title_tokens)
-            author_tokens = list(self.get_author_tokens(authors,
-                    only_first_author=True))
+            author_tokens = list(
+                self.get_author_tokens(authors, only_first_author=True)
+            )
             if author_tokens:
-                q += ((' ' if q != '' else '') +
-                    build_term('author', author_tokens))
+                q += ((' ' if q != '' else '') + build_term('author', author_tokens))
             t = 'search'
         q = q.strip()
         if isinstance(q, type(u'')):
@@ -231,24 +205,40 @@ class Douban(Source):
             url = SUBJECT_URL + q
         else:
             url = SEARCH_URL + urlencode({
-                    'q': q,
-                    })
+                'q': q,
+            })
         if self.DOUBAN_API_KEY and self.DOUBAN_API_KEY != '':
             if t == "isbn" or t == "subject":
                 url = url + "?apikey=" + self.DOUBAN_API_KEY
             else:
                 url = url + "&apikey=" + self.DOUBAN_API_KEY
         return url
+
     # }}}
 
-    def download_cover(self, log, result_queue, abort,  # {{{
-            title=None, authors=None, identifiers={}, timeout=30, get_best_cover=False):
+    def download_cover(
+        self,
+        log,
+        result_queue,
+        abort,  # {{{
+        title=None,
+        authors=None,
+        identifiers={},
+        timeout=30,
+        get_best_cover=False
+    ):
         cached_url = self.get_cached_cover_url(identifiers)
         if cached_url is None:
             log.info('No cached cover found, running identify')
             rq = Queue()
-            self.identify(log, rq, abort, title=title, authors=authors,
-                    identifiers=identifiers)
+            self.identify(
+                log,
+                rq,
+                abort,
+                title=title,
+                authors=authors,
+                identifiers=identifiers
+            )
             if abort.is_set():
                 return
             results = []
@@ -257,8 +247,11 @@ class Douban(Source):
                     results.append(rq.get_nowait())
                 except Empty:
                     break
-            results.sort(key=self.identify_results_keygen(
-                title=title, authors=authors, identifiers=identifiers))
+            results.sort(
+                key=self.identify_results_keygen(
+                    title=title, authors=authors, identifiers=identifiers
+                )
+            )
             for mi in results:
                 cached_url = self.get_cached_cover_url(mi.identifiers)
                 if cached_url is not None:
@@ -291,11 +284,18 @@ class Douban(Source):
             url = self.cached_identifier_to_cover_url(db)
 
         return url
+
     # }}}
 
-    def get_all_details(self, br, log, entries, abort,  # {{{
-            result_queue, timeout):
-        from lxml import etree
+    def get_all_details(
+        self,
+        br,
+        log,
+        entries,
+        abort,  # {{{
+        result_queue,
+        timeout
+    ):
         for relevance, i in enumerate(entries):
             try:
                 ans = self.to_metadata(br, log, i, timeout)
@@ -305,29 +305,31 @@ class Douban(Source):
                     for isbn in getattr(ans, 'all_isbns', []):
                         self.cache_isbn_to_identifier(isbn, db)
                     if ans.has_douban_cover:
-                        self.cache_identifier_to_cover_url(db,
-                                ans.has_douban_cover)
+                        self.cache_identifier_to_cover_url(db, ans.has_douban_cover)
                     self.clean_downloaded_metadata(ans)
                     result_queue.put(ans)
             except:
-                log.exception(
-                    'Failed to get metadata for identify entry:',
-                    etree.tostring(i))
+                log.exception('Failed to get metadata for identify entry:', i)
             if abort.is_set():
                 break
+
     # }}}
 
-    def identify(self, log, result_queue, abort, title=None, authors=None,  # {{{
-            identifiers={}, timeout=30):
-        from lxml import etree
-        from calibre.ebooks.chardet import xml_to_unicode
-        from calibre.utils.cleantext import clean_ascii_chars
+    def identify(
+        self,
+        log,
+        result_queue,
+        abort,
+        title=None,
+        authors=None,  # {{{
+        identifiers={},
+        timeout=30
+    ):
+        import json
 
-        XPath = partial(etree.XPath, namespaces=NAMESPACES)
-        entry          = XPath('//atom:entry')
-
-        query = self.create_query(log, title=title, authors=authors,
-                identifiers=identifiers)
+        query = self.create_query(
+            log, title=title, authors=authors, identifiers=identifiers
+        )
         if not query:
             log.error('Insufficient metadata to construct query')
             return
@@ -335,45 +337,56 @@ class Douban(Source):
         try:
             raw = br.open_novisit(query, timeout=timeout).read()
         except Exception as e:
-            log.exception('Failed to make identify query: %r'%query)
+            log.exception('Failed to make identify query: %r' % query)
             return as_unicode(e)
         try:
-            parser = etree.XMLParser(recover=True, no_network=True)
-            feed = etree.fromstring(xml_to_unicode(clean_ascii_chars(raw),
-                strip_encoding_pats=True)[0], parser=parser)
-            entries = entry(feed)
+            j = json.loads(raw)
         except Exception as e:
             log.exception('Failed to parse identify results')
             return as_unicode(e)
+        if 'books' in j:
+            entries = j['books']
+        else:
+            entries = []
+            entries.append(j)
         if not entries and identifiers and title and authors and \
                 not abort.is_set():
-            return self.identify(log, result_queue, abort, title=title,
-                    authors=authors, timeout=timeout)
-
+            return self.identify(
+                log,
+                result_queue,
+                abort,
+                title=title,
+                authors=authors,
+                timeout=timeout
+            )
         # There is no point running these queries in threads as douban
         # throttles requests returning 403 Forbidden errors
         self.get_all_details(br, log, entries, abort, result_queue, timeout)
 
         return None
+
     # }}}
 
 
 if __name__ == '__main__':  # tests {{{
     # To run these test use: calibre-debug -e src/calibre/ebooks/metadata/sources/douban.py
-    from calibre.ebooks.metadata.sources.test import (test_identify_plugin,
-            title_test, authors_test)
-    test_identify_plugin(Douban.name,
-        [
-            (
-                {'identifiers':{'isbn': '9787536692930'}, 'title':'三体',
-                    'authors':['刘慈欣']},
-                [title_test('三体', exact=True),
-                    authors_test(['刘慈欣'])]
-            ),
-
-            (
-                {'title': 'Linux内核修炼之道', 'authors':['任桥伟']},
-                [title_test('Linux内核修炼之道', exact=False)]
-            ),
-    ])
+    from calibre.ebooks.metadata.sources.test import (
+        test_identify_plugin, title_test, authors_test
+    )
+    test_identify_plugin(
+        Douban.name, [
+            ({
+                'identifiers': {
+                    'isbn': '9787536692930'
+                },
+                'title': '三体',
+                'authors': ['刘慈欣']
+            }, [title_test('三体', exact=True),
+                authors_test(['刘慈欣'])]),
+            ({
+                'title': 'Linux内核修炼之道',
+                'authors': ['任桥伟']
+            }, [title_test('Linux内核修炼之道', exact=False)]),
+        ]
+    )
 # }}}
diff --git a/src/calibre/ebooks/mobi/debug/headers.py b/src/calibre/ebooks/mobi/debug/headers.py
index 308d7c262b..ef4fe8f6eb 100644
--- a/src/calibre/ebooks/mobi/debug/headers.py
+++ b/src/calibre/ebooks/mobi/debug/headers.py
@@ -6,14 +6,14 @@ __license__   = 'GPL v3'
 __copyright__ = '2012, Kovid Goyal '
 __docformat__ = 'restructuredtext en'
 
-import struct, datetime, os, numbers
+import struct, datetime, os, numbers, binascii
 
 from calibre.utils.date import utc_tz
 from calibre.ebooks.mobi.reader.headers import NULL_INDEX
 from calibre.ebooks.mobi.langcodes import main_language, sub_language
 from calibre.ebooks.mobi.debug import format_bytes
 from calibre.ebooks.mobi.utils import get_trailing_data
-from polyglot.builtins import as_bytes, iteritems, range, unicode_type
+from polyglot.builtins import iteritems, range, unicode_type
 
 # PalmDB {{{
 
@@ -210,7 +210,7 @@ class EXTHRecord(object):
             else:
                 self.data, = struct.unpack(b'>L', self.data)
         elif self.type in {209, 300}:
-            self.data = as_bytes(self.data.encode('hex'))
+            self.data = binascii.hexlify(self.data)
 
     def __str__(self):
         return '%s (%d): %r'%(self.name, self.type, self.data)
diff --git a/src/calibre/ebooks/mobi/reader/mobi6.py b/src/calibre/ebooks/mobi/reader/mobi6.py
index 6400295a39..ebd54e326e 100644
--- a/src/calibre/ebooks/mobi/reader/mobi6.py
+++ b/src/calibre/ebooks/mobi/reader/mobi6.py
@@ -10,7 +10,7 @@ import shutil, os, re, struct, textwrap, io
 
 from lxml import html, etree
 
-from calibre import (xml_entity_to_unicode, entity_to_unicode)
+from calibre import xml_entity_to_unicode, entity_to_unicode, guess_type
 from calibre.utils.cleantext import clean_ascii_chars, clean_xml_chars
 from calibre.ebooks import DRMError, unit_convert
 from calibre.ebooks.chardet import strip_encoding_declarations
@@ -21,7 +21,7 @@ from calibre.ebooks.metadata import MetaInformation
 from calibre.ebooks.metadata.opf2 import OPFCreator, OPF
 from calibre.ebooks.metadata.toc import TOC
 from calibre.ebooks.mobi.reader.headers import BookHeader
-from calibre.utils.img import save_cover_data_to
+from calibre.utils.img import save_cover_data_to, gif_data_to_png_data, AnimatedGIF
 from calibre.utils.imghdr import what
 from polyglot.builtins import iteritems, unicode_type, range, map
 
@@ -178,7 +178,7 @@ class MobiReader(object):
         self.processed_html = strip_encoding_declarations(self.processed_html)
         self.processed_html = re.sub(r'&(\S+?);', xml_entity_to_unicode,
             self.processed_html)
-        self.extract_images(processed_records, output_dir)
+        image_name_map = self.extract_images(processed_records, output_dir)
         self.replace_page_breaks()
         self.cleanup_html()
 
@@ -272,7 +272,7 @@ class MobiReader(object):
             head.insert(0, title)
             head.text = '\n\t'
 
-        self.upshift_markup(root)
+        self.upshift_markup(root, image_name_map)
         guides = root.xpath('//guide')
         guide = guides[0] if guides else None
         metadata_elems = root.xpath('//metadata')
@@ -389,8 +389,9 @@ class MobiReader(object):
             raw += unit
         return raw
 
-    def upshift_markup(self, root):
+    def upshift_markup(self, root, image_name_map=None):
         self.log.debug('Converting style information to CSS...')
+        image_name_map = image_name_map or {}
         size_map = {
             'xx-small': '0.5',
             'x-small': '1',
@@ -510,10 +511,11 @@ class MobiReader(object):
                     recindex = attrib.pop(attr, None) or recindex
                 if recindex is not None:
                     try:
-                        recindex = '%05d'%int(recindex)
-                    except:
+                        recindex = int(recindex)
+                    except Exception:
                         pass
-                    attrib['src'] = 'images/%s.jpg' % recindex
+                    else:
+                        attrib['src'] = 'images/' + image_name_map.get(recindex, '%05d.jpg' % recindex)
                 for attr in ('width', 'height'):
                     if attr in attrib:
                         val = attrib[attr]
@@ -674,7 +676,7 @@ class MobiReader(object):
         for i in getattr(self, 'image_names', []):
             path = os.path.join(bp, 'images', i)
             added.add(path)
-            manifest.append((path, 'image/jpeg'))
+            manifest.append((path, guess_type(path)[0] or 'image/jpeg'))
         if cover_copied is not None:
             manifest.append((cover_copied, 'image/jpeg'))
 
@@ -870,6 +872,7 @@ class MobiReader(object):
             os.makedirs(output_dir)
         image_index = 0
         self.image_names = []
+        image_name_map = {}
         start = getattr(self.book_header, 'first_image_index', -1)
         if start > self.num_sections or start < 0:
             # BAEN PRC files have bad headers
@@ -882,18 +885,36 @@ class MobiReader(object):
             image_index += 1
             if data[:4] in {b'FLIS', b'FCIS', b'SRCS', b'\xe9\x8e\r\n',
                     b'RESC', b'BOUN', b'FDST', b'DATP', b'AUDI', b'VIDE'}:
-                # This record is a known non image type, not need to try to
+                # This record is a known non image type, no need to try to
                 # load the image
                 continue
 
-            path = os.path.join(output_dir, '%05d.jpg' % image_index)
             try:
-                if what(None, data) not in {'jpg', 'jpeg', 'gif', 'png', 'bmp'}:
-                    continue
-                save_cover_data_to(data, path, minify_to=(10000, 10000))
+                imgfmt = what(None, data)
             except Exception:
                 continue
+            if imgfmt not in {'jpg', 'jpeg', 'gif', 'png', 'bmp'}:
+                continue
+            if imgfmt == 'jpeg':
+                imgfmt = 'jpg'
+            if imgfmt == 'gif':
+                try:
+                    data = gif_data_to_png_data(data)
+                    imgfmt = 'png'
+                except AnimatedGIF:
+                    pass
+            path = os.path.join(output_dir, '%05d.%s' % (image_index, imgfmt))
+            image_name_map[image_index] = os.path.basename(path)
+            if imgfmt == 'png':
+                with open(path, 'wb') as f:
+                    f.write(data)
+            else:
+                try:
+                    save_cover_data_to(data, path, minify_to=(10000, 10000))
+                except Exception:
+                    continue
             self.image_names.append(os.path.basename(path))
+        return image_name_map
 
 
 def test_mbp_regex():
diff --git a/src/calibre/ebooks/mobi/utils.py b/src/calibre/ebooks/mobi/utils.py
index faf7e5a6b5..0121777702 100644
--- a/src/calibre/ebooks/mobi/utils.py
+++ b/src/calibre/ebooks/mobi/utils.py
@@ -10,7 +10,7 @@ import struct, string, zlib, os
 from collections import OrderedDict
 from io import BytesIO
 
-from calibre.utils.img import save_cover_data_to, scale_image, image_to_data, image_from_data, resize_image
+from calibre.utils.img import save_cover_data_to, scale_image, image_to_data, image_from_data, resize_image, png_data_to_gif_data
 from calibre.utils.imghdr import what
 from calibre.ebooks import normalize
 from polyglot.builtins import unicode_type, range, as_bytes, map
@@ -417,13 +417,8 @@ def to_base(num, base=32, min_num_digits=None):
 def mobify_image(data):
     'Convert PNG images to GIF as the idiotic Kindle cannot display some PNG'
     fmt = what(None, data)
-
     if fmt == 'png':
-        from PIL import Image
-        im = Image.open(BytesIO(data))
-        buf = BytesIO()
-        im.save(buf, 'gif')
-        data = buf.getvalue()
+        data = png_data_to_gif_data(data)
     return data
 
 # Font records {{{
diff --git a/src/calibre/ebooks/mobi/writer8/skeleton.py b/src/calibre/ebooks/mobi/writer8/skeleton.py
index 1fab295273..286083331c 100644
--- a/src/calibre/ebooks/mobi/writer8/skeleton.py
+++ b/src/calibre/ebooks/mobi/writer8/skeleton.py
@@ -16,7 +16,7 @@ from lxml import etree
 from calibre import my_unichr
 from calibre.ebooks.oeb.base import XHTML_NS, extract
 from calibre.ebooks.mobi.utils import to_base, PolyglotDict
-from polyglot.builtins import iteritems, unicode_type
+from polyglot.builtins import iteritems, unicode_type, as_bytes
 
 CHUNK_SIZE = 8192
 
@@ -397,7 +397,7 @@ class Chunker(object):
             pos, fid = to_base(pos, min_num_digits=4), to_href(fid)
             return ':off:'.join((pos, fid)).encode('utf-8')
 
-        placeholder_map = {k:to_placeholder(v) for k, v in
+        placeholder_map = {as_bytes(k):to_placeholder(v) for k, v in
                 iteritems(self.placeholder_map)}
 
         # Now update the links
diff --git a/src/calibre/ebooks/oeb/polish/check/css.py b/src/calibre/ebooks/oeb/polish/check/css.py
index f7d7ea3b68..efce567d32 100644
--- a/src/calibre/ebooks/oeb/polish/check/css.py
+++ b/src/calibre/ebooks/oeb/polish/check/css.py
@@ -222,7 +222,12 @@ class Pool(object):
             self.working = False
 
     def shutdown(self):
-        tuple(map(sip.delete, self.workers))
+
+        def safe_delete(x):
+            if not sip.isdeleted(x):
+                sip.delete(x)
+
+        tuple(map(safe_delete, self.workers))
         self.workers = []
 
 
diff --git a/src/calibre/ebooks/oeb/polish/check/main.py b/src/calibre/ebooks/oeb/polish/check/main.py
index b9c43ff6f2..f4715f50dc 100644
--- a/src/calibre/ebooks/oeb/polish/check/main.py
+++ b/src/calibre/ebooks/oeb/polish/check/main.py
@@ -48,16 +48,18 @@ def run_checks(container):
     xml_items, html_items, raster_images, stylesheets = [], [], [], []
     for name, mt in iteritems(container.mime_map):
         items = None
+        decode = False
         if mt in XML_TYPES:
             items = xml_items
         elif mt in OEB_DOCS:
             items = html_items
         elif mt in OEB_STYLES:
+            decode = True
             items = stylesheets
         elif is_raster_image(mt):
             items = raster_images
         if items is not None:
-            items.append((name, mt, container.open(name, 'rb').read()))
+            items.append((name, mt, container.raw_data(name, decode=decode)))
     errors.extend(run_checkers(check_html_size, html_items))
     errors.extend(run_checkers(check_xml_parsing, xml_items))
     errors.extend(run_checkers(check_xml_parsing, html_items))
diff --git a/src/calibre/ebooks/oeb/transforms/flatcss.py b/src/calibre/ebooks/oeb/transforms/flatcss.py
index 3c12a28080..43c2b191ca 100644
--- a/src/calibre/ebooks/oeb/transforms/flatcss.py
+++ b/src/calibre/ebooks/oeb/transforms/flatcss.py
@@ -351,7 +351,7 @@ class CSSFlattener(object):
                         value = 0.0
                     cssdict[property] = "%0.5fem" % (value / fsize)
 
-    def flatten_node(self, node, stylizer, names, styles, pseudo_styles, psize, item_id):
+    def flatten_node(self, node, stylizer, names, styles, pseudo_styles, psize, item_id, recurse=True):
         if not isinstance(node.tag, string_or_bytes) \
            or namespace(node.tag) != XHTML_NS:
             return
@@ -569,8 +569,9 @@ class CSSFlattener(object):
             del node.attrib['class']
         if 'style' in node.attrib:
             del node.attrib['style']
-        for child in node:
-            self.flatten_node(child, stylizer, names, styles, pseudo_styles, psize, item_id)
+        if recurse:
+            for child in node:
+                self.flatten_node(child, stylizer, names, styles, pseudo_styles, psize, item_id)
 
     def flatten_head(self, item, href, global_href):
         html = item.data
@@ -660,9 +661,9 @@ class CSSFlattener(object):
             stylizer = self.stylizers[item]
             if self.specializer is not None:
                 self.specializer(item, stylizer)
-            body = html.find(XHTML('body'))
             fsize = self.context.dest.fbase
-            self.flatten_node(body, stylizer, names, styles, pseudo_styles, fsize, item.id)
+            self.flatten_node(html, stylizer, names, styles, pseudo_styles, fsize, item.id, recurse=False)
+            self.flatten_node(html.find(XHTML('body')), stylizer, names, styles, pseudo_styles, fsize, item.id)
         items = sorted(((key, val) for (val, key) in iteritems(styles)), key=lambda x:numeric_sort_key(x[0]))
         # :hover must come after link and :active must come after :hover
         psels = sorted(pseudo_styles, key=lambda x :
diff --git a/src/calibre/ebooks/pdf/html_writer.py b/src/calibre/ebooks/pdf/html_writer.py
index 17db4bc616..c3abe969b4 100644
--- a/src/calibre/ebooks/pdf/html_writer.py
+++ b/src/calibre/ebooks/pdf/html_writer.py
@@ -18,7 +18,7 @@ from operator import attrgetter, itemgetter
 
 from html5_parser import parse
 from PyQt5.Qt import (
-    QApplication, QMarginsF, QObject, QPageLayout, QTimer, QUrl, pyqtSignal
+    QApplication, QMarginsF, QObject, QPageLayout, Qt, QTimer, QUrl, pyqtSignal
 )
 from PyQt5.QtWebEngineCore import QWebEngineUrlRequestInterceptor
 from PyQt5.QtWebEngineWidgets import QWebEnginePage, QWebEngineProfile
@@ -39,6 +39,7 @@ from calibre.srv.render_book import check_for_maths
 from calibre.utils.fonts.sfnt.container import Sfnt, UnsupportedFont
 from calibre.utils.fonts.sfnt.merge import merge_truetype_fonts_for_pdf
 from calibre.utils.logging import default_log
+from calibre.utils.monotonic import monotonic
 from calibre.utils.podofo import (
     dedup_type3_fonts, get_podofo, remove_unused_fonts, set_metadata_implementation
 )
@@ -49,6 +50,7 @@ from polyglot.builtins import (
 from polyglot.urllib import urlparse
 
 OK, KILL_SIGNAL = range(0, 2)
+HANG_TIME = 60  # seconds
 # }}}
 
 
@@ -172,10 +174,26 @@ class Renderer(QWebEnginePage):
 
         self.titleChanged.connect(self.title_changed)
         self.loadStarted.connect(self.load_started)
+        self.loadProgress.connect(self.load_progress)
         self.loadFinished.connect(self.load_finished)
+        self.load_hang_check_timer = t = QTimer(self)
+        self.load_started_at = 0
+        t.setTimerType(Qt.VeryCoarseTimer)
+        t.setInterval(HANG_TIME * 1000)
+        t.setSingleShot(True)
+        t.timeout.connect(self.on_load_hang)
 
     def load_started(self):
+        self.load_started_at = monotonic()
         self.load_complete = False
+        self.load_hang_check_timer.start()
+
+    def load_progress(self, amt):
+        self.load_hang_check_timer.start()
+
+    def on_load_hang(self):
+        self.log(self.log_prefix, 'Loading not complete after {} seconds, aborting.'.format(int(monotonic() - self.load_started_at)))
+        self.load_finished(False)
 
     def title_changed(self, title):
         if self.wait_for_title and title == self.wait_for_title and self.load_complete:
@@ -187,6 +205,7 @@ class Renderer(QWebEnginePage):
 
     def load_finished(self, ok):
         self.load_complete = True
+        self.load_hang_check_timer.stop()
         if not ok:
             self.working = False
             self.work_done.emit(self, 'Load of {} failed'.format(self.url().toString()))
@@ -900,7 +919,7 @@ def fonts_are_identical(fonts):
     return True
 
 
-def merge_font(fonts):
+def merge_font(fonts, log):
     # choose the largest font as the base font
     fonts.sort(key=lambda f: len(f['Data'] or b''), reverse=True)
     base_font = fonts[0]
@@ -913,7 +932,7 @@ def merge_font(fonts):
     cmaps = list(filter(None, (f['ToUnicode'] for f in t0_fonts)))
     if cmaps:
         t0_font['ToUnicode'] = as_bytes(merge_cmaps(cmaps))
-    base_font['sfnt'], width_for_glyph_id, height_for_glyph_id = merge_truetype_fonts_for_pdf(*(f['sfnt'] for f in descendant_fonts))
+    base_font['sfnt'], width_for_glyph_id, height_for_glyph_id = merge_truetype_fonts_for_pdf(tuple(f['sfnt'] for f in descendant_fonts), log)
     widths = []
     arrays = tuple(filter(None, (f['W'] for f in descendant_fonts)))
     if arrays:
@@ -928,7 +947,7 @@ def merge_font(fonts):
     return t0_font, base_font, references_to_drop
 
 
-def merge_fonts(pdf_doc):
+def merge_fonts(pdf_doc, log):
     all_fonts = pdf_doc.list_fonts(True)
     base_font_map = {}
 
@@ -957,7 +976,7 @@ def merge_fonts(pdf_doc):
     items = []
     for name, fonts in iteritems(base_font_map):
         if mergeable(fonts):
-            t0_font, base_font, references_to_drop = merge_font(fonts)
+            t0_font, base_font, references_to_drop = merge_font(fonts, log)
             for ref in references_to_drop:
                 replacements[ref] = t0_font['Reference']
             data = base_font['sfnt']()[0]
@@ -1227,7 +1246,7 @@ def convert(opf_path, opts, metadata=None, output_path=None, log=default_log, co
         page_number_display_map, page_layout, page_margins_map,
         pdf_metadata, report_progress, toc if has_toc else None)
 
-    merge_fonts(pdf_doc)
+    merge_fonts(pdf_doc, log)
     num_removed = dedup_type3_fonts(pdf_doc)
     if num_removed:
         log('Removed', num_removed, 'duplicated Type3 glyphs')
diff --git a/src/calibre/ebooks/rtf2xml/convert_to_tags.py b/src/calibre/ebooks/rtf2xml/convert_to_tags.py
index 5ea516bb6b..b9c11754da 100644
--- a/src/calibre/ebooks/rtf2xml/convert_to_tags.py
+++ b/src/calibre/ebooks/rtf2xml/convert_to_tags.py
@@ -1,6 +1,5 @@
 from __future__ import unicode_literals, absolute_import, print_function, division
 import os, sys
-from codecs import EncodedFile
 
 from calibre.ebooks.rtf2xml import copy, check_encoding
 from calibre.ptempfile import better_mktemp
@@ -274,15 +273,10 @@ class ConvertToTags:
         if self.__convert_utf or self.__bad_encoding:
             copy_obj = copy.Copy(bug_handler=self.__bug_handler)
             copy_obj.rename(self.__write_to, self.__file)
-            file_encoding = "utf-8"
-            if self.__bad_encoding:
-                file_encoding = "us-ascii"
             with open_for_read(self.__file) as read_obj:
                 with open_for_write(self.__write_to) as write_obj:
-                    write_objenc = EncodedFile(write_obj, self.__encoding,
-                                    file_encoding, 'replace')
                     for line in read_obj:
-                        write_objenc.write(line)
+                        write_obj.write(line)
         copy_obj = copy.Copy(bug_handler=self.__bug_handler)
         if self.__copy:
             copy_obj.copy_file(self.__write_to, "convert_to_tags.data")
diff --git a/src/calibre/gui2/__init__.py b/src/calibre/gui2/__init__.py
index 87534b996d..a081926eba 100644
--- a/src/calibre/gui2/__init__.py
+++ b/src/calibre/gui2/__init__.py
@@ -978,13 +978,16 @@ class Application(QApplication):
         if not geom:
             return
         restored = widget.restoreGeometry(geom)
+        self.ensure_window_on_screen(widget)
+        return restored
+
+    def ensure_window_on_screen(self, widget):
         screen_rect = self.desktop().availableGeometry(widget)
         if not widget.geometry().intersects(screen_rect):
             w = min(widget.width(), screen_rect.width() - 10)
             h = min(widget.height(), screen_rect.height() - 10)
             widget.resize(w, h)
             widget.move((screen_rect.width() - w) // 2, (screen_rect.height() - h) // 2)
-        return restored
 
     def setup_ui_font(self):
         f = QFont(QApplication.font())
diff --git a/src/calibre/gui2/actions/add.py b/src/calibre/gui2/actions/add.py
index 97f23c3e5f..c22d5d2bf5 100644
--- a/src/calibre/gui2/actions/add.py
+++ b/src/calibre/gui2/actions/add.py
@@ -232,21 +232,25 @@ class AddAction(InterfaceAction):
                         return
 
         for id_ in ids:
-            from calibre.ebooks.oeb.polish.create import create_book
-            pt = PersistentTemporaryFile(suffix='.' + format_)
-            pt.close()
-            try:
-                mi = db.new_api.get_metadata(id_, get_cover=False,
-                                    get_user_categories=False, cover_as_data=False)
-                create_book(mi, pt.name, fmt=format_)
-                db.add_format_with_hooks(id_, format_, pt.name, index_is_id=True, notify=True)
-            finally:
-                os.remove(pt.name)
+            self.add_empty_format_to_book(id_, format_)
 
         current_idx = self.gui.library_view.currentIndex()
         if current_idx.isValid():
             view.model().current_changed(current_idx, current_idx)
 
+    def add_empty_format_to_book(self, book_id, fmt):
+        from calibre.ebooks.oeb.polish.create import create_book
+        db = self.gui.current_db
+        pt = PersistentTemporaryFile(suffix='.' + fmt.lower())
+        pt.close()
+        try:
+            mi = db.new_api.get_metadata(book_id, get_cover=False,
+                                get_user_categories=False, cover_as_data=False)
+            create_book(mi, pt.name, fmt=fmt.lower())
+            db.add_format_with_hooks(book_id, fmt, pt.name, index_is_id=True, notify=True)
+        finally:
+            os.remove(pt.name)
+
     def add_archive(self, single):
         paths = choose_files(
             self.gui, 'recursive-archive-add', _('Choose archive file'),
diff --git a/src/calibre/gui2/actions/edit_metadata.py b/src/calibre/gui2/actions/edit_metadata.py
index 657d7cc1fc..f385defff9 100644
--- a/src/calibre/gui2/actions/edit_metadata.py
+++ b/src/calibre/gui2/actions/edit_metadata.py
@@ -312,7 +312,7 @@ class EditMetadataAction(InterfaceAction):
                 intro_msg=_('The downloaded metadata is on the left and the original metadata'
                             ' is on the right. If a downloaded value is blank or unknown,'
                             ' the original value is used.'),
-                action_button=(_('&View Book'), I('view.png'), self.gui.iactions['View'].view_historical),
+                action_button=(_('&View book'), I('view.png'), self.gui.iactions['View'].view_historical),
                 db=db
             )
             if d.exec_() == d.Accepted:
diff --git a/src/calibre/gui2/actions/store.py b/src/calibre/gui2/actions/store.py
index ba948ac738..7b889ec51b 100644
--- a/src/calibre/gui2/actions/store.py
+++ b/src/calibre/gui2/actions/store.py
@@ -13,6 +13,7 @@ from PyQt5.Qt import QIcon, QSize
 from calibre.gui2 import error_dialog
 from calibre.gui2.actions import InterfaceAction
 from calibre.gui2.dialogs.confirm_delete import confirm
+from calibre.utils.localization import localize_user_manual_link
 
 
 class StoreAction(InterfaceAction):
@@ -146,8 +147,9 @@ class StoreAction(InterfaceAction):
             'buying from. Be sure to double check that any books you get '
             'will work with your e-book reader, especially if the book you '
             'are buying has '
-            'DRM.'
-            )), 'about_get_books_msg',
+            'DRM.'
+            ).format(localize_user_manual_link(
+                'https://manual.calibre-ebook.com/drm.html'))), 'about_get_books_msg',
             parent=self.gui, show_cancel_button=False,
             confirm_msg=_('Show this message again'),
             pixmap='dialog_information.png', title=_('About Get books'))
diff --git a/src/calibre/gui2/actions/tweak_epub.py b/src/calibre/gui2/actions/tweak_epub.py
index b8cfd8b1ce..d4c343cf6f 100644
--- a/src/calibre/gui2/actions/tweak_epub.py
+++ b/src/calibre/gui2/actions/tweak_epub.py
@@ -10,7 +10,7 @@ import time
 
 from PyQt5.Qt import QTimer, QDialog, QDialogButtonBox, QCheckBox, QVBoxLayout, QLabel, Qt
 
-from calibre.gui2 import error_dialog
+from calibre.gui2 import error_dialog, question_dialog
 from calibre.gui2.actions import InterfaceAction
 
 
@@ -105,13 +105,23 @@ class TweakEpubAction(InterfaceAction):
         from calibre.ebooks.oeb.polish.main import SUPPORTED
         db = self.gui.library_view.model().db
         fmts = db.formats(book_id, index_is_id=True) or ''
-        fmts = [x.upper().strip() for x in fmts.split(',')]
+        fmts = [x.upper().strip() for x in fmts.split(',') if x]
         tweakable_fmts = set(fmts).intersection(SUPPORTED)
         if not tweakable_fmts:
-            return error_dialog(self.gui, _('Cannot edit book'),
-                    _('The book must be in the %s formats to edit.'
-                        '\n\nFirst convert the book to one of these formats.') % (_(' or ').join(SUPPORTED)),
-                    show=True)
+            if not fmts:
+                if not question_dialog(self.gui, _('No editable formats'),
+                    _('Do you want to create an empty EPUB file to edit?')):
+                    return
+                tweakable_fmts = {'EPUB'}
+                self.gui.iactions['Add Books'].add_empty_format_to_book(book_id, 'EPUB')
+                current_idx = self.gui.library_view.currentIndex()
+                if current_idx.isValid():
+                    self.gui.library_view.model().current_changed(current_idx, current_idx)
+            else:
+                return error_dialog(self.gui, _('Cannot edit book'), _(
+                    'The book must be in the %s formats to edit.'
+                    '\n\nFirst convert the book to one of these formats.'
+                ) % (_(' or ').join(SUPPORTED)), show=True)
         from calibre.gui2.tweak_book import tprefs
         tprefs.refresh()  # In case they were changed in a Tweak Book process
         if len(tweakable_fmts) > 1:
diff --git a/src/calibre/gui2/actions/virtual_library.py b/src/calibre/gui2/actions/virtual_library.py
index 596b697528..496836257f 100644
--- a/src/calibre/gui2/actions/virtual_library.py
+++ b/src/calibre/gui2/actions/virtual_library.py
@@ -4,7 +4,7 @@
 
 from __future__ import absolute_import, division, print_function, unicode_literals
 
-from PyQt5.Qt import QToolButton
+from PyQt5.Qt import QToolButton, QAction
 
 from calibre.gui2.actions import InterfaceAction
 
@@ -24,6 +24,13 @@ class VirtualLibraryAction(InterfaceAction):
     def genesis(self):
         self.menu = m = self.qaction.menu()
         m.aboutToShow.connect(self.about_to_show_menu)
+        self.qs_action = QAction(self.gui)
+        self.gui.addAction(self.qs_action)
+        self.qs_action.triggered.connect(self.gui.choose_vl_triggerred)
+        self.gui.keyboard.register_shortcut(self.unique_name + ' - ' + 'quick-select-vl',
+            _('Quick select Virtual library'), default_keys=('Ctrl+T',),
+            action=self.qs_action, description=_('Quick select a Virtual library'),
+            group=self.action_spec[0])
 
     def about_to_show_menu(self):
         self.gui.build_virtual_library_menu(self.menu, add_tabs_action=False)
diff --git a/src/calibre/gui2/bars.py b/src/calibre/gui2/bars.py
index 046004557b..61e358ac6f 100644
--- a/src/calibre/gui2/bars.py
+++ b/src/calibre/gui2/bars.py
@@ -413,6 +413,7 @@ if isosx:
                 ia = iactions[what]
                 ac = ia.qaction
                 if not ac.menu() and hasattr(ia, 'shortcut_action_for_context_menu'):
+                    ia.shortcut_action_for_context_menu.setIcon(ac.icon())
                     ac = ia.shortcut_action_for_context_menu
                 m.addAction(CloneAction(ac, m))
 
@@ -506,6 +507,7 @@ else:
                 ia = iactions[what]
                 ac = ia.qaction
                 if not ac.menu() and hasattr(ia, 'shortcut_action_for_context_menu'):
+                    ia.shortcut_action_for_context_menu.setIcon(ac.icon())
                     ac = ia.shortcut_action_for_context_menu
                 m.addAction(ac)
 
diff --git a/src/calibre/gui2/book_details.py b/src/calibre/gui2/book_details.py
index 30c6471eed..5b77323a8f 100644
--- a/src/calibre/gui2/book_details.py
+++ b/src/calibre/gui2/book_details.py
@@ -212,7 +212,7 @@ def add_format_entries(menu, data, book_info):
         else:
             m.addSeparator()
             m.addAction(_('Add other application for %s files...') % fmt.upper(), partial(book_info.choose_open_with, book_id, fmt))
-            m.addAction(_('Edit Open With applications...'), partial(edit_programs, fmt, book_info))
+            m.addAction(_('Edit Open with applications...'), partial(edit_programs, fmt, book_info))
             menu.addMenu(m)
             menu.ow = m
         if fmt.upper() in SUPPORTED:
@@ -279,7 +279,7 @@ def add_item_specific_entries(menu, data, book_info):
 def details_context_menu_event(view, ev, book_info):
     url = view.anchorAt(ev.pos())
     menu = view.createStandardContextMenu()
-    menu.addAction(QIcon(I('edit-copy.png')), _('Copy &all'), partial(copy_all, book_info))
+    menu.addAction(QIcon(I('edit-copy.png')), _('Copy &all'), partial(copy_all, view))
     search_internet_added = False
     if url and url.startswith('action:'):
         data = json_loads(from_hex_bytes(url.split(':', 1)[1]))
diff --git a/src/calibre/gui2/convert/page_setup.ui b/src/calibre/gui2/convert/page_setup.ui
index a2cb9cfc10..e3b759a81a 100644
--- a/src/calibre/gui2/convert/page_setup.ui
+++ b/src/calibre/gui2/convert/page_setup.ui
@@ -96,7 +96,10 @@
        
         Margins
        
-       
+       
+        
+         QFormLayout::FieldsStayAtSizeHint
+        
         
          
           
diff --git a/src/calibre/gui2/dbus_export/menu.py b/src/calibre/gui2/dbus_export/menu.py
index 96daa6fe28..d6b4164a9e 100644
--- a/src/calibre/gui2/dbus_export/menu.py
+++ b/src/calibre/gui2/dbus_export/menu.py
@@ -167,7 +167,7 @@ class DBusMenu(QObject):
     def eventFilter(self, obj, ev):
         ac = getattr(obj, 'menuAction', lambda : None)()
         ac_id = self.action_to_id(ac)
-        if ac_id is not None:
+        if ac_id is not None and hasattr(ev, 'action'):
             etype = ev.type()
             if etype == QEvent.ActionChanged:
                 ac_id = self.action_to_id(ev.action())
diff --git a/src/calibre/gui2/dialogs/choose_format.py b/src/calibre/gui2/dialogs/choose_format.py
index c9ef37118b..f8c3d54d1f 100644
--- a/src/calibre/gui2/dialogs/choose_format.py
+++ b/src/calibre/gui2/dialogs/choose_format.py
@@ -41,6 +41,7 @@ class ChooseFormatDialog(QDialog):
         bb.accepted.connect(self.accept), bb.rejected.connect(self.reject)
         h.addStretch(10), h.addWidget(self.buttonBox)
 
+        formats = list(formats)
         for format in formats:
             self.formats.addItem(QListWidgetItem(file_icon_provider().icon_from_ext(format.lower()),
                                                  format.upper()))
diff --git a/src/calibre/gui2/dialogs/drm_error.ui b/src/calibre/gui2/dialogs/drm_error.ui
index 08d3f3d60a..6d3a39e519 100644
--- a/src/calibre/gui2/dialogs/drm_error.ui
+++ b/src/calibre/gui2/dialogs/drm_error.ui
@@ -44,8 +44,7 @@
     
      
       <p>This book is locked by <b>DRM</b>. To learn more about DRM and why you cannot read or convert this book in calibre, 
-          <a href="https://drmfree.calibre-ebook.com/about#drm">click here</a>.<p>A large number of recent, DRM free releases are 
-          available at <a href="https://drmfree.calibre-ebook.com">Open Books</a>.
+          <a href="https://manual.calibre-ebook.com/drm.html">click here</a>.<p>
      
      
       true
diff --git a/src/calibre/gui2/dialogs/metadata_bulk.py b/src/calibre/gui2/dialogs/metadata_bulk.py
index 24d73f465e..42a6b7f960 100644
--- a/src/calibre/gui2/dialogs/metadata_bulk.py
+++ b/src/calibre/gui2/dialogs/metadata_bulk.py
@@ -369,6 +369,7 @@ class MyBlockingBusy(QDialog):  # {{{
         if args.clear_series:
             self.progress_next_step_range.emit(0)
             cache.set_field('series', {bid: '' for bid in self.ids})
+            cache.set_field('series_index', {bid:1.0 for bid in self.ids})
             self.progress_finished_cur_step.emit()
 
         if args.pubdate is not None:
diff --git a/src/calibre/gui2/dialogs/saved_search_editor.py b/src/calibre/gui2/dialogs/saved_search_editor.py
index ec100c7f8f..1b51f8060c 100644
--- a/src/calibre/gui2/dialogs/saved_search_editor.py
+++ b/src/calibre/gui2/dialogs/saved_search_editor.py
@@ -94,7 +94,7 @@ class SavedSearchEditor(Dialog):
     def __init__(self, parent, initial_search=None):
         self.initial_search = initial_search
         Dialog.__init__(
-            self, _('Manage saved searches'), 'manage-saved-searches', parent)
+            self, _('Manage Saved searches'), 'manage-saved-searches', parent)
 
     def setup_ui(self):
         from calibre.gui2.ui import get_gui
diff --git a/src/calibre/gui2/dialogs/tag_categories.py b/src/calibre/gui2/dialogs/tag_categories.py
index af3caeafdb..6436626713 100644
--- a/src/calibre/gui2/dialogs/tag_categories.py
+++ b/src/calibre/gui2/dialogs/tag_categories.py
@@ -10,7 +10,7 @@ from calibre.gui2.dialogs.tag_categories_ui import Ui_TagCategories
 from calibre.gui2.dialogs.confirm_delete import confirm
 from calibre.gui2 import error_dialog
 from calibre.constants import islinux
-from calibre.utils.icu import sort_key, strcmp
+from calibre.utils.icu import sort_key, strcmp, primary_contains
 from polyglot.builtins import iteritems, unicode_type
 
 
@@ -72,9 +72,11 @@ class TagCategories(QDialog, Ui_TagCategories):
                            lambda: [t.original_name.replace('|', ',') for t in self.db_categories['authors']],
                            lambda: [t.original_name for t in self.db_categories['series']],
                            lambda: [t.original_name for t in self.db_categories['publisher']],
-                           lambda: [t.original_name for t in self.db_categories['tags']]
+                           lambda: [t.original_name for t in self.db_categories['tags']],
+                           lambda: [t.original_name for t in self.db_categories['languages']]
                           ]
-        category_names  = ['', _('Authors'), ngettext('Series', 'Series', 2), _('Publishers'), _('Tags')]
+        category_names  = ['', _('Authors'), ngettext('Series', 'Series', 2),
+                            _('Publishers'), _('Tags'), _('Languages')]
 
         for key,cc in iteritems(self.db.custom_field_metadata()):
             if cc['datatype'] in ['text', 'series', 'enumeration']:
@@ -106,6 +108,7 @@ class TagCategories(QDialog, Ui_TagCategories):
         self.category_box.currentIndexChanged[int].connect(self.select_category)
         self.category_filter_box.currentIndexChanged[int].connect(
                                                 self.display_filtered_categories)
+        self.item_filter_box.textEdited.connect(self.display_filtered_items)
         self.delete_category_button.clicked.connect(self.del_category)
         if islinux:
             self.available_items_box.itemDoubleClicked.connect(self.apply_tags)
@@ -168,14 +171,19 @@ class TagCategories(QDialog, Ui_TagCategories):
         w.setToolTip(_('Category lookup name: ') + item.label)
         return w
 
+    def display_filtered_items(self, text):
+        self.display_filtered_categories(None)
+
     def display_filtered_categories(self, idx):
         idx = idx if idx is not None else self.category_filter_box.currentIndex()
         self.available_items_box.clear()
         self.applied_items_box.clear()
+        item_filter = self.item_filter_box.text()
         for item in self.all_items_sorted:
             if idx == 0 or item.label == self.category_labels[idx]:
                 if item.index not in self.applied_items and item.exists:
-                    self.available_items_box.addItem(self.make_list_widget(item))
+                    if primary_contains(item_filter, item.name):
+                        self.available_items_box.addItem(self.make_list_widget(item))
         for index in self.applied_items:
             self.applied_items_box.addItem(self.make_list_widget(self.all_items[index]))
 
diff --git a/src/calibre/gui2/dialogs/tag_categories.ui b/src/calibre/gui2/dialogs/tag_categories.ui
index 198a99d4b2..7274a9bf88 100644
--- a/src/calibre/gui2/dialogs/tag_categories.ui
+++ b/src/calibre/gui2/dialogs/tag_categories.ui
@@ -33,7 +33,7 @@
           category_box
          
         
-     
+       
        
         
          
@@ -64,6 +64,26 @@
          
         
        
+       
+        
+         
+          Item &filter: 
+         
+         
+          Qt::AlignLeading|Qt::AlignLeft|Qt::AlignVCenter
+         
+         
+          item_filter_box
+         
+        
+       
+       
+        
+         
+          Enter text to filter the available items. Case and accents are ignored.
+         
+        
+       
       
      
      
@@ -136,6 +156,13 @@
          
         
        
+       
+        
+         
+           
+         
+        
+       
       
      
     
@@ -152,7 +179,7 @@
        
       
      
-     
+     
       
        
         true
@@ -165,7 +192,7 @@
        
       
      
-     
+     
       
        
         Apply tags to current tag category
@@ -189,7 +216,7 @@
        
       
      
-     
+     
       
        
         true
@@ -199,7 +226,7 @@
        
       
      
-     
+     
       
        
         Unapply (remove) tag from current tag category
@@ -213,7 +240,7 @@
        
       
      
-     
+     
       
        
         Qt::Horizontal
diff --git a/src/calibre/gui2/dnd.py b/src/calibre/gui2/dnd.py
index 6ead535d45..fbeb64ec8c 100644
--- a/src/calibre/gui2/dnd.py
+++ b/src/calibre/gui2/dnd.py
@@ -198,14 +198,7 @@ def dnd_has_extension(md, extensions, allow_all_extensions=False):
     return bool(exts.intersection(frozenset(extensions)))
 
 
-def dnd_get_image(md, image_exts=None):
-    '''
-    Get the image in the QMimeData object md.
-
-    :return: None, None if no image is found
-             QPixmap, None if an image is found, the pixmap is guaranteed not null
-             url, filename if a URL that points to an image is found
-    '''
+def dnd_get_local_image_and_pixmap(md, image_exts=None):
     if md.hasImage():
         for x in md.formats():
             x = unicode_type(x)
@@ -214,14 +207,13 @@ def dnd_get_image(md, image_exts=None):
                 pmap = QPixmap()
                 pmap.loadFromData(cdata)
                 if not pmap.isNull():
-                    return pmap, None
-                break
+                    return pmap, cdata
     if md.hasFormat('application/octet-stream'):
         cdata = bytes(md.data('application/octet-stream'))
         pmap = QPixmap()
         pmap.loadFromData(cdata)
         if not pmap.isNull():
-            return pmap, None
+            return pmap, cdata
 
     if image_exts is None:
         image_exts = image_extensions()
@@ -229,23 +221,40 @@ def dnd_get_image(md, image_exts=None):
     # No image, look for an URL pointing to an image
     urls = urls_from_md(md)
     paths = [path_from_qurl(u) for u in urls]
-    # First look for a local file
+    # Look for a local file
     images = [xi for xi in paths if
             posixpath.splitext(unquote(xi))[1][1:].lower() in
             image_exts]
     images = [xi for xi in images if os.path.exists(xi)]
-    p = QPixmap()
     for path in images:
         try:
             with open(path, 'rb') as f:
-                p.loadFromData(f.read())
+                cdata = f.read()
         except Exception:
             continue
+        p = QPixmap()
+        p.loadFromData(cdata)
         if not p.isNull():
-            return p, None
+            return p, cdata
 
-    # No local images, look for remote ones
+    return None, None
 
+
+def dnd_get_image(md, image_exts=None):
+    '''
+    Get the image in the QMimeData object md.
+
+    :return: None, None if no image is found
+             QPixmap, None if an image is found, the pixmap is guaranteed not null
+             url, filename if a URL that points to an image is found
+    '''
+    if image_exts is None:
+        image_exts = image_extensions()
+    pmap, data = dnd_get_local_image_and_pixmap(md, image_exts)
+    if pmap is not None:
+        return pmap, None
+    # Look for a remote image
+    urls = urls_from_md(md)
     # First, see if this is from Firefox
     rurl, fname = get_firefox_rurl(md, image_exts)
 
diff --git a/src/calibre/gui2/init.py b/src/calibre/gui2/init.py
index 9f3fa08602..4e7bbcba45 100644
--- a/src/calibre/gui2/init.py
+++ b/src/calibre/gui2/init.py
@@ -531,7 +531,7 @@ class VLTabs(QTabBar):  # {{{
             s = m._s = m.addMenu(_('Restore hidden tabs'))
             for x in hidden:
                 s.addAction(x, partial(self.restore, x))
-        m.addAction(_('Hide virtual library tabs'), self.disable_bar)
+        m.addAction(_('Hide Virtual library tabs'), self.disable_bar)
         if gprefs['vl_tabs_closable']:
             m.addAction(_('Lock virtual library tabs'), self.lock_tab)
         else:
diff --git a/src/calibre/gui2/library/alternate_views.py b/src/calibre/gui2/library/alternate_views.py
index bf78e5791e..9b16c65f93 100644
--- a/src/calibre/gui2/library/alternate_views.py
+++ b/src/calibre/gui2/library/alternate_views.py
@@ -1049,12 +1049,15 @@ class GridView(QListView):
     def number_of_columns(self):
         # Number of columns currently visible in the grid
         if self._ncols is None:
+            dpr = self.device_pixel_ratio
+            width = int(dpr * self.delegate.cover_size.width())
+            height = int(dpr * self.delegate.cover_size.height())
             step = max(10, self.spacing())
-            for y in range(step, 500, step):
-                for x in range(step, 500, step):
+            for y in range(step, 2 * height, step):
+                for x in range(step, 2 * width, step):
                     i = self.indexAt(QPoint(x, y))
                     if i.isValid():
-                        for x in range(self.viewport().width() - step, self.viewport().width() - 300, -step):
+                        for x in range(self.viewport().width() - step, self.viewport().width() - width, -step):
                             j = self.indexAt(QPoint(x, y))
                             if j.isValid():
                                 self._ncols = j.row() - i.row() + 1
@@ -1070,7 +1073,8 @@ class GridView(QListView):
             if not ci.isValid():
                 return
             c = ci.row()
-            delta = {Qt.Key_Left: -1, Qt.Key_Right: 1, Qt.Key_Up: -self.number_of_columns(), Qt.Key_Down: self.number_of_columns()}[k]
+            ncols = self.number_of_columns() or 1
+            delta = {Qt.Key_Left: -1, Qt.Key_Right: 1, Qt.Key_Up: -ncols, Qt.Key_Down: ncols}[k]
             n = max(0, min(c + delta, self.model().rowCount(None) - 1))
             if n == c:
                 return
diff --git a/src/calibre/gui2/metadata/basic_widgets.py b/src/calibre/gui2/metadata/basic_widgets.py
index ef0555bcd6..93a61e15b6 100644
--- a/src/calibre/gui2/metadata/basic_widgets.py
+++ b/src/calibre/gui2/metadata/basic_widgets.py
@@ -30,6 +30,7 @@ from calibre.utils.date import (
     local_tz, qt_to_dt, as_local_time, UNDEFINED_DATE, is_date_undefined,
     utcfromtimestamp, parse_only_date, internal_iso_format_string)
 from calibre import strftime
+from calibre.constants import ispy3
 from calibre.ebooks import BOOK_EXTENSIONS
 from calibre.customize.ui import run_plugins_on_import
 from calibre.gui2.comments_editor import Editor
@@ -52,7 +53,7 @@ def save_dialog(parent, title, msg, det_msg=''):
 
 
 def clean_text(x):
-    return re.sub(r'\s', ' ', x.strip())
+    return re.sub(r'\s', ' ', x.strip(), flags=re.ASCII if ispy3 else 0)
 
 
 '''
@@ -221,7 +222,6 @@ class TitleEdit(EnLineEdit, ToMetadataMixin):
 
     @property
     def current_val(self):
-
         title = clean_text(unicode_type(self.text()))
         if not title:
             title = self.get_default()
diff --git a/src/calibre/gui2/metadata/single_download.py b/src/calibre/gui2/metadata/single_download.py
index 998cd25d31..1b157e8a20 100644
--- a/src/calibre/gui2/metadata/single_download.py
+++ b/src/calibre/gui2/metadata/single_download.py
@@ -20,7 +20,7 @@ from PyQt5.Qt import (
     QWidget, QTableView, QGridLayout, QPalette, QTimer, pyqtSignal,
     QAbstractTableModel, QSize, QListView, QPixmap, QModelIndex,
     QAbstractListModel, QRect, QTextBrowser, QStringListModel, QMenu,
-    QCursor, QHBoxLayout, QPushButton, QSizePolicy)
+    QCursor, QHBoxLayout, QPushButton, QSizePolicy, QSplitter)
 
 from calibre.customize.ui import metadata_plugins
 from calibre.ebooks.metadata import authors_to_string, rating_to_stars
@@ -317,8 +317,6 @@ class Comments(HTMLDisplay):  # {{{
     def __init__(self, parent=None):
         HTMLDisplay.__init__(self, parent)
         self.setAcceptDrops(False)
-        self.setMaximumWidth(300)
-        self.setMinimumWidth(300)
         self.wait_timer = QTimer(self)
         self.wait_timer.timeout.connect(self.update_wait)
         self.wait_timer.setInterval(800)
@@ -374,13 +372,6 @@ class Comments(HTMLDisplay):  # {{{
         
         '''%(c,)
         self.setHtml(templ%html)
-
-    def sizeHint(self):
-        # This is needed, because on windows the dialog cannot be resized to
-        # so that this widgets height become < sizeHint().height(). Qt sets the
-        # sizeHint to (800, 600), which makes the dialog unusable on smaller
-        # screens.
-        return QSize(800, 300)
 # }}}
 
 
@@ -454,31 +445,41 @@ class IdentifyWidget(QWidget):  # {{{
         self.abort = Event()
         self.caches = {}
 
-        self.l = l = QGridLayout()
-        self.setLayout(l)
+        self.l = l = QVBoxLayout(self)
 
         names = [''+p.name+'' for p in metadata_plugins(['identify']) if
                 p.is_configured()]
         self.top = QLabel('
'+_('calibre is downloading metadata from: ') +
             ', '.join(names))
         self.top.setWordWrap(True)
-        l.addWidget(self.top, 0, 0)
+        l.addWidget(self.top)
 
+        self.splitter = s = QSplitter(self)
+        s.setChildrenCollapsible(False)
+        l.addWidget(s, 100)
         self.results_view = ResultsView(self)
         self.results_view.book_selected.connect(self.emit_book_selected)
         self.get_result = self.results_view.get_result
-        l.addWidget(self.results_view, 1, 0)
+        s.addWidget(self.results_view)
 
         self.comments_view = Comments(self)
-        l.addWidget(self.comments_view, 1, 1)
+        s.addWidget(self.comments_view)
+        s.setStretchFactor(0, 2)
+        s.setStretchFactor(1, 1)
 
         self.results_view.show_details_signal.connect(self.comments_view.show_data)
 
         self.query = QLabel('download starting...')
         self.query.setWordWrap(True)
-        l.addWidget(self.query, 2, 0, 1, 2)
+        l.addWidget(self.query)
 
         self.comments_view.show_wait()
+        state = gprefs.get('metadata-download-identify-widget-splitter-state')
+        if state is not None:
+            s.restoreState(state)
+
+    def save_state(self):
+        gprefs['metadata-download-identify-widget-splitter-state'] = bytearray(self.splitter.saveState())
 
     def emit_book_selected(self, book):
         self.book_selected.emit(book, self.caches)
@@ -1091,6 +1092,7 @@ class FullFetch(QDialog):  # {{{
     def accept(self):
         # Prevent the usual dialog accept mechanisms from working
         gprefs['metadata_single_gui_geom'] = bytearray(self.saveGeometry())
+        self.identify_widget.save_state()
         if DEBUG_DIALOG:
             if self.stack.currentIndex() == 2:
                 return QDialog.accept(self)
diff --git a/src/calibre/gui2/open_with.py b/src/calibre/gui2/open_with.py
index 8eed3f1757..c829fe8972 100644
--- a/src/calibre/gui2/open_with.py
+++ b/src/calibre/gui2/open_with.py
@@ -444,7 +444,7 @@ def register_keyboard_shortcuts(gui=None, finalize=False):
             unique_name = application['uuid']
             func = partial(gui.open_with_action_triggerred, filetype, application)
             ac.triggered.connect(func)
-            gui.keyboard.register_shortcut(unique_name, name, action=ac, group=_('Open With'))
+            gui.keyboard.register_shortcut(unique_name, name, action=ac, group=_('Open with'))
             gui.addAction(ac)
             registered_shortcuts[unique_name] = ac
     if finalize:
diff --git a/src/calibre/gui2/preferences/behavior.py b/src/calibre/gui2/preferences/behavior.py
index d390d7a216..e93a449952 100644
--- a/src/calibre/gui2/preferences/behavior.py
+++ b/src/calibre/gui2/preferences/behavior.py
@@ -7,6 +7,7 @@ __copyright__ = '2010, Kovid Goyal '
 __docformat__ = 'restructuredtext en'
 
 import re
+from functools import partial
 
 from PyQt5.Qt import Qt, QListWidgetItem
 
@@ -22,6 +23,13 @@ from calibre.utils.icu import sort_key
 from polyglot.builtins import unicode_type, range
 
 
+def input_order_drop_event(self, ev):
+    ret = self.opt_input_order.__class__.dropEvent(self.opt_input_order, ev)
+    if ev.isAccepted():
+        self.changed_signal.emit()
+    return ret
+
+
 class OutputFormatSetting(Setting):
 
     CHOICES_SEARCH_FLAGS = Qt.MatchFixedString
@@ -62,6 +70,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
 
         self.input_up_button.clicked.connect(self.up_input)
         self.input_down_button.clicked.connect(self.down_input)
+        self.opt_input_order.dropEvent = partial(input_order_drop_event, self)
         for signal in ('Activated', 'Changed', 'DoubleClicked', 'Clicked'):
             signal = getattr(self.opt_internally_viewed_formats, 'item'+signal)
             signal.connect(self.internally_viewed_formats_changed)
@@ -147,7 +156,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
         for format in input_map + list(all_formats.difference(input_map)):
             item = QListWidgetItem(format, self.opt_input_order)
             item.setData(Qt.UserRole, (format))
-            item.setFlags(Qt.ItemIsEnabled|Qt.ItemIsSelectable)
+            item.setFlags(Qt.ItemIsEnabled|Qt.ItemIsSelectable|Qt.ItemIsDragEnabled)
 
     def up_input(self, *args):
         idx = self.opt_input_order.currentRow()
@@ -175,6 +184,6 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
 
 
 if __name__ == '__main__':
-    from PyQt5.Qt import QApplication
-    app = QApplication([])
+    from calibre.gui2 import Application
+    app = Application([])
     test_widget('Interface', 'Behavior')
diff --git a/src/calibre/gui2/preferences/behavior.ui b/src/calibre/gui2/preferences/behavior.ui
index 72bc8958e2..d700aa33c9 100644
--- a/src/calibre/gui2/preferences/behavior.ui
+++ b/src/calibre/gui2/preferences/behavior.ui
@@ -104,6 +104,15 @@
               0
              
             
+            
+             true
+            
+            
+             QAbstractItemView::InternalMove
+            
+            
+             Qt::MoveAction
+            
             
              true
             
diff --git a/src/calibre/gui2/preferences/metadata_sources.ui b/src/calibre/gui2/preferences/metadata_sources.ui
index 5165482c2a..3d93a19491 100644
--- a/src/calibre/gui2/preferences/metadata_sources.ui
+++ b/src/calibre/gui2/preferences/metadata_sources.ui
@@ -119,7 +119,7 @@
                Restore your own subset of checked fields that you define using the 'Set as default' button
               
               
-               &Select default
+               Select &default
               
              
             
diff --git a/src/calibre/gui2/preferences/tweaks.py b/src/calibre/gui2/preferences/tweaks.py
index 3ff445f6ee..a63c88b667 100644
--- a/src/calibre/gui2/preferences/tweaks.py
+++ b/src/calibre/gui2/preferences/tweaks.py
@@ -372,6 +372,7 @@ class TweaksView(QListView):
         self.setAlternatingRowColors(True)
         self.setSpacing(5)
         self.setVerticalScrollMode(self.ScrollPerPixel)
+        self.setMinimumWidth(300)
 
     def currentChanged(self, cur, prev):
         QListView.currentChanged(self, cur, prev)
diff --git a/src/calibre/gui2/search_restriction_mixin.py b/src/calibre/gui2/search_restriction_mixin.py
index 23847ded5e..f73afb5b13 100644
--- a/src/calibre/gui2/search_restriction_mixin.py
+++ b/src/calibre/gui2/search_restriction_mixin.py
@@ -99,9 +99,9 @@ class CreateVirtualLibrary(QDialog):  # {{{
         self.existing_names = existing_names
 
         if editing:
-            self.setWindowTitle(_('Edit virtual library'))
+            self.setWindowTitle(_('Edit Virtual library'))
         else:
-            self.setWindowTitle(_('Create virtual library'))
+            self.setWindowTitle(_('Create Virtual library'))
         self.setWindowIcon(QIcon(I('lt.png')))
 
         gl = QGridLayout()
@@ -127,7 +127,7 @@ class CreateVirtualLibrary(QDialog):  # {{{
         gl.addWidget(self.vl_text, 1, 1)
         self.vl_text.setText(_build_full_search_string(self.gui))
 
-        self.sl = sl = QLabel('
'+_('Create a virtual library based on: ')+
+        self.sl = sl = QLabel('
'+_('Create a Virtual library based on: ')+
             ('{0}, '
             '{1}, '
             '{2}, '
@@ -143,11 +143,11 @@ class CreateVirtualLibrary(QDialog):  # {{{
         self.hl = hl = QLabel(_('''
             
Virtual libraries

 
-            
With virtual libraries, you can restrict calibre to only show
-            you books that match a search. When a virtual library is in effect, calibre
+            
With Virtual libraries, you can restrict calibre to only show
+            you books that match a search. When a Virtual library is in effect, calibre
             behaves as though the library contains only the matched books. The Tag browser
             display only the tags/authors/series/etc. that belong to the matched books and any searches
-            you do will only search within the books in the virtual library. This
+            you do will only search within the books in the Virtual library. This
             is a good way to partition your large library into smaller and easier to work with subsets.

 
             
For example you can use a Virtual library to only show you books with the Tag "Unread"
@@ -225,7 +225,7 @@ class CreateVirtualLibrary(QDialog):  # {{{
         if self.editing and (self.vl_text.text() != self.original_search or
                              self.new_name != self.editing):
             if not question_dialog(self.gui, _('Search text changed'),
-                         _('The virtual library name or the search text has changed. '
+                         _('The Virtual library name or the search text has changed. '
                            'Do you want to discard these changes?'),
                          default_yes=False):
                 self.vl_name.blockSignals(True)
@@ -263,13 +263,13 @@ class CreateVirtualLibrary(QDialog):  # {{{
         n = unicode_type(self.vl_name.currentText()).strip()
         if not n:
             error_dialog(self.gui, _('No name'),
-                         _('You must provide a name for the new virtual library'),
+                         _('You must provide a name for the new Virtual library'),
                          show=True)
             return
 
         if n.startswith('*'):
             error_dialog(self.gui, _('Invalid name'),
-                         _('A virtual library name cannot begin with "*"'),
+                         _('A Virtual library name cannot begin with "*"'),
                          show=True)
             return
 
@@ -283,7 +283,7 @@ class CreateVirtualLibrary(QDialog):  # {{{
         v = unicode_type(self.vl_text.text()).strip()
         if not v:
             error_dialog(self.gui, _('No search string'),
-                         _('You must provide a search to define the new virtual library'),
+                         _('You must provide a search to define the new Virtual library'),
                          show=True)
             return
 
@@ -298,7 +298,7 @@ class CreateVirtualLibrary(QDialog):  # {{{
 
         if not recs and not question_dialog(
                 self.gui, _('Search found no books'),
-                _('The search found no books, so the virtual library '
+                _('The search found no books, so the Virtual library '
                 'will be empty. Do you really want to use that search?'),
                 default_yes=False):
             return
@@ -380,6 +380,8 @@ class SearchRestrictionMixin(object):
         self.build_virtual_library_list(a, self.remove_vl_triggered)
         m.addMenu(a)
 
+        m.addAction(_('Quick select Virtual library'), self.choose_vl_triggerred)
+
         if add_tabs_action:
             if gprefs['show_vl_tabs']:
                 m.addAction(_('Hide virtual library tabs'), self.vl_tabs.disable_bar)
@@ -419,8 +421,12 @@ class SearchRestrictionMixin(object):
 
         virt_libs = db.prefs.get('virtual_libraries', {})
         for vl in sorted(virt_libs.keys(), key=sort_key):
-            a = m.addAction(self.checked if vl == current_lib else self.empty, vl.replace('&', '&&'))
-            a.triggered.connect(partial(self.apply_virtual_library, library=vl))
+            is_current = vl == current_lib
+            a = m.addAction(self.checked if is_current else self.empty, vl.replace('&', '&&'))
+            if is_current:
+                a.triggered.connect(self.apply_virtual_library)
+            else:
+                a.triggered.connect(partial(self.apply_virtual_library, library=vl))
 
     def virtual_library_menu_about_to_show(self):
         self.build_virtual_library_menu(self.virtual_library_menu)
@@ -492,6 +498,29 @@ class SearchRestrictionMixin(object):
             return
         self._remove_vl(name, reapply=True)
 
+    def choose_vl_triggerred(self):
+        from calibre.gui2.tweak_book.widgets import QuickOpen, emphasis_style
+        db = self.library_view.model().db
+        virt_libs = db.prefs.get('virtual_libraries', {})
+        if not virt_libs:
+            return error_dialog(self, _('No virtual libraries'), _(
+                'No Virtual libraries present, create some first'), show=True)
+        example = '
{0}S{1}ome {0}B{1}ook {0}C{1}ollection
'.format(
+            '' % emphasis_style(), '')
+        chars = '
sbc
' % emphasis_style()
+        help_text = _('''
Quickly choose a Virtual library by typing in just a few characters from the library name into the field above.
+        For example, if want to choose the VL:
+        {example}
+        Simply type in the characters:
+        {chars}
+        and press Enter.''').format(example=example, chars=chars)
+
+        d = QuickOpen(
+                sorted(virt_libs.keys(), key=sort_key), parent=self, title=_('Choose Virtual library'),
+                name='vl-open', level1=' ', help_text=help_text)
+        if d.exec_() == d.Accepted and d.selected_result:
+            self.apply_virtual_library(library=d.selected_result)
+
     def _remove_vl(self, name, reapply=True):
         db = self.library_view.model().db
         virt_libs = db.prefs.get('virtual_libraries', {})
diff --git a/src/calibre/gui2/store/config/chooser/models.py b/src/calibre/gui2/store/config/chooser/models.py
index 365565f0a7..fe836e12ce 100644
--- a/src/calibre/gui2/store/config/chooser/models.py
+++ b/src/calibre/gui2/store/config/chooser/models.py
@@ -6,27 +6,51 @@ __license__ = 'GPL 3'
 __copyright__ = '2011, John Schember '
 __docformat__ = 'restructuredtext en'
 
-from PyQt5.Qt import (Qt, QAbstractItemModel, QIcon, QModelIndex, QSize)
 
-from calibre.customize.ui import is_disabled, disable_plugin, enable_plugin
-from calibre.db.search import _match, CONTAINS_MATCH, EQUALS_MATCH, REGEXP_MATCH
+from PyQt5.Qt import (
+    QAbstractItemModel, QIcon, QModelIndex, QStyledItemDelegate, Qt
+)
+
+from calibre import fit_image
+from calibre.customize.ui import disable_plugin, enable_plugin, is_disabled
+from calibre.db.search import CONTAINS_MATCH, EQUALS_MATCH, REGEXP_MATCH, _match
 from calibre.utils.config_base import prefs
 from calibre.utils.icu import sort_key
 from calibre.utils.search_query_parser import SearchQueryParser
-from polyglot.builtins import unicode_type, range
+from polyglot.builtins import range, unicode_type
+
+
+class Delegate(QStyledItemDelegate):
+
+    def paint(self, painter, option, index):
+        icon = index.data(Qt.DecorationRole)
+        if icon and not icon.isNull():
+            QStyledItemDelegate.paint(self, painter, option, QModelIndex())
+            pw, ph = option.rect.width(), option.rect.height()
+            scaled, w, h = fit_image(option.decorationSize.width(), option.decorationSize.height(), pw, ph)
+            r = option.rect
+            if pw > w:
+                x = (pw - w) // 2
+                r = r.adjusted(x, 0, -x, 0)
+            if ph > h:
+                y = (ph - h) // 2
+                r = r.adjusted(0, y, 0, -y)
+            painter.drawPixmap(r, icon.pixmap(w, h))
+        else:
+            QStyledItemDelegate.paint(self, painter, option, index)
 
 
 class Matches(QAbstractItemModel):
 
     HEADERS = [_('Enabled'), _('Name'), _('No DRM'), _('Headquarters'), _('Affiliate'), _('Formats')]
-    HTML_COLS = [1]
+    HTML_COLS = (1,)
+    CENTERED_COLUMNS = (2, 3, 4)
 
     def __init__(self, plugins):
         QAbstractItemModel.__init__(self)
 
         self.NO_DRM_ICON = QIcon(I('ok.png'))
-        self.DONATE_ICON = QIcon()
-        self.DONATE_ICON.addFile(I('donate.png'), QSize(16, 16))
+        self.DONATE_ICON = QIcon(I('donate.png'))
 
         self.all_matches = plugins
         self.matches = plugins
@@ -123,6 +147,10 @@ class Matches(QAbstractItemModel):
                 if is_disabled(result):
                     return Qt.Unchecked
                 return Qt.Checked
+        elif role == Qt.TextAlignmentRole:
+            if col in self.CENTERED_COLUMNS:
+                return Qt.AlignHCenter
+            return Qt.AlignLeft
         elif role == Qt.ToolTipRole:
             if col == 0:
                 if is_disabled(result):
@@ -182,9 +210,7 @@ class Matches(QAbstractItemModel):
         if not self.matches:
             return
         descending = order == Qt.DescendingOrder
-        self.matches.sort(None,
-            lambda x: sort_key(unicode_type(self.data_as_text(x, col))),
-            descending)
+        self.matches.sort(key=lambda x: sort_key(unicode_type(self.data_as_text(x, col))), reverse=descending)
         if reset:
             self.beginResetModel(), self.endResetModel()
 
diff --git a/src/calibre/gui2/store/config/chooser/results_view.py b/src/calibre/gui2/store/config/chooser/results_view.py
index 432a6b6448..208d23e64d 100644
--- a/src/calibre/gui2/store/config/chooser/results_view.py
+++ b/src/calibre/gui2/store/config/chooser/results_view.py
@@ -12,7 +12,7 @@ from PyQt5.Qt import (Qt, QTreeView, QSize, QMenu)
 
 from calibre.customize.ui import store_plugins
 from calibre.gui2.metadata.single_download import RichTextDelegate
-from calibre.gui2.store.config.chooser.models import Matches
+from calibre.gui2.store.config.chooser.models import Matches, Delegate
 from polyglot.builtins import range
 
 
@@ -27,6 +27,8 @@ class ResultsView(QTreeView):
         self.setIconSize(QSize(24, 24))
 
         self.rt_delegate = RichTextDelegate(self)
+        self.delegate = Delegate()
+        self.setItemDelegate(self.delegate)
 
         for i in self._model.HTML_COLS:
             self.setItemDelegateForColumn(i, self.rt_delegate)
diff --git a/src/calibre/gui2/store/search/search.py b/src/calibre/gui2/store/search/search.py
index b4a09d610a..09eb34c472 100644
--- a/src/calibre/gui2/store/search/search.py
+++ b/src/calibre/gui2/store/search/search.py
@@ -392,7 +392,7 @@ class SearchDialog(QDialog, Ui_Dialog):
             self.open_store(result)
 
     def download_book(self, result):
-        d = ChooseFormatDialog(self, _('Choose format to download to your library.'), result.downloads.keys())
+        d = ChooseFormatDialog(self, _('Choose format to download to your library.'), list(result.downloads.keys()))
         if d.exec_() == d.Accepted:
             ext = d.format()
             fname = result.title[:60] + '.' + ext.lower()
diff --git a/src/calibre/gui2/store/stores/open_books_plugin.py b/src/calibre/gui2/store/stores/open_books_plugin.py
deleted file mode 100644
index 8f8c5dd305..0000000000
--- a/src/calibre/gui2/store/stores/open_books_plugin.py
+++ /dev/null
@@ -1,80 +0,0 @@
-# -*- coding: utf-8 -*-
-from __future__ import absolute_import, division, print_function, unicode_literals
-
-store_version = 1  # Needed for dynamic plugin loading
-
-__license__ = 'GPL 3'
-__copyright__ = '2011, John Schember '
-__docformat__ = 'restructuredtext en'
-
-from contextlib import closing
-try:
-    from urllib.parse import quote_plus
-except ImportError:
-    from urllib import quote_plus
-
-from lxml import html
-
-from PyQt5.Qt import QUrl
-
-from calibre import browser, url_slash_cleaner
-from calibre.gui2 import open_url
-from calibre.gui2.store import StorePlugin
-from calibre.gui2.store.basic_config import BasicStoreConfig
-from calibre.gui2.store.search_result import SearchResult
-from calibre.gui2.store.web_store_dialog import WebStoreDialog
-
-
-class OpenBooksStore(BasicStoreConfig, StorePlugin):
-
-    def open(self, parent=None, detail_item=None, external=False):
-        url = 'https://drmfree.calibre-ebook.com/'
-
-        if external or self.config.get('open_external', False):
-            open_url(QUrl(url_slash_cleaner(detail_item if detail_item else url)))
-        else:
-            d = WebStoreDialog(self.gui, url, parent, detail_item)
-            d.setWindowTitle(self.name)
-            d.set_tags(self.config.get('tags', ''))
-            d.exec_()
-
-    def search(self, query, max_results=10, timeout=60):
-        url = 'https://drmfree.calibre-ebook.com/search/?q=' + quote_plus(query)
-
-        br = browser()
-
-        counter = max_results
-        with closing(br.open(url, timeout=timeout)) as f:
-            doc = html.fromstring(f.read())
-            for data in doc.xpath('//ul[@id="object_list"]//li'):
-                if counter <= 0:
-                    break
-
-                id = ''.join(data.xpath('.//div[@class="links"]/a[1]/@href'))
-                id = id.strip()
-                if not id:
-                    continue
-
-                cover_url = ''.join(data.xpath('.//div[@class="cover"]/img/@src'))
-
-                price = ''.join(data.xpath('.//div[@class="price"]/text()'))
-                a, b, price = price.partition('Price:')
-                price = price.strip()
-                if not price:
-                    continue
-
-                title = ''.join(data.xpath('.//div/strong/text()'))
-                author = ''.join(data.xpath('.//div[@class="author"]//text()'))
-                author = author.partition('by')[-1]
-
-                counter -= 1
-
-                s = SearchResult()
-                s.cover_url = cover_url
-                s.title = title.strip()
-                s.author = author.strip()
-                s.price = price.strip()
-                s.detail_item = id.strip()
-                s.drm = SearchResult.DRM_UNLOCKED
-
-                yield s
diff --git a/src/calibre/gui2/tag_browser/ui.py b/src/calibre/gui2/tag_browser/ui.py
index 86f37160eb..33cc4d9a55 100644
--- a/src/calibre/gui2/tag_browser/ui.py
+++ b/src/calibre/gui2/tag_browser/ui.py
@@ -114,7 +114,7 @@ class TagBrowserMixin(object):  # {{{
         if new_category_name is not None:
             new_name = new_category_name.replace('.', '')
         else:
-            new_name = _('New Category').replace('.', '')
+            new_name = _('New category').replace('.', '')
         n = new_name
         while True:
             new_cat = on_category_key[1:] + '.' + n
diff --git a/src/calibre/gui2/tweak_book/boss.py b/src/calibre/gui2/tweak_book/boss.py
index 9e9062409a..17f88e6723 100644
--- a/src/calibre/gui2/tweak_book/boss.py
+++ b/src/calibre/gui2/tweak_book/boss.py
@@ -150,6 +150,8 @@ class Boss(QObject):
         self.gui.preview.split_requested.connect(self.split_requested)
         self.gui.preview.link_clicked.connect(self.link_clicked)
         self.gui.preview.render_process_restarted.connect(self.report_render_process_restart)
+        self.gui.preview.open_file_with.connect(self.open_file_with)
+        self.gui.preview.edit_file.connect(self.edit_file_requested)
         self.gui.check_book.item_activated.connect(self.check_item_activated)
         self.gui.check_book.check_requested.connect(self.check_requested)
         self.gui.check_book.fix_requested.connect(self.fix_requested)
@@ -1374,7 +1376,7 @@ class Boss(QObject):
         from calibre.gui2.open_with import run_program
         run_program(entry, dest.name, self)
         if question_dialog(self.gui, _('File opened'), _(
-            'When you are done editing {0} click "Update" to update'
+            'When you are done editing {0} click "Import" to update'
             ' the file in the book or "Discard" to lose any changes.').format(file_name),
             yes_text=_('Import'), no_text=_('Discard')
         ):
diff --git a/src/calibre/gui2/tweak_book/editor/smarts/html.py b/src/calibre/gui2/tweak_book/editor/smarts/html.py
index 91f031cd29..45c4af2915 100644
--- a/src/calibre/gui2/tweak_book/editor/smarts/html.py
+++ b/src/calibre/gui2/tweak_book/editor/smarts/html.py
@@ -352,6 +352,18 @@ class Smarts(NullSmarts):
         editor.setTextCursor(c)
         return True
 
+    def select_tag_contents(self, editor):
+        editor.highlighter.join()
+        start = self.last_matched_tag
+        end = self.last_matched_closing_tag
+        if start is None or end is None:
+            return False
+        c = editor.textCursor()
+        c.setPosition(start.start_block.position() + start.end_offset + 1)
+        c.setPosition(end.start_block.position() + end.start_offset, c.KeepAnchor)
+        editor.setTextCursor(c)
+        return True
+
     def remove_tag(self, editor):
         editor.highlighter.join()
         if not self.last_matched_closing_tag and not self.last_matched_tag:
@@ -662,6 +674,8 @@ class Smarts(NullSmarts):
             if int(mods & Qt.ControlModifier):
                 if self.jump_to_enclosing_tag(editor, key == Qt.Key_BraceLeft):
                     return True
+        if key == Qt.Key_T and int(ev.modifiers() & (Qt.ControlModifier | Qt.AltModifier)):
+            return self.select_tag_contents(editor)
 
         return False
 
diff --git a/src/calibre/gui2/tweak_book/editor/syntax/css.py b/src/calibre/gui2/tweak_book/editor/syntax/css.py
index f641b9f66e..1f1d117221 100644
--- a/src/calibre/gui2/tweak_book/editor/syntax/css.py
+++ b/src/calibre/gui2/tweak_book/editor/syntax/css.py
@@ -53,6 +53,7 @@ content_tokens = [(re.compile(k), v, n) for k, v, n in [
     r'outline-style|outline-width|overflow(?:-x|-y)?|padding-bottom|'
     r'padding-left|padding-right|padding-top|padding|'
     r'page-break-after|page-break-before|page-break-inside|'
+    r'break-before|break-after|'
     r'pause-after|pause-before|pause|pitch|pitch-range|'
     r'play-during|position|pre-wrap|pre-line|pre|quotes|richness|right|size|'
     r'speak-header|speak-numeral|speak-punctuation|speak|'
diff --git a/src/calibre/gui2/tweak_book/file_list.py b/src/calibre/gui2/tweak_book/file_list.py
index 3fffe3c3c4..d533947a73 100644
--- a/src/calibre/gui2/tweak_book/file_list.py
+++ b/src/calibre/gui2/tweak_book/file_list.py
@@ -188,7 +188,40 @@ class ItemDelegate(QStyledItemDelegate):  # {{{
 # }}}
 
 
-class FileList(QTreeWidget):
+class OpenWithHandler(object):  # {{{
+
+    def add_open_with_actions(self, menu, file_name):
+        from calibre.gui2.open_with import populate_menu, edit_programs
+        fmt = file_name.rpartition('.')[-1].lower()
+        if not fmt:
+            return
+        m = QMenu(_('Open %s with...') % file_name)
+
+        def connect_action(ac, entry):
+            connect_lambda(ac.triggered, self, lambda self: self.open_with(file_name, fmt, entry))
+
+        populate_menu(m, connect_action, fmt)
+        if len(m.actions()) == 0:
+            menu.addAction(_('Open %s with...') % file_name, partial(self.choose_open_with, file_name, fmt))
+        else:
+            m.addSeparator()
+            m.addAction(_('Add other application for %s files...') % fmt.upper(), partial(self.choose_open_with, file_name, fmt))
+            m.addAction(_('Edit Open with applications...'), partial(edit_programs, fmt, self))
+            menu.addMenu(m)
+            menu.ow = m
+
+    def choose_open_with(self, file_name, fmt):
+        from calibre.gui2.open_with import choose_program
+        entry = choose_program(fmt, self)
+        if entry is not None:
+            self.open_with(file_name, fmt, entry)
+
+    def open_with(self, file_name, fmt, entry):
+        raise NotImplementedError()
+# }}}
+
+
+class FileList(QTreeWidget, OpenWithHandler):
 
     delete_requested = pyqtSignal(object, object)
     reorder_spine = pyqtSignal(object)
@@ -586,26 +619,6 @@ class FileList(QTreeWidget):
         if len(list(m.actions())) > 0:
             m.popup(self.mapToGlobal(point))
 
-    def add_open_with_actions(self, menu, file_name):
-        from calibre.gui2.open_with import populate_menu, edit_programs
-        fmt = file_name.rpartition('.')[-1].lower()
-        if not fmt:
-            return
-        m = QMenu(_('Open %s with...') % file_name)
-
-        def connect_action(ac, entry):
-            connect_lambda(ac.triggered, self, lambda self: self.open_with(file_name, fmt, entry))
-
-        populate_menu(m, connect_action, fmt)
-        if len(m.actions()) == 0:
-            menu.addAction(_('Open %s with...') % file_name, partial(self.choose_open_with, file_name, fmt))
-        else:
-            m.addSeparator()
-            m.addAction(_('Add other application for %s files...') % fmt.upper(), partial(self.choose_open_with, file_name, fmt))
-            m.addAction(_('Edit Open With applications...'), partial(edit_programs, fmt, file_name))
-            menu.addMenu(m)
-            menu.ow = m
-
     def choose_open_with(self, file_name, fmt):
         from calibre.gui2.open_with import choose_program
         entry = choose_program(fmt, self)
diff --git a/src/calibre/gui2/tweak_book/preview.py b/src/calibre/gui2/tweak_book/preview.py
index 7839bad725..01ba7ceaee 100644
--- a/src/calibre/gui2/tweak_book/preview.py
+++ b/src/calibre/gui2/tweak_book/preview.py
@@ -3,10 +3,6 @@
 # License: GPLv3 Copyright: 2015, Kovid Goyal 
 from __future__ import absolute_import, division, print_function, unicode_literals
 
-# TODO:
-# live css
-# check that clicking on both internal and external links works
-
 import textwrap
 import time
 from collections import defaultdict
@@ -30,6 +26,7 @@ from calibre.ebooks.oeb.base import OEB_DOCS, XHTML_MIME, serialize
 from calibre.ebooks.oeb.polish.parsing import parse
 from calibre.gui2 import NO_URL_FORMATTING, error_dialog, open_url
 from calibre.gui2.tweak_book import TOP, actions, current_container, editors, tprefs
+from calibre.gui2.tweak_book.file_list import OpenWithHandler
 from calibre.gui2.viewer.web_view import send_reply
 from calibre.gui2.webengine import (
     Bridge, RestartingWebEngineView, create_script, from_js, insert_scripts,
@@ -337,7 +334,7 @@ class Inspector(QWidget):
         return QSize(1280, 600)
 
 
-class WebView(RestartingWebEngineView):
+class WebView(RestartingWebEngineView, OpenWithHandler):
 
     def __init__(self, parent=None):
         RestartingWebEngineView.__init__(self, parent)
@@ -398,8 +395,27 @@ class WebView(RestartingWebEngineView):
         menu.addAction(QIcon(I('debug.png')), _('Inspect element'), self.inspect)
         if url.partition(':')[0].lower() in {'http', 'https'}:
             menu.addAction(_('Open link'), partial(open_url, data.linkUrl()))
+        if data.MediaTypeImage <= data.mediaType() <= data.MediaTypeFile:
+            url = data.mediaUrl()
+            if url.scheme() == FAKE_PROTOCOL:
+                href = url.path().lstrip('/')
+                if href:
+                    c = current_container()
+                    resource_name = c.href_to_name(href)
+                    if resource_name and c.exists(resource_name) and resource_name not in c.names_that_must_not_be_changed:
+                        self.add_open_with_actions(menu, resource_name)
+                        if data.mediaType() == data.MediaTypeImage:
+                            mime = c.mime_map[resource_name]
+                            if mime.startswith('image/'):
+                                menu.addAction(_('Edit %s') % resource_name, partial(self.edit_image, resource_name))
         menu.exec_(ev.globalPos())
 
+    def open_with(self, file_name, fmt, entry):
+        self.parent().open_file_with.emit(file_name, fmt, entry)
+
+    def edit_image(self, resource_name):
+        self.parent().edit_file.emit(resource_name)
+
 
 class Preview(QWidget):
 
@@ -411,6 +427,8 @@ class Preview(QWidget):
     refreshed = pyqtSignal()
     live_css_data = pyqtSignal(object)
     render_process_restarted = pyqtSignal()
+    open_file_with = pyqtSignal(object, object, object)
+    edit_file = pyqtSignal(object)
 
     def __init__(self, parent=None):
         QWidget.__init__(self, parent)
diff --git a/src/calibre/gui2/tweak_book/ui.py b/src/calibre/gui2/tweak_book/ui.py
index 0ce8f2f59c..008fa5186c 100644
--- a/src/calibre/gui2/tweak_book/ui.py
+++ b/src/calibre/gui2/tweak_book/ui.py
@@ -359,7 +359,6 @@ class Main(MainWindow):
         self.action_edit_previous_file = treg('arrow-up.png', _('Edit &previous file'), partial(self.boss.edit_next_file, backwards=True),
                 'edit-previous-file', 'Ctrl+Alt+Up', _('Edit the previous file in the spine'))
         # Qt does not generate shortcut overrides for cmd+arrow on os x which
-        # Qt does not generate shortcut overrides for cmd+arrow on os x which
         # means these shortcuts interfere with editing
         self.action_global_undo = treg('back.png', _('&Revert to before'), self.boss.do_global_undo, 'global-undo', () if isosx else 'Ctrl+Left',
                                       _('Revert book to before the last action (Undo)'))
@@ -370,7 +369,7 @@ class Main(MainWindow):
         self.action_save_copy = treg('save.png', _('Save a ©'), self.boss.save_copy, 'save-copy', 'Ctrl+Alt+S', _('Save a copy of the book'))
         self.action_quit = treg('window-close.png', _('&Quit'), self.boss.quit, 'quit', 'Ctrl+Q', _('Quit'))
         self.action_preferences = treg('config.png', _('&Preferences'), self.boss.preferences, 'preferences', 'Ctrl+P', _('Preferences'))
-        self.action_new_book = treg('plus.png', _('Create &new, empty book'), self.boss.new_book, 'new-book', (), _('Create a new, empty book'))
+        self.action_new_book = treg('plus.png', _('Create new, &empty book'), self.boss.new_book, 'new-book', (), _('Create a new, empty book'))
         self.action_import_book = treg('add_book.png', _('&Import an HTML or DOCX file as a new book'),
                                       self.boss.import_book, 'import-book', (), _('Import an HTML or DOCX file as a new book'))
         self.action_quick_edit = treg('modified.png', _('&Quick open a file to edit'), self.boss.quick_open, 'quick-open', ('Ctrl+T'), _(
@@ -474,7 +473,7 @@ class Main(MainWindow):
                                          'find', {'direction':'up'}, ('Shift+F3', 'Shift+Ctrl+G'), _('Find previous match'))
         self.action_replace = sreg('replace', _('&Replace'),
                                    'replace', keys=('Ctrl+R'), description=_('Replace current match'))
-        self.action_replace_next = sreg('replace-next', _('&Replace and find next'),
+        self.action_replace_next = sreg('replace-next', _('Replace and find ne&xt'),
                                         'replace-find', {'direction':'down'}, ('Ctrl+]'), _('Replace current match and find next'))
         self.action_replace_previous = sreg('replace-previous', _('R&eplace and find previous'),
                                         'replace-find', {'direction':'up'}, ('Ctrl+['), _('Replace current match and find previous'))
@@ -493,7 +492,7 @@ class Main(MainWindow):
 
         # Check Book actions
         group = _('Check book')
-        self.action_check_book = treg('debug.png', _('&Check book'), self.boss.check_requested, 'check-book', ('F7'), _('Check book for errors'))
+        self.action_check_book = treg('debug.png', _('C&heck book'), self.boss.check_requested, 'check-book', ('F7'), _('Check book for errors'))
         self.action_spell_check_book = treg('spell-check.png', _('Check &spelling'), self.boss.spell_check_requested, 'spell-check-book', ('Alt+F7'), _(
             'Check book for spelling errors'))
         self.action_check_book_next = reg('forward.png', _('&Next error'), partial(
@@ -512,7 +511,7 @@ class Main(MainWindow):
             'window-close.png', _('&Close current tab'), self.central.close_current_editor, 'close-current-tab', 'Ctrl+W', _(
                 'Close the currently open tab'))
         self.action_close_all_but_current_tab = reg(
-            'edit-clear.png', _('&Close other tabs'), self.central.close_all_but_current_editor, 'close-all-but-current-tab', 'Ctrl+Alt+W', _(
+            'edit-clear.png', _('C&lose other tabs'), self.central.close_all_but_current_editor, 'close-all-but-current-tab', 'Ctrl+Alt+W', _(
                 'Close all tabs except the current tab'))
         self.action_help = treg(
             'help.png', _('User &Manual'), lambda : open_url(QUrl(localize_user_manual_link(
diff --git a/src/calibre/gui2/tweak_book/widgets.py b/src/calibre/gui2/tweak_book/widgets.py
index 1135135fff..a0f302a4d2 100644
--- a/src/calibre/gui2/tweak_book/widgets.py
+++ b/src/calibre/gui2/tweak_book/widgets.py
@@ -24,7 +24,7 @@ from calibre.gui2 import error_dialog, choose_files, choose_save_file, info_dial
 from calibre.gui2.tweak_book import tprefs, current_container
 from calibre.gui2.widgets2 import Dialog as BaseDialog, HistoryComboBox, to_plain_text, PARAGRAPH_SEPARATOR
 from calibre.utils.icu import primary_sort_key, sort_key, primary_contains, numeric_sort_key
-from calibre.utils.matcher import get_char, Matcher
+from calibre.utils.matcher import get_char, Matcher, DEFAULT_LEVEL1, DEFAULT_LEVEL2, DEFAULT_LEVEL3
 from calibre.gui2.complete2 import EditWithComplete
 from polyglot.builtins import iteritems, unicode_type, zip, getcwd, filter as ignore_me
 
@@ -275,9 +275,13 @@ def make_highlighted_text(emph, text, positions):
     return text
 
 
+def emphasis_style():
+    pal = QApplication.instance().palette()
+    return 'color: {}; font-weight: bold'.format(pal.color(pal.Link).name())
+
+
 class Results(QWidget):
 
-    EMPH = "color:magenta; font-weight:bold"
     MARGIN = 4
 
     item_selected = pyqtSignal()
@@ -355,7 +359,7 @@ class Results(QWidget):
         self.update()
 
     def make_text(self, text, positions):
-        text = QStaticText(make_highlighted_text(self.EMPH, text, positions))
+        text = QStaticText(make_highlighted_text(emphasis_style(), text, positions))
         text.setTextOption(self.text_option)
         text.setTextFormat(Qt.RichText)
         return text
@@ -408,11 +412,12 @@ class Results(QWidget):
 
 class QuickOpen(Dialog):
 
-    def __init__(self, items, parent=None):
-        self.matcher = Matcher(items)
+    def __init__(self, items, parent=None, title=None, name='quick-open', level1=DEFAULT_LEVEL1, level2=DEFAULT_LEVEL2, level3=DEFAULT_LEVEL3, help_text=None):
+        self.matcher = Matcher(items, level1=level1, level2=level2, level3=level3)
         self.matches = ()
         self.selected_result = None
-        Dialog.__init__(self, _('Choose file to edit'), 'quick-open', parent=parent)
+        self.help_text = help_text or self.default_help_text()
+        Dialog.__init__(self, title or _('Choose file to edit'), name, parent=parent)
 
     def sizeHint(self):
         ans = Dialog.sizeHint(self)
@@ -420,25 +425,29 @@ class QuickOpen(Dialog):
         ans.setHeight(max(600, ans.height()))
         return ans
 
+    def default_help_text(self):
+        example = '
{0}i{1}mages/{0}c{1}hapter1/{0}s{1}cene{0}3{1}.jpg
'.format(
+            '' % emphasis_style(), '')
+        chars = '
ics3
' % emphasis_style()
+
+        return _('''
Quickly choose a file by typing in just a few characters from the file name into the field above.
+        For example, if want to choose the file:
+        {example}
+        Simply type in the characters:
+        {chars}
+        and press Enter.''').format(example=example, chars=chars)
+
     def setup_ui(self):
         self.l = l = QVBoxLayout(self)
         self.setLayout(l)
 
         self.text = t = QLineEdit(self)
         t.textEdited.connect(self.update_matches)
+        t.setClearButtonEnabled(True)
+        t.setPlaceholderText(_('Search'))
         l.addWidget(t, alignment=Qt.AlignTop)
 
-        example = '
{0}i{1}mages/{0}c{1}hapter1/{0}s{1}cene{0}3{1}.jpg
'.format(
-            '' % Results.EMPH, '')
-        chars = '
ics3
' % Results.EMPH
-
-        self.help_label = hl = QLabel(_(
-            '''
Quickly choose a file by typing in just a few characters from the file name into the field above.
-        For example, if want to choose the file:
-        {example}
-        Simply type in the characters:
-        {chars}
-        and press Enter.''').format(example=example, chars=chars))
+        self.help_label = hl = QLabel(self.help_text)
         hl.setContentsMargins(50, 50, 50, 50), hl.setAlignment(Qt.AlignTop | Qt.AlignHCenter)
         l.addWidget(hl)
         self.results = Results(self)
@@ -506,7 +515,7 @@ class NamesDelegate(QStyledItemDelegate):
             to.setWrapMode(to.NoWrap)
             to.setAlignment(Qt.AlignLeft | Qt.AlignVCenter)
             positions = sorted(set(positions) - {-1}, reverse=True)
-            text = '%s' % make_highlighted_text(Results.EMPH, text, positions)
+            text = '%s' % make_highlighted_text(emphasis_style(), text, positions)
             doc = QTextDocument()
             c = 'rgb(%d, %d, %d)'%c.getRgb()[:3]
             doc.setDefaultStyleSheet(' body { color: %s }'%c)
diff --git a/src/calibre/gui2/viewer/search.py b/src/calibre/gui2/viewer/search.py
index 1bca57b0e8..957c991846 100644
--- a/src/calibre/gui2/viewer/search.py
+++ b/src/calibre/gui2/viewer/search.py
@@ -10,9 +10,9 @@ from threading import Thread
 
 import regex
 from PyQt5.Qt import (
-    QCheckBox, QComboBox, QHBoxLayout, QIcon, QLabel, QListWidget, QListWidgetItem,
-    QStaticText, QStyle, QStyledItemDelegate, Qt, QToolButton, QVBoxLayout, QWidget,
-    pyqtSignal
+    QCheckBox, QComboBox, QHBoxLayout, QIcon, QLabel, QListWidget,
+    QListWidgetItem, QStaticText, QStyle, QStyledItemDelegate, Qt, QToolButton,
+    QVBoxLayout, QWidget, pyqtSignal
 )
 
 from calibre.ebooks.conversion.search_replace import REGEX_FLAGS
@@ -52,16 +52,21 @@ class BusySpinner(QWidget):  # {{{
 
 quote_map= {'"':'"“”', "'": "'‘’"}
 qpat = regex.compile(r'''(['"])''')
+spat = regex.compile(r'(\s+)')
 
 
 def text_to_regex(text):
     ans = []
-    for part in qpat.split(text):
-        r = quote_map.get(part)
-        if r is not None:
-            ans.append('[' + r + ']')
+    for wpart in spat.split(text):
+        if not wpart.strip():
+            ans.append(r'\s+')
         else:
-            ans.append(regex.escape(part))
+            for part in qpat.split(wpart):
+                r = quote_map.get(part)
+                if r is not None:
+                    ans.append('[' + r + ']')
+                else:
+                    ans.append(regex.escape(part))
     return ''.join(ans)
 
 
@@ -74,6 +79,8 @@ class Search(object):
         self._regex = None
 
     def __eq__(self, other):
+        if not isinstance(other, Search):
+            return False
         return self.text == other.text and self.mode == other.mode and self.case_sensitive == other.case_sensitive
 
     @property
@@ -94,6 +101,11 @@ class Search(object):
             self._regex = regex.compile(expr, flags)
         return self._regex
 
+    def __str__(self):
+        from collections import namedtuple
+        s = ('text', 'mode', 'case_sensitive', 'backwards')
+        return str(namedtuple('Search', s)(*tuple(getattr(self, x) for x in s)))
+
 
 class SearchFinished(object):
 
@@ -103,14 +115,16 @@ class SearchFinished(object):
 
 class SearchResult(object):
 
-    __slots__ = ('search_query', 'before', 'text', 'after', 'spine_idx', 'index', 'file_name', '_static_text')
+    __slots__ = ('search_query', 'before', 'text', 'after', 'q', 'spine_idx', 'index', 'file_name', '_static_text', 'is_hidden')
 
-    def __init__(self, search_query, before, text, after, name, spine_idx, index):
+    def __init__(self, search_query, before, text, after, q, name, spine_idx, index):
         self.search_query = search_query
+        self.q = q
         self.before, self.text, self.after = before, text, after
         self.spine_idx, self.index = spine_idx, index
         self.file_name = name
         self._static_text = None
+        self.is_hidden = False
 
     @property
     def static_text(self):
@@ -134,11 +148,16 @@ class SearchResult(object):
     def for_js(self):
         return {
             'file_name': self.file_name, 'spine_idx': self.spine_idx, 'index': self.index, 'text': self.text,
-            'before': self.before, 'after': self.after, 'mode': self.search_query.mode
+            'before': self.before, 'after': self.after, 'mode': self.search_query.mode, 'q': self.q
         }
 
-    def is_or_is_after(self, result_from_js):
-        return result_from_js['spine_idx'] == self.spine_idx and self.index >= result_from_js['index'] and result_from_js['text'] == self.text
+    def is_result(self, result_from_js):
+        return result_from_js['spine_idx'] == self.spine_idx and self.index == result_from_js['index'] and result_from_js['q'] == self.q
+
+    def __str__(self):
+        from collections import namedtuple
+        s = self.__slots__[:-1]
+        return str(namedtuple('SearchResult', s)(*tuple(getattr(self, x) for x in s)))
 
 
 @lru_cache(maxsize=None)
@@ -166,10 +185,7 @@ def searchable_text_for_name(name):
             stack.append(tail)
         if children:
             stack.extend(reversed(children))
-    # Normalize whitespace to a single space, this will cause failures
-    # when searching over spaces in pre nodes, but that is a lesser evil
-    # since the DOM converts \n, \t etc to a single space
-    return regex.sub(r'\s+', ' ', ''.join(ans))
+    return ''.join(ans)
 
 
 def search_in_name(name, search_query, ctx_size=50):
@@ -328,8 +344,14 @@ class ResultsDelegate(QStyledItemDelegate):  # {{{
         c = p.color(group, c)
         painter.setClipRect(option.rect)
         painter.setPen(c)
+        height = result.static_text.size().height()
+        tl = option.rect.topLeft()
+        x, y = tl.x(), tl.y()
+        y += (option.rect.height() - height) // 2
+        if result.is_hidden:
+            x += option.decorationSize.width() + 4
         try:
-            painter.drawStaticText(option.rect.topLeft(), result.static_text)
+            painter.drawStaticText(x, y, result.static_text)
         except Exception:
             import traceback
             traceback.print_exc()
@@ -344,21 +366,23 @@ class Results(QListWidget):  # {{{
     def __init__(self, parent=None):
         QListWidget.__init__(self, parent)
         self.setFocusPolicy(Qt.NoFocus)
-        self.setStyleSheet('QListWidget::item { padding: 3px; }')
         self.delegate = ResultsDelegate(self)
         self.setItemDelegate(self.delegate)
         self.itemClicked.connect(self.item_activated)
+        self.blank_icon = QIcon(I('blank.png'))
 
     def add_result(self, result):
         i = QListWidgetItem(' ', self)
         i.setData(Qt.UserRole, result)
+        i.setIcon(self.blank_icon)
         return self.count()
 
     def item_activated(self):
         i = self.currentItem()
         if i:
             sr = i.data(Qt.UserRole)
-            self.show_search_result.emit(sr)
+            if not sr.is_hidden:
+                self.show_search_result.emit(sr)
 
     def find_next(self, previous):
         if self.count() < 1:
@@ -370,26 +394,20 @@ class Results(QListWidget):  # {{{
         self.item_activated()
 
     def search_result_not_found(self, sr):
-        remove = []
         for i in range(self.count()):
             item = self.item(i)
             r = item.data(Qt.UserRole)
-            if r.is_or_is_after(sr):
-                remove.append(i)
-        if remove:
-            last_i = remove[-1]
-            if last_i < self.count() - 1:
-                self.setCurrentRow(last_i + 1)
-                self.item_activated()
-            elif remove[0] > 0:
-                self.setCurrentRow(remove[0] - 1)
-                self.item_activated()
-            for i in reversed(remove):
-                self.takeItem(i)
-            if self.count():
-                warning_dialog(self, _('Hidden text'), _(
-                    'Some search results were for hidden text, they have been removed.'), show=True)
+            if r.is_result(sr):
+                r.is_hidden = True
+                item.setIcon(QIcon(I('dialog_warning.png')))
+                break
 
+    @property
+    def current_result_is_hidden(self):
+        item = self.currentItem()
+        if item and item.data(Qt.UserRole) and item.data(Qt.UserRole).is_hidden:
+            return True
+        return False
 # }}}
 
 
@@ -401,6 +419,7 @@ class SearchPanel(QWidget):  # {{{
 
     def __init__(self, parent=None):
         QWidget.__init__(self, parent)
+        self.last_hidden_text_warning = None
         self.current_search = None
         self.l = l = QVBoxLayout(self)
         l.setContentsMargins(0, 0, 0, 0)
@@ -412,10 +431,19 @@ class SearchPanel(QWidget):  # {{{
         l.addWidget(si)
         self.results = r = Results(self)
         r.show_search_result.connect(self.do_show_search_result, type=Qt.QueuedConnection)
+        r.currentRowChanged.connect(self.update_hidden_message)
         l.addWidget(r, 100)
         self.spinner = s = BusySpinner(self)
         s.setVisible(False)
         l.addWidget(s)
+        self.hidden_message = la = QLabel(_('This text is hidden in the book and cannot be displayed'))
+        la.setStyleSheet('QLabel { margin-left: 1ex }')
+        la.setWordWrap(True)
+        la.setVisible(False)
+        l.addWidget(la)
+
+    def update_hidden_message(self):
+        self.hidden_message.setVisible(self.results.current_result_is_hidden)
 
     def focus_input(self):
         self.search_input.focus_input()
@@ -429,8 +457,10 @@ class SearchPanel(QWidget):  # {{{
             self.searcher.daemon = True
             self.searcher.start()
         self.results.clear()
+        self.hidden_message.setVisible(False)
         self.spinner.start()
         self.current_search = search_query
+        self.last_hidden_text_warning = None
         self.search_tasks.put((search_query, current_name))
 
     def run_searches(self):
@@ -457,8 +487,9 @@ class SearchPanel(QWidget):  # {{{
                 try:
                     for i, result in enumerate(search_in_name(name, search_query)):
                         before, text, after = result
-                        self.results_found.emit(SearchResult(search_query, before, text, after, name, spine_idx, counter[text]))
-                        counter[text] += 1
+                        q = (before or '')[-5:] + text + (after or '')[:5]
+                        self.results_found.emit(SearchResult(search_query, before, text, after, q, name, spine_idx, counter[q]))
+                        counter[q] += 1
                 except Exception:
                     import traceback
                     traceback.print_exc()
@@ -476,6 +507,7 @@ class SearchPanel(QWidget):  # {{{
             # first result
             self.results.setCurrentRow(0)
             self.results.item_activated()
+        self.update_hidden_message()
 
     def visibility_changed(self, visible):
         if visible:
@@ -483,6 +515,7 @@ class SearchPanel(QWidget):  # {{{
 
     def clear_searches(self):
         self.current_search = None
+        self.last_hidden_text_warning = None
         searchable_text_for_name.cache_clear()
         self.spinner.stop()
         self.results.clear()
@@ -491,6 +524,7 @@ class SearchPanel(QWidget):  # {{{
         self.search_tasks.put(None)
         self.spinner.stop()
         self.current_search = None
+        self.last_hidden_text_warning = None
         self.searcher = None
 
     def find_next_requested(self, previous):
@@ -501,11 +535,9 @@ class SearchPanel(QWidget):  # {{{
 
     def search_result_not_found(self, sr):
         self.results.search_result_not_found(sr)
-        if not self.results.count() and not self.spinner.is_running:
-            self.show_no_results_found()
+        self.update_hidden_message()
 
     def show_no_results_found(self):
-        if self.current_search:
-            warning_dialog(self, _('No matches found'), _(
-                'No matches were found for: {}').format(self.current_search.text), show=True)
+        msg = _('No matches were found for:')
+        warning_dialog(self, _('No matches found'), msg + '  {}'.format(self.current_search.text), show=True)
 # }}}
diff --git a/src/calibre/gui2/viewer/ui.py b/src/calibre/gui2/viewer/ui.py
index 119d1777b8..92fcf3b7bd 100644
--- a/src/calibre/gui2/viewer/ui.py
+++ b/src/calibre/gui2/viewer/ui.py
@@ -13,8 +13,8 @@ from hashlib import sha256
 from threading import Thread
 
 from PyQt5.Qt import (
-    QApplication, QDockWidget, QEvent, QMimeData, QModelIndex, QPixmap, QScrollBar,
-    Qt, QToolBar, QUrl, QVBoxLayout, QWidget, pyqtSignal
+    QApplication, QCursor, QDockWidget, QEvent, QMenu, QMimeData, QModelIndex,
+    QPixmap, Qt, QTimer, QToolBar, QUrl, QVBoxLayout, QWidget, pyqtSignal
 )
 
 from calibre import prints
@@ -78,13 +78,6 @@ def path_key(path):
     return sha256(as_bytes(path)).hexdigest()
 
 
-class ScrollBar(QScrollBar):
-
-    def paintEvent(self, ev):
-        if self.isEnabled():
-            return QScrollBar.paintEvent(self, ev)
-
-
 class EbookViewer(MainWindow):
 
     msg_from_anotherinstance = pyqtSignal(object)
@@ -94,11 +87,14 @@ class EbookViewer(MainWindow):
 
     def __init__(self, open_at=None, continue_reading=None, force_reload=False):
         MainWindow.__init__(self, None)
-        self.shutting_down = False
+        self.shutting_down = self.close_forced = False
         self.force_reload = force_reload
         connect_lambda(self.book_preparation_started, self, lambda self: self.loading_overlay(_(
             'Preparing book for first read, please wait')), type=Qt.QueuedConnection)
         self.maximized_at_last_fullscreen = False
+        self.save_pos_timer = t = QTimer(self)
+        t.setSingleShot(True), t.setInterval(3000), t.setTimerType(Qt.VeryCoarseTimer)
+        connect_lambda(t.timeout, self, lambda self: self.save_annotations(in_book_file=False))
         self.pending_open_at = open_at
         self.base_window_title = _('E-book viewer')
         self.setWindowTitle(self.base_window_title)
@@ -176,7 +172,10 @@ class EbookViewer(MainWindow):
         self.web_view.show_error.connect(self.show_error)
         self.web_view.print_book.connect(self.print_book, type=Qt.QueuedConnection)
         self.web_view.reset_interface.connect(self.reset_interface, type=Qt.QueuedConnection)
+        self.web_view.quit.connect(self.quit, type=Qt.QueuedConnection)
         self.web_view.shortcuts_changed.connect(self.shortcuts_changed)
+        self.web_view.scrollbar_context_menu.connect(self.scrollbar_context_menu)
+        self.web_view.close_prep_finished.connect(self.close_prep_finished)
         self.actions_toolbar.initialize(self.web_view, self.search_dock.toggleViewAction())
         self.setCentralWidget(self.web_view)
         self.loading_overlay = LoadingOverlay(self)
@@ -192,14 +191,40 @@ class EbookViewer(MainWindow):
             rmap[v].append(k)
         self.actions_toolbar.set_tooltips(rmap)
 
-    def toggle_inspector(self):
-        visible = self.inspector_dock.toggleViewAction().isChecked()
-        self.inspector_dock.setVisible(not visible)
-
     def resizeEvent(self, ev):
         self.loading_overlay.resize(self.size())
         return MainWindow.resizeEvent(self, ev)
 
+    def scrollbar_context_menu(self, x, y, frac):
+        m = QMenu(self)
+        amap = {}
+
+        def a(text, name):
+            m.addAction(text)
+            amap[text] = name
+
+        a(_('Scroll here'), 'here')
+        m.addSeparator()
+        a(_('Start of book'), 'start_of_book')
+        a(_('End of book'), 'end_of_book')
+        m.addSeparator()
+        a(_('Previous section'), 'previous_section')
+        a(_('Next section'), 'next_section')
+        m.addSeparator()
+        a(_('Start of current file'), 'start_of_file')
+        a(_('End of current file'), 'end_of_file')
+        m.addSeparator()
+        a(_('Hide this scrollbar'), 'toggle_scrollbar')
+
+        q = m.exec_(QCursor.pos())
+        if not q:
+            return
+        q = amap[q.text()]
+        if q == 'here':
+            self.web_view.goto_frac(frac)
+        else:
+            self.web_view.trigger_shortcut(q)
+
     # IPC {{{
     def handle_commandline_arg(self, arg):
         if arg:
@@ -245,6 +270,10 @@ class EbookViewer(MainWindow):
 
     # Docks (ToC, Bookmarks, Lookup, etc.) {{{
 
+    def toggle_inspector(self):
+        visible = self.inspector_dock.toggleViewAction().isChecked()
+        self.inspector_dock.setVisible(not visible)
+
     def toggle_toc(self):
         self.toc_dock.setVisible(not self.toc_dock.isVisible())
 
@@ -508,6 +537,7 @@ class EbookViewer(MainWindow):
             return
         self.current_book_data['annotations_map']['last-read'] = [{
             'pos': cfi, 'pos_type': 'epubcfi', 'timestamp': utcnow()}]
+        self.save_pos_timer.start()
     # }}}
 
     # State serialization {{{
@@ -536,6 +566,8 @@ class EbookViewer(MainWindow):
         geom = vprefs['main_window_geometry']
         if geom and get_session_pref('remember_window_geometry', default=False):
             QApplication.instance().safe_restore_geometry(self, geom)
+        else:
+            QApplication.instance().ensure_window_on_screen(self)
         if state:
             self.restoreState(state, self.MAIN_WINDOW_STATE_VERSION)
             self.inspector_dock.setVisible(False)
@@ -543,7 +575,24 @@ class EbookViewer(MainWindow):
     def quit(self):
         self.close()
 
+    def force_close(self):
+        if not self.close_forced:
+            self.close_forced = True
+            self.quit()
+
+    def close_prep_finished(self, cfi):
+        if cfi:
+            self.cfi_changed(cfi)
+        self.force_close()
+
     def closeEvent(self, ev):
+        if self.current_book_data and self.web_view.view_is_ready and not self.close_forced:
+            ev.ignore()
+            if not self.shutting_down:
+                self.shutting_down = True
+                QTimer.singleShot(2000, self.force_close)
+                self.web_view.prepare_for_close()
+            return
         self.shutting_down = True
         self.search_widget.shutdown()
         try:
diff --git a/src/calibre/gui2/viewer/web_view.py b/src/calibre/gui2/viewer/web_view.py
index 6dd4923ee8..e5830a4159 100644
--- a/src/calibre/gui2/viewer/web_view.py
+++ b/src/calibre/gui2/viewer/web_view.py
@@ -34,7 +34,7 @@ from calibre.srv.code import get_translations_data
 from calibre.utils.config import JSONConfig
 from calibre.utils.serialize import json_loads
 from calibre.utils.shared_file import share_open
-from polyglot.builtins import as_bytes, iteritems
+from polyglot.builtins import as_bytes, iteritems, unicode_type
 
 try:
     from PyQt5 import sip
@@ -270,7 +270,10 @@ class ViewerBridge(Bridge):
     print_book = from_js()
     clear_history = from_js()
     reset_interface = from_js()
+    quit = from_js()
     customize_toolbar = from_js()
+    scrollbar_context_menu = from_js(object, object, object)
+    close_prep_finished = from_js(object)
 
     create_view = to_js()
     start_book_load = to_js()
@@ -284,6 +287,8 @@ class ViewerBridge(Bridge):
     trigger_shortcut = to_js()
     set_system_palette = to_js()
     show_search_result = to_js()
+    prepare_for_close = to_js()
+    viewer_font_size_changed = to_js()
 
 
 def apply_font_settings(page_or_view):
@@ -305,6 +310,8 @@ def apply_font_settings(page_or_view):
     sf = fs.get('standard_font') or 'serif'
     sf = getattr(s, {'serif': 'SerifFont', 'sans': 'SansSerifFont', 'mono': 'FixedFont'}[sf])
     s.setFontFamily(s.StandardFont, s.fontFamily(sf))
+    old_minimum = s.fontSize(s.MinimumFontSize)
+    old_base = s.fontSize(s.DefaultFontSize)
     mfs = fs.get('minimum_font_size')
     if mfs is None:
         s.resetFontSize(s.MinimumFontSize)
@@ -314,6 +321,10 @@ def apply_font_settings(page_or_view):
     if bfs is not None:
         s.setFontSize(s.DefaultFontSize, bfs)
 
+    font_size_changed = old_minimum != s.fontSize(s.MinimumFontSize) or old_base != s.fontSize(s.DefaultFontSize)
+    if font_size_changed and hasattr(page_or_view, 'execute_when_ready'):
+        page_or_view.execute_when_ready('viewer_font_size_changed')
+
     return s
 
 
@@ -440,7 +451,10 @@ class WebView(RestartingWebEngineView):
     show_error = pyqtSignal(object, object, object)
     print_book = pyqtSignal()
     reset_interface = pyqtSignal()
+    quit = pyqtSignal()
     customize_toolbar = pyqtSignal()
+    scrollbar_context_menu = pyqtSignal(object, object, object)
+    close_prep_finished = pyqtSignal(object)
     shortcuts_changed = pyqtSignal(object)
     paged_mode_changed = pyqtSignal()
     standalone_misc_settings_changed = pyqtSignal(object)
@@ -459,6 +473,7 @@ class WebView(RestartingWebEngineView):
         self.show_home_page_on_ready = True
         self._size_hint = QSize(int(w/3), int(w/2))
         self._page = WebPage(self)
+        self.view_is_ready = False
         self.bridge.bridge_ready.connect(self.on_bridge_ready)
         self.bridge.view_created.connect(self.on_view_created)
         self.bridge.content_file_changed.connect(self.on_content_file_changed)
@@ -487,7 +502,10 @@ class WebView(RestartingWebEngineView):
         self.bridge.print_book.connect(self.print_book)
         self.bridge.clear_history.connect(self.clear_history)
         self.bridge.reset_interface.connect(self.reset_interface)
+        self.bridge.quit.connect(self.quit)
         self.bridge.customize_toolbar.connect(self.customize_toolbar)
+        self.bridge.scrollbar_context_menu.connect(self.scrollbar_context_menu)
+        self.bridge.close_prep_finished.connect(self.close_prep_finished)
         self.bridge.export_shortcut_map.connect(self.set_shortcut_map)
         self.shortcut_map = {}
         self.bridge.report_cfi.connect(self.call_callback)
@@ -572,6 +590,7 @@ class WebView(RestartingWebEngineView):
 
     def on_view_created(self, data):
         self.view_created.emit(data)
+        self.view_is_ready = True
 
     def on_content_file_changed(self, data):
         self.current_content_file = data
@@ -598,7 +617,7 @@ class WebView(RestartingWebEngineView):
     def set_session_data(self, key, val):
         if key == '*' and val is None:
             vprefs['session_data'] = {}
-            apply_font_settings(self._page)
+            apply_font_settings(self)
             self.paged_mode_changed.emit()
             self.standalone_misc_settings_changed.emit()
         elif key != '*':
@@ -606,7 +625,7 @@ class WebView(RestartingWebEngineView):
             sd[key] = val
             vprefs['session_data'] = sd
             if key in ('standalone_font_settings', 'base_font_size'):
-                apply_font_settings(self._page)
+                apply_font_settings(self)
             elif key == 'read_mode':
                 self.paged_mode_changed.emit()
             elif key == 'standalone_misc_settings':
@@ -621,7 +640,7 @@ class WebView(RestartingWebEngineView):
             vprefs['local_storage'] = sd
 
     def do_callback(self, func_name, callback):
-        cid = next(self.callback_id_counter)
+        cid = unicode_type(next(self.callback_id_counter))
         self.callback_map[cid] = callback
         self.execute_when_ready('get_current_cfi', cid)
 
@@ -663,3 +682,6 @@ class WebView(RestartingWebEngineView):
 
     def palette_changed(self):
         self.execute_when_ready('set_system_palette', system_colors())
+
+    def prepare_for_close(self):
+        self.execute_when_ready('prepare_for_close')
diff --git a/src/calibre/gui2/widgets.py b/src/calibre/gui2/widgets.py
index f2c55472c4..5ef65a5d1c 100644
--- a/src/calibre/gui2/widgets.py
+++ b/src/calibre/gui2/widgets.py
@@ -23,7 +23,7 @@ from calibre.ebooks import BOOK_EXTENSIONS
 from calibre.utils.config import prefs, XMLConfig
 from calibre.gui2.progress_indicator import ProgressIndicator as _ProgressIndicator
 from calibre.gui2.dnd import (dnd_has_image, dnd_get_image, dnd_get_files,
-    image_extensions, dnd_has_extension, DownloadDialog)
+    image_extensions, dnd_has_extension, dnd_get_local_image_and_pixmap, DownloadDialog)
 from calibre.utils.localization import localize_user_manual_link
 from polyglot.builtins import native_string_type, unicode_type, range
 
@@ -238,6 +238,10 @@ class ImageDropMixin(object):  # {{{
     def dropEvent(self, event):
         event.setDropAction(Qt.CopyAction)
         md = event.mimeData()
+        pmap, data = dnd_get_local_image_and_pixmap(md)
+        if pmap is not None:
+            self.handle_image_drop(pmap, data)
+            return
 
         x, y = dnd_get_image(md)
         if x is not None:
@@ -520,7 +524,7 @@ class EnLineEdit(LineEditECM, QLineEdit):  # {{{
     def event(self, ev):
         # See https://bugreports.qt.io/browse/QTBUG-46911
         if ev.type() == ev.ShortcutOverride and (
-                ev.key() in (Qt.Key_Left, Qt.Key_Right) and (ev.modifiers() & ~Qt.KeypadModifier) == Qt.ControlModifier):
+                hasattr(ev, 'key') and ev.key() in (Qt.Key_Left, Qt.Key_Right) and (ev.modifiers() & ~Qt.KeypadModifier) == Qt.ControlModifier):
             ev.accept()
         return QLineEdit.event(self, ev)
 
diff --git a/src/calibre/gui2/wizard/library.ui b/src/calibre/gui2/wizard/library.ui
index 74fccf0dd2..55b037320d 100644
--- a/src/calibre/gui2/wizard/library.ui
+++ b/src/calibre/gui2/wizard/library.ui
@@ -6,7 +6,7 @@
    
     0
     0
-    481
+    614
     300
    
   
@@ -19,18 +19,38 @@
   
    The one stop solution to all your e-book needs.
   
-  
-   
-    
-     
-      Choose your &language:
-     
-     
-      language
-     
-    
+  
+   
+    
+     
+      
+       
+        Choose your &language:
+       
+       
+        language
+       
+      
+     
+     
+      
+     
+     
+      
+       
+        Qt::Horizontal
+       
+       
+        
+         40
+         20
+        
+       
+      
+     
+    
    
-   
+   
     
      
       Qt::Vertical
@@ -43,31 +63,45 @@
      
     
    
-   
-    
+   
+    
      
-      &Change
-     
-    
-   
-   
-    
-     
-      If a calibre library already exists at the new location, calibre will use it automatically.
+      <p>Choose a location for your books. When you add books to calibre, they will be copied here. Use an <b>empty folder</b> for a new calibre library:
      
      
       true
      
     
    
-   
-    
-     
+   
+    
+     
+      
+       
+        true
+       
+      
+     
+     
+      
+       
+        &Change
+       
+      
+     
+    
+   
+   
+    
+     
+      If a calibre library already exists at the newly selected location, calibre will use it automatically.
+     
+     
       true
      
     
    
-   
+   
     
      
       Qt::Vertical
@@ -80,20 +114,7 @@
      
     
    
-   
-    
-   
-   
-    
-     
-      <p>Choose a location for your books. When you add books to calibre, they will be copied here. Use an <b>empty folder</b> for a new calibre library:
-     
-     
-      true
-     
-    
-   
-   
+   
     
      
       
diff --git a/src/calibre/utils/cleantext.py b/src/calibre/utils/cleantext.py
index 2d62e06858..d7c9d74c20 100644
--- a/src/calibre/utils/cleantext.py
+++ b/src/calibre/utils/cleantext.py
@@ -8,15 +8,13 @@ from polyglot.builtins import codepoint_to_chr, map, range, filter
 from polyglot.html_entities import name2codepoint
 from calibre.constants import plugins, preferred_encoding
 
-try:
-    _ncxc = plugins['speedup'][0].clean_xml_chars
-except AttributeError:
-    native_clean_xml_chars = None
-else:
-    def native_clean_xml_chars(x):
-        if isinstance(x, bytes):
-            x = x.decode(preferred_encoding)
-        return _ncxc(x)
+_ncxc = plugins['speedup'][0].clean_xml_chars
+
+
+def native_clean_xml_chars(x):
+    if isinstance(x, bytes):
+        x = x.decode(preferred_encoding)
+    return _ncxc(x)
 
 
 def ascii_pat(for_binary=False):
diff --git a/src/calibre/utils/fonts/sfnt/merge.py b/src/calibre/utils/fonts/sfnt/merge.py
index 1da10537fd..d86dc61cf0 100644
--- a/src/calibre/utils/fonts/sfnt/merge.py
+++ b/src/calibre/utils/fonts/sfnt/merge.py
@@ -12,7 +12,7 @@ class GlyphSizeMismatch(ValueError):
     pass
 
 
-def merge_truetype_fonts_for_pdf(*fonts):
+def merge_truetype_fonts_for_pdf(fonts, log=None):
     # only merges the glyf and loca tables, ignoring all other tables
     all_glyphs = {}
     ans = fonts[0]
@@ -28,7 +28,8 @@ def merge_truetype_fonts_for_pdf(*fonts):
                     all_glyphs[glyph_id] = glyf.glyph_data(offset, sz, as_raw=True)
                 else:
                     if abs(sz - len(prev_glyph_data)) > 8:
-                        raise GlyphSizeMismatch('Size mismatch for glyph id: {} prev_sz: {} sz: {}'.format(glyph_id, len(prev_glyph_data), sz))
+                        if log is not None:
+                            log('Size mismatch for glyph id: {} prev_sz: {} sz: {}'.format(glyph_id, len(prev_glyph_data), sz))
 
     glyf = ans[b'glyf']
     head = ans[b'head']
diff --git a/src/calibre/utils/img.py b/src/calibre/utils/img.py
index bf10b2b7af..e6f14e3a62 100644
--- a/src/calibre/utils/img.py
+++ b/src/calibre/utils/img.py
@@ -67,6 +67,46 @@ def load_jxr_data(data):
 
 # }}}
 
+# png <-> gif {{{
+
+
+def png_data_to_gif_data(data):
+    from PIL import Image
+    img = Image.open(BytesIO(data))
+    buf = BytesIO()
+    if img.mode in ('p', 'P'):
+        transparency = img.info.get('transparency')
+        if transparency is not None:
+            img.save(buf, 'gif', transparency=transparency)
+        else:
+            img.save(buf, 'gif')
+    elif img.mode in ('rgba', 'RGBA'):
+        alpha = img.split()[3]
+        mask = Image.eval(alpha, lambda a: 255 if a <=128 else 0)
+        img = img.convert('RGB').convert('P', palette=Image.ADAPTIVE, colors=255)
+        img.paste(255, mask)
+        img.save(buf, 'gif', transparency=255)
+    else:
+        img = img.convert('P', palette=Image.ADAPTIVE)
+        img.save(buf, 'gif')
+    return buf.getvalue()
+
+
+class AnimatedGIF(ValueError):
+    pass
+
+
+def gif_data_to_png_data(data, discard_animation=False):
+    from PIL import Image
+    img = Image.open(BytesIO(data))
+    if img.is_animated and not discard_animation:
+        raise AnimatedGIF()
+    buf = BytesIO()
+    img.save(buf, 'png')
+    return buf.getvalue()
+
+# }}}
+
 # Loading images {{{
 
 
@@ -140,11 +180,7 @@ def image_to_data(img, compression_quality=95, fmt='JPEG', png_compression_level
         w.setQuality(90)
         if not w.write(img):
             raise ValueError('Failed to export image as ' + fmt + ' with error: ' + w.errorString())
-        from PIL import Image
-        im = Image.open(BytesIO(ba.data()))
-        buf = BytesIO()
-        im.save(buf, 'gif')
-        return buf.getvalue()
+        return png_data_to_gif_data(ba.data())
     is_jpeg = fmt in ('JPG', 'JPEG')
     w = QImageWriter(buf, fmt.encode('ascii'))
     if is_jpeg:
diff --git a/src/calibre/utils/speedup.c b/src/calibre/utils/speedup.c
index ca990150d1..30b82638c7 100644
--- a/src/calibre/utils/speedup.c
+++ b/src/calibre/utils/speedup.c
@@ -394,8 +394,11 @@ clean_xml_chars(PyObject *self, PyObject *text) {
         // based on https://en.wikipedia.org/wiki/Valid_characters_in_XML#Non-restricted_characters
         // python 3.3+ unicode strings never contain surrogate pairs, since if
         // they did, they would be represented as UTF-32
-        if ((0x20 <= ch && ch <= 0xd7ff && ch != 0x7f) ||
-                ch == 9 || ch == 10 || ch == 13 ||
+        if ((0x20 <= ch && ch <= 0x7e) ||
+                ch == 0x9 || ch == 0xa || ch == 0xd || ch == 0x85 ||
+				(0x00A0 <= ch && ch <= 0xD7FF) ||
+				(0xE000 <= ch && ch <= 0xFDCF) ||
+				(0xFDF0 <= ch && ch <= 0xFFFD) ||
                 (0xffff < ch && ch <= 0x10ffff)) {
             PyUnicode_WRITE(text_kind, result_text, target_i, ch);
             target_i += 1;
diff --git a/src/pyj/book_list/local_books.pyj b/src/pyj/book_list/local_books.pyj
index c35dc0d852..9b36bc08a3 100644
--- a/src/pyj/book_list/local_books.pyj
+++ b/src/pyj/book_list/local_books.pyj
@@ -3,16 +3,16 @@
 from __python__ import bound_methods, hash_literals
 
 from elementmaker import E
-from gettext import gettext as _
+from gettext import gettext as _, ngettext
 
 from book_list.globals import get_db
 from book_list.router import home, open_book
-from book_list.top_bar import create_top_bar
+from book_list.top_bar import add_button, create_top_bar
 from book_list.ui import set_panel_handler
-from book_list.views import DEFAULT_MODE, setup_view_mode, get_view_mode
+from book_list.views import DEFAULT_MODE, get_view_mode, setup_view_mode
 from dom import clear, ensure_id
 from modals import create_custom_dialog, error_dialog
-from utils import conditional_timeout
+from utils import conditional_timeout, safe_set_inner_html
 from widgets import create_button
 
 CLASS_NAME = 'local-books-list'
@@ -49,6 +49,68 @@ def delete_book(book, book_idx):
     )
 
 
+def confirm_delete_all():
+    num_of_books = book_list_data.books?.length
+    if not num_of_books:
+        return
+    create_custom_dialog(_('Are you sure?'), def(parent, close_modal):
+
+        def action(doit):
+            if doit:
+                clear(parent)
+                delete_all(parent, close_modal)
+            else:
+                close_modal()
+
+        msg = ngettext(
+            'This will remove the downloaded book from local storage. Are you sure?',
+            'This will remove all {} downloaded books from local storage. Are you sure?',
+            num_of_books).format(num_of_books)
+        m = E.div()
+        safe_set_inner_html(m, msg)
+        parent.appendChild(E.div(
+            m,
+            E.div(class_='button-box',
+                create_button(_('OK'), None, action.bind(None, True)),
+                '\xa0',
+                create_button(_('Cancel'), None, action.bind(None, False), highlight=True),
+            )
+        ))
+    )
+
+
+
+def delete_all(msg_parent, close_modal):
+    db = get_db()
+    books = list(book_list_data.books)
+
+    def refresh():
+        show_recent_stage2.call(book_list_data.container_id, [book_list_data.book_data[i] for i in books])
+
+    def delete_one():
+        if not books.length:
+            close_modal()
+            refresh()
+            return
+        clear(msg_parent)
+        safe_set_inner_html(msg_parent, ngettext(
+            'Deleting one book, please wait...',
+            'Deleting {} books, please wait...',
+            books.length or 0).format(books.length)
+        )
+        book_to_delete = books.pop()
+        db.delete_book(book_list_data.book_data[book_to_delete], def(book, err_string):
+            if err_string:
+                close_modal()
+                refresh()
+                error_dialog(_('Failed to delete book'), err_string)
+            else:
+                delete_one()
+    )
+    delete_one()
+
+
+
 def on_select(book, book_idx):
     title = this
 
@@ -121,27 +183,29 @@ def apply_view_mode(mode):
 
 
 def create_books_list(container, books):
+    clear(container)
     book_list_data.container_id = ensure_id(container)
     book_list_data.book_data = {i:book for i, book in enumerate(books)}
     book_list_data.books = list(range(books.length))
     book_list_data.mode = None
     book_list_data.thumbnail_cache = {}
-    container.appendChild(E.div(data_component='book_list'))
-    apply_view_mode(get_view_mode())
-
-def show_recent_stage2(books):
-    container = document.getElementById(this)
-    if not container:
-        return
-    clear(container)
     if not books.length:
         container.appendChild(E.div(
             style='margin: 1rem 1rem',
             _('No downloaded books present')
         ))
+    else:
+        container.appendChild(E.div(data_component='book_list'))
+        apply_view_mode(get_view_mode())
+
+
+def show_recent_stage2(books):
+    container = document.getElementById(this)
+    if not container:
         return
     create_books_list(container, books)
 
+
 def show_recent():
     container = this
     db = get_db()
@@ -158,6 +222,7 @@ def show_recent():
 def init(container_id):
     container = document.getElementById(container_id)
     create_top_bar(container, title=_('Downloaded books'), action=home, icon='home')
+    add_button(container, 'trash', confirm_delete_all, _('Delete all downloaded books'))
     # book list
     recent = E.div(class_=CLASS_NAME)
     recent_container_id = ensure_id(recent)
diff --git a/src/pyj/book_list/prefs.pyj b/src/pyj/book_list/prefs.pyj
index 750dfb5425..aa8c9d8989 100644
--- a/src/pyj/book_list/prefs.pyj
+++ b/src/pyj/book_list/prefs.pyj
@@ -214,7 +214,7 @@ def create_prefs_widget(container, prefs_data):
     if state.widgets.length:
         container.appendChild(
             E.div(
-                style='margin:1ex 1em; padding: 1em; text-align:center',
+                style='margin:1rem;',
                 create_button(_('Restore default settings'), 'refresh', reset_to_defaults)
             )
         )
@@ -232,8 +232,10 @@ def prefs_panel_handler(title, get_prefs_data, on_close=None, icon='close'):
             on_close()
         back()
 
-    return def init_prefs_panel(container_id):  # noqa:unused-local
+    def init_prefs_panel(container_id):
         container = document.getElementById(container_id)
-        create_top_bar(container, title, action=close_action, icon=icon)
+        create_top_bar(container, title=title, action=close_action, icon=icon)
         container.appendChild(E.div())
         create_prefs_widget(container.lastChild, get_prefs_data())
+
+    return init_prefs_panel
diff --git a/src/pyj/book_list/search.pyj b/src/pyj/book_list/search.pyj
index 250c015c51..071d9bc04f 100644
--- a/src/pyj/book_list/search.pyj
+++ b/src/pyj/book_list/search.pyj
@@ -466,7 +466,7 @@ def get_prefs():
 
         {
             'name':'partition_method',
-            'text':_('Tag browser category partitioning method'),
+            'text':_('Category partitioning method'),
             'choices':[('first letter', _('First Letter')), ('disable', _('Disable')), ('partition', _('Partition'))],
             'tooltip':_('Choose how Tag browser subcategories are displayed when'
                         ' there are more items than the limit. Select by first'
diff --git a/src/pyj/read_book/find.pyj b/src/pyj/read_book/find.pyj
new file mode 100644
index 0000000000..03ea74f69d
--- /dev/null
+++ b/src/pyj/read_book/find.pyj
@@ -0,0 +1,128 @@
+# vim:fileencoding=utf-8
+# License: GPL v3 Copyright: 2020, Kovid Goyal 
+from __python__ import bound_methods, hash_literals
+
+
+def build_text_map():
+    node_list = v'[]'
+    flat_text = ''
+    ignored_tags = {
+        'style': True, 'script': True, 'noscript': True, 'title': True, 'meta': True, 'head': True, 'link': True, 'html': True,
+        'img': True
+    }
+
+    def process_node(node):
+        nonlocal flat_text
+        if node.nodeType is Node.TEXT_NODE:
+            text = node.nodeValue
+            if text and text.length:
+                node_list.push({'node': node, 'offset': flat_text.length, 'length': text.length})
+                flat_text += text
+        elif node.nodeType is Node.ELEMENT_NODE:
+            if not node.hasChildNodes():
+                return
+            tag = node.tagName.toLowerCase()
+            if ignored_tags[tag]:
+                return
+            style = window.getComputedStyle(node)
+            if style.display is 'none' or style.visibility is 'hidden':
+                return
+            children = node.childNodes
+            for i in range(children.length):
+                process_node(v'children[i]')
+
+    process_node(document.body)
+    return {'timestamp': window.performance.now(), 'flat_text': flat_text, 'node_list': node_list}
+
+
+def find_node_for_index_binary(node_list, idx_in_flat_text, start):
+    # Do a binary search for idx
+    start = start or 0
+    end = node_list.length - 1
+    while start <= end:
+        mid = Math.floor((start + end)/2)
+        q = node_list[mid]
+        limit = q.offset + q.length
+        if q.offset <= idx_in_flat_text and limit > idx_in_flat_text:
+            start_node = q.node
+            start_offset = idx_in_flat_text - q.offset
+            return start_node, start_offset, mid
+        if limit <= idx_in_flat_text:
+            start = mid + 1
+        else:
+            end = mid - 1
+    return None, None, None
+
+
+def find_node_for_index_linear(node_list, idx_in_flat_text, start):
+    start = start or 0
+    for i in range(start, node_list.length):
+        q = node_list[i]
+        limit = q.offset + q.length
+        if q.offset <= idx_in_flat_text and limit > idx_in_flat_text:
+            start_node = q.node
+            start_offset = idx_in_flat_text - q.offset
+            return start_node, start_offset, i
+    return None, None, None
+
+
+def find_specific_occurrence(q, num, before_len, after_len, text_map):
+    if not q or not q.length:
+        return
+    from_idx = 0
+    flat_text = text_map.flat_text
+    pos = 0
+    match_num = -1
+    while True:
+        idx = flat_text.indexOf(q, from_idx)
+        if idx < 0:
+            break
+        match_num += 1
+        from_idx = idx + 1
+        if num < match_num:
+            continue
+        start_node, start_offset, node_pos = find_node_for_index_binary(text_map.node_list, idx + before_len, pos)
+        if start_node is not None:
+            pos = node_pos
+            end_node, end_offset, node_pos = find_node_for_index_linear(text_map.node_list, idx + q.length - after_len, pos)
+            if end_node is not None:
+                return {
+                    'start_node': start_node, 'start_offset': start_offset, 'start_pos': pos,
+                    'end_node': end_node, 'end_offset': end_offset, 'end_pos': node_pos,
+                    'idx_in_flat_text': idx
+                }
+        break
+
+
+cache = {}
+
+
+def reset_find_caches():
+    nonlocal cache
+    cache = {}
+
+
+def select_find_result(match):
+    sel = window.getSelection()
+    sel.setBaseAndExtent(match.start_node, match.start_offset, match.end_node, match.end_offset)
+
+
+def select_search_result(sr):
+    window.getSelection().removeAllRanges()
+    if not cache.text_map:
+        cache.text_map = build_text_map()
+    q = ''
+    before_len = after_len = 0
+    if sr.before:
+        q = sr.before[-5:]
+        before_len = q.length
+    q += sr.text
+    if sr.after:
+        after = sr.after[:5]
+        after_len = after.length
+        q += after
+    match = find_specific_occurrence(q, int(sr.index), before_len, after_len, cache.text_map)
+    if not match:
+        return False
+    select_find_result(match)
+    return True
diff --git a/src/pyj/read_book/flow_mode.pyj b/src/pyj/read_book/flow_mode.pyj
index 4b810a2d8c..80bc81d235 100644
--- a/src/pyj/read_book/flow_mode.pyj
+++ b/src/pyj/read_book/flow_mode.pyj
@@ -142,7 +142,7 @@ def scroll_by_page(direction):
 
 
 def is_auto_scroll_active():
-    return scroll_animator.auto and scroll_animator.is_running()
+    return scroll_animator.is_active()
 
 
 def start_autoscroll():
@@ -150,9 +150,9 @@ def start_autoscroll():
 
 
 def toggle_autoscroll():
-    running = False
     if is_auto_scroll_active():
         cancel_scroll()
+        running = False
     else:
         start_autoscroll()
         running = True
@@ -236,9 +236,14 @@ class ScrollAnimator:
     def __init__(self):
         self.animation_id = None
         self.auto = False
+        self.auto_timer = None
+        self.paused = False
 
     def is_running(self):
-        return self.animation_id is not None
+        return self.animation_id is not None or self.auto_timer is not None
+
+    def is_active(self):
+        return self.is_running() and (self.auto or self.auto_timer is not None)
 
     def start(self, direction, auto):
         if self.wait:
@@ -246,7 +251,7 @@ class ScrollAnimator:
 
         now = window.performance.now()
         self.end_time = now + self.DURATION
-        clearTimeout(self.auto_timer)
+        self.stop_auto_spine_transition()
 
         if not self.is_running() or direction is not self.direction or auto is not self.auto:
             if self.auto and not auto:
@@ -298,12 +303,14 @@ class ScrollAnimator:
         if scroll_finished:
             self.pause()
             if opts.scroll_auto_boundary_delay >= 0:
-                self.auto_timer = setTimeout(def():
-                        get_boss().send_message('next_spine_item', previous=self.direction is DIRECTION.Up)
-                    , opts.scroll_auto_boundary_delay * 1000)
+                self.auto_timer = setTimeout(self.request_next_spine_item, opts.scroll_auto_boundary_delay * 1000)
         else:
             self.animation_id = window.requestAnimationFrame(self.auto_scroll)
 
+    def request_next_spine_item(self):
+        self.auto_timer = None
+        get_boss().send_message('next_spine_item', previous=self.direction is DIRECTION.Up)
+
     def report(self):
         amt = window.pageYOffset - self.start_offset
         if abs(amt) > 0 and self.csi_idx is current_spine_item().index:
@@ -324,6 +331,13 @@ class ScrollAnimator:
             window.cancelAnimationFrame(self.animation_id)
             self.animation_id = None
             self.report()
+        self.stop_auto_spine_transition()
+
+    def stop_auto_spine_transition(self):
+        if self.auto_timer is not None:
+            clearTimeout(self.auto_timer)
+            self.auto_timer = None
+            self.paused = False
 
     def pause(self):
         if self.auto:
@@ -451,3 +465,15 @@ def auto_scroll_action(action):
     elif action is 'resume':
         auto_scroll_resume()
     return is_auto_scroll_active()
+
+
+def ensure_selection_visible():
+    s = window.getSelection()
+    if not s.anchorNode:
+        return
+    p = s.anchorNode
+    while p:
+        if p.scrollIntoView:
+            p.scrollIntoView()
+            return
+        p = p.parentNode
diff --git a/src/pyj/read_book/footnotes.pyj b/src/pyj/read_book/footnotes.pyj
index 459708b6db..e72a7fb758 100644
--- a/src/pyj/read_book/footnotes.pyj
+++ b/src/pyj/read_book/footnotes.pyj
@@ -14,6 +14,12 @@ def elem_roles(elem):
     return {k.toLowerCase(): True for k in (elem.getAttribute('role') or '').split(' ')}
 
 
+def epub_type(elem):
+    for a in elem.attributes:
+        if a.nodeName.toLowerCase() == 'epub:type' and a.nodeValue:
+            return a.nodeValue
+
+
 def get_containing_block(node):
     while node and node.tagName and block_names[node.tagName.toLowerCase()] is not True:
         node = node.parentNode
@@ -26,6 +32,8 @@ def is_footnote_link(a, dest_name, dest_frag, src_name, link_to_map):
         return True
     if roles['doc-link']:
         return False
+    if epub_type(a) is 'noteref':
+        return True
 
     # Check if node or any of its first few parents have vertical-align set
     x, num = a, 3
@@ -50,7 +58,7 @@ def is_footnote_link(a, dest_name, dest_frag, src_name, link_to_map):
     eid = a.getAttribute('id') or a.getAttribute('name')
     files_linking_to_self = link_to_map[src_name]
     if eid and files_linking_to_self:
-        files_linking_to_anchor = files_linking_to_self[eid]
+        files_linking_to_anchor = files_linking_to_self[eid] or v'[]'
         if files_linking_to_anchor.length > 1 or (files_linking_to_anchor.length == 1 and files_linking_to_anchor[0] is not src_name):
             # An  link that is linked back from some other
             # file in the spine, most likely an endnote. We exclude links that are
@@ -77,6 +85,11 @@ is_footnote_link.vert_aligns = {'sub': True, 'super': True, 'top': True, 'bottom
 
 
 def is_epub_footnote(node):
+    et = epub_type(node)
+    if et:
+        et = et.toLowerCase()
+        if et is 'note' or et is 'footnote' or et is 'rearnote':
+            return True
     roles = elem_roles(node)
     if roles['doc-note'] or roles['doc-footnote'] or roles['doc-rearnote']:
         return True
diff --git a/src/pyj/read_book/iframe.pyj b/src/pyj/read_book/iframe.pyj
index 1130130c88..cf4d256249 100644
--- a/src/pyj/read_book/iframe.pyj
+++ b/src/pyj/read_book/iframe.pyj
@@ -9,11 +9,12 @@ from fs_images import fix_fullscreen_svg_images
 from iframe_comm import IframeClient
 from read_book.cfi import scroll_to as scroll_to_cfi
 from read_book.extract import get_elements
+from read_book.find import reset_find_caches, select_search_result
 from read_book.flow_mode import (
     anchor_funcs as flow_anchor_funcs, auto_scroll_action as flow_auto_scroll_action,
     flow_onwheel, flow_to_scroll_fraction, handle_gesture as flow_handle_gesture,
     handle_shortcut as flow_handle_shortcut, layout as flow_layout,
-    scroll_by_page as flow_scroll_by_page
+    scroll_by_page as flow_scroll_by_page, ensure_selection_visible
 )
 from read_book.footnotes import is_footnote_link
 from read_book.globals import (
@@ -24,13 +25,14 @@ from read_book.mathjax import apply_mathjax
 from read_book.paged_mode import (
     anchor_funcs as paged_anchor_funcs,
     auto_scroll_action as paged_auto_scroll_action, calc_columns_per_screen,
-    current_cfi, handle_gesture as paged_handle_gesture,
+    current_cfi, get_columns_per_screen_data, handle_gesture as paged_handle_gesture,
     handle_shortcut as paged_handle_shortcut, jump_to_cfi as paged_jump_to_cfi,
     layout as paged_layout, onwheel as paged_onwheel,
     prepare_for_resize as paged_prepare_for_resize, progress_frac,
     reset_paged_mode_globals, resize_done as paged_resize_done,
     scroll_by_page as paged_scroll_by_page, scroll_to_elem,
-    scroll_to_fraction as paged_scroll_to_fraction, snap_to_selection
+    scroll_to_fraction as paged_scroll_to_fraction, snap_to_selection,
+    will_columns_per_screen_change
 )
 from read_book.referencing import (
     elem_for_ref, end_reference_mode, start_reference_mode
@@ -44,11 +46,11 @@ from read_book.shortcuts import (
     create_shortcut_map, keyevent_as_shortcut, shortcut_for_key_event
 )
 from read_book.toc import update_visible_toc_anchors
-from read_book.touch import create_handlers as create_touch_handlers
-from read_book.viewport import scroll_viewport
-from utils import (
-    apply_cloned_selection, clone_selection, debounce, html_escape, is_ios
+from read_book.touch import (
+    create_handlers as create_touch_handlers, reset_handlers as reset_touch_handlers
 )
+from read_book.viewport import scroll_viewport
+from utils import debounce, html_escape, is_ios
 
 FORCE_FLOW_MODE = False
 CALIBRE_VERSION = '__CALIBRE_VERSION__'
@@ -98,6 +100,8 @@ class IframeBoss:
         handlers = {
             'change_color_scheme': self.change_color_scheme,
             'change_font_size': self.change_font_size,
+            'change_number_of_columns': self.change_number_of_columns,
+            'number_of_columns_changed': self.number_of_columns_changed,
             'change_scroll_speed': self.change_scroll_speed,
             'display': self.display,
             'find': self.find,
@@ -115,6 +119,7 @@ class IframeBoss:
             'window_size': self.received_window_size,
             'overlay_visibility_changed': self.on_overlay_visibility_changed,
             'show_search_result': self.show_search_result,
+            'handle_navigation_shortcut': self.on_handle_navigation_shortcut,
         }
         self.comm = IframeClient(handlers)
         self.last_window_ypos = 0
@@ -262,10 +267,44 @@ class IframeBoss:
         else:
             paged_scroll_by_page(backwards, data.all_pages_on_screen)
 
+
     def change_font_size(self, data):
         if data.base_font_size? and data.base_font_size != opts.base_font_size:
             opts.base_font_size = data.base_font_size
             apply_font_size()
+            if not runtime.is_standalone_viewer:
+                # in the standalone viewer this is a separate event as
+                # apply_font_size() is a no-op
+                self.relayout_on_font_size_change()
+
+    def change_number_of_columns(self, data):
+        if current_layout_mode() is 'flow':
+            self.send_message('error', title=_('In flow mode'), msg=_(
+                'Cannot change number of pages per screen in flow mode, switch to paged mode first.'))
+            return
+        cdata = get_columns_per_screen_data()
+        delta = int(data.delta)
+        if delta is 0:
+            new_val = 0
+        else:
+            new_val = max(1, cdata.cps + delta)
+        opts.columns_per_screen[cdata.which] = new_val
+        self.relayout_on_font_size_change()
+        self.send_message('columns_per_screen_changed', which=cdata.which, cps=new_val)
+
+    def relayout_on_font_size_change(self):
+        if current_layout_mode() is not 'flow' and will_columns_per_screen_change():
+            self.do_layout(self.is_titlepage)
+            if self.last_cfi:
+                cfi = self.last_cfi[len('epubcfi(/'):-1].partition('/')[2]
+                if cfi:
+                    paged_jump_to_cfi('/' + cfi)
+            self.update_cfi()
+            self.update_toc_position()
+
+    def number_of_columns_changed(self, data):
+        opts.columns_per_screen = data.columns_per_screen
+        self.relayout_on_font_size_change()
 
     def change_scroll_speed(self, data):
         if data.lines_per_sec_auto?:
@@ -299,6 +338,7 @@ class IframeBoss:
         self.content_loaded_stage2()
 
     def content_loaded_stage2(self):
+        reset_find_caches()
         self.connect_links()
         self.content_ready = True
         # this is the loading styles used to suppress scrollbars during load
@@ -334,6 +374,8 @@ class IframeBoss:
         self.send_message('content_loaded', progress_frac=self.calculate_progress_frac(), file_progress_frac=progress_frac())
         self.last_cfi = None
         self.auto_scroll_action('resume')
+        reset_touch_handlers() # Needed to mitigate issue https://bugs.chromium.org/p/chromium/issues/detail?id=464579
+
         window.setTimeout(self.update_cfi, 0)
         window.setTimeout(self.update_toc_position, 0)
 
@@ -455,6 +497,10 @@ class IframeBoss:
                 else:
                     self.send_message('handle_shortcut', name=sc_name)
 
+    def on_handle_navigation_shortcut(self, data):
+        self.handle_navigation_shortcut(data.name, data.key or {
+            'key': '', 'altKey': False, 'ctrlKey': False, 'shiftKey': False, 'metaKey': False})
+
     def oncontextmenu(self, evt):
         if self.content_ready:
             evt.preventDefault()
@@ -534,39 +580,13 @@ class IframeBoss:
                 self.send_message('find_in_spine', text=data.text, backwards=data.backwards, searched_in_spine=data.searched_in_spine)
 
     def show_search_result(self, data, from_load):
-        sr = data.search_result
-        idx = -1
-        window.getSelection().removeAllRanges()
-        while idx < sr.index:
-            if not window.find(sr.text, True, False, False, False, False):
-                self.send_message('search_result_not_found', search_result=sr)
-                break
-            if sr.mode is not 'normal':
-                # verify we have the correct match since regexes can have
-                # boundary conditions
-                sel = window.getSelection()
-                ranges = clone_selection(sel)
-                r = ranges[0]
-                if sr.before:
-                    p = r.cloneRange()
-                    p.collapse(True)
-                    sel = apply_cloned_selection(v'[p]')
-                    sel.modify('extend', 'left', 'character')
-                    if sel.toString() is not sr.before[-1]:
-                        apply_cloned_selection(ranges)
-                        continue
-                if sr.after:
-                    p = r.cloneRange()
-                    p.collapse(False)
-                    sel = apply_cloned_selection(v'[p]')
-                    sel.modify('extend', 'right', 'character')
-                    if sel.toString() is not sr.after[0]:
-                        apply_cloned_selection(ranges)
-                        continue
-                apply_cloned_selection(ranges)
-            idx += 1
-        if idx > -1 and current_layout_mode() is not 'flow':
-            snap_to_selection()
+        if select_search_result(data.search_result):
+            if current_layout_mode() is 'flow':
+                ensure_selection_visible()
+            else:
+                snap_to_selection()
+        else:
+            self.send_message('search_result_not_found', search_result=data.search_result)
 
     def reference_item_changed(self, ref_num_or_none):
         self.send_message('reference_item_changed', refnum=ref_num_or_none, index=current_spine_item().index)
diff --git a/src/pyj/read_book/overlay.pyj b/src/pyj/read_book/overlay.pyj
index e223d4c221..17b7d25aa6 100644
--- a/src/pyj/read_book/overlay.pyj
+++ b/src/pyj/read_book/overlay.pyj
@@ -247,7 +247,7 @@ class MainOverlay:  # {{{
         sync_action = ac(_('Sync'), _('Get last read position and annotations from the server'), self.overlay.sync_book, 'cloud-download')
         delete_action = ac(_('Delete'), _('Delete this book from the device'), self.overlay.delete_book, 'trash')
         reload_action = ac(_('Reload'), _('Reload this book from the {}').format( _('computer') if runtime.is_standalone_viewer else _('server')), self.overlay.reload_book, 'refresh')
-        home_action = ac(_('Home'), _('Return to list of books'), def(): home();, 'home')
+        home_action = ac(_('Home'), _('Return to the home page'), def(): home();, 'home')
         back_action = ac(_('Back'), None, self.back, 'arrow-left')
         forward_action = ac(_('Forward'), None, self.forward, 'arrow-right')
         if runtime.is_standalone_viewer:
@@ -281,7 +281,7 @@ class MainOverlay:  # {{{
 
             E.ul(
                 ac(_('Font size'), _('Change text size'), self.overlay.show_font_size_chooser, 'Aa', True),
-                ac(_('Preferences'), _('Configure the book reader'), self.overlay.show_prefs, 'cogs'),
+                ac(_('Preferences'), _('Configure the book viewer'), self.overlay.show_prefs, 'cogs'),
             ),
 
             class_=MAIN_OVERLAY_ACTIONS_CLASS
@@ -339,7 +339,9 @@ class MainOverlay:  # {{{
                 ac(_('Inspector'), _('Show the content inspector'),
                    def(): self.overlay.hide(), ui_operations.toggle_inspector();, 'bug'),
                 ac(_('Reset interface'), _('Reset viewer panels, toolbars and scrollbars to defaults'),
-                   def(): self.overlay.hide(), ui_operations.reset_interface();, 'remove'),
+                   def(): self.overlay.hide(), ui_operations.reset_interface();, 'window-restore'),
+                ac(_('Quit'), _('Close the viewer'),
+                   def(): self.overlay.hide(), ui_operations.quit();, 'remove'),
             ))
         container.appendChild(set_css(E.div(class_=MAIN_OVERLAY_TS_CLASS,  # top section
             onclick=def (evt):evt.stopPropagation();,
@@ -461,7 +463,7 @@ class PrefsOverlay: # {{{
     def on_hide(self):
         if self.changes_occurred:
             self.changes_occurred = False
-            ui_operations.redisplay_book()
+            self.overlay.view.preferences_changed()
 
 # }}}
 
@@ -479,20 +481,30 @@ class OpenBook: # {{{
     def __init__(self, overlay, closeable):
         self.overlay = overlay
         self.closeable = closeable
-        self.is_not_escapable = not closeable  # prevent Esc key from closing
+
+    def handle_escape(self):
+        if self.closeable:
+            self.overlay.hide_current_panel()
+        else:
+            ui_operations.quit()
 
     def on_container_click(self, evt):
         pass  # Dont allow panel to be closed by a click
 
     def show(self, container):
         container.style.backgroundColor = get_color('window-background')
-        close_button_style = '' if self.closeable else 'display: none'
         container.appendChild(E.div(
             style='padding: 1ex 1em; border-bottom: solid 1px currentColor; display:flex; justify-content: space-between',
                 E.h2(_('Open a new book')),
                 E.div(
-                    svgicon('close'), style=f'cursor:pointer; {close_button_style}',
-                    onclick=def(event):event.preventDefault(), event.stopPropagation(), self.overlay.hide_current_panel(event);,
+                    svgicon('close'), style=f'cursor:pointer',
+                    onclick=def(event):
+                        event.preventDefault(), event.stopPropagation()
+                        if self.closeable:
+                            self.overlay.hide_current_panel(event)
+                        else:
+                            ui_operations.quit()
+                    ,
                     class_='simple-link'),
             ))
         create_open_book(container, self.overlay.view?.book)
diff --git a/src/pyj/read_book/paged_mode.pyj b/src/pyj/read_book/paged_mode.pyj
index a7135a5758..e2bb712378 100644
--- a/src/pyj/read_book/paged_mode.pyj
+++ b/src/pyj/read_book/paged_mode.pyj
@@ -119,6 +119,26 @@ def fit_images():
         set_elem_data(img, 'height-limited', True)
 
 
+def cps_by_em_size():
+    ans = cps_by_em_size.ans
+    fs = window.getComputedStyle(document.body).fontSize
+    if not ans or cps_by_em_size.at_font_size is not fs:
+        d = document.createElement('span')
+        d.style.position = 'absolute'
+        d.style.visibility = 'hidden'
+        d.style.width = '1rem'
+        d.style.fontSize = '1rem'
+        d.style.paddingTop = d.style.paddingBottom = d.style.paddingLeft = d.style.paddingRight = '0'
+        d.style.marginTop = d.style.marginBottom = d.style.marginLeft = d.style.marginRight = '0'
+        d.style.borderStyle = 'none'
+        document.body.appendChild(d)
+        w = d.clientWidth
+        document.body.removeChild(d)
+        ans = cps_by_em_size.ans = max(2, w)
+        cps_by_em_size.at_font_size = fs
+    return ans
+
+
 def calc_columns_per_screen():
     cps = opts.columns_per_screen or {}
     cps = cps.landscape if scroll_viewport.width() > scroll_viewport.height() else cps.portrait
@@ -127,11 +147,20 @@ def calc_columns_per_screen():
     except:
         cps = 0
     if not cps:
-        cps = int(Math.floor(scroll_viewport.width() / 500.0))
+        cps = int(Math.floor(scroll_viewport.width() / (35 * cps_by_em_size())))
     cps = max(1, min(cps or 1, 20))
     return cps
 
 
+def get_columns_per_screen_data():
+    which = 'landscape' if scroll_viewport.width() > scroll_viewport.height() else 'portrait'
+    return {'which': which, 'cps': calc_columns_per_screen()}
+
+
+def will_columns_per_screen_change():
+    return calc_columns_per_screen() != cols_per_screen
+
+
 def layout(is_single_page, on_resize):
     nonlocal _in_paged_mode, col_width, col_and_gap, screen_height, gap, screen_width, is_full_screen_layout, cols_per_screen, number_of_cols
     body_style = window.getComputedStyle(document.body)
diff --git a/src/pyj/read_book/prefs/font_size.pyj b/src/pyj/read_book/prefs/font_size.pyj
index 7ad7c24c7e..5970979aa7 100644
--- a/src/pyj/read_book/prefs/font_size.pyj
+++ b/src/pyj/read_book/prefs/font_size.pyj
@@ -18,6 +18,7 @@ add_extra_css(def():
     style = rule(QUICK, 'li.current', background_color=get_color('window-background2'))
     style += rule(QUICK, 'li:hover', background_color=get_color('window-background2'))
     style += rule(CONTAINER, 'a:hover', color=get_color('window-hover-foreground'))
+    style += rule(CONTAINER, 'a.calibre-push-button:hover', color=get_color('button-text'))
     return style
 )
 
@@ -40,7 +41,7 @@ def change_font_size_by(amt):
     sd = get_session_data()
     sz = sd.get('base_font_size')
     nsz = sz + amt
-    nsz = max(8, min(nsz, 40))
+    nsz = max(8, min(nsz, 80))
     change_font_size(nsz)
 
 def show_custom_size(ev):
diff --git a/src/pyj/read_book/prefs/scrolling.pyj b/src/pyj/read_book/prefs/scrolling.pyj
index 3ae6947910..71a6511eb6 100644
--- a/src/pyj/read_book/prefs/scrolling.pyj
+++ b/src/pyj/read_book/prefs/scrolling.pyj
@@ -13,14 +13,14 @@ from session import defaults
 CONTAINER = unique_id('standalone-scrolling-settings')
 
 # Scroll speeds in lines/sec
-MIN_SCROLL_SPEED_AUTO = 0.25
+MIN_SCROLL_SPEED_AUTO = 0.05
 MAX_SCROLL_SPEED_AUTO = 5
 
 MIN_SCROLL_AUTO_DELAY = -1
-MAX_SCROLL_AUTO_DELAY = 10
+MAX_SCROLL_AUTO_DELAY = 50
 
-MIN_SCROLL_SPEED_SMOOTH = 10
-MAX_SCROLL_SPEED_SMOOTH = 50
+MIN_SCROLL_SPEED_SMOOTH = 5
+MAX_SCROLL_SPEED_SMOOTH = 80
 
 def restore_defaults():
     container = get_container()
diff --git a/src/pyj/read_book/scrollbar.pyj b/src/pyj/read_book/scrollbar.pyj
index cd7d89e835..639cf6dae6 100644
--- a/src/pyj/read_book/scrollbar.pyj
+++ b/src/pyj/read_book/scrollbar.pyj
@@ -7,7 +7,7 @@ from elementmaker import E
 from book_list.globals import get_session_data
 from book_list.theme import cached_color_to_rgba
 from dom import unique_id
-
+from read_book.globals import ui_operations
 
 SIZE = 10
 
@@ -31,7 +31,7 @@ class BookScrollbar:
         return E.div(
             id=self.container_id,
             style=f'height: 100vh; background-color: #aaa; width: {SIZE}px; border-radius: 5px',
-            onclick=self.bar_clicked,
+            onclick=self.bar_clicked, oncontextmenu=self.context_menu,
             E.div(
                 style=f'position: relative; width: 100%; height: {int(2.2*SIZE)}px; background-color: #444; border-radius: 5px',
                 onmousedown=self.on_bob_mousedown,
@@ -41,6 +41,16 @@ class BookScrollbar:
             )
         )
 
+    def context_menu(self, ev):
+        if ui_operations.scrollbar_context_menu:
+            ev.preventDefault(), ev.stopPropagation()
+            c = self.container
+            bob = c.firstChild
+            height = c.clientHeight - bob.clientHeight
+            top = max(0, min(ev.clientY - bob.clientHeight, height))
+            frac = max(0, min(top / height, 1))
+            ui_operations.scrollbar_context_menu(ev.screenX, ev.screenY, frac)
+
     def bar_clicked(self, evt):
         if evt.button is 0:
             c = self.container
diff --git a/src/pyj/read_book/search.pyj b/src/pyj/read_book/search.pyj
index a3ce46f81d..94f5ac2d04 100644
--- a/src/pyj/read_book/search.pyj
+++ b/src/pyj/read_book/search.pyj
@@ -18,7 +18,7 @@ add_extra_css(def():
     sel = '.' + CLASS_NAME
     style = build_rule(sel, text_align='right', user_select='none')
     sel += ' > div '
-    style += build_rule(sel, display='inline-flex', pointer_events='auto', background_color=get_color('window-background'), padding='1ex')
+    style += build_rule(sel, display='inline-flex', align_items='center', pointer_events='auto', background_color=get_color('window-background'), padding='1ex')
     return style
 )
 
diff --git a/src/pyj/read_book/shortcuts.pyj b/src/pyj/read_book/shortcuts.pyj
index 7345b359f9..fdc6c93c28 100644
--- a/src/pyj/read_book/shortcuts.pyj
+++ b/src/pyj/read_book/shortcuts.pyj
@@ -51,10 +51,13 @@ def get_key_text(evt):
         cc = key.charCodeAt(0)
         # on windows in webengine pressing ctrl+ascii char gives us an ascii
         # control code
-        if (0 < cc < 32 or key is 'Enter') and evt.ctrlKey and not evt.metaKey and not evt.altKey:
+        if (0 < cc < 32 or key is 'Enter' or key is 'Tab') and evt.ctrlKey and not evt.metaKey and not evt.altKey:
             if key is 'Enter':
                 if evt.code and evt.code is not 'Enter':
                     key = 'm'
+            elif key is 'Tab':
+                if evt.code and evt.code is not 'Tab':
+                    key = 'i'
             else:
                 key = chr(96 + cc)
     return key
@@ -216,6 +219,24 @@ def shortcuts_definition():
         _('Decrease font size'),
     ),
 
+    'increase_number_of_columns': desc(
+        v"['Ctrl+]']",
+        'ui',
+        _('Increase number of pages per screen'),
+    ),
+
+    'decrease_number_of_columns': desc(
+        v"['Ctrl+[']",
+        'ui',
+        _('Decrease number of pages per screen'),
+    ),
+
+    'reset_number_of_columns': desc(
+        v"['Ctrl+Alt+c']",
+        'ui',
+        _('Make number of pages per screen automatic'),
+    ),
+
     'toggle_full_screen': desc(
         v"['F11', 'Ctrl+Shift+F']",
         'ui',
@@ -228,6 +249,12 @@ def shortcuts_definition():
         _('Toggle between Paged mode and Flow mode for text layout')
     ),
 
+    'toggle_scrollbar': desc(
+        'Ctrl+w',
+        'ui',
+        _('Toggle the scrollbar')
+    ),
+
     'toggle_reference_mode': desc(
         'Ctrl+x',
         'ui',
diff --git a/src/pyj/read_book/toc.pyj b/src/pyj/read_book/toc.pyj
index e1981b20d5..3b99d4445b 100644
--- a/src/pyj/read_book/toc.pyj
+++ b/src/pyj/read_book/toc.pyj
@@ -159,7 +159,7 @@ def create_toc_panel(book, container, onclick):
     t = _('Search Table of Contents')
     search_bar = create_search_bar(do_search.bind(toc_panel_id), 'search-book-toc', button=search_button, placeholder=t)
     set_css(search_bar, flex_grow='10', margin_right='1em')
-    container.appendChild(E.div(style='margin: 1ex 1em; display: flex;', search_bar, search_button))
+    container.appendChild(E.div(style='margin: 1ex 1em; display: flex; align-items: center', search_bar, search_button))
 
 
 def current_toc_anchor_map(tam, anchor_funcs):
diff --git a/src/pyj/read_book/touch.pyj b/src/pyj/read_book/touch.pyj
index 8858350fd3..fe1b12821b 100644
--- a/src/pyj/read_book/touch.pyj
+++ b/src/pyj/read_book/touch.pyj
@@ -6,7 +6,7 @@ from read_book.globals import get_boss, ui_operations
 from read_book.viewport import scroll_viewport
 
 HOLD_THRESHOLD = 750  # milliseconds
-TAP_THRESHOLD = 7  # pixels
+TAP_THRESHOLD = 8  # pixels
 TAP_LINK_THRESHOLD = 5  # pixels
 PINCH_THRESHOLD = 10  # pixels
 LONG_TAP_THRESHOLD = 500  # milliseconds
@@ -140,6 +140,12 @@ class TouchHandler:
                 return True
         return False
 
+    def reset_handlers(self):
+        self.stop_hold_timer()
+        self.ongoing_touches = {}
+        self.gesture_id = None
+        self.handled_tap_hold = False
+
     def start_hold_timer(self):
         self.stop_hold_timer()
         self.hold_timer = window.setTimeout(self.check_for_hold, 100)
@@ -199,9 +205,7 @@ class TouchHandler:
         self.prune_expired_touches()
         if not self.has_active_touches:
             self.dispatch_gesture()
-            self.ongoing_touches = {}
-            self.gesture_id = None
-            self.handled_tap_hold = False
+            self.reset_handlers()
 
     def handle_touchcancel(self, ev):
         ev.preventDefault(), ev.stopPropagation()
@@ -278,6 +282,12 @@ def create_handlers():
     # Safari does not work if we register the handler
     # on window instead of document
     install_handlers(document, main_touch_handler)
+    # See https://github.com/kovidgoyal/calibre/pull/1101
+    # for why we need touchAction none
+    document.body.style.touchAction = 'none'
+
+def reset_handlers():
+    main_touch_handler.reset_handlers()
 
 def set_left_margin_handler(elem):
     install_handlers(elem, left_margin_handler)
diff --git a/src/pyj/read_book/view.pyj b/src/pyj/read_book/view.pyj
index 16cd2da6c6..fe306e30ef 100644
--- a/src/pyj/read_book/view.pyj
+++ b/src/pyj/read_book/view.pyj
@@ -127,9 +127,23 @@ def show_controls_help():
 
 # }}}
 
+
+def maximum_font_size():
+    ans = maximum_font_size.ans
+    if not ans:
+        q = window.getComputedStyle(document.body).fontSize
+        if q and q.endsWith('px'):
+            q = parseInt(q)
+            if q and not isNaN(q):
+                ans = maximum_font_size.ans = q
+                return ans
+        ans = maximum_font_size.ans = 12
+    return ans
+
+
 def margin_elem(sd, which, id, onclick, oncontextmenu):
     sz = sd.get(which, 20)
-    fsz = min(max(0, sz - 6), 12)
+    fsz = min(max(0, sz - 6), maximum_font_size())
     s = '; text-overflow: ellipsis; white-space: nowrap; overflow: hidden'
     ans = E.div(
         style=f'height:{sz}px; overflow: hidden; font-size:{fsz}px; width:100%; padding: 0; display: flex; justify-content: space-between; align-items: center',
@@ -172,7 +186,7 @@ class View:
         self.current_progress_frac = self.current_file_progress_frac = 0
         self.current_toc_node = self.current_toc_toplevel_node = None
         self.report_cfi_callbacks = {}
-        self.show_chrome_counter = 0
+        self.get_cfi_counter = 0
         self.show_loading_callback_timer = None
         self.timer_ids = {'clock': 0}
         self.book_scrollbar = BookScrollbar(self)
@@ -237,6 +251,7 @@ class View:
             'request_size': self.on_request_size,
             'scroll_to_anchor': self.on_scroll_to_anchor,
             'selectionchange': self.on_selection_change,
+            'columns_per_screen_changed': self.on_columns_per_screen_changed,
             'show_chrome': self.show_chrome,
             'show_footnote': self.on_show_footnote,
             'update_cfi': self.on_update_cfi,
@@ -269,6 +284,15 @@ class View:
     def reference_mode_overlay(self):
         return document.getElementById('reference-mode-overlay')
 
+    def set_scrollbar_visibility(self, visible):
+        sd = get_session_data()
+        sd.set('book_scrollbar', bool(visible))
+        self.book_scrollbar.apply_visibility()
+
+    def toggle_scrollbar(self):
+        sd = get_session_data()
+        self.set_scrollbar_visibility(not sd.get('book_scrollbar'))
+
     def on_lookup_word(self, data):
         if runtime.is_standalone_viewer:
             ui_operations.selection_changed(data.word)
@@ -388,6 +412,8 @@ class View:
             self.toggle_paged_mode()
         elif data.name is 'toggle_toolbar':
             self.toggle_toolbar()
+        elif data.name is 'toggle_scrollbar':
+            self.toggle_scrollbar()
         elif data.name is 'quit':
             ui_operations.quit()
         elif data.name is 'start_search':
@@ -425,7 +451,7 @@ class View:
         elif data.name is 'print':
             ui_operations.print_book()
         elif data.name is 'preferences':
-            self.overlay.show_prefs()
+            self.show_chrome({'initial_panel': 'show_prefs'})
         elif data.name is 'metadata':
             self.overlay.show_metadata()
         elif data.name is 'goto_location':
@@ -442,12 +468,26 @@ class View:
             self.toggle_autoscroll()
         elif data.name.startsWith('switch_color_scheme:'):
             self.switch_color_scheme(data.name.partition(':')[-1])
+        elif data.name is 'increase_number_of_columns':
+            self.iframe_wrapper.send_message('change_number_of_columns', delta=1)
+        elif data.name is 'decrease_number_of_columns':
+            self.iframe_wrapper.send_message('change_number_of_columns', delta=-1)
+        elif data.name is 'reset_number_of_columns':
+            self.iframe_wrapper.send_message('change_number_of_columns', delta=0)
+        else:
+            self.iframe_wrapper.send_message('handle_navigation_shortcut', name=data.name)
 
     def on_selection_change(self, data):
         self.currently_showing.selected_text = data.text
         if ui_operations.selection_changed:
             ui_operations.selection_changed(data.text)
 
+    def on_columns_per_screen_changed(self, data):
+        sd = get_session_data()
+        cps = sd.get('columns_per_screen') or {}
+        cps[data.which] = int(data.cps)
+        sd.set('columns_per_screen', cps)
+
     def switch_color_scheme(self, name):
         get_session_data().set('current_color_scheme', name)
         ui_operations.redisplay_book()
@@ -519,16 +559,26 @@ class View:
         self.iframe.contentWindow.focus()
 
     def show_chrome(self, data):
-        self.show_chrome_counter += 1
         elements = {}
         if data and data.elements:
             elements = data.elements
-        self.get_current_cfi('show-chrome-' + self.show_chrome_counter, self.do_show_chrome.bind(None, elements))
+        initial_panel = data?.initial_panel or None
+        self.get_current_cfi('show-chrome', self.do_show_chrome.bind(None, elements, initial_panel))
 
-    def do_show_chrome(self, elements, request_id, cfi_data):
+    def do_show_chrome(self, elements, initial_panel, request_id, cfi_data):
         self.hide_overlays()
         self.update_cfi_data(cfi_data)
-        self.overlay.show(elements)
+        if initial_panel:
+            getattr(self.overlay, initial_panel)()
+        else:
+            self.overlay.show(elements)
+
+    def prepare_for_close(self):
+
+        def close_prepared(request_id, cfi_data):
+            ui_operations.close_prep_finished(cfi_data.cfi)
+
+        self.get_current_cfi('prepare-close', close_prepared)
 
     def show_search(self):
         self.hide_overlays()
@@ -721,6 +771,10 @@ class View:
             show_controls_help()
             sd.set('controls_help_shown_count', c + 1)
 
+    def preferences_changed(self):
+        ui_operations.update_url_state(True)
+        ui_operations.redisplay_book()
+
     def redisplay_book(self):
         # redisplay_book() is called when settings are changed
         sd = get_session_data()
@@ -897,6 +951,8 @@ class View:
             self.goto_named_destination(toc_node.dest, toc_node.frag)
 
     def get_current_cfi(self, request_id, callback):
+        self.get_cfi_counter += 1
+        request_id += ':' + self.get_cfi_counter
         self.report_cfi_callbacks[request_id] = callback
         self.iframe_wrapper.send_message('get_current_cfi', request_id=request_id)
 
@@ -917,7 +973,7 @@ class View:
     def on_report_cfi(self, data):
         cb = self.report_cfi_callbacks[data.request_id]
         if cb:
-            cb(data.request_id, {
+            cb(data.request_id.rpartition(':')[0], {
                 'cfi': data.cfi,
                 'progress_frac': data.progress_frac,
                 'file_progress_frac': data.file_progress_frac,
@@ -1018,7 +1074,6 @@ class View:
             self.iframe_wrapper.init()
 
     def show_spine_item_stage2(self, resource_data):
-        self.currently_showing.loading = False
         # We cannot encrypt this message because the resource data contains
         # Blob objects which do not survive encryption
         self.processing_spine_item_display = True
@@ -1032,6 +1087,7 @@ class View:
 
     def on_content_loaded(self, data):
         self.processing_spine_item_display = False
+        self.currently_showing.loading = False
         self.hide_loading()
         self.set_progress_frac(data.progress_frac, data.file_progress_frac)
         self.update_header_footer()
@@ -1052,6 +1108,9 @@ class View:
     def update_font_size(self):
         self.iframe_wrapper.send_message('change_font_size', base_font_size=get_session_data().get('base_font_size'))
 
+    def viewer_font_size_changed(self):
+        self.iframe_wrapper.send_message('viewer_font_size_changed', base_font_size=get_session_data().get('base_font_size'))
+
     def update_scroll_speed(self, amt):
         self.iframe_wrapper.send_message('change_scroll_speed', lines_per_sec_auto=change_scroll_speed(amt))
 
diff --git a/src/pyj/utils.pyj b/src/pyj/utils.pyj
index 625f5524e3..93e1747e10 100644
--- a/src/pyj/utils.pyj
+++ b/src/pyj/utils.pyj
@@ -252,21 +252,6 @@ def sandboxed_html(html, style, sandbox):
     return ans
 
 
-def clone_selection(sel):
-    ans = v'[]'
-    for i in range(sel.rangeCount):
-        ans.push(sel.getRangeAt(i).cloneRange())
-    return ans
-
-
-def apply_cloned_selection(ranges):
-    sel = window.getSelection()
-    sel.removeAllRanges()
-    for r in ranges:
-        sel.addRange(r)
-    return sel
-
-
 if __name__ is '__main__':
     from pythonize import strings
     strings()
diff --git a/src/pyj/viewer-main.pyj b/src/pyj/viewer-main.pyj
index 4f47f8381d..bc916008cf 100644
--- a/src/pyj/viewer-main.pyj
+++ b/src/pyj/viewer-main.pyj
@@ -293,6 +293,19 @@ def show_search_result(sr):
     if view:
         view.show_search_result(sr)
 
+@from_python
+def prepare_for_close():
+    if view:
+        view.prepare_for_close()
+    else:
+        ui_operations.close_prep_finished(None)
+
+
+@from_python
+def viewer_font_size_changed():
+    if view:
+        view.viewer_font_size_changed()
+
 
 def onerror(msg, script_url, line_number, column_number, error_object):
     if not error_object:
@@ -361,6 +374,8 @@ if window is window.top:
         sd.set('footer', defaults.footer)
         view.update_header_footer()
         to_python.reset_interface()
+    ui_operations.quit = def():
+        to_python.quit()
     ui_operations.toggle_lookup = def():
         to_python.toggle_lookup()
     ui_operations.selection_changed = def(selected_text):
@@ -401,6 +416,10 @@ if window is window.top:
         to_python.autoscroll_state_changed(active)
     ui_operations.search_result_not_found = def(sr):
         to_python.search_result_not_found(sr)
+    ui_operations.scrollbar_context_menu = def(x, y, frac):
+        to_python.scrollbar_context_menu(x, y, frac)
+    ui_operations.close_prep_finished = def(cfi):
+        to_python.close_prep_finished(cfi)
 
     document.body.appendChild(E.div(id='view'))
     window.onerror = onerror