From 5b7608983987f1fc782325f392de8a4338d8fe8c Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Wed, 13 Mar 2019 06:40:38 +0530
Subject: [PATCH] Various py3 related fixes exposed by the unicode patch

---
 src/calibre/db/backend.py                     |   18 +-
 src/calibre/db/categories.py                  |   15 +-
 src/calibre/db/search.py                      |   10 +-
 src/calibre/db/utils.py                       |    6 +-
 src/calibre/db/write.py                       |    4 +
 src/calibre/devices/prst1/driver.py           |    4 +-
 .../devices/smart_device_app/driver.py        |    6 +-
 .../ebooks/conversion/plugins/epub_output.py  |    7 +-
 src/calibre/ebooks/lrf/html/convert_from.py   |   11 -
 src/calibre/ebooks/lrf/meta.py                |    4 +-
 src/calibre/ebooks/lrf/objects.py             |  103 +-
 src/calibre/ebooks/metadata/book/base.py      |   14 +-
 src/calibre/ebooks/metadata/fb2.py            |    2 +-
 src/calibre/ebooks/metadata/opf2.py           |   19 +-
 src/calibre/ebooks/mobi/mobiml.py             |   26 +-
 src/calibre/ebooks/mobi/reader/mobi6.py       |   67 +-
 src/calibre/ebooks/mobi/writer2/serializer.py |   51 +-
 src/calibre/ebooks/oeb/base.py                |   74 +-
 src/calibre/ebooks/oeb/iterator/spine.py      |    2 +-
 src/calibre/ebooks/oeb/parse_utils.py         |    4 +-
 src/calibre/ebooks/oeb/stylizer.py            |    8 +-
 src/calibre/ebooks/txt/processor.py           |   28 +-
 src/calibre/ebooks/unihandecode/__init__.py   |   16 +-
 .../ebooks/unihandecode/pykakasi/jisyo.py     |    6 +-
 src/calibre/ebooks/unihandecode/unidecoder.py |   10 +-
 src/calibre/gui2/comments_editor.py           |    2 +-
 src/calibre/gui2/dbus_export/gtk.py           |    3 +-
 src/calibre/gui2/dialogs/quickview.py         |    4 +-
 src/calibre/gui2/init.py                      |    7 +-
 src/calibre/gui2/layout.py                    |    2 +-
 src/calibre/gui2/lrf_renderer/main.py         |    5 +-
 src/calibre/gui2/main.py                      |    2 +-
 src/calibre/gui2/main_window.py               |   11 +-
 src/calibre/gui2/notify.py                    |   10 +-
 src/calibre/gui2/store/search/search.py       |    4 +-
 .../gui2/tweak_book/function_replace.py       |    4 +-
 src/calibre/library/add_to_library.py         |    5 +-
 src/calibre/library/caches.py                 |   16 +-
 src/calibre/library/catalogs/csv_xml.py       |    6 +-
 src/calibre/library/custom_columns.py         |   14 +-
 src/calibre/library/database2.py              |   14 +-
 src/calibre/srv/routes.py                     |    2 +-
 src/calibre/utils/apsw_shell.py               | 2958 -----------------
 src/calibre/utils/date.py                     |    4 +-
 src/calibre/utils/formatter_functions.py      |    2 +-
 src/calibre/utils/ipc/simple_worker.py        |    4 +-
 src/calibre/web/feeds/recipes/__init__.py     |    4 +-
 src/polyglot/builtins.py                      |    2 +
 48 files changed, 390 insertions(+), 3210 deletions(-)
 delete mode 100644 src/calibre/utils/apsw_shell.py

diff --git a/src/calibre/db/backend.py b/src/calibre/db/backend.py
index c1d7e6b9b1..146bb8eba4 100644
--- a/src/calibre/db/backend.py
+++ b/src/calibre/db/backend.py
@@ -670,7 +670,7 @@ class DB(object):
             if d['is_multiple']:
                 if x is None:
                     return []
-                if isinstance(x, (str, unicode_type, bytes)):
+                if isinstance(x, (unicode_type, bytes)):
                     x = x.split(d['multiple_seps']['ui_to_list'])
                 x = [y.strip() for y in x if y.strip()]
                 x = [y.decode(preferred_encoding, 'replace') if not isinstance(y,
@@ -681,12 +681,16 @@ class DB(object):
                         x.decode(preferred_encoding, 'replace')
 
         def adapt_datetime(x, d):
-            if isinstance(x, (str, unicode_type, bytes)):
+            if isinstance(x, (unicode_type, bytes)):
+                if isinstance(x, bytes):
+                    x = x.decode(preferred_encoding, 'replace')
                 x = parse_date(x, assume_utc=False, as_utc=False)
             return x
 
         def adapt_bool(x, d):
-            if isinstance(x, (str, unicode_type, bytes)):
+            if isinstance(x, (unicode_type, bytes)):
+                if isinstance(x, bytes):
+                    x = x.decode(preferred_encoding, 'replace')
                 x = x.lower()
                 if x == 'true':
                     x = True
@@ -707,7 +711,9 @@ class DB(object):
         def adapt_number(x, d):
             if x is None:
                 return None
-            if isinstance(x, (str, unicode_type, bytes)):
+            if isinstance(x, (unicode_type, bytes)):
+                if isinstance(x, bytes):
+                    x = x.decode(preferred_encoding, 'replace')
                 if x.lower() == 'none':
                     return None
             if d['datatype'] == 'int':
@@ -1083,7 +1089,7 @@ class DB(object):
 
     def dump_and_restore(self, callback=None, sql=None):
         import codecs
-        from calibre.utils.apsw_shell import Shell
+        from apsw import Shell
         from contextlib import closing
         if callback is None:
             callback = lambda x: x
@@ -1096,7 +1102,7 @@ class DB(object):
                     shell = Shell(db=self.conn, stdout=buf)
                     shell.process_command('.dump')
             else:
-                with open(fname, 'wb') as buf:
+                with lopen(fname, 'wb') as buf:
                     buf.write(sql if isinstance(sql, bytes) else sql.encode('utf-8'))
 
             with TemporaryFile(suffix='_tmpdb.db', dir=os.path.dirname(self.dbpath)) as tmpdb:
diff --git a/src/calibre/db/categories.py b/src/calibre/db/categories.py
index 377eab54ba..0ae1cee97f 100644
--- a/src/calibre/db/categories.py
+++ b/src/calibre/db/categories.py
@@ -11,6 +11,7 @@ import copy
 from functools import partial
 from polyglot.builtins import unicode_type, map
 
+from calibre.constants import ispy3
 from calibre.ebooks.metadata import author_to_author_sort
 from calibre.utils.config_base import tweaks
 from calibre.utils.icu import sort_key, collation_order
@@ -43,11 +44,19 @@ class Tag(object):
         self.search_expression = search_expression
         self.original_categories = None
 
-    def __unicode__(self):
+    @property
+    def string_representation(self):
         return u'%s:%s:%s:%s:%s'%(self.name, self.count, self.id, self.state, self.category)
 
-    def __str__(self):
-        return unicode_type(self).encode('utf-8')
+    if ispy3:
+        def __str__(self):
+            return self.string_representation
+    else:
+        def __str__(self):
+            return self.string_representation.encode('utf-8')
+
+        def __unicode__(self):
+            return self.string_representation
 
     def __repr__(self):
         return str(self)
diff --git a/src/calibre/db/search.py b/src/calibre/db/search.py
index 175cd36d6e..da021d0e36 100644
--- a/src/calibre/db/search.py
+++ b/src/calibre/db/search.py
@@ -80,7 +80,7 @@ def _match(query, value, matchkind, use_primary_find_in_search=True, case_sensit
                     if primary_contains(query, t):
                         return True
                 elif query in t:
-                        return True
+                    return True
         except re.error:
             pass
     return False
@@ -149,7 +149,9 @@ class DateSearch(object):  # {{{
 
         if query == 'false':
             for v, book_ids in field_iter():
-                if isinstance(v, (str, unicode_type)):
+                if isinstance(v, (bytes, unicode_type)):
+                    if isinstance(v, bytes):
+                        v = v.decode(preferred_encoding, 'replace')
                     v = parse_date(v)
                 if v is None or v <= UNDEFINED_DATE:
                     matches |= book_ids
@@ -157,7 +159,9 @@ class DateSearch(object):  # {{{
 
         if query == 'true':
             for v, book_ids in field_iter():
-                if isinstance(v, (str, unicode_type)):
+                if isinstance(v, (bytes, unicode_type)):
+                    if isinstance(v, bytes):
+                        v = v.decode(preferred_encoding, 'replace')
                     v = parse_date(v)
                 if v is not None and v > UNDEFINED_DATE:
                     matches |= book_ids
diff --git a/src/calibre/db/utils.py b/src/calibre/db/utils.py
index 99f281ad59..65d3e678d2 100644
--- a/src/calibre/db/utils.py
+++ b/src/calibre/db/utils.py
@@ -13,13 +13,15 @@ from polyglot.builtins import map, unicode_type
 from threading import Lock
 
 from calibre import as_unicode, prints
-from calibre.constants import cache_dir, get_windows_number_formats, iswindows
+from calibre.constants import cache_dir, get_windows_number_formats, iswindows, preferred_encoding
 
 from calibre.utils.localization import canonicalize_lang
 
 
 def force_to_bool(val):
-    if isinstance(val, (str, unicode_type)):
+    if isinstance(val, (bytes, unicode_type)):
+        if isinstance(val, bytes):
+            val = val.decode(preferred_encoding, 'replace')
         try:
             val = icu_lower(val)
             if not val:
diff --git a/src/calibre/db/write.py b/src/calibre/db/write.py
index dc8caace86..5670fff3f5 100644
--- a/src/calibre/db/write.py
+++ b/src/calibre/db/write.py
@@ -88,6 +88,8 @@ def adapt_number(typ, x):
     if x is None:
         return None
     if isinstance(x, (unicode_type, bytes)):
+        if isinstance(x, bytes):
+            x = x.decode(preferred_encoding, 'replace')
         if not x or x.lower() == 'none':
             return None
     return typ(x)
@@ -95,6 +97,8 @@ def adapt_number(typ, x):
 
 def adapt_bool(x):
     if isinstance(x, (unicode_type, bytes)):
+        if isinstance(x, bytes):
+            x = x.decode(preferred_encoding, 'replace')
         x = x.lower()
         if x == 'true':
             x = True
diff --git a/src/calibre/devices/prst1/driver.py b/src/calibre/devices/prst1/driver.py
index 12120249d3..3dbb353713 100644
--- a/src/calibre/devices/prst1/driver.py
+++ b/src/calibre/devices/prst1/driver.py
@@ -171,7 +171,7 @@ class PRST1(USBMS):
 
         with closing(sqlite.connect(dbpath)) as connection:
             # Replace undecodable characters in the db instead of erroring out
-            connection.text_factory = lambda x: unicode_type(x, "utf-8", "replace")
+            connection.text_factory = lambda x: x if isinstance(x, unicode_type) else x.decode('utf-8', 'replace')
 
             cursor = connection.cursor()
             # Query collections
@@ -758,7 +758,7 @@ class PRST1(USBMS):
 
         thumbnail_path = THUMBPATH%book.bookId
 
-        prefix = self._main_prefix if source_id is 0 else self._card_a_prefix
+        prefix = self._main_prefix if source_id == 0 else self._card_a_prefix
         thumbnail_file_path = os.path.join(prefix, *thumbnail_path.split('/'))
         thumbnail_dir_path = os.path.dirname(thumbnail_file_path)
         if not os.path.exists(thumbnail_dir_path):
diff --git a/src/calibre/devices/smart_device_app/driver.py b/src/calibre/devices/smart_device_app/driver.py
index 9619a82f04..7d60bf2916 100644
--- a/src/calibre/devices/smart_device_app/driver.py
+++ b/src/calibre/devices/smart_device_app/driver.py
@@ -398,7 +398,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
                     if isinstance(a, dict):
                         printable = {}
                         for k,v in a.iteritems():
-                            if isinstance(v, (str, unicode_type)) and len(v) > 50:
+                            if isinstance(v, (bytes, unicode_type)) and len(v) > 50:
                                 printable[k] = 'too long'
                             else:
                                 printable[k] = v
@@ -666,7 +666,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
             if v:
                 v = json.loads(v, object_hook=from_json)
                 if print_debug_info and extra_debug:
-                        self._debug('receive after decode')  # , v)
+                    self._debug('receive after decode')  # , v)
                 return (self.reverse_opcodes[v[0]], v[1])
             self._debug('protocol error -- empty json string')
         except socket.timeout:
@@ -1155,7 +1155,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
                                       (self.DEFAULT_THUMBNAIL_HEIGHT/3) * 4)
                 self._debug('cover width', self.THUMBNAIL_WIDTH)
             elif hasattr(self, 'THUMBNAIL_WIDTH'):
-                    delattr(self, 'THUMBNAIL_WIDTH')
+                delattr(self, 'THUMBNAIL_WIDTH')
 
             self.is_read_sync_col = result.get('isReadSyncCol', None)
             self._debug('Device is_read sync col', self.is_read_sync_col)
diff --git a/src/calibre/ebooks/conversion/plugins/epub_output.py b/src/calibre/ebooks/conversion/plugins/epub_output.py
index a6967a8645..25f6c3b2aa 100644
--- a/src/calibre/ebooks/conversion/plugins/epub_output.py
+++ b/src/calibre/ebooks/conversion/plugins/epub_output.py
@@ -12,7 +12,6 @@ from calibre.customize.conversion import (OutputFormatPlugin,
         OptionRecommendation)
 from calibre.ptempfile import TemporaryDirectory
 from calibre import CurrentDir
-from calibre.constants import filesystem_encoding
 from polyglot.builtins import unicode_type
 
 block_level_tags = (
@@ -41,7 +40,7 @@ block_level_tags = (
       'pre',
       'table',
       'ul',
-      )
+)
 
 
 class EPUBOutput(OutputFormatPlugin):
@@ -326,13 +325,11 @@ class EPUBOutput(OutputFormatPlugin):
             fonts = []
             for uri in list(uris.keys()):
                 path = uris[uri]
-                if isinstance(path, unicode_type):
-                    path = path.encode(filesystem_encoding)
                 if not os.path.exists(path):
                     uris.pop(uri)
                     continue
                 self.log.debug('Encrypting font:', uri)
-                with open(path, 'r+b') as f:
+                with lopen(path, 'r+b') as f:
                     data = f.read(1024)
                     if len(data) >= 1024:
                         f.seek(0)
diff --git a/src/calibre/ebooks/lrf/html/convert_from.py b/src/calibre/ebooks/lrf/html/convert_from.py
index 9ccfe8f2a5..9244ebb686 100644
--- a/src/calibre/ebooks/lrf/html/convert_from.py
+++ b/src/calibre/ebooks/lrf/html/convert_from.py
@@ -55,11 +55,7 @@ def munge_paths(basepath, url):
     if not path:
         path = basepath
     elif not os.path.isabs(path):
-        if isinstance(path, unicode_type):
-            path = path.encode(sys.getfilesystemencoding())
         dn = os.path.dirname(basepath)
-        if isinstance(dn, unicode_type):
-            dn = dn.encode(sys.getfilesystemencoding())
         path = os.path.join(dn, path)
     return os.path.normpath(path), fragment
 
@@ -1480,11 +1476,6 @@ class HTMLConverter(object):
                         ext = os.path.splitext(path)[1]
                         if ext:
                             ext = ext[1:].lower()
-                        enc = sys.getfilesystemencoding()
-                        if not enc:
-                            enc = 'utf8'
-                        if isinstance(path, unicode_type):
-                            path = path.encode(enc, 'replace')
                         if os.access(path, os.R_OK) and os.path.isfile(path):
                             if ext in ['png', 'jpg', 'bmp', 'jpeg']:
                                 self.process_image(path, tag_css)
@@ -1811,8 +1802,6 @@ class HTMLConverter(object):
 
 
 def process_file(path, options, logger):
-    if not isinstance(path, unicode_type):
-        path = path.decode(sys.getfilesystemencoding())
     path = os.path.abspath(path)
     default_title = filename_to_utf8(os.path.splitext(os.path.basename(path))[0])
     dirpath = os.path.dirname(path)
diff --git a/src/calibre/ebooks/lrf/meta.py b/src/calibre/ebooks/lrf/meta.py
index 49e944a70d..882762a78c 100644
--- a/src/calibre/ebooks/lrf/meta.py
+++ b/src/calibre/ebooks/lrf/meta.py
@@ -196,8 +196,8 @@ class xml_field(object):
 
         if not val:
             val = u''
-        if isinstance(val, unicode_type):
-            val = unicode_type(val, 'utf-8')
+        if not isinstance(val, unicode_type):
+            val = val.decode('utf-8')
 
         elems = document.getElementsByTagName(self.tag_name)
         elem = None
diff --git a/src/calibre/ebooks/lrf/objects.py b/src/calibre/ebooks/lrf/objects.py
index c23b1136ac..83895b8860 100644
--- a/src/calibre/ebooks/lrf/objects.py
+++ b/src/calibre/ebooks/lrf/objects.py
@@ -4,6 +4,7 @@ __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 import struct, array, zlib, cStringIO, collections, re
 
 from calibre.ebooks.lrf import LRFParseError, PRS500_PROFILE
+from calibre.constants import ispy3
 from calibre import entity_to_unicode, prepare_string_for_xml
 from calibre.ebooks.lrf.tags import Tag
 from polyglot.builtins import unicode_type
@@ -88,11 +89,8 @@ class LRFObject(object):
         for i in range(0):
             yield i
 
-    def __unicode__(self):
-        return unicode_type(self.__class__.__name__)
-
     def __str__(self):
-        return unicode_type(self).encode('utf-8')
+        return self.__class__.__name__
 
 
 class LRFContentObject(LRFObject):
@@ -204,12 +202,15 @@ class StyleObject(object):
                 s += u'%s="%s" '%(attr, getattr(self, attr))
         return s
 
-    def __unicode__(self):
+    def __str__(self):
         s = u'<%s objid="%s" stylelabel="%s" '%(self.__class__.__name__.replace('Attr', 'Style'), self.id, self.id)
         s += self._tags_to_xml()
         s += u'/>\n'
         return s
 
+    if not ispy3:
+        __unicode__ = __str__
+
     def as_dict(self):
         d = {}
         for h in self.tag_map.values():
@@ -252,11 +253,11 @@ class Color(object):
     def __init__(self, val):
         self.a, self.r, self.g, self.b = val & 0xFF, (val>>8)&0xFF, (val>>16)&0xFF, (val>>24)&0xFF
 
-    def __unicode__(self):
+    def __str__(self):
         return u'0x%02x%02x%02x%02x'%(self.a, self.r, self.g, self.b)
 
-    def __str__(self):
-        return unicode_type(self)
+    if not ispy3:
+        __unicode__ = __str__
 
     def __len__(self):
         return 4
@@ -284,10 +285,13 @@ class PageDiv(EmptyPageElement):
         self.pain, self.spacesize, self.linewidth = pain, spacesize, linewidth
         self.linecolor = Color(linecolor)
 
-    def __unicode__(self):
+    def __str__(self):
         return u'\n<PageDiv pain="%s" spacesize="%s" linewidth="%s" linecolor="%s" />\n'%\
                 (self.pain, self.spacesize, self.linewidth, self.color)
 
+    if not ispy3:
+        __unicode__ = __str__
+
 
 class RuledLine(EmptyPageElement):
 
@@ -299,19 +303,25 @@ class RuledLine(EmptyPageElement):
         self.linecolor = Color(linecolor)
         self.id = -1
 
-    def __unicode__(self):
+    def __str__(self):
         return u'\n<RuledLine linelength="%s" linetype="%s" linewidth="%s" linecolor="%s" />\n'%\
                 (self.linelength, self.linetype, self.linewidth, self.linecolor)
 
+    if not ispy3:
+        __unicode__ = __str__
+
 
 class Wait(EmptyPageElement):
 
     def __init__(self, time):
         self.time = time
 
-    def __unicode__(self):
+    def __str__(self):
         return u'\n<Wait time="%d" />\n'%(self.time)
 
+    if not ispy3:
+        __unicode__ = __str__
+
 
 class Locate(EmptyPageElement):
 
@@ -320,19 +330,25 @@ class Locate(EmptyPageElement):
     def __init__(self, pos):
         self.pos = self.pos_map[pos]
 
-    def __unicode__(self):
+    def __str__(self):
         return u'\n<Locate pos="%s" />\n'%(self.pos)
 
+    if not ispy3:
+        __unicode__ = __str__
+
 
 class BlockSpace(EmptyPageElement):
 
     def __init__(self, xspace, yspace):
         self.xspace, self.yspace = xspace, yspace
 
-    def __unicode__(self):
+    def __str__(self):
         return u'\n<BlockSpace xspace="%d" yspace="%d" />\n'%\
                 (self.xspace, self.yspace)
 
+    if not ispy3:
+        __unicode__ = __str__
+
 
 class Page(LRFStream):
     tag_map = {
@@ -427,15 +443,15 @@ class Page(LRFStream):
         for i in self.content:
             yield i
 
-    def __unicode__(self):
+    def __str__(self):
         s = u'\n<Page pagestyle="%d" objid="%d">\n'%(self.style_id, self.id)
         for i in self:
             s += unicode_type(i)
         s += '\n</Page>\n'
         return s
 
-    def __str__(self):
-        return unicode_type(self)
+    if not ispy3:
+        __unicode__ = __str__
 
     def to_html(self):
         s = u''
@@ -612,7 +628,7 @@ class Block(LRFStream, TextCSS):
             if hasattr(self, attr):
                 self.attrs[attr] = getattr(self, attr)
 
-    def __unicode__(self):
+    def __str__(self):
         s = u'\n<%s objid="%d" blockstyle="%d" '%(self.name, self.id, self.style_id)
         if hasattr(self, 'textstyle_id'):
             s += 'textstyle="%d" '%(self.textstyle_id,)
@@ -625,6 +641,9 @@ class Block(LRFStream, TextCSS):
             return s
         return s.rstrip() + ' />\n'
 
+    if not ispy3:
+        __unicode__ = __str__
+
     def to_html(self):
         if self.name == 'TextBlock':
             return u'<div class="block%s text%s">%s</div>'%(self.style_id, self.textstyle_id, self.content.to_html())
@@ -697,12 +716,15 @@ class Text(LRFStream):
             self.attrs = attrs
             self.self_closing = self_closing
 
-        def __unicode__(self):
+        def __str__(self):
             s = u'<%s '%(self.name,)
             for name, val in self.attrs.items():
                 s += '%s="%s" '%(name, val)
             return s.rstrip() + (u' />' if self.self_closing else u'>')
 
+        if not ispy3:
+            __unicode__ = __str__
+
         def to_html(self):
             s = u''
             return s
@@ -878,7 +900,7 @@ class Text(LRFStream):
             self.close_containers()
         self.stream = None
 
-    def __unicode__(self):
+    def __str__(self):
         s = u''
         open_containers = collections.deque()
         for c in self.content:
@@ -900,6 +922,9 @@ class Text(LRFStream):
                 raise LRFParseError('Malformed text stream %s'%([i.name for i in open_containers if isinstance(i, Text.TextTag)],))
         return s
 
+    if not ispy3:
+        __unicode__ = __str__
+
     def to_html(self):
         s = u''
         open_containers = collections.deque()
@@ -944,10 +969,13 @@ class Image(LRFObject):
     encoding = property(fget=lambda self : self._document.objects[self.refstream].encoding)
     data = property(fget=lambda self : self._document.objects[self.refstream].stream)
 
-    def __unicode__(self):
+    def __str__(self):
         return u'<Image objid="%s" x0="%d" y0="%d" x1="%d" y1="%d" xsize="%d" ysize="%d" refstream="%d" />\n'%\
         (self.id, self.x0, self.y0, self.x1, self.y1, self.xsize, self.ysize, self.refstream)
 
+    if not ispy3:
+        __unicode__ = __str__
+
 
 class PutObj(EmptyPageElement):
 
@@ -955,9 +983,12 @@ class PutObj(EmptyPageElement):
         self.x1, self.y1, self.refobj = x1, y1, refobj
         self.object = objects[refobj]
 
-    def __unicode__(self):
+    def __str__(self):
         return u'<PutObj x1="%d" y1="%d" refobj="%d" />'%(self.x1, self.y1, self.refobj)
 
+    if not ispy3:
+        __unicode__ = __str__
+
 
 class Canvas(LRFStream):
     tag_map = {
@@ -996,7 +1027,7 @@ class Canvas(LRFStream):
             except struct.error:
                 print('Canvas object has errors, skipping.')
 
-    def __unicode__(self):
+    def __str__(self):
         s = '\n<%s objid="%s" '%(self.__class__.__name__, self.id,)
         for attr in self.attrs:
             s += '%s="%s" '%(attr, self.attrs[attr])
@@ -1006,6 +1037,9 @@ class Canvas(LRFStream):
         s += '</%s>\n'%(self.__class__.__name__,)
         return s
 
+    if not ispy3:
+        __unicode__ = __str__
+
     def __iter__(self):
         for i in self._contents:
             yield i
@@ -1039,10 +1073,13 @@ class ImageStream(LRFStream):
         if self._document is not None:
             self._document.image_map[self.id] = self
 
-    def __unicode__(self):
+    def __str__(self):
         return u'<ImageStream objid="%s" encoding="%s" file="%s" />\n'%\
             (self.id, self.encoding, self.file)
 
+    if not ispy3:
+        __unicode__ = __str__
+
 
 class Import(LRFStream):
     pass
@@ -1118,7 +1155,7 @@ class Button(LRFObject):
                 return i[1:][0]
         return (None, None)
 
-    def __unicode__(self):
+    def __str__(self):
         s = u'<Button objid="%s">\n'%(self.id,)
         if self.button_flags & 0x10 != 0:
             s += '<PushButton '
@@ -1132,6 +1169,9 @@ class Button(LRFObject):
         s += '</Button>\n'
         return s
 
+    if not ispy3:
+        __unicode__ = __str__
+
     refpage = property(fget=lambda self : self.jump_action(2)[0])
     refobj = property(fget=lambda self : self.jump_action(2)[1])
 
@@ -1192,7 +1232,7 @@ class BookAttr(StyleObject, LRFObject):
     def add_font(self, tag, f):
         self.font_link_list.append(tag.dword)
 
-    def __unicode__(self):
+    def __str__(self):
         s = u'<BookStyle objid="%s" stylelabel="%s">\n'%(self.id, self.id)
         s += u'<SetDefault %s />\n'%(self._tags_to_xml(),)
         doc = self._document
@@ -1203,6 +1243,9 @@ class BookAttr(StyleObject, LRFObject):
         s += '</BookStyle>\n'
         return s
 
+    if not ispy3:
+        __unicode__ = __str__
+
 
 class SimpleText(Text):
     pass
@@ -1213,9 +1256,12 @@ class TocLabel(object):
     def __init__(self, refpage, refobject, label):
         self.refpage, self.refobject, self.label = refpage, refobject, label
 
-    def __unicode__(self):
+    def __str__(self):
         return u'<TocLabel refpage="%s" refobj="%s">%s</TocLabel>\n'%(self.refpage, self.refobject, self.label)
 
+    if not ispy3:
+        __unicode__ = __str__
+
 
 class TOCObject(LRFStream):
 
@@ -1237,12 +1283,15 @@ class TOCObject(LRFStream):
         for i in self._contents:
             yield i
 
-    def __unicode__(self):
+    def __str__(self):
         s = u'<TOC>\n'
         for i in self:
             s += unicode_type(i)
         return s + '</TOC>\n'
 
+    if not ispy3:
+        __unicode__ = __str__
+
 
 object_map = [
     None,  # 00
diff --git a/src/calibre/ebooks/metadata/book/base.py b/src/calibre/ebooks/metadata/book/base.py
index 0175dbdb80..00f60a7298 100644
--- a/src/calibre/ebooks/metadata/book/base.py
+++ b/src/calibre/ebooks/metadata/book/base.py
@@ -8,7 +8,7 @@ __docformat__ = 'restructuredtext en'
 import copy, traceback
 
 from calibre import prints
-from calibre.constants import DEBUG
+from calibre.constants import DEBUG, ispy3
 from calibre.ebooks.metadata.book import (SC_COPYABLE_FIELDS,
         SC_FIELDS_COPY_NOT_NULL, STANDARD_METADATA_FIELDS,
         TOP_LEVEL_IDENTIFIERS, ALL_METADATA_FIELDS)
@@ -709,7 +709,7 @@ class Metadata(object):
 
         return (None, None, None, None)
 
-    def __unicode__(self):
+    def __unicode__representation__(self):
         '''
         A string representation of this object, suitable for printing to
         console
@@ -791,11 +791,17 @@ class Metadata(object):
             ans[i] = u'<tr><td><b>%s</b></td><td>%s</td></tr>'%x
         return u'<table>%s</table>'%u'\n'.join(ans)
 
-    def __str__(self):
-        return self.__unicode__().encode('utf-8')
+    if ispy3:
+        __str__ = __unicode__representation__
+    else:
+        __unicode__ = __unicode__representation__
+
+        def __str__(self):
+            return self.__unicode__().encode('utf-8')
 
     def __nonzero__(self):
         return bool(self.title or self.author or self.comments or self.tags)
+    __bool__ = __nonzero__
 
     # }}}
 
diff --git a/src/calibre/ebooks/metadata/fb2.py b/src/calibre/ebooks/metadata/fb2.py
index 949b76aae5..8889333944 100644
--- a/src/calibre/ebooks/metadata/fb2.py
+++ b/src/calibre/ebooks/metadata/fb2.py
@@ -115,7 +115,7 @@ def get_metadata(stream):
     if book_title:
         book_title = unicode_type(book_title)
     else:
-        book_title = force_unicode_type(os.path.splitext(
+        book_title = force_unicode(os.path.splitext(
             os.path.basename(getattr(stream, 'name',
                 _('Unknown'))))[0])
     mi = MetaInformation(book_title, authors)
diff --git a/src/calibre/ebooks/metadata/opf2.py b/src/calibre/ebooks/metadata/opf2.py
index 0af4a40724..35c41533f3 100644
--- a/src/calibre/ebooks/metadata/opf2.py
+++ b/src/calibre/ebooks/metadata/opf2.py
@@ -15,7 +15,7 @@ from urlparse import urlparse
 from lxml import etree
 
 from calibre.ebooks import escape_xpath_attr
-from calibre.constants import __appname__, __version__, filesystem_encoding
+from calibre.constants import __appname__, __version__, filesystem_encoding, ispy3
 from calibre.ebooks.metadata.toc import TOC
 from calibre.ebooks.metadata.utils import parse_opf, pretty_print_opf as _pretty_print
 from calibre.ebooks.metadata import string_to_authors, MetaInformation, check_isbn
@@ -73,7 +73,7 @@ class Resource(object):  # {{{
             path = href_or_path
             if not os.path.isabs(path):
                 path = os.path.abspath(os.path.join(basedir, path))
-            if isinstance(path, str):
+            if isinstance(path, bytes):
                 path = path.decode(sys.getfilesystemencoding())
             self.path = path
         else:
@@ -112,8 +112,8 @@ class Resource(object):  # {{{
             rpath = os.path.relpath(self.path, basedir)
         except ValueError:  # On windows path and basedir could be on different drives
             rpath = self.path
-        if isinstance(rpath, unicode_type):
-            rpath = rpath.encode('utf-8')
+        if isinstance(rpath, bytes):
+            rpath = rpath.decode(filesystem_encoding)
         return rpath.replace(os.sep, '/')+frag
 
     def set_basedir(self, path):
@@ -203,11 +203,16 @@ class ManifestItem(Resource):  # {{{
             self.mime_type = val
         return property(fget=fget, fset=fset)
 
-    def __unicode__(self):
+    def __unicode__representation__(self):
         return u'<item id="%s" href="%s" media-type="%s" />'%(self.id, self.href(), self.media_type)
 
-    def __str__(self):
-        return unicode_type(self).encode('utf-8')
+    if ispy3:
+        __str__ = __unicode__representation__
+    else:
+        __unicode__ = __unicode__representation__
+
+        def __str__(self):
+            return unicode_type(self).encode('utf-8')
 
     def __repr__(self):
         return unicode_type(self)
diff --git a/src/calibre/ebooks/mobi/mobiml.py b/src/calibre/ebooks/mobi/mobiml.py
index 16e3926084..4db55b204e 100644
--- a/src/calibre/ebooks/mobi/mobiml.py
+++ b/src/calibre/ebooks/mobi/mobiml.py
@@ -229,7 +229,9 @@ class MobiMLizer(object):
                 while vspace > 0:
                     wrapper.addprevious(etree.Element(XHTML('br')))
                     vspace -= 1
-            if istate.halign != 'auto' and isinstance(istate.halign, (str, unicode_type)):
+            if istate.halign != 'auto' and isinstance(istate.halign, (bytes, unicode_type)):
+                if isinstance(istate.halign, bytes):
+                    istate.halign = istate.halign.decode('utf-8')
                 para.attrib['align'] = istate.halign
         istate.rendered = True
         pstate = bstate.istate
@@ -568,17 +570,17 @@ class MobiMLizer(object):
             self.opts.mobi_ignore_margins = False
 
         if (text or tag in CONTENT_TAGS or tag in NESTABLE_TAGS or (
-            # We have an id but no text and no children, the id should still
-            # be added.
-            istate.ids and tag in ('a', 'span', 'i', 'b', 'u') and
-            len(elem)==0)):
-                if tag == 'li' and len(istates) > 1 and 'value' in elem.attrib:
-                    try:
-                        value = int(elem.attrib['value'])
-                        istates[-2].list_num = value - 1
-                    except:
-                        pass
-                self.mobimlize_content(tag, text, bstate, istates)
+                # We have an id but no text and no children, the id should still
+                # be added.
+                istate.ids and tag in ('a', 'span', 'i', 'b', 'u') and
+                len(elem)==0)):
+            if tag == 'li' and len(istates) > 1 and 'value' in elem.attrib:
+                try:
+                    value = int(elem.attrib['value'])
+                    istates[-2].list_num = value - 1
+                except:
+                    pass
+            self.mobimlize_content(tag, text, bstate, istates)
         for child in elem:
             self.mobimlize_elem(child, stylizer, bstate, istates)
             tail = None
diff --git a/src/calibre/ebooks/mobi/reader/mobi6.py b/src/calibre/ebooks/mobi/reader/mobi6.py
index 16b8a938b6..5f309dd364 100644
--- a/src/calibre/ebooks/mobi/reader/mobi6.py
+++ b/src/calibre/ebooks/mobi/reader/mobi6.py
@@ -283,24 +283,29 @@ class MobiReader(object):
                     ref.attrib['href'] = os.path.basename(htmlfile) + ref.attrib['href']
         except AttributeError:
             pass
+
+        def write_as_utf8(path, data):
+            if isinstance(data, unicode_type):
+                data = data.encode('utf-8')
+            with lopen(path, 'wb') as f:
+                f.write(data)
+
         parse_cache[htmlfile] = root
         self.htmlfile = htmlfile
         ncx = cStringIO.StringIO()
         opf, ncx_manifest_entry = self.create_opf(htmlfile, guide, root)
         self.created_opf_path = os.path.splitext(htmlfile)[0] + '.opf'
-        opf.render(open(self.created_opf_path, 'wb'), ncx,
+        opf.render(lopen(self.created_opf_path, 'wb'), ncx,
             ncx_manifest_entry=ncx_manifest_entry)
         ncx = ncx.getvalue()
         if ncx:
             ncx_path = os.path.join(os.path.dirname(htmlfile), 'toc.ncx')
-            open(ncx_path, 'wb').write(ncx)
+            write_as_utf8(ncx_path, ncx)
 
-        with open('styles.css', 'wb') as s:
-            s.write(self.base_css_rules + '\n\n')
-            for cls, rule in self.tag_css_rules.items():
-                if isinstance(rule, unicode_type):
-                    rule = rule.encode('utf-8')
-                s.write('.%s { %s }\n\n' % (cls, rule))
+        css = [self.base_css_rules, '\n\n']
+        for cls, rule in self.tag_css_rules.items():
+            css.append('.%s { %s }\n\n' % (cls, rule))
+        write_as_utf8('styles.css', ''.join(css))
 
         if self.book_header.exth is not None or self.embedded_mi is not None:
             self.log.debug('Creating OPF...')
@@ -310,7 +315,7 @@ class MobiReader(object):
                 ncx_manifest_entry)
             ncx = ncx.getvalue()
             if ncx:
-                open(os.path.splitext(htmlfile)[0] + '.ncx', 'wb').write(ncx)
+                write_as_utf8(os.path.splitext(htmlfile)[0] + '.ncx', ncx)
 
     def read_embedded_metadata(self, root, elem, guide):
         raw = '<?xml version="1.0" encoding="utf-8" ?>\n<package>' + \
@@ -423,24 +428,25 @@ class MobiReader(object):
                     styles.append(style)
             if 'height' in attrib:
                 height = attrib.pop('height').strip()
-                if height and '<' not in height and '>' not in height and \
-                    re.search(r'\d+', height):
-                        if tag.tag in ('table', 'td', 'tr'):
-                            pass
-                        elif tag.tag == 'img':
-                            tag.set('height', height)
+                if (
+                        height and '<' not in height and '>' not in height and
+                        re.search(r'\d+', height)):
+                    if tag.tag in ('table', 'td', 'tr'):
+                        pass
+                    elif tag.tag == 'img':
+                        tag.set('height', height)
+                    else:
+                        if tag.tag == 'div' and not tag.text and \
+                                (not tag.tail or not tag.tail.strip()) and \
+                                not len(list(tag.iterdescendants())):
+                            # Paragraph spacer
+                            # Insert nbsp so that the element is never
+                            # discarded by a renderer
+                            tag.text = u'\u00a0'  # nbsp
+                            styles.append('height: %s' %
+                                    self.ensure_unit(height))
                         else:
-                            if tag.tag == 'div' and not tag.text and \
-                                    (not tag.tail or not tag.tail.strip()) and \
-                                    not len(list(tag.iterdescendants())):
-                                # Paragraph spacer
-                                # Insert nbsp so that the element is never
-                                # discarded by a renderer
-                                tag.text = u'\u00a0'  # nbsp
-                                styles.append('height: %s' %
-                                        self.ensure_unit(height))
-                            else:
-                                styles.append('margin-top: %s' % self.ensure_unit(height))
+                            styles.append('margin-top: %s' % self.ensure_unit(height))
             if 'width' in attrib:
                 width = attrib.pop('width').strip()
                 if width and re.search(r'\d+', width):
@@ -837,11 +843,10 @@ class MobiReader(object):
             anchor = '<a id="filepos%d"></a>'
             if r > -1 and (r < l or l == end or l == -1):
                 p = self.mobi_html.rfind('<', 0, end + 1)
-                if pos < end and p > -1 and \
-                    not end_tag_re.match(self.mobi_html[p:r]) and \
-                    not self.mobi_html[p:r + 1].endswith('/>'):
-                        anchor = ' filepos-id="filepos%d"'
-                        end = r
+                if (pos < end and p > -1 and not end_tag_re.match(self.mobi_html[p:r]) and
+                        not self.mobi_html[p:r + 1].endswith('/>')):
+                    anchor = ' filepos-id="filepos%d"'
+                    end = r
                 else:
                     end = r + 1
             processed_html.write(self.mobi_html[pos:end] + (anchor % oend))
diff --git a/src/calibre/ebooks/mobi/writer2/serializer.py b/src/calibre/ebooks/mobi/writer2/serializer.py
index ab6551d522..c86dfff2be 100644
--- a/src/calibre/ebooks/mobi/writer2/serializer.py
+++ b/src/calibre/ebooks/mobi/writer2/serializer.py
@@ -1,23 +1,32 @@
 #!/usr/bin/env python2
 # vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
-from __future__ import (unicode_literals, division, absolute_import,
-                        print_function)
+from __future__ import absolute_import, division, print_function, unicode_literals
 
 __license__   = 'GPL v3'
 __copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import re, unicodedata
 
-from calibre.ebooks.oeb.base import (OEB_DOCS, XHTML, XHTML_NS, XML_NS,
-        namespace, prefixname, urlnormalize)
+import re
+import unicodedata
+from collections import defaultdict
+from io import BytesIO
+from urlparse import urldefrag
+
 from calibre.ebooks.mobi.mobiml import MBP_NS
 from calibre.ebooks.mobi.utils import is_guide_ref_start
+from calibre.ebooks.oeb.base import (
+    OEB_DOCS, XHTML, XHTML_NS, XML_NS, namespace, prefixname, urlnormalize
+)
 from polyglot.builtins import unicode_type
 
-from collections import defaultdict
-from urlparse import urldefrag
-from cStringIO import StringIO
+
+class Buf(BytesIO):
+
+    def write(self, x):
+        if isinstance(x, unicode_type):
+            x = x.encode('utf-8')
+        BytesIO.write(self, x)
 
 
 class Serializer(object):
@@ -116,7 +125,7 @@ class Serializer(object):
         '''
         Return the document serialized as a single UTF-8 encoded bytestring.
         '''
-        buf = self.buf = StringIO()
+        buf = self.buf = Buf()
         buf.write(b'<html>')
         self.serialize_head()
         self.serialize_body()
@@ -214,22 +223,22 @@ class Serializer(object):
             # if href is provided add a link ref to the toc level output (e.g. feed_0/index.html)
             if href is not None:
                 # resolve the section url in id_offsets
-                buf.write('<mbp:pagebreak />')
+                buf.write(b'<mbp:pagebreak />')
                 self.id_offsets[urlnormalize(href)] = buf.tell()
 
             if tocref.klass == "periodical":
-                buf.write('<div> <div height="1em"></div>')
+                buf.write(b'<div> <div height="1em"></div>')
             else:
                 t = tocref.title
                 if isinstance(t, unicode_type):
                     t = t.encode('utf-8')
-                buf.write('<div></div> <div> <h2 height="1em"><font size="+2"><b>' + t +
-                          '</b></font></h2> <div height="1em"></div>')
+                buf.write(b'<div></div> <div> <h2 height="1em"><font size="+2"><b>' + t +
+                          b'</b></font></h2> <div height="1em"></div>')
 
-            buf.write('<ul>')
+            buf.write(b'<ul>')
 
             for tocitem in tocref.nodes:
-                buf.write('<li><a filepos=')
+                buf.write(b'<li><a filepos=')
                 itemhref = tocitem.href
                 if tocref.klass == 'periodical':
                     # This is a section node.
@@ -238,15 +247,15 @@ class Serializer(object):
                     # so we change the href.
                     itemhref = re.sub(r'article_\d+/', '', itemhref)
                 self.href_offsets[itemhref].append(buf.tell())
-                buf.write('0000000000')
-                buf.write(' ><font size="+1"><b><u>')
+                buf.write(b'0000000000')
+                buf.write(b' ><font size="+1"><b><u>')
                 t = tocitem.title
                 if isinstance(t, unicode_type):
                     t = t.encode('utf-8')
                 buf.write(t)
-                buf.write('</u></b></font></a></li>')
+                buf.write(b'</u></b></font></a></li>')
 
-            buf.write('</ul><div height="1em"></div></div><mbp:pagebreak />')
+            buf.write(b'</ul><div height="1em"></div></div><mbp:pagebreak />')
 
         self.anchor_offset = buf.tell()
         buf.write(b'<body>')
@@ -350,7 +359,7 @@ class Serializer(object):
                 if child.tail:
                     self.anchor_offset = None
                     self.serialize_text(child.tail)
-        buf.write(b'</%s>' % tag.encode('utf-8'))
+        buf.write(('</%s>' % tag).encode('utf-8'))
 
     def serialize_text(self, text, quot=False):
         text = text.replace('&', '&amp;')
@@ -384,4 +393,4 @@ class Serializer(object):
                     self.start_offset = ioff
                 for hoff in hoffs:
                     buf.seek(hoff)
-                    buf.write(b'%010d' % ioff)
+                    buf.write(('%010d' % ioff).encode('utf-8'))
diff --git a/src/calibre/ebooks/oeb/base.py b/src/calibre/ebooks/oeb/base.py
index d6a7013f57..9c16e54ee4 100644
--- a/src/calibre/ebooks/oeb/base.py
+++ b/src/calibre/ebooks/oeb/base.py
@@ -13,7 +13,7 @@ from urlparse import urldefrag, urlparse, urlunparse, urljoin
 from urllib import unquote
 
 from lxml import etree, html
-from calibre.constants import filesystem_encoding, __version__
+from calibre.constants import filesystem_encoding, __version__, ispy3
 from calibre.translations.dynamic import translate
 from calibre.ebooks.chardet import xml_to_unicode
 from calibre.ebooks.conversion.preprocess import CSSPreProcessor
@@ -107,13 +107,35 @@ self_closing_bad_tags = {'a', 'abbr', 'address', 'article', 'aside', 'audio', 'b
 'span', 'strong', 'sub', 'summary', 'sup', 'textarea', 'time', 'ul', 'var',
 'video', 'title', 'script', 'style'}
 
-_self_closing_pat = re.compile(
-    r'<(?P<tag>%s)(?=[\s/])(?P<arg>[^>]*)/>'%('|'.join(self_closing_bad_tags)),
-    re.IGNORECASE)
+
+def as_string_type(pat, for_unicode):
+    if for_unicode:
+        if isinstance(pat, bytes):
+            pat = pat.decode('utf-8')
+    else:
+        if isinstance(pat, unicode_type):
+            pat = pat.encode('utf-8')
+    return pat
+
+
+def self_closing_pat(for_unicode):
+    attr = 'unicode_ans' if for_unicode else 'bytes_ans'
+    ans = getattr(self_closing_pat, attr, None)
+    if ans is None:
+        sub = '|'.join(self_closing_bad_tags)
+        template = r'<(?P<tag>%s)(?=[\s/])(?P<arg>[^>]*)/>'
+        pat = template % sub
+        pat = as_string_type(pat, for_unicode)
+        ans = re.compile(pat, flags=re.IGNORECASE)
+        setattr(self_closing_pat, attr, ans)
+    return ans
 
 
 def close_self_closing_tags(raw):
-    return _self_closing_pat.sub(r'<\g<tag>\g<arg>></\g<tag>>', raw)
+    for_unicode = isinstance(raw, unicode_type)
+    repl = as_string_type(r'<\g<tag>\g<arg>></\g<tag>>', for_unicode)
+    pat = self_closing_pat(for_unicode)
+    return pat.sub(repl, raw)
 
 
 def uuid_id():
@@ -745,11 +767,15 @@ class Metadata(object):
             return 'Item(term=%r, value=%r, attrib=%r)' \
                 % (barename(self.term), self.value, self.attrib)
 
-        def __str__(self):
-            return unicode_type(self.value).encode('ascii', 'xmlcharrefreplace')
+        if ispy3:
+            def __str__(self):
+                return as_unicode(self.value)
+        else:
+            def __str__(self):
+                return unicode_type(self.value).encode('ascii', 'xmlcharrefreplace')
 
-        def __unicode__(self):
-            return as_unicode(self.value)
+            def __unicode__(self):
+                return as_unicode(self.value)
 
         def to_opf1(self, dcmeta=None, xmeta=None, nsrmap={}):
             attrib = {}
@@ -1075,19 +1101,27 @@ class Manifest(object):
                     self._loader = loader2
                 self._data = None
 
-        def __str__(self):
-            return serialize(self.data, self.media_type, pretty_print=self.oeb.pretty_print)
-
-        def __unicode__(self):
+        @property
+        def unicode_representation(self):
             data = self.data
             if isinstance(data, etree._Element):
                 return xml2unicode(data, pretty_print=self.oeb.pretty_print)
             if isinstance(data, unicode_type):
                 return data
             if hasattr(data, 'cssText'):
-                return data.cssText
+                return unicode_type(data.cssText, 'utf-8', 'replace')
             return unicode_type(data)
 
+        if ispy3:
+            def __str__(self):
+                return self.unicode_representation
+        else:
+            def __unicode__(self):
+                return self.unicode_representation
+
+            def __str__(self):
+                return serialize(self.data, self.media_type, pretty_print=self.oeb.pretty_print)
+
         def __eq__(self, other):
             return id(self) == id(other)
 
@@ -1616,11 +1650,15 @@ class TOC(object):
             ans.extend(child.get_lines(lvl+1))
         return ans
 
-    def __str__(self):
-        return b'\n'.join([x.encode('utf-8') for x in self.get_lines()])
+    if ispy3:
+        def __str__(self):
+            return u'\n'.join(self.get_lines())
+    else:
+        def __unicode__(self):
+            return u'\n'.join(self.get_lines())
 
-    def __unicode__(self):
-        return u'\n'.join(self.get_lines())
+        def __str__(self):
+            return b'\n'.join([x.encode('utf-8') for x in self.get_lines()])
 
     def to_opf1(self, tour):
         for node in self.nodes:
diff --git a/src/calibre/ebooks/oeb/iterator/spine.py b/src/calibre/ebooks/oeb/iterator/spine.py
index f81a2244d0..0159970662 100644
--- a/src/calibre/ebooks/oeb/iterator/spine.py
+++ b/src/calibre/ebooks/oeb/iterator/spine.py
@@ -53,7 +53,7 @@ class SpineItem(unicode_type):
         if not os.path.exists(path) and os.path.exists(ppath):
             path = ppath
         obj = super(SpineItem, cls).__new__(cls, path)
-        with open(path, 'rb') as f:
+        with lopen(path, 'rb') as f:
             raw = f.read()
         if from_epub:
             # According to the spec, HTML in EPUB must be encoded in utf-8 or
diff --git a/src/calibre/ebooks/oeb/parse_utils.py b/src/calibre/ebooks/oeb/parse_utils.py
index 94b369471e..2dd153799d 100644
--- a/src/calibre/ebooks/oeb/parse_utils.py
+++ b/src/calibre/ebooks/oeb/parse_utils.py
@@ -99,7 +99,7 @@ def html5_parse(data, max_nesting_depth=100):
     # Check that the asinine HTML 5 algorithm did not result in a tree with
     # insane nesting depths
     for x in data.iterdescendants():
-        if isinstance(x.tag, basestring) and len(x) is 0:  # Leaf node
+        if isinstance(x.tag, basestring) and not len(x):  # Leaf node
             depth = node_depth(x)
             if depth > max_nesting_depth:
                 raise ValueError('HTML 5 parsing resulted in a tree with nesting'
@@ -259,7 +259,7 @@ def parse_html(data, log=None, decoder=None, preprocessor=None,
         nroot = etree.fromstring('<html></html>')
         has_body = False
         for child in list(data):
-            if isinstance(child.tag, (unicode_type, str)) and barename(child.tag) == 'body':
+            if isinstance(child.tag, (unicode_type, bytes)) and barename(child.tag) == 'body':
                 has_body = True
                 break
         parent = nroot
diff --git a/src/calibre/ebooks/oeb/stylizer.py b/src/calibre/ebooks/oeb/stylizer.py
index 1e7aaf6343..0b4b825657 100644
--- a/src/calibre/ebooks/oeb/stylizer.py
+++ b/src/calibre/ebooks/oeb/stylizer.py
@@ -607,12 +607,12 @@ class Style(object):
                 result = base
             else:
                 result = self._unit_convert(width, base=base)
-            if isinstance(result, (unicode_type, str, bytes)):
+            if isinstance(result, (unicode_type, bytes)):
                 result = self._profile.width
             self._width = result
             if 'max-width' in self._style:
                 result = self._unit_convert(self._style['max-width'], base=base)
-                if isinstance(result, (unicode_type, str, bytes)):
+                if isinstance(result, (unicode_type, bytes)):
                     result = self._width
                 if result < self._width:
                     self._width = result
@@ -644,12 +644,12 @@ class Style(object):
                 result = base
             else:
                 result = self._unit_convert(height, base=base)
-            if isinstance(result, (unicode_type, str, bytes)):
+            if isinstance(result, (unicode_type, bytes)):
                 result = self._profile.height
             self._height = result
             if 'max-height' in self._style:
                 result = self._unit_convert(self._style['max-height'], base=base)
-                if isinstance(result, (unicode_type, str, bytes)):
+                if isinstance(result, (unicode_type, bytes)):
                     result = self._height
                 if result < self._height:
                     self._height = result
diff --git a/src/calibre/ebooks/txt/processor.py b/src/calibre/ebooks/txt/processor.py
index b21d342aab..e2b272c54b 100644
--- a/src/calibre/ebooks/txt/processor.py
+++ b/src/calibre/ebooks/txt/processor.py
@@ -15,7 +15,7 @@ from calibre.ebooks.metadata.opf2 import OPFCreator
 
 from calibre.ebooks.conversion.preprocess import DocAnalysis
 from calibre.utils.cleantext import clean_ascii_chars
-from polyglot.builtins import unicode_type
+from polyglot.builtins import unicode_type, map, range
 
 HTML_TEMPLATE = u'<html><head><meta http-equiv="Content-Type" content="text/html; charset=utf-8"/><title>%s </title></head><body>\n%s\n</body></html>'
 
@@ -55,7 +55,7 @@ def split_txt(txt, epub_split_size_kb=0):
     result in the entire document being one giant
     paragraph. In this case the EPUB parser will not
     be able to determine where to split the file
-    to accomidate the EPUB file size limitation
+    to accommodate the EPUB file size limitation
     and will fail.
     '''
     # Takes care if there is no point to split
@@ -66,9 +66,12 @@ def split_txt(txt, epub_split_size_kb=0):
         # Calculating the average chunk value for easy splitting as EPUB (+2 as a safe margin)
         chunk_size = long(length_byte / (int(length_byte / (epub_split_size_kb * 1024)) + 2))
         # if there are chunks with a superior size then go and break
-        if (len(filter(lambda x: len(x) > chunk_size, txt.split('\n\n')))) :
-            txt = '\n\n'.join([split_string_separator(line, chunk_size)
-                for line in txt.split('\n\n')])
+        parts = txt.split(b'\n\n')
+        lengths = tuple(map(len, parts))
+        if lengths and max(lengths) > chunk_size:
+            txt = b'\n\n'.join([
+                split_string_separator(line, chunk_size) for line in parts
+            ])
     if isbytestring(txt):
         txt = txt.decode('utf-8')
 
@@ -227,7 +230,7 @@ def opf_writer(path, opf_name, manifest, spine, mi):
     opf = OPFCreator(path, mi)
     opf.create_manifest(manifest)
     opf.create_spine(spine)
-    with open(os.path.join(path, opf_name), 'wb') as opffile:
+    with lopen(os.path.join(path, opf_name), 'wb') as opffile:
         opf.render(opffile)
 
 
@@ -236,9 +239,16 @@ def split_string_separator(txt, size):
     Splits the text by putting \n\n at the point size.
     '''
     if len(txt) > size:
-        txt = ''.join([re.sub(type(u'')(r'\.(?P<ends>[^.]*)$'), r'.\n\n\g<ends>',
-            txt[i:i+size], 1) for i in
-            xrange(0, len(txt), size)])
+        size -= 2
+        txt = []
+        for part in (txt[i * size: (i + 1) * size] for i in range(0, len(txt), size)):
+            idx = part.rfind('.')
+            if idx == -1:
+                part += b'\n\n'
+            else:
+                part = part[:idx + 1] + b'\n\n' + part[idx:]
+            txt.append(part)
+        txt = b''.join(txt)
     return txt
 
 
diff --git a/src/calibre/ebooks/unihandecode/__init__.py b/src/calibre/ebooks/unihandecode/__init__.py
index 1ce7d8e5d2..153d8221fe 100644
--- a/src/calibre/ebooks/unihandecode/__init__.py
+++ b/src/calibre/ebooks/unihandecode/__init__.py
@@ -19,8 +19,6 @@ Tranliterate the string from unicode characters to ASCII in Chinese and others.
 '''
 import unicodedata
 
-from calibre.constants import ispy3
-
 
 class Unihandecoder(object):
     preferred_encoding = None
@@ -43,15 +41,11 @@ class Unihandecoder(object):
             self.decoder = Unidecoder()
 
     def decode(self, text):
-        if not ispy3:
-            if not isinstance(text, unicode):
-                try:
-                    text = unicode(text)
-                except:
-                    try:
-                        text = text.decode(self.preferred_encoding)
-                    except:
-                        text = text.decode('utf-8', 'replace')
+        if isinstance(text, bytes):
+            try:
+                text = text.decode(self.preferred_encoding)
+            except Exception:
+                text = text.decode('utf-8', 'replace')
         # at first unicode normalize it. (see Unicode standards)
         ntext = unicodedata.normalize('NFKC', text)
         return self.decoder.decode(ntext)
diff --git a/src/calibre/ebooks/unihandecode/pykakasi/jisyo.py b/src/calibre/ebooks/unihandecode/pykakasi/jisyo.py
index 6101817a7b..6ff55ad90d 100644
--- a/src/calibre/ebooks/unihandecode/pykakasi/jisyo.py
+++ b/src/calibre/ebooks/unihandecode/pykakasi/jisyo.py
@@ -4,7 +4,7 @@
 # Copyright 2011 Hiroshi Miura <miurahr@linux.com>
 from zlib import decompress
 
-from calibre.constants import ispy3
+from polyglot.builtins import unicode_type
 
 
 class jisyo (object):
@@ -34,8 +34,8 @@ class jisyo (object):
                 P('localization/pykakasi/kanadict2.calibre_msgpack', data=True))
 
     def load_jisyo(self, char):
-        if not ispy3:
-            char = unicode(char)
+        if not isinstance(char, unicode_type):
+            char = unicode_type(char, 'utf-8')
         key = "%04x"%ord(char)
 
         try:  # already exist?
diff --git a/src/calibre/ebooks/unihandecode/unidecoder.py b/src/calibre/ebooks/unihandecode/unidecoder.py
index 74b1229ede..748fcb64f1 100644
--- a/src/calibre/ebooks/unihandecode/unidecoder.py
+++ b/src/calibre/ebooks/unihandecode/unidecoder.py
@@ -60,9 +60,9 @@ it under the same terms as Perl itself.
 '''
 
 import re
-from calibre.constants import ispy3
 from calibre.ebooks.unihandecode.unicodepoints import CODEPOINTS
 from calibre.ebooks.unihandecode.zhcodepoints import CODEPOINTS as HANCODES
+from polyglot.builtins import unicode_type
 
 
 class Unidecoder(object):
@@ -95,8 +95,8 @@ class Unidecoder(object):
         Find what group character is a part of.
         '''
         # Code groups withing CODEPOINTS take the form 'xAB'
-        if not ispy3:
-            character = unicode(character)
+        if not isinstance(character, unicode_type):
+            character = unicode_type(character, "utf-8")
         return 'x%02x' % (ord(character) >> 8)
 
     def grouped_point(self, character):
@@ -104,6 +104,6 @@ class Unidecoder(object):
         Return the location the replacement character is in the list for a
         the group character is a part of.
         '''
-        if not ispy3:
-            character = unicode(character)
+        if not isinstance(character, unicode_type):
+            character = unicode_type(character, "utf-8")
         return ord(character) & 255
diff --git a/src/calibre/gui2/comments_editor.py b/src/calibre/gui2/comments_editor.py
index 300f024669..b5c9a73606 100644
--- a/src/calibre/gui2/comments_editor.py
+++ b/src/calibre/gui2/comments_editor.py
@@ -347,7 +347,7 @@ class EditorWidget(QWebView, LineEditECM):  # {{{
                     return unicode_type(self.page().mainFrame().toHtml())
                 check = unicode_type(self.page().mainFrame().toPlainText()).strip()
                 raw = unicode_type(self.page().mainFrame().toHtml())
-                raw = xml_to_unicode_type(raw, strip_encoding_pats=True,
+                raw = xml_to_unicode(raw, strip_encoding_pats=True,
                                     resolve_entities=True)[0]
                 raw = self.comments_pat.sub('', raw)
                 if not check and '<img' not in raw.lower():
diff --git a/src/calibre/gui2/dbus_export/gtk.py b/src/calibre/gui2/dbus_export/gtk.py
index 2bb1af75b6..f92b1d2a9d 100644
--- a/src/calibre/gui2/dbus_export/gtk.py
+++ b/src/calibre/gui2/dbus_export/gtk.py
@@ -201,7 +201,7 @@ class MenuExampleWindow(Gtk.ApplicationWindow):
 
 
 def convert(v):
-    if isinstance(v, basestring):
+    if isinstance(v, (unicode_type, bytes)):
         return unicode_type(v)
     if isinstance(v, dbus.Struct):
         return tuple(convert(val) for val in v)
@@ -309,6 +309,7 @@ class MyApplication(Gtk.Application):
     def do_startup(self):
         Gtk.Application.do_startup(self)
 
+
 app = MyApplication(application_id='com.calibre-ebook.test-gtk')
 signal.signal(signal.SIGINT, signal.SIG_DFL)
 sys.exit(app.run(sys.argv))
diff --git a/src/calibre/gui2/dialogs/quickview.py b/src/calibre/gui2/dialogs/quickview.py
index 8514408eeb..0be1e162ac 100644
--- a/src/calibre/gui2/dialogs/quickview.py
+++ b/src/calibre/gui2/dialogs/quickview.py
@@ -43,7 +43,7 @@ class TableItem(QTableWidgetItem):
             # self is not None and other is None therefore self >= other
             return True
 
-        if isinstance(self.sort, (str, unicode_type)):
+        if isinstance(self.sort, (bytes, unicode_type)):
             l = sort_key(self.sort)
             r = sort_key(other.sort)
         else:
@@ -66,7 +66,7 @@ class TableItem(QTableWidgetItem):
             # self is not None therefore self > other
             return False
 
-        if isinstance(self.sort, (str, unicode_type)):
+        if isinstance(self.sort, (bytes, unicode_type)):
             l = sort_key(self.sort)
             r = sort_key(other.sort)
         else:
diff --git a/src/calibre/gui2/init.py b/src/calibre/gui2/init.py
index 03faa4c4f0..e03c35b7a2 100644
--- a/src/calibre/gui2/init.py
+++ b/src/calibre/gui2/init.py
@@ -13,7 +13,7 @@ from PyQt5.Qt import (Qt, QApplication, QStackedWidget, QMenu, QTimer,
 
 from calibre.utils.config import prefs
 from calibre.utils.icu import sort_key
-from calibre.constants import (isosx, __appname__, preferred_encoding,
+from calibre.constants import (__appname__, preferred_encoding,
     get_version)
 from calibre.gui2 import config, is_widescreen, gprefs, error_dialog, open_url
 from calibre.gui2.library.views import BooksView, DeviceBooksView
@@ -323,11 +323,6 @@ class StatusBar(QStatusBar):  # {{{
     def show_message(self, msg, timeout=0, show_notification=True):
         self.showMessage(msg, timeout)
         if self.notifier is not None and not config['disable_tray_notification'] and show_notification:
-            if isosx and isinstance(msg, unicode_type):
-                try:
-                    msg = msg.encode(preferred_encoding)
-                except UnicodeEncodeError:
-                    msg = msg.encode('utf-8')
             self.notifier(msg)
 
     def clear_message(self):
diff --git a/src/calibre/gui2/layout.py b/src/calibre/gui2/layout.py
index 6c902fb9c8..f4a27c85af 100644
--- a/src/calibre/gui2/layout.py
+++ b/src/calibre/gui2/layout.py
@@ -129,7 +129,7 @@ class LocationManager(QObject):  # {{{
         had_device = self.has_device
         if cp is None:
             cp = (None, None)
-        if isinstance(cp, (str, unicode_type)):
+        if isinstance(cp, (bytes, unicode_type)):
             cp = (cp, None)
         if len(fs) < 3:
             fs = list(fs) + [0]
diff --git a/src/calibre/gui2/lrf_renderer/main.py b/src/calibre/gui2/lrf_renderer/main.py
index 985babf440..5aab2b3155 100644
--- a/src/calibre/gui2/lrf_renderer/main.py
+++ b/src/calibre/gui2/lrf_renderer/main.py
@@ -6,7 +6,7 @@ import sys, logging, os, traceback, time
 from PyQt5.Qt import (
     QKeySequence, QPainter, QDialog, QSpinBox, QSlider, QIcon, Qt, QCoreApplication, QThread, QScrollBar)
 
-from calibre import __appname__, setup_cli_handlers, islinux, isbsd
+from calibre import __appname__, setup_cli_handlers, islinux, isbsd, as_unicode
 from calibre.ebooks.lrf.lrfparser import LRFDocument
 
 from calibre.gui2 import error_dialog, \
@@ -17,7 +17,6 @@ from calibre.gui2.lrf_renderer.config_ui import Ui_ViewerConfig
 from calibre.gui2.main_window import MainWindow
 from calibre.gui2.lrf_renderer.document import Document
 from calibre.gui2.search_box import SearchBox2
-from polyglot.builtins import unicode_type
 
 
 class RenderWorker(QThread):
@@ -201,7 +200,7 @@ class Main(MainWindow, Ui_MainWindow):
             print('Error rendering document', file=sys.stderr)
             print(exception, file=sys.stderr)
             print(self.renderer.formatted_traceback, file=sys.stderr)
-            msg =  u'<p><b>%s</b>: '%(exception.__class__.__name__,) + unicode_type(str(exception), 'utf8', 'replace') + u'</p>'
+            msg =  u'<p><b>%s</b>: '%(exception.__class__.__name__,) + as_unicode(exception) + u'</p>'
             msg += u'<p>Failed to render document</p>'
             msg += u'<p>Detailed <b>traceback</b>:<pre>'
             msg += self.renderer.formatted_traceback + '</pre>'
diff --git a/src/calibre/gui2/main.py b/src/calibre/gui2/main.py
index 99bcc5c4b4..24a1ec8326 100644
--- a/src/calibre/gui2/main.py
+++ b/src/calibre/gui2/main.py
@@ -132,7 +132,7 @@ def get_default_library_path():
         fname = 'Calibre Library'
     if isinstance(fname, unicode_type):
         try:
-            fname = fname.encode(filesystem_encoding)
+            fname.encode(filesystem_encoding)
         except:
             fname = 'Calibre Library'
     x = os.path.expanduser('~'+os.sep+fname)
diff --git a/src/calibre/gui2/main_window.py b/src/calibre/gui2/main_window.py
index be4c90c7a8..80ee19e1d5 100644
--- a/src/calibre/gui2/main_window.py
+++ b/src/calibre/gui2/main_window.py
@@ -5,14 +5,14 @@ __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 
 
-import StringIO, traceback, sys, gc, weakref
+import traceback, sys, gc, weakref
+from io import BytesIO
 
 from PyQt5.Qt import (QMainWindow, QTimer, QAction, QMenu, QMenuBar, QIcon,
                       QObject)
 from calibre.utils.config import OptionParser
 from calibre.gui2 import error_dialog
-from calibre import prints
-from polyglot.builtins import unicode_type
+from calibre import prints, force_unicode
 
 
 def option_parser(usage='''\
@@ -134,7 +134,7 @@ class MainWindow(QMainWindow):
         if type is KeyboardInterrupt:
             return
         try:
-            sio = StringIO.StringIO()
+            sio = BytesIO()
             try:
                 from calibre.debug import print_basic_debug_info
                 print_basic_debug_info(out=sio)
@@ -145,7 +145,8 @@ class MainWindow(QMainWindow):
                 prints(value.locking_debug_msg, file=sio)
             fe = sio.getvalue()
             prints(fe, file=sys.stderr)
-            msg = '<b>%s</b>:'%type.__name__ + unicode_type(str(value), 'utf8', 'replace')
+            fe = force_unicode(fe)
+            msg = '<b>%s</b>:'%type.__name__ + force_unicode(value)
             error_dialog(self, _('Unhandled exception'), msg, det_msg=fe,
                     show=True)
         except BaseException:
diff --git a/src/calibre/gui2/notify.py b/src/calibre/gui2/notify.py
index 779ada9a62..171daa2370 100644
--- a/src/calibre/gui2/notify.py
+++ b/src/calibre/gui2/notify.py
@@ -9,7 +9,7 @@ __docformat__ = 'restructuredtext en'
 
 import time
 from calibre import prints
-from calibre.constants import islinux, isosx, get_osx_version, DEBUG
+from calibre.constants import islinux, isosx, get_osx_version, DEBUG, ispy3
 from polyglot.builtins import unicode_type
 
 
@@ -145,8 +145,12 @@ class AppleNotifier(Notifier):
 
     def notify(self, body, summary):
         def encode(x):
-            if isinstance(x, unicode_type):
-                x = x.encode('utf-8')
+            if ispy3:
+                if isinstance(x, bytes):
+                    x = x.decode('utf-8')
+            else:
+                if isinstance(x, unicode_type):
+                    x = x.encode('utf-8')
             return x
 
         cmd = [self.exe, '-activate',
diff --git a/src/calibre/gui2/store/search/search.py b/src/calibre/gui2/store/search/search.py
index a552bad3e0..24bde0cf62 100644
--- a/src/calibre/gui2/store/search/search.py
+++ b/src/calibre/gui2/store/search/search.py
@@ -67,7 +67,7 @@ class SearchDialog(QDialog, Ui_Dialog):
         self.setup_store_checks()
 
         # Set the search query
-        if isinstance(query, (str, unicode_type)):
+        if isinstance(query, (bytes, unicode_type)):
             self.search_edit.setText(query)
         elif isinstance(query, dict):
             if 'author' in query:
@@ -233,7 +233,7 @@ class SearchDialog(QDialog, Ui_Dialog):
         query = query.replace('<', '')
         # Remove the prefix.
         for loc in ('all', 'author', 'author2', 'authors', 'title', 'title2'):
-            query = re.sub(r'%s:"(?P<a>[^\s"]+)"' % loc, '\g<a>', query)
+            query = re.sub(r'%s:"(?P<a>[^\s"]+)"' % loc, r'\g<a>', query)
             query = query.replace('%s:' % loc, '')
         # Remove the prefix and search text.
         for loc in ('cover', 'download', 'downloads', 'drm', 'format', 'formats', 'price', 'store'):
diff --git a/src/calibre/gui2/tweak_book/function_replace.py b/src/calibre/gui2/tweak_book/function_replace.py
index 20f6f1d1cd..c588f63a77 100644
--- a/src/calibre/gui2/tweak_book/function_replace.py
+++ b/src/calibre/gui2/tweak_book/function_replace.py
@@ -30,8 +30,8 @@ user_functions = JSONConfig('editor-search-replace-functions')
 
 def compile_code(src, name='<string>'):
     if not isinstance(src, unicode_type):
-        match = re.search(r'coding[:=]\s*([-\w.]+)', src[:200])
-        enc = match.group(1) if match else 'utf-8'
+        match = re.search(br'coding[:=]\s*([-\w.]+)', src[:200])
+        enc = match.group(1).decode('utf-8') if match else 'utf-8'
         src = src.decode(enc)
     if not src or not src.strip():
         src = EMPTY_FUNC
diff --git a/src/calibre/library/add_to_library.py b/src/calibre/library/add_to_library.py
index fc3e6dcaad..cb117e5f40 100644
--- a/src/calibre/library/add_to_library.py
+++ b/src/calibre/library/add_to_library.py
@@ -9,7 +9,6 @@ import os
 from hashlib import sha1
 
 from calibre.ebooks import BOOK_EXTENSIONS
-from polyglot.builtins import unicode_type
 
 
 def find_folders_under(root, db, add_root=True,  # {{{
@@ -106,11 +105,9 @@ class FormatCollection(object):  # {{{
 
 def books_in_folder(folder, one_per_folder,  # {{{
         cancel_callback=lambda : False):
-    assert not isinstance(folder, unicode_type)
-
     dirpath = os.path.abspath(folder)
     if one_per_folder:
-        formats = set([])
+        formats = set()
         for path in os.listdir(dirpath):
             if cancel_callback():
                 return []
diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py
index 238fddac59..daea1f0bb5 100644
--- a/src/calibre/library/caches.py
+++ b/src/calibre/library/caches.py
@@ -19,7 +19,7 @@ from calibre.utils.localization import (canonicalize_lang, lang_map, get_udc)
 from calibre.db.search import CONTAINS_MATCH, EQUALS_MATCH, REGEXP_MATCH, _match
 from calibre.ebooks.metadata import title_sort, author_to_author_sort
 from calibre.ebooks.metadata.opf2 import metadata_to_opf
-from calibre import prints
+from calibre import prints, force_unicode
 from polyglot.builtins import unicode_type
 
 
@@ -137,7 +137,9 @@ del y, c, n, u
 
 
 def force_to_bool(val):
-    if isinstance(val, (str, unicode_type)):
+    if isinstance(val, (bytes, unicode_type)):
+        if isinstance(val, bytes):
+            val = force_unicode(val)
         try:
             val = icu_lower(val)
             if not val:
@@ -348,7 +350,7 @@ class ResultCache(SearchQueryParser):  # {{{
                 if item is None:
                     continue
                 v = item[loc]
-                if isinstance(v, (str, unicode_type)):
+                if isinstance(v, (bytes, unicode_type)):
                     v = parse_date(v)
                 if v is None or v <= UNDEFINED_DATE:
                     matches.add(item[0])
@@ -359,7 +361,7 @@ class ResultCache(SearchQueryParser):  # {{{
                 if item is None:
                     continue
                 v = item[loc]
-                if isinstance(v, (str, unicode_type)):
+                if isinstance(v, (bytes, unicode_type)):
                     v = parse_date(v)
                 if v is not None and v > UNDEFINED_DATE:
                     matches.add(item[0])
@@ -371,7 +373,7 @@ class ResultCache(SearchQueryParser):  # {{{
                 (p, relop) = self.date_search_relops[k]
                 query = query[p:]
         if relop is None:
-                (p, relop) = self.date_search_relops['=']
+            (p, relop) = self.date_search_relops['=']
 
         if query in self.local_today:
             qd = now()
@@ -403,7 +405,7 @@ class ResultCache(SearchQueryParser):  # {{{
             if item is None or item[loc] is None:
                 continue
             v = item[loc]
-            if isinstance(v, (str, unicode_type)):
+            if isinstance(v, (bytes, unicode_type)):
                 v = parse_date(v)
             if relop(v, qd, field_count):
                 matches.add(item[0])
@@ -448,7 +450,7 @@ class ResultCache(SearchQueryParser):  # {{{
                     (p, relop) = self.numeric_search_relops[k]
                     query = query[p:]
             if relop is None:
-                    (p, relop) = self.numeric_search_relops['=']
+                (p, relop) = self.numeric_search_relops['=']
 
             if dt == 'int':
                 cast = lambda x: int(x)
diff --git a/src/calibre/library/catalogs/csv_xml.py b/src/calibre/library/catalogs/csv_xml.py
index 5364a9f9be..bed17ff771 100644
--- a/src/calibre/library/catalogs/csv_xml.py
+++ b/src/calibre/library/catalogs/csv_xml.py
@@ -154,7 +154,7 @@ class CSV_XML(CatalogPlugin):
                         item = u'%.2g' % (item / 2.0)
 
                     # Convert HTML to markdown text
-                    if type(item) is unicode_type:
+                    if isinstance(item, unicode_type):
                         opening_tag = re.search('<(\\w+)(\x20|>)', item)
                         if opening_tag:
                             closing_tag = re.search('<\\/%s>$' % opening_tag.group(1), item)
@@ -177,7 +177,7 @@ class CSV_XML(CatalogPlugin):
                 for field in fields:
                     if field.startswith('#'):
                         val = db.get_field(r['id'], field, index_is_id=True)
-                        if not isinstance(val, (str, unicode_type)):
+                        if not isinstance(val, unicode_type):
                             val = unicode_type(val)
                         item = getattr(E, field.replace('#', '_'))(val)
                         record.append(item)
@@ -188,7 +188,7 @@ class CSV_XML(CatalogPlugin):
                         val = r[field]
                         if not val:
                             continue
-                        if not isinstance(val, (str, unicode_type)):
+                        if not isinstance(val, (bytes, unicode_type)):
                             if (fm.get(field, {}).get('datatype', None) ==
                                     'rating' and val):
                                 val = u'%.2g' % (val / 2.0)
diff --git a/src/calibre/library/custom_columns.py b/src/calibre/library/custom_columns.py
index 3a29ea73ef..ca14b24330 100644
--- a/src/calibre/library/custom_columns.py
+++ b/src/calibre/library/custom_columns.py
@@ -9,7 +9,7 @@ __docformat__ = 'restructuredtext en'
 import json, re
 from functools import partial
 
-from calibre import prints
+from calibre import prints, force_unicode
 from calibre.constants import preferred_encoding
 from calibre.library.field_metadata import FieldMetadata
 from calibre.utils.date import parse_date
@@ -131,7 +131,7 @@ class CustomColumns(object):
             if d['is_multiple']:
                 if x is None:
                     return []
-                if isinstance(x, (str, unicode_type, bytes)):
+                if isinstance(x, (unicode_type, bytes)):
                     x = x.split(d['multiple_seps']['ui_to_list'])
                 x = [y.strip() for y in x if y.strip()]
                 x = [y.decode(preferred_encoding, 'replace') if not isinstance(y,
@@ -142,12 +142,14 @@ class CustomColumns(object):
                         x.decode(preferred_encoding, 'replace')
 
         def adapt_datetime(x, d):
-            if isinstance(x, (str, unicode_type, bytes)):
+            if isinstance(x, (unicode_type, bytes)):
                 x = parse_date(x, assume_utc=False, as_utc=False)
             return x
 
         def adapt_bool(x, d):
-            if isinstance(x, (str, unicode_type, bytes)):
+            if isinstance(x, (unicode_type, bytes)):
+                if isinstance(x, bytes):
+                    x = force_unicode(x)
                 x = x.lower()
                 if x == 'true':
                     x = True
@@ -168,7 +170,9 @@ class CustomColumns(object):
         def adapt_number(x, d):
             if x is None:
                 return None
-            if isinstance(x, (str, unicode_type, bytes)):
+            if isinstance(x, (unicode_type, bytes)):
+                if isinstance(x, bytes):
+                    x = force_unicode(x)
                 if x.lower() == 'none':
                     return None
             if d['datatype'] == 'int':
diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py
index 88a26e28a7..5ed44f5d89 100644
--- a/src/calibre/library/database2.py
+++ b/src/calibre/library/database2.py
@@ -24,7 +24,7 @@ from calibre.library.custom_columns import CustomColumns
 from calibre.library.sqlite import connect, IntegrityError
 from calibre.library.prefs import DBPrefs
 from calibre.ebooks.metadata.book.base import Metadata
-from calibre.constants import preferred_encoding, iswindows, filesystem_encoding
+from calibre.constants import preferred_encoding, iswindows, filesystem_encoding, ispy3
 from calibre.ptempfile import (PersistentTemporaryFile,
         base_dir, SpooledTemporaryFile)
 from calibre.customize.ui import (run_plugins_on_import,
@@ -1754,12 +1754,14 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
             self.rc = rc
             self.id = id
 
-        def __str__(self):
-            return unicode_type(self)
+        def __unicode_representation__(self):
+            return u'n=%s s=%s c=%d rt=%d rc=%d id=%s' % (
+                self.n, self.s, self.c, self.rt, self.rc, self.id)
 
-        def __unicode__(self):
-            return 'n=%s s=%s c=%d rt=%d rc=%d id=%s'%\
-                            (self.n, self.s, self.c, self.rt, self.rc, self.id)
+        if ispy3:
+            __str__ = __unicode_representation__
+        else:
+            __str__ = __unicode__ = __unicode_representation__
 
     def clean_user_categories(self):
         user_cats = self.prefs.get('user_categories', {})
diff --git a/src/calibre/srv/routes.py b/src/calibre/srv/routes.py
index 8623c77489..6d96ee0891 100644
--- a/src/calibre/srv/routes.py
+++ b/src/calibre/srv/routes.py
@@ -202,7 +202,7 @@ class Route(object):
             raise RouteError('The variable(s) %s are not part of the route: %s' % (','.join(unknown), self.endpoint.route))
 
         def quoted(x):
-            if not isinstance(x, unicode_type) and not isinstance(x, bytes):
+            if not isinstance(x, (unicode_type, bytes)):
                 x = unicode_type(x)
             if isinstance(x, unicode_type):
                 x = x.encode('utf-8')
diff --git a/src/calibre/utils/apsw_shell.py b/src/calibre/utils/apsw_shell.py
deleted file mode 100644
index 43b2356737..0000000000
--- a/src/calibre/utils/apsw_shell.py
+++ /dev/null
@@ -1,2958 +0,0 @@
-#!/usr/bin/env python2
-# This is a patched version of sheel.py to fix
-# https://code.google.com/p/apsw/issues/detail?id=142
-
-import sys
-import apsw
-import shlex
-import os
-import csv
-import re
-import textwrap
-import time
-import codecs
-import base64
-
-from polyglot.builtins import unicode_type
-
-if sys.platform=="win32":
-    _win_colour=False
-    try:
-        import colorama
-        colorama.init()
-        del colorama
-        _win_colour=True
-    except:  # there are several failure reasons, ignore them all
-        pass
-
-
-class Shell(object):
-    """Implements a SQLite shell
-
-    :param stdin: Where to read input from (default sys.stdin)
-    :param stdout: Where to send output (default sys.stdout)
-    :param stderr: Where to send errors (default sys.stderr)
-    :param encoding: Default encoding for files opened/created by the
-      Shell.  If you want stdin/out/err to use a particular encoding
-      then you need to provide them `already configured <http://docs.python.org/library/codecs.html#codecs.open>`__ that way.
-    :param args: This should be program arguments only (ie if
-      passing in sys.argv do not include sys.argv[0] which is the
-      program name.  You can also pass in None and then call
-      :meth:`process_args` if you want to catch any errors
-      in handling the arguments yourself.
-    :param db: A existing :class:`Connection` you wish to use
-
-    The commands and behaviour are modelled after the `interactive
-    shell <https://sqlite.org/sqlite.html>`__ that is part of
-    SQLite.
-
-    You can inherit from this class to embed in your own code and user
-    interface.  Internally everything is handled as unicode.
-    Conversions only happen at the point of input or output which you
-    can override in your own code.
-
-    This implementation fixes a number of bugs/quirks present in the
-    sqlite shell.  Its control-C handling is also friendlier.  Some
-    examples of issues not present in this implementation:
-
-    * https://sqlite.org/src/info/c25aab7e7e
-    * https://sqlite.org/src/info/7b61b6c6ce
-    * https://sqlite.org/src/info/ee19e690ec
-    * https://sqlite.org/src/info/2466653295
-
-    Errors and diagnostics are only ever sent to error output
-    (self.stderr) and never to the regular output (self.stdout).  This
-    means using shell output is always easy and consistent.
-
-    Shell commands begin with a dot (eg .help).  They are implemented
-    as a method named after the command (eg command_help).  The method
-    is passed one parameter which is the list of arguments to the
-    command.
-
-    Output modes are implemented by functions named after the mode (eg
-    output_column).
-
-    When you request help the help information is automatically
-    generated from the docstrings for the command and output
-    functions.
-
-    You should not use a Shell object concurrently from multiple
-    threads.  It is one huge set of state information which would
-    become inconsistent if used simultaneously, and then give baffling
-    errors.  It is safe to call methods one at a time from different
-    threads.  ie it doesn't care what thread calls methods as long as
-    you don't call more than one concurrently.
-    """
-
-    class Error(Exception):
-        """Class raised on errors.  The expectation is that the error
-        will be displayed by the shell as text so there are no
-        specific subclasses as the distinctions between different
-        types of errors doesn't matter."""
-        pass
-
-    def __init__(self, stdin=None, stdout=None, stderr=None, encoding="utf8", args=None, db=None):
-        """Create instance, set defaults and do argument processing."""
-        super(Shell, self).__init__()
-        # The parameter doc has to be in main class doc as sphinx
-        # ignores any described here
-        self.exceptions=False
-        self.history_file="~/.sqlite_history"
-        self._db=None
-        self.dbfilename=None
-        if db:
-            self.db=db, db.filename
-        else:
-            self.db=None, None
-        self.prompt=    "sqlite> "
-        self.moreprompt="    ..> "
-        self.separator="|"
-        self.bail=False
-        self.echo=False
-        self.timer=False
-        self.header=False
-        self.nullvalue=""
-        self.output=self.output_list
-        self._output_table=self._fmt_sql_identifier("table")
-        self.widths=[]
-        # do we truncate output in list mode?  (explain doesn't, regular does)
-        self.truncate=True
-        # a stack of previous outputs. turning on explain saves previous, off restores
-        self._output_stack=[]
-
-        # other stuff
-        self.set_encoding(encoding)
-        if stdin is None:
-            stdin=sys.stdin
-        if stdout is None:
-            stdout=sys.stdout
-        if stderr is None:
-            stderr=sys.stderr
-        self.stdin=stdin
-        self.stdout=stdout
-        self._original_stdout=stdout
-        self.stderr=stderr
-        # we don't become interactive until the command line args are
-        # successfully parsed and acted upon
-        self.interactive=None
-        # current colouring object
-        self.command_colour()  # set to default
-        self._using_readline=False
-        self._input_stack=[]
-        self.input_line_number=0
-        self.push_input()
-        self.push_output()
-        self._input_descriptions=[]
-
-        if args:
-            try:
-                self.process_args(args)
-            except:
-                if len(self._input_descriptions):
-                    self._input_descriptions.append("Processing command line arguments")
-                self.handle_exception()
-                raise
-
-        if self.interactive is None:
-            self.interactive=getattr(self.stdin, "isatty", False) and self.stdin.isatty() and getattr(self.stdout, "isatty", False) and self.stdout.isatty()
-
-    def _ensure_db(self):
-        "The database isn't opened until first use.  This function ensures it is now open."
-        if not self._db:
-            if not self.dbfilename:
-                self.dbfilename=":memory:"
-            self._db=apsw.Connection(self.dbfilename, flags=apsw.SQLITE_OPEN_URI | apsw.SQLITE_OPEN_READWRITE | apsw.SQLITE_OPEN_CREATE)
-        return self._db
-
-    def _set_db(self, newv):
-        "Sets the open database (or None) and filename"
-        (db, dbfilename)=newv
-        if self._db:
-            self._db.close()
-            self._db=None
-        self._db=db
-        self.dbfilename=dbfilename
-
-    db=property(_ensure_db, _set_db, None, "The current :class:`Connection`")
-
-    def process_args(self, args):
-        """Process command line options specified in args.  It is safe to
-        call this multiple times.  We try to be compatible with SQLite shell
-        argument parsing.
-
-        :param args: A list of string options.  Do not include the
-           program as args[0]
-
-        :returns: A tuple of (databasefilename, initfiles,
-           sqlncommands).  This is provided for informational purposes
-           only - they have already been acted upon.  An example use
-           is that the SQLite shell does not enter the main interactive
-           loop if any sql/commands were provided.
-
-        The first non-option is the database file name.  Each
-        remaining non-option is treated as a complete input (ie it
-        isn't joined with others looking for a trailing semi-colon).
-
-        The SQLite shell uses single dash in front of options.  We
-        allow both single and double dashes.  When an unrecognized
-        argument is encountered then
-        :meth:`process_unknown_args` is called.
-        """
-        # we don't use optparse as we need to use single dashes for
-        # options - all hand parsed
-        if not args:
-            return None, [], []
-
-        # are options still valid?
-        options=True
-        # have we seen the database name?
-        havedbname=False
-        # List of init files to read
-        inits=[]
-        # List of sql/dot commands
-        sqls=[]
-
-        while args:
-            if not options or not args[0].startswith("-"):
-                options=False
-                if not havedbname:
-                    # grab new database
-                    self.db=None, args[0]
-                    havedbname=True
-                else:
-                    sqls.append(args[0])
-                args=args[1:]
-                continue
-
-            # remove initial single or double dash
-            args[0]=args[0][1:]
-            if args[0].startswith("-"):
-                args[0]=args[0][1:]
-
-            if args[0]=="init":
-                if len(args)<2:
-                    raise self.Error("You need to specify a filename after -init")
-                inits.append(args[1])
-                args=args[2:]
-                continue
-
-            if args[0]=="header" or args[0]=="noheader":
-                self.header=args[0]=="header"
-                args=args[1:]
-                continue
-
-            if args[0] in ("echo", "bail", "interactive"):
-                setattr(self, args[0], True)
-                args=args[1:]
-                continue
-
-            if args[0]=="batch":
-                self.interactive=False
-                args=args[1:]
-                continue
-
-            if args[0] in ("separator", "nullvalue", "encoding"):
-                if len(args)<2:
-                    raise self.Error("You need to specify a value after -"+args[0])
-                getattr(self, "command_"+args[0])([args[1]])
-                args=args[2:]
-                continue
-
-            if args[0]=="version":
-                self.write(self.stdout, apsw.sqlitelibversion()+"\n")
-                # A pretty gnarly thing to do
-                sys.exit(0)
-
-            if args[0]=="help":
-                self.write(self.stderr, self.usage())
-                sys.exit(0)
-
-            if args[0] in ("no-colour", "no-color", "nocolour", "nocolor"):
-                self.colour_scheme="off"
-                self._out_colour()
-                args=args[1:]
-                continue
-
-            # only remaining known args are output modes
-            if getattr(self, "output_"+args[0], None):
-                self.command_mode(args[:1])
-                args=args[1:]
-                continue
-
-            newargs=self.process_unknown_args(args)
-            if newargs is None:
-                raise self.Error("Unrecognized argument '"+args[0]+"'")
-            args=newargs
-
-        for f in inits:
-            self.command_read([f])
-
-        for s in sqls:
-            self.process_complete_line(s)
-
-        return self.dbfilename, inits, sqls
-
-    def process_unknown_args(self, args):
-        """This is called when :meth:`process_args` encounters an
-        argument it doesn't understand.  Override this method if you
-        want to be able to understand additional command line arguments.
-
-        :param args: A list of the remaining arguments.  The initial one will
-           have had the leading dashes removed (eg if it was --foo on the command
-           line then args[0] will be "foo"
-        :returns: None if you don't recognize the argument either.  Otherwise
-           return the list of remaining arguments after you have processed
-           yours.
-        """
-        return None
-
-    def usage(self):
-        "Returns the usage message.  Make sure it is newline terminated"
-
-        msg="""
-Usage: program [OPTIONS] FILENAME [SQL|CMD] [SQL|CMD]...
-FILENAME is the name of a SQLite database. A new database is
-created if the file does not exist.
-OPTIONS include:
-   -init filename       read/process named file
-   -echo                print commands before execution
-   -[no]header          turn headers on or off
-   -bail                stop after hitting an error
-   -interactive         force interactive I/O
-   -batch               force batch I/O
-   -column              set output mode to 'column'
-   -csv                 set output mode to 'csv'
-   -html                set output mode to 'html'
-   -line                set output mode to 'line'
-   -list                set output mode to 'list'
-   -python              set output mode to 'python'
-   -separator 'x'       set output field separator (|)
-   -nullvalue 'text'    set text string for NULL values
-   -version             show SQLite version
-   -encoding 'name'     the encoding to use for files
-                        opened via .import, .read & .output
-   -nocolour            disables colour output to screen
-"""
-        return msg.lstrip()
-
-    ###
-    # Value formatting routines.  They take a value and return a
-    # text formatting of them.  Mostly used by the various output's
-    # but also by random other pieces of code.
-    ###
-
-    _binary_type = eval(("buffer", "bytes")[sys.version_info>=(3,0)])
-    _basestring = eval(("basestring", "str")[sys.version_info>=(3,0)])
-
-    # bytes that are ok in C strings - no need for quoting
-    _printable=[ord(x) for x in
-                "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789~!@#$%^&*()`_-+={}[]:;,.<>/?|"
-                ]
-
-    def _fmt_c_string(self, v):
-        "Format as a C string including surrounding double quotes"
-        if isinstance(v, self._basestring):
-            op=['"']
-            for c in v:
-                if c=="\\":
-                    op.append("\\\\")
-                elif c=="\r":
-                    op.append("\\r")
-                elif c=="\n":
-                    op.append("\\n")
-                elif c=="\t":
-                    op.append("\\t")
-                elif ord(c) not in self._printable:
-                    op.append("\\"+c)
-                else:
-                    op.append(c)
-            op.append('"')
-            return "".join(op)
-        elif v is None:
-            return '"'+self.nullvalue+'"'
-        elif isinstance(v, self._binary_type):
-            if sys.version_info<(3,0):
-                o=lambda x: ord(x)
-                fromc=lambda x: x
-            else:
-                o=lambda x: x
-                fromc=lambda x: chr(x)
-            res=['"']
-            for c in v:
-                if o(c) in self._printable:
-                    res.append(fromc(c))
-                else:
-                    res.append("\\x%02X" % (o(c),))
-            res.append('"')
-            return "".join(res)
-        else:
-            # number of some kind
-            return '"%s"' % (v,)
-
-    def _fmt_html_col(self, v):
-        "Format as HTML (mainly escaping &/</>"
-        return self._fmt_text_col(v).\
-           replace("&", "&amp;"). \
-           replace(">", "&gt;"). \
-           replace("<", "&lt;"). \
-           replace("'", "&apos;"). \
-           replace('"', "&quot;")
-
-    def _fmt_json_value(self, v):
-        "Format a value."
-        if isinstance(v, self._basestring):
-            # we assume utf8 so only some characters need to be escaed
-            op=['"']
-            for c in v:
-                if c=="\\":
-                    op.append("\\\\")
-                elif c=="\r":
-                    op.append("\\r")
-                elif c=="\n":
-                    op.append("\\n")
-                elif c=="\t":
-                    op.append("\\t")
-                elif c=="/":  # yes you have to escape forward slash for some reason
-                    op.append("\\/")
-                elif c=='"':
-                    op.append("\\"+c)
-                elif c=="\\b":
-                    op.append("\\b")
-                elif c=="\\f":
-                    op.append("\\f")
-                else:
-                    # It isn't clear when \u sequences *must* be used.
-                    # Assuming not needed due to utf8 output which
-                    # corresponds to what rfc4627 implies.
-                    op.append(c)
-            op.append('"')
-            return "".join(op)
-        elif v is None:
-            return 'null'
-        elif isinstance(v, self._binary_type):
-            if sys.version_info<(3,0):
-                o=base64.encodestring(v)
-            else:
-                o=base64.encodebytes(v).decode("ascii")
-            if o[-1]=="\n":
-                o=o[:-1]
-            return '"'+o+'"'
-        else:
-            # number of some kind
-            return '%s' % (v,)
-
-    def _fmt_python(self, v):
-        "Format as python literal"
-        if v is None:
-            return "None"
-        elif isinstance(v, self._basestring):
-            return repr(v)
-        elif isinstance(v, self._binary_type):
-            if sys.version_info<(3,0):
-                res=["buffer(\""]
-                for i in v:
-                    if ord(i) in self._printable:
-                        res.append(i)
-                    else:
-                        res.append("\\x%02X" % (ord(i),))
-                res.append("\")")
-                return "".join(res)
-            else:
-                res=['b"']
-                for i in v:
-                    if i in self._printable:
-                        res.append(chr(i))
-                    else:
-                        res.append("\\x%02X" % (i,))
-                res.append('"')
-                return "".join(res)
-        else:
-            return "%s" % (v,)
-
-    def _fmt_sql_identifier(self, v):
-        "Return the identifier quoted in SQL syntax if needed (eg table and column names)"
-        if not len(v):  # yes sqlite does allow zero length identifiers
-            return '""'
-        nonalnum=re.sub("[A-Za-z_0-9]+", "", v)
-        if len(nonalnum)==0:
-            if v.upper() not in self._sqlite_reserved:
-                # Ok providing it doesn't start with a digit
-                if v[0] not in "0123456789":
-                    return v
-        # double quote it unless there are any double quotes in it
-        if '"' in nonalnum:
-            return "[%s]" % (v,)
-        return '"%s"' % (v,)
-
-    def _fmt_text_col(self, v):
-        "Regular text formatting"
-        if v is None:
-            return self.nullvalue
-        elif isinstance(v, self._basestring):
-            return v
-        elif isinstance(v, self._binary_type):
-            # sqlite gives back raw bytes!
-            return "<Binary data>"
-        else:
-            return "%s" % (v,)
-
-    ###
-    # The various output routines.  They are always called with the
-    # header irrespective of the setting allowing for some per query
-    # setup. (see output_column for example).  The doc strings are
-    # used to generate help.
-    ###
-
-    def output_column(self, header, line):
-        """
-        Items left aligned in space padded columns.  They are
-        truncated if they do not fit. If the width hasn't been
-        specified for a column then 10 is used unless the column name
-        (header) is longer in which case that width is used.  Use the
-        .width command to change column sizes.
-        """
-        # as an optimization we calculate self._actualwidths which is
-        # reset for each query
-        if header:
-            def gw(n):
-                if n<len(self.widths) and self.widths[n]!=0:
-                    return self.widths[n]
-                # if width is not present or 0 then autosize
-                text=self._fmt_text_col(line[n])
-                return max(len(text), 10)
-
-            widths=[gw(i) for i in range(len(line))]
-
-            if self.truncate:
-                self._actualwidths=["%"+("-%d.%ds", "%d.%ds")[w<0]%(abs(w), abs(w)) for w in widths]
-            else:
-                self._actualwidths=["%"+("-%ds", "%ds")[w<0]%(abs(w),) for w in widths]
-
-            if self.header:
-                # output the headers
-                c=self.colour
-                cols=[c.header+(self._actualwidths[i] % (self._fmt_text_col(line[i]),))+c.header_ for i in range(len(line))]
-                # sqlite shell uses two spaces between columns
-                self.write(self.stdout, "  ".join(cols)+"\n")
-                if c is self._colours["off"]:
-                    self.output_column(False, ["-"*abs(widths[i]) for i in range(len(widths))])
-            return
-        cols=[self.colour.colour_value(line[i], self._actualwidths[i] % (self._fmt_text_col(line[i]),)) for i in range(len(line))]
-        # sqlite shell uses two spaces between columns
-        self.write(self.stdout, "  ".join(cols)+"\n")
-
-    def output_csv(self, header, line):
-        """
-        Items in csv format (comma separated).  Use tabs mode for tab
-        separated.  You can use the .separator command to use a
-        different one after switching mode.  A separator of comma uses
-        double quotes for quoting while other separators do not do any
-        quoting.  The Python csv library used for this only supports
-        single character separators.
-        """
-
-        # we use self._csv for the work, setup when header is
-        # supplied. _csv is a tuple of a StringIO and the csv.writer
-        # instance.
-
-        # Sigh
-        if sys.version_info<(3,0):
-            fixdata=lambda x: x.encode("utf8")
-        else:
-            fixdata=lambda x: x
-
-        if header:
-            if sys.version_info<(3,0):
-                import StringIO as io
-            else:
-                import io
-            s=io.StringIO()
-            kwargs={}
-            if self.separator==",":
-                kwargs["dialect"]="excel"
-            elif self.separator=="\t":
-                kwargs["dialect"]="excel-tab"
-            else:
-                kwargs["quoting"]=csv.QUOTE_NONE
-                kwargs["delimiter"]=fixdata(self.separator)
-                kwargs["doublequote"]=False
-                # csv module is bug ridden junk - I already say no
-                # quoting so it still looks for the quotechar and then
-                # gets upset that it can't be quoted.  Which bit of no
-                # quoting was ambiguous?
-                kwargs["quotechar"]="\x00"
-
-            writer=csv.writer(s, **kwargs)
-            self._csv=(s, writer)
-            if self.header:
-                self.output_csv(None, line)
-            return
-
-        if header is None:
-            c=self.colour
-            line=[c.header+fixdata(self._fmt_text_col(l))+c.header_ for l in line]
-        else:
-            fmt=lambda x: self.colour.colour_value(x, fixdata(self._fmt_text_col(x)))
-            line=[fmt(l) for l in line]
-        self._csv[1].writerow(line)
-        t=self._csv[0].getvalue()
-        if sys.version_info<(3,0):
-            t=t.decode("utf8")
-        # csv lib always does DOS eol
-        assert(t.endswith("\r\n"))
-        t=t[:-2]
-        # should not be other eol irregularities
-        assert(not t.endswith("\r") and not t.endswith("\n"))
-        self.write(self.stdout, t+"\n")
-        self._csv[0].truncate(0)
-        self._csv[0].seek(0)
-
-    def output_html(self, header, line):
-        "HTML table style"
-        if header:
-            if not self.header:
-                return
-            fmt=lambda x: self.colour.header+self._fmt_html_col(x)+self.colour.header_
-        else:
-            fmt=lambda x: self.colour.colour_value(x, self._fmt_html_col(x))
-        line=[fmt(l) for l in line]
-        out=["<TR>"]
-        for l in line:
-            out.append(("<TD>","<TH>")[header])
-            out.append(l)
-            out.append(("</TD>\n","</TH>\n")[header])
-        out.append("</TR>\n")
-        self.write(self.stdout, "".join(out))
-
-    def output_insert(self, header, line):
-        """
-        Lines as SQL insert statements.  The table name is "table"
-        unless you specified a different one as the second parameter
-        to the .mode command.
-        """
-        if header:
-            return
-        fmt=lambda x: self.colour.colour_value(x, apsw.format_sql_value(x))
-        out="INSERT INTO "+self._output_table+" VALUES("+",".join([fmt(l) for l in line])+");\n"
-        self.write(self.stdout, out)
-
-    def output_json(self, header, line):
-        """
-        Each line as a JSON object with a trailing comma.  Blobs are
-        output as base64 encoded strings.  You should be using UTF8
-        output encoding.
-        """
-        if header:
-            self._output_json_cols=line
-            return
-        fmt=lambda x: self.colour.colour_value(x, self._fmt_json_value(x))
-        out=["%s: %s" % (self._fmt_json_value(k), fmt(line[i])) for i,k in enumerate(self._output_json_cols)]
-        self.write(self.stdout, "{ "+", ".join(out)+"},\n")
-
-    def output_line(self, header, line):
-        """
-        One value per line in the form 'column = value' with a blank
-        line between rows.
-        """
-        if header:
-            w=5
-            for l in line:
-                if len(l)>w:
-                    w=len(l)
-            self._line_info=(w, line)
-            return
-        fmt=lambda x: self.colour.colour_value(x, self._fmt_text_col(x))
-        w=self._line_info[0]
-        for i in range(len(line)):
-            self.write(self.stdout, "%*s = %s\n" % (w, self._line_info[1][i], fmt(line[i])))
-        self.write(self.stdout, "\n")
-
-    def output_list(self, header, line):
-        "All items on one line with separator"
-        if header:
-            if not self.header:
-                return
-            c=self.colour
-            fmt=lambda x: c.header+x+c.header_
-        else:
-            fmt=lambda x: self.colour.colour_value(x, self._fmt_text_col(x))
-        self.write(self.stdout, self.separator.join([fmt(x) for x in line])+"\n")
-
-    def output_python(self, header, line):
-        "Tuples in Python source form for each row"
-        if header:
-            if not self.header:
-                return
-            c=self.colour
-            fmt=lambda x: c.header+self._fmt_python(x)+c.header_
-        else:
-            fmt=lambda x: self.colour.colour_value(x, self._fmt_python(x))
-        self.write(self.stdout, '('+", ".join([fmt(l) for l in line])+"),\n")
-
-    def output_tcl(self, header, line):
-        "Outputs TCL/C style strings using current separator"
-        # In theory you could paste the output into your source ...
-        if header:
-            if not self.header:
-                return
-            c=self.colour
-            fmt=lambda x: c.header+self._fmt_c_string(x)+c.header_
-        else:
-            fmt=lambda x: self.colour.colour_value(x, self._fmt_c_string(x))
-        self.write(self.stdout, self.separator.join([fmt(l) for l in line])+"\n")
-
-    def _output_summary(self, summary):
-        # internal routine to output a summary line or two
-        self.write(self.stdout, self.colour.summary+summary+self.colour.summary_)
-
-    ###
-    # Various routines
-    ###
-
-    def cmdloop(self, intro=None):
-        """Runs the main interactive command loop.
-
-        :param intro: Initial text banner to display instead of the
-           default.  Make sure you newline terminate it.
-        """
-        if intro is None:
-            intro="""
-SQLite version %s (APSW %s)
-Enter ".help" for instructions
-Enter SQL statements terminated with a ";"
-""" % (apsw.sqlitelibversion(), apsw.apswversion())
-            intro=intro.lstrip()
-        if self.interactive and intro:
-            if sys.version_info<(3,0):
-                intro=unicode_type(intro)
-            c=self.colour
-            self.write(self.stdout, c.intro+intro+c.intro_)
-
-        using_readline=False
-        try:
-            if self.interactive and self.stdin is sys.stdin:
-                import readline
-                old_completer=readline.get_completer()
-                readline.set_completer(self.complete)
-                readline.parse_and_bind("tab: complete")
-                using_readline=True
-                try:
-                    readline.read_history_file(os.path.expanduser(self.history_file))
-                except:
-                    # We only expect IOError here but if the history
-                    # file does not exist and this code has been
-                    # compiled into the module it is possible to get
-                    # an IOError that doesn't match the IOError from
-                    # Python parse time resulting in an IOError
-                    # exception being raised.  Consequently we just
-                    # catch all exceptions.
-                    pass
-        except ImportError:
-            pass
-
-        try:
-            while True:
-                self._input_descriptions=[]
-                if using_readline:
-                    # we drop completion cache because it contains
-                    # table and column names which could have changed
-                    # with last executed SQL
-                    self._completion_cache=None
-                    self._using_readline=True
-                try:
-                    command=self.getcompleteline()
-                    if command is None:  # EOF
-                        return
-                    self.process_complete_line(command)
-                except:
-                    self._append_input_description()
-                    try:
-                        self.handle_exception()
-                    except UnicodeDecodeError:
-                        self.handle_exception()
-        finally:
-            if using_readline:
-                readline.set_completer(old_completer)
-                readline.set_history_length(256)
-                readline.write_history_file(os.path.expanduser(self.history_file))
-
-    def handle_exception(self):
-        """Handles the current exception, printing a message to stderr as appropriate.
-        It will reraise the exception if necessary (eg if bail is true)"""
-        eclass,eval,etb=sys.exc_info()  # py2&3 compatible way of doing this
-        if isinstance(eval, SystemExit):
-            eval._handle_exception_saw_this=True
-            raise
-
-        self._out_colour()
-        self.write(self.stderr, self.colour.error)
-
-        if isinstance(eval, KeyboardInterrupt):
-            self.handle_interrupt()
-            text="Interrupted"
-        else:
-            text=str(eval)
-
-        if not text.endswith("\n"):
-            text=text+"\n"
-
-        if len(self._input_descriptions):
-            for i in range(len(self._input_descriptions)):
-                if i==0:
-                    pref="At "
-                else:
-                    pref=" "*i+"From "
-                self.write(self.stderr, pref+self._input_descriptions[i]+"\n")
-
-        self.write(self.stderr, text)
-        if self.exceptions:
-            stack=[]
-            while etb:
-                stack.append(etb.tb_frame)
-                etb = etb.tb_next
-
-            for frame in stack:
-                self.write(self.stderr, "\nFrame %s in %s at line %d\n" %
-                           (frame.f_code.co_name, frame.f_code.co_filename,
-                            frame.f_lineno))
-                vars=list(frame.f_locals.items())
-                vars.sort()
-                for k,v in vars:
-                    try:
-                        v=repr(v)[:80]
-                    except:
-                        v="<Unable to convert to string>"
-                    self.write(self.stderr, "%10s = %s\n" % (k,v))
-            self.write(self.stderr, "\n%s: %s\n" % (eclass, repr(eval)))
-
-        self.write(self.stderr, self.colour.error_)
-
-        eval._handle_exception_saw_this=True
-        if self.bail:
-            raise
-
-    def process_sql(self, sql, bindings=None, internal=False, summary=None):
-        """Processes SQL text consisting of one or more statements
-
-        :param sql: SQL to execute
-
-        :param bindings: bindings for the *sql*
-
-        :param internal: If True then this is an internal execution
-          (eg the .tables or .database command).  When executing
-          internal sql timings are not shown nor is the SQL echoed.
-
-        :param summary: If not None then should be a tuple of two
-          items.  If the ``sql`` returns any data then the first item
-          is printed before the first row, and the second item is
-          printed after the last row.  An example usage is the .find
-          command which shows table names.
-        """
-        cur=self.db.cursor()
-        # we need to know when each new statement is executed
-        state={'newsql': True, 'timing': None}
-
-        def et(cur, sql, bindings):
-            state['newsql']=True
-            # if time reporting, do so now
-            if not internal and self.timer:
-                if state['timing']:
-                    self.display_timing(state['timing'], self.get_resource_usage())
-            # print statement if echo is on
-            if not internal and self.echo:
-                # ? should we strip leading and trailing whitespace? backslash quote stuff?
-                if bindings:
-                    self.write(self.stderr, "%s [%s]\n" % (sql, bindings))
-                else:
-                    self.write(self.stderr, sql+"\n")
-            # save resource from begining of command (ie don't include echo time above)
-            if not internal and self.timer:
-                state['timing']=self.get_resource_usage()
-            return True
-        cur.setexectrace(et)
-        # processing loop
-        try:
-            for row in cur.execute(sql, bindings):
-                if state['newsql']:
-                    # summary line?
-                    if summary:
-                        self._output_summary(summary[0])
-                    # output a header always
-                    cols=[h for h,d in cur.getdescription()]
-                    self.output(True, cols)
-                    state['newsql']=False
-                self.output(False, row)
-            if not state['newsql'] and summary:
-                self._output_summary(summary[1])
-        except:
-            # If echo is on and the sql to execute is a syntax error
-            # then the exec tracer won't have seen it so it won't be
-            # printed and the user will be wondering exactly what sql
-            # had the error.  We look in the traceback and deduce if
-            # the error was happening in a prepare or not.  Also we
-            # need to ignore the case where SQLITE_SCHEMA happened and
-            # a reprepare is being done since the exec tracer will
-            # have been called in that situation.
-            if not internal and self.echo:
-                tb=sys.exc_info()[2]
-                last=None
-                while tb:
-                    last=tb.tb_frame
-                    tb=tb.tb_next
-
-                if last and last.f_code.co_name=="sqlite3_prepare" \
-                   and last.f_code.co_filename.endswith("statementcache.c") \
-                   and "sql" in last.f_locals:
-                    self.write(self.stderr, last.f_locals["sql"]+"\n")
-            raise
-
-        if not internal and self.timer:
-            self.display_timing(state['timing'], self.get_resource_usage())
-
-    def process_command(self, cmd):
-        """Processes a dot command.  It is split into parts using the
-        `shlex.split
-        <http://docs.python.org/library/shlex.html#shlex.split>`__
-        function which is roughly the same method used by Unix/POSIX
-        shells.
-        """
-        if self.echo:
-            self.write(self.stderr, cmd+"\n")
-        # broken with unicode on Python 2!!!
-        if sys.version_info<(3,0):
-            cmd=cmd.encode("utf8")
-            cmd=[c.decode("utf8") for c in shlex.split(cmd)]
-        else:
-            cmd=shlex.split(cmd)
-        assert cmd[0][0]=="."
-        cmd[0]=cmd[0][1:]
-        fn=getattr(self, "command_"+cmd[0], None)
-        if not fn:
-            raise self.Error("Unknown command \"%s\".  Enter \".help\" for help" % (cmd[0],))
-        fn(cmd[1:])
-
-    ###
-    # Commands start here
-    ###
-
-    def _boolean_command(self, name, cmd):
-        "Parse and verify boolean parameter"
-        if len(cmd)!=1 or cmd[0].lower() not in ("on", "off"):
-            raise self.Error(name+" expected ON or OFF")
-        return cmd[0].lower()=="on"
-
-    # Note that doc text is used for generating help output.
-
-    def command_backup(self, cmd):
-        """backup ?DB? FILE: Backup DB (default "main") to FILE
-
-        Copies the contents of the current database to FILE
-        overwriting whatever was in FILE.  If you have attached databases
-        then you can specify their name instead of the default of "main".
-
-        The backup is done at the page level - SQLite copies the pages
-        as is.  There is no round trip through SQL code.
-        """
-        dbname="main"
-        if len(cmd)==1:
-            fname=cmd[0]
-        elif len(cmd)==2:
-            dbname=cmd[0]
-            fname=cmd[1]
-        else:
-            raise self.Error("Backup takes one or two parameters")
-        out=apsw.Connection(fname)
-        b=out.backup("main", self.db, dbname)
-        try:
-            while not b.done:
-                b.step()
-        finally:
-            b.finish()
-            out.close()
-
-    def command_bail(self, cmd):
-        """bail ON|OFF: Stop after hitting an error (default OFF)
-
-        If an error is encountered while processing commands or SQL
-        then exit.  (Note this is different than SQLite shell which
-        only exits for errors in SQL.)
-        """
-        self.bail=self._boolean_command("bail", cmd)
-
-    def command_colour(self, cmd=[]):
-        """colour SCHEME: Selects a colour scheme
-
-        Residents of both countries that have not adopted the metric
-        system may also spell this command without a 'u'.  If using a
-        colour terminal in interactive mode then output is
-        automatically coloured to make it more readable.  Use 'off' to
-        turn off colour, and no name or 'default' for the default.
-        """
-        if len(cmd)>1:
-            raise self.Error("Too many colour schemes")
-        c=cmd and cmd[0] or "default"
-        if c not in self._colours:
-            raise self.Error("No such colour scheme: "+c)
-        self.colour_scheme=c
-        self._out_colour()
-
-    command_color=command_colour
-
-    def command_databases(self, cmd):
-        """databases: Lists names and files of attached databases
-
-        """
-        if len(cmd):
-            raise self.Error("databases command doesn't take any parameters")
-        self.push_output()
-        self.header=True
-        self.output=self.output_column
-        self.truncate=False
-        self.widths=[3,15,58]
-        try:
-            self.process_sql("pragma database_list", internal=True)
-        finally:
-            self.pop_output()
-
-    def command_dump(self, cmd):
-        """dump ?TABLE? [TABLE...]: Dumps all or specified tables in SQL text format
-
-        The table name is treated as like pattern so you can use % as
-        a wildcard.  You can use dump to make a text based backup of
-        the database.  It is also useful for comparing differences or
-        making the data available to other databases.  Indices and
-        triggers for the table(s) are also dumped.  Finally views
-        matching the table pattern name are dumped (it isn't possible
-        to work out which views access which table and views can
-        access multiple tables anyway).
-
-        Note that if you are dumping virtual tables such as used by
-        the FTS3 module then they may use other tables to store
-        information.  For example if you create a FTS3 table named
-        *recipes* then it also creates *recipes_content*,
-        *recipes_segdir* etc.  Consequently to dump this example
-        correctly use::
-
-           .dump recipes recipes_%
-
-        If the database is empty or no tables/views match then there
-        is no output.
-        """
-        # Simple tables are easy to dump.  More complicated is dealing
-        # with virtual tables, foreign keys etc.
-
-        # Lock the database while doing the dump so nothing changes
-        # under our feet
-        self.process_sql("BEGIN IMMEDIATE", internal=True)
-
-        try:
-            # first pass -see if virtual tables or foreign keys are in
-            # use.  If they are we emit pragmas to deal with them, but
-            # prefer not to emit them
-            v={"virtuals": False,
-               "foreigns": False}
-
-            def check(name, sql):
-                if name.lower().startswith("sqlite_"):
-                    return False
-                sql=sql.lower()
-                if re.match(r"^\s*create\s+virtual\s+.*", sql):
-                    v["virtuals"]=True
-                # pragma table_info doesn't tell us if foreign keys
-                # are involved so we guess if any the various strings are
-                # in the sql somewhere
-                if re.match(r".*\b(foreign\s*key|references)\b.*", sql):
-                    v["foreigns"]=True
-                return True
-
-            if len(cmd)==0:
-                cmd=["%"]
-
-            tables=[]
-            for pattern in cmd:
-                for name,sql in self.db.cursor().execute("SELECT name,sql FROM sqlite_master "
-                                                         "WHERE sql NOT NULL AND type IN ('table','view') "
-                                                         "AND tbl_name LIKE ?1", (pattern,)):
-                    if check(name, sql) and name not in tables:
-                        tables.append(name)
-
-            if not tables:
-                return
-
-            # will we need to analyze anything later?
-            analyze_needed=[]
-            for stat in self.db.cursor().execute("select name from sqlite_master where sql not null and type='table' and tbl_name like 'sqlite_stat%'"):
-                for name in tables:
-                    if len(self.db.cursor().execute("select * from "+self._fmt_sql_identifier(stat[0])+" WHERE tbl=?", (name,)).fetchall()):
-                        if name not in analyze_needed:
-                            analyze_needed.append(name)
-            analyze_needed.sort()
-
-            def blank():
-                self.write(self.stdout, "\n")
-
-            def comment(s):
-                if isinstance(s, bytes):
-                    s = s.decode('utf-8', 'replace')
-                self.write(self.stdout, textwrap.fill(s, 78, initial_indent="-- ", subsequent_indent="-- ")+"\n")
-
-            pats=", ".join([(x,"(All)")[x=="%"] for x in cmd])
-            comment("SQLite dump (by APSW %s)" % (apsw.apswversion(),))
-            comment("SQLite version " + apsw.sqlitelibversion())
-            comment("Date: " +time.ctime())
-            comment("Tables like: "+pats)
-            comment("Database: "+self.db.filename)
-            try:
-                import getpass
-                import socket
-                comment("User: %s @ %s" % (getpass.getuser(), socket.gethostname()))
-            except ImportError:
-                pass
-            blank()
-
-            comment("The values of various per-database settings")
-            comment("PRAGMA page_size="+str(self.db.cursor().execute("pragma page_size").fetchall()[0][0])+";\n")
-            comment("PRAGMA encoding='"+self.db.cursor().execute("pragma encoding").fetchall()[0][0]+"';\n")
-            vac={0: "NONE", 1: "FULL", 2: "INCREMENTAL"}
-            vacvalue=self.db.cursor().execute("pragma auto_vacuum").fetchall()[0][0]
-            comment("PRAGMA auto_vacuum="+vac.get(vacvalue, str(vacvalue))+";\n")
-            comment("PRAGMA max_page_count="+str(self.db.cursor().execute("pragma max_page_count").fetchall()[0][0])+";\n")
-            blank()
-
-            # different python versions have different requirements
-            # about specifying cmp to sort routine so we use this
-            # portable workaround with a decorated list instead
-            dectables=[(x.lower(), x) for x in tables]
-            dectables.sort()
-            tables=[y for x,y in dectables]
-
-            virtuals=v["virtuals"]
-            foreigns=v["foreigns"]
-
-            if virtuals:
-                comment("This pragma is needed to restore virtual tables")
-                self.write(self.stdout, "PRAGMA writable_schema=ON;\n")
-            if foreigns:
-                comment("This pragma turns off checking of foreign keys "
-                        "as tables would be inconsistent while restoring.  It was introduced "
-                        "in SQLite 3.6.19.")
-                self.write(self.stdout, "PRAGMA foreign_keys=OFF;\n")
-
-            if virtuals or foreigns:
-                blank()
-
-            self.write(self.stdout, "BEGIN TRANSACTION;\n")
-            blank()
-
-            def sqldef(s):
-                # return formatted sql watching out for embedded
-                # comments at the end forcing trailing ; onto next
-                # line https://sqlite.org/src/info/c04a8b8a4f
-                if "--" in s.split("\n")[-1]:
-                    nl="\n"
-                else:
-                    nl=""
-                return s+nl+";\n"
-
-            # do the table dumping loops
-            oldtable=self._output_table
-            try:
-                self.push_output()
-                self.output=self.output_insert
-                # Dump the table
-                for table in tables:
-                    for sql in self.db.cursor().execute("SELECT sql FROM sqlite_master WHERE name=?1 AND type='table'", (table,)):
-                        comment("Table  "+table)
-                        # Special treatment for virtual tables - they
-                        # get called back on drops and creates and
-                        # could thwart us so we have to manipulate
-                        # sqlite_master directly
-                        if sql[0].lower().split()[:3]==["create", "virtual", "table"]:
-                            self.write(self.stdout, "DELETE FROM sqlite_master WHERE name="+apsw.format_sql_value(table)+" AND type='table';\n")
-                            self.write(self.stdout, "INSERT INTO sqlite_master(type,name,tbl_name,rootpage,sql) VALUES('table',%s,%s,0,%s);\n"
-                                       % (apsw.format_sql_value(table), apsw.format_sql_value(table), apsw.format_sql_value(sql[0])))
-                        else:
-                            self.write(self.stdout, "DROP TABLE IF EXISTS "+self._fmt_sql_identifier(table)+";\n")
-                            self.write(self.stdout, sqldef(sql[0]))
-                            self._output_table=self._fmt_sql_identifier(table)
-                            self.process_sql("select * from "+self._fmt_sql_identifier(table), internal=True)
-                        # Now any indices or triggers
-                        first=True
-                        for name,sql in self.db.cursor().execute("SELECT name,sql FROM sqlite_master "
-                                                                 "WHERE sql NOT NULL AND type IN ('index', 'trigger') "
-                                                                 "AND tbl_name=?1 AND name NOT LIKE 'sqlite_%' "
-                                                                 "ORDER BY lower(name)", (table,)):
-                            if first:
-                                comment("Triggers and indices on  "+table)
-                                first=False
-                            self.write(self.stdout, sqldef(sql))
-                        blank()
-                # Views done last.  They have to be done in the same order as they are in sqlite_master
-                # as they could refer to each other
-                first=True
-                for name,sql in self.db.cursor().execute("SELECT name,sql FROM sqlite_master "
-                                                         "WHERE sql NOT NULL AND type='view' "
-                                                         "AND name IN ( "+",".join([apsw.format_sql_value(i) for i in tables])+
-                                                         ") ORDER BY _ROWID_"):
-                    if first:
-                        comment("Views")
-                        first=False
-                    self.write(self.stdout, "DROP VIEW IF EXISTS %s;\n" % (self._fmt_sql_identifier(name),))
-                    self.write(self.stdout, sqldef(sql))
-                if not first:
-                    blank()
-
-                # sqlite sequence
-                # does it exist
-                if len(self.db.cursor().execute("select * from sqlite_master where name='sqlite_sequence'").fetchall()):
-                    first=True
-                    for t in tables:
-                        v=self.db.cursor().execute("select seq from main.sqlite_sequence where name=?1", (t,)).fetchall()
-                        if len(v):
-                            assert len(v)==1
-                            if first:
-                                comment("For primary key autoincrements the next id "
-                                        "to use is stored in sqlite_sequence")
-                                first=False
-                            self.write(self.stdout, 'DELETE FROM main.sqlite_sequence WHERE name=%s;\n' % (apsw.format_sql_value(t),))
-                            self.write(self.stdout, 'INSERT INTO main.sqlite_sequence VALUES (%s, %s);\n' % (apsw.format_sql_value(t), v[0][0]))
-                    if not first:
-                        blank()
-            finally:
-                self.pop_output()
-                self._output_table=oldtable
-
-            # analyze
-            if analyze_needed:
-                comment("You had used the analyze command on these tables before.  Rerun for this new data.")
-                for n in analyze_needed:
-                    self.write(self.stdout, "ANALYZE "+self._fmt_sql_identifier(n)+";\n")
-                blank()
-
-            # user version pragma
-            uv=self.db.cursor().execute("pragma user_version").fetchall()[0][0]
-            if uv:
-                comment("Your database may need this.  It is sometimes used to keep track of the schema version (eg Firefox does this).")
-                comment("pragma user_version=%d;" % (uv,))
-                blank()
-
-            # Save it all
-            self.write(self.stdout, "COMMIT TRANSACTION;\n")
-
-            # cleanup pragmas
-            if foreigns:
-                blank()
-                comment("Restoring foreign key checking back on.  Note that SQLite 3.6.19 is off by default")
-                self.write(self.stdout, "PRAGMA foreign_keys=ON;\n")
-            if virtuals:
-                blank()
-                comment("Restoring writable schema back to default")
-                self.write(self.stdout, "PRAGMA writable_schema=OFF;\n")
-                # schema reread
-                blank()
-                comment("We need to force SQLite to reread the schema because otherwise it doesn't know that "
-                        "the virtual tables we inserted directly into sqlite_master exist.  See "
-                        "last comments of https://sqlite.org/cvstrac/tktview?tn=3425")
-                self.write(self.stdout, "BEGIN;\nCREATE TABLE no_such_table(x,y,z);\nROLLBACK;\n")
-
-        finally:
-            self.process_sql("END", internal=True)
-
-    def command_echo(self, cmd):
-        """echo ON|OFF: If ON then each SQL statement or command is printed before execution (default OFF)
-
-        The SQL statement or command is sent to error output so that
-        it is not intermingled with regular output.
-        """
-        self.echo=self._boolean_command("echo", cmd)
-
-    def set_encoding(self, enc):
-        """Saves *enc* as the default encoding, after verifying that
-        it is valid.  You can also include :error to specify error
-        handling - eg 'cp437:replace'
-
-        Raises an exception on invalid encoding or error
-        """
-        enc=enc.split(":", 1)
-        if len(enc)>1:
-            enc, errors=enc
-        else:
-            enc=enc[0]
-            errors=None
-        try:
-            codecs.lookup(enc)
-        except LookupError:
-            raise self.Error("No known encoding '%s'" % (enc,))
-        try:
-            if errors is not None:
-                codecs.lookup_error(errors)
-        except LookupError:
-            raise self.Error("No known codec error handler '%s'" % (errors,))
-        self.encoding=enc, errors
-
-    def command_encoding(self, cmd):
-        """encoding ENCODING: Set the encoding used for new files opened via .output and imports
-
-        SQLite and APSW work internally using Unicode and characters.
-        Files however are a sequence of bytes.  An encoding describes
-        how to convert between bytes and characters.  The default
-        encoding is utf8 and that is generally the best value to use
-        when other programs give you a choice.
-
-        You can also specify an error handler.  For example
-        'cp437:replace' will use code page 437 and any Unicode
-        codepoints not present in cp437 will be replaced (typically
-        with something like a question mark).  Other error handlers
-        include 'ignore', 'strict' (default) and 'xmlcharrefreplace'.
-
-        For the default input/output/error streams on startup the
-        shell defers to Python's detection of encoding.  For example
-        on Windows it asks what code page is in use and on Unix it
-        looks at the LC_CTYPE environment variable.  You can set the
-        PYTHONIOENCODING environment variable to override this
-        detection.
-
-        This command affects files opened after setting the encoding
-        as well as imports.
-
-        See the online APSW documentation for more details.
-        """
-        if len(cmd)!=1:
-            raise self.Error("Encoding takes one argument")
-        self.set_encoding(cmd[0])
-
-    def command_exceptions(self, cmd):
-        """exceptions ON|OFF: If ON then detailed tracebacks are shown on exceptions (default OFF)
-
-        Normally when an exception occurs the error string only is
-        displayed.  However it is sometimes useful to get a full
-        traceback.  An example would be when you are developing
-        virtual tables and using the shell to exercise them.  In
-        addition to displaying each stack frame, the local variables
-        within each frame are also displayed.
-        """
-        self.exceptions=self._boolean_command("exceptions", cmd)
-
-    def command_exit(self, cmd):
-        """exit:Exit this program"""
-        if len(cmd):
-            raise self.Error("Exit doesn't take any parameters")
-        sys.exit(0)
-
-    def command_quit(self, cmd):
-        """quit:Exit this program"""
-        if len(cmd):
-            raise self.Error("Quit doesn't take any parameters")
-        sys.exit(0)
-
-    def command_explain(self, cmd):
-        """explain ON|OFF: Set output mode suitable for explain (default OFF)
-
-        Explain shows the underlying SQLite virtual machine code for a
-        statement.  You need to prefix the SQL with explain.  For example:
-
-           explain select * from table;
-
-        This output mode formats the explain output nicely.  If you do
-        '.explain OFF' then the output mode and settings in place when
-        you did '.explain ON' are restored.
-        """
-        if len(cmd)==0 or self._boolean_command("explain", cmd):
-            self.push_output()
-            self.header=True
-            self.widths=[4,13,4,4,4,13,2,13]
-            self.truncate=False
-            self.output=self.output_column
-        else:
-            self.pop_output()
-
-    def command_find(self, cmd):
-        """find what ?TABLE?: Searches all columns of all tables for a value
-
-        The find command helps you locate data across your database
-        for example to find a string or any references to an id.
-
-        You can specify a like pattern to limit the search to a subset
-        of tables (eg specifying 'CUSTOMER%' for all tables beginning
-        with CUSTOMER).
-
-        The what value will be treated as a string and/or integer if
-        possible.  If what contains % or _ then it is also treated as
-        a like pattern.
-
-        This command will take a long time to execute needing to read
-        all of the relevant tables.
-        """
-        if len(cmd)<1 or len(cmd)>2:
-            raise self.Error("At least one argument required and at most two accepted")
-        tablefilter="%"
-        if len(cmd)==2:
-            tablefilter=cmd[1]
-        querytemplate=[]
-        queryparams=[]
-
-        def qp():  # binding for current queryparams
-            return "?"+str(len(queryparams))
-        s=cmd[0]
-        if '%' in s or '_' in s:
-            queryparams.append(s)
-            querytemplate.append("%s LIKE "+qp())
-        queryparams.append(s)
-        querytemplate.append("%s = "+qp())
-        try:
-            i=int(s)
-            queryparams.append(i)
-            querytemplate.append("%s = "+qp())
-        except ValueError:
-            pass
-        querytemplate=" OR ".join(querytemplate)
-        for (table,) in self.db.cursor().execute("SELECT name FROM sqlite_master WHERE type='table' AND name LIKE ?1", (tablefilter,)):
-            t=self._fmt_sql_identifier(table)
-            query="SELECT * from %s WHERE " % (t,)
-            colq=[]
-            for _,column,_,_,_,_ in self.db.cursor().execute("pragma table_info(%s)" % (t,)):
-                colq.append(querytemplate % ((self._fmt_sql_identifier(column),)*len(queryparams)))
-            query=query+" OR ".join(colq)
-            self.process_sql(query, queryparams, internal=True, summary=("Table "+table+"\n", "\n"))
-
-    def command_header(self, cmd):
-        """header(s) ON|OFF: Display the column names in output (default OFF)
-
-        """
-        self.header=self._boolean_command("header", cmd)
-
-    command_headers=command_header
-
-    _help_info=None
-
-    def command_help(self, cmd):
-        """help ?COMMAND?: Shows list of commands and their usage.  If COMMAND
-        is specified then shows detail about that COMMAND.  ('.help all' will
-        show detailed help about all commands.)
-        """
-        if not self._help_info:
-            # buildup help database
-            self._help_info={}
-            for c in dir(self):
-                if not c.startswith("command_"):
-                    continue
-                # help is 3 parts
-                # - the syntax string (eg backup ?dbname? filename)
-                # - the one liner description (eg saves database to filename)
-                # - the multi-liner detailed description
-                # We grab this from the doc string for the function in the form
-                #   syntax: one liner\nmulti\nliner
-                d=getattr(self, c).__doc__
-                assert d, c+" command must have documentation"
-                c=c[len("command_"):]
-                if c in ("headers", "color"):
-                    continue
-                while d[0]=="\n":
-                    d=d[1:]
-                parts=d.split("\n", 1)
-                firstline=parts[0].strip().split(":", 1)
-                assert len(firstline)==2, c+" command must have usage: description doc"
-                if len(parts)==1 or len(parts[1].strip())==0:  # work around textwrap bug
-                    multi=""
-                else:
-                    multi=textwrap.dedent(parts[1])
-                if c=="mode":
-                    if not self._output_modes:
-                        self._cache_output_modes()
-                    firstline[1]=firstline[1]+" "+" ".join(self._output_modes)
-                    multi=multi+"\n\n"+"\n\n".join(self._output_modes_detail)
-                if c=="colour":
-                    colours=list(self._colours.keys())
-                    colours.sort()
-                    firstline[1]=firstline[1]+" from "+", ".join(colours)
-                if len(multi.strip())==0:  # All whitespace
-                    multi=None
-                else:
-                    multi=multi.strip("\n")
-                    # we need to keep \n\n as a newline but turn all others into spaces
-                    multi=multi.replace("\n\n", "\x00")
-                    multi=multi.replace("\n", " ")
-                    multi=multi.replace("\x00", "\n\n")
-                    multi=multi.split("\n\n")
-                self._help_info[c]=('.'+firstline[0].strip(), firstline[1].strip(), multi)
-
-        self.write(self.stderr, "\n")
-
-        tw=self._terminal_width()
-        if tw<32:
-            tw=32
-        if len(cmd)==0:
-            commands=list(self._help_info.keys())
-            commands.sort()
-            w=0
-            for command in commands:
-                if len(self._help_info[command][0])>w:
-                    w=len(self._help_info[command][0])
-            out=[]
-            for command in commands:
-                hi=self._help_info[command]
-                # usage string
-                out.append(hi[0])
-                # space padding (including 2 for between columns)
-                out.append(" "*(2+w-len(hi[0])))
-                # usage message wrapped if need be
-                out.append(("\n"+" "*(2+w)).join(textwrap.wrap(hi[1], tw-w-2)))
-                # newline
-                out.append("\n")
-            self.write(self.stderr, "".join(out))
-        else:
-            if cmd[0]=="all":
-                cmd=list(self._help_info.keys())
-                cmd.sort()
-            w=0
-            for command in self._help_info:
-                if len(self._help_info[command][0])>w:
-                    w=len(self._help_info[command][0])
-
-            for command in cmd:
-                if command=="headers":
-                    command="header"
-                if command not in self._help_info:
-                    raise self.Error("No such command \"%s\"" % (command,))
-                out=[]
-                hi=self._help_info[command]
-                # usage string
-                out.append(hi[0])
-                # space padding (2)
-                out.append(" "*(2+w-len(hi[0])))
-                # usage message wrapped if need be
-                out.append(("\n"+" "*(2+w)).join(textwrap.wrap(hi[1], tw-w-2))+"\n")
-                if hi[2]:
-                    # newlines
-                    out.append("\n")
-                    # detailed message
-                    for i,para in enumerate(hi[2]):
-                        out.append(textwrap.fill(para, tw)+"\n")
-                        if i<len(hi[2])-1:
-                            out.append("\n")
-                # if not first one then print separator header
-                if command!=cmd[0]:
-                    self.write(self.stderr, "\n"+"="*tw+"\n")
-                self.write(self.stderr, "".join(out))
-        self.write(self.stderr, "\n")
-
-    def command_import(self, cmd):
-        """import FILE TABLE: Imports separated data from FILE into TABLE
-
-        Reads data from the file into the named table using the
-        current separator and encoding.  For example if the separator
-        is currently a comma then the file should be CSV (comma
-        separated values).
-
-        All values read in are supplied to SQLite as strings.  If you
-        want SQLite to treat them as other types then declare your
-        columns appropriately.  For example declaring a column 'REAL'
-        will result in the values being stored as floating point if
-        they can be safely converted.  See this page for more details:
-
-          https://sqlite.org/datatype3.html
-
-        Another alternative is to create a tempory table, insert the
-        values into that and then use casting.
-
-          CREATE TEMPORARY TABLE import(a,b,c);
-
-          .import filename import
-
-          CREATE TABLE final AS SELECT cast(a as BLOB), cast(b as INTEGER), cast(c as CHAR) from import;
-
-          DROP TABLE import;
-
-        You can also get more sophisticated using the SQL CASE
-        operator.  For example this will turn zero length strings into
-        null:
-
-          SELECT CASE col WHEN '' THEN null ELSE col END FROM ...
-        """
-        if len(cmd)!=2:
-            raise self.Error("import takes two parameters")
-
-        try:
-            final=None
-            # start transaction so database can't be changed
-            # underneath us
-            self.db.cursor().execute("BEGIN IMMEDIATE")
-            final="ROLLBACK"
-
-            # how many columns?
-            ncols=len(self.db.cursor().execute("pragma table_info("+self._fmt_sql_identifier(cmd[1])+")").fetchall())
-            if ncols<1:
-                raise self.Error("No such table '%s'" % (cmd[1],))
-
-            cur=self.db.cursor()
-            sql="insert into %s values(%s)" % (self._fmt_sql_identifier(cmd[1]), ",".join("?"*ncols))
-
-            kwargs={}
-            if self.separator==",":
-                kwargs["dialect"]="excel"
-            elif self.separator=="\t":
-                kwargs["dialect"]="excel-tab"
-            else:
-                kwargs["quoting"]=csv.QUOTE_NONE
-                kwargs["delimiter"]=self.separator
-                kwargs["doublequote"]=False
-                kwargs["quotechar"]="\x00"
-            row=1
-            for line in self._csvin_wrapper(cmd[0], kwargs):
-                if len(line)!=ncols:
-                    raise self.Error("row %d has %d columns but should have %d" % (row, len(line), ncols))
-                try:
-                    cur.execute(sql, line)
-                except:
-                    self.write(self.stderr, "Error inserting row %d" % (row,))
-                    raise
-                row+=1
-            self.db.cursor().execute("COMMIT")
-
-        except:
-            if final:
-                self.db.cursor().execute(final)
-            raise
-
-    def _csvin_wrapper(self, filename, dialect):
-        # Returns a csv reader that works around python bugs and uses
-        # dialect dict to configure reader
-
-        # Very easy for python 3
-        if sys.version_info>=(3,0):
-            thefile=codecs.open(filename, "r", self.encoding[0])
-            for line in csv.reader(thefile, **dialect.copy()):
-                yield line
-            thefile.close()
-            return
-
-        ###
-        # csv module is not good at unicode so we have to
-        # indirect unless utf8 is in use
-        ###
-        if self.encoding[0].lower()=="utf8":  # no need for tempfile
-            thefile=open(filename, "rb")
-        else:
-            import tempfile
-            thefile=tempfile.TemporaryFile(prefix="apsw_import")
-            thefile.write(codecs.open(filename, "r", self.encoding[0]).read().encode("utf8"))
-            # move back to beginning
-            thefile.seek(0,0)
-
-        # Ensure all values are utf8 not unicode
-        for k,v in dialect.items():
-            if isinstance(v, unicode_type):
-                dialect[k]=v.encode("utf8")
-        for line in csv.reader(thefile, **dialect):
-            # back to unicode again
-            yield [x.decode("utf8") for x in line]
-        thefile.close()
-
-    def command_autoimport(self, cmd):
-        """autoimport FILENAME ?TABLE?: Imports filename creating a table and automatically working out separators and data types (alternative to .import command)
-
-        The import command requires that you precisely pre-setup the
-        table and schema, and set the data separators (eg commas or
-        tabs).  In many cases this information can be automatically
-        deduced from the file contents which is what this command
-        does.  There must be at least two columns and two rows.
-
-        If the table is not specified then the basename of the file
-        will be used.
-
-        Additionally the type of the contents of each column is also
-        deduced - for example if it is a number or date.  Empty values
-        are turned into nulls.  Dates are normalized into YYYY-MM-DD
-        format and DateTime are normalized into ISO8601 format to
-        allow easy sorting and searching.  4 digit years must be used
-        to detect dates.  US (swapped day and month) versus rest of
-        the world is also detected providing there is at least one
-        value that resolves the ambiguity.
-
-        Care is taken to ensure that columns looking like numbers are
-        only treated as numbers if they do not have unnecessary
-        leading zeroes or plus signs.  This is to avoid treating phone
-        numbers and similar number like strings as integers.
-
-        This command can take quite some time on large files as they
-        are effectively imported twice.  The first time is to
-        determine the format and the types for each column while the
-        second pass actually imports the data.
-        """
-        if len(cmd)<1 or len(cmd)>2:
-            raise self.Error("Expected one or two parameters")
-        if not os.path.exists(cmd[0]):
-            raise self.Error("File \"%s\" does not exist" % (cmd[0],))
-        if len(cmd)==2:
-            tablename=cmd[1]
-        else:
-            tablename=None
-        try:
-            final=None
-            c=self.db.cursor()
-            c.execute("BEGIN IMMEDIATE")
-            final="ROLLBACK"
-
-            if not tablename:
-                tablename=os.path.splitext(os.path.basename(cmd[0]))[0]
-
-            if c.execute("pragma table_info(%s)" % (self._fmt_sql_identifier(tablename),)).fetchall():
-                raise self.Error("Table \"%s\" already exists" % (tablename,))
-
-            # The types we support deducing
-            def DateUS(v):  # US formatted date with wrong ordering of day and month
-                return DateWorld(v, switchdm=True)
-
-            def DateWorld(v, switchdm=False):  # Sensibly formatted date as used anywhere else in the world
-                y,m,d=self._getdate(v)
-                if switchdm:
-                    m,d=d,m
-                if m<1 or m>12 or d<1 or d>31:
-                    raise ValueError
-                return "%d-%02d-%02d" % (y,m,d)
-
-            def DateTimeUS(v):  # US date and time
-                return DateTimeWorld(v, switchdm=True)
-
-            def DateTimeWorld(v, switchdm=False):  # Sensible date and time
-                y,m,d,h,M,s=self._getdatetime(v)
-                if switchdm:
-                    m,d=d,m
-                if m<1 or m>12 or d<1 or d>31 or h<0 or h>23 or M<0 or M>59 or s<0 or s>65:
-                    raise ValueError
-                return "%d-%02d-%02dT%02d:%02d:%02d" % (y,m,d,h,M,s)
-
-            def Number(v):  # we really don't want phone numbers etc to match
-                # Python's float & int constructors allow whitespace which we don't
-                if re.search(r"\s", v):
-                    raise ValueError
-                if v=="0":
-                    return 0
-                if v[0]=="+":  # idd prefix
-                    raise ValueError
-                if re.match("^[0-9]+$", v):
-                    if v[0]=="0":
-                        raise ValueError  # also a phone number
-                    return int(v)
-                if v[0]=="0" and not v.startswith("0."):  # deceptive not a number
-                    raise ValueError
-                return float(v)
-
-            # Work out the file format
-            formats=[
-                {"dialect": "excel"},
-                {"dialect": "excel-tab"}]
-            seps=["|", ";", ":"]
-            if self.separator not in seps:
-                seps.append(self.separator)
-            for sep in seps:
-                formats.append(
-                    {"quoting": csv.QUOTE_NONE,
-                     "delimiter": sep,
-                     "doublequote": False,
-                     "quotechar": "\x00"}
-                    )
-            possibles=[]
-            errors=[]
-            encodingissue=False
-            # format is copy() on every use.  This appears bizarre and
-            # unnecessary.  However Python 2.3 and 2.4 somehow manage
-            # to empty it if not copied.
-            for format in formats:
-                ncols=-1
-                lines=0
-                try:
-                    for line in self._csvin_wrapper(cmd[0], format.copy()):
-                        if lines==0:
-                            lines=1
-                            ncols=len(line)
-                            # data type guess setup
-                            datas=[]
-                            for i in range(ncols):
-                                datas.append([DateUS, DateWorld, DateTimeUS, DateTimeWorld, Number])
-                            allblanks=[True]*ncols
-                            continue
-                        if len(line)!=ncols:
-                            raise ValueError("Expected %d columns - got %d" % (ncols, len(line)))
-                        lines+=1
-                        for i in range(ncols):
-                            if not line[i]:
-                                continue
-                            allblanks[i]=False
-                            if not datas[i]:
-                                continue
-                            # remove datas that give ValueError
-                            d=[]
-                            for dd in datas[i]:
-                                try:
-                                    dd(line[i])
-                                    d.append(dd)
-                                except ValueError:
-                                    pass
-                            datas[i]=d
-                    if ncols>1 and lines>1:
-                        # if a particular column was allblank then clear datas for it
-                        for i in range(ncols):
-                            if allblanks[i]:
-                                datas[i]=[]
-                        possibles.append((format.copy(), ncols, lines, datas))
-                except UnicodeDecodeError:
-                    encodingissue=True
-                except:
-                    s=str(sys.exc_info()[1])
-                    if s not in errors:
-                        errors.append(s)
-
-            if len(possibles)==0:
-                if encodingissue:
-                    raise self.Error("The file is probably not in the current encoding \"%s\" and didn't match a known file format" % (self.encoding[0],))
-                v="File doesn't appear to match a known type."
-                if len(errors):
-                    v+="  Errors reported:\n"+"\n".join(["  "+e for e in errors])
-                raise self.Error(v)
-            if len(possibles)>1:
-                raise self.Error("File matches more than one type!")
-            format, ncols, lines, datas=possibles[0]
-            fmt=format.get("dialect", None)
-            if fmt is None:
-                fmt="(delimited by \"%s\")" % (format["delimiter"],)
-            self.write(self.stdout, "Detected Format %s  Columns %d  Rows %d\n" % (fmt, ncols, lines))
-            # Header row
-            reader=self._csvin_wrapper(cmd[0], format)
-            for header in reader:
-                break
-            # Check schema
-            identity=lambda x:x
-            for i in range(ncols):
-                if len(datas[i])>1:
-                    raise self.Error("Column #%d \"%s\" has ambiguous data format - %s" % (i+1, header[i], ", ".join([dl.__name__ for dl in datas[i]])))
-                if datas[i]:
-                    datas[i]=datas[i][0]
-                else:
-                    datas[i]=identity
-            # Make the table
-            sql="CREATE TABLE %s(%s)" % (self._fmt_sql_identifier(tablename), ", ".join([self._fmt_sql_identifier(h) for h in header]))
-            c.execute(sql)
-            # prep work for each row
-            sql="INSERT INTO %s VALUES(%s)" % (self._fmt_sql_identifier(tablename), ",".join(["?"]*ncols))
-            for line in reader:
-                vals=[]
-                for i in range(ncols):
-                    l=line[i]
-                    if not l:
-                        vals.append(None)
-                    else:
-                        vals.append(datas[i](l))
-                c.execute(sql, vals)
-
-            c.execute("COMMIT")
-            self.write(self.stdout, "Auto-import into table \"%s\" complete\n" % (tablename,))
-        except:
-            if final:
-                self.db.cursor().execute(final)
-            raise
-
-    def _getdate(self, v):
-        # Returns a tuple of 3 items y,m,d from string v
-        m=re.match(r"^([0-9]+)[^0-9]([0-9]+)[^0-9]([0-9]+)$", v)
-        if not m:
-            raise ValueError
-        y,m,d=int(m.group(1)), int(m.group(2)), int(m.group(3))
-        if d>1000:  # swap order
-            y,m,d=d,m,y
-        if y<1000 or y>9999:
-            raise ValueError
-        return y,m,d
-
-    def _getdatetime(self, v):
-        # must be at least HH:MM
-        m=re.match(r"^([0-9]+)[^0-9]([0-9]+)[^0-9]([0-9]+)[^0-9]+([0-9]+)[^0-9]([0-9]+)([^0-9]([0-9]+))?$", v)
-        if not m:
-            raise ValueError
-        items=list(m.group(1,2,3,4,5,7))
-        for i in range(len(items)):
-            if items[i] is None:
-                items[i]=0
-        items=[int(i) for i in items]
-        if items[2]>1000:
-            items=[items[2], items[1], items[0]]+items[3:]
-        if items[0]<1000 or items[0]>9999:
-            raise ValueError
-        return items
-
-    def command_indices(self, cmd):
-        """indices TABLE: Lists all indices on table TABLE
-
-        """
-        if len(cmd)!=1:
-            raise self.Error("indices takes one table name")
-        self.push_output()
-        self.header=False
-        self.output=self.output_list
-        try:
-            self.process_sql("SELECT name FROM sqlite_master WHERE type='index' AND tbl_name LIKE ?1 "
-                             "UNION ALL SELECT name FROM sqlite_temp_master WHERE type='index' AND tbl_name LIKE "
-                             "?1 ORDER by name", cmd, internal=True)
-        finally:
-            self.pop_output()
-
-    def command_load(self, cmd):
-        """load FILE ?ENTRY?: Loads a SQLite extension library
-
-        Note: Extension loading may not be enabled in the SQLite
-        library version you are using.
-
-        Extensions are an easy way to add new functions and
-        functionality.  For a useful extension look at the bottom of
-        https://sqlite.org/contrib
-
-        By default sqlite3_extension_init is called in the library but
-        you can specify an alternate entry point.
-
-        If you get an error about the extension not being found you
-        may need to explicitly specify the directory.  For example if
-        it is in the current directory then use:
-
-          .load ./extension.so
-        """
-        if len(cmd)<1 or len(cmd)>2:
-            raise self.Error("load takes one or two parameters")
-        try:
-            self.db.enableloadextension(True)
-        except:
-            raise self.Error("Extension loading is not supported")
-
-        self.db.loadextension(*cmd)
-
-    _output_modes=None
-
-    def command_mode(self, cmd):
-        """mode MODE ?TABLE?: Sets output mode to one of"""
-        if len(cmd) in (1,2):
-            w=cmd[0]
-            if w=="tabs":
-                w="list"
-            m=getattr(self, "output_"+w, None)
-            if w!="insert":
-                if len(cmd)==2:
-                    raise self.Error("Output mode %s doesn't take parameters" % (cmd[0]))
-            if m:
-                self.output=m
-                # set some defaults
-                self.truncate=True
-                if cmd[0]=="csv":
-                    self.separator=","
-                elif cmd[0]=="tabs":
-                    self.separator="\t"
-                else:
-                    pass
-                    # self.separator=self._output_stack[0]["separator"]
-                if w=="insert":
-                    if len(cmd)==2:
-                        self._output_table=cmd[1]
-                    else:
-                        self._output_table="table"
-                    self._output_table=self._fmt_sql_identifier(self._output_table)
-                return
-        if not self._output_modes:
-            self._cache_output_modes()
-        raise self.Error("Expected a valid output mode: "+", ".join(self._output_modes))
-
-    # needed so command completion and help can use it
-    def _cache_output_modes(self):
-        modes=[m[len("output_"):] for m in dir(self) if m.startswith("output_")]
-        modes.append("tabs")
-        modes.sort()
-        self._output_modes=modes
-
-        detail=[]
-
-        for m in modes:
-            if m=='tabs':
-                continue
-            d=getattr(self, "output_"+m).__doc__
-            assert d, "output mode "+m+" needs doc"
-            d=d.replace("\n", " ").strip()
-            while "  " in d:
-                d=d.replace("  ", " ")
-            detail.append(m+": "+d)
-        self._output_modes_detail=detail
-
-    def command_nullvalue(self, cmd):
-        """nullvalue STRING: Print STRING in place of null values
-
-        This affects textual output modes like column and list and
-        sets how SQL null values are shown.  The default is a zero
-        length string.  Insert mode and dumps are not affected by this
-        setting.  You can use double quotes to supply a zero length
-        string.  For example:
-
-          .nullvalue ""         # the default
-          .nullvalue <NULL>     # rather obvious
-          .nullvalue " \\t "     # A tab surrounded by spaces
-        """
-        if len(cmd)!=1:
-            raise self.Error("nullvalue takes exactly one parameter")
-        self.nullvalue=self.fixup_backslashes(cmd[0])
-
-    def command_output(self, cmd):
-        """output FILENAME: Send output to FILENAME (or stdout)
-
-        If the FILENAME is stdout then output is sent to standard
-        output from when the shell was started.  The file is opened
-        using the current encoding (change with .encoding command).
-        """
-        # Flush everything
-        self.stdout.flush()
-        self.stderr.flush()
-        if hasattr(self.stdin, "flush"):
-            try:
-                self.stdin.flush()
-            except IOError:  # see issue 117
-                pass
-
-        # we will also close stdout but only do so once we have a
-        # replacement so that stdout is always valid
-
-        if len(cmd)!=1:
-            raise self.Error("You must specify a filename")
-
-        try:
-            fname=cmd[0]
-            if fname=="stdout":
-                old=None
-                if self.stdout!=self._original_stdout:
-                    old=self.stdout
-                self.stdout=self._original_stdout
-                if old is not None:  # done here in case close raises exception
-                    old.close()
-                return
-
-            newf=codecs.open(fname, "w", self.encoding[0], self.encoding[1])
-            old=None
-            if self.stdout!=self._original_stdout:
-                old=self.stdout
-            self.stdout=newf
-            if old is not None:
-                old.close()
-        finally:
-            self._out_colour()
-
-    def command_print(self, cmd):
-        """print STRING: print the literal STRING
-
-        If more than one argument is supplied then they are printed
-        space separated.  You can use backslash escapes such as \\n
-        and \\t.
-        """
-        self.write(self.stdout, " ".join([self.fixup_backslashes(i) for i in cmd])+"\n")
-
-    def command_prompt(self, cmd):
-        """prompt MAIN ?CONTINUE?: Changes the prompts for first line and continuation lines
-
-        The default is to print 'sqlite> ' for the main prompt where
-        you can enter a dot command or a SQL statement.  If the SQL
-        statement is complete (eg not ; terminated) then you are
-        prompted for more using the continuation prompt which defaults
-        to ' ..> '.  Example:
-
-          .prompt "Yes, Master> " "More, Master> "
-
-        You can use backslash escapes such as \\n and \\t.
-        """
-        if len(cmd)<1 or len(cmd)>2:
-            raise self.Error("prompt takes one or two arguments")
-        self.prompt=self.fixup_backslashes(cmd[0])
-        if len(cmd)==2:
-            self.moreprompt=self.fixup_backslashes(cmd[1])
-
-    def command_read(self, cmd):
-        """read FILENAME: Processes SQL and commands in FILENAME (or Python if FILENAME ends with .py)
-
-        Treats the specified file as input (a mixture or SQL and/or
-        dot commands).  If the filename ends in .py then it is treated
-        as Python code instead.
-
-        For Python code the symbol 'shell' refers to the instance of
-        the shell and 'apsw' is the apsw module.
-        """
-        if len(cmd)!=1:
-            raise self.Error("read takes a single filename")
-        if cmd[0].lower().endswith(".py"):
-            g={}
-            g.update({'apsw': apsw, 'shell': self})
-            if sys.version_info<(3,0):
-                execfile(cmd[0], g, g)
-            else:
-                # compile step is needed to associate name with code
-                exec(compile(open(cmd[0]).read(), cmd[0], 'exec'), g, g)
-        else:
-            f=codecs.open(cmd[0], "rU", self.encoding[0])
-            try:
-                try:
-                    self.push_input()
-                    self.stdin=f
-                    self.interactive=False
-                    self.input_line_number=0
-                    while True:
-                        line=self.getcompleteline()
-                        if line is None:
-                            break
-                        self.process_complete_line(line)
-                except:
-                    eval=sys.exc_info()[1]
-                    if not isinstance(eval, SystemExit):
-                        self._append_input_description()
-                    raise
-
-            finally:
-                self.pop_input()
-                f.close()
-
-    def command_restore(self, cmd):
-        """restore ?DB? FILE: Restore database from FILE into DB (default "main")
-
-        Copies the contents of FILE to the current database (default "main").
-        The backup is done at the page level - SQLite copies the pages as
-        is.  There is no round trip through SQL code.
-        """
-        dbname="main"
-        if len(cmd)==1:
-            fname=cmd[0]
-        elif len(cmd)==2:
-            dbname=cmd[0]
-            fname=cmd[1]
-        else:
-            raise self.Error("Restore takes one or two parameters")
-        input=apsw.Connection(fname)
-        b=self.db.backup(dbname, input, "main")
-        try:
-            while not b.done:
-                b.step()
-        finally:
-            b.finish()
-            input.close()
-
-    def command_schema(self, cmd):
-        """schema ?TABLE? [TABLE...]: Shows SQL for table
-
-        If you give one or more tables then their schema is listed
-        (including indices).  If you don't specify any then all
-        schemas are listed. TABLE is a like pattern so you can % for
-        wildcards.
-        """
-        self.push_output()
-        self.output=self.output_list
-        self.header=False
-        try:
-            if len(cmd)==0:
-                cmd=['%']
-            for n in cmd:
-                self.process_sql("SELECT sql||';' FROM "
-                                 "(SELECT sql sql, type type, tbl_name tbl_name, name name "
-                                 "FROM sqlite_master UNION ALL "
-                                 "SELECT sql, type, tbl_name, name FROM sqlite_temp_master) "
-                                 "WHERE tbl_name LIKE ?1 AND type!='meta' AND sql NOTNULL AND name NOT LIKE 'sqlite_%' "
-                                 "ORDER BY substr(type,2,1), name", (n,), internal=True)
-        finally:
-            self.pop_output()
-
-    def command_separator(self, cmd):
-        """separator STRING: Change separator for output mode and .import
-
-        You can use quotes and backslashes.  For example to set the
-        separator to space tab space you can use:
-
-          .separator " \\t "
-
-        The setting is automatically changed when you switch to csv or
-        tabs output mode.  You should also set it before doing an
-        import (ie , for CSV and \\t for TSV).
-        """
-        if len(cmd)!=1:
-            raise self.Error("separator takes exactly one parameter")
-        self.separator=self.fixup_backslashes(cmd[0])
-
-    _shows=("echo", "explain", "headers", "mode", "nullvalue", "output", "separator", "width", "exceptions", "encoding")
-
-    def command_show(self, cmd):
-        """show: Show the current values for various settings."""
-        if len(cmd)>1:
-            raise self.Error("show takes at most one parameter")
-        if len(cmd):
-            what=cmd[0]
-            if what not in self._shows:
-                raise self.Error("Unknown show: '%s'" % (what,))
-        else:
-            what=None
-
-        outs=[]
-        for i in self._shows:
-            k=i
-            if what and i!=what:
-                continue
-            # boolean settings
-            if i in ("echo", "headers", "exceptions"):
-                if i=="headers":
-                    i="header"
-                v="off"
-                if getattr(self, i):
-                    v="on"
-            elif i=="explain":
-                # we cheat by looking at truncate setting!
-                v="on"
-                if self.truncate:
-                    v="off"
-            elif i in ("nullvalue", "separator"):
-                v=self._fmt_c_string(getattr(self, i))
-            elif i=="mode":
-                if not self._output_modes:
-                    self._cache_output_modes()
-                for v in self._output_modes:
-                    if self.output==getattr(self, "output_"+v):
-                        break
-                else:
-                    assert False, "Bug: didn't find output mode"
-            elif i=="output":
-                if self.stdout is self._original_stdout:
-                    v="stdout"
-                else:
-                    v=getattr(self.stdout, "name", "<unknown stdout>")
-            elif i=="width":
-                v=" ".join(["%d"%(i,) for i in self.widths])
-            elif i=="encoding":
-                v=self.encoding[0]
-                if self.encoding[1]:
-                    v+=" (Errors "+self.encoding[1]+")"
-            else:
-                assert False, "Bug: unknown show handling"
-            outs.append((k,v))
-
-        # find width of k column
-        l=0
-        for k,v in outs:
-            if len(k)>l:
-                l=len(k)
-
-        for k,v in outs:
-            self.write(self.stderr, "%*.*s: %s\n" % (l,l, k, v))
-
-    def command_tables(self, cmd):
-        """tables ?PATTERN?: Lists names of tables matching LIKE pattern
-
-        This also returns views.
-        """
-        self.push_output()
-        self.output=self.output_list
-        self.header=False
-        try:
-            if len(cmd)==0:
-                cmd=['%']
-
-            # The SQLite shell code filters out sqlite_ prefixes if
-            # you specified an argument else leaves them in.  It also
-            # has a hand coded output mode that does space separation
-            # plus wrapping at 80 columns.
-            for n in cmd:
-                self.process_sql("SELECT name FROM sqlite_master "
-                                 "WHERE type IN ('table', 'view') AND name NOT LIKE 'sqlite_%' "
-                                 "AND name like ?1 "
-                                 "UNION ALL "
-                                 "SELECT name FROM sqlite_temp_master "
-                                 "WHERE type IN ('table', 'view') AND name NOT LIKE 'sqlite_%' "
-                                 "ORDER BY 1", (n,), internal=True)
-        finally:
-            self.pop_output()
-
-    def command_timeout(self, cmd):
-        """timeout MS: Try opening locked tables for MS milliseconds
-
-        If a database is locked by another process SQLite will keep
-        retrying.  This sets how many thousandths of a second it will
-        keep trying for.  If you supply zero or a negative number then
-        all busy handlers are disabled.
-        """
-        if len(cmd)!=1:
-            raise self.Error("timeout takes a number")
-        try:
-            t=int(cmd[0])
-        except:
-            raise self.Error("%s is not a number" % (cmd[0],))
-        self.db.setbusytimeout(t)
-
-    def command_timer(self, cmd):
-        """timer ON|OFF: Control printing of time and resource usage after each query
-
-        The values displayed are in seconds when shown as floating
-        point or an absolute count.  Only items that have changed
-        since starting the query are shown.  On non-Windows platforms
-        considerably more information can be shown.
-        """
-        if self._boolean_command("timer", cmd):
-            try:
-                self.get_resource_usage()
-            except:
-                raise self.Error("Timing not supported by this Python version/platform")
-            self.timer=True
-        else:
-            self.timer=False
-
-    def command_width(self, cmd):
-        """width NUM NUM ...: Set the column widths for "column" mode
-
-        In "column" output mode, each column is a fixed width with values truncated to
-        fit.  Specify new widths using this command.  Use a negative number
-        to right justify and zero for default column width.
-        """
-        if len(cmd)==0:
-            raise self.Error("You need to specify some widths!")
-        w=[]
-        for i in cmd:
-            try:
-                w.append(int(i))
-            except:
-                raise self.Error("'%s' is not a valid number" % (i,))
-        self.widths=w
-
-    def _terminal_width(self):
-        """Works out the terminal width which is used for word wrapping
-        some output (eg .help)"""
-        try:
-            if sys.platform=="win32":
-                import ctypes, struct
-                h=ctypes.windll.kernel32.GetStdHandle(-12)  # -12 is stderr
-                buf=ctypes.create_string_buffer(22)
-                if ctypes.windll.kernel32.GetConsoleScreenBufferInfo(h, buf):
-                    _,_,_,_,_,left,top,right,bottom,_,_=struct.unpack("hhhhHhhhhhh", buf.raw)
-                    return right-left
-                raise Exception()
-            else:
-                # posix
-                import struct, fcntl, termios
-                s=struct.pack('HHHH', 0,0,0,0)
-                x=fcntl.ioctl(2, termios.TIOCGWINSZ, s)
-                return struct.unpack('HHHH', x)[1]
-        except:
-            try:
-                v=int(os.getenv("COLUMNS"))
-                if v<10:
-                    return 80
-                return v
-            except:
-                return 80
-
-    def push_output(self):
-        """Saves the current output settings onto a stack.  See
-        :meth:`pop_output` for more details as to why you would use
-        this."""
-        o={}
-        for k in "separator", "header", "nullvalue", "output", "widths", "truncate":
-            o[k]=getattr(self, k)
-        self._output_stack.append(o)
-
-    def pop_output(self):
-        """Restores most recently pushed output.  There are many
-        output parameters such as nullvalue, mode
-        (list/tcl/html/insert etc), column widths, header etc.  If you
-        temporarily need to change some settings then
-        :meth:`push_output`, change the settings and then pop the old
-        ones back.
-
-        A simple example is implementing a command like .dump.  Push
-        the current output, change the mode to insert so we get SQL
-        inserts printed and then pop to go back to what was there
-        before.
-
-        """
-        # first item should always be present
-        assert len(self._output_stack)
-        if len(self._output_stack)==1:
-            o=self._output_stack[0]
-        else:
-            o=self._output_stack.pop()
-        for k,v in o.items():
-            setattr(self,k,v)
-
-    def _append_input_description(self):
-        """When displaying an error in :meth:`handle_exception` we
-        want to give context such as when the commands being executed
-        came from a .read command (which in turn could execute another
-        .read).
-        """
-        if self.interactive:
-            return
-        res=[]
-        res.append("Line %d" % (self.input_line_number,))
-        res.append(": "+getattr(self.stdin, "name", "<stdin>"))
-        self._input_descriptions.append(" ".join(res))
-
-    def fixup_backslashes(self, s):
-        """Implements the various backlash sequences in s such as
-        turning backslash t into a tab.
-
-        This function is needed because shlex does not do it for us.
-        """
-        if "\\" not in s:
-            return s
-        # See the resolve_backslashes function in SQLite shell source
-        res=[]
-        i=0
-        while i<len(s):
-            if s[i]!="\\":
-                res.append(s[i])
-                i+=1
-                continue
-            i+=1
-            if i>=len(s):
-                raise self.Error("Backslash with nothing following")
-            c=s[i]
-            res.append({
-                "\\": "\\",
-                "r": "\r",
-                "n": "\n",
-                "t": "\t"
-                }.get(c, None))
-            i+=1  # advance again
-            if res[-1] is None:
-                raise self.Error("Unknown backslash sequence \\"+c)
-        return "".join(res)
-
-    if sys.version_info<(3,0):
-        def write(self, dest, text):
-            """Writes text to dest.  dest will typically be one of self.stdout or self.stderr."""
-            # ensure text is unicode to catch codeset issues here
-            if type(text)!=unicode_type:
-                text=unicode_type(text)
-            try:
-                dest.write(text)
-            except UnicodeEncodeError:
-                ev=sys.exc_info()[1]
-                # See issue108 and try to work around it
-                if ev.args[0]=="ascii" and dest.encoding and ev.args[0]!=dest.encoding and hasattr(dest, "fileno") and \
-                   isinstance(dest.fileno(), int) and dest.fileno()>=0:
-                    args=[dest.encoding,]
-                    if dest.errors:
-                        args.append(dest.errors)
-                    dest.write(text.encode(*args))
-                else:
-                    raise
-
-        _raw_input=raw_input
-    else:
-        def write(self, dest, text):
-            "Writes text to dest.  dest will typically be one of self.stdout or self.stderr."
-            dest.write(text)
-        _raw_input=input
-
-    def getline(self, prompt=""):
-        """Returns a single line of input (may be incomplete SQL) from self.stdin.
-
-        If EOF is reached then return None.  Do not include trailing
-        newline in return.
-        """
-        self.stdout.flush()
-        self.stderr.flush()
-        try:
-            if self.interactive:
-                if self.stdin is sys.stdin:
-                    c=self.colour.prompt, self.colour.prompt_
-                    if self._using_readline and sys.platform!="win32":
-                        # these are needed so that readline knows they are non-printing characters
-                        c="\x01"+c[0]+"\x02", "\x01"+c[1]+"\x02",
-                    line=self._raw_input(c[0]+prompt+c[1])+"\n"  # raw_input excludes newline
-                else:
-                    self.write(self.stdout, prompt)
-                    line=self.stdin.readline()  # includes newline unless last line of file doesn't have one
-            else:
-                line=self.stdin.readline()  # includes newline unless last line of file doesn't have one
-            self.input_line_number+=1
-            if sys.version_info<(3,0):
-                if type(line)!=unicode_type:
-                    enc=getattr(self.stdin, "encoding", self.encoding[0])
-                    if not enc:
-                        enc=self.encoding[0]
-                    line=line.decode(enc)
-        except EOFError:
-            return None
-        if len(line)==0:  # always a \n on the end normally so this is EOF
-            return None
-        if line[-1]=="\n":
-            line=line[:-1]
-        return line
-
-    def getcompleteline(self):
-        """Returns a complete input.
-
-        For dot commands it will be one line.  For SQL statements it
-        will be as many as is necessary to have a
-        :meth:`~apsw.complete` statement (ie semicolon terminated).
-        Returns None on end of file."""
-        try:
-            self._completion_first=True
-            command=self.getline(self.prompt)
-            if command is None:
-                return None
-            if len(command.strip())==0:
-                return ""
-            if command[0]=="?":
-                command=".help "+command[1:]
-            # incomplete SQL?
-            while command[0]!="." and not apsw.complete(command):
-                self._completion_first=False
-                line=self.getline(self.moreprompt)
-                if line is None:  # unexpected eof
-                    raise self.Error("Incomplete SQL (line %d of %s): %s\n" % (self.input_line_number, getattr(self.stdin, "name", "<stdin>"), command))
-                if line in ("go", "/"):
-                    break
-                command=command+"\n"+line
-            return command
-        except KeyboardInterrupt:
-            self.handle_interrupt()
-            return ""
-
-    def handle_interrupt(self):
-        """Deal with keyboard interrupt (typically Control-C).  It
-        will :meth:`~Connection.interrupt` the database and print"^C" if interactive."""
-        self.db.interrupt()
-        if not self.bail and self.interactive:
-            self.write(self.stderr, "^C\n")
-            return
-        raise
-
-    def process_complete_line(self, command):
-        """Given some text will call the appropriate method to process
-        it (eg :meth:`process_sql` or :meth:`process_command`)"""
-        try:
-            if len(command.strip())==0:
-                return
-            if command[0]==".":
-                self.process_command(command)
-            else:
-                self.process_sql(command)
-        except KeyboardInterrupt:
-            self.handle_interrupt()
-
-    def push_input(self):
-        """Saves the current input paramaters to a stack.  See :meth:`pop_input`."""
-        d={}
-        for i in "interactive", "stdin", "input_line_number":
-            d[i]=getattr(self, i)
-        self._input_stack.append(d)
-
-    def pop_input(self):
-        """Restore most recently pushed input parameters (interactive,
-        self.stdin, linenumber etc).  Use this if implementing a
-        command like read.  Push the current input, read the file and
-        then pop the input to go back to before.
-        """
-        assert(len(self._input_stack))>1
-        d=self._input_stack.pop()
-        for k,v in d.items():
-            setattr(self, k, v)
-
-    def complete(self, token, state):
-        """Return a possible completion for readline
-
-        This function is called with state starting at zero to get the
-        first completion, then one/two/three etc until you return None.  The best
-        implementation is to generate the list when state==0, save it,
-        and provide members on each increase.
-
-        The default implementation extracts the current full input
-        from readline and then calls :meth:`complete_command` or
-        :meth:`complete_sql` as appropriate saving the results for
-        subsequent calls.
-        """
-        if state==0:
-            import readline
-            # the whole line
-            line=readline.get_line_buffer()
-            # begining and end(+1) of the token in line
-            beg=readline.get_begidx()
-            end=readline.get_endidx()
-            # Are we matching a command?
-            try:
-                if self._completion_first and line.startswith("."):
-                    self.completions=self.complete_command(line, token, beg, end)
-                else:
-                    self.completions=self.complete_sql(line, token, beg, end)
-            except:
-                # Readline swallows any exceptions we raise.  We
-                # shouldn't be raising any so this is to catch that
-                import traceback
-                traceback.print_exc()
-                raise
-
-        if state>len(self.completions):
-            return None
-        return self.completions[state]
-
-    # Taken from https://sqlite.org/lang_keywords.html
-    _sqlite_keywords="""ABORTADD AFTER ALL ALTER ANALYZE AND AS ASC ATTACH AUTOINCREMENT
-           BEFORE BEGIN BETWEEN BY CASCADE CASE CAST CHECK COLLATE COLUMN COMMIT
-           CONFLICT CONSTRAINT CREATE CROSS CURRENT_DATE CURRENT_TIME
-           CURRENT_TIMESTAMP DATABASE DEFAULT DEFERRABLE DEFERRED DELETE DESC
-           DETACH DISTINCT DROP EACH ELSE END ESCAPE EXCEPT EXCLUSIVE EXISTS
-           EXPLAIN FAIL FOR FOREIGN FROM FULL GLOB GROUP HAVING IF IGNORE
-           IMMEDIATE IN INDEX INDEXED INITIALLY INNER INSERT INSTEAD INTERSECT
-           INTO IS ISNULL JOIN KEY LEFT LIKE LIMIT MATCH NATURAL NOT NOTNULL NULL
-           OF OFFSET ON OR ORDER OUTER PLAN PRAGMA PRIMARY QUERY RAISE REFERENCES
-           REGEXP REINDEX RELEASE RENAME REPLACE RESTRICT RIGHT ROLLBACK ROW
-           SAVEPOINT SELECT SET TABLE TEMP TEMPORARY THEN TO TRANSACTION TRIGGER
-           UNION UNIQUE UPDATE USING VACUUM VALUES VIEW VIRTUAL WHEN WHERE""".split()
-    # reserved words need to be quoted.  Only a subset of the above are reserved
-    # but what the heck
-    _sqlite_reserved=_sqlite_keywords
-    # add a space after each of them except functions which get parentheses
-    _sqlite_keywords=[x+(" ", "(")[x in ("VALUES", "CAST")] for x in _sqlite_keywords]
-
-    _sqlite_special_names="""_ROWID_ OID ROWID SQLITE_MASTER
-           SQLITE_SEQUENCE""".split()
-
-    _sqlite_functions="""abs( changes() char( coalesce( glob( ifnull(
-           hex( instr( last_insert_rowid() length( like(
-           load_extension( lower( ltrim( max( min( nullif( quote(
-           random() randomblob( replace( round( rtrim( soundex(
-           sqlite_compileoption_get( sqlite_compileoption_used(
-           sqlite_source_id() sqlite_version() substr( total_changes()
-           trim( typeof( unicode( upper( zeroblob( date( time( datetime(
-           julianday( strftime(  avg( count( group_concat( sum( total(""".split()
-
-    _pragmas_bool=("yes", "true", "on", "no", "false", "off")
-    _pragmas={"application_id": None,
-              "auto_vacuum=": ("NONE", "FULL", "INCREMENTAL"),
-              "automatic_index=": _pragmas_bool,
-              "cache_size=": None,
-              "case_sensitive_like=": _pragmas_bool,
-              "checkpoint_fullfsync=": _pragmas_bool,
-              "collation_list": None,
-              "compile_options": None,
-              "database_list": None,
-              "default_cache_size=": None,
-              "encoding=": None,
-              # ('"UTF-8"', '"UTF-16"', '"UTF-16le"', '"UTF16-16be"'),
-              # too hard to get " to be part of token just in this special case
-              "foreign_key_check": None,
-              "foreign_key_list(": None,
-              "foreign_keys": _pragmas_bool,
-              "freelist_count": None,
-              "fullfsync=": _pragmas_bool,
-              "ignore_check_constraints": _pragmas_bool,
-              "incremental_vacuum(": None,
-              "index_info(": None,
-              "index_list(": None,
-              "integrity_check": None,
-              "journal_mode=": ("DELETE", "TRUNCATE", "PERSIST", "MEMORY", "OFF", "WAL"),
-              "journal_size_limit=": None,
-              "legacy_file_format=": _pragmas_bool,
-              "locking_mode=": ("NORMAL", "EXCLUSIVE"),
-              "max_page_count=": None,
-              "page_count;": None,
-              "page_size=": None,
-              "quick_check": None,
-              "read_uncommitted=": _pragmas_bool,
-              "recursive_triggers=": _pragmas_bool,
-              "reverse_unordered_selects=": _pragmas_bool,
-              "schema_version": None,
-              "secure_delete=": _pragmas_bool,
-              "shrink_memory": None,
-              "synchronous=": ("OFF", "NORMAL", "FULL"),
-              "table_info(": None,
-              "temp_store=": ("DEFAULT", "FILE", "MEMORY"),
-              "temp_store_directory=": None,
-              "wal_autocheckpoint=": None,
-              "wal_checkpoint": None,
-              "writable_schema": _pragmas_bool,
-              }
-
-    def _get_prev_tokens(self, line, end):
-        "Returns the tokens prior to pos end in the line"
-        return re.findall(r'"?\w+"?', line[:end])
-
-    def complete_sql(self, line, token, beg, end):
-        """Provide some completions for SQL
-
-        :param line: The current complete input line
-        :param token: The word readline is looking for matches
-        :param beg: Integer offset of token in line
-        :param end: Integer end of token in line
-        :return: A list of completions, or an empty list if none
-        """
-        if self._completion_cache is None:
-            cur=self.db.cursor()
-            collations=[row[1] for row in cur.execute("pragma collation_list")]
-            databases=[row[1] for row in cur.execute("pragma database_list")]
-            other=[]
-            for db in databases:
-                if db=="temp":
-                    master="sqlite_temp_master"
-                else:
-                    master="[%s].sqlite_master" % (db,)
-                for row in cur.execute("select * from "+master).fetchall():
-                    for col in (1,2):
-                        if row[col] not in other and not row[col].startswith("sqlite_"):
-                            other.append(row[col])
-                    if row[0]=="table":
-                        try:
-                            for table in cur.execute("pragma [%s].table_info([%s])" % (db, row[1],)).fetchall():
-                                if table[1] not in other:
-                                    other.append(table[1])
-                                for item in table[2].split():
-                                    if item not in other:
-                                        other.append(item)
-                        except apsw.SQLError:
-                            # See https://code.google.com/p/apsw/issues/detail?id=86
-                            pass
-
-            self._completion_cache=[self._sqlite_keywords, self._sqlite_functions, self._sqlite_special_names, collations, databases, other]
-            for i in range(len(self._completion_cache)):
-                self._completion_cache[i].sort()
-
-        # be somewhat sensible about pragmas
-        if "pragma " in line.lower():
-            t=self._get_prev_tokens(line.lower(), end)
-
-            # pragma foo = bar
-            if len(t)>2 and t[-3]=="pragma":
-                # t[-2] should be a valid one
-                for p in self._pragmas:
-                    if p.replace("=","")==t[-2]:
-                        vals=self._pragmas[p]
-                        if not vals:
-                            return []
-                        return [x+";" for x in vals if x.startswith(token)]
-            # at equals?
-            if len(t)>1 and t[-2]=="pragma" and line[:end].replace(" ","").endswith("="):
-                for p in self._pragmas:
-                    if p.replace("=","")==t[-1]:
-                        vals=self._pragmas[p]
-                        if not vals:
-                            return []
-                        return vals
-            # pragma foo
-            if len(t)>1 and t[-2]=="pragma":
-                res=[x for x in self._pragmas.keys() if x.startswith(token)]
-                res.sort()
-                return res
-
-            # pragma
-            if len(t) and t[-1]=="pragma":
-                res=list(self._pragmas.keys())
-                res.sort()
-                return res
-
-        # This is currently not context sensitive (eg it doesn't look
-        # to see if last token was 'FROM' and hence next should only
-        # be table names.  That is a SMOP like pragmas above
-        res=[]
-        ut=token.upper()
-        for corpus in self._completion_cache:
-            for word in corpus:
-                if word.upper().startswith(ut):
-                    # potential match - now match case
-                    if word.startswith(token):  # exact
-                        if word not in res:
-                            res.append(word)
-                    elif word.lower().startswith(token):  # lower
-                        if word.lower() not in res:
-                            res.append(word.lower())
-                    elif word.upper().startswith(token):  # upper
-                        if word.upper() not in res:
-                            res.append(word.upper())
-                    else:
-                        # match letter by letter otherwise readline mangles what was typed in
-                        w=token+word[len(token):]
-                        if w not in res:
-                            res.append(w)
-        return res
-
-    _builtin_commands=None
-
-    def complete_command(self, line, token, beg, end):
-        """Provide some completions for dot commands
-
-        :param line: The current complete input line
-        :param token: The word readline is looking for matches
-        :param beg: Integer offset of token in line
-        :param end: Integer end of token in line
-        :return: A list of completions, or an empty list if none
-        """
-        if not self._builtin_commands:
-            self._builtin_commands=["."+x[len("command_"):] for x in dir(self) if x.startswith("command_") and x!="command_headers"]
-        if beg==0:
-            # some commands don't need a space because they take no
-            # params but who cares?
-            return [x+" " for x in self._builtin_commands if x.startswith(token)]
-        return None
-
-    def get_resource_usage(self):
-        """Return a dict of various numbers (ints or floats).  The
-        .timer command shows the difference between before and after
-        results of what this returns by calling :meth:`display_timing`"""
-        if sys.platform=="win32":
-            import ctypes, time, platform
-            ctypes.windll.kernel32.GetProcessTimes.argtypes=[
-                platform.architecture()[0]=='64bit' and ctypes.c_int64 or ctypes.c_int32,
-                ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p]
-
-            # All 4 out params have to be present.  FILETIME is really
-            # just a 64 bit quantity in 100 nanosecond granularity
-            dummy=ctypes.c_ulonglong()
-            utime=ctypes.c_ulonglong()
-            stime=ctypes.c_ulonglong()
-            rc=ctypes.windll.kernel32.GetProcessTimes(
-                ctypes.windll.kernel32.GetCurrentProcess(),
-                ctypes.byref(dummy),  # creation time
-                ctypes.byref(dummy),  # exit time
-                ctypes.byref(stime),
-                ctypes.byref(utime))
-            if rc:
-                return {'Wall clock': time.time(),
-                        'User time': float(utime.value)/10000000,
-                        'System time': float(stime.value)/10000000}
-            return {}
-        else:
-            import resource, time
-            r=resource.getrusage(resource.RUSAGE_SELF)
-            res={'Wall clock': time.time()}
-            for i,desc in (("utime", "User time"),
-                       ("stime", "System time"),
-                       ("maxrss", "Max rss"),
-                       ("idrss", "Memory"),
-                       ("isrss", "Stack"),
-                       ("ixrss", "Shared Memory"),
-                       ("minflt", "PF (no I/O)"),
-                       ("majflt", "PF (I/O)"),
-                       ("inblock", "Blocks in"),
-                       ("oublock", "Blocks out"),
-                       ("nsignals", "Signals"),
-                       ("nvcsw", "Voluntary context switches"),
-                       ("nivcsw", "Involunary context switches"),
-                       ("msgrcv", "Messages received"),
-                       ("msgsnd", "Messages sent"),
-                       ("nswap", "Swaps"),
-                       ):
-                f="ru_"+i
-                if hasattr(r, f):
-                    res[desc]=getattr(r,f)
-            return res
-
-    def display_timing(self, b4, after):
-        """Writes the difference between b4 and after to self.stderr.
-        The data is dictionaries returned from
-        :meth:`get_resource_usage`."""
-        v=list(b4.keys())
-        for i in after:
-            if i not in v:
-                v.append(i)
-        v.sort()
-        for k in v:
-            if k in b4 and k in after:
-                one=b4[k]
-                two=after[k]
-                val=two-one
-                if val:
-                    if type(val)==float:
-                        self.write(self.stderr, "+ %s: %.4f\n" % (k, val))
-                    else:
-                        self.write(self.stderr, "+ %s: %d\n" % (k, val))
-
-    # Colour support
-
-    def _out_colour(self):
-        # Sets up color for output.  Input being interactive doesn't
-        # matter.  This method needs to be called on all changes to
-        # output.
-        if getattr(self.stdout, "isatty", False) and self.stdout.isatty():
-            self.colour=self._colours[self.colour_scheme]
-        else:
-            self.colour=self._colours["off"]
-
-    # This class returns an empty string for all undefined attributes
-    # so that it doesn't matter if a colour scheme leaves something
-    # out.
-    class _colourscheme:
-
-        def __init__(self, **kwargs):
-            for k,v in kwargs.items():
-                setattr(self, k, v)
-
-        def __nonzero__(self):
-            return True
-
-        def __str__(self):
-            return "_colourscheme("+str(self.__dict__)+")"
-
-        def __getattr__(self, k):
-            return ""
-
-        def colour_value(self, val, formatted):
-            self.colour
-            if val is None:
-                return self.vnull+formatted+self.vnull_
-            if isinstance(val, Shell._basestring):
-                return self.vstring+formatted+self.vstring_
-            if isinstance(val, Shell._binary_type):
-                return self.vblob+formatted+self.vblob_
-            # must be a number - we don't distinguish between float/int
-            return self.vnumber+formatted+self.vnumber_
-
-    # The colour definitions - the convention is the name to turn
-    # something on and the name with an underscore suffix to turn it
-    # off
-    d=_colourscheme(**dict([(v, "\x1b["+str(n)+"m") for n,v in {0: "reset", 1: "bold", 4: "underline", 22: "bold_", 24: "underline_",
-     7: "inverse", 27: "inverse_",
-     30: "fg_black", 31: "fg_red", 32: "fg_green", 33: "fg_yellow", 34: "fg_blue", 35: "fg_magenta", 36: "fg_cyan", 37: "fg_white", 39: "fg_",
-     40: "bg_black", 41: "bg_red", 42: "bg_green", 43: "bg_yellow", 44: "bg_blue", 45: "bg_magenta", 46: "bg_cyan", 47: "bg_white", 49: "bg_"}.items()]))
-
-    _colours={"off": _colourscheme(colour_value=lambda x,y: y)}
-
-    _colours["default"]=_colourscheme(prompt=d.bold, prompt_=d.bold_,
-                                      error=d.fg_red+d.bold, error_=d.bold_+d.fg_,
-                                      intro=d.fg_blue+d.bold, intro_=d.bold_+d.fg_,
-                                      summary=d.fg_blue+d.bold, summary_=d.bold_+d.fg_,
-                                      header=sys.platform=="win32" and d.inverse or d.underline,
-                                      header_=sys.platform=="win32" and d.inverse_ or d.underline_,
-                                      vnull=d.fg_red, vnull_=d.fg_,
-                                      vstring=d.fg_yellow, vstring_=d.fg_,
-                                      vblob=d.fg_blue, vblob_=d.fg_,
-                                      vnumber=d.fg_magenta, vnumber_=d.fg_)
-    if sys.platform=="win32":
-        if not _win_colour:
-            for k in _colours:
-                _colours[k]=_colours["off"]
-    # unpollute namespace
-    del d
-    del _colourscheme
-    try:
-        del n
-        del x
-        del v
-    except:
-        pass
-
-
-def main():
-    # Docstring must start on second line so dedenting works correctly
-    """
-    Call this to run the interactive shell.  It automatically passes
-    in sys.argv[1:] and exits Python when done.
-
-    """
-    try:
-        s=Shell()
-        _,_,cmds=s.process_args(sys.argv[1:])
-        if len(cmds)==0:
-            s.cmdloop()
-    except:
-        v=sys.exc_info()[1]
-        if getattr(v, "_handle_exception_saw_this", False):
-            pass
-        else:
-            # Where did this exception come from?
-            import traceback
-            traceback.print_exc()
-        sys.exit(1)
-
-if __name__=='__main__':
-    main()
diff --git a/src/calibre/utils/date.py b/src/calibre/utils/date.py
index d518da9ee4..396ed9e7b4 100644
--- a/src/calibre/utils/date.py
+++ b/src/calibre/utils/date.py
@@ -11,7 +11,7 @@ from datetime import datetime, time as dtime, timedelta, MINYEAR, MAXYEAR
 from functools import partial
 
 from calibre import strftime
-from calibre.constants import iswindows, isosx, plugins
+from calibre.constants import iswindows, isosx, plugins, preferred_encoding
 from calibre.utils.iso8601 import utc_tz, local_tz, UNDEFINED_DATE
 from calibre.utils.localization import lcdata
 from polyglot.builtins import unicode_type
@@ -101,6 +101,8 @@ def parse_date(date_string, assume_utc=False, as_utc=True, default=None):
     from dateutil.parser import parse
     if not date_string:
         return UNDEFINED_DATE
+    if isinstance(date_string, bytes):
+        date_string = date_string.decode(preferred_encoding, 'replace')
     if default is None:
         func = datetime.utcnow if assume_utc else datetime.now
         default = func().replace(day=15, hour=0, minute=0, second=0, microsecond=0,
diff --git a/src/calibre/utils/formatter_functions.py b/src/calibre/utils/formatter_functions.py
index 953840639a..e8a54c6f4f 100644
--- a/src/calibre/utils/formatter_functions.py
+++ b/src/calibre/utils/formatter_functions.py
@@ -132,7 +132,7 @@ class FormatterFunction(object):
 
     def eval_(self, formatter, kwargs, mi, locals, *args):
         ret = self.evaluate(formatter, kwargs, mi, locals, *args)
-        if isinstance(ret, (str, unicode_type)):
+        if isinstance(ret, (bytes, unicode_type)):
             return ret
         if isinstance(ret, list):
             return ','.join(ret)
diff --git a/src/calibre/utils/ipc/simple_worker.py b/src/calibre/utils/ipc/simple_worker.py
index bb721a5279..decaa9d554 100644
--- a/src/calibre/utils/ipc/simple_worker.py
+++ b/src/calibre/utils/ipc/simple_worker.py
@@ -253,8 +253,8 @@ def offload_worker(env={}, priority='normal', cwd=None):
 def compile_code(src):
     import re, io
     if not isinstance(src, unicode_type):
-        match = re.search(r'coding[:=]\s*([-\w.]+)', src[:200])
-        enc = match.group(1) if match else 'utf-8'
+        match = re.search(br'coding[:=]\s*([-\w.]+)', src[:200])
+        enc = match.group(1).decode('utf-8') if match else 'utf-8'
         src = src.decode(enc)
     # Python complains if there is a coding declaration in a unicode string
     src = re.sub(r'^#.*coding\s*[:=]\s*([-\w.]+)', '#', src, flags=re.MULTILINE)
diff --git a/src/calibre/web/feeds/recipes/__init__.py b/src/calibre/web/feeds/recipes/__init__.py
index 0b08171b6e..a55f4795bf 100644
--- a/src/calibre/web/feeds/recipes/__init__.py
+++ b/src/calibre/web/feeds/recipes/__init__.py
@@ -32,8 +32,8 @@ def compile_recipe(src):
     :return: Recipe class or None, if no such class was found in src
     '''
     if not isinstance(src, unicode_type):
-        match = re.search(r'coding[:=]\s*([-\w.]+)', src[:200])
-        enc = match.group(1) if match else 'utf-8'
+        match = re.search(br'coding[:=]\s*([-\w.]+)', src[:200])
+        enc = match.group(1).decode('utf-8') if match else 'utf-8'
         src = src.decode(enc)
     # Python complains if there is a coding declaration in a unicode string
     src = re.sub(r'^#.*coding\s*[:=]\s*([-\w.]+)', '#', src.lstrip(u'\ufeff'), flags=re.MULTILINE)
diff --git a/src/polyglot/builtins.py b/src/polyglot/builtins.py
index 61a4bb3ccf..d4fbe45933 100644
--- a/src/polyglot/builtins.py
+++ b/src/polyglot/builtins.py
@@ -25,6 +25,7 @@ if is_py3:
     zip = builtins.__dict__['zip']
     map = builtins.__dict__['map']
     filter = builtins.__dict__['filter']
+    range = builtins.__dict__['range']
 
     codepoint_to_chr = chr
     unicode_type = str
@@ -47,6 +48,7 @@ else:
 """)
 
     from future_builtins import zip, map, filter  # noqa
+    range = xrange
     import __builtin__ as builtins
 
     codepoint_to_chr = unichr