python3: migrate lots of code to io.(Bytes|String)IO instead of c?StringIO

This is the only IO representation on python3, and it is backported to
python2 to aid in porting but requires determining whether to handle
bytes or unicode...  which is sort of the point of porting, so let's
handle this properly everywhere we can.

recipes/bloomberg_columns.recipe

@@ -7,7 +7,7 @@ chron.com
 '''
 from datetime import datetime, timedelta
 from lxml import html, etree
-from StringIO import StringIO
+import io
 from calibre.web.feeds.recipes import BasicNewsRecipe
 import urllib2
 from collections import OrderedDict
@@ -31,7 +31,7 @@ def get_article_parsed(this_url):
     page = urllib2.urlopen(req)
     content = page.read()
     parser = etree.HTMLParser()
-    parsed = html.parse(StringIO(content), parser)
+    parsed = html.parse(io.BytesIO(bytes(content)), parser)
     return parsed
 
 

recipes/houston_chronicle.recipe

@@ -9,7 +9,7 @@ chron.com
 import re
 import time
 import urllib2
-from StringIO import StringIO
+import io
 from datetime import datetime
 import traceback
 import sys
@@ -66,7 +66,7 @@ def get_article_parsed(this_url):
     page = urllib2.urlopen(this_url)
     content = page.read()
     parser = etree.HTMLParser()
-    parsed = html.parse(StringIO(content), parser)
+    parsed = html.parse(io.BytesIO(bytes(content)), parser)
     return parsed
 
 

recipes/zeitde_sub.recipe

@@ -13,7 +13,7 @@ Die Zeit EPUB
 import os
 import zipfile
 import re
-import cStringIO
+import io
 from calibre.web.feeds.news import BasicNewsRecipe
 from calibre.ptempfile import PersistentTemporaryFile
 from calibre import walk
@@ -272,7 +272,7 @@ class ZeitEPUBAbo(BasicNewsRecipe):
             with closing(browser.open(cover_url)) as r:
                 cdata = r.read()
             from calibre.ebooks.metadata.pdf import get_metadata
-            stream = cStringIO.StringIO(cdata)
+            stream = io.BytesIO(cdata)
             cdata = None
             mi = get_metadata(stream)
             if mi.cover_data and mi.cover_data[1]:

src/calibre/devices/android/driver.py

@@ -4,10 +4,9 @@ __license__   = 'GPL v3'
 __copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
+import io
 import os
 
-import cStringIO
-
 from calibre import fsync
 from calibre.devices.usbms.driver import USBMS
 from polyglot.builtins import string_or_bytes
@@ -391,12 +390,12 @@ class WEBOS(USBMS):
 
         coverdata = getattr(metadata, 'thumbnail', None)
         if coverdata and coverdata[2]:
-            cover = Image.open(cStringIO.StringIO(coverdata[2]))
+            cover = Image.open(io.BytesIO(coverdata[2]))
         else:
             coverdata = lopen(I('library.png'), 'rb').read()
 
             cover = Image.new('RGB', (120,160), 'black')
-            im = Image.open(cStringIO.StringIO(coverdata))
+            im = Image.open(io.BytesIO(coverdata))
             im.thumbnail((120, 160), Image.ANTIALIAS)
 
             x, y = im.size
@@ -406,7 +405,7 @@ class WEBOS(USBMS):
             draw.text((1, 10), metadata.get('title', _('Unknown')).encode('ascii', 'ignore'))
             draw.text((1, 140), metadata.get('authors', _('Unknown'))[0].encode('ascii', 'ignore'))
 
-        data = cStringIO.StringIO()
+        data = io.BytesIO()
         cover.save(data, 'JPEG')
         coverdata = data.getvalue()
 
@@ -416,12 +415,12 @@ class WEBOS(USBMS):
 
         coverdata = getattr(metadata, 'thumbnail', None)
         if coverdata and coverdata[2]:
-            cover = Image.open(cStringIO.StringIO(coverdata[2]))
+            cover = Image.open(io.BytesIO(coverdata[2]))
         else:
             coverdata = lopen(I('library.png'), 'rb').read()
 
             cover = Image.new('RGB', (52,69), 'black')
-            im = Image.open(cStringIO.StringIO(coverdata))
+            im = Image.open(io.BytesIO(coverdata))
             im.thumbnail((52, 69), Image.ANTIALIAS)
 
             x, y = im.size
@@ -429,7 +428,7 @@ class WEBOS(USBMS):
 
         cover2 = cover.resize((52, 69), Image.ANTIALIAS).convert('RGB')
 
-        data = cStringIO.StringIO()
+        data = io.BytesIO()
         cover2.save(data, 'JPEG')
         coverdata = data.getvalue()
 

src/calibre/devices/kindle/bookmark.py

@@ -5,7 +5,7 @@ __license__   = 'GPL v3'
 __docformat__ = 'restructuredtext en'
 
 import os
-from cStringIO import StringIO
+import io
 from struct import unpack
 
 
@@ -51,7 +51,7 @@ class Bookmark():  # {{{
         if self.bookmark_extension == 'mbp':
             MAGIC_MOBI_CONSTANT = 150
             with lopen(self.path,'rb') as f:
-                stream = StringIO(f.read())
+                stream = io.BytesIO(f.read())
                 data = StreamSlicer(stream)
                 self.timestamp, = unpack('>I', data[0x24:0x28])
                 bpar_offset, = unpack('>I', data[0x4e:0x52])
@@ -148,7 +148,7 @@ class Bookmark():  # {{{
                 # This will find the first instance of a clipping only
                 book_fs = self.path.replace('.%s' % self.bookmark_extension,'.%s' % self.book_format)
                 with lopen(book_fs,'rb') as f2:
-                    stream = StringIO(f2.read())
+                    stream = io.BytesIO(f2.read())
                     mi = get_topaz_metadata(stream)
                 my_clippings = self.path
                 split = my_clippings.find('documents') + len('documents/')
@@ -179,7 +179,7 @@ class Bookmark():  # {{{
             MAGIC_TOPAZ_CONSTANT = 33.33
             self.timestamp = os.path.getmtime(self.path)
             with lopen(self.path,'rb') as f:
-                stream = StringIO(f.read())
+                stream = io.BytesIO(f.read())
                 data = StreamSlicer(stream)
                 self.last_read = int(unpack('>I', data[5:9])[0])
                 self.last_read_location = self.last_read/MAGIC_TOPAZ_CONSTANT + 1
@@ -220,7 +220,7 @@ class Bookmark():  # {{{
         elif self.bookmark_extension == 'pdr':
             self.timestamp = os.path.getmtime(self.path)
             with lopen(self.path,'rb') as f:
-                stream = StringIO(f.read())
+                stream = io.BytesIO(f.read())
                 data = StreamSlicer(stream)
                 self.last_read = int(unpack('>I', data[5:9])[0])
                 entries, = unpack('>I', data[9:13])
@@ -289,7 +289,7 @@ class Bookmark():  # {{{
             # Read the book len from the header
             try:
                 with lopen(book_fs,'rb') as f:
-                    self.stream = StringIO(f.read())
+                    self.stream = io.BytesIO(f.read())
                     self.data = StreamSlicer(self.stream)
                     self.nrecs, = unpack('>H', self.data[76:78])
                     record0 = self.record(0)

src/calibre/devices/nook/driver.py

@@ -8,9 +8,7 @@ __docformat__ = 'restructuredtext en'
 Device driver for Barns and Nobel's Nook
 '''
 
-import os, errno
+import io, os, errno
-
-import cStringIO
 
 from calibre import fsync, prints
 from calibre.constants import DEBUG
@@ -57,12 +55,12 @@ class NOOK(USBMS):
 
         coverdata = getattr(metadata, 'thumbnail', None)
         if coverdata and coverdata[2]:
-            cover = Image.open(cStringIO.StringIO(coverdata[2]))
+            cover = Image.open(io.BytesIO(coverdata[2]))
         else:
             coverdata = lopen(I('library.png'), 'rb').read()
 
             cover = Image.new('RGB', (96, 144), 'black')
-            im = Image.open(cStringIO.StringIO(coverdata))
+            im = Image.open(io.BytesIO(coverdata))
             im.thumbnail((96, 144), Image.ANTIALIAS)
 
             x, y = im.size
@@ -72,7 +70,7 @@ class NOOK(USBMS):
             draw.text((1, 15), metadata.get('title', _('Unknown')).encode('ascii', 'ignore'))
             draw.text((1, 115), metadata.get('authors', _('Unknown')).encode('ascii', 'ignore'))
 
-        data = cStringIO.StringIO()
+        data = io.BytesIO()
         cover.save(data, 'JPEG')
         coverdata = data.getvalue()
 

src/calibre/ebooks/chm/metadata.py

@@ -126,10 +126,10 @@ def _get_cover(soup, rdr):
                 ans = None
         if ans is not None:
             from PIL import Image
-            from cStringIO import StringIO
+            import io
-            buf = StringIO()
+            buf = io.BytesIO()
             try:
-                Image.open(StringIO(ans)).convert('RGB').save(buf, 'JPEG')
+                Image.open(io.BytesIO(ans)).convert('RGB').save(buf, 'JPEG')
                 ans = buf.getvalue()
             except:
                 ans = None

src/calibre/ebooks/compression/palmdoc.py

@@ -5,7 +5,7 @@ from __future__ import print_function
 __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 
-from cStringIO import StringIO
+import io
 from struct import pack
 
 from calibre.constants import plugins
@@ -50,7 +50,7 @@ def test():
 
 
 def py_compress_doc(data):
-    out = StringIO()
+    out = io.BytesIO()
     i = 0
     ldata = len(data)
     while i < ldata:

src/calibre/ebooks/conversion/plugins/htmlz_output.py

@@ -6,9 +6,8 @@ __license__ = 'GPL 3'
 __copyright__ = '2011, John Schember <john@nachtimwald.com>'
 __docformat__ = 'restructuredtext en'
 
+import io
 import os
-from cStringIO import StringIO
-
 
 from calibre.customize.conversion import OutputFormatPlugin, \
     OptionRecommendation
@@ -127,7 +126,7 @@ class HTMLZOutput(OutputFormatPlugin):
 
             # Metadata
             with open(os.path.join(tdir, u'metadata.opf'), 'wb') as mdataf:
-                opf = OPF(StringIO(etree.tostring(oeb_book.metadata.to_opf1())))
+                opf = OPF(io.BytesIO(etree.tostring(oeb_book.metadata.to_opf1())))
                 mi = opf.to_book_metadata()
                 if cover_path:
                     mi.cover = u'cover.jpg'

src/calibre/ebooks/conversion/plugins/pml_output.py

@@ -4,7 +4,7 @@ __license__ = 'GPL 3'
 __copyright__ = '2009, John Schember <john@nachtimwald.com>'
 __docformat__ = 'restructuredtext en'
 
-import os, cStringIO
+import os, io
 
 from calibre.customize.conversion import (OutputFormatPlugin,
         OptionRecommendation)
@@ -65,12 +65,12 @@ class PMLOutput(OutputFormatPlugin):
         for item in manifest:
             if item.media_type in OEB_RASTER_IMAGES and item.href in image_hrefs.keys():
                 if opts.full_image_depth:
-                    im = Image.open(cStringIO.StringIO(item.data))
+                    im = Image.open(io.BytesIO(item.data))
                 else:
-                    im = Image.open(cStringIO.StringIO(item.data)).convert('P')
+                    im = Image.open(io.BytesIO(item.data)).convert('P')
                     im.thumbnail((300,300), Image.ANTIALIAS)
 
-                data = cStringIO.StringIO()
+                data = io.BytesIO()
                 im.save(data, 'PNG')
                 data = data.getvalue()
 

src/calibre/ebooks/conversion/plumber.py

@@ -923,12 +923,12 @@ OptionRecommendation(name='search_replace',
     def download_cover(self, url):
         from calibre import browser
         from PIL import Image
-        from cStringIO import StringIO
+        import io
         from calibre.ptempfile import PersistentTemporaryFile
         self.log('Downloading cover from %r'%url)
         br = browser()
         raw = br.open_novisit(url).read()
-        buf = StringIO(raw)
+        buf = io.BytesIO(raw)
         pt = PersistentTemporaryFile('.jpg')
         pt.close()
         img = Image.open(buf)

src/calibre/ebooks/lit/reader.py

@@ -8,9 +8,8 @@ __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net> ' \
     'and Marshall T. Vandegrift <llasram@gmail.com>'
 
-import struct, os, functools, re
+import io, struct, os, functools, re
 from urlparse import urldefrag
-from cStringIO import StringIO
 from urllib import unquote as urlunquote
 
 from lxml import etree
@@ -143,7 +142,7 @@ class UnBinary(object):
         self.is_html = map is HTML_MAP
         self.tag_atoms, self.attr_atoms = atoms
         self.dir = os.path.dirname(path)
-        buf = StringIO()
+        buf = io.BytesIO()
         self.binary_to_text(bin, buf)
         self.raw = buf.getvalue().lstrip()
         self.escape_reserved()

src/calibre/ebooks/lit/writer.py

@@ -7,9 +7,9 @@ from __future__ import print_function
 __license__   = 'GPL v3'
 __copyright__ = '2008, Marshall T. Vandegrift <llasram@gmail.com>'
 
-from cStringIO import StringIO
 from struct import pack
 from itertools import izip, count, chain
+import io
 import time
 import random
 import re
@@ -151,7 +151,7 @@ class ReBinary(object):
         self.logger = oeb.logger
         self.manifest = oeb.manifest
         self.tags, self.tattrs = map
-        self.buf = StringIO()
+        self.buf = io.BytesIO()
         self.anchors = []
         self.page_breaks = []
         self.is_html  = is_html = map is HTML_MAP
@@ -282,7 +282,7 @@ class ReBinary(object):
         if len(self.anchors) > 6:
             self.logger.warn("More than six anchors in file %r. "
                 "Some links may not work properly." % self.item.href)
-        data = StringIO()
+        data = io.BytesIO()
         data.write(codepoint_to_chr(len(self.anchors)).encode('utf-8'))
         for anchor, offset in self.anchors:
             data.write(codepoint_to_chr(len(anchor)).encode('utf-8'))
@@ -333,7 +333,7 @@ class LitWriter(object):
         self._oeb = oeb
         self._logger = oeb.logger
         self._stream = stream
-        self._sections = [StringIO() for i in range(4)]
+        self._sections = [io.BytesIO() for i in range(4)]
         self._directory = []
         self._meta = None
         self._litize_oeb()
@@ -403,7 +403,7 @@ class LitWriter(object):
         piece2_offset = self._tell()
         self._write('IFCM', pack('<IIIQQ',
             1, CCHUNK_SIZE, 0x20000, ULL_NEG1, 1))
-        cchunk = StringIO()
+        cchunk = io.BytesIO()
         last = 0
         for i, dcount in izip(count(), dcounts):
             cchunk.write(decint(last))
@@ -505,7 +505,7 @@ class LitWriter(object):
                 manifest['css'].append(item)
             elif item.media_type in LIT_IMAGES:
                 manifest['images'].append(item)
-        data = StringIO()
+        data = io.BytesIO()
         data.write(pack('<Bc', 1, '\\'))
         offset = 0
         for state in states:
@@ -533,9 +533,9 @@ class LitWriter(object):
         self._add_file('/manifest', data.getvalue())
 
     def _build_page_breaks(self):
-        pb1 = StringIO()
+        pb1 = io.BytesIO()
-        pb2 = StringIO()
+        pb2 = io.BytesIO()
-        pb3 = StringIO()
+        pb3 = io.BytesIO()
         pb3cur = 0
         bits = 0
         linear = []
@@ -591,7 +591,7 @@ class LitWriter(object):
         self._add_file('/Version', pack('<HH', 8, 1))
 
     def _build_namelist(self):
-        data = StringIO()
+        data = io.BytesIO()
         data.write(pack('<HH', 0x3c, len(self._sections)))
         names = ['Uncompressed', 'MSCompressed', 'EbEncryptDS',
                  'EbEncryptOnlyDS']
@@ -628,7 +628,7 @@ class LitWriter(object):
                     unlen = len(data)
                     lzx = Compressor(17)
                     data, rtable = lzx.compress(data, flush=True)
-                    rdata = StringIO()
+                    rdata = io.BytesIO()
                     rdata.write(pack('<IIIIQQQQ',
                         3, len(rtable), 8, 0x28, unlen, len(data), 0x8000, 0))
                     for uncomp, comp in rtable[:-1]:
@@ -673,7 +673,7 @@ class LitWriter(object):
         directory.sort(cmp=lambda x, y:
             cmp(x.name.lower(), y.name.lower()))
         qrn = 1 + (1 << 2)
-        dchunk = StringIO()
+        dchunk = io.BytesIO()
         dcount = 0
         quickref = []
         name = directory[0].name
@@ -685,7 +685,7 @@ class LitWriter(object):
             usedlen = dchunk.tell() + len(next) + (len(quickref) * 2) + 52
             if usedlen >= DCHUNK_SIZE:
                 ddata.append((dchunk.getvalue(), quickref, dcount, name))
-                dchunk = StringIO()
+                dchunk = io.BytesIO()
                 dcount = 0
                 quickref = []
                 name = en
@@ -700,9 +700,9 @@ class LitWriter(object):
         dcounts = []
         ichunk = None
         if len(ddata) > 1:
-            ichunk = StringIO()
+            ichunk = io.BytesIO()
         for cid, (content, quickref, dcount, name) in izip(count(), ddata):
-            dchunk = StringIO()
+            dchunk = io.BytesIO()
             prev = cid - 1 if cid > 0 else ULL_NEG1
             next = cid + 1 if cid < cidmax else ULL_NEG1
             rem = DCHUNK_SIZE - (len(content) + 50)

src/calibre/ebooks/lrf/meta.py

@@ -13,9 +13,8 @@ to get and set meta information. For example:
 >>> lrf.category = "History"
 """
 
-import struct, zlib, sys, os
+import io, struct, zlib, sys, os
 from shutil import copyfileobj
-from cStringIO import StringIO
 import xml.dom.minidom as dom
 from functools import wraps
 
@@ -238,7 +237,7 @@ def insert_into_file(fileobj, data, start, end):
     @param end:     The position in fileobj of data that must not be overwritten
     @return:        C{start + len(data) - end}
     """
-    buffer = StringIO()
+    buffer = io.BytesIO()
     fileobj.seek(end)
     copyfileobj(fileobj, buffer, -1)
     buffer.flush()

src/calibre/ebooks/lrf/objects.py

@@ -1,7 +1,7 @@
 from __future__ import print_function
 __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
-import struct, array, zlib, cStringIO, collections, re
+import struct, array, zlib, io, collections, re
 
 from calibre.ebooks.lrf import LRFParseError, PRS500_PROFILE
 from calibre.constants import ispy3
@@ -98,7 +98,7 @@ class LRFContentObject(LRFObject):
     tag_map = {}
 
     def __init__(self, bytes, objects):
-        self.stream = bytes if hasattr(bytes, 'read') else cStringIO.StringIO(bytes)
+        self.stream = bytes if hasattr(bytes, 'read') else io.BytesIO(bytes)
         length = self.stream_size()
         self.objects = objects
         self._contents = []
@@ -601,7 +601,7 @@ class Block(LRFStream, TextCSS):
 
     def initialize(self):
         self.attrs = {}
-        stream = cStringIO.StringIO(self.stream)
+        stream = io.BytesIO(self.stream)
         tag = Tag(stream)
         if tag.id != 0xF503:
             raise LRFParseError("Bad block content")
@@ -836,7 +836,7 @@ class Text(LRFStream):
 
     def initialize(self):
         self.content = collections.deque()
-        stream = cStringIO.StringIO(self.stream)
+        stream = io.BytesIO(self.stream)
         length = len(self.stream)
         style = self.style.as_dict()
         current_style = style.copy()
@@ -1017,7 +1017,7 @@ class Canvas(LRFStream):
             if hasattr(self, attr):
                 self.attrs[attr] = getattr(self, attr)
         self._contents = []
-        stream = cStringIO.StringIO(self.stream)
+        stream = io.BytesIO(self.stream)
         while stream.tell() < len(self.stream):
             tag = Tag(stream)
             try:
@@ -1266,7 +1266,7 @@ class TocLabel(object):
 class TOCObject(LRFStream):
 
     def initialize(self):
-        stream = cStringIO.StringIO(self.stream)
+        stream = io.BytesIO(self.stream)
         c = struct.unpack("<H", stream.read(2))[0]
         stream.seek(4*(c+1))
         self._contents = []

src/calibre/ebooks/lrf/pylrs/pylrf.py

@@ -5,7 +5,7 @@
 """
 import struct
 import zlib
-import StringIO
+import io
 import codecs
 import os
 
@@ -473,7 +473,7 @@ class LrfTagStream(LrfStreamBase):
 
     def getStreamTags(self, encoding,
             optimizeTags=False, optimizeCompression=False):
-        stream = StringIO.StringIO()
+        stream = io.BytesIO()
         if optimizeTags:
             tagListOptimizer(self.tags)
 
@@ -587,7 +587,7 @@ class LrfToc(LrfObject):
         self.tags.extend(stream.getStreamTags())
 
     def _makeTocStream(self, toc, se):
-        stream = StringIO.StringIO()
+        stream = io.BytesIO()
         nEntries = len(toc)
 
         writeDWord(stream, nEntries)
@@ -770,4 +770,3 @@ class LrfWriter(object):
     def writeObjectTable(self, lrf):
         for tableEntry in self.objectTable:
             tableEntry.write(lrf)
-

src/calibre/ebooks/metadata/epub.py

@@ -6,8 +6,7 @@ __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 
 '''Read meta information from epub files'''
 
-import os, re, posixpath
+import io, os, re, posixpath
-from cStringIO import StringIO
 from contextlib import closing
 
 from calibre.utils.zipfile import ZipFile, BadZipfile, safe_replace
@@ -151,7 +150,7 @@ class OCFZipReader(OCFReader):
     def open(self, name, mode='r'):
         if isinstance(self.archive, LocalZipFile):
             return self.archive.open(name)
-        return StringIO(self.archive.read(name))
+        return io.BytesIO(self.archive.read(name))
 
     def read_bytes(self, name):
         return self.archive.read(name)
@@ -324,5 +323,3 @@ def set_metadata(stream, mi, apply_null=False, update_timestamp=False, force_ide
             os.remove(replacements[cpath].name)
     except:
         pass
-
-

src/calibre/ebooks/metadata/extz.py

@@ -7,10 +7,9 @@ __copyright__ = '2011, John Schember <john@nachtimwald.com>'
 Read meta information from extZ (TXTZ, HTMLZ...) files.
 '''
 
+import io
 import os
 
-from cStringIO import StringIO
-
 from calibre.ebooks.metadata import MetaInformation
 from calibre.ebooks.metadata.opf2 import OPF
 from calibre.ptempfile import PersistentTemporaryFile
@@ -26,7 +25,7 @@ def get_metadata(stream, extract_cover=True):
     try:
         with ZipFile(stream) as zf:
             opf_name = get_first_opf_name(zf)
-            opf_stream = StringIO(zf.read(opf_name))
+            opf_stream = io.BytesIO(zf.read(opf_name))
             opf = OPF(opf_stream)
             mi = opf.to_book_metadata()
             if extract_cover:
@@ -53,7 +52,7 @@ def set_metadata(stream, mi):
     # Get the OPF in the archive.
     with ZipFile(stream) as zf:
         opf_path = get_first_opf_name(zf)
-        opf_stream = StringIO(zf.read(opf_path))
+        opf_stream = io.BytesIO(zf.read(opf_path))
     opf = OPF(opf_stream)
 
     # Cover.
@@ -77,7 +76,7 @@ def set_metadata(stream, mi):
 
     # Update the metadata.
     opf.smart_update(mi, replace_metadata=True)
-    newopf = StringIO(opf.render())
+    newopf = io.BytesIO(opf.render())
     safe_replace(stream, opf_path, newopf, extra_replacements=replacements, add_missing=True)
 
     # Cleanup temporary files.

src/calibre/ebooks/metadata/lit.py

@@ -4,7 +4,7 @@ __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 Support for reading the metadata from a LIT file.
 '''
 
-import cStringIO, os
+import io, os
 
 from calibre.ebooks.metadata.opf2 import OPF
 
@@ -15,7 +15,7 @@ def get_metadata(stream):
     litfile = LitContainer(stream, Log())
     src = litfile.get_metadata().encode('utf-8')
     litfile = litfile._litfile
-    opf = OPF(cStringIO.StringIO(src), os.getcwdu())
+    opf = OPF(io.BytesIO(src), os.getcwdu())
     mi = opf.to_book_metadata()
     covers = []
     for item in opf.iterguide():
@@ -39,4 +39,3 @@ def get_metadata(stream):
             idx = 1
     mi.cover_data = ('jpg', covers[idx][0])
     return mi
-

src/calibre/ebooks/metadata/odt.py

@@ -21,9 +21,8 @@
 #
 from __future__ import division
 
-import zipfile, re
+import zipfile, re, io
 import xml.sax.saxutils
-from cStringIO import StringIO
 
 from odf.namespaces import OFFICENS, DCNS, METANS
 from odf.opendocument import load as odLoad
@@ -168,7 +167,7 @@ def get_metadata(stream, extract_cover=True):
     parser.setFeature(xml.sax.handler.feature_external_ges, False)
     parser.setContentHandler(odfs)
     content = zin.read('meta.xml')
-    parser.parse(StringIO(content))
+    parser.parse(io.BytesIO(content))
     data = odfs.seenfields
     mi = MetaInformation(None, [])
     if 'title' in data:

src/calibre/ebooks/metadata/opf2.py

@@ -8,7 +8,7 @@ __docformat__ = 'restructuredtext en'
 lxml based OPF parser.
 '''
 
-import re, sys, unittest, functools, os, uuid, glob, cStringIO, json, copy
+import re, sys, unittest, functools, os, uuid, glob, io, json, copy
 from urllib import unquote
 from urlparse import urlparse
 
@@ -1723,7 +1723,6 @@ def metadata_to_opf(mi, as_string=True, default_lang=None):
 
 def test_m2o():
     from calibre.utils.date import now as nowf
-    from cStringIO import StringIO
     mi = MetaInformation('test & title', ['a"1', "a'2"])
     mi.title_sort = 'a\'"b'
     mi.author_sort = 'author sort'
@@ -1742,7 +1741,7 @@ def test_m2o():
     mi.cover = os.path.abspath('asd.jpg')
     opf = metadata_to_opf(mi)
     print(opf)
-    newmi = MetaInformation(OPF(StringIO(opf)))
+    newmi = MetaInformation(OPF(io.BytesIO(opf)))
     for attr in ('author_sort', 'title_sort', 'comments',
                     'publisher', 'series', 'series_index', 'rating',
                     'isbn', 'tags', 'cover_data', 'application_id',
@@ -1760,7 +1759,7 @@ def test_m2o():
 class OPFTest(unittest.TestCase):
 
     def setUp(self):
-        self.stream = cStringIO.StringIO(
+        self.stream = io.BytesIO(
 '''\
 <?xml version="1.0"  encoding="UTF-8"?>
 <package version="2.0" xmlns="http://www.idpf.org/2007/opf" >
@@ -1814,10 +1813,10 @@ class OPFTest(unittest.TestCase):
 
     def testCreator(self):
         opf = OPFCreator(os.getcwdu(), self.opf)
-        buf = cStringIO.StringIO()
+        buf = io.BytesIO()
         opf.render(buf)
         raw = buf.getvalue()
-        self.testReading(opf=OPF(cStringIO.StringIO(raw), os.getcwdu()))
+        self.testReading(opf=OPF(io.BytesIO(raw), os.getcwdu()))
 
     def testSmartUpdate(self):
         self.opf.smart_update(MetaInformation(self.opf))
@@ -1833,7 +1832,6 @@ def test():
 
 
 def test_user_metadata():
-    from cStringIO import StringIO
     mi = Metadata('Test title', ['test author1', 'test author2'])
     um = {
         '#myseries': {'#value#': u'test series\xe4', 'datatype':'text',
@@ -1846,12 +1844,12 @@ def test_user_metadata():
     mi.set_all_user_metadata(um)
     raw = metadata_to_opf(mi)
     opfc = OPFCreator(os.getcwdu(), other=mi)
-    out = StringIO()
+    out = io.BytesIO()
     opfc.render(out)
     raw2 = out.getvalue()
-    f = StringIO(raw)
+    f = io.BytesIO(raw)
     opf = OPF(f)
-    f2 = StringIO(raw2)
+    f2 = io.BytesIO(raw2)
     opf2 = OPF(f2)
     assert um == opf._user_metadata_
     assert um == opf2._user_metadata_

src/calibre/ebooks/metadata/snb.py

@@ -6,7 +6,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2010, Li Fanxi <lifanxi@freemindworld.com>'
 
 import os
-from StringIO import StringIO
+import io
 from calibre.ebooks.metadata import MetaInformation
 from calibre.ebooks.snb.snbfile import SNBFile
 from lxml import etree
@@ -19,7 +19,7 @@ def get_metadata(stream, extract_cover=True):
 
     try:
         if not hasattr(stream, 'write'):
-            snbFile.Parse(StringIO(stream), True)
+            snbFile.Parse(io.BytesIO(stream), True)
         else:
             stream.seek(0)
             snbFile.Parse(stream, True)

src/calibre/ebooks/metadata/topaz.py

@@ -5,7 +5,7 @@ __copyright__ = '2010, Greg Riker <griker@hotmail.com>'
 __docformat__ = 'restructuredtext en'
 
 ''' Read/write metadata from Amazon's topaz format '''
-import StringIO, sys, numbers
+import io, sys, numbers
 from struct import pack
 
 from calibre.ebooks.metadata import MetaInformation
@@ -194,7 +194,7 @@ class MetadataUpdater(object):
                 else:
                     return None
         dkey = self.topaz_headers[x]
-        dks = StringIO.StringIO()
+        dks = io.StringIO()
         dks.write(self.encode_vwi(len(dkey['tag'])))
         offset += 1
         dks.write(dkey['tag'])
@@ -290,7 +290,7 @@ class MetadataUpdater(object):
         delta = updated_md_len - original_md_len
 
         # Copy the first 5 bytes of the file: sig + num_recs
-        ths = StringIO.StringIO()
+        ths = io.StringIO()
         ths.write(self.data[:5])
 
         # Rewrite the offsets for hdr_offsets > metadata offset
@@ -377,9 +377,8 @@ if __name__ == '__main__':
         print(get_metadata(open(sys.argv[1], 'rb')))
     else:
         # Test set_metadata()
-        import cStringIO
         data = open(sys.argv[1], 'rb')
-        stream = cStringIO.StringIO()
+        stream = io.BytesIO()
         stream.write(data.read())
         mi = MetaInformation(title="Updated Title", authors=['Author, Random'])
         set_metadata(stream, mi)

src/calibre/ebooks/mobi/writer2/indexer.py

@@ -10,7 +10,7 @@ __docformat__ = 'restructuredtext en'
 
 import numbers
 from struct import pack
-from cStringIO import StringIO
+import io
 from collections import OrderedDict, defaultdict
 
 from calibre.ebooks.mobi.utils import (encint, encode_number_as_hex,
@@ -166,7 +166,7 @@ class IndexEntry(object):
 
     @property
     def bytestring(self):
-        buf = StringIO()
+        buf = io.BytesIO()
         if isinstance(self.index, numbers.Integral):
             buf.write(encode_number_as_hex(self.index))
         else:
@@ -294,7 +294,7 @@ class TBS(object):  # {{{
                 self.book_tbs(data, first)
 
     def periodical_tbs(self, data, first, depth_map):
-        buf = StringIO()
+        buf = io.BytesIO()
 
         has_section_start = (depth_map[1] and
                 set(depth_map[1]).intersection(set(data['starts'])))
@@ -496,7 +496,7 @@ class Indexer(object):  # {{{
 
     def create_index_record(self, secondary=False):  # {{{
         header_length = 192
-        buf = StringIO()
+        buf = io.BytesIO()
         indices = list(SecondaryIndexEntry.entries()) if secondary else self.indices
 
         # Write index entries
@@ -539,7 +539,7 @@ class Indexer(object):  # {{{
     # }}}
 
     def create_header(self, secondary=False):  # {{{
-        buf = StringIO()
+        buf = io.BytesIO()
         if secondary:
             tagx_block = TAGX().secondary
         else:

src/calibre/ebooks/mobi/writer2/main.py

@@ -7,8 +7,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import random, time
+import io, random, time
-from cStringIO import StringIO
 from struct import pack
 
 from calibre.ebooks import normalize
@@ -130,7 +129,7 @@ class MobiWriter(object):
             pbreak = 0
             running = offset
 
-            buf = StringIO()
+            buf = io.BytesIO()
 
             while breaks and (breaks[0] - offset) < RECORD_SIZE:
                 pbreak = (breaks.pop(0) - running) >> 3
@@ -163,7 +162,7 @@ class MobiWriter(object):
                 write_page_breaks_after_item=self.write_page_breaks_after_item)
         text = self.serializer()
         self.text_length = len(text)
-        text = StringIO(text)
+        text = io.BytesIO(text)
         nrecords = 0
         records_size = 0
 
@@ -228,7 +227,7 @@ class MobiWriter(object):
         # EOF record
         self.records.append(b'\xE9\x8E\x0D\x0A')
 
-        record0 = StringIO()
+        record0 = io.BytesIO()
         # The MOBI Header
         record0.write(pack(b'>HHIHHHH',
             self.compression,  # compression type # compression type

src/calibre/ebooks/oeb/reader.py

@@ -7,7 +7,7 @@ from __future__ import print_function
 __license__   = 'GPL v3'
 __copyright__ = '2008, Marshall T. Vandegrift <llasram@gmail.com>'
 
-import sys, os, uuid, copy, re, cStringIO
+import sys, os, uuid, copy, re, io
 from itertools import izip
 from urlparse import urldefrag, urlparse
 from urllib import unquote as urlunquote
@@ -138,7 +138,7 @@ class OEBReader(object):
     def _metadata_from_opf(self, opf):
         from calibre.ebooks.metadata.opf2 import OPF
         from calibre.ebooks.oeb.transforms.metadata import meta_info_to_oeb_metadata
-        stream = cStringIO.StringIO(etree.tostring(opf, xml_declaration=True, encoding='utf-8'))
+        stream = io.BytesIO(etree.tostring(opf, xml_declaration=True, encoding='utf-8'))
         o = OPF(stream)
         pwm = o.primary_writing_mode
         if pwm:

src/calibre/ebooks/pdb/ereader/writer.py

@@ -8,6 +8,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2009, John Schember <john@nachtimwald.com>'
 __docformat__ = 'restructuredtext en'
 
+import io
 import re
 import struct
 import zlib
@@ -18,8 +19,6 @@ try:
 except ImportError:
     import Image
 
-import cStringIO
-
 from calibre.ebooks.pdb.formatwriter import FormatWriter
 from calibre.ebooks.pdb.header import PdbHeaderBuilder
 from calibre.ebooks.pml.pmlml import PMLMLizer
@@ -141,10 +140,10 @@ class Writer(FormatWriter):
         for item in manifest:
             if item.media_type in OEB_RASTER_IMAGES and item.href in image_hrefs.keys():
                 try:
-                    im = Image.open(cStringIO.StringIO(item.data)).convert('P')
+                    im = Image.open(io.BytesIO(item.data)).convert('P')
                     im.thumbnail((300,300), Image.ANTIALIAS)
 
-                    data = cStringIO.StringIO()
+                    data = io.BytesIO()
                     im.save(data, 'PNG')
                     data = data.getvalue()
 

src/calibre/ebooks/rb/writer.py

@@ -4,6 +4,7 @@ __license__ = 'GPL 3'
 __copyright__ = '2009, John Schember <john@nachtimwald.com>'
 __docformat__ = 'restructuredtext en'
 
+import io
 import struct
 import zlib
 
@@ -13,8 +14,6 @@ try:
 except ImportError:
     import Image
 
-import cStringIO
-
 from calibre.ebooks.rb.rbml import RBMLizer
 from calibre.ebooks.rb import HEADER
 from calibre.ebooks.rb import unique_name
@@ -121,8 +120,8 @@ class RBWriter(object):
                 try:
                     data = ''
 
-                    im = Image.open(cStringIO.StringIO(item.data)).convert('L')
+                    im = Image.open(io.BytesIO(item.data)).convert('L')
-                    data = cStringIO.StringIO()
+                    data = io.BytesIO()
                     im.save(data, 'PNG')
                     data = data.getvalue()
 
@@ -152,4 +151,3 @@ class RBWriter(object):
         text += 'BODY=index.html\n'
 
         return text
-

src/calibre/ebooks/rtf/rtfml.py

@@ -10,7 +10,7 @@ Transform OEB content into RTF markup
 
 import os
 import re
-import cStringIO
+import io
 
 from lxml import etree
 
@@ -79,15 +79,15 @@ def txt2rtf(text):
     if not isinstance(text, unicode_type):
         return text
 
-    buf = cStringIO.StringIO()
+    buf = io.StringIO()
     for x in text:
         val = ord(x)
         if val == 160:
-            buf.write('\\~')
+            buf.write(u'\\~')
         elif val <= 127:
-            buf.write(x)
+            buf.write(unicode_type(x))
         else:
-            c = r'\u{0:d}?'.format(val)
+            c = unicode_type(r'\u{0:d}?'.format(val))
             buf.write(c)
     return buf.getvalue()
 

src/calibre/gui2/__init__.py

@@ -1261,7 +1261,7 @@ def form_to_compiled_form(form):
 
 
 def build_forms(srcdir, info=None, summary=False, check_for_migration=False):
-    import re, cStringIO
+    import re, io
     from PyQt5.uic import compileUi
     forms = find_forms(srcdir)
     if info is None:
@@ -1286,7 +1286,7 @@ def build_forms(srcdir, info=None, summary=False, check_for_migration=False):
         if force_compile or not os.path.exists(compiled_form) or os.stat(form).st_mtime > os.stat(compiled_form).st_mtime:
             if not summary:
                 info('\tCompiling form', form)
-            buf = cStringIO.StringIO()
+            buf = io.BytesIO()
             compileUi(form, buf)
             dat = buf.getvalue()
             dat = dat.replace('import images_rc', '')

src/calibre/gui2/device.py

@@ -3,7 +3,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 
 # Imports {{{
-import os, traceback, Queue, time, cStringIO, re, sys, weakref
+import os, traceback, Queue, time, io, re, sys, weakref
 from threading import Thread, Event
 
 from PyQt5.Qt import (
@@ -113,7 +113,7 @@ class DeviceJob(BaseJob):  # {{{
 
     @property
     def log_file(self):
-        return cStringIO.StringIO(self._details.encode('utf-8'))
+        return io.BytesIO(self._details.encode('utf-8'))
 
     # }}}
 

src/calibre/gui2/dialogs/catalog.py

@@ -22,7 +22,7 @@ class Catalog(QDialog, Ui_Dialog):
     ''' Catalog Dialog builder'''
 
     def __init__(self, parent, dbspec, ids, db):
-        import re, cStringIO
+        import re, io
         from calibre import prints as info
         from PyQt5.uic import compileUi
 
@@ -68,7 +68,7 @@ class Catalog(QDialog, Ui_Dialog):
                     # Compile the .ui form provided in plugin.zip
                     if not os.path.exists(compiled_form):
                         # info('\tCompiling form', form)
-                        buf = cStringIO.StringIO()
+                        buf = io.BytesIO()
                         compileUi(form, buf)
                         dat = buf.getvalue()
                         dat = re.compile(r'QtGui.QApplication.translate\(.+?,\s+"(.+?)(?<!\\)",.+?\)',

src/calibre/translations/dynamic.py

@@ -5,7 +5,7 @@ Dynamic language lookup of translations for user-visible strings.
 __license__   = 'GPL v3'
 __copyright__ = '2008, Marshall T. Vandegrift <llasram@gmail.com>'
 
-import cStringIO
+import io
 from gettext import GNUTranslations
 from calibre.utils.localization import get_lc_messages_path
 from zipfile import ZipFile
@@ -25,7 +25,7 @@ def translate(lang, text):
             with ZipFile(P('localization/locales.zip',
                 allow_user_override=False), 'r') as zf:
                 try:
-                    buf = cStringIO.StringIO(zf.read(mpath + '/messages.mo'))
+                    buf = io.BytesIO(zf.read(mpath + '/messages.mo'))
                 except:
                     pass
                 else:

src/calibre/utils/ipc/job.py

@@ -8,7 +8,7 @@ __docformat__ = 'restructuredtext en'
 
 _count = 0
 
-import time, cStringIO
+import time, io
 from Queue import Queue, Empty
 
 from calibre import prints
@@ -146,7 +146,7 @@ class BaseJob(object):
     def log_file(self):
         if self.log_path:
             return open(self.log_path, 'rb')
-        return cStringIO.StringIO(_('No details available.').encode('utf-8',
+        return io.BytesIO(_('No details available.').encode('utf-8',
             'replace'))
 
     @property
@@ -159,6 +159,3 @@ class ParallelJob(BaseJob):
     def __init__(self, name, description, done, args=[], kwargs={}):
         self.name, self.args, self.kwargs = name, args, kwargs
         BaseJob.__init__(self, description, done)
-
-
-

src/calibre/utils/localization.py

@@ -6,7 +6,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import os, locale, re, cStringIO
+import os, locale, re, io
 from gettext import GNUTranslations, NullTranslations
 
 from polyglot.builtins import unicode_type
@@ -129,14 +129,14 @@ def get_all_translators():
         for lang in available_translations():
             mpath = get_lc_messages_path(lang)
             if mpath is not None:
-                buf = cStringIO.StringIO(zf.read(mpath + '/messages.mo'))
+                buf = io.BytesIO(zf.read(mpath + '/messages.mo'))
                 yield lang, GNUTranslations(buf)
 
 
 def get_single_translator(mpath, which='messages'):
     from zipfile import ZipFile
     with ZipFile(P('localization/locales.zip', allow_user_override=False), 'r') as zf:
-        buf = cStringIO.StringIO(zf.read(mpath + '/%s.mo' % which))
+        buf = io.BytesIO(zf.read(mpath + '/%s.mo' % which))
         return GNUTranslations(buf)
 
 
@@ -185,14 +185,14 @@ lcdata = {
 
 def load_po(path):
     from calibre.translations.msgfmt import make
-    buf = cStringIO.StringIO()
+    buf = io.BytesIO()
     try:
         make(path, buf)
     except Exception:
         print (('Failed to compile translations file: %s, ignoring') % path)
         buf = None
     else:
-        buf = cStringIO.StringIO(buf.getvalue())
+        buf = io.BytesIO(buf.getvalue())
     return buf
 
 
@@ -216,12 +216,12 @@ def set_translators():
             with ZipFile(P('localization/locales.zip',
                 allow_user_override=False), 'r') as zf:
                 if buf is None:
-                    buf = cStringIO.StringIO(zf.read(mpath + '/messages.mo'))
+                    buf = io.BytesIO(zf.read(mpath + '/messages.mo'))
                 if mpath == 'nds':
                     mpath = 'de'
                 isof = mpath + '/iso639.mo'
                 try:
-                    iso639 = cStringIO.StringIO(zf.read(isof))
+                    iso639 = io.BytesIO(zf.read(isof))
                 except:
                     pass  # No iso639 translations for this lang
                 if buf is not None:

src/calibre/utils/zipfile.py

@@ -4,7 +4,7 @@ a zip archive, detecting filename encoding, updating zip files, etc.
 """
 from __future__ import print_function
 import struct, os, time, sys, shutil, stat, re, io
-import binascii, cStringIO
+import binascii
 from contextlib import closing
 from tempfile import SpooledTemporaryFile
 
@@ -835,7 +835,7 @@ class ZipFile:
         self.start_dir = offset_cd + concat
         fp.seek(self.start_dir, 0)
         data = fp.read(size_cd)
-        fp = cStringIO.StringIO(data)
+        fp = io.BytesIO(data)
         total = 0
         while total < size_cd:
             centdir = fp.read(sizeCentralDir)

src/calibre/web/feeds/news.py

@@ -7,7 +7,7 @@ Defines various abstract base classes that can be subclassed to create powerful
 __docformat__ = "restructuredtext en"
 
 
-import os, time, traceback, re, urlparse, sys, cStringIO
+import os, time, traceback, re, urlparse, sys, io
 from collections import defaultdict
 from functools import partial
 from contextlib import nested, closing
@@ -1305,7 +1305,7 @@ class BasicNewsRecipe(Recipe):
             ext = cu.split('/')[-1].rpartition('.')[-1].lower().strip()
             if ext == 'pdf':
                 from calibre.ebooks.metadata.pdf import get_metadata
-                stream = cStringIO.StringIO(cdata)
+                stream = io.BytesIO(cdata)
                 cdata = None
                 mi = get_metadata(stream)
                 if mi.cover_data and mi.cover_data[1]:
@@ -1807,7 +1807,7 @@ class CalibrePeriodical(BasicNewsRecipe):
                         ' Either your subscription has expired or you have'
                         ' exceeded the maximum allowed downloads for today.'))
             raise
-        f = cStringIO.StringIO(raw)
+        f = io.BytesIO(raw)
         from calibre.utils.zipfile import ZipFile
         zf = ZipFile(f)
         zf.extractall()