py3: More unicode_literals and str() porting

src/calibre/devices/eb600/driver.py

@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*-
+from __future__ import absolute_import, division, print_function, unicode_literals
 
 __license__   = 'GPL v3'
 __copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'

src/calibre/devices/kindle/apnx.py

@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*-
+from __future__ import absolute_import, division, print_function, unicode_literals
 
 __license__   = 'GPL v3'
 __copyright__ = '2011, John Schember <john at nachtimwald.com>'
@@ -17,7 +18,7 @@ from calibre.ebooks.mobi.reader.headers import MetadataHeader
 from calibre.utils.logging import default_log
 from calibre import prints, fsync
 from calibre.constants import DEBUG
-from polyglot.builtins import range, as_unicode, as_bytes
+from polyglot.builtins import range, as_unicode, as_bytes, unicode_type, map
 
 
 class APNXBuilder(object):
@@ -32,7 +33,7 @@ class APNXBuilder(object):
         using either the fast or accurate algorithm.
         '''
         import uuid
-        apnx_meta = {'guid': str(uuid.uuid4()).replace('-', '')[:8], 'asin':
+        apnx_meta = {'guid': unicode_type(uuid.uuid4()).replace('-', '')[:8], 'asin':
                 '', 'cdetype': 'EBOK', 'format': 'MOBI_7', 'acr': ''}
 
         with lopen(mobi_file_path, 'rb') as mf:
@@ -52,11 +53,11 @@ class APNXBuilder(object):
             if mh.exth is None or not mh.exth.cdetype:
                 apnx_meta['cdetype'] = 'EBOK'
             else:
-                apnx_meta['cdetype'] = str(mh.exth.cdetype)
+                apnx_meta['cdetype'] = unicode_type(mh.exth.cdetype)
             if mh.exth is None or not mh.exth.uuid:
                 apnx_meta['asin'] = ''
             else:
-                apnx_meta['asin'] = str(mh.exth.uuid)
+                apnx_meta['asin'] = unicode_type(mh.exth.uuid)
 
         # Get the pages depending on the chosen parser
         pages = []
@@ -113,16 +114,18 @@ class APNXBuilder(object):
 
         if DEBUG:
             prints('APNX Content Header:', content_header)
+        content_header = as_bytes(content_header)
+        page_header = as_bytes(page_header)
 
         apnx += struct.pack('>I', 65537)
         apnx += struct.pack('>I', 12 + len(content_header))
         apnx += struct.pack('>I', len(content_header))
-        apnx += as_bytes(content_header)
+        apnx += content_header
         apnx += struct.pack('>H', 1)
         apnx += struct.pack('>H', len(page_header))
         apnx += struct.pack('>H', len(pages))
         apnx += struct.pack('>H', 32)
-        apnx += as_bytes(page_header)
+        apnx += page_header
 
         # Write page values to APNX.
         for page in pages:
@@ -234,15 +237,17 @@ class APNXBuilder(object):
         # not modifying the text. In this case the case
         # doesn't matter just the absolute character and
         # the position within the stream.
-        for c in mr.mobi_html.lower():
+        data = bytearray(as_bytes(mr.mobi_html.lower()))
+        slash, p, lt, gt = map(ord, '/p<>')
+        for c in data:
             pos += 1
 
             # Check if we are starting or stopping a p tag.
             if check_p:
-                if c == '/':
+                if c == slash:
                     closing = True
                     continue
-                elif c == 'p':
+                elif c == p:
                     if closing:
                         in_p = False
                     else:
@@ -252,11 +257,11 @@ class APNXBuilder(object):
                 closing = False
                 continue
 
-            if c == '<':
+            if c == lt:
                 in_tag = True
                 check_p = True
                 continue
-            elif c == '>':
+            elif c == gt:
                 in_tag = False
                 check_p = False
                 continue
@@ -287,8 +292,8 @@ class APNXBuilder(object):
             return self.get_pages_fast(mobi_file_path)
         mr.extract_text()
 
-        html = mr.mobi_html.lower()
+        html = as_bytes(mr.mobi_html.lower())
-        for m in re.finditer('<[^>]*pagebreak[^>]*>', html):
+        for m in re.finditer(b'<[^>]*pagebreak[^>]*>', html):
             pages.append(m.end())
 
         return pages

src/calibre/devices/kindle/bookmark.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 
-from __future__ import print_function
+from __future__ import absolute_import, division, print_function, unicode_literals
+
 __license__   = 'GPL v3'
 __docformat__ = 'restructuredtext en'
 
@@ -57,7 +58,7 @@ class Bookmark():  # {{{
                 bpar_offset, = unpack('>I', data[0x4e:0x52])
                 lrlo = bpar_offset + 0x0c
                 self.last_read = int(unpack('>I', data[lrlo:lrlo+4])[0])
-                self.last_read_location = self.last_read/MAGIC_MOBI_CONSTANT + 1
+                self.last_read_location = self.last_read // MAGIC_MOBI_CONSTANT + 1
                 entries, = unpack('>I', data[0x4a:0x4e])
 
                 # Store the annotations/locations
@@ -73,13 +74,13 @@ class Bookmark():  # {{{
                 sig = data[eo:eo+4]
                 previous_block = None
 
-                while sig == 'DATA':
+                while sig == b'DATA':
                     text = None
                     entry_type = None
                     rec_len, = unpack('>I', data[eo+4:eo+8])
                     if rec_len == 0:
                         current_block = "empty_data"
-                    elif data[eo+8:eo+12] == "EBAR":
+                    elif data[eo+8:eo+12] == b"EBAR":
                         current_block = "data_header"
                         # entry_type = "data_header"
                         location, = unpack('>I', data[eo+0x34:eo+0x38])
@@ -93,7 +94,7 @@ class Bookmark():  # {{{
                         text = data[eo+8:eo+8+rec_len].decode('utf-16-be')
 
                     if entry_type:
-                        displayed_location = location/MAGIC_MOBI_CONSTANT + 1
+                        displayed_location = location // MAGIC_MOBI_CONSTANT + 1
                         user_notes[location] = dict(id=self.id,
                                                     displayed_location=displayed_location,
                                                     type=entry_type,
@@ -104,7 +105,7 @@ class Bookmark():  # {{{
                     previous_block = current_block
                     sig = data[eo:eo+4]
 
-                while sig == 'BKMK':
+                while sig == b'BKMK':
                     # Fix start location for Highlights using BKMK data
                     end_loc, = unpack('>I', data[eo+0x10:eo+0x14])
 
@@ -121,7 +122,7 @@ class Bookmark():  # {{{
                                                                     start,
                                                                     start//MAGIC_MOBI_CONSTANT + 1)
                         '''
-                        user_notes[start]['displayed_location'] = start/MAGIC_MOBI_CONSTANT + 1
+                        user_notes[start]['displayed_location'] = start // MAGIC_MOBI_CONSTANT + 1
                         user_notes.pop(end_loc)
                     else:
                         # If a bookmark coincides with a user annotation, the locs could
@@ -129,7 +130,7 @@ class Bookmark():  # {{{
                         # Skip bookmark for last_read_location
                         if end_loc != self.last_read:
                             # print " adding Bookmark at 0x%x (%d)" % (end_loc, end_loc/MAGIC_MOBI_CONSTANT + 1)
-                            displayed_location = end_loc/MAGIC_MOBI_CONSTANT + 1
+                            displayed_location = end_loc // MAGIC_MOBI_CONSTANT + 1
                             user_notes[end_loc - 1] = dict(id=self.id,
                                                            displayed_location=displayed_location,
                                                            type='Bookmark',
@@ -154,7 +155,7 @@ class Bookmark():  # {{{
                 split = my_clippings.find('documents') + len('documents/')
                 my_clippings = my_clippings[:split] + "My Clippings.txt"
                 try:
-                    with lopen(my_clippings, 'r') as f2:
+                    with io.open(my_clippings, encoding='utf-8', errors='replace') as f2:
                         marker_found = 0
                         text = ''
                         search_str1 = '%s' % (mi.title)

src/calibre/devices/kindle/driver.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-from __future__ import print_function
+from __future__ import absolute_import, division, print_function, unicode_literals
 __license__   = 'GPL v3'
 __copyright__ = '2009, John Schember <john at nachtimwald.com>'
 __docformat__ = 'restructuredtext en'
@@ -9,13 +9,13 @@ __docformat__ = 'restructuredtext en'
 Device driver for Amazon's Kindle
 '''
 
-import datetime, os, re, sys, json, hashlib, errno
+import datetime, os, re, json, hashlib, errno
 
-from calibre.constants import DEBUG
+from calibre.constants import DEBUG, filesystem_encoding
 from calibre.devices.kindle.bookmark import Bookmark
 from calibre.devices.usbms.driver import USBMS
 from calibre import strftime, fsync, prints
-from polyglot.builtins import unicode_type, as_bytes
+from polyglot.builtins import unicode_type, as_bytes, as_unicode
 
 '''
 Notes on collections:
@@ -111,12 +111,10 @@ class KINDLE(USBMS):
         else:
             mi = cls.metadata_from_formats([path])
         if mi.title == _('Unknown') or ('-asin' in mi.title and '-type' in mi.title):
+            path = as_unicode(path, filesystem_encoding, 'replace')
             match = cls.WIRELESS_FILE_NAME_PATTERN.match(os.path.basename(path))
             if match is not None:
                 mi.title = match.group('title')
-                if not isinstance(mi.title, unicode_type):
-                    mi.title = mi.title.decode(sys.getfilesystemencoding(),
-                                               'replace')
         return mi
 
     def get_annotations(self, path_map):
@@ -428,7 +426,7 @@ class KINDLE2(KINDLE):
             for x in items:
                 x = x[-40:]
                 if x not in path_map:
-                    path_map[x] = set([])
+                    path_map[x] = set()
                 path_map[x].add(col)
         if path_map:
             for book in bl:

src/calibre/devices/nook/driver.py

@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*-
+from __future__ import absolute_import, division, print_function, unicode_literals
 
 __license__   = 'GPL v3'
 __copyright__ = '2009, John Schember <john at nachtimwald.com>'

src/calibre/devices/smart_device_app/__init__.py

@@ -1,9 +1,8 @@
 #!/usr/bin/env python2
 # vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
 
+from __future__ import absolute_import, division, print_function, unicode_literals
+
 __license__   = 'GPL v3'
 __copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
-
-
-

src/calibre/devices/smart_device_app/driver.py

@@ -113,16 +113,16 @@ class ConnectionListener(Thread):
                         try:
                             packet = self.driver.broadcast_socket.recvfrom(100)
                             remote = packet[1]
-                            content_server_port = b''
+                            content_server_port = ''
                             try:
                                 from calibre.srv.opts import server_config
-                                content_server_port = str(server_config().port)
+                                content_server_port = unicode_type(server_config().port)
                             except Exception:
                                 pass
-                            message = str(self.driver.ZEROCONF_CLIENT_STRING + b' (on ' +
+                            message = (self.driver.ZEROCONF_CLIENT_STRING + ' (on ' +
-                                            str(socket.gethostname().partition('.')[0]) +
+                                            unicode_type(socket.gethostname().partition('.')[0]) +
-                                            b');' + content_server_port +
+                                            ');' + content_server_port +
-                                            b',' + str(self.driver.port))
+                                            ',' + unicode_type(self.driver.port)).encode('utf-8')
                             self.driver._debug('received broadcast', packet, message)
                             self.driver.broadcast_socket.sendto(message, remote)
                         except:
@@ -234,7 +234,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
 
     CURRENT_CC_VERSION          = 128
 
-    ZEROCONF_CLIENT_STRING      = b'calibre wireless device client'
+    ZEROCONF_CLIENT_STRING      = 'calibre wireless device client'
 
     # A few "random" port numbers to use for detecting clients using broadcast
     # The clients are expected to broadcast a UDP 'hi there' on all of these
@@ -481,7 +481,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
         opts = config().parse()
         if not isinstance(template, unicode_type):
             template = template.decode('utf-8')
-        app_id = str(getattr(mdata, 'application_id', ''))
+        app_id = unicode_type(getattr(mdata, 'application_id', ''))
         id_ = mdata.get('id', fname)
         extra_components = get_components(template, mdata, id_,
                 timefmt=opts.send_timefmt, length=maxlen-len(app_id)-1,
@@ -1885,7 +1885,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
                             'between 50 and 99. Forced to be %d.')%self.DEFAULT_THUMBNAIL_COMPRESSION_QUALITY
                 self._debug(message)
                 self.set_option('thumbnail_compression_quality',
-                                str(self.DEFAULT_THUMBNAIL_COMPRESSION_QUALITY))
+                                unicode_type(self.DEFAULT_THUMBNAIL_COMPRESSION_QUALITY))
 
             try:
                 self.listen_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

src/calibre/devices/user_defined/driver.py

@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*-
+from __future__ import absolute_import, division, print_function, unicode_literals
 
 __license__   = 'GPL v3'
 __copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'