From 839b68d4b61a75b29ed6ce06ac7eecf0c66a76b1 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Sun, 16 Sep 2012 13:13:36 +0200
Subject: [PATCH 01/12] new formatter functions. formats_paths and
 current_library_path

---
 src/calibre/library/__init__.py          |  9 +++++--
 src/calibre/library/database2.py         |  1 +
 src/calibre/utils/formatter_functions.py | 31 +++++++++++++++++++++++-
 3 files changed, 38 insertions(+), 3 deletions(-)

diff --git a/src/calibre/library/__init__.py b/src/calibre/library/__init__.py
index 605d062de3..3ae237c919 100644
--- a/src/calibre/library/__init__.py
+++ b/src/calibre/library/__init__.py
@@ -61,13 +61,18 @@ def generate_test_db(library_path, # {{{
     print 'Time per record:', t/float(num_of_records)
 # }}}
 
-def current_library_name():
+def current_library_path():
     from calibre.utils.config import prefs
-    import posixpath
     path = prefs['library_path']
     if path:
         path = path.replace('\\', '/')
         while path.endswith('/'):
             path = path[:-1]
+        return path
+
+def current_library_name():
+    import posixpath
+    path = current_library_path()
+    if path:
         return posixpath.basename(path)
 
diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py
index 0b23e3f0a4..8e8eb691ca 100644
--- a/src/calibre/library/database2.py
+++ b/src/calibre/library/database2.py
@@ -1262,6 +1262,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         ans = {}
         if path is not None:
             stat = os.stat(path)
+            ans['path'] = path;
             ans['size'] = stat.st_size
             ans['mtime'] = utcfromtimestamp(stat.st_mtime)
             self.format_metadata_cache[id_][fmt] = ans
diff --git a/src/calibre/utils/formatter_functions.py b/src/calibre/utils/formatter_functions.py
index fb92d04932..d57e50006e 100644
--- a/src/calibre/utils/formatter_functions.py
+++ b/src/calibre/utils/formatter_functions.py
@@ -629,6 +629,22 @@ class BuiltinFormatsSizes(BuiltinFormatterFunction):
         fmt_data = mi.get('format_metadata', {})
         return ','.join(k.upper()+':'+str(v['size']) for k,v in fmt_data.iteritems())
 
+class BuiltinFormatsPaths(BuiltinFormatterFunction):
+    name = 'formats_paths'
+    arg_count = 0
+    category = 'Get values from metadata'
+    __doc__ = doc = _('formats_paths() -- return a comma-separated list of '
+                      'colon_separated items representing full path to '
+                      'the formats of a book. You can use the select '
+                      'function to get the path for a specific '
+                      'format. Note that format names are always uppercase, '
+                      'as in EPUB.'
+            )
+
+    def evaluate(self, formatter, kwargs, mi, locals):
+        fmt_data = mi.get('format_metadata', {})
+        return ','.join(k.upper()+':'+str(v['path']) for k,v in fmt_data.iteritems())
+
 class BuiltinHumanReadable(BuiltinFormatterFunction):
     name = 'human_readable'
     arg_count = 1
@@ -1146,6 +1162,18 @@ class BuiltinCurrentLibraryName(BuiltinFormatterFunction):
         from calibre.library import current_library_name
         return current_library_name()
 
+class BuiltinCurrentLibraryPath(BuiltinFormatterFunction):
+    name = 'current_library_path'
+    arg_count = 0
+    category = 'Get values from metadata'
+    __doc__ = doc = _('current_library_path() -- '
+            'return the path to the current calibre library. This function can '
+            'be called in template program mode using the template '
+            '"{:\'current_library_path()\'}".')
+    def evaluate(self, formatter, kwargs, mi, locals):
+        from calibre.library import current_library_path
+        return current_library_path()
+
 class BuiltinFinishFormatting(BuiltinFormatterFunction):
     name = 'finish_formatting'
     arg_count = 4
@@ -1168,7 +1196,8 @@ _formatter_builtins = [
     BuiltinCurrentLibraryName(),
     BuiltinDaysBetween(), BuiltinDivide(), BuiltinEval(), BuiltinFirstNonEmpty(),
     BuiltinField(), BuiltinFinishFormatting(), BuiltinFormatDate(),
-    BuiltinFormatNumber(), BuiltinFormatsModtimes(), BuiltinFormatsSizes(),
+    BuiltinFormatNumber(), BuiltinFormatsModtimes(), BuiltinFormatsPaths(),
+    BuiltinFormatsSizes(),
     BuiltinHasCover(), BuiltinHumanReadable(), BuiltinIdentifierInList(),
     BuiltinIfempty(), BuiltinLanguageCodes(), BuiltinLanguageStrings(),
     BuiltinInList(), BuiltinListDifference(), BuiltinListEquals(),

From a0942353444f9e6ccf41e551207e1d387cc32f0e Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Sun, 16 Sep 2012 19:12:38 +0200
Subject: [PATCH 02/12] First try at a threaded listener that can tell CC if
 calibre is busy serving another device.

---
 .../devices/smart_device_app/driver.py        | 173 ++++++++++++++----
 1 file changed, 136 insertions(+), 37 deletions(-)

diff --git a/src/calibre/devices/smart_device_app/driver.py b/src/calibre/devices/smart_device_app/driver.py
index 676cdf145c..1f44b40b66 100644
--- a/src/calibre/devices/smart_device_app/driver.py
+++ b/src/calibre/devices/smart_device_app/driver.py
@@ -10,9 +10,12 @@ Created on 29 Jun 2012
 import socket, select, json, inspect, os, traceback, time, sys, random
 import posixpath
 import hashlib, threading
+import Queue
+
 from base64 import b64encode, b64decode
 from functools import wraps
 from errno import EAGAIN, EINTR
+from threading import Thread
 
 from calibre import prints
 from calibre.constants import numeric_version, DEBUG
@@ -48,6 +51,109 @@ def synchronous(tlockname):
     return _synched
 
 
+class ConnectionListener (Thread):
+
+    def __init__(self, driver):
+        Thread.__init__(self)
+        self.daemon = True
+        self.driver = driver
+        self.keep_running = True
+
+    def stop(self):
+        self.keep_running = False
+
+    def run(self):
+        queue_not_serviced_count = 0
+        device_socket = None
+        while self.keep_running:
+            try:
+                time.sleep(1) # Limit to one book per two seconds
+            except:
+                # Happens during interpreter shutdown
+                break
+
+            if not self.keep_running:
+                break
+
+            if not self.driver.connection_queue.empty():
+                self.driver._debug('queue not empty')
+                queue_not_serviced_count += 1
+                if queue_not_serviced_count >= 3:
+                    self.driver._debug('queue not serviced')
+                    try:
+                        sock = self.driver.connection_queue.get_nowait()
+                        s = self.driver._json_encode(
+                                        self.driver.opcodes['CALIBRE_BUSY'], {})
+                        self.driver._send_byte_string(device_socket, (b'%d' % len(s)) + s)
+                        sock.close()
+                    except Queue.Empty:
+                        pass
+                    queue_not_serviced_count = 0
+
+            if getattr(self.driver, 'broadcast_socket', None) is not None:
+                while True:
+                    ans = select.select((self.driver.broadcast_socket,), (), (), 0)
+                    if len(ans[0]) > 0:
+                        try:
+                            packet = self.driver.broadcast_socket.recvfrom(100)
+                            remote = packet[1]
+                            message = str(self.driver.ZEROCONF_CLIENT_STRING + b' (on ' +
+                                            str(socket.gethostname().partition('.')[0]) +
+                                            b'),' + str(self.driver.port))
+                            self.driver._debug('received broadcast', packet, message)
+                            self.driver.broadcast_socket.sendto(message, remote)
+                        except:
+                            pass
+                    else:
+                        break
+
+            if self.driver.connection_queue.empty() and \
+                        getattr(self.driver, 'listen_socket', None) is not None:
+                ans = select.select((self.driver.listen_socket,), (), (), 0)
+                if len(ans[0]) > 0:
+                    # timeout in 10 ms to detect rare case where the socket went
+                    # way between the select and the accept
+                    try:
+                        self.driver._debug('attempt to open device socket')
+                        device_socket = None
+                        self.driver.listen_socket.settimeout(0.010)
+                        device_socket, ign = eintr_retry_call(
+                                self.driver.listen_socket.accept)
+                        self.driver.listen_socket.settimeout(None)
+                        device_socket.settimeout(None)
+
+                        try:
+                            peer = self.driver.device_socket.getpeername()[0]
+                            attempts = self.drjver.connection_attempts.get(peer, 0)
+                            if attempts >= self.MAX_UNSUCCESSFUL_CONNECTS:
+                                self.driver._debug('too many connection attempts from', peer)
+                                device_socket.close()
+                                device_socket = None
+#                                raise InitialConnectionError(_('Too many connection attempts from %s') % peer)
+                            else:
+                                self.driver.connection_attempts[peer] = attempts + 1
+                        except InitialConnectionError:
+                            raise
+                        except:
+                            pass
+
+                        try:
+                            self.driver.connection_queue.put_nowait(device_socket)
+                        except Queue.Full:
+                            device_socket.close();
+                            device_socket = None
+                            self.driver._debug('driver is not answering')
+
+                    except socket.timeout:
+                        pass
+                    except socket.error:
+                        x = sys.exc_info()[1]
+                        self.driver._debug('unexpected socket exception', x.args[0])
+                        device_socket.close()
+                        device_socket = None
+#                        raise
+
+
 class SDBook(Book):
     def __init__(self, prefix, lpath, size=None, other=None):
         Book.__init__(self, prefix, lpath, size=size, other=other)
@@ -112,6 +218,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
         'OK'                     : 0,
         'BOOK_DATA'              : 10,
         'BOOK_DONE'              : 11,
+        'CALIBRE_BUSY'           : 18,
         'DELETE_BOOK'            : 13,
         'DISPLAY_MESSAGE'        : 17,
         'FREE_SPACE'             : 5,
@@ -373,7 +480,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
             pos += len(v)
         return data
 
-    def _send_byte_string(self, s):
+    def _send_byte_string(self, sock, s):
         if not isinstance(s, bytes):
             self._debug('given a non-byte string!')
             raise PacketError("Internal error: found a string that isn't bytes")
@@ -382,11 +489,11 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
         while sent_len < total_len:
             try:
                 if sent_len == 0:
-                    amt_sent = self.device_socket.send(s)
+                    amt_sent = sock.send(s)
                 else:
-                    amt_sent = self.device_socket.send(s[sent_len:])
+                    amt_sent = sock.send(s[sent_len:])
                 if amt_sent <= 0:
-                    raise IOError('Bad write on device socket')
+                    raise IOError('Bad write on socket')
                 sent_len += amt_sent
             except socket.error as e:
                 self._debug('socket error', e, e.errno)
@@ -410,7 +517,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
             if print_debug_info and extra_debug:
                 self._debug('send string', s)
             self.device_socket.settimeout(self.MAX_CLIENT_COMM_TIMEOUT)
-            self._send_byte_string((b'%d' % len(s)) + s)
+            self._send_byte_string(self.device_socket, (b'%d' % len(s)) + s)
             if not wait_for_response:
                 return None, None
             return self._receive_from_client(print_debug_info=print_debug_info)
@@ -620,39 +727,26 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
                     break
 
         if getattr(self, 'listen_socket', None) is not None:
-            ans = select.select((self.listen_socket,), (), (), 0)
-            if len(ans[0]) > 0:
-                # timeout in 10 ms to detect rare case where the socket went
-                # way between the select and the accept
+            try:
+                ans = self.connection_queue.get_nowait()
+                self.device_socket = ans
+                self.is_connected = True
                 try:
-                    self.device_socket = None
-                    self.listen_socket.settimeout(0.010)
-                    self.device_socket, ign = eintr_retry_call(
-                            self.listen_socket.accept)
-                    self.listen_socket.settimeout(None)
-                    self.device_socket.settimeout(None)
-                    self.is_connected = True
-                    try:
-                        peer = self.device_socket.getpeername()[0]
-                        attempts = self.connection_attempts.get(peer, 0)
-                        if attempts >= self.MAX_UNSUCCESSFUL_CONNECTS:
-                            self._debug('too many connection attempts from', peer)
-                            self._close_device_socket()
-                            raise InitialConnectionError(_('Too many connection attempts from %s') % peer)
-                        else:
-                            self.connection_attempts[peer] = attempts + 1
-                    except InitialConnectionError:
-                        raise
-                    except:
-                        pass
-                except socket.timeout:
-                    self._close_device_socket()
-                except socket.error:
-                    x = sys.exc_info()[1]
-                    self._debug('unexpected socket exception', x.args[0])
-                    self._close_device_socket()
+                    peer = self.device_socket.getpeername()[0]
+                    attempts = self.connection_attempts.get(peer, 0)
+                    if attempts >= self.MAX_UNSUCCESSFUL_CONNECTS:
+                        self._debug('too many connection attempts from', peer)
+                        self._close_device_socket()
+                        raise InitialConnectionError(_('Too many connection attempts from %s') % peer)
+                    else:
+                        self.connection_attempts[peer] = attempts + 1
+                except InitialConnectionError:
                     raise
-                return (self.is_connected, self)
+                except:
+                    pass
+            except Queue.Empty:
+                self.is_connected = False
+            return (self.is_connected, self)
         return (False, None)
 
     @synchronous('sync_lock')
@@ -1127,17 +1221,22 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
                 self._debug('broadcast socket listening on port', port)
                 break
 
+        message = None
         if port == 0:
             self.broadcast_socket.close()
             self.broadcast_socket = None
             message = 'attaching port to broadcast socket failed. This is not fatal.'
             self._debug(message)
-            return message
 
+        self.connection_queue = Queue.Queue(1)
+        self.connection_listener = ConnectionListener(self)
+        self.connection_listener.start()
+        return message
 
     @synchronous('sync_lock')
     def shutdown(self):
         if getattr(self, 'listen_socket', None) is not None:
+            self.connection_listener.stop()
             unpublish_zeroconf('calibre smart device client',
                              '_calibresmartdeviceapp._tcp', self.port, {})
             self._close_listen_socket()

From 810aa02b74d85074f3c20548e086dc5539af00d8 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Mon, 17 Sep 2012 13:53:35 +0200
Subject: [PATCH 03/12] Send books to the device using a pure binary protocol

---
 .../devices/smart_device_app/driver.py        | 25 +++++++++++++------
 1 file changed, 17 insertions(+), 8 deletions(-)

diff --git a/src/calibre/devices/smart_device_app/driver.py b/src/calibre/devices/smart_device_app/driver.py
index 1f44b40b66..10f1f6ea40 100644
--- a/src/calibre/devices/smart_device_app/driver.py
+++ b/src/calibre/devices/smart_device_app/driver.py
@@ -89,6 +89,8 @@ class ConnectionListener (Thread):
                     except Queue.Empty:
                         pass
                     queue_not_serviced_count = 0
+            else:
+                queue_not_serviced_count = 0
 
             if getattr(self.driver, 'broadcast_socket', None) is not None:
                 while True:
@@ -140,7 +142,7 @@ class ConnectionListener (Thread):
                         try:
                             self.driver.connection_queue.put_nowait(device_socket)
                         except Queue.Full:
-                            device_socket.close();
+                            device_socket.close()
                             device_socket = None
                             self.driver._debug('driver is not answering')
 
@@ -577,7 +579,8 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
         opcode, result = self._call_client('SEND_BOOK', {'lpath': lpath, 'length': length,
                                'metadata': book_metadata, 'thisBook': this_book,
                                'totalBooks': total_books,
-                               'willStreamBooks': self.client_can_stream_books},
+                               'willStreamBooks': self.client_can_stream_books,
+                               'willStreamBinary' : self.client_can_receive_book_binary},
                           print_debug_info=False,
                           wait_for_response=(not self.client_can_stream_books))
 
@@ -590,17 +593,21 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
                 blen = len(b)
                 if not b:
                     break
-                b = b64encode(b)
-                opcode, result = self._call_client('BOOK_DATA',
-                                {'lpath': lpath, 'position': pos, 'data': b},
-                                print_debug_info=False,
-                                wait_for_response=(not self.client_can_stream_books))
+                if self.client_can_stream_books and self.client_can_receive_book_binary:
+                    self._send_byte_string(self.device_socket, b)
+                else:
+                    b = b64encode(b)
+                    opcode, result = self._call_client('BOOK_DATA',
+                                    {'lpath': lpath, 'position': pos, 'data': b},
+                                    print_debug_info=False,
+                                    wait_for_response=(not self.client_can_stream_books))
                 pos += blen
                 if not self.client_can_stream_books and opcode != 'OK':
                     self._debug('protocol error', opcode)
                     failed = True
                     break
-        self._call_client('BOOK_DONE', {'lpath': lpath})
+        if not (self.client_can_stream_books and self.client_can_receive_book_binary):
+            self._call_client('BOOK_DONE', {'lpath': lpath})
         self.time = None
         if close_:
             infile.close()
@@ -799,6 +806,8 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
             self._debug('Device can stream books', self.client_can_stream_books)
             self.client_can_stream_metadata = result.get('canStreamMetadata', False)
             self._debug('Device can stream metadata', self.client_can_stream_metadata)
+            self.client_can_receive_book_binary = result.get('canReceiveBookBinary', False)
+            self._debug('Device can receive book binary', self.client_can_stream_metadata)
 
             self.max_book_packet_len = result.get('maxBookContentPacketLen',
                                                   self.BASE_PACKET_LEN)

From 653df51062176c86eb34f992314dea2fe75fddc1 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Tue, 18 Sep 2012 11:00:55 +0200
Subject: [PATCH 04/12] Add UUID as a possible id type to content server
 ajax/book

---
 src/calibre/library/database2.py   |  5 +++++
 src/calibre/library/server/ajax.py | 14 ++++++++++----
 2 files changed, 15 insertions(+), 4 deletions(-)

diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py
index 8e8eb691ca..f76d13f27c 100644
--- a/src/calibre/library/database2.py
+++ b/src/calibre/library/database2.py
@@ -2564,6 +2564,11 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
             if notify:
                 self.notify('metadata', [id])
 
+    def get_id_from_uuid(self, uuid):
+        if uuid:
+            return self.conn.get('SELECT id FROM books WHERE uuid=?', (uuid,),
+                                 all=False)
+
     # Convenience methods for tags_list_editor
     # Note: we generally do not need to refresh_ids because library_view will
     # refresh everything.
diff --git a/src/calibre/library/server/ajax.py b/src/calibre/library/server/ajax.py
index 98b4be1fb0..4d06d9073b 100644
--- a/src/calibre/library/server/ajax.py
+++ b/src/calibre/library/server/ajax.py
@@ -181,7 +181,7 @@ class AjaxServer(object):
         return data, mi.last_modified
 
     @Endpoint(set_last_modified=False)
-    def ajax_book(self, book_id, category_urls='true'):
+    def ajax_book(self, book_id, category_urls='true', id_is_uuid='false'):
         '''
         Return the metadata of the book as a JSON dictionary.
 
@@ -192,7 +192,10 @@ class AjaxServer(object):
         cherrypy.response.timeout = 3600
 
         try:
-            book_id = int(book_id)
+            if id_is_uuid == 'true':
+                book_id = self.db.get_id_from_uuid(book_id)
+            else:
+                book_id = int(book_id)
             data, last_modified = self.ajax_book_to_json(book_id,
                     get_category_urls=category_urls.lower()=='true')
         except:
@@ -204,7 +207,7 @@ class AjaxServer(object):
         return data
 
     @Endpoint(set_last_modified=False)
-    def ajax_books(self, ids=None, category_urls='true'):
+    def ajax_books(self, ids=None, category_urls='true', id_is_uuid='false'):
         '''
         Return the metadata for a list of books specified as a comma separated
         list of ids. The metadata is returned as a dictionary mapping ids to
@@ -218,7 +221,10 @@ class AjaxServer(object):
         if ids is None:
             raise cherrypy.HTTPError(404, 'Must specify some ids')
         try:
-            ids = set(int(x.strip()) for x in ids.split(','))
+            if id_is_uuid == 'true':
+                ids = set(self.db.get_id_from_uuid(x) for x in ids.split(','))
+            else:
+                ids = set(int(x.strip()) for x in ids.split(','))
         except:
             raise cherrypy.HTTPError(404, 'ids must be a comma separated list'
                     ' of integers')

From edc3feba18cba22fad086fea3cfbdc3b07525986 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Tue, 18 Sep 2012 11:56:24 +0200
Subject: [PATCH 05/12] Limit lpath length on windows machine to a guestimate
 that will result in a path usable by windows explorer.

---
 .../devices/smart_device_app/driver.py        | 22 ++++++++++++++-----
 1 file changed, 17 insertions(+), 5 deletions(-)

diff --git a/src/calibre/devices/smart_device_app/driver.py b/src/calibre/devices/smart_device_app/driver.py
index 10f1f6ea40..be186748e3 100644
--- a/src/calibre/devices/smart_device_app/driver.py
+++ b/src/calibre/devices/smart_device_app/driver.py
@@ -18,7 +18,7 @@ from errno import EAGAIN, EINTR
 from threading import Thread
 
 from calibre import prints
-from calibre.constants import numeric_version, DEBUG
+from calibre.constants import numeric_version, DEBUG, iswindows
 from calibre.devices.errors import (OpenFailed, ControlError, TimeoutError,
                                     InitialConnectionError, PacketError)
 from calibre.devices.interface import DevicePlugin
@@ -189,6 +189,9 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
     SUPPORTS_USE_AUTHOR_SORT    = False
     WANTS_UPDATED_THUMBNAILS    = True
     MAX_PATH_LEN                = 250
+    # guess of length of MTP name. The length of the full path to the folder
+    # on the device is added to this. That path includes device the mount point.
+    WINDOWS_PATH_FUDGE_FACTOR   = 25
     THUMBNAIL_HEIGHT            = 160
     PREFIX                      = ''
     BACKLOADING_ERROR_MESSAGE   = None
@@ -357,7 +360,14 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
     # remove the 'path' argument and all its uses. Also removed the calls to
     # filename_callback and sanitize_path_components
     def _create_upload_path(self, mdata, fname, create_dirs=True):
-        maxlen = self.MAX_PATH_LEN
+        fname = sanitize(fname)
+        ext = os.path.splitext(fname)[1]
+
+        if iswindows:
+            maxlen = 225 - max(25, self.exts_path_lengths.get(ext, 25))
+        else:
+            maxlen = self.MAX_PATH_LEN
+        self._debug('max path length', maxlen)
 
         special_tag = None
         if mdata.tags:
@@ -378,9 +388,6 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
             template = "{title}_%d-%d-%d" % date
         use_subdirs = self.SUPPORTS_SUB_DIRS and settings.use_subdirs
 
-        fname = sanitize(fname)
-        ext = os.path.splitext(fname)[1]
-
         from calibre.library.save_to_disk import get_components
         from calibre.library.save_to_disk import config
         opts = config().parse()
@@ -816,9 +823,14 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
                 self._debug('Protocol error - bogus accepted extensions')
                 self._close_device_socket()
                 return False
+
             config = self._configProxy()
             config['format_map'] = exts
             self._debug('selected formats', config['format_map'])
+
+            self.exts_path_lengths = result.get('extensionPathLengths', {})
+            self._debug('extension path lengths', self.exts_path_lengths)
+
             if password:
                 returned_hash = result.get('passwordHash', None)
                 if result.get('passwordHash', None) is None:

From 145ab86f38689d0dab299138bb8a6bbe4802d3c7 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Tue, 18 Sep 2012 12:50:42 +0200
Subject: [PATCH 06/12] Upload books using a binary protocol

---
 .../devices/smart_device_app/driver.py        | 81 +++++++++++++------
 1 file changed, 56 insertions(+), 25 deletions(-)

diff --git a/src/calibre/devices/smart_device_app/driver.py b/src/calibre/devices/smart_device_app/driver.py
index be186748e3..71805b8692 100644
--- a/src/calibre/devices/smart_device_app/driver.py
+++ b/src/calibre/devices/smart_device_app/driver.py
@@ -367,7 +367,6 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
             maxlen = 225 - max(25, self.exts_path_lengths.get(ext, 25))
         else:
             maxlen = self.MAX_PATH_LEN
-        self._debug('max path length', maxlen)
 
         special_tag = None
         if mdata.tags:
@@ -462,6 +461,13 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
         return json.dumps([op, res], encoding='utf-8')
 
     # Network functions
+
+    def _read_binary_from_net(self, length):
+        self.device_socket.settimeout(self.MAX_CLIENT_COMM_TIMEOUT)
+        v = self.device_socket.recv(length)
+        self.device_socket.settimeout(None)
+        return v
+
     def _read_string_from_net(self):
         data = bytes(0)
         while True:
@@ -470,9 +476,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
                 break
             # recv seems to return a pointer into some internal buffer.
             # Things get trashed if we don't make a copy of the data.
-            self.device_socket.settimeout(self.MAX_CLIENT_COMM_TIMEOUT)
-            v = self.device_socket.recv(2)
-            self.device_socket.settimeout(None)
+            v = self._read_binary_from_net(2)
             if len(v) == 0:
                 return '' # documentation says the socket is broken permanently.
             data += v
@@ -480,9 +484,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
         data = data[dex:]
         pos = len(data)
         while pos < total_len:
-            self.device_socket.settimeout(self.MAX_CLIENT_COMM_TIMEOUT)
-            v = self.device_socket.recv(total_len - pos)
-            self.device_socket.settimeout(None)
+            v = self._read_binary_from_net(total_len - pos)
             if len(v) == 0:
                 return '' # documentation says the socket is broken permanently.
             data += v
@@ -582,7 +584,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
         length = infile.tell()
         book_metadata.size = length
         infile.seek(0)
-        self._debug(lpath, length)
+
         opcode, result = self._call_client('SEND_BOOK', {'lpath': lpath, 'length': length,
                                'metadata': book_metadata, 'thisBook': this_book,
                                'totalBooks': total_books,
@@ -818,6 +820,8 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
 
             self.max_book_packet_len = result.get('maxBookContentPacketLen',
                                                   self.BASE_PACKET_LEN)
+            self._debug('max_book_packet_len', self.max_book_packet_len)
+
             exts = result.get('acceptedExtensions', None)
             if exts is None or not isinstance(exts, list) or len(exts) == 0:
                 self._debug('Protocol error - bogus accepted extensions')
@@ -1024,7 +1028,10 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
     @synchronous('sync_lock')
     def upload_books(self, files, names, on_card=None, end_session=True,
                      metadata=None):
-        self._debug(names)
+        if self.settings().extra_customization[self.OPT_EXTRA_DEBUG]:
+            self._debug(names)
+        else:
+            self._debug()
 
         paths = []
         names = iter(names)
@@ -1071,7 +1078,11 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
 
     @synchronous('sync_lock')
     def delete_books(self, paths, end_session=True):
-        self._debug(paths)
+        if self.settings().extra_customization[self.OPT_EXTRA_DEBUG]:
+            self._debug(paths)
+        else:
+            self._debug()
+
         for path in paths:
             # the path has the prefix on it (I think)
             path = self._strip_prefix(path)
@@ -1083,7 +1094,11 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
 
     @synchronous('sync_lock')
     def remove_books_from_metadata(self, paths, booklists):
-        self._debug(paths)
+        if self.settings().extra_customization[self.OPT_EXTRA_DEBUG]:
+            self._debug(paths)
+        else:
+            self._debug()
+
         for i, path in enumerate(paths):
             path = self._strip_prefix(path)
             self.report_progress((i + 1) / float(len(paths)), _('Removing books from device metadata listing...'))
@@ -1098,29 +1113,45 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
 
     @synchronous('sync_lock')
     def get_file(self, path, outfile, end_session=True, this_book=None, total_books=None):
-        self._debug(path)
+        if self.settings().extra_customization[self.OPT_EXTRA_DEBUG]:
+            self._debug(path)
+        else:
+            self._debug()
+
         eof = False
         position = 0
         while not eof:
             opcode, result = self._call_client('GET_BOOK_FILE_SEGMENT',
                                     {'lpath' : path, 'position': position,
                                      'thisBook': this_book, 'totalBooks': total_books,
-                                     'canStream':True},
+                                     'canStream':True, 'canStreamBinary': True},
                                     print_debug_info=False)
             if opcode == 'OK':
                 client_will_stream = 'willStream' in result
-                while not eof:
-                    if not result['eof']:
-                        data = b64decode(result['data'])
-                        if len(data) != result['next_position'] - position:
-                            self._debug('position mismatch', result['next_position'], position)
-                        position = result['next_position']
-                        outfile.write(data)
-                        opcode, result = self._receive_from_client(print_debug_info=True)
-                    else:
-                        eof = True
-                    if not client_will_stream:
-                        break
+                client_will_stream_binary = 'willStreamBinary' in result
+
+                if (client_will_stream_binary):
+                    length = result.get('fileLength');
+                    remaining = length
+
+                    while remaining > 0:
+                        v = self._read_binary_from_net(min(remaining, self.max_book_packet_len))
+                        outfile.write(v)
+                        remaining -= len(v)
+                    eof = True
+                else:
+                    while not eof:
+                        if not result['eof']:
+                            data = b64decode(result['data'])
+                            if len(data) != result['next_position'] - position:
+                                self._debug('position mismatch', result['next_position'], position)
+                            position = result['next_position']
+                            outfile.write(data)
+                            opcode, result = self._receive_from_client(print_debug_info=True)
+                        else:
+                            eof = True
+                        if not client_will_stream:
+                            break
             else:
                 raise ControlError(desc='request for book data failed')
 

From 3c36e87b86a3da162c7bca813b956a08d3ad292a Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Tue, 18 Sep 2012 13:02:15 +0200
Subject: [PATCH 07/12] Improve the path fudge factor stuff.

---
 src/calibre/devices/smart_device_app/driver.py | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/src/calibre/devices/smart_device_app/driver.py b/src/calibre/devices/smart_device_app/driver.py
index 71805b8692..4befe970d5 100644
--- a/src/calibre/devices/smart_device_app/driver.py
+++ b/src/calibre/devices/smart_device_app/driver.py
@@ -188,10 +188,16 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
     NEWS_IN_FOLDER              = True
     SUPPORTS_USE_AUTHOR_SORT    = False
     WANTS_UPDATED_THUMBNAILS    = True
-    MAX_PATH_LEN                = 250
+    NOT_WINDOWS_MAX_PATH_LEN    = 250
+
+    # Guess about the max length on windows. This number will be reduced by
+    # the length of the path on the client, and by the fudge factor below
+    WINDOWS_MAX_PATH_LEN        = 250
     # guess of length of MTP name. The length of the full path to the folder
     # on the device is added to this. That path includes device the mount point.
-    WINDOWS_PATH_FUDGE_FACTOR   = 25
+    # making this number effectively around 10 to 15 larger.
+    WINDOWS_PATH_FUDGE_FACTOR   = 40
+
     THUMBNAIL_HEIGHT            = 160
     PREFIX                      = ''
     BACKLOADING_ERROR_MESSAGE   = None
@@ -364,7 +370,10 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
         ext = os.path.splitext(fname)[1]
 
         if iswindows:
-            maxlen = 225 - max(25, self.exts_path_lengths.get(ext, 25))
+            maxlen = (self.WINDOWS_MAX_PATH_LEN -
+                      (self.WINDOWS_PATH_FUDGE_FACTOR +
+                       self.exts_path_lengths.get(ext,
+                                        self.WINDOWS_PATH_FUDGE_FACTOR)))
         else:
             maxlen = self.MAX_PATH_LEN
 

From 58f4a37f27a3ae33d9cbffb41095a8f6462767b2 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Tue, 18 Sep 2012 13:14:35 +0200
Subject: [PATCH 08/12] Use the path computation on all platforms because the
 device might connect to several over time.

---
 .../devices/smart_device_app/driver.py        | 21 ++++++++-----------
 1 file changed, 9 insertions(+), 12 deletions(-)

diff --git a/src/calibre/devices/smart_device_app/driver.py b/src/calibre/devices/smart_device_app/driver.py
index 4befe970d5..f678ecf576 100644
--- a/src/calibre/devices/smart_device_app/driver.py
+++ b/src/calibre/devices/smart_device_app/driver.py
@@ -18,7 +18,7 @@ from errno import EAGAIN, EINTR
 from threading import Thread
 
 from calibre import prints
-from calibre.constants import numeric_version, DEBUG, iswindows
+from calibre.constants import numeric_version, DEBUG
 from calibre.devices.errors import (OpenFailed, ControlError, TimeoutError,
                                     InitialConnectionError, PacketError)
 from calibre.devices.interface import DevicePlugin
@@ -188,15 +188,16 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
     NEWS_IN_FOLDER              = True
     SUPPORTS_USE_AUTHOR_SORT    = False
     WANTS_UPDATED_THUMBNAILS    = True
-    NOT_WINDOWS_MAX_PATH_LEN    = 250
 
     # Guess about the max length on windows. This number will be reduced by
-    # the length of the path on the client, and by the fudge factor below
-    WINDOWS_MAX_PATH_LEN        = 250
+    # the length of the path on the client, and by the fudge factor below. We
+    # use this on all platforms because the device might be connected to windows
+    # in the future.
+    MAX_PATH_LEN                = 250
     # guess of length of MTP name. The length of the full path to the folder
     # on the device is added to this. That path includes device the mount point.
     # making this number effectively around 10 to 15 larger.
-    WINDOWS_PATH_FUDGE_FACTOR   = 40
+    PATH_FUDGE_FACTOR   = 40
 
     THUMBNAIL_HEIGHT            = 160
     PREFIX                      = ''
@@ -369,13 +370,9 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
         fname = sanitize(fname)
         ext = os.path.splitext(fname)[1]
 
-        if iswindows:
-            maxlen = (self.WINDOWS_MAX_PATH_LEN -
-                      (self.WINDOWS_PATH_FUDGE_FACTOR +
-                       self.exts_path_lengths.get(ext,
-                                        self.WINDOWS_PATH_FUDGE_FACTOR)))
-        else:
-            maxlen = self.MAX_PATH_LEN
+        maxlen = (self.WINDOWS_MAX_PATH_LEN -
+                  (self.WINDOWS_PATH_FUDGE_FACTOR +
+                   self.exts_path_lengths.get(ext, self.WINDOWS_PATH_FUDGE_FACTOR)))
 
         special_tag = None
         if mdata.tags:

From ef34e380356b151be9525ab95493692c5aa66396 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Tue, 18 Sep 2012 13:25:19 +0200
Subject: [PATCH 09/12] ...

---
 src/calibre/devices/smart_device_app/driver.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/calibre/devices/smart_device_app/driver.py b/src/calibre/devices/smart_device_app/driver.py
index f678ecf576..10df19509c 100644
--- a/src/calibre/devices/smart_device_app/driver.py
+++ b/src/calibre/devices/smart_device_app/driver.py
@@ -195,9 +195,9 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
     # in the future.
     MAX_PATH_LEN                = 250
     # guess of length of MTP name. The length of the full path to the folder
-    # on the device is added to this. That path includes device the mount point.
+    # on the device is added to this. That path includes the device's mount point
     # making this number effectively around 10 to 15 larger.
-    PATH_FUDGE_FACTOR   = 40
+    PATH_FUDGE_FACTOR           = 40
 
     THUMBNAIL_HEIGHT            = 160
     PREFIX                      = ''

From dde1512191039085ef6c15e3ae967ee12509af54 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Tue, 18 Sep 2012 14:53:40 +0200
Subject: [PATCH 10/12] Back out only doing the smart_update if the date
 changes. This prevents cover thumbnails from being regenerated for different
 devices.

---
 src/calibre/gui2/device.py | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/src/calibre/gui2/device.py b/src/calibre/gui2/device.py
index 9299c285d9..e092f819c6 100644
--- a/src/calibre/gui2/device.py
+++ b/src/calibre/gui2/device.py
@@ -1673,11 +1673,10 @@ class DeviceMixin(object): # {{{
                     if update_metadata:
                         mi = db.get_metadata(id_, index_is_id=True,
                                              get_cover=get_covers)
-                        if book.get('last_modified', None) != mi.last_modified:
-                            book.smart_update(db.get_metadata(id_,
-                                                          index_is_id=True,
-                                                          get_cover=get_covers),
-                                          replace_metadata=True)
+                        book.smart_update(db.get_metadata(id_,
+                                                      index_is_id=True,
+                                                      get_cover=get_covers),
+                                      replace_metadata=True)
                     book.in_library = 'UUID'
                     # ensure that the correct application_id is set
                     book.application_id = id_

From db967f62baa7ef521d7e393daa2acae50a4d49d0 Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Tue, 18 Sep 2012 14:54:13 +0200
Subject: [PATCH 11/12] Fix path problem (typo in name). Make cover size device
 selectable.

---
 .../devices/smart_device_app/driver.py        | 20 ++++++++++++++-----
 1 file changed, 15 insertions(+), 5 deletions(-)

diff --git a/src/calibre/devices/smart_device_app/driver.py b/src/calibre/devices/smart_device_app/driver.py
index 10df19509c..cbf2a6d0d4 100644
--- a/src/calibre/devices/smart_device_app/driver.py
+++ b/src/calibre/devices/smart_device_app/driver.py
@@ -200,6 +200,8 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
     PATH_FUDGE_FACTOR           = 40
 
     THUMBNAIL_HEIGHT            = 160
+    DEFAULT_THUMBNAIL_HEIGHT    = 160
+
     PREFIX                      = ''
     BACKLOADING_ERROR_MESSAGE   = None
 
@@ -370,9 +372,8 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
         fname = sanitize(fname)
         ext = os.path.splitext(fname)[1]
 
-        maxlen = (self.WINDOWS_MAX_PATH_LEN -
-                  (self.WINDOWS_PATH_FUDGE_FACTOR +
-                   self.exts_path_lengths.get(ext, self.WINDOWS_PATH_FUDGE_FACTOR)))
+        maxlen = (self.MAX_PATH_LEN - (self.PATH_FUDGE_FACTOR +
+                   self.exts_path_lengths.get(ext, self.PATH_FUDGE_FACTOR)))
 
         special_tag = None
         if mdata.tags:
@@ -646,7 +647,8 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
         v = self.known_metadata.get(book.lpath, None)
         if v is not None:
             return (v.get('uuid', None) == book.get('uuid', None) and
-                    v.get('last_modified', None) == book.get('last_modified', None))
+                    v.get('last_modified', None) == book.get('last_modified', None) and
+                    v.get('thumbnail', None) == book.get('thumbnail', None))
         return False
 
     def _set_known_metadata(self, book, remove=False):
@@ -841,6 +843,14 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
             self.exts_path_lengths = result.get('extensionPathLengths', {})
             self._debug('extension path lengths', self.exts_path_lengths)
 
+            self.THUMBNAIL_HEIGHT = result.get('coverHeight', self.DEFAULT_THUMBNAIL_HEIGHT)
+            if 'coverWidth' in result:
+                # Setting this field forces the aspect ratio
+                self.THUMBNAIL_WIDTH = result.get('coverWidth',
+                                      (self.DEFAULT_THUMBNAIL_HEIGHT/3) * 4)
+            elif hasattr(self, 'THUMBNAIL_WIDTH'):
+                    delattr(self, 'THUMBNAIL_WIDTH')
+
             if password:
                 returned_hash = result.get('passwordHash', None)
                 if result.get('passwordHash', None) is None:
@@ -1137,7 +1147,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
                 client_will_stream_binary = 'willStreamBinary' in result
 
                 if (client_will_stream_binary):
-                    length = result.get('fileLength');
+                    length = result.get('fileLength')
                     remaining = length
 
                     while remaining > 0:

From 25cf777a53f92faa84463fab1fece9d44ee763ac Mon Sep 17 00:00:00 2001
From: Charles Haley <>
Date: Tue, 18 Sep 2012 15:47:03 +0200
Subject: [PATCH 12/12] Comments and debug statement repair

---
 src/calibre/devices/smart_device_app/driver.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/calibre/devices/smart_device_app/driver.py b/src/calibre/devices/smart_device_app/driver.py
index cbf2a6d0d4..47bc4b3185 100644
--- a/src/calibre/devices/smart_device_app/driver.py
+++ b/src/calibre/devices/smart_device_app/driver.py
@@ -67,7 +67,7 @@ class ConnectionListener (Thread):
         device_socket = None
         while self.keep_running:
             try:
-                time.sleep(1) # Limit to one book per two seconds
+                time.sleep(1)
             except:
                 # Happens during interpreter shutdown
                 break
@@ -76,7 +76,6 @@ class ConnectionListener (Thread):
                 break
 
             if not self.driver.connection_queue.empty():
-                self.driver._debug('queue not empty')
                 queue_not_serviced_count += 1
                 if queue_not_serviced_count >= 3:
                     self.driver._debug('queue not serviced')