Wireless device connections: Use a streamed mode for improved networking performance leading to much less time spent sending metadata to/from the device. Also make it easier to specify a fixed port directly in the dialog used to start the connection.

2025-07-09 03:04:10 -04:00 · 2012-08-23 11:58:32 +05:30 · 2012-08-23 11:58:32 +05:30 · 881587066a
commit 881587066a
parent 492915a94d 6bb6900d85
5 changed files with 198 additions and 55 deletions
--- a/src/calibre/devices/smart_device_app/driver.py
+++ b/src/calibre/devices/smart_device_app/driver.py
@ -89,6 +89,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
    SEND_NOOP_EVERY_NTH_PROBE   = 5
    DISCONNECT_AFTER_N_SECONDS  = 30*60 # 30 minutes

+
    opcodes = {
        'NOOP'                   : 12,
        'OK'                     : 0,
@ -163,11 +164,13 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
    OPT_COLLECTIONS             = 8
    OPT_AUTODISCONNECT          = 10

+
    def __init__(self, path):
        self.sync_lock = threading.RLock()
        self.noop_counter = 0
        self.debug_start_time = time.time()
        self.debug_time = time.time()
+        self.client_can_stream_books = False

    def _debug(self, *args):
        if not DEBUG:
@ -343,7 +346,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
            # recv seems to return a pointer into some internal buffer.
            # Things get trashed if we don't make a copy of the data.
            self.device_socket.settimeout(self.MAX_CLIENT_COMM_TIMEOUT)
-            v = self.device_socket.recv(self.BASE_PACKET_LEN)
+            v = self.device_socket.recv(2)
            self.device_socket.settimeout(None)
            if len(v) == 0:
                return '' # documentation says the socket is broken permanently.
@ -382,7 +385,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
                    raise
                time.sleep(0.1) # lets not hammer the OS too hard

-    def _call_client(self, op, arg, print_debug_info=True):
+    def _call_client(self, op, arg, print_debug_info=True, wait_for_response=True):
        if op != 'NOOP':
            self.noop_counter = 0
        extra_debug = self.settings().extra_customization[self.OPT_EXTRA_DEBUG]
@ -398,7 +401,28 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
            if print_debug_info and extra_debug:
                self._debug('send string', s)
            self.device_socket.settimeout(self.MAX_CLIENT_COMM_TIMEOUT)
-            self._send_byte_string((b'%d' % len(s))+s)
+            self._send_byte_string((b'%d' % len(s)) + s)
+            if not wait_for_response:
+                return None, None
+            return self._receive_from_client(print_debug_info=print_debug_info)
+        except socket.timeout:
+            self._debug('timeout communicating with device')
+            self._close_device_socket()
+            raise TimeoutError('Device did not respond in reasonable time')
+        except socket.error:
+            self._debug('device went away')
+            self._close_device_socket()
+            raise ControlError(desc='Device closed the network connection')
+        except:
+            self._debug('other exception')
+            traceback.print_exc()
+            self._close_device_socket()
+            raise
+        raise ControlError(desc='Device responded with incorrect information')
+
+    def _receive_from_client(self, print_debug_info=True):
+        extra_debug = self.settings().extra_customization[self.OPT_EXTRA_DEBUG]
+        try:
            v = self._read_string_from_net()
            self.device_socket.settimeout(None)
            if print_debug_info and extra_debug:
@ -434,9 +458,13 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
        book_metadata.size = length
        infile.seek(0)
        self._debug(lpath, length)
-        self._call_client('SEND_BOOK', {'lpath': lpath, 'length': length,
+        opcode, result = self._call_client('SEND_BOOK', {'lpath': lpath, 'length': length,
                               'metadata': book_metadata, 'thisBook': this_book,
-                               'totalBooks': total_books}, print_debug_info=False)
+                               'totalBooks': total_books,
+                               'willStreamBooks': self.client_can_stream_books},
+                          print_debug_info=False,
+                          wait_for_response=(not self.client_can_stream_books))
+
        self._set_known_metadata(book_metadata)
        pos = 0
        failed = False
@ -449,9 +477,10 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
                b = b64encode(b)
                opcode, result = self._call_client('BOOK_DATA',
                                {'lpath': lpath, 'position': pos, 'data': b},
-                                print_debug_info=False)
+                                print_debug_info=False,
+                                wait_for_response=(not self.client_can_stream_books))
                pos += blen
-                if opcode != 'OK':
+                if not self.client_can_stream_books and opcode != 'OK':
                    self._debug('protocol error', opcode)
                    failed = True
                    break
@ -466,6 +495,10 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
            return self.OPT_PASSWORD
        elif opt_string == 'autostart':
            return self.OPT_AUTOSTART
+        elif opt_string == 'use_fixed_port':
+            return self.OPT_USE_PORT
+        elif opt_string == 'port_number':
+            return self.OPT_PORT_NUMBER
        else:
            return None

@ -510,6 +543,19 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
            self.device_socket = None
        self.is_connected = False

+    def _attach_to_port(self, port):
+        try:
+            self._debug('try port', port)
+            self.listen_socket.bind(('', port))
+        except socket.error:
+            self._debug('socket error on port', port)
+            port = 0
+        except:
+            self._debug('Unknown exception while allocating listen socket')
+            traceback.print_exc()
+            raise
+        return port
+
    # The public interface methods.

    @synchronous('sync_lock')
@ -560,7 +606,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
                        if attempts >= self.MAX_UNSUCCESSFUL_CONNECTS:
                            self._debug('too many connection attempts from', peer)
                            self._close_device_socket()
-                            raise InitialConnectionError(_('Too many connection attempts from %s')%peer)
+                            raise InitialConnectionError(_('Too many connection attempts from %s') % peer)
                        else:
                            self.connection_attempts[peer] = attempts + 1
                    except InitialConnectionError:
@ -619,6 +665,8 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
                self._close_device_socket()
                return False
            self._debug('CC version #:', result.get('ccVersionNumber', 'unknown'))
+            self.client_can_stream_books = result.get('canStreamBooks', False)
+            self._debug('CC can stream', self.client_can_stream_books);
            self.max_book_packet_len = result.get('maxBookContentPacketLen',
                                                  self.BASE_PACKET_LEN)
            exts = result.get('acceptedExtensions', None)
@ -725,11 +773,15 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
        self._debug(oncard)
        if oncard is not None:
            return CollectionsBookList(None, None, None)
-        opcode, result = self._call_client('GET_BOOK_COUNT', {})
+        opcode, result = self._call_client('GET_BOOK_COUNT', {'canStream':True})
        bl = CollectionsBookList(None, self.PREFIX, self.settings)
        if opcode == 'OK':
            count = result['count']
+            will_stream = 'willStream' in result;
            for i in range(0, count):
+                if will_stream:
+                    opcode, result = self._receive_from_client(print_debug_info=False)
+                else:
                    opcode, result = self._call_client('GET_BOOK_METADATA', {'index': i},
                                                  print_debug_info=False)
                if opcode == 'OK':
@ -760,11 +812,10 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
        # probably need to send two booklists, one with calibre's data that is
        # given back by "books", and one that has been plugboarded.
        self._call_client('SEND_BOOKLISTS', { 'count': len(booklists[0]),
-                                              'collections': coldict} )
+                                              'collections': coldict})
        for i,book in enumerate(booklists[0]):
            if not self._metadata_already_on_device(book):
                self._set_known_metadata(book)
-                self._debug('syncing book', book.lpath)
                opcode, result = self._call_client('SEND_BOOK_METADATA',
                                                  {'index': i, 'data': book},
                                                  print_debug_info=False)
@ -802,19 +853,19 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
            paths.append((lpath, length))
            # No need to deal with covers. The client will get the thumbnails
            # in the mi structure
-            self.report_progress((i+1) / float(len(files)), _('Transferring books to device...'))
+            self.report_progress((i + 1) / float(len(files)), _('Transferring books to device...'))

        self.report_progress(1.0, _('Transferring books to device...'))
-        self._debug('finished uploading %d books'%(len(files)))
+        self._debug('finished uploading %d books' % (len(files)))
        return paths

    @synchronous('sync_lock')
    def add_books_to_metadata(self, locations, metadata, booklists):
-        self._debug('adding metadata for %d books'%(len(metadata)))
+        self._debug('adding metadata for %d books' % (len(metadata)))

        metadata = iter(metadata)
        for i, location in enumerate(locations):
-            self.report_progress((i+1) / float(len(locations)),
+            self.report_progress((i + 1) / float(len(locations)),
                                 _('Adding books to device metadata listing...'))
            info = metadata.next()
            lpath = location[0]
@ -846,14 +897,14 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
        self._debug(paths)
        for i, path in enumerate(paths):
            path = self._strip_prefix(path)
-            self.report_progress((i+1) / float(len(paths)), _('Removing books from device metadata listing...'))
+            self.report_progress((i + 1) / float(len(paths)), _('Removing books from device metadata listing...'))
            for bl in booklists:
                for book in bl:
                    if path == book.path:
                        bl.remove_book(book)
                        self._set_known_metadata(book, remove=True)
        self.report_progress(1.0, _('Removing books from device metadata listing...'))
-        self._debug('finished removing metadata for %d books'%(len(paths)))
+        self._debug('finished removing metadata for %d books' % (len(paths)))


    @synchronous('sync_lock')
@ -864,7 +915,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
        while not eof:
            opcode, result = self._call_client('GET_BOOK_FILE_SEGMENT',
                                    {'lpath' : path, 'position': position},
-                                    print_debug_info=False )
+                                    print_debug_info=False)
            if opcode == 'OK':
                if not result['eof']:
                    data = b64decode(result['data'])
@ -904,57 +955,71 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
        self.max_book_packet_len = 0
        self.noop_counter = 0
        self.connection_attempts = {}
+        self.client_can_stream_books = False
+
+        message = None
        try:
            self.listen_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        except:
-            self._debug('creation of listen socket failed')
-            return
+            message = 'creation of listen socket failed'
+            self._debug(message)
+            return message

        i = 0
-        while i < 100: # try up to 100 random port numbers
+
        if self.settings().extra_customization[self.OPT_USE_PORT]:
-                i = 100
            try:
-                    port = int(self.settings().extra_customization[self.OPT_PORT_NUMBER])
+                opt_port = int(self.settings().extra_customization[self.OPT_PORT_NUMBER])
            except:
-                    port = 0
-            else:
-                i += 1
-                port = random.randint(8192, 32000)
-            try:
-                self._debug('try port', port)
-                self.listen_socket.bind(('', port))
-                break
-            except socket.error:
-                port = 0
-            except:
-                self._debug('Unknown exception while allocating listen socket')
-                traceback.print_exc()
-                raise
-        if port == 0:
-            self._debug('Failed to allocate a port');
+                message = _('Invalid port in options: %s')% \
+                            self.settings().extra_customization[self.OPT_PORT_NUMBER]
+                self.debug(message)
                self.listen_socket.close()
                self.listen_socket = None
                self.is_connected = False
-            return
+                return message
+
+            port = self._attach_to_port(opt_port)
+            if port == 0:
+                message = 'Failed to connect to port %d'%opt_port
+                self._debug(message);
+                self.listen_socket.close()
+                self.listen_socket = None
+                self.is_connected = False
+                return message
+        else:
+            while i < 100: # try up to 100 random port numbers
+                i += 1
+                port = self._attach_to_port(random.randint(8192, 32000))
+                if port != 0:
+                    break;
+            if port == 0:
+                message = _('Failed to allocate a random port')
+                self._debug(message);
+                self.listen_socket.close()
+                self.listen_socket = None
+                self.is_connected = False
+                return message

        try:
            self.listen_socket.listen(0)
        except:
-            self._debug('listen on socket failed', port)
+            message = 'listen on port %d failed' % port
+            self._debug(message)
            self.listen_socket.close()
            self.listen_socket = None
            self.is_connected = False
-            return
+            return message

        try:
            do_zeroconf(publish_zeroconf, port)
        except:
-            self._debug('registration with bonjour failed')
+            message = 'registration with bonjour failed'
+            self._debug(message)
            self.listen_socket.close()
            self.listen_socket = None
            self.is_connected = False
-            return
+            return message

        self._debug('listening on port', port)
        self.port = port
@ -975,7 +1040,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):

    @synchronous('sync_lock')
    def start_plugin(self):
-        self.startup_on_demand()
+        return self.startup_on_demand()

    @synchronous('sync_lock')
    def stop_plugin(self):
--- a/src/calibre/gui2/actions/device.py
+++ b/src/calibre/gui2/actions/device.py
@ -238,11 +238,20 @@ class ConnectShareAction(InterfaceAction):

    def set_smartdevice_action_state(self):
        from calibre.utils.mdns import get_external_ip
-        running = self.gui.device_manager.is_running('smartdevice')
+        dm = self.gui.device_manager
+
+        running = dm.is_running('smartdevice')
        if not running:
            text = self.share_conn_menu.DEVICE_MSGS[0]
+        else:
+            use_fixed_port = dm.get_option('smartdevice', 'use_fixed_port')
+            port_number = dm.get_option('smartdevice', 'port_number')
+            if use_fixed_port:
+                text = self.share_conn_menu.DEVICE_MSGS[1]  + ' [%s port %s]'%(
+                                            get_external_ip(), port_number)
            else:
                text = self.share_conn_menu.DEVICE_MSGS[1]  + ' [%s]'%get_external_ip()
+
        icon = 'green' if running else 'red'
        ac = self.share_conn_menu.control_smartdevice_action
        ac.setIcon(QIcon(I('dot_%s.png'%icon)))
--- a/src/calibre/gui2/device.py
+++ b/src/calibre/gui2/device.py
@ -554,7 +554,7 @@ class DeviceManager(Thread): # {{{
    # will switch to the device thread before calling the plugin.

    def start_plugin(self, name):
-        self._call_request(name, 'start_plugin')
+        return self._call_request(name, 'start_plugin')

    def stop_plugin(self, name):
        self._call_request(name, 'stop_plugin')
--- a/src/calibre/gui2/dialogs/smartdevice.py
+++ b/src/calibre/gui2/dialogs/smartdevice.py
@ -7,6 +7,7 @@ __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'

 from PyQt4.Qt import (QDialog, QLineEdit, Qt)

+from calibre.gui2 import error_dialog
 from calibre.gui2.dialogs.smartdevice_ui import Ui_Dialog

 class SmartdeviceDialog(QDialog, Ui_Dialog):
@ -27,7 +28,21 @@ class SmartdeviceDialog(QDialog, Ui_Dialog):
              'smart device interface when calibre starts. You should not do '
              'this if you are using a network that is not secure and you '
              'are not setting a password.') + '</p>')
+
+        self.use_fixed_port.setToolTip('<p>' +
+            _('Check this box if you want calibre to use a fixed network '
+              'port. Normally you will not need to do this. However, if '
+              'your device consistently fails to connect to calibre, '
+              'try checking this box.') + '</p>')
+
+        self.fixed_port.setToolTip('<p>' +
+            _('A port number must be a 4-digit integer less than 32,000. No '
+              'two network applications on the same computer can use '
+              'the same port number. If calibre says that it fails to connect '
+              'to the port, try a different number.') + '</p>')
+
        self.show_password.stateChanged[int].connect(self.toggle_password)
+        self.use_fixed_port.stateChanged[int].connect(self.use_fixed_port_changed)

        self.device_manager = parent.device_manager

@ -37,8 +52,22 @@ class SmartdeviceDialog(QDialog, Ui_Dialog):
        pw = self.device_manager.get_option('smartdevice', 'password')
        if pw:
            self.password_box.setText(pw)
+
+        use_fixed_port = self.device_manager.get_option('smartdevice', 'use_fixed_port')
+        port_number = self.device_manager.get_option('smartdevice', 'port_number')
+        self.fixed_port.setText(port_number)
+        self.use_fixed_port.setChecked(use_fixed_port);
+        if not use_fixed_port:
+            self.fixed_port.setEnabled(False);
+
+        if pw:
+            self.password_box.setText(pw)
+
        self.resize(self.sizeHint())

+    def use_fixed_port_changed(self, state):
+        self.fixed_port.setEnabled(state == Qt.Checked)
+
    def toggle_password(self, state):
        self.password_box.setEchoMode(QLineEdit.Password if state ==
                Qt.Unchecked else QLineEdit.Normal)
@ -48,6 +77,16 @@ class SmartdeviceDialog(QDialog, Ui_Dialog):
                                       unicode(self.password_box.text()))
        self.device_manager.set_option('smartdevice', 'autostart',
                                       self.autostart_box.isChecked())
-        self.device_manager.start_plugin('smartdevice')
+        self.device_manager.set_option('smartdevice', 'use_fixed_port',
+                                       self.use_fixed_port.isChecked())
+        self.device_manager.set_option('smartdevice', 'port_number',
+                                       unicode(self.fixed_port.text()))
+
+        message = self.device_manager.start_plugin('smartdevice')
+
+        if not self.device_manager.is_running('smartdevice'):
+            error_dialog(self, _('Problem starting smartdevice'),
+                _('The snart device driver did not start. It said "%s"')%message, show=True)
+        else:
            QDialog.accept(self)

--- a/src/calibre/gui2/dialogs/smartdevice.ui
+++ b/src/calibre/gui2/dialogs/smartdevice.ui
@ -71,6 +71,36 @@
     </property>
    </widget>
   </item>
+   <item row="2" column="0">
+    <widget class="QLabel" name="label_21">
+     <property name="text">
+      <string>Optional &amp;fixed port:</string>
+     </property>
+     <property name="buddy">
+      <cstring>fixed_port</cstring>
+     </property>
+    </widget>
+   </item>
+   <item row="2" column="1">
+    <widget class="QLineEdit" name="fixed_port">
+     <property name="sizePolicy">
+      <sizepolicy hsizetype="Expanding" vsizetype="Expanding">
+       <horstretch>100</horstretch>
+       <verstretch>0</verstretch>
+      </sizepolicy>
+     </property>
+     <property name="placeholderText">
+      <string>Optional port number</string>
+     </property>
+    </widget>
+   </item>
+   <item row="2" column="2">
+    <widget class="QCheckBox" name="use_fixed_port">
+     <property name="text">
+      <string>&amp;Use a fixed port</string>
+     </property>
+    </widget>
+   </item>
   <item row="4" column="0" colspan="3">
    <widget class="QDialogButtonBox" name="buttonBox">
     <property name="orientation">