Wireless device driver: Make detecting and connecting to devices easier on networks where mdns is disabled (many home routers disable it)

src/calibre/devices/smart_device_app/driver.py

@@ -8,6 +8,7 @@ Created on 29 Jun 2012
 @author: charles
 '''
 import socket, select, json, inspect, os, traceback, time, sys, random
+import posixpath
 import hashlib, threading
 from base64 import b64encode, b64decode
 from functools import wraps
@@ -26,6 +27,7 @@ from calibre.ebooks.metadata import title_sort
 from calibre.ebooks.metadata.book.base import Metadata
 from calibre.ebooks.metadata.book.json_codec import JsonCodec
 from calibre.library import current_library_name
+from calibre.ptempfile import PersistentTemporaryFile
 from calibre.utils.ipc import eintr_retry_call
 from calibre.utils.config import from_json, tweaks
 from calibre.utils.date import isoformat, now
@@ -49,6 +51,12 @@ def do_zeroconf(f, port):
         '_calibresmartdeviceapp._tcp', port, {})
 
 
+class SDBook(Book):
+    def __init__(self, prefix, lpath, size=None, other=None):
+        Book.__init__(self, prefix, lpath, size=size, other=other)
+        path = getattr(self, 'path', lpath)
+        self.path = path.replace('\\', '/')
+
 class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
     name = 'SmartDevice App Interface'
     gui_name = _('SmartDevice')
@@ -70,14 +78,15 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
     DEVICE_PLUGBOARD_NAME       = 'SMART_DEVICE_APP'
     CAN_SET_METADATA            = []
     CAN_DO_DEVICE_DB_PLUGBOARD  = False
-    SUPPORTS_SUB_DIRS           = False
+    SUPPORTS_SUB_DIRS           = True
     MUST_READ_METADATA          = True
     NEWS_IN_FOLDER              = False
     SUPPORTS_USE_AUTHOR_SORT    = False
     WANTS_UPDATED_THUMBNAILS    = True
-    MAX_PATH_LEN                = 100
+    MAX_PATH_LEN                = 250
     THUMBNAIL_HEIGHT            = 160
     PREFIX                      = ''
+    BACKLOADING_ERROR_MESSAGE   = None
 
     # Some network protocol constants
     BASE_PACKET_LEN             = 4096
@@ -88,6 +97,16 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
     SEND_NOOP_EVERY_NTH_PROBE   = 5
     DISCONNECT_AFTER_N_SECONDS  = 30*60 # 30 minutes
 
+    ZEROCONF_CLIENT_STRING      = b'calibre smart device client'
+
+    # A few "random" port numbers to use for detecting clients using broadcast
+    # The clients are expected to broadcast a UDP 'hi there' on all of these
+    # ports when they attempt to connect. Calibre will respond with the port
+    # number the client should use. This scheme backs up mdns. And yes, we
+    # must hope that no other application on the machine is using one of these
+    # ports in datagram mode.
+    # If you change the ports here, all clients will also need to change.
+    BROADCAST_PORTS             = [54982, 48123, 39001, 44044, 59678]
 
     opcodes = {
         'NOOP'                   : 12,
@@ -196,25 +215,6 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
         print()
         self.debug_time = time.time()
 
-    # Various methods required by the plugin architecture
-    @classmethod
-    def _default_save_template(cls):
-        from calibre.library.save_to_disk import config
-        st = cls.SAVE_TEMPLATE if cls.SAVE_TEMPLATE else \
-            config().parse().send_template
-        if st:
-            st = os.path.basename(st)
-        return st
-
-    @classmethod
-    def save_template(cls):
-        st = cls.settings().save_template
-        if st:
-            st = os.path.basename(st)
-        else:
-            st = cls._default_save_template()
-        return st
-
     # local utilities
 
     # copied from USBMS. Perhaps this could be a classmethod in usbms?
@@ -276,6 +276,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
             extra_components.append(sanitize(fname))
         else:
             extra_components[-1] = sanitize(extra_components[-1]+ext)
+        self._debug('1', extra_components)
 
         if extra_components[-1] and extra_components[-1][0] in ('.', '_'):
             extra_components[-1] = 'x' + extra_components[-1][1:]
@@ -308,7 +309,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
 
         extra_components = list(map(remove_trailing_periods, extra_components))
         components = shorten_components_to(maxlen, extra_components)
-        filepath = os.path.join(*components)
+        filepath = posixpath.join(*components)
         return filepath
 
     def _strip_prefix(self, path):
@@ -525,19 +526,27 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
             self.device_socket = None
         self.is_connected = False
 
-    def _attach_to_port(self, port):
+    def _attach_to_port(self, sock, port):
         try:
             self._debug('try port', port)
-            self.listen_socket.bind(('', port))
+            sock.bind(('', port))
         except socket.error:
             self._debug('socket error on port', port)
             port = 0
         except:
-            self._debug('Unknown exception while allocating listen socket')
+            self._debug('Unknown exception while attaching port to socket')
             traceback.print_exc()
             raise
         return port
 
+    def _close_listen_socket(self):
+        self.listen_socket.close()
+        self.listen_socket = None
+        self.is_connected = False
+        if getattr(self, 'broadcast_socket', None) is not None:
+            self.broadcast_socket.close()
+            self.broadcast_socket = None
+
     # The public interface methods.
 
     @synchronous('sync_lock')
@@ -569,6 +578,23 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
                 except:
                     self._close_device_socket()
             return (self.is_connected, self)
+        if getattr(self, 'broadcast_socket', None) is not None:
+            while True:
+                ans = select.select((self.broadcast_socket,), (), (), 0)
+                if len(ans[0]) > 0:
+                    try:
+                        packet = self.broadcast_socket.recvfrom(100)
+                        remote = packet[1]
+                        message = str(self.ZEROCONF_CLIENT_STRING + b' (on ' +
+                                        str(socket.gethostname().partition('.')[0]) +
+                                        b'),' + str(self.port))
+                        self._debug('received broadcast', packet, message)
+                        self.broadcast_socket.sendto(message, remote)
+                    except:
+                        pass
+                else:
+                    break
+
         if getattr(self, 'listen_socket', None) is not None:
             ans = select.select((self.listen_socket,), (), (), 0)
             if len(ans[0]) > 0:
@@ -629,7 +655,10 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
                     'validExtensions': self.ALL_FORMATS,
                     'passwordChallenge': challenge,
                     'currentLibraryName': self.current_library_name,
-                                'currentLibraryUUID': library_uuid})
+                    'currentLibraryUUID': library_uuid,
+                    'pubdateFormat': tweaks['gui_pubdate_display_format'],
+                    'timestampFormat': tweaks['gui_timestamp_display_format'],
+                    'lastModifiedFormat': tweaks['gui_last_modified_display_format']})
             if opcode != 'OK':
                 # Something wrong with the return. Close the socket
                 # and continue.
@@ -775,7 +804,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
                 if opcode == 'OK':
                     if '_series_sort_' in result:
                         del result['_series_sort_']
-                    book = self.json_codec.raw_to_book(result, Book, self.PREFIX)
+                    book = self.json_codec.raw_to_book(result, SDBook, self.PREFIX)
                     self._set_known_metadata(book)
                     bl.add_book(book, replace_metadata=True)
                 else:
@@ -847,7 +876,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
             lpath = self._create_upload_path(mdata, fname, create_dirs=False)
             if not hasattr(infile, 'read'):
                 infile = USBMS.normalize_path(infile)
-            book = Book(self.PREFIX, lpath, other=mdata)
+            book = SDBook(self.PREFIX, lpath, other=mdata)
             length = self._put_file(infile, lpath, book, i, len(files))
             if length < 0:
                 raise ControlError(desc='Sending book %s to device failed' % lpath)
@@ -872,7 +901,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
             lpath = location[0]
             length = location[1]
             lpath = self._strip_prefix(lpath)
-            book = Book(self.PREFIX, lpath, other=info)
+            book = SDBook(self.PREFIX, lpath, other=info)
             if book.size is None:
                 book.size = length
             b = booklists[0].add_book(book, replace_metadata=True)
@@ -929,6 +958,15 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
             else:
                 raise ControlError(desc='request for book data failed')
 
+    @synchronous('sync_lock')
+    def prepare_addable_books(self, paths):
+        for idx, path in enumerate(paths):
+            (ign, ext) = os.path.splitext(path)
+            tf = PersistentTemporaryFile(suffix=ext)
+            self.get_file(path, tf)
+            paths[idx] = tf.name
+        return paths
+
     @synchronous('sync_lock')
     def set_plugboards(self, plugboards, pb_func):
         self._debug()
@@ -976,31 +1014,26 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
                 message = _('Invalid port in options: %s')% \
                             self.settings().extra_customization[self.OPT_PORT_NUMBER]
                 self._debug(message)
-                self.listen_socket.close()
+                self._close_listen_socket()
-                self.listen_socket = None
-                self.is_connected = False
                 return message
 
-            port = self._attach_to_port(opt_port)
+            port = self._attach_to_port(self.listen_socket, opt_port)
             if port == 0:
                 message = _('Failed to connect to port %d. Try a different value.')%opt_port
                 self._debug(message)
-                self.listen_socket.close()
+                self._close_listen_socket()
-                self.listen_socket = None
-                self.is_connected = False
                 return message
         else:
             while i < 100: # try up to 100 random port numbers
                 i += 1
-                port = self._attach_to_port(random.randint(8192, 32000))
+                port = self._attach_to_port(self.listen_socket,
+                                            random.randint(8192, 32000))
                 if port != 0:
                     break
             if port == 0:
                 message = _('Failed to allocate a random port')
                 self._debug(message)
-                self.listen_socket.close()
+                self._close_listen_socket()
-                self.listen_socket = None
-                self.is_connected = False
                 return message
 
         try:
@@ -1008,9 +1041,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
         except:
             message = 'listen on port %d failed' % port
             self._debug(message)
-            self.listen_socket.close()
+            self._close_listen_socket()
-            self.listen_socket = None
-            self.is_connected = False
             return message
 
         try:
@@ -1018,21 +1049,40 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
         except:
             message = 'registration with bonjour failed'
             self._debug(message)
-            self.listen_socket.close()
+            self._close_listen_socket()
-            self.listen_socket = None
-            self.is_connected = False
             return message
 
         self._debug('listening on port', port)
         self.port = port
 
+        # Now try to open a UDP socket to receive broadcasts on
+
+        try:
+            self.broadcast_socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+        except:
+            message = 'creation of broadcast socket failed. This is not fatal.'
+            self._debug(message)
+            return message
+
+        for p in self.BROADCAST_PORTS:
+            port = self._attach_to_port(self.broadcast_socket, p)
+            if port != 0:
+                self._debug('broadcast socket listening on port', port)
+                break
+
+        if port == 0:
+            self.broadcast_socket.close()
+            self.broadcast_socket = None
+            message = 'attaching port to broadcast socket failed. This is not fatal.'
+            self._debug(message)
+            return message
+
+
     @synchronous('sync_lock')
     def shutdown(self):
         if getattr(self, 'listen_socket', None) is not None:
             do_zeroconf(unpublish_zeroconf, self.port)
-            self.listen_socket.close()
+            self._close_listen_socket()
-            self.listen_socket = None
-            self.is_connected = False
 
     # Methods for dynamic control
 

src/calibre/gui2/actions/device.py

@@ -237,20 +237,28 @@ class ConnectShareAction(InterfaceAction):
             self.share_conn_menu.hide_smartdevice_menus()
 
     def set_smartdevice_action_state(self):
-        from calibre.utils.mdns import get_external_ip
+        from calibre.gui2.dialogs.smartdevice import get_all_ip_addresses
         dm = self.gui.device_manager
 
+        all_ips = get_all_ip_addresses()
+        if len(all_ips) > 3:
+            formatted_addresses = _('Many IP addresses. See Start/Stop dialog.')
+            show_port = False
+        else:
+            formatted_addresses = ' or '.join(get_all_ip_addresses())
+            show_port = True
+
         running = dm.is_running('smartdevice')
         if not running:
             text = self.share_conn_menu.DEVICE_MSGS[0]
         else:
             use_fixed_port = dm.get_option('smartdevice', 'use_fixed_port')
             port_number = dm.get_option('smartdevice', 'port_number')
-            if use_fixed_port:
+            if show_port and use_fixed_port:
-                text = self.share_conn_menu.DEVICE_MSGS[1]  + ' [%s port %s]'%(
+                text = self.share_conn_menu.DEVICE_MSGS[1]  + ' [%s, port %s]'%(
-                                            get_external_ip(), port_number)
+                                            formatted_addresses, port_number)
             else:
-                text = self.share_conn_menu.DEVICE_MSGS[1]  + ' [%s]'%get_external_ip()
+                text = self.share_conn_menu.DEVICE_MSGS[1] + ' [' + formatted_addresses + ']'
 
         icon = 'green' if running else 'red'
         ac = self.share_conn_menu.control_smartdevice_action

src/calibre/gui2/dialogs/smartdevice.py

@@ -5,10 +5,35 @@ from __future__ import (unicode_literals, division, absolute_import,
 __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 
-from PyQt4.Qt import (QDialog, QLineEdit, Qt)
+from PyQt4.Qt import (QDialog, QLineEdit, Qt, QPushButton, QDialogButtonBox)
 
 from calibre.gui2 import error_dialog
 from calibre.gui2.dialogs.smartdevice_ui import Ui_Dialog
+from calibre.utils.config import prefs
+from calibre.utils.mdns import get_all_ips
+
+def _cmp_ipaddr(l, r):
+    lparts = ['%3s'%x for x in l.split('.')]
+    rparts = ['%3s'%x for x in r.split('.')]
+
+    if lparts[0] in ['192', '170', ' 10']:
+        if rparts[0] not in ['192', '170', '10']:
+            return -1
+        return cmp(rparts, lparts)
+
+    if rparts[0] in ['192', '170', ' 10']:
+        return 1
+
+    return cmp(lparts, rparts)
+
+def get_all_ip_addresses():
+    ipaddrs = list()
+    for iface in get_all_ips().itervalues():
+        for addrs in iface:
+            if 'broadcast' in addrs and addrs['addr'] != '127.0.0.1':
+                ipaddrs.append(addrs['addr'])
+    ipaddrs.sort(cmp=_cmp_ipaddr)
+    return ipaddrs
 
 class SmartdeviceDialog(QDialog, Ui_Dialog):
 
@@ -40,6 +65,15 @@ class SmartdeviceDialog(QDialog, Ui_Dialog):
               'to the port, try another number. You can use any number between '
               '8,000 and 32,000.') + '</p>')
 
+
+        self.ip_addresses.setToolTip('<p>' +
+            _('These are the IP addresses for this computer. If you decide to have your device connect to '
+              'calibre using a fixed IP address, one of these addresses should '
+              'be the one you use. It is unlikely but possible that the correct '
+              'IP address is not listed here, in which case you will need to go '
+              "to your computer's control panel to get a complete list of "
+              "your computer's network interfaces and IP addresses.") + '</p>')
+
         self.show_password.stateChanged[int].connect(self.toggle_password)
         self.use_fixed_port.stateChanged[int].connect(self.use_fixed_port_changed)
 
@@ -57,15 +91,39 @@ class SmartdeviceDialog(QDialog, Ui_Dialog):
         self.orig_port_number = self.device_manager.get_option('smartdevice',
                                                           'port_number')
         self.fixed_port.setText(self.orig_port_number)
-        self.use_fixed_port.setChecked(self.orig_fixed_port);
+        self.use_fixed_port.setChecked(self.orig_fixed_port)
         if not self.orig_fixed_port:
-            self.fixed_port.setEnabled(False);
+            self.fixed_port.setEnabled(False)
 
         if pw:
             self.password_box.setText(pw)
 
+        self.auto_mgmt_button = QPushButton(_('Enable automatic metadata management'))
+        self.auto_mgmt_button.clicked.connect(self.auto_mgmt_button_clicked)
+        self.auto_mgmt_button.setToolTip('<p>' +
+            _('Enabling automatic metadata management tells calibre to send any '
+              'changes you made to books\' metadata when your device is '
+              'connected, which is the most useful setting when using the wireless '
+              'device interface. If automatic metadata management is not '
+              'enabled, changes are sent only when you re-send the book. You can '
+              'get more information or change this preference to some other '
+              'choice at Preferences -> Sending books to devices -> '
+              'Metadata management')
+                                                    + '</p>')
+        self.buttonBox.addButton(self.auto_mgmt_button, QDialogButtonBox.ActionRole)
+        if prefs['manage_device_metadata'] == 'on_connect':
+            self.auto_mgmt_button.setText(_('Automatic metadata management is enabled'))
+            self.auto_mgmt_button.setEnabled(False)
+
+        self.ip_addresses.setText(', '.join(get_all_ip_addresses()))
+
         self.resize(self.sizeHint())
 
+    def auto_mgmt_button_clicked(self):
+        self.auto_mgmt_button.setText(_('Automatic metadata management is enabled'))
+        self.auto_mgmt_button.setEnabled(False)
+        prefs.set('manage_device_metadata', 'on_connect')
+
     def use_fixed_port_changed(self, state):
         self.fixed_port.setEnabled(state == Qt.Checked)
 

src/calibre/gui2/dialogs/smartdevice.ui

@@ -38,7 +38,34 @@
      </property>
     </widget>
    </item>
+   <item row="1" column="0">
+    <widget class="QLabel" name="label_23">
+     <property name="text">
+      <string>Calibre IP addresses:</string>
+     </property>
+    </widget>
+   </item>
    <item row="1" column="1">
+    <widget class="QLabel" name="ip_addresses">
+     <property name="text">
+      <string>Possibe IP addresses:</string>
+     </property>
+     <property name="wordWrap">
+      <bool>true</bool>
+     </property>
+    </widget>
+   </item>
+   <item row="2" column="0">
+    <widget class="QLabel" name="label_2">
+     <property name="text">
+      <string>Optional &amp;password:</string>
+     </property>
+     <property name="buddy">
+      <cstring>password_box</cstring>
+     </property>
+    </widget>
+   </item>
+   <item row="2" column="1">
     <widget class="QLineEdit" name="password_box">
      <property name="sizePolicy">
       <sizepolicy hsizetype="Expanding" vsizetype="Expanding">
@@ -54,24 +81,14 @@
      </property>
     </widget>
    </item>
-   <item row="1" column="0">
+   <item row="2" column="2">
-    <widget class="QLabel" name="label_2">
-     <property name="text">
-      <string>Optional &amp;password:</string>
-     </property>
-     <property name="buddy">
-      <cstring>password_box</cstring>
-     </property>
-    </widget>
-   </item>
-   <item row="1" column="2">
     <widget class="QCheckBox" name="show_password">
      <property name="text">
       <string>&amp;Show password</string>
      </property>
     </widget>
    </item>
-   <item row="2" column="0">
+   <item row="4" column="0">
     <widget class="QLabel" name="label_21">
      <property name="text">
       <string>Optional &amp;fixed port:</string>
@@ -81,7 +98,7 @@
      </property>
     </widget>
    </item>
-   <item row="2" column="1">
+   <item row="4" column="1">
     <widget class="QLineEdit" name="fixed_port">
      <property name="sizePolicy">
       <sizepolicy hsizetype="Expanding" vsizetype="Expanding">
@@ -94,14 +111,21 @@
      </property>
     </widget>
    </item>
-   <item row="2" column="2">
+   <item row="4" column="2">
     <widget class="QCheckBox" name="use_fixed_port">
      <property name="text">
       <string>&amp;Use a fixed port</string>
      </property>
     </widget>
    </item>
-   <item row="4" column="0" colspan="3">
+   <item row="6" column="0" colspan="3">
+    <widget class="QCheckBox" name="autostart_box">
+     <property name="text">
+      <string>&amp;Automatically allow connections at calibre startup</string>
+     </property>
+    </widget>
+   </item>
+   <item row="10" column="0" colspan="3">
     <widget class="QDialogButtonBox" name="buttonBox">
      <property name="orientation">
       <enum>Qt::Horizontal</enum>
@@ -111,13 +135,6 @@
      </property>
     </widget>
    </item>
-   <item row="3" column="0" colspan="3">
-    <widget class="QCheckBox" name="autostart_box">
-     <property name="text">
-      <string>&amp;Automatically allow connections at calibre startup</string>
-     </property>
-    </widget>
-   </item>
   </layout>
  </widget>
  <resources>