Wireless device connections: Use a streamed mode for improved networking performance leading to much less time spent sending metadata to/from the device. Also make it easier to specify a fixed port directly in the dialog used to start the connection.

This commit is contained in:
Kovid Goyal 2012-08-23 11:58:32 +05:30
commit 881587066a
5 changed files with 198 additions and 55 deletions

View File

@ -89,6 +89,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
SEND_NOOP_EVERY_NTH_PROBE = 5 SEND_NOOP_EVERY_NTH_PROBE = 5
DISCONNECT_AFTER_N_SECONDS = 30*60 # 30 minutes DISCONNECT_AFTER_N_SECONDS = 30*60 # 30 minutes
opcodes = { opcodes = {
'NOOP' : 12, 'NOOP' : 12,
'OK' : 0, 'OK' : 0,
@ -163,11 +164,13 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
OPT_COLLECTIONS = 8 OPT_COLLECTIONS = 8
OPT_AUTODISCONNECT = 10 OPT_AUTODISCONNECT = 10
def __init__(self, path): def __init__(self, path):
self.sync_lock = threading.RLock() self.sync_lock = threading.RLock()
self.noop_counter = 0 self.noop_counter = 0
self.debug_start_time = time.time() self.debug_start_time = time.time()
self.debug_time = time.time() self.debug_time = time.time()
self.client_can_stream_books = False
def _debug(self, *args): def _debug(self, *args):
if not DEBUG: if not DEBUG:
@ -343,7 +346,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
# recv seems to return a pointer into some internal buffer. # recv seems to return a pointer into some internal buffer.
# Things get trashed if we don't make a copy of the data. # Things get trashed if we don't make a copy of the data.
self.device_socket.settimeout(self.MAX_CLIENT_COMM_TIMEOUT) self.device_socket.settimeout(self.MAX_CLIENT_COMM_TIMEOUT)
v = self.device_socket.recv(self.BASE_PACKET_LEN) v = self.device_socket.recv(2)
self.device_socket.settimeout(None) self.device_socket.settimeout(None)
if len(v) == 0: if len(v) == 0:
return '' # documentation says the socket is broken permanently. return '' # documentation says the socket is broken permanently.
@ -382,7 +385,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
raise raise
time.sleep(0.1) # lets not hammer the OS too hard time.sleep(0.1) # lets not hammer the OS too hard
def _call_client(self, op, arg, print_debug_info=True): def _call_client(self, op, arg, print_debug_info=True, wait_for_response=True):
if op != 'NOOP': if op != 'NOOP':
self.noop_counter = 0 self.noop_counter = 0
extra_debug = self.settings().extra_customization[self.OPT_EXTRA_DEBUG] extra_debug = self.settings().extra_customization[self.OPT_EXTRA_DEBUG]
@ -398,7 +401,28 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
if print_debug_info and extra_debug: if print_debug_info and extra_debug:
self._debug('send string', s) self._debug('send string', s)
self.device_socket.settimeout(self.MAX_CLIENT_COMM_TIMEOUT) self.device_socket.settimeout(self.MAX_CLIENT_COMM_TIMEOUT)
self._send_byte_string((b'%d' % len(s))+s) self._send_byte_string((b'%d' % len(s)) + s)
if not wait_for_response:
return None, None
return self._receive_from_client(print_debug_info=print_debug_info)
except socket.timeout:
self._debug('timeout communicating with device')
self._close_device_socket()
raise TimeoutError('Device did not respond in reasonable time')
except socket.error:
self._debug('device went away')
self._close_device_socket()
raise ControlError(desc='Device closed the network connection')
except:
self._debug('other exception')
traceback.print_exc()
self._close_device_socket()
raise
raise ControlError(desc='Device responded with incorrect information')
def _receive_from_client(self, print_debug_info=True):
extra_debug = self.settings().extra_customization[self.OPT_EXTRA_DEBUG]
try:
v = self._read_string_from_net() v = self._read_string_from_net()
self.device_socket.settimeout(None) self.device_socket.settimeout(None)
if print_debug_info and extra_debug: if print_debug_info and extra_debug:
@ -434,9 +458,13 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
book_metadata.size = length book_metadata.size = length
infile.seek(0) infile.seek(0)
self._debug(lpath, length) self._debug(lpath, length)
self._call_client('SEND_BOOK', {'lpath': lpath, 'length': length, opcode, result = self._call_client('SEND_BOOK', {'lpath': lpath, 'length': length,
'metadata': book_metadata, 'thisBook': this_book, 'metadata': book_metadata, 'thisBook': this_book,
'totalBooks': total_books}, print_debug_info=False) 'totalBooks': total_books,
'willStreamBooks': self.client_can_stream_books},
print_debug_info=False,
wait_for_response=(not self.client_can_stream_books))
self._set_known_metadata(book_metadata) self._set_known_metadata(book_metadata)
pos = 0 pos = 0
failed = False failed = False
@ -449,9 +477,10 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
b = b64encode(b) b = b64encode(b)
opcode, result = self._call_client('BOOK_DATA', opcode, result = self._call_client('BOOK_DATA',
{'lpath': lpath, 'position': pos, 'data': b}, {'lpath': lpath, 'position': pos, 'data': b},
print_debug_info=False) print_debug_info=False,
wait_for_response=(not self.client_can_stream_books))
pos += blen pos += blen
if opcode != 'OK': if not self.client_can_stream_books and opcode != 'OK':
self._debug('protocol error', opcode) self._debug('protocol error', opcode)
failed = True failed = True
break break
@ -466,6 +495,10 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
return self.OPT_PASSWORD return self.OPT_PASSWORD
elif opt_string == 'autostart': elif opt_string == 'autostart':
return self.OPT_AUTOSTART return self.OPT_AUTOSTART
elif opt_string == 'use_fixed_port':
return self.OPT_USE_PORT
elif opt_string == 'port_number':
return self.OPT_PORT_NUMBER
else: else:
return None return None
@ -510,6 +543,19 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
self.device_socket = None self.device_socket = None
self.is_connected = False self.is_connected = False
def _attach_to_port(self, port):
try:
self._debug('try port', port)
self.listen_socket.bind(('', port))
except socket.error:
self._debug('socket error on port', port)
port = 0
except:
self._debug('Unknown exception while allocating listen socket')
traceback.print_exc()
raise
return port
# The public interface methods. # The public interface methods.
@synchronous('sync_lock') @synchronous('sync_lock')
@ -560,7 +606,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
if attempts >= self.MAX_UNSUCCESSFUL_CONNECTS: if attempts >= self.MAX_UNSUCCESSFUL_CONNECTS:
self._debug('too many connection attempts from', peer) self._debug('too many connection attempts from', peer)
self._close_device_socket() self._close_device_socket()
raise InitialConnectionError(_('Too many connection attempts from %s')%peer) raise InitialConnectionError(_('Too many connection attempts from %s') % peer)
else: else:
self.connection_attempts[peer] = attempts + 1 self.connection_attempts[peer] = attempts + 1
except InitialConnectionError: except InitialConnectionError:
@ -619,6 +665,8 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
self._close_device_socket() self._close_device_socket()
return False return False
self._debug('CC version #:', result.get('ccVersionNumber', 'unknown')) self._debug('CC version #:', result.get('ccVersionNumber', 'unknown'))
self.client_can_stream_books = result.get('canStreamBooks', False)
self._debug('CC can stream', self.client_can_stream_books);
self.max_book_packet_len = result.get('maxBookContentPacketLen', self.max_book_packet_len = result.get('maxBookContentPacketLen',
self.BASE_PACKET_LEN) self.BASE_PACKET_LEN)
exts = result.get('acceptedExtensions', None) exts = result.get('acceptedExtensions', None)
@ -725,12 +773,16 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
self._debug(oncard) self._debug(oncard)
if oncard is not None: if oncard is not None:
return CollectionsBookList(None, None, None) return CollectionsBookList(None, None, None)
opcode, result = self._call_client('GET_BOOK_COUNT', {}) opcode, result = self._call_client('GET_BOOK_COUNT', {'canStream':True})
bl = CollectionsBookList(None, self.PREFIX, self.settings) bl = CollectionsBookList(None, self.PREFIX, self.settings)
if opcode == 'OK': if opcode == 'OK':
count = result['count'] count = result['count']
will_stream = 'willStream' in result;
for i in range(0, count): for i in range(0, count):
opcode, result = self._call_client('GET_BOOK_METADATA', {'index': i}, if will_stream:
opcode, result = self._receive_from_client(print_debug_info=False)
else:
opcode, result = self._call_client('GET_BOOK_METADATA', {'index': i},
print_debug_info=False) print_debug_info=False)
if opcode == 'OK': if opcode == 'OK':
if '_series_sort_' in result: if '_series_sort_' in result:
@ -760,11 +812,10 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
# probably need to send two booklists, one with calibre's data that is # probably need to send two booklists, one with calibre's data that is
# given back by "books", and one that has been plugboarded. # given back by "books", and one that has been plugboarded.
self._call_client('SEND_BOOKLISTS', { 'count': len(booklists[0]), self._call_client('SEND_BOOKLISTS', { 'count': len(booklists[0]),
'collections': coldict} ) 'collections': coldict})
for i,book in enumerate(booklists[0]): for i,book in enumerate(booklists[0]):
if not self._metadata_already_on_device(book): if not self._metadata_already_on_device(book):
self._set_known_metadata(book) self._set_known_metadata(book)
self._debug('syncing book', book.lpath)
opcode, result = self._call_client('SEND_BOOK_METADATA', opcode, result = self._call_client('SEND_BOOK_METADATA',
{'index': i, 'data': book}, {'index': i, 'data': book},
print_debug_info=False) print_debug_info=False)
@ -802,19 +853,19 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
paths.append((lpath, length)) paths.append((lpath, length))
# No need to deal with covers. The client will get the thumbnails # No need to deal with covers. The client will get the thumbnails
# in the mi structure # in the mi structure
self.report_progress((i+1) / float(len(files)), _('Transferring books to device...')) self.report_progress((i + 1) / float(len(files)), _('Transferring books to device...'))
self.report_progress(1.0, _('Transferring books to device...')) self.report_progress(1.0, _('Transferring books to device...'))
self._debug('finished uploading %d books'%(len(files))) self._debug('finished uploading %d books' % (len(files)))
return paths return paths
@synchronous('sync_lock') @synchronous('sync_lock')
def add_books_to_metadata(self, locations, metadata, booklists): def add_books_to_metadata(self, locations, metadata, booklists):
self._debug('adding metadata for %d books'%(len(metadata))) self._debug('adding metadata for %d books' % (len(metadata)))
metadata = iter(metadata) metadata = iter(metadata)
for i, location in enumerate(locations): for i, location in enumerate(locations):
self.report_progress((i+1) / float(len(locations)), self.report_progress((i + 1) / float(len(locations)),
_('Adding books to device metadata listing...')) _('Adding books to device metadata listing...'))
info = metadata.next() info = metadata.next()
lpath = location[0] lpath = location[0]
@ -846,14 +897,14 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
self._debug(paths) self._debug(paths)
for i, path in enumerate(paths): for i, path in enumerate(paths):
path = self._strip_prefix(path) path = self._strip_prefix(path)
self.report_progress((i+1) / float(len(paths)), _('Removing books from device metadata listing...')) self.report_progress((i + 1) / float(len(paths)), _('Removing books from device metadata listing...'))
for bl in booklists: for bl in booklists:
for book in bl: for book in bl:
if path == book.path: if path == book.path:
bl.remove_book(book) bl.remove_book(book)
self._set_known_metadata(book, remove=True) self._set_known_metadata(book, remove=True)
self.report_progress(1.0, _('Removing books from device metadata listing...')) self.report_progress(1.0, _('Removing books from device metadata listing...'))
self._debug('finished removing metadata for %d books'%(len(paths))) self._debug('finished removing metadata for %d books' % (len(paths)))
@synchronous('sync_lock') @synchronous('sync_lock')
@ -864,7 +915,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
while not eof: while not eof:
opcode, result = self._call_client('GET_BOOK_FILE_SEGMENT', opcode, result = self._call_client('GET_BOOK_FILE_SEGMENT',
{'lpath' : path, 'position': position}, {'lpath' : path, 'position': position},
print_debug_info=False ) print_debug_info=False)
if opcode == 'OK': if opcode == 'OK':
if not result['eof']: if not result['eof']:
data = b64decode(result['data']) data = b64decode(result['data'])
@ -904,57 +955,71 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
self.max_book_packet_len = 0 self.max_book_packet_len = 0
self.noop_counter = 0 self.noop_counter = 0
self.connection_attempts = {} self.connection_attempts = {}
self.client_can_stream_books = False
message = None
try: try:
self.listen_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.listen_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
except: except:
self._debug('creation of listen socket failed') message = 'creation of listen socket failed'
return self._debug(message)
return message
i = 0 i = 0
while i < 100: # try up to 100 random port numbers
if self.settings().extra_customization[self.OPT_USE_PORT]: if self.settings().extra_customization[self.OPT_USE_PORT]:
i = 100
try:
port = int(self.settings().extra_customization[self.OPT_PORT_NUMBER])
except:
port = 0
else:
i += 1
port = random.randint(8192, 32000)
try: try:
self._debug('try port', port) opt_port = int(self.settings().extra_customization[self.OPT_PORT_NUMBER])
self.listen_socket.bind(('', port))
break
except socket.error:
port = 0
except: except:
self._debug('Unknown exception while allocating listen socket') message = _('Invalid port in options: %s')% \
traceback.print_exc() self.settings().extra_customization[self.OPT_PORT_NUMBER]
raise self.debug(message)
if port == 0: self.listen_socket.close()
self._debug('Failed to allocate a port'); self.listen_socket = None
self.listen_socket.close() self.is_connected = False
self.listen_socket = None return message
self.is_connected = False
return port = self._attach_to_port(opt_port)
if port == 0:
message = 'Failed to connect to port %d'%opt_port
self._debug(message);
self.listen_socket.close()
self.listen_socket = None
self.is_connected = False
return message
else:
while i < 100: # try up to 100 random port numbers
i += 1
port = self._attach_to_port(random.randint(8192, 32000))
if port != 0:
break;
if port == 0:
message = _('Failed to allocate a random port')
self._debug(message);
self.listen_socket.close()
self.listen_socket = None
self.is_connected = False
return message
try: try:
self.listen_socket.listen(0) self.listen_socket.listen(0)
except: except:
self._debug('listen on socket failed', port) message = 'listen on port %d failed' % port
self._debug(message)
self.listen_socket.close() self.listen_socket.close()
self.listen_socket = None self.listen_socket = None
self.is_connected = False self.is_connected = False
return return message
try: try:
do_zeroconf(publish_zeroconf, port) do_zeroconf(publish_zeroconf, port)
except: except:
self._debug('registration with bonjour failed') message = 'registration with bonjour failed'
self._debug(message)
self.listen_socket.close() self.listen_socket.close()
self.listen_socket = None self.listen_socket = None
self.is_connected = False self.is_connected = False
return return message
self._debug('listening on port', port) self._debug('listening on port', port)
self.port = port self.port = port
@ -975,7 +1040,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
@synchronous('sync_lock') @synchronous('sync_lock')
def start_plugin(self): def start_plugin(self):
self.startup_on_demand() return self.startup_on_demand()
@synchronous('sync_lock') @synchronous('sync_lock')
def stop_plugin(self): def stop_plugin(self):

View File

@ -238,11 +238,20 @@ class ConnectShareAction(InterfaceAction):
def set_smartdevice_action_state(self): def set_smartdevice_action_state(self):
from calibre.utils.mdns import get_external_ip from calibre.utils.mdns import get_external_ip
running = self.gui.device_manager.is_running('smartdevice') dm = self.gui.device_manager
running = dm.is_running('smartdevice')
if not running: if not running:
text = self.share_conn_menu.DEVICE_MSGS[0] text = self.share_conn_menu.DEVICE_MSGS[0]
else: else:
text = self.share_conn_menu.DEVICE_MSGS[1] + ' [%s]'%get_external_ip() use_fixed_port = dm.get_option('smartdevice', 'use_fixed_port')
port_number = dm.get_option('smartdevice', 'port_number')
if use_fixed_port:
text = self.share_conn_menu.DEVICE_MSGS[1] + ' [%s port %s]'%(
get_external_ip(), port_number)
else:
text = self.share_conn_menu.DEVICE_MSGS[1] + ' [%s]'%get_external_ip()
icon = 'green' if running else 'red' icon = 'green' if running else 'red'
ac = self.share_conn_menu.control_smartdevice_action ac = self.share_conn_menu.control_smartdevice_action
ac.setIcon(QIcon(I('dot_%s.png'%icon))) ac.setIcon(QIcon(I('dot_%s.png'%icon)))

View File

@ -554,7 +554,7 @@ class DeviceManager(Thread): # {{{
# will switch to the device thread before calling the plugin. # will switch to the device thread before calling the plugin.
def start_plugin(self, name): def start_plugin(self, name):
self._call_request(name, 'start_plugin') return self._call_request(name, 'start_plugin')
def stop_plugin(self, name): def stop_plugin(self, name):
self._call_request(name, 'stop_plugin') self._call_request(name, 'stop_plugin')

View File

@ -7,6 +7,7 @@ __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
from PyQt4.Qt import (QDialog, QLineEdit, Qt) from PyQt4.Qt import (QDialog, QLineEdit, Qt)
from calibre.gui2 import error_dialog
from calibre.gui2.dialogs.smartdevice_ui import Ui_Dialog from calibre.gui2.dialogs.smartdevice_ui import Ui_Dialog
class SmartdeviceDialog(QDialog, Ui_Dialog): class SmartdeviceDialog(QDialog, Ui_Dialog):
@ -27,7 +28,21 @@ class SmartdeviceDialog(QDialog, Ui_Dialog):
'smart device interface when calibre starts. You should not do ' 'smart device interface when calibre starts. You should not do '
'this if you are using a network that is not secure and you ' 'this if you are using a network that is not secure and you '
'are not setting a password.') + '</p>') 'are not setting a password.') + '</p>')
self.use_fixed_port.setToolTip('<p>' +
_('Check this box if you want calibre to use a fixed network '
'port. Normally you will not need to do this. However, if '
'your device consistently fails to connect to calibre, '
'try checking this box.') + '</p>')
self.fixed_port.setToolTip('<p>' +
_('A port number must be a 4-digit integer less than 32,000. No '
'two network applications on the same computer can use '
'the same port number. If calibre says that it fails to connect '
'to the port, try a different number.') + '</p>')
self.show_password.stateChanged[int].connect(self.toggle_password) self.show_password.stateChanged[int].connect(self.toggle_password)
self.use_fixed_port.stateChanged[int].connect(self.use_fixed_port_changed)
self.device_manager = parent.device_manager self.device_manager = parent.device_manager
@ -37,8 +52,22 @@ class SmartdeviceDialog(QDialog, Ui_Dialog):
pw = self.device_manager.get_option('smartdevice', 'password') pw = self.device_manager.get_option('smartdevice', 'password')
if pw: if pw:
self.password_box.setText(pw) self.password_box.setText(pw)
use_fixed_port = self.device_manager.get_option('smartdevice', 'use_fixed_port')
port_number = self.device_manager.get_option('smartdevice', 'port_number')
self.fixed_port.setText(port_number)
self.use_fixed_port.setChecked(use_fixed_port);
if not use_fixed_port:
self.fixed_port.setEnabled(False);
if pw:
self.password_box.setText(pw)
self.resize(self.sizeHint()) self.resize(self.sizeHint())
def use_fixed_port_changed(self, state):
self.fixed_port.setEnabled(state == Qt.Checked)
def toggle_password(self, state): def toggle_password(self, state):
self.password_box.setEchoMode(QLineEdit.Password if state == self.password_box.setEchoMode(QLineEdit.Password if state ==
Qt.Unchecked else QLineEdit.Normal) Qt.Unchecked else QLineEdit.Normal)
@ -48,6 +77,16 @@ class SmartdeviceDialog(QDialog, Ui_Dialog):
unicode(self.password_box.text())) unicode(self.password_box.text()))
self.device_manager.set_option('smartdevice', 'autostart', self.device_manager.set_option('smartdevice', 'autostart',
self.autostart_box.isChecked()) self.autostart_box.isChecked())
self.device_manager.start_plugin('smartdevice') self.device_manager.set_option('smartdevice', 'use_fixed_port',
QDialog.accept(self) self.use_fixed_port.isChecked())
self.device_manager.set_option('smartdevice', 'port_number',
unicode(self.fixed_port.text()))
message = self.device_manager.start_plugin('smartdevice')
if not self.device_manager.is_running('smartdevice'):
error_dialog(self, _('Problem starting smartdevice'),
_('The snart device driver did not start. It said "%s"')%message, show=True)
else:
QDialog.accept(self)

View File

@ -71,6 +71,36 @@
</property> </property>
</widget> </widget>
</item> </item>
<item row="2" column="0">
<widget class="QLabel" name="label_21">
<property name="text">
<string>Optional &amp;fixed port:</string>
</property>
<property name="buddy">
<cstring>fixed_port</cstring>
</property>
</widget>
</item>
<item row="2" column="1">
<widget class="QLineEdit" name="fixed_port">
<property name="sizePolicy">
<sizepolicy hsizetype="Expanding" vsizetype="Expanding">
<horstretch>100</horstretch>
<verstretch>0</verstretch>
</sizepolicy>
</property>
<property name="placeholderText">
<string>Optional port number</string>
</property>
</widget>
</item>
<item row="2" column="2">
<widget class="QCheckBox" name="use_fixed_port">
<property name="text">
<string>&amp;Use a fixed port</string>
</property>
</widget>
</item>
<item row="4" column="0" colspan="3"> <item row="4" column="0" colspan="3">
<widget class="QDialogButtonBox" name="buttonBox"> <widget class="QDialogButtonBox" name="buttonBox">
<property name="orientation"> <property name="orientation">