Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-07 18:24:30 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' of https://github.com/cbhaley/calibre

Browse Source
This commit is contained in:
Kovid Goyal 2013-09-19 18:36:00 +05:30
commit 2e99bf845e
1 changed files with 90 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

109
src/calibre/devices/smart_device_app/driver.py
View File

@ -7,7 +7,7 @@ Created on 29 Jun 2012
@author: charles @author: charles
''' '''
import socket, select, json, os, traceback, time, sys, random import socket, select, json, os, traceback, time, sys, random, cPickle
import posixpath import posixpath
import hashlib, threading import hashlib, threading
import Queue import Queue
@ -34,7 +34,8 @@ from calibre.library import current_library_name
from calibre.library.server import server_config as content_server_config from calibre.library.server import server_config as content_server_config
from calibre.ptempfile import PersistentTemporaryFile from calibre.ptempfile import PersistentTemporaryFile
from calibre.utils.ipc import eintr_retry_call from calibre.utils.ipc import eintr_retry_call
from calibre.utils.config_base import tweaks from calibre.utils.config_base import config_dir, tweaks
from calibre.utils.date import parse_date
from calibre.utils.filenames import ascii_filename as sanitize, shorten_components_to from calibre.utils.filenames import ascii_filename as sanitize, shorten_components_to
from calibre.utils.mdns import (publish as publish_zeroconf, unpublish as from calibre.utils.mdns import (publish as publish_zeroconf, unpublish as
unpublish_zeroconf, get_all_ips) unpublish_zeroconf, get_all_ips)
@ -108,7 +109,7 @@ class ConnectionListener(Thread):
try: try:
packet = self.driver.broadcast_socket.recvfrom(100) packet = self.driver.broadcast_socket.recvfrom(100)
remote = packet[1] remote = packet[1]
content_server_port = b''; content_server_port = b''
try : try :
content_server_port = \ content_server_port = \
str(content_server_config().parse().port) str(content_server_config().parse().port)
@ -214,11 +215,11 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
# Some network protocol constants # Some network protocol constants
BASE_PACKET_LEN = 4096 BASE_PACKET_LEN = 4096
PROTOCOL_VERSION = 1 PROTOCOL_VERSION = 1
MAX_CLIENT_COMM_TIMEOUT = 300.0 # Wait at most N seconds for an answer MAX_CLIENT_COMM_TIMEOUT = 300.0 # Wait at most N seconds for an answer
MAX_UNSUCCESSFUL_CONNECTS = 5 MAX_UNSUCCESSFUL_CONNECTS = 5
SEND_NOOP_EVERY_NTH_PROBE = 5 SEND_NOOP_EVERY_NTH_PROBE = 5
DISCONNECT_AFTER_N_SECONDS = 30*60 # 30 minutes DISCONNECT_AFTER_N_SECONDS = 30*60 # 30 minutes
ZEROCONF_CLIENT_STRING = b'calibre wireless device client' ZEROCONF_CLIENT_STRING = b'calibre wireless device client'
@ -330,7 +331,6 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
OPT_OVERWRITE_BOOKS_UUID = 12 OPT_OVERWRITE_BOOKS_UUID = 12
OPT_COMPRESSION_QUALITY = 13 OPT_COMPRESSION_QUALITY = 13
def __init__(self, path): def __init__(self, path):
self.sync_lock = threading.RLock() self.sync_lock = threading.RLock()
self.noop_counter = 0 self.noop_counter = 0
@ -457,7 +457,8 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
else: else:
for c in tag.split('/'): for c in tag.split('/'):
c = sanitize(c) c = sanitize(c)
if not c: continue if not c:
continue
extra_components.append(c) extra_components.append(c)
extra_components.append(name) extra_components.append(name)
@ -521,7 +522,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
# Things get trashed if we don't make a copy of the data. # Things get trashed if we don't make a copy of the data.
v = self._read_binary_from_net(2) v = self._read_binary_from_net(2)
if len(v) == 0: if len(v) == 0:
return '' # documentation says the socket is broken permanently. return '' # documentation says the socket is broken permanently.
data += v data += v
total_len = int(data[:dex]) total_len = int(data[:dex])
data = data[dex:] data = data[dex:]
@ -529,7 +530,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
while pos < total_len: while pos < total_len:
v = self._read_binary_from_net(total_len - pos) v = self._read_binary_from_net(total_len - pos)
if len(v) == 0: if len(v) == 0:
return '' # documentation says the socket is broken permanently. return '' # documentation says the socket is broken permanently.
data += v data += v
pos += len(v) pos += len(v)
return data return data
@ -553,7 +554,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
self._debug('socket error', e, e.errno) self._debug('socket error', e, e.errno)
if e.args[0] != EAGAIN and e.args[0] != EINTR: if e.args[0] != EAGAIN and e.args[0] != EINTR:
raise raise
time.sleep(0.1) # lets not hammer the OS too hard time.sleep(0.1) # lets not hammer the OS too hard
def _call_client(self, op, arg, print_debug_info=True, wait_for_response=True): def _call_client(self, op, arg, print_debug_info=True, wait_for_response=True):
if op != 'NOOP': if op != 'NOOP':
@ -601,7 +602,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
if v: if v:
v = json.loads(v, object_hook=from_json) v = json.loads(v, object_hook=from_json)
if print_debug_info and extra_debug: if print_debug_info and extra_debug:
self._debug('receive after decode') #, v) self._debug('receive after decode') # , v)
return (self.reverse_opcodes[v[0]], v[1]) return (self.reverse_opcodes[v[0]], v[1])
self._debug('protocol error -- empty json string') self._debug('protocol error -- empty json string')
except socket.timeout: except socket.timeout:
@ -680,6 +681,18 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
else: else:
return None return None
def _metadata_in_cache(self, uuid, ext, lastmod):
key = uuid+ext
if isinstance(lastmod, unicode):
lastmod = parse_date(lastmod)
# if key in self.known_uuids:
# self._debug(key, lastmod, self.known_uuids[key].last_modified)
# else:
# self._debug(key, 'not in known uuids')
if key in self.known_uuids and self.known_uuids[key].last_modified == lastmod:
return self.known_uuids[key].deepcopy()
return None
def _metadata_already_on_device(self, book): def _metadata_already_on_device(self, book):
v = self.known_metadata.get(book.lpath, None) v = self.known_metadata.get(book.lpath, None)
if v is not None: if v is not None:
@ -701,6 +714,32 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
except: except:
return None return None
def _read_metadata_cache(self):
cache_file_name = os.path.join(config_dir,
'device_drivers_' + self.__class__.__name__ +
'_metadata_cache.pickle')
if os.path.exists(cache_file_name):
with open(cache_file_name, mode='rb') as fd:
json_metadata = cPickle.load(fd)
for uuid,json_book in json_metadata.iteritems():
book = self.json_codec.raw_to_book(json_book, SDBook, self.PREFIX)
self.known_uuids[uuid] = book
lpath = book.get('lpath')
if lpath in self.known_metadata:
self.known_uuids.pop(uuid, None)
else:
self.known_metadata[lpath] = book
def _write_metadata_cache(self):
cache_file_name = os.path.join(config_dir,
'device_drivers_' + self.__class__.__name__ +
'_metadata_cache.pickle')
json_metadata = {}
for uuid,book in self.known_uuids.iteritems():
json_metadata[uuid] = self.json_codec.encode_book_metadata(book)
with open(cache_file_name, mode='wb') as fd:
cPickle.dump(json_metadata, fd, -1)
def _set_known_metadata(self, book, remove=False): def _set_known_metadata(self, book, remove=False):
lpath = book.lpath lpath = book.lpath
ext = os.path.splitext(lpath)[1] ext = os.path.splitext(lpath)[1]
@ -721,6 +760,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
except: except:
pass pass
self.device_socket = None self.device_socket = None
self._write_metadata_cache()
self.is_connected = False self.is_connected = False
def _attach_to_port(self, sock, port): def _attach_to_port(self, sock, port):
@ -868,6 +908,8 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
self._debug('Device can receive book binary', self.client_can_stream_metadata) self._debug('Device can receive book binary', self.client_can_stream_metadata)
self.client_can_delete_multiple = result.get('canDeleteMultipleBooks', False) self.client_can_delete_multiple = result.get('canDeleteMultipleBooks', False)
self._debug('Device can delete multiple books', self.client_can_delete_multiple) self._debug('Device can delete multiple books', self.client_can_delete_multiple)
self.client_can_use_metadata_cache = result.get('canUseCachedMetadata', False)
self._debug('Device can use cached metadata', self.client_can_use_metadata_cache)
self.client_device_kind = result.get('deviceKind', '') self.client_device_kind = result.get('deviceKind', '')
self._debug('Client device kind', self.client_device_kind) self._debug('Client device kind', self.client_device_kind)
@ -875,6 +917,9 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
self.client_device_name = result.get('deviceName', self.client_device_kind) self.client_device_name = result.get('deviceName', self.client_device_kind)
self._debug('Client device name', self.client_device_name) self._debug('Client device name', self.client_device_name)
self.client_app_name = result.get('appName', "")
self._debug('Client app name', self.client_app_name)
self.max_book_packet_len = result.get('maxBookContentPacketLen', self.max_book_packet_len = result.get('maxBookContentPacketLen',
self.BASE_PACKET_LEN) self.BASE_PACKET_LEN)
self._debug('max_book_packet_len', self.max_book_packet_len) self._debug('max_book_packet_len', self.max_book_packet_len)
@ -888,7 +933,6 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
self.client_wants_uuid_file_names = result.get('useUuidFileNames', False) self.client_wants_uuid_file_names = result.get('useUuidFileNames', False)
self._debug('Device wants UUID file names', self.client_wants_uuid_file_names) self._debug('Device wants UUID file names', self.client_wants_uuid_file_names)
config = self._configProxy() config = self._configProxy()
config['format_map'] = exts config['format_map'] = exts
self._debug('selected formats', config['format_map']) self._debug('selected formats', config['format_map'])
@ -933,8 +977,6 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
except: except:
pass pass
self.known_metadata = {}
self.known_uuids = {}
return True return True
except socket.timeout: except socket.timeout:
self._close_device_socket() self._close_device_socket()
@ -1019,13 +1061,41 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
self._debug(oncard) self._debug(oncard)
if oncard is not None: if oncard is not None:
return CollectionsBookList(None, None, None) return CollectionsBookList(None, None, None)
opcode, result = self._call_client('GET_BOOK_COUNT', {'canStream':True, opcode, result = self._call_client('GET_BOOK_COUNT',
'canScan':True}) {'canStream':True,
'canScan':True,
'willUseCachedMetadata': self.client_can_use_metadata_cache})
bl = CollectionsBookList(None, self.PREFIX, self.settings) bl = CollectionsBookList(None, self.PREFIX, self.settings)
if opcode == 'OK': if opcode == 'OK':
count = result['count'] count = result['count']
will_stream = 'willStream' in result will_stream = 'willStream' in result
will_scan = 'willScan' in result will_scan = 'willScan' in result
will_use_cache = self.client_can_use_metadata_cache
if will_use_cache:
books_on_device = {}
self._debug('caching. count=', count)
for i in range(0, count):
opcode, result = self._receive_from_client(print_debug_info=False)
books_on_device[result.get('uuid')] = result
books_to_send = []
for u,v in books_on_device.iteritems():
book = self._metadata_in_cache(u, v['extension'], v['last_modified'])
if book:
bl.add_book(book, replace_metadata=True)
else:
books_to_send.append(v['priKey'])
count = len(books_to_send)
self._debug('caching. Need count from device', count)
self._call_client('NOOP', {'count': count},
print_debug_info=False, wait_for_response=False)
for priKey in books_to_send:
self._call_client('NOOP', {'priKey':priKey},
print_debug_info=False, wait_for_response=False)
for i in range(0, count): for i in range(0, count):
if (i % 100) == 0: if (i % 100) == 0:
self._debug('getting book metadata. Done', i, 'of', count) self._debug('getting book metadata. Done', i, 'of', count)
@ -1086,7 +1156,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
books_to_send.append(book) books_to_send.append(book)
count = len(books_to_send) count = len(books_to_send)
self._call_client('SEND_BOOKLISTS', { 'count': count, self._call_client('SEND_BOOKLISTS', {'count': count,
'collections': coldict, 'collections': coldict,
'willStreamMetadata': self.client_can_stream_metadata}, 'willStreamMetadata': self.client_can_stream_metadata},
wait_for_response=not self.client_can_stream_metadata) wait_for_response=not self.client_can_stream_metadata)
@ -1209,7 +1279,6 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
self.report_progress(1.0, _('Removing books from device metadata listing...')) self.report_progress(1.0, _('Removing books from device metadata listing...'))
self._debug('finished removing metadata for %d books' % (len(paths))) self._debug('finished removing metadata for %d books' % (len(paths)))
@synchronous('sync_lock') @synchronous('sync_lock')
def get_file(self, path, outfile, end_session=True, this_book=None, total_books=None): def get_file(self, path, outfile, end_session=True, this_book=None, total_books=None):
if self.settings().extra_customization[self.OPT_EXTRA_DEBUG]: if self.settings().extra_customization[self.OPT_EXTRA_DEBUG]:
@ -1340,7 +1409,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
self._close_listen_socket() self._close_listen_socket()
return message return message
else: else:
while i < 100: # try 9090 then up to 99 random port numbers while i < 100: # try 9090 then up to 99 random port numbers
i += 1 i += 1
port = self._attach_to_port(self.listen_socket, port = self._attach_to_port(self.listen_socket,
9090 if i == 1 else random.randint(8192, 32000)) 9090 if i == 1 else random.randint(8192, 32000))
@ -1397,6 +1466,8 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
self.connection_queue = Queue.Queue(1) self.connection_queue = Queue.Queue(1)
self.connection_listener = ConnectionListener(self) self.connection_listener = ConnectionListener(self)
self.connection_listener.start() self.connection_listener.start()
self._read_metadata_cache()
return message return message
@synchronous('sync_lock') @synchronous('sync_lock')