Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-09 03:04:10 -04:00
Code Issues Packages Projects Releases Wiki Activity

Template language: Add a couple of new functions to get the path to individual book formats and the calibre library as a whole

Browse Source
This commit is contained in:
Kovid Goyal 2012-09-18 22:56:45 +05:30
commit 5821ac063a
6 changed files with 296 additions and 85 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

312
src/calibre/devices/smart_device_app/driver.py
View File

@ -10,9 +10,12 @@ Created on 29 Jun 2012
import socket, select, json, inspect, os, traceback, time, sys, random
import posixpath
import hashlib, threading
import Queue
from base64 import b64encode, b64decode
from functools import wraps
from errno import EAGAIN, EINTR
from threading import Thread
from calibre import prints
from calibre.constants import numeric_version, DEBUG
@ -48,6 +51,110 @@ def synchronous(tlockname):
return _synched
class ConnectionListener (Thread):
def __init__(self, driver):
Thread.__init__(self)
self.daemon = True
self.driver = driver
self.keep_running = True
def stop(self):
self.keep_running = False
def run(self):
queue_not_serviced_count = 0
device_socket = None
while self.keep_running:
try:
time.sleep(1)
except:
# Happens during interpreter shutdown
break
if not self.keep_running:
break
if not self.driver.connection_queue.empty():
queue_not_serviced_count += 1
if queue_not_serviced_count >= 3:
self.driver._debug('queue not serviced')
try:
sock = self.driver.connection_queue.get_nowait()
s = self.driver._json_encode(
self.driver.opcodes['CALIBRE_BUSY'], {})
self.driver._send_byte_string(device_socket, (b'%d' % len(s)) + s)
sock.close()
except Queue.Empty:
pass
queue_not_serviced_count = 0
else:
queue_not_serviced_count = 0
if getattr(self.driver, 'broadcast_socket', None) is not None:
while True:
ans = select.select((self.driver.broadcast_socket,), (), (), 0)
if len(ans[0]) > 0:
try:
packet = self.driver.broadcast_socket.recvfrom(100)
remote = packet[1]
message = str(self.driver.ZEROCONF_CLIENT_STRING + b' (on ' +
str(socket.gethostname().partition('.')[0]) +
b'),' + str(self.driver.port))
self.driver._debug('received broadcast', packet, message)
self.driver.broadcast_socket.sendto(message, remote)
except:
pass
else:
break
if self.driver.connection_queue.empty() and \
getattr(self.driver, 'listen_socket', None) is not None:
ans = select.select((self.driver.listen_socket,), (), (), 0)
if len(ans[0]) > 0:
# timeout in 10 ms to detect rare case where the socket went
# way between the select and the accept
try:
self.driver._debug('attempt to open device socket')
device_socket = None
self.driver.listen_socket.settimeout(0.010)
device_socket, ign = eintr_retry_call(
self.driver.listen_socket.accept)
self.driver.listen_socket.settimeout(None)
device_socket.settimeout(None)
try:
peer = self.driver.device_socket.getpeername()[0]
attempts = self.drjver.connection_attempts.get(peer, 0)
if attempts >= self.MAX_UNSUCCESSFUL_CONNECTS:
self.driver._debug('too many connection attempts from', peer)
device_socket.close()
device_socket = None
# raise InitialConnectionError(_('Too many connection attempts from %s') % peer)
else:
self.driver.connection_attempts[peer] = attempts + 1
except InitialConnectionError:
raise
except:
pass
try:
self.driver.connection_queue.put_nowait(device_socket)
except Queue.Full:
device_socket.close()
device_socket = None
self.driver._debug('driver is not answering')
except socket.timeout:
pass
except socket.error:
x = sys.exc_info()[1]
self.driver._debug('unexpected socket exception', x.args[0])
device_socket.close()
device_socket = None
# raise
class SDBook(Book):
def __init__(self, prefix, lpath, size=None, other=None):
Book.__init__(self, prefix, lpath, size=size, other=other)
@ -80,8 +187,20 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
NEWS_IN_FOLDER = True
SUPPORTS_USE_AUTHOR_SORT = False
WANTS_UPDATED_THUMBNAILS = True
# Guess about the max length on windows. This number will be reduced by
# the length of the path on the client, and by the fudge factor below. We
# use this on all platforms because the device might be connected to windows
# in the future.
MAX_PATH_LEN = 250
# guess of length of MTP name. The length of the full path to the folder
# on the device is added to this. That path includes the device's mount point
# making this number effectively around 10 to 15 larger.
PATH_FUDGE_FACTOR = 40
THUMBNAIL_HEIGHT = 160
DEFAULT_THUMBNAIL_HEIGHT = 160
PREFIX = ''
BACKLOADING_ERROR_MESSAGE = None
@ -112,6 +231,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
'OK' : 0,
'BOOK_DATA' : 10,
'BOOK_DONE' : 11,
'CALIBRE_BUSY' : 18,
'DELETE_BOOK' : 13,
'DISPLAY_MESSAGE' : 17,
'FREE_SPACE' : 5,
@ -248,7 +368,11 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
# remove the 'path' argument and all its uses. Also removed the calls to
# filename_callback and sanitize_path_components
def _create_upload_path(self, mdata, fname, create_dirs=True):
maxlen = self.MAX_PATH_LEN
fname = sanitize(fname)
ext = os.path.splitext(fname)[1]
maxlen = (self.MAX_PATH_LEN - (self.PATH_FUDGE_FACTOR +
self.exts_path_lengths.get(ext, self.PATH_FUDGE_FACTOR)))
special_tag = None
if mdata.tags:
@ -269,9 +393,6 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
template = "{title}_%d-%d-%d" % date
use_subdirs = self.SUPPORTS_SUB_DIRS and settings.use_subdirs
fname = sanitize(fname)
ext = os.path.splitext(fname)[1]
from calibre.library.save_to_disk import get_components
from calibre.library.save_to_disk import config
opts = config().parse()
@ -346,6 +467,13 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
return json.dumps([op, res], encoding='utf-8')
# Network functions
def _read_binary_from_net(self, length):
self.device_socket.settimeout(self.MAX_CLIENT_COMM_TIMEOUT)
v = self.device_socket.recv(length)
self.device_socket.settimeout(None)
return v
def _read_string_from_net(self):
data = bytes(0)
while True:
@ -354,9 +482,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
break
# recv seems to return a pointer into some internal buffer.
# Things get trashed if we don't make a copy of the data.
self.device_socket.settimeout(self.MAX_CLIENT_COMM_TIMEOUT)
v = self.device_socket.recv(2)
self.device_socket.settimeout(None)
v = self._read_binary_from_net(2)
if len(v) == 0:
return '' # documentation says the socket is broken permanently.
data += v
@ -364,16 +490,14 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
data = data[dex:]
pos = len(data)
while pos < total_len:
self.device_socket.settimeout(self.MAX_CLIENT_COMM_TIMEOUT)
v = self.device_socket.recv(total_len - pos)
self.device_socket.settimeout(None)
v = self._read_binary_from_net(total_len - pos)
if len(v) == 0:
return '' # documentation says the socket is broken permanently.
data += v
pos += len(v)
return data
def _send_byte_string(self, s):
def _send_byte_string(self, sock, s):
if not isinstance(s, bytes):
self._debug('given a non-byte string!')
raise PacketError("Internal error: found a string that isn't bytes")
@ -382,11 +506,11 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
while sent_len < total_len:
try:
if sent_len == 0:
amt_sent = self.device_socket.send(s)
amt_sent = sock.send(s)
else:
amt_sent = self.device_socket.send(s[sent_len:])
amt_sent = sock.send(s[sent_len:])
if amt_sent <= 0:
raise IOError('Bad write on device socket')
raise IOError('Bad write on socket')
sent_len += amt_sent
except socket.error as e:
self._debug('socket error', e, e.errno)
@ -410,7 +534,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
if print_debug_info and extra_debug:
self._debug('send string', s)
self.device_socket.settimeout(self.MAX_CLIENT_COMM_TIMEOUT)
self._send_byte_string((b'%d' % len(s)) + s)
self._send_byte_string(self.device_socket, (b'%d' % len(s)) + s)
if not wait_for_response:
return None, None
return self._receive_from_client(print_debug_info=print_debug_info)
@ -466,11 +590,12 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
length = infile.tell()
book_metadata.size = length
infile.seek(0)
self._debug(lpath, length)
opcode, result = self._call_client('SEND_BOOK', {'lpath': lpath, 'length': length,
'metadata': book_metadata, 'thisBook': this_book,
'totalBooks': total_books,
'willStreamBooks': self.client_can_stream_books},
'willStreamBooks': self.client_can_stream_books,
'willStreamBinary' : self.client_can_receive_book_binary},
print_debug_info=False,
wait_for_response=(not self.client_can_stream_books))
@ -483,17 +608,21 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
blen = len(b)
if not b:
break
b = b64encode(b)
opcode, result = self._call_client('BOOK_DATA',
{'lpath': lpath, 'position': pos, 'data': b},
print_debug_info=False,
wait_for_response=(not self.client_can_stream_books))
if self.client_can_stream_books and self.client_can_receive_book_binary:
self._send_byte_string(self.device_socket, b)
else:
b = b64encode(b)
opcode, result = self._call_client('BOOK_DATA',
{'lpath': lpath, 'position': pos, 'data': b},
print_debug_info=False,
wait_for_response=(not self.client_can_stream_books))
pos += blen
if not self.client_can_stream_books and opcode != 'OK':
self._debug('protocol error', opcode)
failed = True
break
self._call_client('BOOK_DONE', {'lpath': lpath})
if not (self.client_can_stream_books and self.client_can_receive_book_binary):
self._call_client('BOOK_DONE', {'lpath': lpath})
self.time = None
if close_:
infile.close()
@ -517,7 +646,8 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
v = self.known_metadata.get(book.lpath, None)
if v is not None:
return (v.get('uuid', None) == book.get('uuid', None) and
v.get('last_modified', None) == book.get('last_modified', None))
v.get('last_modified', None) == book.get('last_modified', None) and
v.get('thumbnail', None) == book.get('thumbnail', None))
return False
def _set_known_metadata(self, book, remove=False):
@ -620,39 +750,26 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
break
if getattr(self, 'listen_socket', None) is not None:
ans = select.select((self.listen_socket,), (), (), 0)
if len(ans[0]) > 0:
# timeout in 10 ms to detect rare case where the socket went
# way between the select and the accept
try:
ans = self.connection_queue.get_nowait()
self.device_socket = ans
self.is_connected = True
try:
self.device_socket = None
self.listen_socket.settimeout(0.010)
self.device_socket, ign = eintr_retry_call(
self.listen_socket.accept)
self.listen_socket.settimeout(None)
self.device_socket.settimeout(None)
self.is_connected = True
try:
peer = self.device_socket.getpeername()[0]
attempts = self.connection_attempts.get(peer, 0)
if attempts >= self.MAX_UNSUCCESSFUL_CONNECTS:
self._debug('too many connection attempts from', peer)
self._close_device_socket()
raise InitialConnectionError(_('Too many connection attempts from %s') % peer)
else:
self.connection_attempts[peer] = attempts + 1
except InitialConnectionError:
raise
except:
pass
except socket.timeout:
self._close_device_socket()
except socket.error:
x = sys.exc_info()[1]
self._debug('unexpected socket exception', x.args[0])
self._close_device_socket()
peer = self.device_socket.getpeername()[0]
attempts = self.connection_attempts.get(peer, 0)
if attempts >= self.MAX_UNSUCCESSFUL_CONNECTS:
self._debug('too many connection attempts from', peer)
self._close_device_socket()
raise InitialConnectionError(_('Too many connection attempts from %s') % peer)
else:
self.connection_attempts[peer] = attempts + 1
except InitialConnectionError:
raise
return (self.is_connected, self)
except:
pass
except Queue.Empty:
self.is_connected = False
return (self.is_connected, self)
return (False, None)
@synchronous('sync_lock')
@ -705,17 +822,34 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
self._debug('Device can stream books', self.client_can_stream_books)
self.client_can_stream_metadata = result.get('canStreamMetadata', False)
self._debug('Device can stream metadata', self.client_can_stream_metadata)
self.client_can_receive_book_binary = result.get('canReceiveBookBinary', False)
self._debug('Device can receive book binary', self.client_can_stream_metadata)
self.max_book_packet_len = result.get('maxBookContentPacketLen',
self.BASE_PACKET_LEN)
self._debug('max_book_packet_len', self.max_book_packet_len)
exts = result.get('acceptedExtensions', None)
if exts is None or not isinstance(exts, list) or len(exts) == 0:
self._debug('Protocol error - bogus accepted extensions')
self._close_device_socket()
return False
config = self._configProxy()
config['format_map'] = exts
self._debug('selected formats', config['format_map'])
self.exts_path_lengths = result.get('extensionPathLengths', {})
self._debug('extension path lengths', self.exts_path_lengths)
self.THUMBNAIL_HEIGHT = result.get('coverHeight', self.DEFAULT_THUMBNAIL_HEIGHT)
if 'coverWidth' in result:
# Setting this field forces the aspect ratio
self.THUMBNAIL_WIDTH = result.get('coverWidth',
(self.DEFAULT_THUMBNAIL_HEIGHT/3) * 4)
elif hasattr(self, 'THUMBNAIL_WIDTH'):
delattr(self, 'THUMBNAIL_WIDTH')
if password:
returned_hash = result.get('passwordHash', None)
if result.get('passwordHash', None) is None:
@ -909,7 +1043,10 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
@synchronous('sync_lock')
def upload_books(self, files, names, on_card=None, end_session=True,
metadata=None):
self._debug(names)
if self.settings().extra_customization[self.OPT_EXTRA_DEBUG]:
self._debug(names)
else:
self._debug()
paths = []
names = iter(names)
@ -956,7 +1093,11 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
@synchronous('sync_lock')
def delete_books(self, paths, end_session=True):
self._debug(paths)
if self.settings().extra_customization[self.OPT_EXTRA_DEBUG]:
self._debug(paths)
else:
self._debug()
for path in paths:
# the path has the prefix on it (I think)
path = self._strip_prefix(path)
@ -968,7 +1109,11 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
@synchronous('sync_lock')
def remove_books_from_metadata(self, paths, booklists):
self._debug(paths)
if self.settings().extra_customization[self.OPT_EXTRA_DEBUG]:
self._debug(paths)
else:
self._debug()
for i, path in enumerate(paths):
path = self._strip_prefix(path)
self.report_progress((i + 1) / float(len(paths)), _('Removing books from device metadata listing...'))
@ -983,29 +1128,45 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
@synchronous('sync_lock')
def get_file(self, path, outfile, end_session=True, this_book=None, total_books=None):
self._debug(path)
if self.settings().extra_customization[self.OPT_EXTRA_DEBUG]:
self._debug(path)
else:
self._debug()
eof = False
position = 0
while not eof:
opcode, result = self._call_client('GET_BOOK_FILE_SEGMENT',
{'lpath' : path, 'position': position,
'thisBook': this_book, 'totalBooks': total_books,
'canStream':True},
'canStream':True, 'canStreamBinary': True},
print_debug_info=False)
if opcode == 'OK':
client_will_stream = 'willStream' in result
while not eof:
if not result['eof']:
data = b64decode(result['data'])
if len(data) != result['next_position'] - position:
self._debug('position mismatch', result['next_position'], position)
position = result['next_position']
outfile.write(data)
opcode, result = self._receive_from_client(print_debug_info=True)
else:
eof = True
if not client_will_stream:
break
client_will_stream_binary = 'willStreamBinary' in result
if (client_will_stream_binary):
length = result.get('fileLength')
remaining = length
while remaining > 0:
v = self._read_binary_from_net(min(remaining, self.max_book_packet_len))
outfile.write(v)
remaining -= len(v)
eof = True
else:
while not eof:
if not result['eof']:
data = b64decode(result['data'])
if len(data) != result['next_position'] - position:
self._debug('position mismatch', result['next_position'], position)
position = result['next_position']
outfile.write(data)
opcode, result = self._receive_from_client(print_debug_info=True)
else:
eof = True
if not client_will_stream:
break
else:
raise ControlError(desc='request for book data failed')
@ -1127,17 +1288,22 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
self._debug('broadcast socket listening on port', port)
break
message = None
if port == 0:
self.broadcast_socket.close()
self.broadcast_socket = None
message = 'attaching port to broadcast socket failed. This is not fatal.'
self._debug(message)
return message
self.connection_queue = Queue.Queue(1)
self.connection_listener = ConnectionListener(self)
self.connection_listener.start()
return message
@synchronous('sync_lock')
def shutdown(self):
if getattr(self, 'listen_socket', None) is not None:
self.connection_listener.stop()
unpublish_zeroconf('calibre smart device client',
'_calibresmartdeviceapp._tcp', self.port, {})
self._close_listen_socket()

9
src/calibre/gui2/device.py
View File

@ -1673,11 +1673,10 @@ class DeviceMixin(object): # {{{
if update_metadata:
mi = db.get_metadata(id_, index_is_id=True,
get_cover=get_covers)
if book.get('last_modified', None) != mi.last_modified:
book.smart_update(db.get_metadata(id_,
index_is_id=True,
get_cover=get_covers),
replace_metadata=True)
book.smart_update(db.get_metadata(id_,
index_is_id=True,
get_cover=get_covers),
replace_metadata=True)
book.in_library = 'UUID'
# ensure that the correct application_id is set
book.application_id = id_

9
src/calibre/library/__init__.py
View File

@ -61,13 +61,18 @@ def generate_test_db(library_path, # {{{
print 'Time per record:', t/float(num_of_records)
# }}}
def current_library_name():
def current_library_path():
from calibre.utils.config import prefs
import posixpath
path = prefs['library_path']
if path:
path = path.replace('\\', '/')
while path.endswith('/'):
path = path[:-1]
return path
def current_library_name():
import posixpath
path = current_library_path()
if path:
return posixpath.basename(path)

6
src/calibre/library/database2.py
View File

@ -1262,6 +1262,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
ans = {}
if path is not None:
stat = os.stat(path)
ans['path'] = path
ans['size'] = stat.st_size
ans['mtime'] = utcfromtimestamp(stat.st_mtime)
self.format_metadata_cache[id_][fmt] = ans
@ -2563,6 +2564,11 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
if notify:
self.notify('metadata', [id])
def get_id_from_uuid(self, uuid):
if uuid:
return self.conn.get('SELECT id FROM books WHERE uuid=?', (uuid,),
all=False)
# Convenience methods for tags_list_editor
# Note: we generally do not need to refresh_ids because library_view will
# refresh everything.

14
src/calibre/library/server/ajax.py
View File

@ -181,7 +181,7 @@ class AjaxServer(object):
return data, mi.last_modified
@Endpoint(set_last_modified=False)
def ajax_book(self, book_id, category_urls='true'):
def ajax_book(self, book_id, category_urls='true', id_is_uuid='false'):
'''
Return the metadata of the book as a JSON dictionary.
@ -192,7 +192,10 @@ class AjaxServer(object):
cherrypy.response.timeout = 3600
try:
book_id = int(book_id)
if id_is_uuid == 'true':
book_id = self.db.get_id_from_uuid(book_id)
else:
book_id = int(book_id)
data, last_modified = self.ajax_book_to_json(book_id,
get_category_urls=category_urls.lower()=='true')
except:
@ -204,7 +207,7 @@ class AjaxServer(object):
return data
@Endpoint(set_last_modified=False)
def ajax_books(self, ids=None, category_urls='true'):
def ajax_books(self, ids=None, category_urls='true', id_is_uuid='false'):
'''
Return the metadata for a list of books specified as a comma separated
list of ids. The metadata is returned as a dictionary mapping ids to
@ -218,7 +221,10 @@ class AjaxServer(object):
if ids is None:
raise cherrypy.HTTPError(404, 'Must specify some ids')
try:
ids = set(int(x.strip()) for x in ids.split(','))
if id_is_uuid == 'true':
ids = set(self.db.get_id_from_uuid(x) for x in ids.split(','))
else:
ids = set(int(x.strip()) for x in ids.split(','))
except:
raise cherrypy.HTTPError(404, 'ids must be a comma separated list'
' of integers')

31
src/calibre/utils/formatter_functions.py
View File

@ -629,6 +629,22 @@ class BuiltinFormatsSizes(BuiltinFormatterFunction):
fmt_data = mi.get('format_metadata', {})
return ','.join(k.upper()+':'+str(v['size']) for k,v in fmt_data.iteritems())
class BuiltinFormatsPaths(BuiltinFormatterFunction):
name = 'formats_paths'
arg_count = 0
category = 'Get values from metadata'
__doc__ = doc = _('formats_paths() -- return a comma-separated list of '
'colon_separated items representing full path to '
'the formats of a book. You can use the select '
'function to get the path for a specific '
'format. Note that format names are always uppercase, '
'as in EPUB.'
)
def evaluate(self, formatter, kwargs, mi, locals):
fmt_data = mi.get('format_metadata', {})
return ','.join(k.upper()+':'+str(v['path']) for k,v in fmt_data.iteritems())
class BuiltinHumanReadable(BuiltinFormatterFunction):
name = 'human_readable'
arg_count = 1
@ -1146,6 +1162,18 @@ class BuiltinCurrentLibraryName(BuiltinFormatterFunction):
from calibre.library import current_library_name
return current_library_name()
class BuiltinCurrentLibraryPath(BuiltinFormatterFunction):
name = 'current_library_path'
arg_count = 0
category = 'Get values from metadata'
__doc__ = doc = _('current_library_path() -- '
'return the path to the current calibre library. This function can '
'be called in template program mode using the template '
'"{:\'current_library_path()\'}".')
def evaluate(self, formatter, kwargs, mi, locals):
from calibre.library import current_library_path
return current_library_path()
class BuiltinFinishFormatting(BuiltinFormatterFunction):
name = 'finish_formatting'
arg_count = 4
@ -1168,7 +1196,8 @@ _formatter_builtins = [
BuiltinCurrentLibraryName(),
BuiltinDaysBetween(), BuiltinDivide(), BuiltinEval(), BuiltinFirstNonEmpty(),
BuiltinField(), BuiltinFinishFormatting(), BuiltinFormatDate(),
BuiltinFormatNumber(), BuiltinFormatsModtimes(), BuiltinFormatsSizes(),
BuiltinFormatNumber(), BuiltinFormatsModtimes(), BuiltinFormatsPaths(),
BuiltinFormatsSizes(),
BuiltinHasCover(), BuiltinHumanReadable(), BuiltinIdentifierInList(),
BuiltinIfempty(), BuiltinLanguageCodes(), BuiltinLanguageStrings(),
BuiltinInList(), BuiltinListDifference(), BuiltinListEquals(),