Template language: Add a couple of new functions to get the path to individual book formats and the calibre library as a whole

src/calibre/devices/smart_device_app/driver.py

@@ -10,9 +10,12 @@ Created on 29 Jun 2012
 import socket, select, json, inspect, os, traceback, time, sys, random
 import posixpath
 import hashlib, threading
+import Queue
+
 from base64 import b64encode, b64decode
 from functools import wraps
 from errno import EAGAIN, EINTR
+from threading import Thread
 
 from calibre import prints
 from calibre.constants import numeric_version, DEBUG
@@ -48,6 +51,110 @@ def synchronous(tlockname):
     return _synched
 
 
+class ConnectionListener (Thread):
+
+    def __init__(self, driver):
+        Thread.__init__(self)
+        self.daemon = True
+        self.driver = driver
+        self.keep_running = True
+
+    def stop(self):
+        self.keep_running = False
+
+    def run(self):
+        queue_not_serviced_count = 0
+        device_socket = None
+        while self.keep_running:
+            try:
+                time.sleep(1)
+            except:
+                # Happens during interpreter shutdown
+                break
+
+            if not self.keep_running:
+                break
+
+            if not self.driver.connection_queue.empty():
+                queue_not_serviced_count += 1
+                if queue_not_serviced_count >= 3:
+                    self.driver._debug('queue not serviced')
+                    try:
+                        sock = self.driver.connection_queue.get_nowait()
+                        s = self.driver._json_encode(
+                                        self.driver.opcodes['CALIBRE_BUSY'], {})
+                        self.driver._send_byte_string(device_socket, (b'%d' % len(s)) + s)
+                        sock.close()
+                    except Queue.Empty:
+                        pass
+                    queue_not_serviced_count = 0
+            else:
+                queue_not_serviced_count = 0
+
+            if getattr(self.driver, 'broadcast_socket', None) is not None:
+                while True:
+                    ans = select.select((self.driver.broadcast_socket,), (), (), 0)
+                    if len(ans[0]) > 0:
+                        try:
+                            packet = self.driver.broadcast_socket.recvfrom(100)
+                            remote = packet[1]
+                            message = str(self.driver.ZEROCONF_CLIENT_STRING + b' (on ' +
+                                            str(socket.gethostname().partition('.')[0]) +
+                                            b'),' + str(self.driver.port))
+                            self.driver._debug('received broadcast', packet, message)
+                            self.driver.broadcast_socket.sendto(message, remote)
+                        except:
+                            pass
+                    else:
+                        break
+
+            if self.driver.connection_queue.empty() and \
+                        getattr(self.driver, 'listen_socket', None) is not None:
+                ans = select.select((self.driver.listen_socket,), (), (), 0)
+                if len(ans[0]) > 0:
+                    # timeout in 10 ms to detect rare case where the socket went
+                    # way between the select and the accept
+                    try:
+                        self.driver._debug('attempt to open device socket')
+                        device_socket = None
+                        self.driver.listen_socket.settimeout(0.010)
+                        device_socket, ign = eintr_retry_call(
+                                self.driver.listen_socket.accept)
+                        self.driver.listen_socket.settimeout(None)
+                        device_socket.settimeout(None)
+
+                        try:
+                            peer = self.driver.device_socket.getpeername()[0]
+                            attempts = self.drjver.connection_attempts.get(peer, 0)
+                            if attempts >= self.MAX_UNSUCCESSFUL_CONNECTS:
+                                self.driver._debug('too many connection attempts from', peer)
+                                device_socket.close()
+                                device_socket = None
+#                                raise InitialConnectionError(_('Too many connection attempts from %s') % peer)
+                            else:
+                                self.driver.connection_attempts[peer] = attempts + 1
+                        except InitialConnectionError:
+                            raise
+                        except:
+                            pass
+
+                        try:
+                            self.driver.connection_queue.put_nowait(device_socket)
+                        except Queue.Full:
+                            device_socket.close()
+                            device_socket = None
+                            self.driver._debug('driver is not answering')
+
+                    except socket.timeout:
+                        pass
+                    except socket.error:
+                        x = sys.exc_info()[1]
+                        self.driver._debug('unexpected socket exception', x.args[0])
+                        device_socket.close()
+                        device_socket = None
+#                        raise
+
+
 class SDBook(Book):
     def __init__(self, prefix, lpath, size=None, other=None):
         Book.__init__(self, prefix, lpath, size=size, other=other)
@@ -80,8 +187,20 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
     NEWS_IN_FOLDER              = True
     SUPPORTS_USE_AUTHOR_SORT    = False
     WANTS_UPDATED_THUMBNAILS    = True
+
+    # Guess about the max length on windows. This number will be reduced by
+    # the length of the path on the client, and by the fudge factor below. We
+    # use this on all platforms because the device might be connected to windows
+    # in the future.
     MAX_PATH_LEN                = 250
+    # guess of length of MTP name. The length of the full path to the folder
+    # on the device is added to this. That path includes the device's mount point
+    # making this number effectively around 10 to 15 larger.
+    PATH_FUDGE_FACTOR           = 40
+
     THUMBNAIL_HEIGHT            = 160
+    DEFAULT_THUMBNAIL_HEIGHT    = 160
+
     PREFIX                      = ''
     BACKLOADING_ERROR_MESSAGE   = None
 
@@ -112,6 +231,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
         'OK'                     : 0,
         'BOOK_DATA'              : 10,
         'BOOK_DONE'              : 11,
+        'CALIBRE_BUSY'           : 18,
         'DELETE_BOOK'            : 13,
         'DISPLAY_MESSAGE'        : 17,
         'FREE_SPACE'             : 5,
@@ -248,7 +368,11 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
     # remove the 'path' argument and all its uses. Also removed the calls to
     # filename_callback and sanitize_path_components
     def _create_upload_path(self, mdata, fname, create_dirs=True):
-        maxlen = self.MAX_PATH_LEN
+        fname = sanitize(fname)
+        ext = os.path.splitext(fname)[1]
+
+        maxlen = (self.MAX_PATH_LEN - (self.PATH_FUDGE_FACTOR +
+                   self.exts_path_lengths.get(ext, self.PATH_FUDGE_FACTOR)))
 
         special_tag = None
         if mdata.tags:
@@ -269,9 +393,6 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
             template = "{title}_%d-%d-%d" % date
         use_subdirs = self.SUPPORTS_SUB_DIRS and settings.use_subdirs
 
-        fname = sanitize(fname)
-        ext = os.path.splitext(fname)[1]
-
         from calibre.library.save_to_disk import get_components
         from calibre.library.save_to_disk import config
         opts = config().parse()
@@ -346,6 +467,13 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
         return json.dumps([op, res], encoding='utf-8')
 
     # Network functions
+
+    def _read_binary_from_net(self, length):
+        self.device_socket.settimeout(self.MAX_CLIENT_COMM_TIMEOUT)
+        v = self.device_socket.recv(length)
+        self.device_socket.settimeout(None)
+        return v
+
     def _read_string_from_net(self):
         data = bytes(0)
         while True:
@@ -354,9 +482,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
                 break
             # recv seems to return a pointer into some internal buffer.
             # Things get trashed if we don't make a copy of the data.
-            self.device_socket.settimeout(self.MAX_CLIENT_COMM_TIMEOUT)
-            v = self.device_socket.recv(2)
-            self.device_socket.settimeout(None)
+            v = self._read_binary_from_net(2)
             if len(v) == 0:
                 return '' # documentation says the socket is broken permanently.
             data += v
@@ -364,16 +490,14 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
         data = data[dex:]
         pos = len(data)
         while pos < total_len:
-            self.device_socket.settimeout(self.MAX_CLIENT_COMM_TIMEOUT)
-            v = self.device_socket.recv(total_len - pos)
-            self.device_socket.settimeout(None)
+            v = self._read_binary_from_net(total_len - pos)
             if len(v) == 0:
                 return '' # documentation says the socket is broken permanently.
             data += v
             pos += len(v)
         return data
 
-    def _send_byte_string(self, s):
+    def _send_byte_string(self, sock, s):
         if not isinstance(s, bytes):
             self._debug('given a non-byte string!')
             raise PacketError("Internal error: found a string that isn't bytes")
@@ -382,11 +506,11 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
         while sent_len < total_len:
             try:
                 if sent_len == 0:
-                    amt_sent = self.device_socket.send(s)
+                    amt_sent = sock.send(s)
                 else:
-                    amt_sent = self.device_socket.send(s[sent_len:])
+                    amt_sent = sock.send(s[sent_len:])
                 if amt_sent <= 0:
-                    raise IOError('Bad write on device socket')
+                    raise IOError('Bad write on socket')
                 sent_len += amt_sent
             except socket.error as e:
                 self._debug('socket error', e, e.errno)
@@ -410,7 +534,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
             if print_debug_info and extra_debug:
                 self._debug('send string', s)
             self.device_socket.settimeout(self.MAX_CLIENT_COMM_TIMEOUT)
-            self._send_byte_string((b'%d' % len(s)) + s)
+            self._send_byte_string(self.device_socket, (b'%d' % len(s)) + s)
             if not wait_for_response:
                 return None, None
             return self._receive_from_client(print_debug_info=print_debug_info)
@@ -466,11 +590,12 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
         length = infile.tell()
         book_metadata.size = length
         infile.seek(0)
-        self._debug(lpath, length)
+
         opcode, result = self._call_client('SEND_BOOK', {'lpath': lpath, 'length': length,
                                'metadata': book_metadata, 'thisBook': this_book,
                                'totalBooks': total_books,
-                               'willStreamBooks': self.client_can_stream_books},
+                               'willStreamBooks': self.client_can_stream_books,
+                               'willStreamBinary' : self.client_can_receive_book_binary},
                           print_debug_info=False,
                           wait_for_response=(not self.client_can_stream_books))
 
@@ -483,6 +608,9 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
                 blen = len(b)
                 if not b:
                     break
+                if self.client_can_stream_books and self.client_can_receive_book_binary:
+                    self._send_byte_string(self.device_socket, b)
+                else:
                     b = b64encode(b)
                     opcode, result = self._call_client('BOOK_DATA',
                                     {'lpath': lpath, 'position': pos, 'data': b},
@@ -493,6 +621,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
                     self._debug('protocol error', opcode)
                     failed = True
                     break
+        if not (self.client_can_stream_books and self.client_can_receive_book_binary):
             self._call_client('BOOK_DONE', {'lpath': lpath})
         self.time = None
         if close_:
@@ -517,7 +646,8 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
         v = self.known_metadata.get(book.lpath, None)
         if v is not None:
             return (v.get('uuid', None) == book.get('uuid', None) and
-                    v.get('last_modified', None) == book.get('last_modified', None))
+                    v.get('last_modified', None) == book.get('last_modified', None) and
+                    v.get('thumbnail', None) == book.get('thumbnail', None))
         return False
 
     def _set_known_metadata(self, book, remove=False):
@@ -620,17 +750,9 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
                     break
 
         if getattr(self, 'listen_socket', None) is not None:
-            ans = select.select((self.listen_socket,), (), (), 0)
-            if len(ans[0]) > 0:
-                # timeout in 10 ms to detect rare case where the socket went
-                # way between the select and the accept
             try:
-                    self.device_socket = None
-                    self.listen_socket.settimeout(0.010)
-                    self.device_socket, ign = eintr_retry_call(
-                            self.listen_socket.accept)
-                    self.listen_socket.settimeout(None)
-                    self.device_socket.settimeout(None)
+                ans = self.connection_queue.get_nowait()
+                self.device_socket = ans
                 self.is_connected = True
                 try:
                     peer = self.device_socket.getpeername()[0]
@@ -645,13 +767,8 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
                     raise
                 except:
                     pass
-                except socket.timeout:
-                    self._close_device_socket()
-                except socket.error:
-                    x = sys.exc_info()[1]
-                    self._debug('unexpected socket exception', x.args[0])
-                    self._close_device_socket()
-                    raise
+            except Queue.Empty:
+                self.is_connected = False
             return (self.is_connected, self)
         return (False, None)
 
@@ -705,17 +822,34 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
             self._debug('Device can stream books', self.client_can_stream_books)
             self.client_can_stream_metadata = result.get('canStreamMetadata', False)
             self._debug('Device can stream metadata', self.client_can_stream_metadata)
+            self.client_can_receive_book_binary = result.get('canReceiveBookBinary', False)
+            self._debug('Device can receive book binary', self.client_can_stream_metadata)
 
             self.max_book_packet_len = result.get('maxBookContentPacketLen',
                                                   self.BASE_PACKET_LEN)
+            self._debug('max_book_packet_len', self.max_book_packet_len)
+
             exts = result.get('acceptedExtensions', None)
             if exts is None or not isinstance(exts, list) or len(exts) == 0:
                 self._debug('Protocol error - bogus accepted extensions')
                 self._close_device_socket()
                 return False
+
             config = self._configProxy()
             config['format_map'] = exts
             self._debug('selected formats', config['format_map'])
+
+            self.exts_path_lengths = result.get('extensionPathLengths', {})
+            self._debug('extension path lengths', self.exts_path_lengths)
+
+            self.THUMBNAIL_HEIGHT = result.get('coverHeight', self.DEFAULT_THUMBNAIL_HEIGHT)
+            if 'coverWidth' in result:
+                # Setting this field forces the aspect ratio
+                self.THUMBNAIL_WIDTH = result.get('coverWidth',
+                                      (self.DEFAULT_THUMBNAIL_HEIGHT/3) * 4)
+            elif hasattr(self, 'THUMBNAIL_WIDTH'):
+                    delattr(self, 'THUMBNAIL_WIDTH')
+
             if password:
                 returned_hash = result.get('passwordHash', None)
                 if result.get('passwordHash', None) is None:
@@ -909,7 +1043,10 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
     @synchronous('sync_lock')
     def upload_books(self, files, names, on_card=None, end_session=True,
                      metadata=None):
+        if self.settings().extra_customization[self.OPT_EXTRA_DEBUG]:
             self._debug(names)
+        else:
+            self._debug()
 
         paths = []
         names = iter(names)
@@ -956,7 +1093,11 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
 
     @synchronous('sync_lock')
     def delete_books(self, paths, end_session=True):
+        if self.settings().extra_customization[self.OPT_EXTRA_DEBUG]:
             self._debug(paths)
+        else:
+            self._debug()
+
         for path in paths:
             # the path has the prefix on it (I think)
             path = self._strip_prefix(path)
@@ -968,7 +1109,11 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
 
     @synchronous('sync_lock')
     def remove_books_from_metadata(self, paths, booklists):
+        if self.settings().extra_customization[self.OPT_EXTRA_DEBUG]:
             self._debug(paths)
+        else:
+            self._debug()
+
         for i, path in enumerate(paths):
             path = self._strip_prefix(path)
             self.report_progress((i + 1) / float(len(paths)), _('Removing books from device metadata listing...'))
@@ -983,17 +1128,33 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
 
     @synchronous('sync_lock')
     def get_file(self, path, outfile, end_session=True, this_book=None, total_books=None):
+        if self.settings().extra_customization[self.OPT_EXTRA_DEBUG]:
             self._debug(path)
+        else:
+            self._debug()
+
         eof = False
         position = 0
         while not eof:
             opcode, result = self._call_client('GET_BOOK_FILE_SEGMENT',
                                     {'lpath' : path, 'position': position,
                                      'thisBook': this_book, 'totalBooks': total_books,
-                                     'canStream':True},
+                                     'canStream':True, 'canStreamBinary': True},
                                     print_debug_info=False)
             if opcode == 'OK':
                 client_will_stream = 'willStream' in result
+                client_will_stream_binary = 'willStreamBinary' in result
+
+                if (client_will_stream_binary):
+                    length = result.get('fileLength')
+                    remaining = length
+
+                    while remaining > 0:
+                        v = self._read_binary_from_net(min(remaining, self.max_book_packet_len))
+                        outfile.write(v)
+                        remaining -= len(v)
+                    eof = True
+                else:
                     while not eof:
                         if not result['eof']:
                             data = b64decode(result['data'])
@@ -1127,17 +1288,22 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
                 self._debug('broadcast socket listening on port', port)
                 break
 
+        message = None
         if port == 0:
             self.broadcast_socket.close()
             self.broadcast_socket = None
             message = 'attaching port to broadcast socket failed. This is not fatal.'
             self._debug(message)
-            return message
 
+        self.connection_queue = Queue.Queue(1)
+        self.connection_listener = ConnectionListener(self)
+        self.connection_listener.start()
+        return message
 
     @synchronous('sync_lock')
     def shutdown(self):
         if getattr(self, 'listen_socket', None) is not None:
+            self.connection_listener.stop()
             unpublish_zeroconf('calibre smart device client',
                              '_calibresmartdeviceapp._tcp', self.port, {})
             self._close_listen_socket()

src/calibre/gui2/device.py

@@ -1673,7 +1673,6 @@ class DeviceMixin(object): # {{{
                     if update_metadata:
                         mi = db.get_metadata(id_, index_is_id=True,
                                              get_cover=get_covers)
-                        if book.get('last_modified', None) != mi.last_modified:
                         book.smart_update(db.get_metadata(id_,
                                                       index_is_id=True,
                                                       get_cover=get_covers),

src/calibre/library/__init__.py

@@ -61,13 +61,18 @@ def generate_test_db(library_path, # {{{
     print 'Time per record:', t/float(num_of_records)
 # }}}
 
-def current_library_name():
+def current_library_path():
     from calibre.utils.config import prefs
-    import posixpath
     path = prefs['library_path']
     if path:
         path = path.replace('\\', '/')
         while path.endswith('/'):
             path = path[:-1]
+        return path
+
+def current_library_name():
+    import posixpath
+    path = current_library_path()
+    if path:
         return posixpath.basename(path)
 

src/calibre/library/database2.py

@@ -1262,6 +1262,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         ans = {}
         if path is not None:
             stat = os.stat(path)
+            ans['path'] = path
             ans['size'] = stat.st_size
             ans['mtime'] = utcfromtimestamp(stat.st_mtime)
             self.format_metadata_cache[id_][fmt] = ans
@@ -2563,6 +2564,11 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
             if notify:
                 self.notify('metadata', [id])
 
+    def get_id_from_uuid(self, uuid):
+        if uuid:
+            return self.conn.get('SELECT id FROM books WHERE uuid=?', (uuid,),
+                                 all=False)
+
     # Convenience methods for tags_list_editor
     # Note: we generally do not need to refresh_ids because library_view will
     # refresh everything.

src/calibre/library/server/ajax.py

@@ -181,7 +181,7 @@ class AjaxServer(object):
         return data, mi.last_modified
 
     @Endpoint(set_last_modified=False)
-    def ajax_book(self, book_id, category_urls='true'):
+    def ajax_book(self, book_id, category_urls='true', id_is_uuid='false'):
         '''
         Return the metadata of the book as a JSON dictionary.
 
@@ -192,6 +192,9 @@ class AjaxServer(object):
         cherrypy.response.timeout = 3600
 
         try:
+            if id_is_uuid == 'true':
+                book_id = self.db.get_id_from_uuid(book_id)
+            else:
                 book_id = int(book_id)
             data, last_modified = self.ajax_book_to_json(book_id,
                     get_category_urls=category_urls.lower()=='true')
@@ -204,7 +207,7 @@ class AjaxServer(object):
         return data
 
     @Endpoint(set_last_modified=False)
-    def ajax_books(self, ids=None, category_urls='true'):
+    def ajax_books(self, ids=None, category_urls='true', id_is_uuid='false'):
         '''
         Return the metadata for a list of books specified as a comma separated
         list of ids. The metadata is returned as a dictionary mapping ids to
@@ -218,6 +221,9 @@ class AjaxServer(object):
         if ids is None:
             raise cherrypy.HTTPError(404, 'Must specify some ids')
         try:
+            if id_is_uuid == 'true':
+                ids = set(self.db.get_id_from_uuid(x) for x in ids.split(','))
+            else:
                 ids = set(int(x.strip()) for x in ids.split(','))
         except:
             raise cherrypy.HTTPError(404, 'ids must be a comma separated list'

src/calibre/utils/formatter_functions.py

@@ -629,6 +629,22 @@ class BuiltinFormatsSizes(BuiltinFormatterFunction):
         fmt_data = mi.get('format_metadata', {})
         return ','.join(k.upper()+':'+str(v['size']) for k,v in fmt_data.iteritems())
 
+class BuiltinFormatsPaths(BuiltinFormatterFunction):
+    name = 'formats_paths'
+    arg_count = 0
+    category = 'Get values from metadata'
+    __doc__ = doc = _('formats_paths() -- return a comma-separated list of '
+                      'colon_separated items representing full path to '
+                      'the formats of a book. You can use the select '
+                      'function to get the path for a specific '
+                      'format. Note that format names are always uppercase, '
+                      'as in EPUB.'
+            )
+
+    def evaluate(self, formatter, kwargs, mi, locals):
+        fmt_data = mi.get('format_metadata', {})
+        return ','.join(k.upper()+':'+str(v['path']) for k,v in fmt_data.iteritems())
+
 class BuiltinHumanReadable(BuiltinFormatterFunction):
     name = 'human_readable'
     arg_count = 1
@@ -1146,6 +1162,18 @@ class BuiltinCurrentLibraryName(BuiltinFormatterFunction):
         from calibre.library import current_library_name
         return current_library_name()
 
+class BuiltinCurrentLibraryPath(BuiltinFormatterFunction):
+    name = 'current_library_path'
+    arg_count = 0
+    category = 'Get values from metadata'
+    __doc__ = doc = _('current_library_path() -- '
+            'return the path to the current calibre library. This function can '
+            'be called in template program mode using the template '
+            '"{:\'current_library_path()\'}".')
+    def evaluate(self, formatter, kwargs, mi, locals):
+        from calibre.library import current_library_path
+        return current_library_path()
+
 class BuiltinFinishFormatting(BuiltinFormatterFunction):
     name = 'finish_formatting'
     arg_count = 4
@@ -1168,7 +1196,8 @@ _formatter_builtins = [
     BuiltinCurrentLibraryName(),
     BuiltinDaysBetween(), BuiltinDivide(), BuiltinEval(), BuiltinFirstNonEmpty(),
     BuiltinField(), BuiltinFinishFormatting(), BuiltinFormatDate(),
-    BuiltinFormatNumber(), BuiltinFormatsModtimes(), BuiltinFormatsSizes(),
+    BuiltinFormatNumber(), BuiltinFormatsModtimes(), BuiltinFormatsPaths(),
+    BuiltinFormatsSizes(),
     BuiltinHasCover(), BuiltinHumanReadable(), BuiltinIdentifierInList(),
     BuiltinIfempty(), BuiltinLanguageCodes(), BuiltinLanguageStrings(),
     BuiltinInList(), BuiltinListDifference(), BuiltinListEquals(),