From 60d83fd2b00a575b2ab41abb53587bffc672f008 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Tue, 11 Sep 2012 14:56:33 +0200 Subject: [PATCH 1/3] Support scan for new books --- .../devices/smart_device_app/driver.py | 67 ++++++++++++++----- 1 file changed, 52 insertions(+), 15 deletions(-) diff --git a/src/calibre/devices/smart_device_app/driver.py b/src/calibre/devices/smart_device_app/driver.py index 1dcf74450d..3fae38f66b 100644 --- a/src/calibre/devices/smart_device_app/driver.py +++ b/src/calibre/devices/smart_device_app/driver.py @@ -22,6 +22,7 @@ from calibre.devices.interface import DevicePlugin from calibre.devices.usbms.books import Book, CollectionsBookList from calibre.devices.usbms.deviceconfig import DeviceConfig from calibre.devices.usbms.driver import USBMS +from calibre.devices.utils import build_template_regexp from calibre.ebooks import BOOK_EXTENSIONS from calibre.ebooks.metadata import title_sort from calibre.ebooks.metadata.book.base import Metadata @@ -560,6 +561,16 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): self.broadcast_socket.close() self.broadcast_socket = None + def _read_file_metadata(self, temp_file_name): + from calibre.ebooks.metadata.meta import get_metadata + from calibre.customize.ui import quick_metadata + ext = temp_file_name.rpartition('.')[-1].lower() + with open(temp_file_name, 'rb') as stream: + with quick_metadata: + return get_metadata(stream, stream_type=ext, + force_read_metadata=True, + pattern=build_template_regexp(self.save_template())) + # The public interface methods. @synchronous('sync_lock') @@ -801,11 +812,13 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): self._debug(oncard) if oncard is not None: return CollectionsBookList(None, None, None) - opcode, result = self._call_client('GET_BOOK_COUNT', {'canStream':True}) + opcode, result = self._call_client('GET_BOOK_COUNT', {'canStream':True, + 'canScan':True}) bl = CollectionsBookList(None, self.PREFIX, self.settings) if opcode == 'OK': count = result['count'] will_stream = 'willStream' in result + will_scan = 'willScan' in result for i in range(0, count): if (i % 100) == 0: self._debug('getting book metadata. Done', i, 'of', count) @@ -820,8 +833,24 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): book = self.json_codec.raw_to_book(result, SDBook, self.PREFIX) self._set_known_metadata(book) bl.add_book(book, replace_metadata=True) + if '_new_book_' in result: + book.set('_new_book_', True) else: raise ControlError(desc='book metadata not returned') + + if will_scan: + total = 0 + for book in bl: + if book.get('_new_book_', None): + total += 1 + count = 0; + for book in bl: + if book.get('_new_book_', None): + paths = [book.lpath] + self.prepare_addable_books(paths, this_book=count, total_books=total) + book.smart_update(self._read_file_metadata(paths[0])) + del book._new_book_ + count += 1 self._debug('finished getting book metadata') return bl @@ -951,33 +980,41 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): @synchronous('sync_lock') - def get_file(self, path, outfile, end_session=True): + def get_file(self, path, outfile, end_session=True, this_book=None, total_books=None): self._debug(path) eof = False position = 0 while not eof: opcode, result = self._call_client('GET_BOOK_FILE_SEGMENT', - {'lpath' : path, 'position': position}, + {'lpath' : path, 'position': position, + 'thisBook': this_book, 'totalBooks': total_books, + 'canStream':True}, print_debug_info=False) if opcode == 'OK': - if not result['eof']: - data = b64decode(result['data']) - if len(data) != result['next_position'] - position: - self._debug('position mismatch', result['next_position'], position) - position = result['next_position'] - outfile.write(data) - else: - eof = True + client_will_stream = 'willStream' in result; + while not eof: + if not result['eof']: + data = b64decode(result['data']) + if len(data) != result['next_position'] - position: + self._debug('position mismatch', result['next_position'], position) + position = result['next_position'] + outfile.write(data) + opcode, result = self._receive_from_client(print_debug_info=True) + else: + eof = True + if not client_will_stream: + break else: raise ControlError(desc='request for book data failed') @synchronous('sync_lock') - def prepare_addable_books(self, paths): + def prepare_addable_books(self, paths, this_book=None, total_books=None): for idx, path in enumerate(paths): (ign, ext) = os.path.splitext(path) - tf = PersistentTemporaryFile(suffix=ext) - self.get_file(path, tf) - paths[idx] = tf.name + with PersistentTemporaryFile(suffix=ext) as tf: + self.get_file(path, tf, this_book=this_book, total_books=total_books) + paths[idx] = tf.name + tf.name = path return paths @synchronous('sync_lock') From f51127110b6d8635dc67434099ba6d346c88c4da Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Tue, 11 Sep 2012 16:07:38 +0200 Subject: [PATCH 2/3] Add a book to known metadata only if it isn't new. This ensures that the book's metadata will be sent back to the device. --- src/calibre/devices/smart_device_app/driver.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/calibre/devices/smart_device_app/driver.py b/src/calibre/devices/smart_device_app/driver.py index 3fae38f66b..de285c61ee 100644 --- a/src/calibre/devices/smart_device_app/driver.py +++ b/src/calibre/devices/smart_device_app/driver.py @@ -831,10 +831,11 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): if '_series_sort_' in result: del result['_series_sort_'] book = self.json_codec.raw_to_book(result, SDBook, self.PREFIX) - self._set_known_metadata(book) bl.add_book(book, replace_metadata=True) if '_new_book_' in result: book.set('_new_book_', True) + else: + self._set_known_metadata(book) else: raise ControlError(desc='book metadata not returned') From 383d48ef206bfaabbea533fe6ff6770b5e0536fc Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Wed, 12 Sep 2012 19:32:41 +0200 Subject: [PATCH 3/3] Fix bugs in scanning. --- src/calibre/devices/smart_device_app/driver.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/calibre/devices/smart_device_app/driver.py b/src/calibre/devices/smart_device_app/driver.py index de285c61ee..676cdf145c 100644 --- a/src/calibre/devices/smart_device_app/driver.py +++ b/src/calibre/devices/smart_device_app/driver.py @@ -523,7 +523,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): def _set_known_metadata(self, book, remove=False): lpath = book.lpath if remove: - self.known_metadata[lpath] = None + self.known_metadata.pop(lpath, None) else: self.known_metadata[lpath] = book.deepcopy() @@ -844,10 +844,11 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): for book in bl: if book.get('_new_book_', None): total += 1 - count = 0; + count = 0 for book in bl: if book.get('_new_book_', None): paths = [book.lpath] + self._set_known_metadata(book, remove=True) self.prepare_addable_books(paths, this_book=count, total_books=total) book.smart_update(self._read_file_metadata(paths[0])) del book._new_book_ @@ -992,7 +993,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): 'canStream':True}, print_debug_info=False) if opcode == 'OK': - client_will_stream = 'willStream' in result; + client_will_stream = 'willStream' in result while not eof: if not result['eof']: data = b64decode(result['data'])