From 4201dbeeaca994ebfed1c87fc9a8c2e6f29e1a43 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Mon, 17 May 2010 10:48:47 +0100 Subject: [PATCH 1/3] 1) Ensure that folder_device prefix (the path) end in os.sep 2) Fix regression causing sync of metadata cache on every connect 3) Report correct progress when adding books --- src/calibre/devices/folder_device/driver.py | 5 ++++- src/calibre/devices/usbms/books.py | 2 +- src/calibre/devices/usbms/device.py | 3 +-- src/calibre/devices/usbms/driver.py | 19 ++++++++++++++----- 4 files changed, 20 insertions(+), 9 deletions(-) diff --git a/src/calibre/devices/folder_device/driver.py b/src/calibre/devices/folder_device/driver.py index 6cc825dd9b..6b06cdf092 100644 --- a/src/calibre/devices/folder_device/driver.py +++ b/src/calibre/devices/folder_device/driver.py @@ -54,7 +54,10 @@ class FOLDER_DEVICE(USBMS): def __init__(self, path): if not os.path.isdir(path): raise IOError, 'Path is not a folder' - self._main_prefix = path + if path.endswith(os.sep): + self._main_prefix = path + else: + self._main_prefix = path + os.sep self.booklist_class = BookList self.is_connected = True diff --git a/src/calibre/devices/usbms/books.py b/src/calibre/devices/usbms/books.py index 3ecee3755f..59f098d421 100644 --- a/src/calibre/devices/usbms/books.py +++ b/src/calibre/devices/usbms/books.py @@ -117,7 +117,7 @@ class BookList(_BookList): def add_book(self, book, replace_metadata): if book not in self: self.append(book) - return True + return False # subclasses return True if device metadata has changed def remove_book(self, book): self.remove(book) diff --git a/src/calibre/devices/usbms/device.py b/src/calibre/devices/usbms/device.py index 249733b4e3..9b1da24805 100644 --- a/src/calibre/devices/usbms/device.py +++ b/src/calibre/devices/usbms/device.py @@ -113,8 +113,7 @@ class Device(DeviceConfig, DevicePlugin): def _windows_space(cls, prefix): if not prefix: return 0, 0 - if prefix.endswith(os.sep): - prefix = prefix[:-1] + prefix = prefix[:-1] win32file = __import__('win32file', globals(), locals(), [], -1) try: sectors_per_cluster, bytes_per_sector, free_clusters, total_clusters = \ diff --git a/src/calibre/devices/usbms/driver.py b/src/calibre/devices/usbms/driver.py index 1d5343024c..3a30b3c10e 100644 --- a/src/calibre/devices/usbms/driver.py +++ b/src/calibre/devices/usbms/driver.py @@ -90,6 +90,7 @@ class USBMS(CLI, Device): self.count_found_in_bl += 1 else: item = self.book_from_path(prefix, lpath) + changed = True if metadata.add_book(item, replace_metadata=False): changed = True except: # Probably a filename encoding error @@ -106,12 +107,17 @@ class USBMS(CLI, Device): if not os.path.exists(ebook_dir): continue # Get all books in the ebook_dir directory if self.SUPPORTS_SUB_DIRS: + # build a list of files to check, so we can accurately report progress + flist = [] for path, dirs, files in os.walk(ebook_dir): for filename in files: - self.report_progress(0.5, _('Getting list of books on device...')) - changed = update_booklist(filename, path, prefix) - if changed: - need_sync = True + if filename != self.METADATA_CACHE: + flist.append({'filename':filename, 'path': path}) + for i, f in enumerate(flist): + self.report_progress(i/float(len(flist)), _('Getting list of books on device...')) + changed = update_booklist(f['filename'], f['path'], prefix) + if changed: + need_sync = True else: paths = os.listdir(ebook_dir) for i, filename in enumerate(paths): @@ -123,7 +129,10 @@ class USBMS(CLI, Device): # if count != len(bl) then there were items in it that we did not # find on the device. If need_sync is True then there were either items # on the device that were not in bl or some of the items were changed. - #print "count found in cache: %d, count of files in cache: %d, must_sync_cache: %s" % (self.count_found_in_bl, len(bl), need_sync) + + #print "count found in cache: %d, count of files in cache: %d, need_sync: %s, must_sync_cache: %s" % \ + # (self.count_found_in_bl, len(bl), need_sync, + # need_sync or self.count_found_in_bl != len(bl)) if self.count_found_in_bl != len(bl) or need_sync: if oncard == 'cardb': self.sync_booklists((None, None, metadata)) From 922121f726092b67d864abed9c41c0a3216fe09b Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Mon, 17 May 2010 11:09:27 +0100 Subject: [PATCH 2/3] Calculate author_sort for metadata cache, if it isn't already set --- src/calibre/gui2/device.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/calibre/gui2/device.py b/src/calibre/gui2/device.py index 8f4ff6617f..26afc58068 100644 --- a/src/calibre/gui2/device.py +++ b/src/calibre/gui2/device.py @@ -19,7 +19,7 @@ from calibre.devices.scanner import DeviceScanner from calibre.gui2 import config, error_dialog, Dispatcher, dynamic, \ pixmap_to_data, warning_dialog, \ question_dialog -from calibre.ebooks.metadata import authors_to_string +from calibre.ebooks.metadata import authors_to_string, authors_to_sort_string from calibre import preferred_encoding from calibre.utils.filenames import ascii_filename from calibre.devices.errors import FreeSpaceError @@ -1133,9 +1133,11 @@ class DeviceGUI(object): resend_metadata = True # Set author_sort if it isn't already asort = getattr(book, 'author_sort', None) - if not asort: - pass + if not asort and book.authors: + book.author_sort = authors_to_sort_string(book.authors) + resend_metadata = True + if resend_metadata: - # Correcting metadata cache on device. + # Correct the metadata cache on device. if self.device_manager.is_device_connected: self.device_manager.sync_booklists(None, booklists) From f6f028ee5cf0b06f055c15f84655fa93d110b33f Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Mon, 17 May 2010 12:53:32 +0100 Subject: [PATCH 3/3] Remove race conditions when connecting folders and a device is present --- src/calibre/devices/folder_device/driver.py | 4 ++ src/calibre/gui2/device.py | 51 +++++++++++++-------- src/calibre/gui2/ui.py | 5 +- 3 files changed, 39 insertions(+), 21 deletions(-) diff --git a/src/calibre/devices/folder_device/driver.py b/src/calibre/devices/folder_device/driver.py index 6b06cdf092..792de9ee0a 100644 --- a/src/calibre/devices/folder_device/driver.py +++ b/src/calibre/devices/folder_device/driver.py @@ -61,6 +61,10 @@ class FOLDER_DEVICE(USBMS): self.booklist_class = BookList self.is_connected = True + def reset(self, key='-1', log_packets=False, report_progress=None, + detected_device=None): + pass + def disconnect_from_folder(self): self._main_prefix = '' self.is_connected = False diff --git a/src/calibre/gui2/device.py b/src/calibre/gui2/device.py index 26afc58068..edf0e763f7 100644 --- a/src/calibre/gui2/device.py +++ b/src/calibre/gui2/device.py @@ -86,7 +86,9 @@ class DeviceManager(Thread): self.current_job = None self.scanner = DeviceScanner() self.connected_device = None - self.ejected_devices = set([]) + self.ejected_devices = set([]) + self.connected_device_is_folder = False + self.folder_connection_path = None def report_progress(self, *args): pass @@ -99,7 +101,7 @@ class DeviceManager(Thread): def device(self): return self.connected_device - def do_connect(self, connected_devices): + def do_connect(self, connected_devices, is_folder_device): for dev, detected_device in connected_devices: dev.reset(detected_device=detected_device, report_progress=self.report_progress) @@ -110,7 +112,8 @@ class DeviceManager(Thread): traceback.print_exc() continue self.connected_device = dev - self.connected_slot(True) + self.connected_device_is_folder = is_folder_device + self.connected_slot(True, is_folder_device) return True return False @@ -128,7 +131,7 @@ class DeviceManager(Thread): if self.connected_device in self.ejected_devices: self.ejected_devices.remove(self.connected_device) else: - self.connected_slot(False) + self.connected_slot(False, self.connected_device_is_folder) self.connected_device = None def detect_device(self): @@ -149,17 +152,19 @@ class DeviceManager(Thread): if possibly_connected: possibly_connected_devices.append((device, detected_device)) if possibly_connected_devices: - if not self.do_connect(possibly_connected_devices): + if not self.do_connect(possibly_connected_devices, + is_folder_device=False): print 'Connect to device failed, retrying in 5 seconds...' time.sleep(5) - if not self.do_connect(possibly_connected_devices): + if not self.do_connect(possibly_connected_devices, + is_folder_device=False): print 'Device connect failed again, giving up' def umount_device(self, *args): if self.is_device_connected: self.connected_device.eject() self.ejected_devices.add(self.connected_device) - self.connected_slot(False) + self.connected_slot(False, self.connected_device_is_folder) def next(self): if not self.jobs.empty(): @@ -170,7 +175,18 @@ class DeviceManager(Thread): def run(self): while self.keep_going: - self.detect_device() + if not self.is_device_connected and \ + self.folder_connection_path is not None: + f = self.folder_connection_path + self.folder_connection_path = None # Make sure we try this folder only once + try: + dev = FOLDER_DEVICE(f) + self.do_connect([[dev, None],], is_folder_device=True) + except: + print 'Unable to open folder as device', f + traceback.print_exc() + else: + self.detect_device() while True: job = self.next() if job is not None: @@ -182,7 +198,6 @@ class DeviceManager(Thread): break time.sleep(self.sleep_time) - def create_job(self, func, done, description, args=[], kwargs={}): job = DeviceJob(func, done, self.job_manager, args=args, kwargs=kwargs, description=description) @@ -208,21 +223,19 @@ class DeviceManager(Thread): return self.create_job(self._get_device_information, done, description=_('Get device information')) + # This will be called on the GUI thread. Because of this, we must store + # information that the scanner thread will use to do the real work. def connect_to_folder(self, path): - dev = FOLDER_DEVICE(path) - try: - dev.open() - except: - print 'Unable to open device', dev - traceback.print_exc() - return False - self.connected_device = dev - self.connected_slot(True) - return True + self.folder_connection_path = path + # This is called on the GUI thread. No problem here, because it calls the + # device driver, telling it to tell the scanner when it passes by that the + # folder has disconnected. def disconnect_folder(self): if self.connected_device is not None: if hasattr(self.connected_device, 'disconnect_from_folder'): + # As we are on the wrong thread, this call must *not* do + # anything besides set a flag that the right thread will see. self.connected_device.disconnect_from_folder() def _books(self): diff --git a/src/calibre/gui2/ui.py b/src/calibre/gui2/ui.py index c1e208625b..1aecadba88 100644 --- a/src/calibre/gui2/ui.py +++ b/src/calibre/gui2/ui.py @@ -673,7 +673,6 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI): dir = choose_dir(self, 'Select Device Folder', 'Select folder to open') if dir is not None: self.device_manager.connect_to_folder(dir) - self._sync_menu.disconnect_from_folder_action.setEnabled(True) def disconnect_from_folder(self): self.device_manager.disconnect_folder() @@ -945,12 +944,14 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI): elif model.location_for_row(x) == 'cardb': self.card_b_view.write_settings() - def device_detected(self, connected): + def device_detected(self, connected, is_folder_device): ''' Called when a device is connected to the computer. ''' if connected: self._sync_menu.connect_to_folder_action.setEnabled(False) + if is_folder_device: + self._sync_menu.disconnect_from_folder_action.setEnabled(True) self.device_manager.get_device_information(\ Dispatcher(self.info_read)) self.set_default_thumbnail(\