diff --git a/src/calibre/__init__.py b/src/calibre/__init__.py index ac8e681fed..58390a314a 100644 --- a/src/calibre/__init__.py +++ b/src/calibre/__init__.py @@ -674,7 +674,7 @@ def get_download_filename(url, cookie_file=None): return filename -def human_readable(size): +def human_readable(size, sep=' '): """ Convert a size in bytes into a human readable form """ divisor, suffix = 1, "B" for i, candidate in enumerate(('B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB')): @@ -686,7 +686,7 @@ def human_readable(size): size = size[:size.find(".")+2] if size.endswith('.0'): size = size[:-2] - return size + " " + suffix + return size + sep + suffix def remove_bracketed_text(src, brackets={u'(':u')', u'[':u']', u'{':u'}'}): diff --git a/src/calibre/devices/mtp/filesystem_cache.py b/src/calibre/devices/mtp/filesystem_cache.py index a94172b6b0..cc7d41e09b 100644 --- a/src/calibre/devices/mtp/filesystem_cache.py +++ b/src/calibre/devices/mtp/filesystem_cache.py @@ -7,10 +7,85 @@ __license__ = 'GPL v3' __copyright__ = '2012, Kovid Goyal ' __docformat__ = 'restructuredtext en' +import weakref, sys +from operator import attrgetter +from future_builtins import map + +from calibre import human_readable, prints, force_unicode +from calibre.utils.icu import sort_key + +class FileOrFolder(object): + + def __init__(self, entry, fs_cache, all_storage_ids): + self.object_id = entry['id'] + self.is_folder = entry['is_folder'] + self.name = force_unicode(entry.get('name', '___'), 'utf-8') + self.persistent_id = entry.get('persistent_id', self.object_id) + self.size = entry.get('size', 0) + # self.parent_id is None for storage objects + self.parent_id = entry.get('parent_id', None) + if self.parent_id == 0: + sid = entry['storage_id'] + if sid not in all_storage_ids: + sid = all_storage_ids[0] + self.parent_id = sid + self.is_hidden = entry.get('is_hidden', False) + self.is_system = entry.get('is_system', False) + self.can_delete = entry.get('can_delete', True) + + self.files = [] + self.folders = [] + fs_cache.id_map[self.object_id] = self + self.fs_cache = weakref.ref(fs_cache) + + @property + def id_map(self): + return self.fs_cache().id_map + + @property + def parent(self): + return None if self.parent_id is None else self.id_map[self.parent_id] + + def __iter__(self): + for e in self.folders: + yield e + for e in self.files: + yield e + + def dump(self, prefix='', out=sys.stdout): + c = '+' if self.is_folder else '-' + data = ('%s children'%(sum(map(len, (self.files, self.folders)))) + if self.is_folder else human_readable(self.size)) + line = '%s%s %s [id:%s %s]'%(prefix, c, self.name, self.object_id, data) + prints(line, file=out) + for c in (self.folders, self.files): + for e in sorted(c, key=lambda x:sort_key(x.name)): + e.dump(prefix=prefix+' ', out=out) + class FilesystemCache(object): - def __init__(self, storage_map): - self.tree = {} - for storage_id, id_map in storage_map.iteritems(): - self.tree[storage_id] = self.build_tree(id_map) + def __init__(self, all_storage, entries): + self.entries = [] + self.id_map = {} + + for storage in all_storage: + e = FileOrFolder(storage, self, []) + self.entries.append(e) + + self.entries.sort(key=attrgetter('object_id')) + all_storage_ids = [x.object_id for x in self.entries] + + for entry in entries: + FileOrFolder(entry, self, all_storage_ids) + + for item in self.id_map.itervalues(): + p = item.parent + if p is not None: + t = p.folders if item.is_folder else p.files + t.append(item) + + def dump(self, out=sys.stdout): + for e in self.entries: + e.dump(out=out) + diff --git a/src/calibre/devices/mtp/unix/driver.py b/src/calibre/devices/mtp/unix/driver.py index c94a2e2458..835f2245d0 100644 --- a/src/calibre/devices/mtp/unix/driver.py +++ b/src/calibre/devices/mtp/unix/driver.py @@ -9,77 +9,13 @@ __docformat__ = 'restructuredtext en' import time, operator from threading import RLock -from itertools import chain -from collections import deque, OrderedDict from io import BytesIO -from calibre import prints from calibre.devices.errors import OpenFailed, DeviceError from calibre.devices.mtp.base import MTPDeviceBase, synchronous +from calibre.devices.mtp.filesystem_cache import FilesystemCache from calibre.devices.mtp.unix.detect import MTPDetect -class FilesystemCache(object): - - def __init__(self, files, folders): - self.files = files - self.folders = folders - self.file_id_map = {f['id']:f for f in files} - self.folder_id_map = {f['id']:f for f in self.iterfolders(set_level=0)} - - # Set the parents of each file - self.files_in_root = OrderedDict() - for f in files: - parents = deque() - pid = f['parent_id'] - while pid is not None and pid > 0: - try: - parent = self.folder_id_map[pid] - except KeyError: - break - parents.appendleft(pid) - pid = parent['parent_id'] - f['parents'] = parents - if not parents: - self.files_in_root[f['id']] = f - - # Set the files in each folder - for f in self.iterfolders(): - f['files'] = [i for i in files if i['parent_id'] == - f['id']] - - # Decode the file and folder names - for f in chain(files, folders): - try: - name = f['name'].decode('utf-8') - except UnicodeDecodeError: - name = 'undecodable_%d'%f['id'] - f['name'] = name - - def iterfolders(self, folders=None, set_level=None): - clevel = None if set_level is None else set_level + 1 - if folders is None: - folders = self.folders - for f in folders: - if set_level is not None: - f['level'] = set_level - yield f - for c in f['children']: - for child in self.iterfolders([c], set_level=clevel): - yield child - - def dump_filesystem(self): - indent = 2 - for f in self.iterfolders(): - prefix = ' '*(indent*f['level']) - prints(prefix, '+', f['name'], 'id=%s'%f['id']) - for leaf in f['files']: - prints(prefix, ' '*indent, '-', leaf['name'], - 'id=%d'%leaf['id'], 'size=%d'%leaf['size'], - 'modtime=%d'%leaf['modtime']) - for leaf in self.files_in_root.itervalues(): - prints('-', leaf['name'], 'id=%d'%leaf['id'], - 'size=%d'%leaf['size'], 'modtime=%d'%leaf['modtime']) - class MTP_DEVICE(MTPDeviceBase): supported_platforms = ['linux'] @@ -87,7 +23,7 @@ class MTP_DEVICE(MTPDeviceBase): def __init__(self, *args, **kwargs): MTPDeviceBase.__init__(self, *args, **kwargs) self.dev = None - self.filesystem_cache = None + self._filesystem_cache = None self.lock = RLock() self.blacklisted_devices = set() @@ -129,7 +65,7 @@ class MTP_DEVICE(MTPDeviceBase): @synchronous def post_yank_cleanup(self): - self.dev = self.filesystem_cache = self.current_friendly_name = None + self.dev = self._filesystem_cache = self.current_friendly_name = None @synchronous def startup(self): @@ -140,7 +76,7 @@ class MTP_DEVICE(MTPDeviceBase): @synchronous def shutdown(self): - self.dev = self.filesystem_cache = None + self.dev = self._filesystem_cache = None def format_errorstack(self, errs): return '\n'.join(['%d:%s'%(code, msg.decode('utf-8', 'replace')) for @@ -148,7 +84,7 @@ class MTP_DEVICE(MTPDeviceBase): @synchronous def open(self, connected_device, library_uuid): - self.dev = self.filesystem_cache = None + self.dev = self._filesystem_cache = None def blacklist_device(): d = connected_device self.blacklisted_devices.add((d.busnum, d.devnum, d.vendor_id, @@ -179,23 +115,41 @@ class MTP_DEVICE(MTPDeviceBase): self._carda_id = storage[1]['id'] if len(storage) > 2: self._cardb_id = storage[2]['id'] - self.current_friendly_name = self.dev.name + self.current_friendly_name = self.dev.friendly_name - @synchronous - def read_filesystem_cache(self): - try: - files, errs = self.dev.get_filelist(self) - if errs and not files: - raise DeviceError('Failed to read files from device. Underlying errors:\n' - +self.format_errorstack(errs)) - folders, errs = self.dev.get_folderlist() - if errs and not folders: - raise DeviceError('Failed to read folders from device. Underlying errors:\n' - +self.format_errorstack(errs)) - self.filesystem_cache = FilesystemCache(files, folders) - except: - self.dev = self._main_id = self._carda_id = self._cardb_id = None - raise + @property + def filesystem_cache(self): + if self._filesystem_cache is None: + with self.lock: + files, errs = self.dev.get_filelist(self) + if errs and not files: + raise DeviceError('Failed to read files from device. Underlying errors:\n' + +self.format_errorstack(errs)) + folders, errs = self.dev.get_folderlist() + if errs and not folders: + raise DeviceError('Failed to read folders from device. Underlying errors:\n' + +self.format_errorstack(errs)) + storage = [] + for sid, capacity in zip([self._main_id, self._carda_id, + self._cardb_id], self.total_space()): + if sid is not None: + name = _('Unknown') + for x in self.dev.storage_info: + if x['id'] == sid: + name = x['name'] + break + storage.append({'id':sid, 'size':capacity, + 'is_folder':True, 'name':name}) + all_folders = [] + def recurse(f): + all_folders.append(f) + for c in f['children']: + recurse(c) + + for f in folders: recurse(f) + self._filesystem_cache = FilesystemCache(storage, + all_folders+files) + return self._filesystem_cache @synchronous def get_device_information(self, end_session=True): @@ -246,7 +200,6 @@ if __name__ == '__main__': devs = linux_scanner() mtp_devs = dev.detect(devs) dev.open(list(mtp_devs)[0], 'xxx') - dev.read_filesystem_cache() d = dev.dev print ("Opened device:", dev.get_gui_name()) print ("Storage info:") @@ -257,7 +210,7 @@ if __name__ == '__main__': # fname = b'moose.txt' # src = BytesIO(raw) # print (d.put_file(dev._main_id, 0, fname, src, len(raw), PR())) - dev.filesystem_cache.dump_filesystem() + dev.filesystem_cache.dump() # with open('/tmp/flint.epub', 'wb') as f: # print(d.get_file(786, f, PR())) # print() diff --git a/src/calibre/devices/mtp/unix/libmtp.c b/src/calibre/devices/mtp/unix/libmtp.c index 856fd057bf..ffab2e8b30 100644 --- a/src/calibre/devices/mtp/unix/libmtp.c +++ b/src/calibre/devices/mtp/unix/libmtp.c @@ -315,7 +315,7 @@ libmtp_Device_storage_info(libmtp_Device *self, void *closure) { "capacity", storage->MaxCapacity, "freespace_bytes", storage->FreeSpaceInBytes, "freespace_objects", storage->FreeSpaceInObjects, - "storage_desc", storage->StorageDescription, + "name", storage->StorageDescription, "volume_id", storage->VolumeIdentifier ); @@ -358,13 +358,14 @@ libmtp_Device_get_filelist(libmtp_Device *self, PyObject *args, PyObject *kwargs } for (f=tf; f != NULL; f=f->next) { - fo = Py_BuildValue("{s:k,s:k,s:k,s:s,s:K,s:k}", + fo = Py_BuildValue("{s:k,s:k,s:k,s:s,s:K,s:k,s:O}", "id", f->item_id, "parent_id", f->parent_id, "storage_id", f->storage_id, "name", f->filename, "size", f->filesize, - "modtime", f->modificationdate + "modtime", f->modificationdate, + "is_folder", Py_False ); if (fo == NULL || PyList_Append(ans, fo) != 0) break; Py_DECREF(fo); @@ -393,11 +394,12 @@ int folderiter(LIBMTP_folder_t *f, PyObject *parent) { children = PyList_New(0); if (children == NULL) { PyErr_NoMemory(); return 1;} - folder = Py_BuildValue("{s:k,s:k,s:k,s:s,s:N}", + folder = Py_BuildValue("{s:k,s:k,s:k,s:s,s:O,s:N}", "id", f->folder_id, "parent_id", f->parent_id, "storage_id", f->storage_id, "name", f->name, + "is_folder", Py_True, "children", children); if (folder == NULL) return 1; PyList_Append(parent, folder); diff --git a/src/calibre/devices/mtp/windows/content_enumeration.cpp b/src/calibre/devices/mtp/windows/content_enumeration.cpp index 29d227d710..70cead4893 100644 --- a/src/calibre/devices/mtp/windows/content_enumeration.cpp +++ b/src/calibre/devices/mtp/windows/content_enumeration.cpp @@ -28,7 +28,7 @@ static IPortableDeviceKeyCollection* create_filesystem_properties_collection() { ADDPROP(WPD_OBJECT_PARENT_ID); ADDPROP(WPD_OBJECT_PERSISTENT_UNIQUE_ID); ADDPROP(WPD_OBJECT_NAME); - ADDPROP(WPD_OBJECT_SYNC_ID); + // ADDPROP(WPD_OBJECT_SYNC_ID); ADDPROP(WPD_OBJECT_ISSYSTEM); ADDPROP(WPD_OBJECT_ISHIDDEN); ADDPROP(WPD_OBJECT_CAN_DELETE); @@ -93,7 +93,7 @@ static void set_properties(PyObject *obj, IPortableDeviceValues *values) { set_string_property(obj, WPD_OBJECT_PARENT_ID, "parent_id", values); set_string_property(obj, WPD_OBJECT_NAME, "name", values); - set_string_property(obj, WPD_OBJECT_SYNC_ID, "sync_id", values); + // set_string_property(obj, WPD_OBJECT_SYNC_ID, "sync_id", values); set_string_property(obj, WPD_OBJECT_PERSISTENT_UNIQUE_ID, "persistent_id", values); set_bool_property(obj, WPD_OBJECT_ISHIDDEN, "is_hidden", values); diff --git a/src/calibre/devices/mtp/windows/driver.py b/src/calibre/devices/mtp/windows/driver.py index 9d0ac55e82..51f5bfd60d 100644 --- a/src/calibre/devices/mtp/windows/driver.py +++ b/src/calibre/devices/mtp/windows/driver.py @@ -9,19 +9,27 @@ __docformat__ = 'restructuredtext en' import time, threading from functools import wraps +from future_builtins import zip +from itertools import chain from calibre import as_unicode, prints from calibre.constants import plugins, __appname__, numeric_version from calibre.ptempfile import SpooledTemporaryFile from calibre.devices.errors import OpenFailed from calibre.devices.mtp.base import MTPDeviceBase +from calibre.devices.mtp.filesystem_cache import FilesystemCache + +class ThreadingViolation(Exception): + + def __init__(self): + Exception.__init__('You cannot use the MTP driver from a thread other than the ' + ' thread in which startup() was called') def same_thread(func): @wraps(func) def check_thread(self, *args, **kwargs): if self.start_thread is not threading.current_thread(): - raise Exception('You cannot use %s from a thread other than the ' - ' thread in which startup() was called'%self.__class__.__name__) + raise ThreadingViolation() return func(self, *args, **kwargs) return check_thread @@ -42,6 +50,7 @@ class MTP_DEVICE(MTPDeviceBase): self.wpd = self.wpd_error = None self._main_id = self._carda_id = self._cardb_id = None self.start_thread = None + self._filesystem_cache = None def startup(self): self.start_thread = threading.current_thread() @@ -59,7 +68,7 @@ class MTP_DEVICE(MTPDeviceBase): @same_thread def shutdown(self): - self.dev = self.filesystem_cache = self.start_thread = None + self.dev = self._filesystem_cache = self.start_thread = None if self.wpd is not None: self.wpd.uninit() @@ -130,11 +139,33 @@ class MTP_DEVICE(MTPDeviceBase): return True + @property + def filesystem_cache(self): + if self._filesystem_cache is None: + ts = self.total_space() + all_storage = [] + items = [] + for storage_id, capacity in zip([self._main_id, self._carda_id, + self._cardb_id], ts): + if storage_id is None: continue + name = _('Unknown') + for s in self.dev.data['storage']: + if s['id'] == storage_id: + name = s['name'] + break + storage = {'id':storage_id, 'size':capacity, 'name':name, + 'is_folder':True} + id_map = self.dev.get_filesystem(storage_id) + all_storage.append(storage) + items.append(id_map.itervalues()) + self._filesystem_cache = FilesystemCache(all_storage, chain(*items)) + return self._filesystem_cache + @same_thread def post_yank_cleanup(self): self.currently_connected_pnp_id = self.current_friendly_name = None self._main_id = self._carda_id = self._cardb_id = None - self.dev = self.filesystem_cache = None + self.dev = self._filesystem_cache = None @same_thread def eject(self): @@ -142,11 +173,11 @@ class MTP_DEVICE(MTPDeviceBase): self.ejected_devices.add(self.currently_connected_pnp_id) self.currently_connected_pnp_id = self.current_friendly_name = None self._main_id = self._carda_id = self._cardb_id = None - self.dev = self.filesystem_cache = None + self.dev = self._filesystem_cache = None @same_thread def open(self, connected_device, library_uuid): - self.dev = self.filesystem_cache = None + self.dev = self._filesystem_cache = None try: self.dev = self.wpd.Device(connected_device) except self.wpd.WPDError: diff --git a/src/calibre/devices/mtp/windows/remote.py b/src/calibre/devices/mtp/windows/remote.py index 6f883f8baf..a3686ce88c 100644 --- a/src/calibre/devices/mtp/windows/remote.py +++ b/src/calibre/devices/mtp/windows/remote.py @@ -70,10 +70,10 @@ def main(): print ('Connected to:', dev.get_gui_name()) print ('Total space', dev.total_space()) print ('Free space', dev.free_space()) - # pprint.pprint(dev.dev.get_filesystem(dev._main_id)) - print ('Fetching file: oFF (198214 bytes)') - stream = dev.get_file('oFF') - print ("Fetched size: ", stream.tell()) + dev.filesystem_cache.dump() + # print ('Fetching file: oFF (198214 bytes)') + # stream = dev.get_file('oFF') + # print ("Fetched size: ", stream.tell()) finally: dev.shutdown()