diff --git a/src/calibre/devices/mtp/base.py b/src/calibre/devices/mtp/base.py index 4f8bbc991f..3b71e40619 100644 --- a/src/calibre/devices/mtp/base.py +++ b/src/calibre/devices/mtp/base.py @@ -7,10 +7,17 @@ __license__ = 'GPL v3' __copyright__ = '2012, Kovid Goyal ' __docformat__ = 'restructuredtext en' -from functools import wraps +import re +from functools import wraps, partial +from calibre import prints +from calibre.constants import DEBUG from calibre.devices.interface import DevicePlugin +def debug(*args, **kwargs): + if DEBUG: + prints('MTP:', *args, **kwargs) + def synchronous(func): @wraps(func) def synchronizer(self, *args, **kwargs): @@ -53,4 +60,28 @@ class MTPDeviceBase(DevicePlugin): # return False return False + def build_template_regexp(self): + return None + # TODO: Implement this + def replfunc(match, seen=None): + v = match.group(1) + if v in ['authors', 'author_sort']: + v = 'author' + if v in ('title', 'series', 'series_index', 'isbn', 'author'): + if v not in seen: + seen.add(v) + return '(?P<' + v + '>.+?)' + return '(.+?)' + s = set() + f = partial(replfunc, seen=s) + template = None + try: + template = self.save_template().rpartition('/')[2] + return re.compile(re.sub('{([^}]*)}', f, template) + '([_\d]*$)') + except: + prints(u'Failed to parse template: %r'%template) + template = u'{title} - {authors}' + return re.compile(re.sub('{([^}]*)}', f, template) + '([_\d]*$)') + + diff --git a/src/calibre/devices/mtp/books.py b/src/calibre/devices/mtp/books.py new file mode 100644 index 0000000000..c02923702e --- /dev/null +++ b/src/calibre/devices/mtp/books.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:fdm=marker:ai +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +__license__ = 'GPL v3' +__copyright__ = '2012, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + +import os + +from calibre.devices.interface import BookList as BL +from calibre.ebooks.metadata.book.base import Metadata +from calibre.ebooks.metadata.book.json_codec import JsonCodec + +class BookList(BL): + + def __init__(self, storage_id): + self.storage_id = storage_id + + def supports_collections(self): + return False + +class Book(Metadata): + + def __init__(self, storage_id, lpath, other=None): + Metadata.__init__(self, _('Unknown'), other=other) + self.storage_id, self.lpath = storage_id, lpath + self.lpath = self.lpath.replace(os.sep, '/') + self.mtp_relpath = tuple([icu_lower(x) for x in self.lpath.split('/')]) + + def matches_file(self, mtp_file): + return (self.storage_id == mtp_file.storage_id and + self.mtp_relpath == mtp_file.mtp_relpath) + +class JSONCodec(JsonCodec): + pass + diff --git a/src/calibre/devices/mtp/driver.py b/src/calibre/devices/mtp/driver.py index c3e34a2be5..f1e9bdbcff 100644 --- a/src/calibre/devices/mtp/driver.py +++ b/src/calibre/devices/mtp/driver.py @@ -7,10 +7,13 @@ __license__ = 'GPL v3' __copyright__ = '2012, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import json, pprint +import json, pprint, traceback from io import BytesIO +from calibre import prints from calibre.constants import iswindows, numeric_version +from calibre.devices.mtp.base import debug +from calibre.ptempfile import SpooledTemporaryFile from calibre.utils.config import from_json, to_json from calibre.utils.date import now, isoformat @@ -25,6 +28,7 @@ class MTP_DEVICE(BASE): METADATA_CACHE = 'metadata.calibre' DRIVEINFO = 'driveinfo.calibre' + CAN_SET_METADATA = [] def _update_drive_info(self, storage, location_code, name=None): import uuid @@ -81,6 +85,98 @@ class MTP_DEVICE(BASE): self._update_drive_info(self.filesystem_cache.storage(sid), location_code, name=name) + def books(self, oncard=None, end_session=True): + from calibre.devices.mtp.books import JSONCodec + from calibre.devices.mtp.books import BookList, Book + sid = {'carda':self._carda_id, 'cardb':self._cardb_id}.get(oncard, + self._main_id) + if sid is None: + return BookList(None) + + bl = BookList(sid) + # If True then there is a mismatch between the ebooks on the device and + # the metadata cache + need_sync = False + all_books = list(self.filesystem_cache.iterebooks(sid)) + steps = len(all_books) + 2 + count = 0 + + self.report_progress(0, _('Reading metadata from device')) + # Read the cache if it exists + storage = self.filesystem_cache.storage(sid) + cache = storage.find_path((self.METADATA_CACHE,)) + if cache is not None: + json_codec = JSONCodec() + try: + stream = self.get_file(cache) + json_codec.decode_from_file(stream, bl, Book, sid) + except: + need_sync = True + + relpath_cache = {b.mtp_relpath:i for i, b in enumerate(bl)} + + for mtp_file in all_books: + count += 1 + relpath = mtp_file.mtp_relpath + idx = relpath_cache.get(relpath, None) + if idx is not None: + cached_metadata = bl[idx] + del relpath_cache[relpath] + if cached_metadata.size == mtp_file.size: + debug('Using cached metadata for', + '/'.join(mtp_file.full_path)) + continue # No need to update metadata + book = cached_metadata + else: + book = Book(sid, '/'.join(relpath)) + bl.append(book) + + need_sync = True + self.report_progress(count/steps, _('Reading metadata from %s')% + ('/'.join(relpath))) + try: + book.smart_update(self.read_file_metadata(mtp_file)) + debug('Read metadata for', '/'.join(mtp_file.full_path)) + except: + prints('Failed to read metadata from', + '/'.join(mtp_file.full_path)) + traceback.print_exc() + book.size = mtp_file.size + + # Remove books in the cache that no longer exist + for idx in sorted(relpath_cache.itervalues(), reverse=True): + del bl[idx] + need_sync = True + + if need_sync: + self.report_progress(count/steps, _('Updating metadata cache on device')) + self.write_metadata_cache(storage, bl) + self.report_progress(1, _('Finished reading metadata from device')) + + def read_file_metadata(self, mtp_file): + from calibre.ebooks.metadata.meta import get_metadata + from calibre.customize.ui import quick_metadata + ext = mtp_file.name.rpartition('.')[-1].lower() + stream = self.get_file(mtp_file) + with quick_metadata: + return get_metadata(stream, stream_type=ext, + force_read_metadata=True, + pattern=self.build_template_regexp()) + + def write_metadata_cache(self, storage, bl): + from calibre.devices.mtp.books import JSONCodec + + if bl.storage_id != storage.storage_id: + # Just a sanity check, should never happen + return + + json_codec = JSONCodec() + stream = SpooledTemporaryFile(10*(1024**2)) + json_codec.encode_to_file(stream, bl) + size = stream.tell() + stream.seek(0) + self.put_file(storage, self.METADATA_CACHE, stream, size) + if __name__ == '__main__': dev = MTP_DEVICE(None) dev.startup() @@ -92,8 +188,9 @@ if __name__ == '__main__': cd = dev.detect_managed_devices(devs) if cd is None: raise ValueError('Failed to detect MTP device') + dev.set_progress_reporter(prints) dev.open(cd, None) - pprint.pprint(dev.get_device_information()) + dev.books() finally: dev.shutdown() diff --git a/src/calibre/devices/mtp/filesystem_cache.py b/src/calibre/devices/mtp/filesystem_cache.py index cd97c5c2ed..ba2206d191 100644 --- a/src/calibre/devices/mtp/filesystem_cache.py +++ b/src/calibre/devices/mtp/filesystem_cache.py @@ -14,6 +14,9 @@ from future_builtins import map from calibre import human_readable, prints, force_unicode from calibre.utils.icu import sort_key, lower +from calibre.ebooks import BOOK_EXTENSIONS + +bexts = frozenset(BOOK_EXTENSIONS) class FileOrFolder(object): @@ -50,6 +53,9 @@ class FileOrFolder(object): if self.storage_id == self.object_id: self.storage_prefix = 'mtp:::%s:::'%self.persistent_id + self.is_ebook = (not self.is_folder and + self.name.rpartition('.')[-1].lower() in bexts) + def __repr__(self): name = 'Folder' if self.is_folder else 'File' try: @@ -147,6 +153,9 @@ class FileOrFolder(object): parent = c return parent + @property + def mtp_relpath(self): + return tuple(x.lower() for x in self.full_path[1:]) class FilesystemCache(object): @@ -192,4 +201,9 @@ class FilesystemCache(object): if e.storage_id == storage_id: return e + def iterebooks(self, storage_id): + for x in self.id_map.itervalues(): + if x.storage_id == storage_id and x.is_ebook: + yield x + diff --git a/src/calibre/devices/mtp/unix/driver.py b/src/calibre/devices/mtp/unix/driver.py index 5d7d767a9b..9244ac198c 100644 --- a/src/calibre/devices/mtp/unix/driver.py +++ b/src/calibre/devices/mtp/unix/driver.py @@ -17,7 +17,6 @@ from calibre.constants import plugins from calibre.ptempfile import SpooledTemporaryFile from calibre.devices.errors import OpenFailed, DeviceError from calibre.devices.mtp.base import MTPDeviceBase, synchronous -from calibre.devices.mtp.filesystem_cache import FilesystemCache MTPDevice = namedtuple('MTPDevice', 'busnum devnum vendor_id product_id ' 'bcd serial manufacturer product') @@ -175,6 +174,7 @@ class MTP_DEVICE(MTPDeviceBase): @property def filesystem_cache(self): if self._filesystem_cache is None: + from calibre.devices.mtp.filesystem_cache import FilesystemCache with self.lock: storage, all_items, all_errs = [], [], [] for sid, capacity in zip([self._main_id, self._carda_id, diff --git a/src/calibre/devices/mtp/windows/driver.py b/src/calibre/devices/mtp/windows/driver.py index 0506f63054..191d69560d 100644 --- a/src/calibre/devices/mtp/windows/driver.py +++ b/src/calibre/devices/mtp/windows/driver.py @@ -17,7 +17,6 @@ from calibre.constants import plugins, __appname__, numeric_version from calibre.ptempfile import SpooledTemporaryFile from calibre.devices.errors import OpenFailed, DeviceError from calibre.devices.mtp.base import MTPDeviceBase -from calibre.devices.mtp.filesystem_cache import FilesystemCache class ThreadingViolation(Exception): @@ -143,6 +142,7 @@ class MTP_DEVICE(MTPDeviceBase): @property def filesystem_cache(self): if self._filesystem_cache is None: + from calibre.devices.mtp.filesystem_cache import FilesystemCache ts = self.total_space() all_storage = [] items = []