MTP: Implement getting list of books and their metadata from device

This commit is contained in:
Kovid Goyal 2012-08-31 18:45:27 +05:30
parent 944fb4a7fe
commit 359813d7eb
6 changed files with 185 additions and 5 deletions

View File

@ -7,10 +7,17 @@ __license__ = 'GPL v3'
__copyright__ = '2012, Kovid Goyal <kovid at kovidgoyal.net>' __copyright__ = '2012, Kovid Goyal <kovid at kovidgoyal.net>'
__docformat__ = 'restructuredtext en' __docformat__ = 'restructuredtext en'
from functools import wraps import re
from functools import wraps, partial
from calibre import prints
from calibre.constants import DEBUG
from calibre.devices.interface import DevicePlugin from calibre.devices.interface import DevicePlugin
def debug(*args, **kwargs):
if DEBUG:
prints('MTP:', *args, **kwargs)
def synchronous(func): def synchronous(func):
@wraps(func) @wraps(func)
def synchronizer(self, *args, **kwargs): def synchronizer(self, *args, **kwargs):
@ -53,4 +60,28 @@ class MTPDeviceBase(DevicePlugin):
# return False # return False
return False return False
def build_template_regexp(self):
return None
# TODO: Implement this
def replfunc(match, seen=None):
v = match.group(1)
if v in ['authors', 'author_sort']:
v = 'author'
if v in ('title', 'series', 'series_index', 'isbn', 'author'):
if v not in seen:
seen.add(v)
return '(?P<' + v + '>.+?)'
return '(.+?)'
s = set()
f = partial(replfunc, seen=s)
template = None
try:
template = self.save_template().rpartition('/')[2]
return re.compile(re.sub('{([^}]*)}', f, template) + '([_\d]*$)')
except:
prints(u'Failed to parse template: %r'%template)
template = u'{title} - {authors}'
return re.compile(re.sub('{([^}]*)}', f, template) + '([_\d]*$)')

View File

@ -0,0 +1,38 @@
#!/usr/bin/env python
# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:fdm=marker:ai
from __future__ import (unicode_literals, division, absolute_import,
print_function)
__license__ = 'GPL v3'
__copyright__ = '2012, Kovid Goyal <kovid at kovidgoyal.net>'
__docformat__ = 'restructuredtext en'
import os
from calibre.devices.interface import BookList as BL
from calibre.ebooks.metadata.book.base import Metadata
from calibre.ebooks.metadata.book.json_codec import JsonCodec
class BookList(BL):
def __init__(self, storage_id):
self.storage_id = storage_id
def supports_collections(self):
return False
class Book(Metadata):
def __init__(self, storage_id, lpath, other=None):
Metadata.__init__(self, _('Unknown'), other=other)
self.storage_id, self.lpath = storage_id, lpath
self.lpath = self.lpath.replace(os.sep, '/')
self.mtp_relpath = tuple([icu_lower(x) for x in self.lpath.split('/')])
def matches_file(self, mtp_file):
return (self.storage_id == mtp_file.storage_id and
self.mtp_relpath == mtp_file.mtp_relpath)
class JSONCodec(JsonCodec):
pass

View File

@ -7,10 +7,13 @@ __license__ = 'GPL v3'
__copyright__ = '2012, Kovid Goyal <kovid at kovidgoyal.net>' __copyright__ = '2012, Kovid Goyal <kovid at kovidgoyal.net>'
__docformat__ = 'restructuredtext en' __docformat__ = 'restructuredtext en'
import json, pprint import json, pprint, traceback
from io import BytesIO from io import BytesIO
from calibre import prints
from calibre.constants import iswindows, numeric_version from calibre.constants import iswindows, numeric_version
from calibre.devices.mtp.base import debug
from calibre.ptempfile import SpooledTemporaryFile
from calibre.utils.config import from_json, to_json from calibre.utils.config import from_json, to_json
from calibre.utils.date import now, isoformat from calibre.utils.date import now, isoformat
@ -25,6 +28,7 @@ class MTP_DEVICE(BASE):
METADATA_CACHE = 'metadata.calibre' METADATA_CACHE = 'metadata.calibre'
DRIVEINFO = 'driveinfo.calibre' DRIVEINFO = 'driveinfo.calibre'
CAN_SET_METADATA = []
def _update_drive_info(self, storage, location_code, name=None): def _update_drive_info(self, storage, location_code, name=None):
import uuid import uuid
@ -81,6 +85,98 @@ class MTP_DEVICE(BASE):
self._update_drive_info(self.filesystem_cache.storage(sid), self._update_drive_info(self.filesystem_cache.storage(sid),
location_code, name=name) location_code, name=name)
def books(self, oncard=None, end_session=True):
from calibre.devices.mtp.books import JSONCodec
from calibre.devices.mtp.books import BookList, Book
sid = {'carda':self._carda_id, 'cardb':self._cardb_id}.get(oncard,
self._main_id)
if sid is None:
return BookList(None)
bl = BookList(sid)
# If True then there is a mismatch between the ebooks on the device and
# the metadata cache
need_sync = False
all_books = list(self.filesystem_cache.iterebooks(sid))
steps = len(all_books) + 2
count = 0
self.report_progress(0, _('Reading metadata from device'))
# Read the cache if it exists
storage = self.filesystem_cache.storage(sid)
cache = storage.find_path((self.METADATA_CACHE,))
if cache is not None:
json_codec = JSONCodec()
try:
stream = self.get_file(cache)
json_codec.decode_from_file(stream, bl, Book, sid)
except:
need_sync = True
relpath_cache = {b.mtp_relpath:i for i, b in enumerate(bl)}
for mtp_file in all_books:
count += 1
relpath = mtp_file.mtp_relpath
idx = relpath_cache.get(relpath, None)
if idx is not None:
cached_metadata = bl[idx]
del relpath_cache[relpath]
if cached_metadata.size == mtp_file.size:
debug('Using cached metadata for',
'/'.join(mtp_file.full_path))
continue # No need to update metadata
book = cached_metadata
else:
book = Book(sid, '/'.join(relpath))
bl.append(book)
need_sync = True
self.report_progress(count/steps, _('Reading metadata from %s')%
('/'.join(relpath)))
try:
book.smart_update(self.read_file_metadata(mtp_file))
debug('Read metadata for', '/'.join(mtp_file.full_path))
except:
prints('Failed to read metadata from',
'/'.join(mtp_file.full_path))
traceback.print_exc()
book.size = mtp_file.size
# Remove books in the cache that no longer exist
for idx in sorted(relpath_cache.itervalues(), reverse=True):
del bl[idx]
need_sync = True
if need_sync:
self.report_progress(count/steps, _('Updating metadata cache on device'))
self.write_metadata_cache(storage, bl)
self.report_progress(1, _('Finished reading metadata from device'))
def read_file_metadata(self, mtp_file):
from calibre.ebooks.metadata.meta import get_metadata
from calibre.customize.ui import quick_metadata
ext = mtp_file.name.rpartition('.')[-1].lower()
stream = self.get_file(mtp_file)
with quick_metadata:
return get_metadata(stream, stream_type=ext,
force_read_metadata=True,
pattern=self.build_template_regexp())
def write_metadata_cache(self, storage, bl):
from calibre.devices.mtp.books import JSONCodec
if bl.storage_id != storage.storage_id:
# Just a sanity check, should never happen
return
json_codec = JSONCodec()
stream = SpooledTemporaryFile(10*(1024**2))
json_codec.encode_to_file(stream, bl)
size = stream.tell()
stream.seek(0)
self.put_file(storage, self.METADATA_CACHE, stream, size)
if __name__ == '__main__': if __name__ == '__main__':
dev = MTP_DEVICE(None) dev = MTP_DEVICE(None)
dev.startup() dev.startup()
@ -92,8 +188,9 @@ if __name__ == '__main__':
cd = dev.detect_managed_devices(devs) cd = dev.detect_managed_devices(devs)
if cd is None: if cd is None:
raise ValueError('Failed to detect MTP device') raise ValueError('Failed to detect MTP device')
dev.set_progress_reporter(prints)
dev.open(cd, None) dev.open(cd, None)
pprint.pprint(dev.get_device_information()) dev.books()
finally: finally:
dev.shutdown() dev.shutdown()

View File

@ -14,6 +14,9 @@ from future_builtins import map
from calibre import human_readable, prints, force_unicode from calibre import human_readable, prints, force_unicode
from calibre.utils.icu import sort_key, lower from calibre.utils.icu import sort_key, lower
from calibre.ebooks import BOOK_EXTENSIONS
bexts = frozenset(BOOK_EXTENSIONS)
class FileOrFolder(object): class FileOrFolder(object):
@ -50,6 +53,9 @@ class FileOrFolder(object):
if self.storage_id == self.object_id: if self.storage_id == self.object_id:
self.storage_prefix = 'mtp:::%s:::'%self.persistent_id self.storage_prefix = 'mtp:::%s:::'%self.persistent_id
self.is_ebook = (not self.is_folder and
self.name.rpartition('.')[-1].lower() in bexts)
def __repr__(self): def __repr__(self):
name = 'Folder' if self.is_folder else 'File' name = 'Folder' if self.is_folder else 'File'
try: try:
@ -147,6 +153,9 @@ class FileOrFolder(object):
parent = c parent = c
return parent return parent
@property
def mtp_relpath(self):
return tuple(x.lower() for x in self.full_path[1:])
class FilesystemCache(object): class FilesystemCache(object):
@ -192,4 +201,9 @@ class FilesystemCache(object):
if e.storage_id == storage_id: if e.storage_id == storage_id:
return e return e
def iterebooks(self, storage_id):
for x in self.id_map.itervalues():
if x.storage_id == storage_id and x.is_ebook:
yield x

View File

@ -17,7 +17,6 @@ from calibre.constants import plugins
from calibre.ptempfile import SpooledTemporaryFile from calibre.ptempfile import SpooledTemporaryFile
from calibre.devices.errors import OpenFailed, DeviceError from calibre.devices.errors import OpenFailed, DeviceError
from calibre.devices.mtp.base import MTPDeviceBase, synchronous from calibre.devices.mtp.base import MTPDeviceBase, synchronous
from calibre.devices.mtp.filesystem_cache import FilesystemCache
MTPDevice = namedtuple('MTPDevice', 'busnum devnum vendor_id product_id ' MTPDevice = namedtuple('MTPDevice', 'busnum devnum vendor_id product_id '
'bcd serial manufacturer product') 'bcd serial manufacturer product')
@ -175,6 +174,7 @@ class MTP_DEVICE(MTPDeviceBase):
@property @property
def filesystem_cache(self): def filesystem_cache(self):
if self._filesystem_cache is None: if self._filesystem_cache is None:
from calibre.devices.mtp.filesystem_cache import FilesystemCache
with self.lock: with self.lock:
storage, all_items, all_errs = [], [], [] storage, all_items, all_errs = [], [], []
for sid, capacity in zip([self._main_id, self._carda_id, for sid, capacity in zip([self._main_id, self._carda_id,

View File

@ -17,7 +17,6 @@ from calibre.constants import plugins, __appname__, numeric_version
from calibre.ptempfile import SpooledTemporaryFile from calibre.ptempfile import SpooledTemporaryFile
from calibre.devices.errors import OpenFailed, DeviceError from calibre.devices.errors import OpenFailed, DeviceError
from calibre.devices.mtp.base import MTPDeviceBase from calibre.devices.mtp.base import MTPDeviceBase
from calibre.devices.mtp.filesystem_cache import FilesystemCache
class ThreadingViolation(Exception): class ThreadingViolation(Exception):
@ -143,6 +142,7 @@ class MTP_DEVICE(MTPDeviceBase):
@property @property
def filesystem_cache(self): def filesystem_cache(self):
if self._filesystem_cache is None: if self._filesystem_cache is None:
from calibre.devices.mtp.filesystem_cache import FilesystemCache
ts = self.total_space() ts = self.total_space()
all_storage = [] all_storage = []
items = [] items = []