mirror of
https://github.com/kovidgoyal/calibre.git
synced 2025-07-09 03:04:10 -04:00
MTP: Implement getting list of books and their metadata from device
This commit is contained in:
parent
944fb4a7fe
commit
359813d7eb
@ -7,10 +7,17 @@ __license__ = 'GPL v3'
|
|||||||
__copyright__ = '2012, Kovid Goyal <kovid at kovidgoyal.net>'
|
__copyright__ = '2012, Kovid Goyal <kovid at kovidgoyal.net>'
|
||||||
__docformat__ = 'restructuredtext en'
|
__docformat__ = 'restructuredtext en'
|
||||||
|
|
||||||
from functools import wraps
|
import re
|
||||||
|
from functools import wraps, partial
|
||||||
|
|
||||||
|
from calibre import prints
|
||||||
|
from calibre.constants import DEBUG
|
||||||
from calibre.devices.interface import DevicePlugin
|
from calibre.devices.interface import DevicePlugin
|
||||||
|
|
||||||
|
def debug(*args, **kwargs):
|
||||||
|
if DEBUG:
|
||||||
|
prints('MTP:', *args, **kwargs)
|
||||||
|
|
||||||
def synchronous(func):
|
def synchronous(func):
|
||||||
@wraps(func)
|
@wraps(func)
|
||||||
def synchronizer(self, *args, **kwargs):
|
def synchronizer(self, *args, **kwargs):
|
||||||
@ -53,4 +60,28 @@ class MTPDeviceBase(DevicePlugin):
|
|||||||
# return False
|
# return False
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
def build_template_regexp(self):
|
||||||
|
return None
|
||||||
|
# TODO: Implement this
|
||||||
|
def replfunc(match, seen=None):
|
||||||
|
v = match.group(1)
|
||||||
|
if v in ['authors', 'author_sort']:
|
||||||
|
v = 'author'
|
||||||
|
if v in ('title', 'series', 'series_index', 'isbn', 'author'):
|
||||||
|
if v not in seen:
|
||||||
|
seen.add(v)
|
||||||
|
return '(?P<' + v + '>.+?)'
|
||||||
|
return '(.+?)'
|
||||||
|
s = set()
|
||||||
|
f = partial(replfunc, seen=s)
|
||||||
|
template = None
|
||||||
|
try:
|
||||||
|
template = self.save_template().rpartition('/')[2]
|
||||||
|
return re.compile(re.sub('{([^}]*)}', f, template) + '([_\d]*$)')
|
||||||
|
except:
|
||||||
|
prints(u'Failed to parse template: %r'%template)
|
||||||
|
template = u'{title} - {authors}'
|
||||||
|
return re.compile(re.sub('{([^}]*)}', f, template) + '([_\d]*$)')
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
38
src/calibre/devices/mtp/books.py
Normal file
38
src/calibre/devices/mtp/books.py
Normal file
@ -0,0 +1,38 @@
|
|||||||
|
#!/usr/bin/env python
|
||||||
|
# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:fdm=marker:ai
|
||||||
|
from __future__ import (unicode_literals, division, absolute_import,
|
||||||
|
print_function)
|
||||||
|
|
||||||
|
__license__ = 'GPL v3'
|
||||||
|
__copyright__ = '2012, Kovid Goyal <kovid at kovidgoyal.net>'
|
||||||
|
__docformat__ = 'restructuredtext en'
|
||||||
|
|
||||||
|
import os
|
||||||
|
|
||||||
|
from calibre.devices.interface import BookList as BL
|
||||||
|
from calibre.ebooks.metadata.book.base import Metadata
|
||||||
|
from calibre.ebooks.metadata.book.json_codec import JsonCodec
|
||||||
|
|
||||||
|
class BookList(BL):
|
||||||
|
|
||||||
|
def __init__(self, storage_id):
|
||||||
|
self.storage_id = storage_id
|
||||||
|
|
||||||
|
def supports_collections(self):
|
||||||
|
return False
|
||||||
|
|
||||||
|
class Book(Metadata):
|
||||||
|
|
||||||
|
def __init__(self, storage_id, lpath, other=None):
|
||||||
|
Metadata.__init__(self, _('Unknown'), other=other)
|
||||||
|
self.storage_id, self.lpath = storage_id, lpath
|
||||||
|
self.lpath = self.lpath.replace(os.sep, '/')
|
||||||
|
self.mtp_relpath = tuple([icu_lower(x) for x in self.lpath.split('/')])
|
||||||
|
|
||||||
|
def matches_file(self, mtp_file):
|
||||||
|
return (self.storage_id == mtp_file.storage_id and
|
||||||
|
self.mtp_relpath == mtp_file.mtp_relpath)
|
||||||
|
|
||||||
|
class JSONCodec(JsonCodec):
|
||||||
|
pass
|
||||||
|
|
@ -7,10 +7,13 @@ __license__ = 'GPL v3'
|
|||||||
__copyright__ = '2012, Kovid Goyal <kovid at kovidgoyal.net>'
|
__copyright__ = '2012, Kovid Goyal <kovid at kovidgoyal.net>'
|
||||||
__docformat__ = 'restructuredtext en'
|
__docformat__ = 'restructuredtext en'
|
||||||
|
|
||||||
import json, pprint
|
import json, pprint, traceback
|
||||||
from io import BytesIO
|
from io import BytesIO
|
||||||
|
|
||||||
|
from calibre import prints
|
||||||
from calibre.constants import iswindows, numeric_version
|
from calibre.constants import iswindows, numeric_version
|
||||||
|
from calibre.devices.mtp.base import debug
|
||||||
|
from calibre.ptempfile import SpooledTemporaryFile
|
||||||
from calibre.utils.config import from_json, to_json
|
from calibre.utils.config import from_json, to_json
|
||||||
from calibre.utils.date import now, isoformat
|
from calibre.utils.date import now, isoformat
|
||||||
|
|
||||||
@ -25,6 +28,7 @@ class MTP_DEVICE(BASE):
|
|||||||
|
|
||||||
METADATA_CACHE = 'metadata.calibre'
|
METADATA_CACHE = 'metadata.calibre'
|
||||||
DRIVEINFO = 'driveinfo.calibre'
|
DRIVEINFO = 'driveinfo.calibre'
|
||||||
|
CAN_SET_METADATA = []
|
||||||
|
|
||||||
def _update_drive_info(self, storage, location_code, name=None):
|
def _update_drive_info(self, storage, location_code, name=None):
|
||||||
import uuid
|
import uuid
|
||||||
@ -81,6 +85,98 @@ class MTP_DEVICE(BASE):
|
|||||||
self._update_drive_info(self.filesystem_cache.storage(sid),
|
self._update_drive_info(self.filesystem_cache.storage(sid),
|
||||||
location_code, name=name)
|
location_code, name=name)
|
||||||
|
|
||||||
|
def books(self, oncard=None, end_session=True):
|
||||||
|
from calibre.devices.mtp.books import JSONCodec
|
||||||
|
from calibre.devices.mtp.books import BookList, Book
|
||||||
|
sid = {'carda':self._carda_id, 'cardb':self._cardb_id}.get(oncard,
|
||||||
|
self._main_id)
|
||||||
|
if sid is None:
|
||||||
|
return BookList(None)
|
||||||
|
|
||||||
|
bl = BookList(sid)
|
||||||
|
# If True then there is a mismatch between the ebooks on the device and
|
||||||
|
# the metadata cache
|
||||||
|
need_sync = False
|
||||||
|
all_books = list(self.filesystem_cache.iterebooks(sid))
|
||||||
|
steps = len(all_books) + 2
|
||||||
|
count = 0
|
||||||
|
|
||||||
|
self.report_progress(0, _('Reading metadata from device'))
|
||||||
|
# Read the cache if it exists
|
||||||
|
storage = self.filesystem_cache.storage(sid)
|
||||||
|
cache = storage.find_path((self.METADATA_CACHE,))
|
||||||
|
if cache is not None:
|
||||||
|
json_codec = JSONCodec()
|
||||||
|
try:
|
||||||
|
stream = self.get_file(cache)
|
||||||
|
json_codec.decode_from_file(stream, bl, Book, sid)
|
||||||
|
except:
|
||||||
|
need_sync = True
|
||||||
|
|
||||||
|
relpath_cache = {b.mtp_relpath:i for i, b in enumerate(bl)}
|
||||||
|
|
||||||
|
for mtp_file in all_books:
|
||||||
|
count += 1
|
||||||
|
relpath = mtp_file.mtp_relpath
|
||||||
|
idx = relpath_cache.get(relpath, None)
|
||||||
|
if idx is not None:
|
||||||
|
cached_metadata = bl[idx]
|
||||||
|
del relpath_cache[relpath]
|
||||||
|
if cached_metadata.size == mtp_file.size:
|
||||||
|
debug('Using cached metadata for',
|
||||||
|
'/'.join(mtp_file.full_path))
|
||||||
|
continue # No need to update metadata
|
||||||
|
book = cached_metadata
|
||||||
|
else:
|
||||||
|
book = Book(sid, '/'.join(relpath))
|
||||||
|
bl.append(book)
|
||||||
|
|
||||||
|
need_sync = True
|
||||||
|
self.report_progress(count/steps, _('Reading metadata from %s')%
|
||||||
|
('/'.join(relpath)))
|
||||||
|
try:
|
||||||
|
book.smart_update(self.read_file_metadata(mtp_file))
|
||||||
|
debug('Read metadata for', '/'.join(mtp_file.full_path))
|
||||||
|
except:
|
||||||
|
prints('Failed to read metadata from',
|
||||||
|
'/'.join(mtp_file.full_path))
|
||||||
|
traceback.print_exc()
|
||||||
|
book.size = mtp_file.size
|
||||||
|
|
||||||
|
# Remove books in the cache that no longer exist
|
||||||
|
for idx in sorted(relpath_cache.itervalues(), reverse=True):
|
||||||
|
del bl[idx]
|
||||||
|
need_sync = True
|
||||||
|
|
||||||
|
if need_sync:
|
||||||
|
self.report_progress(count/steps, _('Updating metadata cache on device'))
|
||||||
|
self.write_metadata_cache(storage, bl)
|
||||||
|
self.report_progress(1, _('Finished reading metadata from device'))
|
||||||
|
|
||||||
|
def read_file_metadata(self, mtp_file):
|
||||||
|
from calibre.ebooks.metadata.meta import get_metadata
|
||||||
|
from calibre.customize.ui import quick_metadata
|
||||||
|
ext = mtp_file.name.rpartition('.')[-1].lower()
|
||||||
|
stream = self.get_file(mtp_file)
|
||||||
|
with quick_metadata:
|
||||||
|
return get_metadata(stream, stream_type=ext,
|
||||||
|
force_read_metadata=True,
|
||||||
|
pattern=self.build_template_regexp())
|
||||||
|
|
||||||
|
def write_metadata_cache(self, storage, bl):
|
||||||
|
from calibre.devices.mtp.books import JSONCodec
|
||||||
|
|
||||||
|
if bl.storage_id != storage.storage_id:
|
||||||
|
# Just a sanity check, should never happen
|
||||||
|
return
|
||||||
|
|
||||||
|
json_codec = JSONCodec()
|
||||||
|
stream = SpooledTemporaryFile(10*(1024**2))
|
||||||
|
json_codec.encode_to_file(stream, bl)
|
||||||
|
size = stream.tell()
|
||||||
|
stream.seek(0)
|
||||||
|
self.put_file(storage, self.METADATA_CACHE, stream, size)
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
dev = MTP_DEVICE(None)
|
dev = MTP_DEVICE(None)
|
||||||
dev.startup()
|
dev.startup()
|
||||||
@ -92,8 +188,9 @@ if __name__ == '__main__':
|
|||||||
cd = dev.detect_managed_devices(devs)
|
cd = dev.detect_managed_devices(devs)
|
||||||
if cd is None:
|
if cd is None:
|
||||||
raise ValueError('Failed to detect MTP device')
|
raise ValueError('Failed to detect MTP device')
|
||||||
|
dev.set_progress_reporter(prints)
|
||||||
dev.open(cd, None)
|
dev.open(cd, None)
|
||||||
pprint.pprint(dev.get_device_information())
|
dev.books()
|
||||||
finally:
|
finally:
|
||||||
dev.shutdown()
|
dev.shutdown()
|
||||||
|
|
||||||
|
@ -14,6 +14,9 @@ from future_builtins import map
|
|||||||
|
|
||||||
from calibre import human_readable, prints, force_unicode
|
from calibre import human_readable, prints, force_unicode
|
||||||
from calibre.utils.icu import sort_key, lower
|
from calibre.utils.icu import sort_key, lower
|
||||||
|
from calibre.ebooks import BOOK_EXTENSIONS
|
||||||
|
|
||||||
|
bexts = frozenset(BOOK_EXTENSIONS)
|
||||||
|
|
||||||
class FileOrFolder(object):
|
class FileOrFolder(object):
|
||||||
|
|
||||||
@ -50,6 +53,9 @@ class FileOrFolder(object):
|
|||||||
if self.storage_id == self.object_id:
|
if self.storage_id == self.object_id:
|
||||||
self.storage_prefix = 'mtp:::%s:::'%self.persistent_id
|
self.storage_prefix = 'mtp:::%s:::'%self.persistent_id
|
||||||
|
|
||||||
|
self.is_ebook = (not self.is_folder and
|
||||||
|
self.name.rpartition('.')[-1].lower() in bexts)
|
||||||
|
|
||||||
def __repr__(self):
|
def __repr__(self):
|
||||||
name = 'Folder' if self.is_folder else 'File'
|
name = 'Folder' if self.is_folder else 'File'
|
||||||
try:
|
try:
|
||||||
@ -147,6 +153,9 @@ class FileOrFolder(object):
|
|||||||
parent = c
|
parent = c
|
||||||
return parent
|
return parent
|
||||||
|
|
||||||
|
@property
|
||||||
|
def mtp_relpath(self):
|
||||||
|
return tuple(x.lower() for x in self.full_path[1:])
|
||||||
|
|
||||||
class FilesystemCache(object):
|
class FilesystemCache(object):
|
||||||
|
|
||||||
@ -192,4 +201,9 @@ class FilesystemCache(object):
|
|||||||
if e.storage_id == storage_id:
|
if e.storage_id == storage_id:
|
||||||
return e
|
return e
|
||||||
|
|
||||||
|
def iterebooks(self, storage_id):
|
||||||
|
for x in self.id_map.itervalues():
|
||||||
|
if x.storage_id == storage_id and x.is_ebook:
|
||||||
|
yield x
|
||||||
|
|
||||||
|
|
||||||
|
@ -17,7 +17,6 @@ from calibre.constants import plugins
|
|||||||
from calibre.ptempfile import SpooledTemporaryFile
|
from calibre.ptempfile import SpooledTemporaryFile
|
||||||
from calibre.devices.errors import OpenFailed, DeviceError
|
from calibre.devices.errors import OpenFailed, DeviceError
|
||||||
from calibre.devices.mtp.base import MTPDeviceBase, synchronous
|
from calibre.devices.mtp.base import MTPDeviceBase, synchronous
|
||||||
from calibre.devices.mtp.filesystem_cache import FilesystemCache
|
|
||||||
|
|
||||||
MTPDevice = namedtuple('MTPDevice', 'busnum devnum vendor_id product_id '
|
MTPDevice = namedtuple('MTPDevice', 'busnum devnum vendor_id product_id '
|
||||||
'bcd serial manufacturer product')
|
'bcd serial manufacturer product')
|
||||||
@ -175,6 +174,7 @@ class MTP_DEVICE(MTPDeviceBase):
|
|||||||
@property
|
@property
|
||||||
def filesystem_cache(self):
|
def filesystem_cache(self):
|
||||||
if self._filesystem_cache is None:
|
if self._filesystem_cache is None:
|
||||||
|
from calibre.devices.mtp.filesystem_cache import FilesystemCache
|
||||||
with self.lock:
|
with self.lock:
|
||||||
storage, all_items, all_errs = [], [], []
|
storage, all_items, all_errs = [], [], []
|
||||||
for sid, capacity in zip([self._main_id, self._carda_id,
|
for sid, capacity in zip([self._main_id, self._carda_id,
|
||||||
|
@ -17,7 +17,6 @@ from calibre.constants import plugins, __appname__, numeric_version
|
|||||||
from calibre.ptempfile import SpooledTemporaryFile
|
from calibre.ptempfile import SpooledTemporaryFile
|
||||||
from calibre.devices.errors import OpenFailed, DeviceError
|
from calibre.devices.errors import OpenFailed, DeviceError
|
||||||
from calibre.devices.mtp.base import MTPDeviceBase
|
from calibre.devices.mtp.base import MTPDeviceBase
|
||||||
from calibre.devices.mtp.filesystem_cache import FilesystemCache
|
|
||||||
|
|
||||||
class ThreadingViolation(Exception):
|
class ThreadingViolation(Exception):
|
||||||
|
|
||||||
@ -143,6 +142,7 @@ class MTP_DEVICE(MTPDeviceBase):
|
|||||||
@property
|
@property
|
||||||
def filesystem_cache(self):
|
def filesystem_cache(self):
|
||||||
if self._filesystem_cache is None:
|
if self._filesystem_cache is None:
|
||||||
|
from calibre.devices.mtp.filesystem_cache import FilesystemCache
|
||||||
ts = self.total_space()
|
ts = self.total_space()
|
||||||
all_storage = []
|
all_storage = []
|
||||||
items = []
|
items = []
|
||||||
|
Loading…
x
Reference in New Issue
Block a user