mirror of
https://github.com/kovidgoyal/calibre.git
synced 2025-07-09 03:04:10 -04:00
Performance enhancements for sony driver and book matching
This commit is contained in:
parent
24c8b1679b
commit
acfa27d4c5
@ -144,52 +144,67 @@ class XMLCache(object):
|
|||||||
if title+str(i) not in seen:
|
if title+str(i) not in seen:
|
||||||
title = title+str(i)
|
title = title+str(i)
|
||||||
playlist.set('title', title)
|
playlist.set('title', title)
|
||||||
|
seen.add(title)
|
||||||
break
|
break
|
||||||
else:
|
else:
|
||||||
seen.add(title)
|
seen.add(title)
|
||||||
|
|
||||||
def build_playlist_id_map(self):
|
|
||||||
debug_print('Start build_playlist_id_map')
|
|
||||||
ans = {}
|
|
||||||
self.ensure_unique_playlist_titles()
|
|
||||||
debug_print('after ensure_unique_playlist_titles')
|
|
||||||
self.prune_empty_playlists()
|
|
||||||
for i, root in self.record_roots.items():
|
|
||||||
debug_print('build_playlist_id_map loop', i)
|
|
||||||
id_map = self.build_id_map(root)
|
|
||||||
ans[i] = []
|
|
||||||
for playlist in root.xpath('//*[local-name()="playlist"]'):
|
|
||||||
items = []
|
|
||||||
for item in playlist:
|
|
||||||
id_ = item.get('id', None)
|
|
||||||
record = id_map.get(id_, None)
|
|
||||||
if record is not None:
|
|
||||||
items.append(record)
|
|
||||||
ans[i].append((playlist.get('title'), items))
|
|
||||||
debug_print('end build_playlist_id_map')
|
|
||||||
return ans
|
|
||||||
|
|
||||||
def build_id_playlist_map(self, bl_index):
|
def build_id_playlist_map(self, bl_index):
|
||||||
|
'''
|
||||||
|
Return a map of the collections in books: {lpaths: [collection names]}
|
||||||
|
'''
|
||||||
debug_print('Start build_id_playlist_map')
|
debug_print('Start build_id_playlist_map')
|
||||||
pmap = self.build_playlist_id_map()[bl_index]
|
self.ensure_unique_playlist_titles()
|
||||||
|
self.prune_empty_playlists()
|
||||||
|
debug_print('after cleaning playlists')
|
||||||
|
root = self.record_roots[bl_index]
|
||||||
|
if root is None:
|
||||||
|
return
|
||||||
|
id_map = self.build_id_map(root)
|
||||||
playlist_map = {}
|
playlist_map = {}
|
||||||
for title, records in pmap:
|
# foreach playlist, get the lpaths for the ids in it, then add to dict
|
||||||
for record in records:
|
for playlist in root.xpath('//*[local-name()="playlist"]'):
|
||||||
path = record.get('path', None)
|
name = playlist.get('title')
|
||||||
if path:
|
if name is None:
|
||||||
if path not in playlist_map:
|
debug_print('build_id_playlist_map: unnamed playlist!')
|
||||||
playlist_map[path] = []
|
continue
|
||||||
playlist_map[path].append(title)
|
for item in playlist:
|
||||||
|
# translate each id into its lpath
|
||||||
|
id_ = item.get('id', None)
|
||||||
|
if id_ is None:
|
||||||
|
debug_print('build_id_playlist_map: id_ is None!')
|
||||||
|
continue
|
||||||
|
bk = id_map.get(id_, None)
|
||||||
|
if bk is None:
|
||||||
|
debug_print('build_id_playlist_map: book is None!', id_)
|
||||||
|
continue
|
||||||
|
lpath = bk.get('path', None)
|
||||||
|
if lpath is None:
|
||||||
|
debug_print('build_id_playlist_map: lpath is None!', id_)
|
||||||
|
continue
|
||||||
|
if lpath not in playlist_map:
|
||||||
|
playlist_map[lpath] = []
|
||||||
|
playlist_map[lpath].append(name)
|
||||||
debug_print('Finish build_id_playlist_map. Found', len(playlist_map))
|
debug_print('Finish build_id_playlist_map. Found', len(playlist_map))
|
||||||
return playlist_map
|
return playlist_map
|
||||||
|
|
||||||
|
def reset_existing_playlists_map(self):
|
||||||
|
self._playlist_to_playlist_id_map = {}
|
||||||
|
|
||||||
def get_or_create_playlist(self, bl_idx, title):
|
def get_or_create_playlist(self, bl_idx, title):
|
||||||
root = self.record_roots[bl_idx]
|
root = self.record_roots[bl_idx]
|
||||||
for playlist in root.xpath('//*[local-name()="playlist"]'):
|
# maintain a private map of playlists to their ids. Don't check if it
|
||||||
if playlist.get('title', None) == title:
|
# exists, because reset_existing_playlist_map must be called before it
|
||||||
return playlist
|
# is used to ensure that deleted playlists are taken into account
|
||||||
if DEBUG:
|
if bl_idx not in self._playlist_to_playlist_id_map:
|
||||||
debug_print('Creating playlist:', title)
|
self._playlist_to_playlist_id_map[bl_idx] = {}
|
||||||
|
for playlist in root.xpath('//*[local-name()="playlist"]'):
|
||||||
|
pl_title = playlist.get('title', None)
|
||||||
|
if pl_title is not None:
|
||||||
|
self._playlist_to_playlist_id_map[bl_idx][pl_title] = playlist
|
||||||
|
if title in self._playlist_to_playlist_id_map[bl_idx]:
|
||||||
|
return self._playlist_to_playlist_id_map[bl_idx][title]
|
||||||
|
debug_print('Creating playlist:', title)
|
||||||
ans = root.makeelement('{%s}playlist'%self.namespaces[bl_idx],
|
ans = root.makeelement('{%s}playlist'%self.namespaces[bl_idx],
|
||||||
nsmap=root.nsmap, attrib={
|
nsmap=root.nsmap, attrib={
|
||||||
'uuid' : uuid(),
|
'uuid' : uuid(),
|
||||||
@ -198,6 +213,7 @@ class XMLCache(object):
|
|||||||
'sourceid': '1'
|
'sourceid': '1'
|
||||||
})
|
})
|
||||||
root.append(ans)
|
root.append(ans)
|
||||||
|
self._playlist_to_playlist_id_map[bl_idx][title] = ans
|
||||||
return ans
|
return ans
|
||||||
# }}}
|
# }}}
|
||||||
|
|
||||||
@ -260,7 +276,9 @@ class XMLCache(object):
|
|||||||
ensure_media_xml_base_ids(root)
|
ensure_media_xml_base_ids(root)
|
||||||
|
|
||||||
idmap = ensure_numeric_ids(root)
|
idmap = ensure_numeric_ids(root)
|
||||||
remap_playlist_references(root, idmap)
|
if len(idmap) > 0:
|
||||||
|
debug_print('fix_ids: found some non-numeric ids')
|
||||||
|
remap_playlist_references(root, idmap)
|
||||||
if i == 0:
|
if i == 0:
|
||||||
sourceid, playlist_sid = 1, 0
|
sourceid, playlist_sid = 1, 0
|
||||||
base = 0
|
base = 0
|
||||||
@ -326,7 +344,9 @@ class XMLCache(object):
|
|||||||
record = lpath_map.get(book.lpath, None)
|
record = lpath_map.get(book.lpath, None)
|
||||||
if record is None:
|
if record is None:
|
||||||
record = self.create_text_record(root, i, book.lpath)
|
record = self.create_text_record(root, i, book.lpath)
|
||||||
self.update_text_record(record, book, path, i)
|
date = self.check_timestamp(record, book, path)
|
||||||
|
if date is not None:
|
||||||
|
self.update_text_record(record, book, date, path, i)
|
||||||
# Ensure the collections in the XML database are recorded for
|
# Ensure the collections in the XML database are recorded for
|
||||||
# this book
|
# this book
|
||||||
if book.device_collections is None:
|
if book.device_collections is None:
|
||||||
@ -352,8 +372,10 @@ class XMLCache(object):
|
|||||||
|
|
||||||
def update_playlists(self, bl_index, root, booklist, collections_attributes):
|
def update_playlists(self, bl_index, root, booklist, collections_attributes):
|
||||||
debug_print('Starting update_playlists', collections_attributes, bl_index)
|
debug_print('Starting update_playlists', collections_attributes, bl_index)
|
||||||
|
self.reset_existing_playlists_map()
|
||||||
collections = booklist.get_collections(collections_attributes)
|
collections = booklist.get_collections(collections_attributes)
|
||||||
lpath_map = self.build_lpath_map(root)
|
lpath_map = self.build_lpath_map(root)
|
||||||
|
debug_print('update_playlists: finished building maps')
|
||||||
for category, books in collections.items():
|
for category, books in collections.items():
|
||||||
records = [lpath_map.get(b.lpath, None) for b in books]
|
records = [lpath_map.get(b.lpath, None) for b in books]
|
||||||
# Remove any books that were not found, although this
|
# Remove any books that were not found, although this
|
||||||
@ -362,24 +384,23 @@ class XMLCache(object):
|
|||||||
debug_print('WARNING: Some elements in the JSON cache were not'
|
debug_print('WARNING: Some elements in the JSON cache were not'
|
||||||
' found in the XML cache')
|
' found in the XML cache')
|
||||||
records = [x for x in records if x is not None]
|
records = [x for x in records if x is not None]
|
||||||
|
ids = set()
|
||||||
for rec in records:
|
for rec in records:
|
||||||
if rec.get('id', None) is None:
|
id = rec.get('id', None)
|
||||||
|
if id is None:
|
||||||
rec.set('id', str(self.max_id(root)+1))
|
rec.set('id', str(self.max_id(root)+1))
|
||||||
ids = [x.get('id', None) for x in records]
|
id = rec.get('id', None)
|
||||||
if None in ids:
|
ids.add(id)
|
||||||
debug_print('WARNING: Some <text> elements do not have ids')
|
# ids cannot contain None, so no reason to check
|
||||||
ids = [x for x in ids if x is not None]
|
|
||||||
playlist = self.get_or_create_playlist(bl_index, category)
|
playlist = self.get_or_create_playlist(bl_index, category)
|
||||||
playlist_ids = []
|
# Reduce ids to books not already in the playlist
|
||||||
for item in playlist:
|
for item in playlist:
|
||||||
id_ = item.get('id', None)
|
id_ = item.get('id', None)
|
||||||
if id_ is not None:
|
if id_ is not None:
|
||||||
playlist_ids.append(id_)
|
ids.discard(id_)
|
||||||
for item in list(playlist):
|
# Add the books in ids that were not already in the playlist
|
||||||
playlist.remove(item)
|
for id_ in ids:
|
||||||
|
|
||||||
extra_ids = [x for x in playlist_ids if x not in ids]
|
|
||||||
for id_ in ids + extra_ids:
|
|
||||||
item = playlist.makeelement(
|
item = playlist.makeelement(
|
||||||
'{%s}item'%self.namespaces[bl_index],
|
'{%s}item'%self.namespaces[bl_index],
|
||||||
nsmap=playlist.nsmap, attrib={'id':id_})
|
nsmap=playlist.nsmap, attrib={'id':id_})
|
||||||
@ -416,11 +437,23 @@ class XMLCache(object):
|
|||||||
root.append(ans)
|
root.append(ans)
|
||||||
return ans
|
return ans
|
||||||
|
|
||||||
def update_text_record(self, record, book, path, bl_index):
|
def check_timestamp(self, record, book, path):
|
||||||
|
'''
|
||||||
|
Checks the timestamp in the Sony DB against the file. If different,
|
||||||
|
return the file timestamp. Otherwise return None.
|
||||||
|
'''
|
||||||
timestamp = os.path.getmtime(path)
|
timestamp = os.path.getmtime(path)
|
||||||
date = strftime(timestamp)
|
date = strftime(timestamp)
|
||||||
if date != record.get('date', None):
|
if date != record.get('date', None):
|
||||||
record.set('date', date)
|
return date
|
||||||
|
return None
|
||||||
|
|
||||||
|
def update_text_record(self, record, book, date, path, bl_index):
|
||||||
|
'''
|
||||||
|
Update the Sony database from the book. This is done if the timestamp in
|
||||||
|
the db differs from the timestamp on the file.
|
||||||
|
'''
|
||||||
|
record.set('date', date)
|
||||||
record.set('size', str(os.stat(path).st_size))
|
record.set('size', str(os.stat(path).st_size))
|
||||||
title = book.title if book.title else _('Unknown')
|
title = book.title if book.title else _('Unknown')
|
||||||
record.set('title', title)
|
record.set('title', title)
|
||||||
|
@ -134,9 +134,16 @@ class CollectionsBookList(BookList):
|
|||||||
def get_collections(self, collection_attributes):
|
def get_collections(self, collection_attributes):
|
||||||
collections = {}
|
collections = {}
|
||||||
series_categories = set([])
|
series_categories = set([])
|
||||||
|
# This map of sets is used to avoid linear searches when testing for
|
||||||
|
# book equality
|
||||||
|
collections_lpaths = {}
|
||||||
for book in self:
|
for book in self:
|
||||||
# The default: leave the book in all existing collections. Do not
|
# Make sure we can identify this book via the lpath
|
||||||
# add any new ones.
|
lpath = getattr(book, 'lpath', None)
|
||||||
|
if lpath is None:
|
||||||
|
continue
|
||||||
|
# Decide how we will build the collections. The default: leave the
|
||||||
|
# book in all existing collections. Do not add any new ones.
|
||||||
attrs = ['device_collections']
|
attrs = ['device_collections']
|
||||||
if getattr(book, '_new_book', False):
|
if getattr(book, '_new_book', False):
|
||||||
if prefs['preserve_user_collections']:
|
if prefs['preserve_user_collections']:
|
||||||
@ -163,11 +170,12 @@ class CollectionsBookList(BookList):
|
|||||||
continue
|
continue
|
||||||
if category not in collections:
|
if category not in collections:
|
||||||
collections[category] = []
|
collections[category] = []
|
||||||
if book not in collections[category]:
|
collections_lpaths[category] = set()
|
||||||
|
if lpath not in collections_lpaths[category]:
|
||||||
|
collections_lpaths[category].add(lpath)
|
||||||
collections[category].append(book)
|
collections[category].append(book)
|
||||||
if attr == 'series':
|
if attr == 'series':
|
||||||
series_categories.add(category)
|
series_categories.add(category)
|
||||||
|
|
||||||
# Sort collections
|
# Sort collections
|
||||||
for category, books in collections.items():
|
for category, books in collections.items():
|
||||||
def tgetter(x):
|
def tgetter(x):
|
||||||
|
@ -1416,7 +1416,6 @@ class DeviceMixin(object): # {{{
|
|||||||
# the application_id, which is really the db key, but as this can
|
# the application_id, which is really the db key, but as this can
|
||||||
# accidentally match across libraries we also verify the title. The
|
# accidentally match across libraries we also verify the title. The
|
||||||
# db_id exists on Sony devices. Fallback is title and author match
|
# db_id exists on Sony devices. Fallback is title and author match
|
||||||
resend_metadata = False
|
|
||||||
for booklist in booklists:
|
for booklist in booklists:
|
||||||
for book in booklist:
|
for book in booklist:
|
||||||
if getattr(book, 'uuid', None) in self.db_book_uuid_cache:
|
if getattr(book, 'uuid', None) in self.db_book_uuid_cache:
|
||||||
@ -1433,12 +1432,10 @@ class DeviceMixin(object): # {{{
|
|||||||
if getattr(book, 'application_id', None) in d['db_ids']:
|
if getattr(book, 'application_id', None) in d['db_ids']:
|
||||||
book.in_library = True
|
book.in_library = True
|
||||||
book.smart_update(d['db_ids'][book.application_id])
|
book.smart_update(d['db_ids'][book.application_id])
|
||||||
resend_metadata = True
|
|
||||||
continue
|
continue
|
||||||
if book.db_id in d['db_ids']:
|
if book.db_id in d['db_ids']:
|
||||||
book.in_library = True
|
book.in_library = True
|
||||||
book.smart_update(d['db_ids'][book.db_id])
|
book.smart_update(d['db_ids'][book.db_id])
|
||||||
resend_metadata = True
|
|
||||||
continue
|
continue
|
||||||
if book.authors:
|
if book.authors:
|
||||||
# Compare against both author and author sort, because
|
# Compare against both author and author sort, because
|
||||||
@ -1448,21 +1445,13 @@ class DeviceMixin(object): # {{{
|
|||||||
if book_authors in d['authors']:
|
if book_authors in d['authors']:
|
||||||
book.in_library = True
|
book.in_library = True
|
||||||
book.smart_update(d['authors'][book_authors])
|
book.smart_update(d['authors'][book_authors])
|
||||||
resend_metadata = True
|
|
||||||
elif book_authors in d['author_sort']:
|
elif book_authors in d['author_sort']:
|
||||||
book.in_library = True
|
book.in_library = True
|
||||||
book.smart_update(d['author_sort'][book_authors])
|
book.smart_update(d['author_sort'][book_authors])
|
||||||
resend_metadata = True
|
|
||||||
# Set author_sort if it isn't already
|
# Set author_sort if it isn't already
|
||||||
asort = getattr(book, 'author_sort', None)
|
asort = getattr(book, 'author_sort', None)
|
||||||
if not asort and book.authors:
|
if not asort and book.authors:
|
||||||
book.author_sort = self.library_view.model().db.author_sort_from_authors(book.authors)
|
book.author_sort = self.library_view.model().db.author_sort_from_authors(book.authors)
|
||||||
resend_metadata = True
|
|
||||||
|
|
||||||
if resend_metadata:
|
|
||||||
# Correct the metadata cache on device.
|
|
||||||
if self.device_manager.is_device_connected:
|
|
||||||
self.device_manager.sync_booklists(None, booklists)
|
|
||||||
|
|
||||||
# }}}
|
# }}}
|
||||||
|
|
||||||
|
Loading…
x
Reference in New Issue
Block a user