mirror of
https://github.com/kovidgoyal/calibre.git
synced 2025-07-08 18:54:09 -04:00
Performance enhancements for sony driver and book matching
This commit is contained in:
Charles Haley
parent
24c8b1679b
commit
acfa27d4c5
@ -144,52 +144,67 @@ class XMLCache(object):
|
||||
if title+str(i) not in seen:
|
||||
title = title+str(i)
|
||||
playlist.set('title', title)
|
||||
seen.add(title)
|
||||
break
|
||||
else:
|
||||
seen.add(title)
|
||||
|
||||
def build_playlist_id_map(self):
|
||||
debug_print('Start build_playlist_id_map')
|
||||
ans = {}
|
||||
self.ensure_unique_playlist_titles()
|
||||
debug_print('after ensure_unique_playlist_titles')
|
||||
self.prune_empty_playlists()
|
||||
for i, root in self.record_roots.items():
|
||||
debug_print('build_playlist_id_map loop', i)
|
||||
id_map = self.build_id_map(root)
|
||||
ans[i] = []
|
||||
for playlist in root.xpath('//*[local-name()="playlist"]'):
|
||||
items = []
|
||||
for item in playlist:
|
||||
id_ = item.get('id', None)
|
||||
record = id_map.get(id_, None)
|
||||
if record is not None:
|
||||
items.append(record)
|
||||
ans[i].append((playlist.get('title'), items))
|
||||
debug_print('end build_playlist_id_map')
|
||||
return ans
|
||||
|
||||
def build_id_playlist_map(self, bl_index):
|
||||
'''
|
||||
Return a map of the collections in books: {lpaths: [collection names]}
|
||||
'''
|
||||
debug_print('Start build_id_playlist_map')
|
||||
pmap = self.build_playlist_id_map()[bl_index]
|
||||
self.ensure_unique_playlist_titles()
|
||||
self.prune_empty_playlists()
|
||||
debug_print('after cleaning playlists')
|
||||
root = self.record_roots[bl_index]
|
||||
if root is None:
|
||||
return
|
||||
id_map = self.build_id_map(root)
|
||||
playlist_map = {}
|
||||
for title, records in pmap:
|
||||
for record in records:
|
||||
path = record.get('path', None)
|
||||
if path:
|
||||
if path not in playlist_map:
|
||||
playlist_map[path] = []
|
||||
playlist_map[path].append(title)
|
||||
# foreach playlist, get the lpaths for the ids in it, then add to dict
|
||||
for playlist in root.xpath('//*[local-name()="playlist"]'):
|
||||
name = playlist.get('title')
|
||||
if name is None:
|
||||
debug_print('build_id_playlist_map: unnamed playlist!')
|
||||
continue
|
||||
for item in playlist:
|
||||
# translate each id into its lpath
|
||||
id_ = item.get('id', None)
|
||||
if id_ is None:
|
||||
debug_print('build_id_playlist_map: id_ is None!')
|
||||
continue
|
||||
bk = id_map.get(id_, None)
|
||||
if bk is None:
|
||||
debug_print('build_id_playlist_map: book is None!', id_)
|
||||
continue
|
||||
lpath = bk.get('path', None)
|
||||
if lpath is None:
|
||||
debug_print('build_id_playlist_map: lpath is None!', id_)
|
||||
continue
|
||||
if lpath not in playlist_map:
|
||||
playlist_map[lpath] = []
|
||||
playlist_map[lpath].append(name)
|
||||
debug_print('Finish build_id_playlist_map. Found', len(playlist_map))
|
||||
return playlist_map
|
||||
|
||||
def reset_existing_playlists_map(self):
|
||||
self._playlist_to_playlist_id_map = {}
|
||||
|
||||
def get_or_create_playlist(self, bl_idx, title):
|
||||
root = self.record_roots[bl_idx]
|
||||
for playlist in root.xpath('//*[local-name()="playlist"]'):
|
||||
if playlist.get('title', None) == title:
|
||||
return playlist
|
||||
if DEBUG:
|
||||
debug_print('Creating playlist:', title)
|
||||
# maintain a private map of playlists to their ids. Don't check if it
|
||||
# exists, because reset_existing_playlist_map must be called before it
|
||||
# is used to ensure that deleted playlists are taken into account
|
||||
if bl_idx not in self._playlist_to_playlist_id_map:
|
||||
self._playlist_to_playlist_id_map[bl_idx] = {}
|
||||
for playlist in root.xpath('//*[local-name()="playlist"]'):
|
||||
pl_title = playlist.get('title', None)
|
||||
if pl_title is not None:
|
||||
self._playlist_to_playlist_id_map[bl_idx][pl_title] = playlist
|
||||
if title in self._playlist_to_playlist_id_map[bl_idx]:
|
||||
return self._playlist_to_playlist_id_map[bl_idx][title]
|
||||
debug_print('Creating playlist:', title)
|
||||
ans = root.makeelement('{%s}playlist'%self.namespaces[bl_idx],
|
||||
nsmap=root.nsmap, attrib={
|
||||
'uuid' : uuid(),
|
||||
@ -198,6 +213,7 @@ class XMLCache(object):
|
||||
'sourceid': '1'
|
||||
})
|
||||
root.append(ans)
|
||||
self._playlist_to_playlist_id_map[bl_idx][title] = ans
|
||||
return ans
|
||||
# }}}
|
||||
|
||||
@ -260,7 +276,9 @@ class XMLCache(object):
|
||||
ensure_media_xml_base_ids(root)
|
||||
|
||||
idmap = ensure_numeric_ids(root)
|
||||
remap_playlist_references(root, idmap)
|
||||
if len(idmap) > 0:
|
||||
debug_print('fix_ids: found some non-numeric ids')
|
||||
remap_playlist_references(root, idmap)
|
||||
if i == 0:
|
||||
sourceid, playlist_sid = 1, 0
|
||||
base = 0
|
||||
@ -326,7 +344,9 @@ class XMLCache(object):
|
||||
record = lpath_map.get(book.lpath, None)
|
||||
if record is None:
|
||||
record = self.create_text_record(root, i, book.lpath)
|
||||
self.update_text_record(record, book, path, i)
|
||||
date = self.check_timestamp(record, book, path)
|
||||
if date is not None:
|
||||
self.update_text_record(record, book, date, path, i)
|
||||
# Ensure the collections in the XML database are recorded for
|
||||
# this book
|
||||
if book.device_collections is None:
|
||||
@ -352,8 +372,10 @@ class XMLCache(object):
|
||||
|
||||
def update_playlists(self, bl_index, root, booklist, collections_attributes):
|
||||
debug_print('Starting update_playlists', collections_attributes, bl_index)
|
||||
self.reset_existing_playlists_map()
|
||||
collections = booklist.get_collections(collections_attributes)
|
||||
lpath_map = self.build_lpath_map(root)
|
||||
debug_print('update_playlists: finished building maps')
|
||||
for category, books in collections.items():
|
||||
records = [lpath_map.get(b.lpath, None) for b in books]
|
||||
# Remove any books that were not found, although this
|
||||
@ -362,24 +384,23 @@ class XMLCache(object):
|
||||
debug_print('WARNING: Some elements in the JSON cache were not'
|
||||
' found in the XML cache')
|
||||
records = [x for x in records if x is not None]
|
||||
ids = set()
|
||||
for rec in records:
|
||||
if rec.get('id', None) is None:
|
||||
id = rec.get('id', None)
|
||||
if id is None:
|
||||
rec.set('id', str(self.max_id(root)+1))
|
||||
ids = [x.get('id', None) for x in records]
|
||||
if None in ids:
|
||||
debug_print('WARNING: Some <text> elements do not have ids')
|
||||
ids = [x for x in ids if x is not None]
|
||||
id = rec.get('id', None)
|
||||
ids.add(id)
|
||||
# ids cannot contain None, so no reason to check
|
||||
|
||||
playlist = self.get_or_create_playlist(bl_index, category)
|
||||
playlist_ids = []
|
||||
# Reduce ids to books not already in the playlist
|
||||
for item in playlist:
|
||||
id_ = item.get('id', None)
|
||||
if id_ is not None:
|
||||
playlist_ids.append(id_)
|
||||
for item in list(playlist):
|
||||
playlist.remove(item)
|
||||
|
||||
extra_ids = [x for x in playlist_ids if x not in ids]
|
||||
for id_ in ids + extra_ids:
|
||||
ids.discard(id_)
|
||||
# Add the books in ids that were not already in the playlist
|
||||
for id_ in ids:
|
||||
item = playlist.makeelement(
|
||||
'{%s}item'%self.namespaces[bl_index],
|
||||
nsmap=playlist.nsmap, attrib={'id':id_})
|
||||
@ -416,11 +437,23 @@ class XMLCache(object):
|
||||
root.append(ans)
|
||||
return ans
|
||||
|
||||
def update_text_record(self, record, book, path, bl_index):
|
||||
def check_timestamp(self, record, book, path):
|
||||
'''
|
||||
Checks the timestamp in the Sony DB against the file. If different,
|
||||
return the file timestamp. Otherwise return None.
|
||||
'''
|
||||
timestamp = os.path.getmtime(path)
|
||||
date = strftime(timestamp)
|
||||
if date != record.get('date', None):
|
||||
record.set('date', date)
|
||||
return date
|
||||
return None
|
||||
|
||||
def update_text_record(self, record, book, date, path, bl_index):
|
||||
'''
|
||||
Update the Sony database from the book. This is done if the timestamp in
|
||||
the db differs from the timestamp on the file.
|
||||
'''
|
||||
record.set('date', date)
|
||||
record.set('size', str(os.stat(path).st_size))
|
||||
title = book.title if book.title else _('Unknown')
|
||||
record.set('title', title)
|
||||
|
||||
@ -134,9 +134,16 @@ class CollectionsBookList(BookList):
|
||||
def get_collections(self, collection_attributes):
|
||||
collections = {}
|
||||
series_categories = set([])
|
||||
# This map of sets is used to avoid linear searches when testing for
|
||||
# book equality
|
||||
collections_lpaths = {}
|
||||
for book in self:
|
||||
# The default: leave the book in all existing collections. Do not
|
||||
# add any new ones.
|
||||
# Make sure we can identify this book via the lpath
|
||||
lpath = getattr(book, 'lpath', None)
|
||||
if lpath is None:
|
||||
continue
|
||||
# Decide how we will build the collections. The default: leave the
|
||||
# book in all existing collections. Do not add any new ones.
|
||||
attrs = ['device_collections']
|
||||
if getattr(book, '_new_book', False):
|
||||
if prefs['preserve_user_collections']:
|
||||
@ -163,11 +170,12 @@ class CollectionsBookList(BookList):
|
||||
continue
|
||||
if category not in collections:
|
||||
collections[category] = []
|
||||
if book not in collections[category]:
|
||||
collections_lpaths[category] = set()
|
||||
if lpath not in collections_lpaths[category]:
|
||||
collections_lpaths[category].add(lpath)
|
||||
collections[category].append(book)
|
||||
if attr == 'series':
|
||||
series_categories.add(category)
|
||||
|
||||
# Sort collections
|
||||
for category, books in collections.items():
|
||||
def tgetter(x):
|
||||
|
||||
@ -1416,7 +1416,6 @@ class DeviceMixin(object): # {{{
|
||||
# the application_id, which is really the db key, but as this can
|
||||
# accidentally match across libraries we also verify the title. The
|
||||
# db_id exists on Sony devices. Fallback is title and author match
|
||||
resend_metadata = False
|
||||
for booklist in booklists:
|
||||
for book in booklist:
|
||||
if getattr(book, 'uuid', None) in self.db_book_uuid_cache:
|
||||
@ -1433,12 +1432,10 @@ class DeviceMixin(object): # {{{
|
||||
if getattr(book, 'application_id', None) in d['db_ids']:
|
||||
book.in_library = True
|
||||
book.smart_update(d['db_ids'][book.application_id])
|
||||
resend_metadata = True
|
||||
continue
|
||||
if book.db_id in d['db_ids']:
|
||||
book.in_library = True
|
||||
book.smart_update(d['db_ids'][book.db_id])
|
||||
resend_metadata = True
|
||||
continue
|
||||
if book.authors:
|
||||
# Compare against both author and author sort, because
|
||||
@ -1448,21 +1445,13 @@ class DeviceMixin(object): # {{{
|
||||
if book_authors in d['authors']:
|
||||
book.in_library = True
|
||||
book.smart_update(d['authors'][book_authors])
|
||||
resend_metadata = True
|
||||
elif book_authors in d['author_sort']:
|
||||
book.in_library = True
|
||||
book.smart_update(d['author_sort'][book_authors])
|
||||
resend_metadata = True
|
||||
# Set author_sort if it isn't already
|
||||
asort = getattr(book, 'author_sort', None)
|
||||
if not asort and book.authors:
|
||||
book.author_sort = self.library_view.model().db.author_sort_from_authors(book.authors)
|
||||
resend_metadata = True
|
||||
|
||||
if resend_metadata:
|
||||
# Correct the metadata cache on device.
|
||||
if self.device_manager.is_device_connected:
|
||||
self.device_manager.sync_booklists(None, booklists)
|
||||
|
||||
# }}}
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user