From 3228374bf935e1b971e197053257762fa5c6488f Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Tue, 22 Jun 2010 22:32:19 +0100 Subject: [PATCH 1/3] Fix yet another problem with sony collections. This one was provoked by doing a sync booklist just after loading the cache. The problem was that the collections had not been computed, so they were all removed. The bug could happen if only one book was out of sync. --- src/calibre/devices/prs505/sony_cache.py | 109 ++++++++++------------- src/calibre/gui2/device.py | 5 ++ src/calibre/gui2/library/models.py | 2 +- 3 files changed, 53 insertions(+), 63 deletions(-) diff --git a/src/calibre/devices/prs505/sony_cache.py b/src/calibre/devices/prs505/sony_cache.py index e7d0e4686c..a704824a3f 100644 --- a/src/calibre/devices/prs505/sony_cache.py +++ b/src/calibre/devices/prs505/sony_cache.py @@ -9,7 +9,6 @@ import os, time from pprint import pprint from base64 import b64decode from uuid import uuid4 - from lxml import etree from calibre import prints, guess_type @@ -151,15 +150,14 @@ class XMLCache(object): else: seen.add(title) - def get_playlist_map(self): - debug_print('Start get_playlist_map') + def build_playlist_id_map(self): + debug_print('Start build_playlist_id_map') ans = {} self.ensure_unique_playlist_titles() debug_print('after ensure_unique_playlist_titles') self.prune_empty_playlists() - debug_print('get_playlist_map loop') for i, root in self.record_roots.items(): - debug_print('get_playlist_map loop', i) + debug_print('build_playlist_id_map loop', i) id_map = self.build_id_map(root) ans[i] = [] for playlist in root.xpath('//*[local-name()="playlist"]'): @@ -170,9 +168,23 @@ class XMLCache(object): if record is not None: items.append(record) ans[i].append((playlist.get('title'), items)) - debug_print('end get_playlist_map') + debug_print('end build_playlist_id_map') return ans + def build_id_playlist_map(self, bl_index): + debug_print('Start build_id_playlist_map') + pmap = self.build_playlist_id_map()[bl_index] + playlist_map = {} + for title, records in pmap: + for record in records: + path = record.get('path', None) + if path: + if path not in playlist_map: + playlist_map[path] = set() + playlist_map[path].add(title) + debug_print('Finish build_id_playlist_map. Found', len(playlist_map)) + return playlist_map + def get_or_create_playlist(self, bl_idx, title): root = self.record_roots[bl_idx] for playlist in root.xpath('//*[local-name()="playlist"]'): @@ -192,8 +204,7 @@ class XMLCache(object): # }}} def fix_ids(self): # {{{ - if DEBUG: - debug_print('Running fix_ids()') + debug_print('Running fix_ids()') def ensure_numeric_ids(root): idmap = {} @@ -276,38 +287,19 @@ class XMLCache(object): def update_booklist(self, bl, bl_index): if bl_index not in self.record_roots: return - if DEBUG: - debug_print('Updating JSON cache:', bl_index) + debug_print('Updating JSON cache:', bl_index) + playlist_map = self.build_id_playlist_map(bl_index) root = self.record_roots[bl_index] - pmap = self.get_playlist_map()[bl_index] - playlist_map = {} - for title, records in pmap: - for record in records: - path = record.get('path', None) - if path: - if path not in playlist_map: - playlist_map[path] = [] - playlist_map[path].append(title) - lpath_map = self.build_lpath_map(root) for book in bl: record = lpath_map.get(book.lpath, None) if record is not None: title = record.get('title', None) if title is not None and title != book.title: - if DEBUG: - debug_print('Renaming title', book.title, 'to', title) + debug_print('Renaming title', book.title, 'to', title) book.title = title -# We shouldn't do this for Sonys, because the reader strips -# all but the first author. -# authors = record.get('author', None) -# if authors is not None: -# authors = string_to_authors(authors) -# if authors != book.authors: -# if DEBUG: -# prints('Renaming authors', book.authors, 'to', -# authors) -# book.authors = authors + # Don't set the author, because the reader strips all but + # the first author. for thumbnail in record.xpath( 'descendant::*[local-name()="thumbnail"]'): for img in thumbnail.xpath( @@ -318,45 +310,45 @@ class XMLCache(object): book.thumbnail = raw break break - if book.lpath in playlist_map: - tags = playlist_map[book.lpath] - book.device_collections = tags + book.device_collections = list(playlist_map.get(book.lpath, set())) debug_print('Finished updating JSON cache:', bl_index) # }}} # Update XML from JSON {{{ def update(self, booklists, collections_attributes): - debug_print('Starting update XML from JSON') - playlist_map = self.get_playlist_map() - + debug_print('In update. Starting update XML from JSON') for i, booklist in booklists.items(): - if DEBUG: - debug_print('Updating XML Cache:', i) + playlist_map = self.build_id_playlist_map(i) + debug_print('Updating XML Cache:', i) root = self.record_roots[i] lpath_map = self.build_lpath_map(root) for book in booklist: path = os.path.join(self.prefixes[i], *(book.lpath.split('/'))) -# record = self.book_by_lpath(book.lpath, root) record = lpath_map.get(book.lpath, None) if record is None: record = self.create_text_record(root, i, book.lpath) self.update_text_record(record, book, path, i) - - bl_pmap = playlist_map[i] - self.update_playlists(i, root, booklist, bl_pmap, - collections_attributes) - - self.fix_ids() - - # This is needed to update device_collections + # Ensure the collections in the XML database are recorded for + # this book + if book.device_collections is None: + book.device_collections = [] + book.device_collections = list(set(book.device_collections) | + playlist_map.get(book.lpath, set())) + self.update_playlists(i, root, booklist, collections_attributes) + # Update the device collections because update playlist could have added + # some new ones. + debug_print('In update/ Starting refresh of device_collections') for i, booklist in booklists.items(): - self.update_booklist(booklist, i) + playlist_map = self.build_id_playlist_map(i) + for book in booklist: + book.device_collections = list(set(book.device_collections) | + playlist_map.get(book.lpath, set())) + self.fix_ids() debug_print('Finished update XML from JSON') - def update_playlists(self, bl_index, root, booklist, playlist_map, - collections_attributes): - debug_print('Starting update_playlists') + def update_playlists(self, bl_index, root, booklist, collections_attributes): + debug_print('Starting update_playlists', collections_attributes) collections = booklist.get_collections(collections_attributes) lpath_map = self.build_lpath_map(root) for category, books in collections.items(): @@ -372,10 +364,8 @@ class XMLCache(object): rec.set('id', str(self.max_id(root)+1)) ids = [x.get('id', None) for x in records] if None in ids: - if DEBUG: - debug_print('WARNING: Some elements do not have ids') - ids = [x for x in ids if x is not None] - + debug_print('WARNING: Some elements do not have ids') + ids = [x for x in ids if x is not None] playlist = self.get_or_create_playlist(bl_index, category) playlist_ids = [] for item in playlist: @@ -544,10 +534,5 @@ class XMLCache(object): break self.namespaces[i] = ns -# if DEBUG: -# debug_print('Found nsmaps:') -# pprint(self.nsmaps) -# debug_print('Found namespaces:') -# pprint(self.namespaces) # }}} diff --git a/src/calibre/gui2/device.py b/src/calibre/gui2/device.py index 850396bc5d..836105aba9 100644 --- a/src/calibre/gui2/device.py +++ b/src/calibre/gui2/device.py @@ -1228,6 +1228,11 @@ class DeviceMixin(object): # {{{ return cp, fs = job.result self.location_view.model().update_devices(cp, fs) + # reset the views so that up-to-date info is shown. These need to be + # here because the sony driver updates collections in sync_booklists + self.memory_view.reset() + self.card_a_view.reset() + self.card_b_view.reset() def upload_books(self, files, names, metadata, on_card=None, memory=None): ''' diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py index 435b5c4c07..a85462e22d 100644 --- a/src/calibre/gui2/library/models.py +++ b/src/calibre/gui2/library/models.py @@ -1118,7 +1118,7 @@ class DeviceBooksModel(BooksModel): # {{{ elif cname == 'collections': tags = self.db[self.map[row]].device_collections if tags: - return QVariant(', '.join(tags)) + return QVariant(', '.join(sorted(tags, key=str.lower))) elif role == Qt.ToolTipRole and index.isValid(): if self.map[row] in self.indices_to_be_deleted(): return QVariant(_('Marked for deletion')) From 456eda2519da966e7160d738d0a2aad98011d964 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Wed, 23 Jun 2010 11:19:15 +0100 Subject: [PATCH 2/3] Kovid-suggested changes to series columns --- src/calibre/gui2/library/models.py | 13 ++---------- src/calibre/library/caches.py | 8 +++---- src/calibre/library/cli.py | 30 +++++++++++++++++++++++---- src/calibre/library/custom_columns.py | 4 ++-- 4 files changed, 34 insertions(+), 21 deletions(-) diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py index a85462e22d..008f024aae 100644 --- a/src/calibre/gui2/library/models.py +++ b/src/calibre/gui2/library/models.py @@ -21,6 +21,7 @@ from calibre.utils.date import dt_factory, qt_to_dt, isoformat from calibre.ebooks.metadata.meta import set_metadata as _set_metadata from calibre.utils.search_query_parser import SearchQueryParser from calibre.library.caches import _match, CONTAINS_MATCH, EQUALS_MATCH, REGEXP_MATCH +from calibre.library.cli import parse_series_string from calibre import strftime, isbytestring, prepare_string_for_xml from calibre.constants import filesystem_encoding from calibre.gui2.library import DEFAULT_SORT @@ -708,17 +709,7 @@ class BooksModel(QAbstractTableModel): # {{{ return False val = qt_to_dt(val, as_utc=False) elif typ == 'series': - val = unicode(value.toString()).strip() - pat = re.compile(r'\[([.0-9]+)\]') - match = pat.search(val) - if match is not None: - val = pat.sub('', val).strip() - s_index = float(match.group(1)) - elif val: - if tweaks['series_index_auto_increment'] == 'next': - s_index = self.db.get_next_cc_series_num_for(val, label=label) - else: - s_index = 1.0 + val, s_index = parse_series_string(self.db, label, value.toString()) self.db.set_custom(self.db.id(row), val, extra=s_index, label=label, num=None, append=False, notify=True) return True diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py index d4afaabcdc..06cf07bb67 100644 --- a/src/calibre/library/caches.py +++ b/src/calibre/library/caches.py @@ -620,6 +620,10 @@ class ResultCache(SearchQueryParser): elif field == 'title': field = 'sort' elif field == 'authors': field = 'author_sort' as_string = field not in ('size', 'rating', 'timestamp') + + if self.first_sort: + subsort = True + self.first_sort = False if self.field_metadata[field]['is_custom']: if self.field_metadata[field]['datatype'] == 'series': fcmp = functools.partial(self.seriescmp, @@ -638,10 +642,6 @@ class ResultCache(SearchQueryParser): else: fcmp = functools.partial(self.cmp, self.FIELD_MAP[field], subsort=subsort, asstr=as_string) - - if self.first_sort: - subsort = True - self.first_sort = False self._map.sort(cmp=fcmp, reverse=not ascending) self._map_filtered = [id for id in self._map if id in self._map_filtered] diff --git a/src/calibre/library/cli.py b/src/calibre/library/cli.py index 3f71c98238..058b879b55 100644 --- a/src/calibre/library/cli.py +++ b/src/calibre/library/cli.py @@ -7,11 +7,11 @@ __docformat__ = 'restructuredtext en' Command line interface to the calibre database. ''' -import sys, os, cStringIO +import sys, os, cStringIO, re from textwrap import TextWrapper from calibre import terminal_controller, preferred_encoding, prints -from calibre.utils.config import OptionParser, prefs +from calibre.utils.config import OptionParser, prefs, tweaks from calibre.ebooks.metadata.meta import get_metadata from calibre.library.database2 import LibraryDatabase2 from calibre.ebooks.metadata.opf2 import OPFCreator, OPF @@ -680,9 +680,31 @@ def command_catalog(args, dbpath): # end of GR additions +def parse_series_string(db, label, value): + val = unicode(value).strip() + s_index = None + pat = re.compile(r'\[([.0-9]+)\]') + match = pat.search(val) + if match is not None: + val = pat.sub('', val).strip() + s_index = float(match.group(1)) + elif val: + if tweaks['series_index_auto_increment'] == 'next': + s_index = db.get_next_cc_series_num_for(val, label=label) + else: + s_index = 1.0 + return val, s_index + def do_set_custom(db, col, id_, val, append): - db.set_custom(id_, val, label=col, append=append) - prints('Data set to: %r'%db.get_custom(id_, label=col, index_is_id=True)) + if db.custom_column_label_map[col]['datatype'] == 'series': + val, s_index = parse_series_string(db, col, val) + db.set_custom(id_, val, extra=s_index, label=col, append=append) + prints('Data set to: %r[%4.2f]'% + (db.get_custom(id_, label=col, index_is_id=True), + db.get_custom_extra(id_, label=col, index_is_id=True))) + else: + db.set_custom(id_, val, label=col, append=append) + prints('Data set to: %r'%db.get_custom(id_, label=col, index_is_id=True)) def set_custom_option_parser(): parser = get_parser(_( diff --git a/src/calibre/library/custom_columns.py b/src/calibre/library/custom_columns.py index 52084fcda1..e039f5a817 100644 --- a/src/calibre/library/custom_columns.py +++ b/src/calibre/library/custom_columns.py @@ -308,8 +308,8 @@ class CustomColumns(object): self.conn.commit() return changed - def set_custom(self, id_, val, extra=None, label=None, num=None, - append=False, notify=True): + def set_custom(self, id_, val, label=None, num=None, + append=False, notify=True, extra=None): if label is not None: data = self.custom_column_label_map[label] if num is not None: From 0c3d4511f831d4cfd05584c148b2fb4c259b5221 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Wed, 23 Jun 2010 12:05:22 +0100 Subject: [PATCH 3/3] Fix problem where collections are not built if the user spells the db fields using uppercase letters. --- src/calibre/devices/prs505/driver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/devices/prs505/driver.py b/src/calibre/devices/prs505/driver.py index 5860826778..023416bdf2 100644 --- a/src/calibre/devices/prs505/driver.py +++ b/src/calibre/devices/prs505/driver.py @@ -145,7 +145,7 @@ class PRS505(USBMS): blists[i] = booklists[i] opts = self.settings() if opts.extra_customization: - collections = [x.strip() for x in + collections = [x.lower().strip() for x in opts.extra_customization.split(',')] else: collections = []