From c793002de7e050750c87da49a24fd8adef753c3b Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Wed, 5 Jan 2011 08:39:39 +0000 Subject: [PATCH 1/8] Make evaluation of composite columns just-in-time, instead of just-in-case. --- src/calibre/library/caches.py | 74 +++++++++++++++++++++++++---------- 1 file changed, 53 insertions(+), 21 deletions(-) diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py index a32c45191f..3a61a8fd5d 100644 --- a/src/calibre/library/caches.py +++ b/src/calibre/library/caches.py @@ -132,6 +132,48 @@ def _match(query, value, matchkind): pass return False +class CacheRow(object): + + def __init__(self, db, composites, val): + self.db = db + self.composites = composites + self._mydata = val + self._must_do = len(composites) > 0 + + def __getitem__(self, col): + rec = self._mydata + if self._must_do and col in self.composites: + self._must_do = False + mi = self.db.get_metadata(rec[0], index_is_id=True) + for c in self.composites: + rec[c] = mi.get(self.composites[c]) + return rec[col] + + def __setitem__ (self, col, val): + self._mydata[col] = val + + def append(self, val): + self._mydata.append(val) + + def get(self, col, default): + try: + return self.__getitem__(col) + except: + return default + + def __len__(self): + return len(self._mydata) + + def __iter__(self): + for v in self._mydata: + yield v + + def __str__(self): + return self.__unicode__() + + def __unicode__(self): + return unicode(self._mydata) + class ResultCache(SearchQueryParser): # {{{ ''' @@ -139,7 +181,12 @@ class ResultCache(SearchQueryParser): # {{{ ''' def __init__(self, FIELD_MAP, field_metadata): self.FIELD_MAP = FIELD_MAP - self._map = self._data = self._map_filtered = [] + self.composites = {} + for key in field_metadata: + if field_metadata[key]['datatype'] == 'composite': + self.composites[field_metadata[key]['rec_index']] = key + self._data = [] + self._map = self._map_filtered = [] self.first_sort = True self.search_restriction = '' self.field_metadata = field_metadata @@ -148,10 +195,6 @@ class ResultCache(SearchQueryParser): # {{{ self.build_date_relop_dict() self.build_numeric_relop_dict() - self.composites = [] - for key in field_metadata: - if field_metadata[key]['datatype'] == 'composite': - self.composites.append((key, field_metadata[key]['rec_index'])) def __getitem__(self, row): return self._data[self._map_filtered[row]] @@ -583,13 +626,10 @@ class ResultCache(SearchQueryParser): # {{{ ''' for id in ids: try: - self._data[id] = db.conn.get('SELECT * from meta2 WHERE id=?', (id,))[0] + self._data[id] = CacheRow(db, self.composites, + db.conn.get('SELECT * from meta2 WHERE id=?', (id,))[0]) self._data[id].append(db.book_on_device_string(id)) self._data[id].append(None) - if len(self.composites) > 0: - mi = db.get_metadata(id, index_is_id=True) - for k,c in self.composites: - self._data[id][c] = mi.get(k, None) except IndexError: return None try: @@ -603,13 +643,10 @@ class ResultCache(SearchQueryParser): # {{{ return self._data.extend(repeat(None, max(ids)-len(self._data)+2)) for id in ids: - self._data[id] = db.conn.get('SELECT * from meta2 WHERE id=?', (id,))[0] + self._data[id] = CacheRow(db, self.composites, + db.conn.get('SELECT * from meta2 WHERE id=?', (id,))[0]) self._data[id].append(db.book_on_device_string(id)) self._data[id].append(None) - if len(self.composites) > 0: - mi = db.get_metadata(id, index_is_id=True) - for k,c in self.composites: - self._data[id][c] = mi.get(k) self._map[0:0] = ids self._map_filtered[0:0] = ids @@ -630,16 +667,11 @@ class ResultCache(SearchQueryParser): # {{{ temp = db.conn.get('SELECT * FROM meta2') self._data = list(itertools.repeat(None, temp[-1][0]+2)) if temp else [] for r in temp: - self._data[r[0]] = r + self._data[r[0]] = CacheRow(db, self.composites, r) for item in self._data: if item is not None: item.append(db.book_on_device_string(item[0])) item.append(None) - if len(self.composites) > 0: - mi = db.get_metadata(item[0], index_is_id=True) - for k,c in self.composites: - item[c] = mi.get(k) - self._map = [i[0] for i in self._data if i is not None] if field is not None: self.sort(field, ascending) From 2662ab485fb978a27dcdc613b68e45613e8c6ac6 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Wed, 5 Jan 2011 09:49:27 +0000 Subject: [PATCH 2/8] Make tags sorted in the meta2 table. --- src/calibre/gui2/library/models.py | 4 ++-- src/calibre/library/custom_columns.py | 9 +++++---- src/calibre/library/database2.py | 2 +- src/calibre/library/sqlite.py | 23 ++++++++++++++++++++++- 4 files changed, 30 insertions(+), 8 deletions(-) diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py index 49cb1ce182..6a48aef9be 100644 --- a/src/calibre/gui2/library/models.py +++ b/src/calibre/gui2/library/models.py @@ -526,7 +526,7 @@ class BooksModel(QAbstractTableModel): # {{{ def tags(r, idx=-1): tags = self.db.data[r][idx] if tags: - return QVariant(', '.join(sorted(tags.split(','), key=sort_key))) + return QVariant(', '.join(tags.split(','))) return None def series_type(r, idx=-1, siix=-1): @@ -577,7 +577,7 @@ class BooksModel(QAbstractTableModel): # {{{ def text_type(r, mult=False, idx=-1): text = self.db.data[r][idx] if text and mult: - return QVariant(', '.join(sorted(text.split('|'),key=sort_key))) + return QVariant(', '.join(text.split('|'))) return QVariant(text) def number_type(r, idx=-1): diff --git a/src/calibre/library/custom_columns.py b/src/calibre/library/custom_columns.py index 07ea407460..558f3b8fc9 100644 --- a/src/calibre/library/custom_columns.py +++ b/src/calibre/library/custom_columns.py @@ -14,6 +14,7 @@ from calibre.constants import preferred_encoding from calibre.library.field_metadata import FieldMetadata from calibre.utils.date import parse_date from calibre.utils.config import tweaks +from calibre.utils.icu import sort_key class CustomColumns(object): @@ -181,8 +182,8 @@ class CustomColumns(object): ans = row[self.FIELD_MAP[data['num']]] if data['is_multiple'] and data['datatype'] == 'text': ans = ans.split('|') if ans else [] - if data['display'].get('sort_alpha', False): - ans.sort(cmp=lambda x,y:cmp(x.lower(), y.lower())) + if data['display'].get('sort_alpha', True): + ans.sort(key=sort_key) return ans def get_custom_extra(self, idx, label=None, num=None, index_is_id=False): @@ -534,8 +535,8 @@ class CustomColumns(object): if data['normalized']: query = '%s.value' if data['is_multiple']: - query = 'group_concat(%s.value, "|")' - if not display.get('sort_alpha', False): + query = 'cc_sortconcat(%s.value)' + if not display.get('sort_alpha', True): query = 'sort_concat(link.id, %s.value)' line = '''(SELECT {query} FROM {lt} AS link INNER JOIN {table} ON(link.value={table}.id) WHERE link.book=books.id) diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 611aa1cc89..0b1c6a6cfb 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -242,7 +242,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): 'timestamp', '(SELECT MAX(uncompressed_size) FROM data WHERE book=books.id) size', ('rating', 'ratings', 'rating', 'ratings.rating'), - ('tags', 'tags', 'tag', 'group_concat(name)'), + ('tags', 'tags', 'tag', 'tags_sortconcat(name)'), '(SELECT text FROM comments WHERE book=books.id) comments', ('series', 'series', 'series', 'name'), ('publisher', 'publishers', 'publisher', 'name'), diff --git a/src/calibre/library/sqlite.py b/src/calibre/library/sqlite.py index 0458ada27b..0c3ae487ea 100644 --- a/src/calibre/library/sqlite.py +++ b/src/calibre/library/sqlite.py @@ -19,7 +19,7 @@ from calibre.ebooks.metadata import title_sort, author_to_author_sort from calibre.utils.date import parse_date, isoformat from calibre import isbytestring, force_unicode from calibre.constants import iswindows, DEBUG -from calibre.utils.icu import strcmp +from calibre.utils.icu import strcmp, sort_key global_lock = RLock() @@ -69,6 +69,25 @@ class Concatenate(object): return None return self.sep.join(self.ans) +class TagsSortConcatenate(object): + '''Sorted string concatenation aggregator for sqlite''' + def __init__(self, sep=','): + self.sep = sep + self.ans = [] + + def step(self, value): + if value is not None: + self.ans.append(value) + + def finalize(self): + if not self.ans: + return None + return self.sep.join(sorted(self.ans, key=sort_key)) + +class CcSortConcatenate(TagsSortConcatenate): + def __init__(self): + TagsSortConcatenate.__init__(self, sep='|') + class SortedConcatenate(object): '''String concatenation aggregator for sqlite, sorted by supplied index''' sep = ',' @@ -155,6 +174,8 @@ class DBThread(Thread): c_ext_loaded = load_c_extensions(self.conn) self.conn.row_factory = sqlite.Row if self.row_factory else lambda cursor, row : list(row) self.conn.create_aggregate('concat', 1, Concatenate) + self.conn.create_aggregate('tags_sortconcat', 1, TagsSortConcatenate) + self.conn.create_aggregate('cc_sortconcat', 1, CcSortConcatenate) if not c_ext_loaded: self.conn.create_aggregate('sortconcat', 2, SortedConcatenate) self.conn.create_aggregate('sort_concat', 2, SafeSortedConcatenate) From 84c6bcac39d515bda5e3344ad577d0ce214576d8 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Wed, 5 Jan 2011 12:59:18 +0000 Subject: [PATCH 3/8] Add ability to manipulate int, float, and bool columns in search replace --- src/calibre/gui2/dialogs/metadata_bulk.py | 5 ++++- src/calibre/library/custom_columns.py | 22 +++++++++++++++++++--- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/src/calibre/gui2/dialogs/metadata_bulk.py b/src/calibre/gui2/dialogs/metadata_bulk.py index ef14c95b1d..e1ee4327f3 100644 --- a/src/calibre/gui2/dialogs/metadata_bulk.py +++ b/src/calibre/gui2/dialogs/metadata_bulk.py @@ -321,7 +321,8 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog): if (f in ['author_sort'] or (fm[f]['datatype'] in ['text', 'series', 'enumeration'] and fm[f].get('search_terms', None) - and f not in ['formats', 'ondevice', 'sort'])): + and f not in ['formats', 'ondevice', 'sort']) or + fm[f]['datatype'] in ['int', 'float', 'bool'] ): self.all_fields.append(f) self.writable_fields.append(f) if f in ['sort'] or fm[f]['datatype'] == 'composite': @@ -431,6 +432,8 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog): val = mi.get('title_sort', None) else: val = mi.get(field, None) + if isinstance(val, (int, float, bool)): + val = str(val) if val is None: val = [] if fm['is_multiple'] else [''] elif not fm['is_multiple']: diff --git a/src/calibre/library/custom_columns.py b/src/calibre/library/custom_columns.py index 558f3b8fc9..ccdd55021d 100644 --- a/src/calibre/library/custom_columns.py +++ b/src/calibre/library/custom_columns.py @@ -134,7 +134,15 @@ class CustomColumns(object): def adapt_bool(x, d): if isinstance(x, (str, unicode, bytes)): - x = bool(int(x)) + x = x.lower() + if x == 'true': + x = True + elif x == 'false': + x = False + elif x == 'none': + x = None + else: + x = bool(int(x)) return x def adapt_enum(x, d): @@ -143,9 +151,17 @@ class CustomColumns(object): v = None return v + def adapt_number(x, d): + if isinstance(x, (str, unicode, bytes)): + if x.lower() == 'none': + return None + if d['datatype'] == 'int': + return int(x) + return float(x) + self.custom_data_adapters = { - 'float': lambda x,d : x if x is None else float(x), - 'int': lambda x,d : x if x is None else int(x), + 'float': adapt_number, + 'int': adapt_number, 'rating':lambda x,d : x if x is None else min(10., max(0., float(x))), 'bool': adapt_bool, 'comments': lambda x,d: adapt_text(x, {'is_multiple':False}), From 3f3944e95b6a4fd7d830d98187b78a45298796e8 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Wed, 5 Jan 2011 14:51:14 +0000 Subject: [PATCH 4/8] Enhancement #8035: Advanced Search, Titel/Author?series/Tag - Type Ahead Word Lists --- src/calibre/gui2/dialogs/search.py | 44 +++++++++++++++++++++++++----- src/calibre/gui2/dialogs/search.ui | 27 ++++++++++++++---- 2 files changed, 59 insertions(+), 12 deletions(-) diff --git a/src/calibre/gui2/dialogs/search.py b/src/calibre/gui2/dialogs/search.py index 8e8fd09652..62a0f8a9f1 100644 --- a/src/calibre/gui2/dialogs/search.py +++ b/src/calibre/gui2/dialogs/search.py @@ -3,7 +3,7 @@ __copyright__ = '2008, Kovid Goyal ' import re, copy -from PyQt4.QtGui import QDialog, QDialogButtonBox +from PyQt4.Qt import QDialog, QDialogButtonBox, QCompleter, Qt from calibre.gui2.dialogs.search_ui import Ui_Dialog from calibre.library.caches import CONTAINS_MATCH, EQUALS_MATCH @@ -22,6 +22,28 @@ class SearchDialog(QDialog, Ui_Dialog): key=lambda x: sort_key(x if x[0] != '#' else x[1:])) self.general_combo.addItems(searchables) + all_authors = db.all_authors() + all_authors.sort(key=lambda x : sort_key(x[1])) + for i in all_authors: + id, name = i + name = name.strip().replace('|', ',') + self.authors_box.addItem(name) + self.authors_box.setEditText('') + self.authors_box.completer().setCompletionMode(QCompleter.PopupCompletion) + self.authors_box.setAutoCompletionCaseSensitivity(Qt.CaseInsensitive) + + all_series = db.all_series() + all_series.sort(key=lambda x : sort_key(x[1])) + for i in all_series: + id, name = i + self.series_box.addItem(name) + self.series_box.setEditText('') + self.series_box.completer().setCompletionMode(QCompleter.PopupCompletion) + self.series_box.setAutoCompletionCaseSensitivity(Qt.CaseInsensitive) + + all_tags = db.all_tags() + self.tags_box.update_tags_cache(all_tags) + self.box_last_values = copy.deepcopy(box_values) if self.box_last_values: for k,v in self.box_last_values.items(): @@ -121,26 +143,34 @@ class SearchDialog(QDialog, Ui_Dialog): return tok def box_search_string(self): + mk = self.matchkind.currentIndex() + if mk == CONTAINS_MATCH: + self.mc = '' + elif mk == EQUALS_MATCH: + self.mc = '=' + else: + self.mc = '~' + ans = [] self.box_last_values = {} title = unicode(self.title_box.text()).strip() self.box_last_values['title_box'] = title if title: - ans.append('title:"' + title + '"') + ans.append('title:"' + self.mc + title + '"') author = unicode(self.authors_box.text()).strip() self.box_last_values['authors_box'] = author if author: - ans.append('author:"' + author + '"') + ans.append('author:"' + self.mc + author + '"') series = unicode(self.series_box.text()).strip() self.box_last_values['series_box'] = series if series: - ans.append('series:"' + series + '"') - self.mc = '=' + ans.append('series:"' + self.mc + series + '"') + tags = unicode(self.tags_box.text()) self.box_last_values['tags_box'] = tags - tags = self.tokens(tags) + tags = [t.strip() for t in tags.split(',') if t.strip()] if tags: - tags = ['tags:' + t for t in tags] + tags = ['tags:"=' + t + '"' for t in tags] ans.append('(' + ' or '.join(tags) + ')') general = unicode(self.general_box.text()) self.box_last_values['general_box'] = general diff --git a/src/calibre/gui2/dialogs/search.ui b/src/calibre/gui2/dialogs/search.ui index 7bb4c15363..6848a45506 100644 --- a/src/calibre/gui2/dialogs/search.ui +++ b/src/calibre/gui2/dialogs/search.ui @@ -21,7 +21,7 @@ - What kind of match to use: + &What kind of match to use: matchkind @@ -228,7 +228,7 @@ - + Enter the title. @@ -265,21 +265,21 @@ - + Enter an author's name. Only one author can be used. - + Enter a series name, without an index. Only one series name can be used. - + Enter tags separated by spaces @@ -348,6 +348,23 @@ + + + EnLineEdit + QLineEdit +
widgets.h
+ + + EnComboBox + QComboBox +
widgets.h
+ + + TagsLineEdit + QLineEdit +
widgets.h
+ + all phrase From 7e7f0954ce61c319b1223e8fe1baee53dfbb58ff Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Wed, 5 Jan 2011 18:51:35 +0000 Subject: [PATCH 5/8] New implementation of CacheRow --- src/calibre/library/caches.py | 54 ++++++++++++++--------------------- 1 file changed, 21 insertions(+), 33 deletions(-) diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py index 3a61a8fd5d..ada1ee0a77 100644 --- a/src/calibre/library/caches.py +++ b/src/calibre/library/caches.py @@ -132,47 +132,35 @@ def _match(query, value, matchkind): pass return False -class CacheRow(object): +class CacheRow(list): def __init__(self, db, composites, val): self.db = db - self.composites = composites - self._mydata = val + self._composites = composites + list.__init__(self, val) self._must_do = len(composites) > 0 def __getitem__(self, col): - rec = self._mydata - if self._must_do and col in self.composites: - self._must_do = False - mi = self.db.get_metadata(rec[0], index_is_id=True) - for c in self.composites: - rec[c] = mi.get(self.composites[c]) - return rec[col] + if self._must_do: + is_comp = False + if isinstance(col, slice): + for c in range(col.start, col.stop): + if c in self._composites: + is_comp = True + break + elif col in self._composites: + is_comp = True + if is_comp: + id = list.__getitem__(self, 0) + self._must_do = False + mi = self.db.get_metadata(id, index_is_id=True) + for c in self._composites: + self[c] = mi.get(self._composites[c]) + return list.__getitem__(self, col) - def __setitem__ (self, col, val): - self._mydata[col] = val + def __getslice__(self, i, j): + return self.__getitem__(slice(i, j)) - def append(self, val): - self._mydata.append(val) - - def get(self, col, default): - try: - return self.__getitem__(col) - except: - return default - - def __len__(self): - return len(self._mydata) - - def __iter__(self): - for v in self._mydata: - yield v - - def __str__(self): - return self.__unicode__() - - def __unicode__(self): - return unicode(self._mydata) class ResultCache(SearchQueryParser): # {{{ From cbd880e8f736fe3ee20c3fea62a1895a8848194f Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Wed, 5 Jan 2011 19:17:01 +0000 Subject: [PATCH 6/8] Take out sorting change --- src/calibre/gui2/library/models.py | 4 ++-- src/calibre/library/custom_columns.py | 31 ++++++--------------------- src/calibre/library/database2.py | 2 +- src/calibre/library/sqlite.py | 23 +------------------- 4 files changed, 11 insertions(+), 49 deletions(-) diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py index 6a48aef9be..49cb1ce182 100644 --- a/src/calibre/gui2/library/models.py +++ b/src/calibre/gui2/library/models.py @@ -526,7 +526,7 @@ class BooksModel(QAbstractTableModel): # {{{ def tags(r, idx=-1): tags = self.db.data[r][idx] if tags: - return QVariant(', '.join(tags.split(','))) + return QVariant(', '.join(sorted(tags.split(','), key=sort_key))) return None def series_type(r, idx=-1, siix=-1): @@ -577,7 +577,7 @@ class BooksModel(QAbstractTableModel): # {{{ def text_type(r, mult=False, idx=-1): text = self.db.data[r][idx] if text and mult: - return QVariant(', '.join(text.split('|'))) + return QVariant(', '.join(sorted(text.split('|'),key=sort_key))) return QVariant(text) def number_type(r, idx=-1): diff --git a/src/calibre/library/custom_columns.py b/src/calibre/library/custom_columns.py index ccdd55021d..07ea407460 100644 --- a/src/calibre/library/custom_columns.py +++ b/src/calibre/library/custom_columns.py @@ -14,7 +14,6 @@ from calibre.constants import preferred_encoding from calibre.library.field_metadata import FieldMetadata from calibre.utils.date import parse_date from calibre.utils.config import tweaks -from calibre.utils.icu import sort_key class CustomColumns(object): @@ -134,15 +133,7 @@ class CustomColumns(object): def adapt_bool(x, d): if isinstance(x, (str, unicode, bytes)): - x = x.lower() - if x == 'true': - x = True - elif x == 'false': - x = False - elif x == 'none': - x = None - else: - x = bool(int(x)) + x = bool(int(x)) return x def adapt_enum(x, d): @@ -151,17 +142,9 @@ class CustomColumns(object): v = None return v - def adapt_number(x, d): - if isinstance(x, (str, unicode, bytes)): - if x.lower() == 'none': - return None - if d['datatype'] == 'int': - return int(x) - return float(x) - self.custom_data_adapters = { - 'float': adapt_number, - 'int': adapt_number, + 'float': lambda x,d : x if x is None else float(x), + 'int': lambda x,d : x if x is None else int(x), 'rating':lambda x,d : x if x is None else min(10., max(0., float(x))), 'bool': adapt_bool, 'comments': lambda x,d: adapt_text(x, {'is_multiple':False}), @@ -198,8 +181,8 @@ class CustomColumns(object): ans = row[self.FIELD_MAP[data['num']]] if data['is_multiple'] and data['datatype'] == 'text': ans = ans.split('|') if ans else [] - if data['display'].get('sort_alpha', True): - ans.sort(key=sort_key) + if data['display'].get('sort_alpha', False): + ans.sort(cmp=lambda x,y:cmp(x.lower(), y.lower())) return ans def get_custom_extra(self, idx, label=None, num=None, index_is_id=False): @@ -551,8 +534,8 @@ class CustomColumns(object): if data['normalized']: query = '%s.value' if data['is_multiple']: - query = 'cc_sortconcat(%s.value)' - if not display.get('sort_alpha', True): + query = 'group_concat(%s.value, "|")' + if not display.get('sort_alpha', False): query = 'sort_concat(link.id, %s.value)' line = '''(SELECT {query} FROM {lt} AS link INNER JOIN {table} ON(link.value={table}.id) WHERE link.book=books.id) diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 0b1c6a6cfb..611aa1cc89 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -242,7 +242,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): 'timestamp', '(SELECT MAX(uncompressed_size) FROM data WHERE book=books.id) size', ('rating', 'ratings', 'rating', 'ratings.rating'), - ('tags', 'tags', 'tag', 'tags_sortconcat(name)'), + ('tags', 'tags', 'tag', 'group_concat(name)'), '(SELECT text FROM comments WHERE book=books.id) comments', ('series', 'series', 'series', 'name'), ('publisher', 'publishers', 'publisher', 'name'), diff --git a/src/calibre/library/sqlite.py b/src/calibre/library/sqlite.py index 0c3ae487ea..0458ada27b 100644 --- a/src/calibre/library/sqlite.py +++ b/src/calibre/library/sqlite.py @@ -19,7 +19,7 @@ from calibre.ebooks.metadata import title_sort, author_to_author_sort from calibre.utils.date import parse_date, isoformat from calibre import isbytestring, force_unicode from calibre.constants import iswindows, DEBUG -from calibre.utils.icu import strcmp, sort_key +from calibre.utils.icu import strcmp global_lock = RLock() @@ -69,25 +69,6 @@ class Concatenate(object): return None return self.sep.join(self.ans) -class TagsSortConcatenate(object): - '''Sorted string concatenation aggregator for sqlite''' - def __init__(self, sep=','): - self.sep = sep - self.ans = [] - - def step(self, value): - if value is not None: - self.ans.append(value) - - def finalize(self): - if not self.ans: - return None - return self.sep.join(sorted(self.ans, key=sort_key)) - -class CcSortConcatenate(TagsSortConcatenate): - def __init__(self): - TagsSortConcatenate.__init__(self, sep='|') - class SortedConcatenate(object): '''String concatenation aggregator for sqlite, sorted by supplied index''' sep = ',' @@ -174,8 +155,6 @@ class DBThread(Thread): c_ext_loaded = load_c_extensions(self.conn) self.conn.row_factory = sqlite.Row if self.row_factory else lambda cursor, row : list(row) self.conn.create_aggregate('concat', 1, Concatenate) - self.conn.create_aggregate('tags_sortconcat', 1, TagsSortConcatenate) - self.conn.create_aggregate('cc_sortconcat', 1, CcSortConcatenate) if not c_ext_loaded: self.conn.create_aggregate('sortconcat', 2, SortedConcatenate) self.conn.create_aggregate('sort_concat', 2, SafeSortedConcatenate) From 9ee7dc27e85f1d6e948210818a4b68082bf5792d Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Wed, 5 Jan 2011 19:21:17 +0000 Subject: [PATCH 7/8] Do slice stepping correctly --- src/calibre/library/caches.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py index ada1ee0a77..2596b494bf 100644 --- a/src/calibre/library/caches.py +++ b/src/calibre/library/caches.py @@ -144,7 +144,9 @@ class CacheRow(list): if self._must_do: is_comp = False if isinstance(col, slice): - for c in range(col.start, col.stop): + start = 0 if col.start is None else col.start + step = 1 if col.stop is None else col.stop + for c in range(start, col.stop, step): if c in self._composites: is_comp = True break From 5adaf263e4c29cc3810a198f44777d90b5999ad5 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Wed, 5 Jan 2011 19:52:55 +0000 Subject: [PATCH 8/8] Ensure that multiples are sorted in alpha order before column sort and during template processing --- src/calibre/ebooks/metadata/book/base.py | 7 ++++--- src/calibre/library/caches.py | 15 ++++++--------- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/src/calibre/ebooks/metadata/book/base.py b/src/calibre/ebooks/metadata/book/base.py index 77df6b00c2..17f2c6705c 100644 --- a/src/calibre/ebooks/metadata/book/base.py +++ b/src/calibre/ebooks/metadata/book/base.py @@ -16,6 +16,7 @@ from calibre.ebooks.metadata.book import TOP_LEVEL_CLASSIFIERS from calibre.ebooks.metadata.book import ALL_METADATA_FIELDS from calibre.library.field_metadata import FieldMetadata from calibre.utils.date import isoformat, format_date +from calibre.utils.icu import sort_key from calibre.utils.formatter import TemplateFormatter @@ -490,7 +491,7 @@ class Metadata(object): return authors_to_string(self.authors) def format_tags(self): - return u', '.join([unicode(t) for t in self.tags]) + return u', '.join([unicode(t) for t in sorted(self.tags, key=sort_key)]) def format_rating(self): return unicode(self.rating) @@ -530,7 +531,7 @@ class Metadata(object): orig_res = res datatype = cmeta['datatype'] if datatype == 'text' and cmeta['is_multiple']: - res = u', '.join(res) + res = u', '.join(sorted(res, key=sort_key)) elif datatype == 'series' and series_with_index: if self.get_extra(key) is not None: res = res + \ @@ -560,7 +561,7 @@ class Metadata(object): elif key == 'series_index': res = self.format_series_index(res) elif datatype == 'text' and fmeta['is_multiple']: - res = u', '.join(res) + res = u', '.join(sorted(res, key=sort_key)) elif datatype == 'series' and series_with_index: res = res + ' [%s]'%self.format_series_index() elif datatype == 'datetime': diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py index 2596b494bf..980c9f1fa9 100644 --- a/src/calibre/library/caches.py +++ b/src/calibre/library/caches.py @@ -691,13 +691,7 @@ class ResultCache(SearchQueryParser): # {{{ fields = [('timestamp', False)] keyg = SortKeyGenerator(fields, self.field_metadata, self._data) - # For efficiency, the key generator returns a plain value if only one - # field is in the sort field list. Because the normal cmp function will - # always assume asc, we must deal with asc/desc here. - if len(fields) == 1: - self._map.sort(key=keyg, reverse=not fields[0][1]) - else: - self._map.sort(key=keyg) + self._map.sort(key=keyg) tmap = list(itertools.repeat(False, len(self._data))) for x in self._map_filtered: @@ -730,8 +724,6 @@ class SortKeyGenerator(object): def __call__(self, record): values = tuple(self.itervals(self.data[record])) - if len(values) == 1: - return values[0] return SortKey(self.orders, values) def itervals(self, record): @@ -754,6 +746,11 @@ class SortKeyGenerator(object): val = (self.string_sort_key(val), sidx) elif dt in ('text', 'comments', 'composite', 'enumeration'): + if val: + sep = fm['is_multiple'] + if sep: + val = sep.join(sorted(val.split(sep), + key=self.string_sort_key)) val = self.string_sort_key(val) elif dt == 'bool':