diff --git a/src/calibre/gui2/dialogs/config/__init__.py b/src/calibre/gui2/dialogs/config/__init__.py index 9d108d3807..4d0dd07746 100644 --- a/src/calibre/gui2/dialogs/config/__init__.py +++ b/src/calibre/gui2/dialogs/config/__init__.py @@ -371,7 +371,7 @@ class ConfigDialog(ResizableDialog, Ui_Dialog): hidden_cols = state['hidden_columns'] positions = state['column_positions'] colmap.sort(cmp=lambda x,y: cmp(positions[x], positions[y])) - self.custcols = copy.deepcopy(self.db.custom_column_label_map) + self.custcols = copy.deepcopy(self.db.field_metadata.get_custom_field_metadata()) for col in colmap: item = QListWidgetItem(self.model.headers[col], self.columns) item.setData(Qt.UserRole, QVariant(col)) @@ -713,20 +713,20 @@ class ConfigDialog(ResizableDialog, Ui_Dialog): must_restart = False for c in self.custcols: - if self.custcols[c]['num'] is None: + if self.custcols[c]['colnum'] is None: self.db.create_custom_column( - label=c, + label=self.custcols[c]['label'], name=self.custcols[c]['name'], datatype=self.custcols[c]['datatype'], is_multiple=self.custcols[c]['is_multiple'], display = self.custcols[c]['display']) must_restart = True elif '*deleteme' in self.custcols[c]: - self.db.delete_custom_column(label=c) + self.db.delete_custom_column(label=self.custcols[c]['label']) must_restart = True elif '*edited' in self.custcols[c]: cc = self.custcols[c] - self.db.set_custom_column_metadata(cc['num'], name=cc['name'], + self.db.set_custom_column_metadata(cc['colnum'], name=cc['name'], label=cc['label'], display = self.custcols[c]['display']) if '*must_restart' in self.custcols[c]: diff --git a/src/calibre/gui2/dialogs/config/create_custom_column.py b/src/calibre/gui2/dialogs/config/create_custom_column.py index 9e040315c9..693f079d12 100644 --- a/src/calibre/gui2/dialogs/config/create_custom_column.py +++ b/src/calibre/gui2/dialogs/config/create_custom_column.py @@ -69,13 +69,14 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn): self.column_name_box.setText(c['label']) self.column_heading_box.setText(c['name']) ct = c['datatype'] if not c['is_multiple'] else '*text' - self.orig_column_number = c['num'] + self.orig_column_number = c['colnum'] self.orig_column_name = col column_numbers = dict(map(lambda x:(self.column_types[x]['datatype'], x), self.column_types)) self.column_type_box.setCurrentIndex(column_numbers[ct]) self.column_type_box.setEnabled(False) if ct == 'datetime': - self.date_format_box.setText(c['display'].get('date_format', '')) + if c['display'].get('date_format', None): + self.date_format_box.setText(c['display'].get('date_format', '')) self.datatype_changed() self.exec_() @@ -90,7 +91,9 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn): def accept(self): - col = unicode(self.column_name_box.text()) + col = unicode(self.column_name_box.text()).lower() + if not col.isalnum(): + return self.simple_error('', _('The label must contain only letters and digits')) col_heading = unicode(self.column_heading_box.text()) col_type = self.column_types[self.column_type_box.currentIndex()]['datatype'] if col_type == '*text': @@ -104,14 +107,14 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn): return self.simple_error('', _('No column heading was provided')) bad_col = False if col in self.parent.custcols: - if not self.editing_col or self.parent.custcols[col]['num'] != self.orig_column_number: + if not self.editing_col or self.parent.custcols[col]['colnum'] != self.orig_column_number: bad_col = True if bad_col: return self.simple_error('', _('The lookup name %s is already used')%col) bad_head = False for t in self.parent.custcols: if self.parent.custcols[t]['name'] == col_heading: - if not self.editing_col or self.parent.custcols[t]['num'] != self.orig_column_number: + if not self.editing_col or self.parent.custcols[t]['colnum'] != self.orig_column_number: bad_head = True for t in self.standard_colheads: if self.standard_colheads[t] == col_heading: @@ -129,14 +132,15 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn): date_format = {'date_format': None} if not self.editing_col: - self.parent.custcols[col] = { + self.parent.db.field_metadata + self.parent.custcols[self.parent.db.field_metadata.custom_field_prefix+col] = { 'label':col, 'name':col_heading, 'datatype':col_type, 'editable':True, 'display':date_format, 'normalized':None, - 'num':None, + 'colnum':None, 'is_multiple':is_multiple, } item = QListWidgetItem(col_heading, self.parent.columns) diff --git a/src/calibre/gui2/dialogs/config/create_custom_column.ui b/src/calibre/gui2/dialogs/config/create_custom_column.ui index 279349f28e..5cb9494845 100644 --- a/src/calibre/gui2/dialogs/config/create_custom_column.ui +++ b/src/calibre/gui2/dialogs/config/create_custom_column.ui @@ -65,7 +65,7 @@ - Used for searching the column. Must be lower case and not contain spaces or colons. + Used for searching the column. Must contain only digits and lower case letters. diff --git a/src/calibre/gui2/dialogs/tag_categories.py b/src/calibre/gui2/dialogs/tag_categories.py index fdec767d4d..fcf517e571 100644 --- a/src/calibre/gui2/dialogs/tag_categories.py +++ b/src/calibre/gui2/dialogs/tag_categories.py @@ -49,7 +49,7 @@ class TagCategories(QDialog, Ui_TagCategories): cc_map = self.db.custom_column_label_map for cc in cc_map: if cc_map[cc]['datatype'] == 'text': - self.category_labels.append(db.tag_browser_categories.get_search_label(cc)) + self.category_labels.append(db.field_metadata.label_to_key(cc)) category_icons.append(cc_icon) category_values.append(lambda col=cc: self.db.all_custom(label=col)) category_names.append(cc_map[cc]['name']) diff --git a/src/calibre/gui2/library/delegates.py b/src/calibre/gui2/library/delegates.py index d908ed01b4..529055ecd2 100644 --- a/src/calibre/gui2/library/delegates.py +++ b/src/calibre/gui2/library/delegates.py @@ -171,7 +171,8 @@ class TagsDelegate(QStyledItemDelegate): # {{{ if not index.model().is_custom_column(col): editor = TagsLineEdit(parent, self.db.all_tags()) else: - editor = TagsLineEdit(parent, sorted(list(self.db.all_custom(label=col)))) + editor = TagsLineEdit(parent, + sorted(list(self.db.all_custom(label=self.db.field_metadata.key_to_label(col))))) return editor else: editor = EnLineEdit(parent) @@ -209,7 +210,7 @@ class CcDateDelegate(QStyledItemDelegate): # {{{ m = index.model() # db col is not named for the field, but for the table number. To get it, # gui column -> column label -> table number -> db column - val = m.db.data[index.row()][m.db.FIELD_MAP[m.custom_columns[m.column_map[index.column()]]['num']]] + val = m.db.data[index.row()][m.custom_columns[m.column_map[index.column()]]['rec_index']] if val is None: val = now() editor.setDate(val) @@ -243,7 +244,7 @@ class CcTextDelegate(QStyledItemDelegate): # {{{ editor.setDecimals(2) else: editor = EnLineEdit(parent) - complete_items = sorted(list(m.db.all_custom(label=col))) + complete_items = sorted(list(m.db.all_custom(label=m.db.field_metadata.key_to_label(col)))) completer = QCompleter(complete_items, self) completer.setCaseSensitivity(Qt.CaseInsensitive) completer.setCompletionMode(QCompleter.PopupCompletion) @@ -260,9 +261,7 @@ class CcCommentsDelegate(QStyledItemDelegate): # {{{ def createEditor(self, parent, option, index): m = index.model() col = m.column_map[index.column()] - # db col is not named for the field, but for the table number. To get it, - # gui column -> column label -> table number -> db column - text = m.db.data[index.row()][m.db.FIELD_MAP[m.custom_columns[col]['num']]] + text = m.db.data[index.row()][m.custom_columns[col]['rec_index']] editor = CommentsDialog(parent, text) d = editor.exec_() if d: @@ -297,9 +296,7 @@ class CcBoolDelegate(QStyledItemDelegate): # {{{ def setEditorData(self, editor, index): m = index.model() - # db col is not named for the field, but for the table number. To get it, - # gui column -> column label -> table number -> db column - val = m.db.data[index.row()][m.db.FIELD_MAP[m.custom_columns[m.column_map[index.column()]]['num']]] + val = m.db.data[index.row()][m.custom_columns[m.column_map[index.column()]]['rec_index']] if tweaks['bool_custom_columns_are_tristate'] == 'no': val = 1 if not val else 0 else: diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py index a871ce2aa3..5490e96169 100644 --- a/src/calibre/gui2/library/models.py +++ b/src/calibre/gui2/library/models.py @@ -111,15 +111,15 @@ class BooksModel(QAbstractTableModel): # {{{ def set_database(self, db): self.db = db - self.custom_columns = self.db.custom_column_label_map + self.custom_columns = self.db.field_metadata.get_custom_field_metadata() self.column_map = list(self.orig_headers.keys()) + \ list(self.custom_columns) def col_idx(name): if name == 'ondevice': return -1 - if name not in self.db.FIELD_MAP: + if name not in self.db.field_metadata: return 100000 - return self.db.FIELD_MAP[name] + return self.db.field_metadata[name]['rec_index'] self.column_map.sort(cmp=lambda x,y: cmp(col_idx(x), col_idx(y))) for col in self.column_map: @@ -232,11 +232,12 @@ class BooksModel(QAbstractTableModel): # {{{ return self.about_to_be_sorted.emit(self.db.id) ascending = order == Qt.AscendingOrder - self.db.sort(self.column_map[col], ascending) + label = self.column_map[col] + self.db.sort(label, ascending) if reset: self.clear_caches() self.reset() - self.sorted_on = (self.column_map[col], order) + self.sorted_on = (label, order) self.sort_history.insert(0, self.sorted_on) self.sorting_done.emit(self.db.index) @@ -551,36 +552,36 @@ class BooksModel(QAbstractTableModel): # {{{ self.dc = { 'title' : functools.partial(text_type, - idx=self.db.FIELD_MAP['title'], mult=False), + idx=self.db.field_metadata['title']['rec_index'], mult=False), 'authors' : functools.partial(authors, - idx=self.db.FIELD_MAP['authors']), + idx=self.db.field_metadata['authors']['rec_index']), 'size' : functools.partial(size, - idx=self.db.FIELD_MAP['size']), + idx=self.db.field_metadata['size']['rec_index']), 'timestamp': functools.partial(datetime_type, - idx=self.db.FIELD_MAP['timestamp']), + idx=self.db.field_metadata['timestamp']['rec_index']), 'pubdate' : functools.partial(datetime_type, - idx=self.db.FIELD_MAP['pubdate']), + idx=self.db.field_metadata['pubdate']['rec_index']), 'rating' : functools.partial(rating_type, - idx=self.db.FIELD_MAP['rating']), + idx=self.db.field_metadata['rating']['rec_index']), 'publisher': functools.partial(text_type, - idx=self.db.FIELD_MAP['publisher'], mult=False), + idx=self.db.field_metadata['publisher']['rec_index'], mult=False), 'tags' : functools.partial(tags, - idx=self.db.FIELD_MAP['tags']), + idx=self.db.field_metadata['tags']['rec_index']), 'series' : functools.partial(series, - idx=self.db.FIELD_MAP['series'], - siix=self.db.FIELD_MAP['series_index']), + idx=self.db.field_metadata['series']['rec_index'], + siix=self.db.field_metadata['series_index']['rec_index']), 'ondevice' : functools.partial(text_type, - idx=self.db.FIELD_MAP['ondevice'], mult=False), + idx=self.db.field_metadata['ondevice']['rec_index'], mult=False), } self.dc_decorator = { 'ondevice':functools.partial(ondevice_decorator, - idx=self.db.FIELD_MAP['ondevice']), + idx=self.db.field_metadata['ondevice']['rec_index']), } # Add the custom columns to the data converters for col in self.custom_columns: - idx = self.db.FIELD_MAP[self.custom_columns[col]['num']] + idx = self.custom_columns[col]['rec_index'] datatype = self.custom_columns[col]['datatype'] if datatype in ('text', 'comments'): self.dc[col] = functools.partial(text_type, idx=idx, mult=self.custom_columns[col]['is_multiple']) @@ -632,8 +633,6 @@ class BooksModel(QAbstractTableModel): # {{{ return None if role == Qt.ToolTipRole: ht = self.column_map[section] - if self.is_custom_column(self.column_map[section]): - ht = self.db.tag_browser_categories.custom_field_prefix + ht if ht == 'timestamp': # change help text because users know this field as 'date' ht = 'date' return QVariant(_('The lookup/search name is "{0}"').format(ht)) @@ -652,7 +651,7 @@ class BooksModel(QAbstractTableModel): # {{{ if colhead in self.editable_cols: flags |= Qt.ItemIsEditable elif self.is_custom_column(colhead): - if self.custom_columns[colhead]['editable']: + if self.custom_columns[colhead]['is_editable']: flags |= Qt.ItemIsEditable return flags @@ -679,7 +678,9 @@ class BooksModel(QAbstractTableModel): # {{{ if not val.isValid(): return False val = qt_to_dt(val, as_utc=False) - self.db.set_custom(self.db.id(row), val, label=colhead, num=None, append=False, notify=True) + self.db.set_custom(self.db.id(row), val, + label=self.db.field_metadata.key_to_label(colhead), + num=None, append=False, notify=True) return True def setData(self, index, value, role): diff --git a/src/calibre/gui2/tag_view.py b/src/calibre/gui2/tag_view.py index 3882e4e174..8ecc26e30c 100644 --- a/src/calibre/gui2/tag_view.py +++ b/src/calibre/gui2/tag_view.py @@ -224,7 +224,7 @@ class TagsModel(QAbstractItemModel): # {{{ data = self.get_node_tree(config['sort_by_popularity']) self.root_item = TagTreeItem() for i, r in enumerate(self.row_map): - if self.db.get_tag_browser_categories()[r]['kind'] != 'user': + if self.db.field_metadata[r]['kind'] != 'user': tt = _('The lookup/search name is "{0}"').format(r) else: tt = '' @@ -248,7 +248,7 @@ class TagsModel(QAbstractItemModel): # {{{ else: data = self.db.get_categories(sort_on_count=sort, icon_map=self.category_icon_map) - tb_categories = self.db.get_tag_browser_categories() + tb_categories = self.db.field_metadata for category in tb_categories: if category in data: # They should always be there, but ... self.row_map.append(category) diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py index 36698533c5..93891ee92b 100644 --- a/src/calibre/library/caches.py +++ b/src/calibre/library/caches.py @@ -150,14 +150,13 @@ class ResultCache(SearchQueryParser): ''' Stores sorted and filtered metadata in memory. ''' - def __init__(self, FIELD_MAP, cc_label_map, tag_browser_categories): + def __init__(self, FIELD_MAP, field_metadata): self.FIELD_MAP = FIELD_MAP - self.custom_column_label_map = cc_label_map self._map = self._map_filtered = self._data = [] self.first_sort = True self.search_restriction = '' - self.tag_browser_categories = tag_browser_categories - self.all_search_locations = tag_browser_categories.get_search_labels() + self.field_metadata = field_metadata + self.all_search_locations = field_metadata.get_search_terms() SearchQueryParser.__init__(self, self.all_search_locations) self.build_date_relop_dict() self.build_numeric_relop_dict() @@ -249,10 +248,10 @@ class ResultCache(SearchQueryParser): query = query[p:] if relop is None: (p, relop) = self.date_search_relops['='] - if location in self.custom_column_label_map: - loc = self.FIELD_MAP[self.custom_column_label_map[location]['num']] - else: - loc = self.FIELD_MAP[{'date':'timestamp', 'pubdate':'pubdate'}[location]] + + if location == 'date': + location = 'timestamp' + loc = self.field_metadata[location]['rec_index'] if query == _('today'): qd = now() @@ -310,22 +309,18 @@ class ResultCache(SearchQueryParser): query = query[p:] if relop is None: (p, relop) = self.numeric_search_relops['='] - if location in self.custom_column_label_map: - loc = self.FIELD_MAP[self.custom_column_label_map[location]['num']] - dt = self.custom_column_label_map[location]['datatype'] - if dt == 'int': - cast = (lambda x: int (x)) - adjust = lambda x: x - elif dt == 'rating': - cast = (lambda x: int (x)) - adjust = lambda x: x/2 - elif dt == 'float': - cast = lambda x : float (x) - adjust = lambda x: x - else: - loc = self.FIELD_MAP['rating'] + + loc = self.field_metadata[location]['rec_index'] + dt = self.field_metadata[location]['datatype'] + if dt == 'int': + cast = (lambda x: int (x)) + adjust = lambda x: x + elif dt == 'rating': cast = (lambda x: int (x)) adjust = lambda x: x/2 + elif dt == 'float': + cast = lambda x : float (x) + adjust = lambda x: x try: q = cast(query) @@ -346,22 +341,21 @@ class ResultCache(SearchQueryParser): def get_matches(self, location, query): matches = set([]) if query and query.strip(): - location = location.lower().strip() + # get metadata key associated with the search term. Eliminates + # dealing with plurals and other aliases + location = self.field_metadata.search_term_to_key(location.lower().strip()) - ### take care of dates special case - if (location in ('pubdate', 'date')) or \ - ((location in self.custom_column_label_map) and \ - self.custom_column_label_map[location]['datatype'] == 'datetime'): + # take care of dates special case + if location in self.field_metadata and \ + self.field_metadata[location]['datatype'] == 'datetime': return self.get_dates_matches(location, query.lower()) - ### take care of numerics special case - if location == 'rating' or \ - (location in self.custom_column_label_map and - self.custom_column_label_map[location]['datatype'] in - ('rating', 'int', 'float')): + # take care of numbers special case + if location in self.field_metadata and \ + self.field_metadata[location]['datatype'] in ('rating', 'int', 'float'): return self.get_numeric_matches(location, query.lower()) - ### everything else + # everything else, or 'all' matches matchkind = CONTAINS_MATCH if (len(query) > 1): if query.startswith('\\'): @@ -372,57 +366,41 @@ class ResultCache(SearchQueryParser): elif query.startswith('~'): matchkind = REGEXP_MATCH query = query[1:] - if matchkind != REGEXP_MATCH: ### leave case in regexps because it can be significant e.g. \S \W \D + if matchkind != REGEXP_MATCH: + # leave case in regexps because it can be significant e.g. \S \W \D query = query.lower() if not isinstance(query, unicode): query = query.decode('utf-8') - if location in ('tag', 'author', 'format', 'comment'): - location += 's' - MAP = {} - # Fields not used when matching against text contents. These are - # the non-text fields - EXCLUDE_FIELDS = [] - - # get the db columns for the standard searchables - for x in self.tag_browser_categories: - if len(self.tag_browser_categories[x]['search_labels']) and \ - not self.tag_browser_categories.is_custom_field(x): - MAP[x] = self.tag_browser_categories[x]['rec_index'] - if self.tag_browser_categories[x]['datatype'] != 'text': - EXCLUDE_FIELDS.append(MAP[x]) - - # add custom columns to MAP. Put the column's type into IS_CUSTOM - IS_CUSTOM = [] + db_col = {} + exclude_fields = [] # fields to not check when matching against text. + col_datatype = [] + is_multiple_cols = {} for x in range(len(self.FIELD_MAP)): - IS_CUSTOM.append('') - # normal and custom ratings columns use the same code - IS_CUSTOM[self.FIELD_MAP['rating']] = 'rating' - for x in self.tag_browser_categories.get_custom_fields(): - if self.tag_browser_categories[x]['datatype'] != "datetime": - MAP[x] = self.FIELD_MAP[self.tag_browser_categories[x]['colnum']] - IS_CUSTOM[MAP[x]] = self.tag_browser_categories[x]['datatype'] - - SPLITABLE_FIELDS = [MAP['authors'], MAP['tags'], MAP['formats']] - for x in self.tag_browser_categories.get_custom_fields(): - if self.tag_browser_categories[x]['is_multiple']: - SPLITABLE_FIELDS.append(MAP[x]) + col_datatype.append('') + for x in self.field_metadata: + if len(self.field_metadata[x]['search_terms']): + db_col[x] = self.field_metadata[x]['rec_index'] + if self.field_metadata[x]['datatype'] not in ['text', 'comments']: + exclude_fields.append(db_col[x]) + col_datatype[db_col[x]] = self.field_metadata[x]['datatype'] + is_multiple_cols[db_col[x]] = self.field_metadata[x]['is_multiple'] try: rating_query = int(query) * 2 except: rating_query = None - location = [location] if location != 'all' else list(MAP.keys()) + location = [location] if location != 'all' else list(db_col.keys()) for i, loc in enumerate(location): - location[i] = MAP[loc] + location[i] = db_col[loc] # get the tweak here so that the string lookup and compare aren't in the loop bools_are_tristate = tweaks['bool_custom_columns_are_tristate'] == 'yes' - for loc in location: - if loc == MAP['authors']: + for loc in location: # location is now an array of field indices + if loc == db_col['authors']: ### DB stores authors with commas changed to bars, so change query q = query.replace(',', '|'); else: @@ -431,7 +409,7 @@ class ResultCache(SearchQueryParser): for item in self._data: if item is None: continue - if IS_CUSTOM[loc] == 'bool': # complexity caused by the two-/three-value tweak + if col_datatype[loc] == 'bool': # complexity caused by the two-/three-value tweak v = item[loc] if not bools_are_tristate: if v is None or not v: # item is None or set to false @@ -466,18 +444,18 @@ class ResultCache(SearchQueryParser): matches.add(item[0]) continue - if IS_CUSTOM[loc] == 'rating': # get here if 'all' query + if col_datatype[loc] == 'rating': # get here if 'all' query if rating_query and rating_query == int(item[loc]): matches.add(item[0]) continue try: # a conversion below might fail - # relationals not supported in 'all' queries - if IS_CUSTOM[loc] == 'float': + # relationals are not supported in 'all' queries + if col_datatype[loc] == 'float': if float(query) == item[loc]: matches.add(item[0]) continue - if IS_CUSTOM[loc] == 'int': + if col_datatype[loc] == 'int': if int(query) == item[loc]: matches.add(item[0]) continue @@ -486,12 +464,9 @@ class ResultCache(SearchQueryParser): # no further match is possible continue - if loc not in EXCLUDE_FIELDS: - if loc in SPLITABLE_FIELDS: - if IS_CUSTOM[loc]: - vals = item[loc].split('|') - else: - vals = item[loc].split(',') + if loc not in exclude_fields: # time for text matching + if is_multiple_cols[loc] is not None: + vals = item[loc].split(is_multiple_cols[loc]) else: vals = [item[loc]] ### make into list to make _match happy if _match(q, vals, matchkind): @@ -622,9 +597,9 @@ class ResultCache(SearchQueryParser): elif field == 'title': field = 'sort' elif field == 'authors': field = 'author_sort' as_string = field not in ('size', 'rating', 'timestamp') - if field in self.custom_column_label_map: - as_string = self.custom_column_label_map[field]['datatype'] in ('comments', 'text') - field = self.custom_column_label_map[field]['num'] + if self.field_metadata[field]['is_custom']: + as_string = self.field_metadata[field]['datatype'] in ('comments', 'text') + field = self.field_metadata[field]['colnum'] if self.first_sort: subsort = True diff --git a/src/calibre/library/custom_columns.py b/src/calibre/library/custom_columns.py index 90af46cc42..83e6b029cb 100644 --- a/src/calibre/library/custom_columns.py +++ b/src/calibre/library/custom_columns.py @@ -144,14 +144,19 @@ class CustomColumns(object): for k in sorted(self.custom_column_label_map.keys()): v = self.custom_column_label_map[k] if v['normalized']: - searchable = True + is_category = True else: - searchable = False + is_category = False + if v['is_multiple']: + is_m = '|' + else: + is_m = None tn = 'custom_column_{0}'.format(v['num']) - self.tag_browser_categories.add_custom_field(label=v['label'], + self.field_metadata.add_custom_field(label=v['label'], table=tn, column='value', datatype=v['datatype'], - is_multiple=v['is_multiple'], colnum=v['num'], - name=v['name'], searchable=searchable) + colnum=v['num'], name=v['name'], display=v['display'], + is_multiple=is_m, is_category=is_category, + is_editable=v['editable']) def get_custom(self, idx, label=None, num=None, index_is_id=False): if label is not None: diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index a69d6bab57..5ba603cc52 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -116,7 +116,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): self.books_list_filter = self.conn.create_dynamic_filter('books_list_filter') def __init__(self, library_path, row_factory=False): - self.tag_browser_categories = FieldMetadata() #.get_tag_browser_categories() + self.field_metadata = FieldMetadata() if not os.path.exists(library_path): os.makedirs(library_path) self.listeners = set([]) @@ -206,20 +206,20 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): 'lccn':16, 'pubdate':17, 'flags':18, 'uuid':19} for k,v in self.FIELD_MAP.iteritems(): - self.tag_browser_categories.set_field_record_index(k, v, prefer_custom=False) + self.field_metadata.set_field_record_index(k, v, prefer_custom=False) base = max(self.FIELD_MAP.values()) for col in custom_cols: self.FIELD_MAP[col] = base = base+1 - self.tag_browser_categories.set_field_record_index( + self.field_metadata.set_field_record_index( self.custom_column_num_map[col]['label'], base, prefer_custom=True) self.FIELD_MAP['cover'] = base+1 - self.tag_browser_categories.set_field_record_index('cover', base+1, prefer_custom=False) + self.field_metadata.set_field_record_index('cover', base+1, prefer_custom=False) self.FIELD_MAP['ondevice'] = base+2 - self.tag_browser_categories.set_field_record_index('ondevice', base+2, prefer_custom=False) + self.field_metadata.set_field_record_index('ondevice', base+2, prefer_custom=False) script = ''' DROP VIEW IF EXISTS meta2; @@ -232,8 +232,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): self.conn.commit() self.book_on_device_func = None - self.data = ResultCache(self.FIELD_MAP, self.custom_column_label_map, - self.tag_browser_categories) + self.data = ResultCache(self.FIELD_MAP, self.field_metadata) self.search = self.data.search self.refresh = functools.partial(self.data.refresh, self) self.sort = self.data.sort @@ -646,9 +645,6 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): def get_recipe(self, id): return self.conn.get('SELECT script FROM feeds WHERE id=?', (id,), all=False) - def get_tag_browser_categories(self): - return self.tag_browser_categories - def get_categories(self, sort_on_count=False, ids=None, icon_map=None): self.books_list_filter.change([] if not ids else ids) @@ -656,11 +652,18 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): if icon_map is not None and type(icon_map) != TagsIcons: raise TypeError('icon_map passed to get_categories must be of type TagIcons') + tb_cats = self.field_metadata + + # remove all user categories from field_metadata. They can + # easily come and go. We will add all the existing ones in below. + for k in tb_cats.keys(): + if tb_cats[k]['kind'] in ['user', 'search']: + del tb_cats[k] + #### First, build the standard and custom-column categories #### - tb_cats = self.tag_browser_categories for category in tb_cats.keys(): cat = tb_cats[category] - if cat['kind'] == 'not_cat': + if not cat['is_category']: continue tn = cat['table'] categories[category] = [] #reserve the position in the ordered list @@ -680,7 +683,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): # icon_map is not None if get_categories is to store an icon and # possibly a tooltip in the tag structure. icon, tooltip = None, '' - label = tb_cats.get_field_label(category) + label = tb_cats.key_to_label(category) if icon_map: if not tb_cats.is_custom_field(category): if category in icon_map: @@ -737,12 +740,6 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): #### Now do the user-defined categories. #### user_categories = prefs['user_categories'] - # remove all user categories from tag_browser_categories. They can - # easily come and go. We will add all the existing ones in below. - for k in tb_cats.keys(): - if tb_cats[k]['kind'] in ['user', 'search']: - del tb_cats[k] - # We want to use same node in the user category as in the source # category. To do that, we need to find the original Tag node. There is # a time/space tradeoff here. By converting the tags into a map, we can diff --git a/src/calibre/library/field_metadata.py b/src/calibre/library/field_metadata.py index 0134db712f..5a175ab2aa 100644 --- a/src/calibre/library/field_metadata.py +++ b/src/calibre/library/field_metadata.py @@ -4,7 +4,6 @@ Created on 25 May 2010 @author: charles ''' -from UserDict import DictMixin from calibre.utils.ordered_dict import OrderedDict class TagsIcons(dict): @@ -22,105 +21,253 @@ class TagsIcons(dict): raise ValueError('Missing category icon [%s]'%a) self[a] = icon_dict[a] -class FieldMetadata(dict, DictMixin): +class FieldMetadata(dict): + ''' + key: the key to the dictionary is: + - for standard fields, the metadata field name. + - for custom fields, the metadata field name prefixed by '#' + This is done to create two 'namespaces' so the names don't clash - # kind == standard: is tag category. May be a search label. Is db col - # or is specially handled (e.g., news) - # kind == not_cat: Is not a tag category. May be a search label. Is db col - # kind == user: user-defined tag category - # kind == search: saved-searches category - # For 'standard', the order below is the order that the categories will - # appear in the tags pane. - # - # label is the column label. key is either the label or in the case of - # custom fields, the label prefixed with 'x'. Because of the prefixing, - # there cannot be a name clash between standard and custom fields, so key - # can be used as the metadata dictionary key. + label: the actual column label. No prefixing. - category_items_ = [ - ('authors', {'table':'authors', 'column':'name', - 'datatype':'text', 'is_multiple':False, - 'kind':'standard', 'name':_('Authors'), - 'search_labels':['authors', 'author'], - 'is_custom':False}), - ('series', {'table':'series', 'column':'name', - 'datatype':'text', 'is_multiple':False, - 'kind':'standard', 'name':_('Series'), - 'search_labels':['series'], - 'is_custom':False}), - ('formats', {'table':None, 'column':None, - 'datatype':'text', 'is_multiple':False, # must think what type this is! - 'kind':'standard', 'name':_('Formats'), - 'search_labels':['formats', 'format'], - 'is_custom':False}), - ('publisher', {'table':'publishers', 'column':'name', - 'datatype':'text', 'is_multiple':False, - 'kind':'standard', 'name':_('Publishers'), - 'search_labels':['publisher'], - 'is_custom':False}), - ('rating', {'table':'ratings', 'column':'rating', - 'datatype':'rating', 'is_multiple':False, - 'kind':'standard', 'name':_('Ratings'), - 'search_labels':['rating'], - 'is_custom':False}), - ('news', {'table':'news', 'column':'name', - 'datatype':None, 'is_multiple':False, - 'kind':'standard', 'name':_('News'), - 'search_labels':[], - 'is_custom':False}), - ('tags', {'table':'tags', 'column':'name', - 'datatype':'text', 'is_multiple':True, - 'kind':'standard', 'name':_('Tags'), - 'search_labels':['tags', 'tag'], - 'is_custom':False}), - ('author_sort',{'table':None, 'column':None, 'datatype':'text', - 'is_multiple':False, 'kind':'not_cat', 'name':None, - 'search_labels':[], 'is_custom':False}), - ('comments', {'table':None, 'column':None, 'datatype':'text', - 'is_multiple':False, 'kind':'not_cat', 'name':None, - 'search_labels':['comments', 'comment'], 'is_custom':False}), - ('cover', {'table':None, 'column':None, 'datatype':None, - 'is_multiple':False, 'kind':'not_cat', 'name':None, - 'search_labels':['cover'], 'is_custom':False}), - ('flags', {'table':None, 'column':None, 'datatype':'text', - 'is_multiple':False, 'kind':'not_cat', 'name':None, - 'search_labels':[], 'is_custom':False}), - ('id', {'table':None, 'column':None, 'datatype':'int', - 'is_multiple':False, 'kind':'not_cat', 'name':None, - 'search_labels':[], 'is_custom':False}), - ('isbn', {'table':None, 'column':None, 'datatype':'text', - 'is_multiple':False, 'kind':'not_cat', 'name':None, - 'search_labels':['isbn'], 'is_custom':False}), - ('lccn', {'table':None, 'column':None, 'datatype':'text', - 'is_multiple':False, 'kind':'not_cat', 'name':None, - 'search_labels':[], 'is_custom':False}), - ('ondevice', {'table':None, 'column':None, 'datatype':'bool', - 'is_multiple':False, 'kind':'not_cat', 'name':None, - 'search_labels':[], 'is_custom':False}), - ('path', {'table':None, 'column':None, 'datatype':'text', - 'is_multiple':False, 'kind':'not_cat', 'name':None, - 'search_labels':[], 'is_custom':False}), - ('pubdate', {'table':None, 'column':None, 'datatype':'datetime', - 'is_multiple':False, 'kind':'not_cat', 'name':None, - 'search_labels':['pubdate'], 'is_custom':False}), - ('series_index',{'table':None, 'column':None, 'datatype':'float', - 'is_multiple':False, 'kind':'not_cat', 'name':None, - 'search_labels':[], 'is_custom':False}), - ('sort', {'table':None, 'column':None, 'datatype':'text', - 'is_multiple':False, 'kind':'not_cat', 'name':None, - 'search_labels':[], 'is_custom':False}), - ('size', {'table':None, 'column':None, 'datatype':'float', - 'is_multiple':False, 'kind':'not_cat', 'name':None, - 'search_labels':[], 'is_custom':False}), - ('timestamp', {'table':None, 'column':None, 'datatype':'datetime', - 'is_multiple':False, 'kind':'not_cat', 'name':None, - 'search_labels':['date'], 'is_custom':False}), - ('title', {'table':None, 'column':None, 'datatype':'text', - 'is_multiple':False, 'kind':'not_cat', 'name':None, - 'search_labels':['title'], 'is_custom':False}), - ('uuid', {'table':None, 'column':None, 'datatype':'text', - 'is_multiple':False, 'kind':'not_cat', 'name':None, - 'search_labels':[], 'is_custom':False}), + datatype: the type of the information in the field. Valid values are float, + int, rating, bool, comments, datetime, text. + is_multiple: valid for the text datatype. If None, the field is to be + treated as a single term. If not None, it contains a string, and the field + is assumed to contain a list of terms separated by that string + + kind == standard: is a db field. + kind == category: standard tag category that isn't a field. see news. + kind == user: user-defined tag category. + kind == search: saved-searches category. + + is_category: is a tag browser category. If true, then: + table: name of the db table used to construct item list + column: name of the column in the connection table to join on + If these are None, then the category constructor must know how + to build the item list (e.g., formats). + The order below is the order that the categories will + appear in the tags pane. + + name: the text that is to be used when displaying the field. Column headings + in the GUI, etc. + + search_terms: the terms that can be used to identify the field when + searching. They can be thought of as aliases for metadata keys, but are only + valid when passed to search(). + + is_custom: the field has been added by the user. + + rec_index: the index of the field in the db metadata record. + + ''' + _field_metadata = [ + ('authors', {'table':'authors', + 'column':'name', + 'datatype':'text', + 'is_multiple':',', + 'kind':'field', + 'name':_('Authors'), + 'search_terms':['authors', 'author'], + 'is_custom':False, + 'is_category':True}), + ('series', {'table':'series', + 'column':'name', + 'datatype':'text', + 'is_multiple':None, + 'kind':'field', + 'name':_('Series'), + 'search_terms':['series'], + 'is_custom':False, + 'is_category':True}), + ('formats', {'table':None, + 'column':None, + 'datatype':'text', + 'is_multiple':',', + 'kind':'field', + 'name':_('Formats'), + 'search_terms':['formats', 'format'], + 'is_custom':False, + 'is_category':True}), + ('publisher', {'table':'publishers', + 'column':'name', + 'datatype':'text', + 'is_multiple':None, + 'kind':'field', + 'name':_('Publishers'), + 'search_terms':['publisher'], + 'is_custom':False, + 'is_category':True}), + ('rating', {'table':'ratings', + 'column':'rating', + 'datatype':'rating', + 'is_multiple':None, + 'kind':'field', + 'name':_('Ratings'), + 'search_terms':['rating'], + 'is_custom':False, + 'is_category':True}), + ('news', {'table':'news', + 'column':'name', + 'datatype':None, + 'is_multiple':None, + 'kind':'category', + 'name':_('News'), + 'search_terms':[], + 'is_custom':False, + 'is_category':True}), + ('tags', {'table':'tags', + 'column':'name', + 'datatype':'text', + 'is_multiple':',', + 'kind':'field', + 'name':_('Tags'), + 'search_terms':['tags', 'tag'], + 'is_custom':False, + 'is_category':True}), + ('author_sort',{'table':None, + 'column':None, + 'datatype':'text', + 'is_multiple':None, + 'kind':'field', + 'name':None, + 'search_terms':[], + 'is_custom':False, + 'is_category':False}), + ('comments', {'table':None, + 'column':None, + 'datatype':'text', + 'is_multiple':None, + 'kind':'field', + 'name':None, + 'search_terms':['comments', 'comment'], + 'is_custom':False, 'is_category':False}), + ('cover', {'table':None, + 'column':None, + 'datatype':None, + 'is_multiple':None, + 'kind':'field', + 'name':None, + 'search_terms':['cover'], + 'is_custom':False, + 'is_category':False}), + ('flags', {'table':None, + 'column':None, + 'datatype':'text', + 'is_multiple':None, + 'kind':'field', + 'name':None, + 'search_terms':[], + 'is_custom':False, + 'is_category':False}), + ('id', {'table':None, + 'column':None, + 'datatype':'int', + 'is_multiple':None, + 'kind':'field', + 'name':None, + 'search_terms':[], + 'is_custom':False, + 'is_category':False}), + ('isbn', {'table':None, + 'column':None, + 'datatype':'text', + 'is_multiple':None, + 'kind':'field', + 'name':None, + 'search_terms':['isbn'], + 'is_custom':False, + 'is_category':False}), + ('lccn', {'table':None, + 'column':None, + 'datatype':'text', + 'is_multiple':None, + 'kind':'field', + 'name':None, + 'search_terms':[], + 'is_custom':False, + 'is_category':False}), + ('ondevice', {'table':None, + 'column':None, + 'datatype':'bool', + 'is_multiple':None, + 'kind':'field', + 'name':None, + 'search_terms':[], + 'is_custom':False, + 'is_category':False}), + ('path', {'table':None, + 'column':None, + 'datatype':'text', + 'is_multiple':None, + 'kind':'field', + 'name':None, + 'search_terms':[], + 'is_custom':False, + 'is_category':False}), + ('pubdate', {'table':None, + 'column':None, + 'datatype':'datetime', + 'is_multiple':None, + 'kind':'field', + 'name':None, + 'search_terms':['pubdate'], + 'is_custom':False, + 'is_category':False}), + ('series_index',{'table':None, + 'column':None, + 'datatype':'float', + 'is_multiple':None, + 'kind':'field', + 'name':None, + 'search_terms':[], + 'is_custom':False, + 'is_category':False}), + ('sort', {'table':None, + 'column':None, + 'datatype':'text', + 'is_multiple':None, + 'kind':'field', + 'name':None, + 'search_terms':[], + 'is_custom':False, + 'is_category':False}), + ('size', {'table':None, + 'column':None, + 'datatype':'float', + 'is_multiple':None, + 'kind':'field', + 'name':None, + 'search_terms':[], + 'is_custom':False, + 'is_category':False}), + ('timestamp', {'table':None, + 'column':None, + 'datatype':'datetime', + 'is_multiple':None, + 'kind':'field', + 'name':None, + 'search_terms':['date'], + 'is_custom':False, + 'is_category':False}), + ('title', {'table':None, + 'column':None, + 'datatype':'text', + 'is_multiple':None, + 'kind':'field', + 'name':None, + 'search_terms':['title'], + 'is_custom':False, + 'is_category':False}), + ('uuid', {'table':None, + 'column':None, + 'datatype':'text', + 'is_multiple':None, + 'kind':'field', + 'name':None, + 'search_terms':[], + 'is_custom':False, + 'is_category':False}), ] # search labels that are not db columns @@ -131,10 +278,15 @@ class FieldMetadata(dict, DictMixin): def __init__(self): self._tb_cats = OrderedDict() - for k,v in self.category_items_: + self._search_term_map = {} + self.custom_label_to_key_map = {} + for k,v in self._field_metadata: self._tb_cats[k] = v + self._tb_cats[k]['label'] = k + self._tb_cats[k]['display'] = {} + self._tb_cats[k]['is_editable'] = True + self._add_search_terms_to_map(k, self._tb_cats[k]['search_terms']) self.custom_field_prefix = '#' - self.get = self._tb_cats.get def __getitem__(self, key): @@ -150,6 +302,12 @@ class FieldMetadata(dict, DictMixin): for key in self._tb_cats: yield key + def __contains__(self, key): + return self.has_key(key) + + def has_key(self, key): + return key in self._tb_cats + def keys(self): return self._tb_cats.keys() @@ -157,44 +315,80 @@ class FieldMetadata(dict, DictMixin): for key in self._tb_cats: yield key + def itervalues(self): + return self._tb_cats.itervalues() + + def values(self): + return self._tb_cats.values() + def iteritems(self): for key in self._tb_cats: yield (key, self._tb_cats[key]) + def items(self): + return list(self.iteritems()) + def is_custom_field(self, key): return key.startswith(self.custom_field_prefix) - def get_field_label(self, key): + def key_to_label(self, key): if 'label' not in self._tb_cats[key]: return key return self._tb_cats[key]['label'] - def get_search_label(self, label): + def label_to_key(self, label, prefer_custom=False): + if prefer_custom: + if label in self.custom_label_to_key_map: + return self.custom_label_to_key_map[label] if 'label' in self._tb_cats: return label - if self.is_custom_field(label): - return self.custom_field_prefix+label + if not prefer_custom: + if label in self.custom_label_to_key_map: + return self.custom_label_to_key_map[label] raise ValueError('Unknown key [%s]'%(label)) def get_custom_fields(self): return [l for l in self._tb_cats if self._tb_cats[l]['is_custom']] - def add_custom_field(self, label, table, column, datatype, - is_multiple, colnum, name, searchable): - fn = self.custom_field_prefix + label - if fn in self._tb_cats: + def get_custom_field_metadata(self): + l = {} + for k in self._tb_cats: + if self._tb_cats[k]['is_custom']: + l[k] = self._tb_cats[k] + return l + + def add_custom_field(self, label, table, column, datatype, colnum, name, + display, is_editable, is_multiple, is_category): + key = self.custom_field_prefix + label + if key in self._tb_cats: raise ValueError('Duplicate custom field [%s]'%(label)) - if searchable: - sl = [fn] - kind = 'standard' - else: - sl = [] - kind = 'not_cat' - self._tb_cats[fn] = {'table':table, 'column':column, - 'datatype':datatype, 'is_multiple':is_multiple, - 'kind':kind, 'name':name, - 'search_labels':sl, 'label':label, - 'colnum':colnum, 'is_custom':True} + self._tb_cats[key] = {'table':table, 'column':column, + 'datatype':datatype, 'is_multiple':is_multiple, + 'kind':'field', 'name':name, + 'search_terms':[key], 'label':label, + 'colnum':colnum, 'display':display, + 'is_custom':True, 'is_category':is_category, + 'is_editable': is_editable,} + self._add_search_terms_to_map(key, [key]) + self.custom_label_to_key_map[label] = key + + def add_user_category(self, label, name): + if label in self._tb_cats: + raise ValueError('Duplicate user field [%s]'%(label)) + self._tb_cats[label] = {'table':None, 'column':None, + 'datatype':None, 'is_multiple':None, + 'kind':'user', 'name':name, + 'search_terms':[], 'is_custom':False, + 'is_category':True} + + def add_search_category(self, label, name): + if label in self._tb_cats: + raise ValueError('Duplicate user field [%s]'%(label)) + self._tb_cats[label] = {'table':None, 'column':None, + 'datatype':None, 'is_multiple':None, + 'kind':'search', 'name':name, + 'search_terms':[], 'is_custom':False, + 'is_category':True} def set_field_record_index(self, label, index, prefer_custom=False): if prefer_custom: @@ -208,21 +402,6 @@ class FieldMetadata(dict, DictMixin): key = self.custom_field_prefix+label self._tb_cats[key]['rec_index'] = index # let the exception fly ... - def add_user_category(self, label, name): - if label in self._tb_cats: - raise ValueError('Duplicate user field [%s]'%(label)) - self._tb_cats[label] = {'table':None, 'column':None, - 'datatype':None, 'is_multiple':False, - 'kind':'user', 'name':name, - 'search_labels':[], 'is_custom':False} - - def add_search_category(self, label, name): - if label in self._tb_cats: - raise ValueError('Duplicate user field [%s]'%(label)) - self._tb_cats[label] = {'table':None, 'column':None, - 'datatype':None, 'is_multiple':False, - 'kind':'search', 'name':name, - 'search_labels':[], 'is_custom':False} # DEFAULT_LOCATIONS = frozenset([ # 'all', @@ -246,14 +425,23 @@ class FieldMetadata(dict, DictMixin): # 'title', # ]) - - def get_search_labels(self): - s_labels = [] + def get_search_terms(self): + s_keys = [] for v in self._tb_cats.itervalues(): - map((lambda x:s_labels.append(x)), v['search_labels']) + map((lambda x:s_keys.append(x)), v['search_terms']) for v in self.search_items: - s_labels.append(v) -# if set(s_labels) != self.DEFAULT_LOCATIONS: + s_keys.append(v) +# if set(s_keys) != self.DEFAULT_LOCATIONS: # print 'search labels and default_locations do not match:' -# print set(s_labels) ^ self.DEFAULT_LOCATIONS - return s_labels +# print set(s_keys) ^ self.DEFAULT_LOCATIONS + return s_keys + + def _add_search_terms_to_map(self, key, terms): + if terms is not None: + for t in terms: + self._search_term_map[t] = key + + def search_term_to_key(self, term): + if term in self._search_term_map: + return self._search_term_map[term] + return term diff --git a/src/calibre/library/schema_upgrades.py b/src/calibre/library/schema_upgrades.py index f1e68b3916..6222a8950f 100644 --- a/src/calibre/library/schema_upgrades.py +++ b/src/calibre/library/schema_upgrades.py @@ -289,6 +289,10 @@ class SchemaUpgrade(object): '''.format(tn=table_name, cn=column_name, vcn=view_column_name)) self.conn.executescript(script) - for tn, cn in self.tag_browser_categories.items(): - if tn != 'news': - create_tag_browser_view(tn, cn[0], cn[1]) + for field in self.field_metadata.itervalues(): + if field['is_category'] and not field['is_custom'] and \ + field['table'] != 'news' and field['table'] is not None: + cn = field['table'][:-1] + if cn == 'serie': + cn += 's' + create_tag_browser_view(field['table'], cn, field['column']) diff --git a/src/calibre/library/server/opds.py b/src/calibre/library/server/opds.py index dd1e2d0bb0..ca635f9c28 100644 --- a/src/calibre/library/server/opds.py +++ b/src/calibre/library/server/opds.py @@ -339,7 +339,7 @@ class OPDSServer(object): raise cherrypy.HTTPError(404, 'Not found') categories = self.categories_cache( self.get_opds_allowed_ids_for_version(version)) - category_meta = self.db.get_tag_browser_categories() + category_meta = self.db.field_metadata cats = [ (_('Newest'), _('Date'), 'Onewest'), (_('Title'), _('Title'), 'Otitle'),