Get rid of the messy restricted names paradigm for user created fields

src/calibre/ebooks/metadata/book/__init__.py

@@ -88,35 +88,6 @@ CALIBRE_METADATA_FIELDS = frozenset([
     ]
 )
 
-CALIBRE_RESERVED_LABELS = frozenset([
-    'all',          # search term
-    'date',         # search term
-    'formats',      # search term
-    'inlibrary',    # search term
-    'news',         # search term
-    'ondevice',     # search term
-    'search',       # search term
-    'format',       # The next four are here for backwards compatibility
-    'tag',          # with searching. The terms can be used without the
-    'author',       # trailing 's'.
-    'comment',      # Sigh ...
-    ]
-)
-
-RESERVED_METADATA_FIELDS = SOCIAL_METADATA_FIELDS.union(
-                           PUBLICATION_METADATA_FIELDS).union(
-                           BOOK_STRUCTURE_FIELDS).union(
-                           USER_METADATA_FIELDS).union(
-                           DEVICE_METADATA_FIELDS).union(
-                           CALIBRE_METADATA_FIELDS).union(
-                           CALIBRE_RESERVED_LABELS)
-
-assert len(RESERVED_METADATA_FIELDS) == sum(map(len, (
-    SOCIAL_METADATA_FIELDS, PUBLICATION_METADATA_FIELDS,
-    BOOK_STRUCTURE_FIELDS, USER_METADATA_FIELDS,
-    DEVICE_METADATA_FIELDS, CALIBRE_METADATA_FIELDS,
-    CALIBRE_RESERVED_LABELS
-    )))
 
 SERIALIZABLE_FIELDS = SOCIAL_METADATA_FIELDS.union(
                       USER_METADATA_FIELDS).union(

src/calibre/ebooks/metadata/book/base.py

@@ -68,13 +68,6 @@ class Metadata(object):
             # Don't abuse this privilege
             self.__dict__[field] = val
 
-    @property
-    def reserved_names(self):
-        'The set of names you cannot use for your own purposes on this object'
-        _data = object.__getattribute__(self, '_data')
-        return frozenset(RESERVED_FIELD_NAMES).union(frozenset(
-            _data['user_metadata'].iterkeys()))
-
     @property
     def user_metadata_names(self):
         'The set of user metadata names this object knows about'
@@ -120,10 +113,8 @@ class Metadata(object):
 
     # }}}
 
-_m = Metadata()
+# We don't need reserved field names for this object any more. Lets just use a
-RESERVED_FIELD_NAMES = \
+# protocol like the last char of a user field label should be _ when using this
-        frozenset(_m.__dict__.iterkeys()).union( # _data
+# object
-        RESERVED_METADATA_FIELDS).union(
+# So mi.tags returns the builtin tags and mi.tags_ returns the user tags
-        frozenset(Metadata.__dict__.iterkeys())) # methods defined in Metadata
-del _m
 

src/calibre/gui2/dialogs/config/create_custom_column.py

@@ -10,7 +10,6 @@ from PyQt4.Qt import QDialog, Qt, QListWidgetItem, QVariant
 
 from calibre.gui2.dialogs.config.create_custom_column_ui import Ui_QCreateCustomColumn
 from calibre.gui2 import error_dialog
-from calibre.ebooks.metadata.book.base import RESERVED_FIELD_NAMES
 
 class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
 
@@ -103,14 +102,10 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
             return self.simple_error('', _('No lookup name was provided'))
         if not col_heading:
             return self.simple_error('', _('No column heading was provided'))
-        if col in RESERVED_FIELD_NAMES:
-            return self.simple_error('', _('The lookup name %s is reserved and cannot be used')%col)
         bad_col = False
         if col in self.parent.custcols:
             if not self.editing_col or self.parent.custcols[col]['num'] != self.orig_column_number:
                 bad_col = True
-        if col in self.standard_colnames:
-            bad_col = True
         if bad_col:
             return self.simple_error('', _('The lookup name %s is already used')%col)
         bad_head = False

src/calibre/gui2/dialogs/tag_categories.py

@@ -49,7 +49,7 @@ class TagCategories(QDialog, Ui_TagCategories):
         cc_map = self.db.custom_column_label_map
         for cc in cc_map:
             if cc_map[cc]['datatype'] == 'text':
-                self.category_labels.append(cc)
+                self.category_labels.append(db.tag_browser_categories.get_search_label(cc))
                 category_icons.append(cc_icon)
                 category_values.append(lambda col=cc: self.db.all_custom(label=col))
                 category_names.append(cc_map[cc]['name'])

src/calibre/gui2/library/models.py

@@ -632,6 +632,8 @@ class BooksModel(QAbstractTableModel): # {{{
                 return None
             if role == Qt.ToolTipRole:
                 ht = self.column_map[section]
+                if self.is_custom_column(self.column_map[section]):
+                    ht = self.db.tag_browser_categories.custom_field_prefix + ht
                 if ht == 'timestamp': # change help text because users know this field as 'date'
                     ht = 'date'
                 return QVariant(_('The lookup/search name is "{0}"').format(ht))

src/calibre/gui2/tag_view.py

@@ -14,7 +14,7 @@ from PyQt4.Qt import Qt, QTreeView, QApplication, pyqtSignal, \
                      QAbstractItemModel, QVariant, QModelIndex
 from calibre.gui2 import config, NONE
 from calibre.utils.config import prefs
-from calibre.ebooks.metadata.book import RESERVED_METADATA_FIELDS
+from calibre.library.field_metadata import TagsIcons
 
 class TagsView(QTreeView): # {{{
 
@@ -205,7 +205,7 @@ class TagsModel(QAbstractItemModel): # {{{
         # must do this here because 'QPixmap: Must construct a QApplication
         # before a QPaintDevice'. The ':' in front avoids polluting either the
         # user-defined categories (':' at end) or columns namespaces (no ':').
-        self.category_icon_map = {
+        self.category_icon_map = TagsIcons({
                     'authors'   : QIcon(I('user_profile.svg')),
                     'series'    : QIcon(I('series.svg')),
                     'formats'   : QIcon(I('book.svg')),
@@ -215,10 +215,8 @@ class TagsModel(QAbstractItemModel): # {{{
                     'tags'      : QIcon(I('tags.svg')),
                     ':custom'   : QIcon(I('column.svg')),
                     ':user'     : QIcon(I('drawer.svg')),
-                    'search'    : QIcon(I('search.svg'))}
+                    'search'    : QIcon(I('search.svg'))})
-        for k in self.category_icon_map.keys():
+
-            if not k.startswith(':') and k not in RESERVED_METADATA_FIELDS:
-                raise ValueError('Tag category [%s] is not a reserved word.' %(k))
         self.icon_state_map = [None, QIcon(I('plus.svg')), QIcon(I('minus.svg'))]
         self.db = db
         self.search_restriction = ''
@@ -226,10 +224,14 @@ class TagsModel(QAbstractItemModel): # {{{
         data = self.get_node_tree(config['sort_by_popularity'])
         self.root_item = TagTreeItem()
         for i, r in enumerate(self.row_map):
+            if self.db.get_tag_browser_categories()[r]['kind'] != 'user':
+                tt = _('The lookup/search name is "{0}"').format(r)
+            else:
+                tt = ''
             c = TagTreeItem(parent=self.root_item,
                     data=self.categories[i],
                     category_icon=self.category_icon_map[r],
-                    tooltip=_('The lookup/search name is "{0}"').format(r))
+                    tooltip=tt)
             for tag in data[r]:
                 TagTreeItem(parent=c, data=tag, icon_map=self.icon_state_map)
 
@@ -247,7 +249,7 @@ class TagsModel(QAbstractItemModel): # {{{
             data = self.db.get_categories(sort_on_count=sort, icon_map=self.category_icon_map)
 
         tb_categories = self.db.get_tag_browser_categories()
-        for category in tb_categories.iterkeys():
+        for category in tb_categories:
             if category in data: # They should always be there, but ...
                 self.row_map.append(category)
                 self.categories.append(tb_categories[category]['name'])

src/calibre/library/caches.py

@@ -17,6 +17,7 @@ from calibre.utils.config import tweaks
 from calibre.utils.date import parse_date, now, UNDEFINED_DATE
 from calibre.utils.search_query_parser import SearchQueryParser
 from calibre.utils.pyparsing import ParseException
+# from calibre.library.field_metadata import FieldMetadata
 
 class CoverCache(QThread):
 
@@ -149,15 +150,15 @@ class ResultCache(SearchQueryParser):
     '''
     Stores sorted and filtered metadata in memory.
     '''
-    def __init__(self, FIELD_MAP, cc_label_map):
+    def __init__(self, FIELD_MAP, cc_label_map, tag_browser_categories):
         self.FIELD_MAP = FIELD_MAP
         self.custom_column_label_map = cc_label_map
         self._map = self._map_filtered = self._data = []
         self.first_sort = True
         self.search_restriction = ''
-        SearchQueryParser.__init__(self,
+        self.tag_browser_categories = tag_browser_categories
-            locations=SearchQueryParser.DEFAULT_LOCATIONS +
+        self.all_search_locations = tag_browser_categories.get_search_labels()
-            [c for c in cc_label_map])
+        SearchQueryParser.__init__(self, self.all_search_locations)
         self.build_date_relop_dict()
         self.build_numeric_relop_dict()
 
@@ -379,25 +380,33 @@ class ResultCache(SearchQueryParser):
             if location in ('tag', 'author', 'format', 'comment'):
                 location += 's'
 
-            all = ('title', 'authors', 'publisher', 'tags', 'comments', 'series',
-                   'formats', 'isbn', 'rating', 'cover', 'ondevice')
             MAP = {}
+            # Fields not used when matching against text contents. These are
+            # the non-text fields
+            EXCLUDE_FIELDS = []
 
-            for x in all: # get the db columns for the standard searchables
+            # get the db columns for the standard searchables
-                MAP[x] = self.FIELD_MAP[x]
+            for x in self.tag_browser_categories:
+                if len(self.tag_browser_categories[x]['search_labels']) and \
+                         not self.tag_browser_categories.is_custom_field(x):
+                    MAP[x] = self.tag_browser_categories[x]['rec_index']
+                    if self.tag_browser_categories[x]['datatype'] != 'text':
+                        EXCLUDE_FIELDS.append(MAP[x])
+
+            # add custom columns to MAP. Put the column's type into IS_CUSTOM
             IS_CUSTOM = []
-            for x in range(len(self.FIELD_MAP)): # build a list containing '' the size of FIELD_MAP
+            for x in range(len(self.FIELD_MAP)):
                 IS_CUSTOM.append('')
-            IS_CUSTOM[self.FIELD_MAP['rating']] = 'rating'  # normal and custom ratings columns use the same code
+            # normal and custom ratings columns use the same code
-            for x in self.custom_column_label_map: # add custom columns to MAP. Put the column's type into IS_CUSTOM
+            IS_CUSTOM[self.FIELD_MAP['rating']] = 'rating'
-                if self.custom_column_label_map[x]['datatype'] != "datetime":
+            for x in self.tag_browser_categories.get_custom_fields():
-                    MAP[x] = self.FIELD_MAP[self.custom_column_label_map[x]['num']]
+                if self.tag_browser_categories[x]['datatype'] != "datetime":
-                    IS_CUSTOM[MAP[x]] = self.custom_column_label_map[x]['datatype']
+                    MAP[x] = self.FIELD_MAP[self.tag_browser_categories[x]['colnum']]
+                    IS_CUSTOM[MAP[x]] = self.tag_browser_categories[x]['datatype']
 
-            EXCLUDE_FIELDS = [MAP['rating'], MAP['cover']]
             SPLITABLE_FIELDS = [MAP['authors'], MAP['tags'], MAP['formats']]
-            for x in self.custom_column_label_map:
+            for x in self.tag_browser_categories.get_custom_fields():
-                if self.custom_column_label_map[x]['is_multiple']:
+                if self.tag_browser_categories[x]['is_multiple']:
                     SPLITABLE_FIELDS.append(MAP[x])
 
             try:

src/calibre/library/custom_columns.py

@@ -144,12 +144,14 @@ class CustomColumns(object):
         for k in sorted(self.custom_column_label_map.keys()):
             v = self.custom_column_label_map[k]
             if v['normalized']:
+                searchable = True
+            else:
+                searchable = False
             tn = 'custom_column_{0}'.format(v['num'])
-                self.tag_browser_categories[v['label']] = {
+            self.tag_browser_categories.add_custom_field(label=v['label'],
-                        'table':tn, 'column':'value',
+                    table=tn, column='value', datatype=v['datatype'],
-                        'type':v['datatype'], 'is_multiple':v['is_multiple'],
+                    is_multiple=v['is_multiple'], colnum=v['num'],
-                        'kind':'custom', 'name':v['name']
+                    name=v['name'], searchable=searchable)
-                    }
 
     def get_custom(self, idx, label=None, num=None, index_is_id=False):
         if label is not None:

src/calibre/library/database2.py

@@ -20,6 +20,7 @@ from PyQt4.QtGui import QImage
 
 from calibre.ebooks.metadata import title_sort
 from calibre.library.database import LibraryDatabase
+from calibre.library.field_metadata import FieldMetadata, TagsIcons
 from calibre.library.schema_upgrades import SchemaUpgrade
 from calibre.library.caches import ResultCache
 from calibre.library.custom_columns import CustomColumns
@@ -33,11 +34,10 @@ from calibre.customize.ui import run_plugins_on_import
 
 from calibre.utils.filenames import ascii_filename
 from calibre.utils.date import utcnow, now as nowf, utcfromtimestamp
-from calibre.utils.ordered_dict import OrderedDict
 from calibre.utils.config import prefs
 from calibre.utils.search_query_parser import saved_searches
 from calibre.ebooks import BOOK_EXTENSIONS, check_ebook_format
-from calibre.ebooks.metadata.book import RESERVED_METADATA_FIELDS
+
 
 if iswindows:
     import calibre.utils.winshell as winshell
@@ -116,6 +116,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         self.books_list_filter = self.conn.create_dynamic_filter('books_list_filter')
 
     def __init__(self, library_path, row_factory=False):
+        self.tag_browser_categories = FieldMetadata() #.get_tag_browser_categories()
         if not os.path.exists(library_path):
             os.makedirs(library_path)
         self.listeners = set([])
@@ -127,36 +128,6 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         if isinstance(self.dbpath, unicode):
             self.dbpath = self.dbpath.encode(filesystem_encoding)
 
-        # Order as has been customary in the tags pane.
-        tag_browser_categories_items = [
-                ('authors',   {'table':'authors', 'column':'name',
-                               'type':'text', 'is_multiple':False,
-                               'kind':'standard', 'name':_('Authors')}),
-                ('series',    {'table':'series', 'column':'name',
-                               'type':None, 'is_multiple':False,
-                               'kind':'standard', 'name':_('Series')}),
-                ('formats',   {'table':None, 'column':None,
-                               'type':None, 'is_multiple':False,
-                               'kind':'standard', 'name':_('Formats')}),
-                ('publisher', {'table':'publishers', 'column':'name',
-                               'type':'text', 'is_multiple':False,
-                               'kind':'standard', 'name':_('Publishers')}),
-                ('rating',    {'table':'ratings', 'column':'rating',
-                               'type':'rating', 'is_multiple':False,
-                               'kind':'standard', 'name':_('Ratings')}),
-                ('news',      {'table':'news', 'column':'name',
-                               'type':None, 'is_multiple':False,
-                               'kind':'standard', 'name':_('News')}),
-                ('tags',      {'table':'tags', 'column':'name',
-                               'type':'text', 'is_multiple':True,
-                               'kind':'standard', 'name':_('Tags')}),
-        ]
-        self.tag_browser_categories = OrderedDict()
-        for k,v in tag_browser_categories_items:
-            if k not in RESERVED_METADATA_FIELDS:
-                raise ValueError('Tag category [%s] is not a reserved word.' %(k))
-            self.tag_browser_categories[k] = v
-
         self.connect()
         self.is_case_sensitive = not iswindows and not isosx and \
             not os.path.exists(self.dbpath.replace('metadata.db', 'MeTAdAtA.dB'))
@@ -224,6 +195,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
             lines.append(line)
 
         custom_map = self.custom_columns_in_meta()
+        # custom col labels are numbers (the id in the custom_columns table)
         custom_cols = list(sorted(custom_map.keys()))
         lines.extend([custom_map[x] for x in custom_cols])
 
@@ -233,12 +205,21 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
              'sort':11, 'author_sort':12, 'formats':13, 'isbn':14, 'path':15,
              'lccn':16, 'pubdate':17, 'flags':18, 'uuid':19}
 
+        for k,v in self.FIELD_MAP.iteritems():
+            self.tag_browser_categories.set_field_record_index(k, v, prefer_custom=False)
+
         base = max(self.FIELD_MAP.values())
         for col in custom_cols:
             self.FIELD_MAP[col] = base = base+1
+            self.tag_browser_categories.set_field_record_index(
+                                        self.custom_column_num_map[col]['label'],
+                                        base,
+                                        prefer_custom=True)
 
         self.FIELD_MAP['cover'] = base+1
+        self.tag_browser_categories.set_field_record_index('cover', base+1, prefer_custom=False)
         self.FIELD_MAP['ondevice'] = base+2
+        self.tag_browser_categories.set_field_record_index('ondevice', base+2, prefer_custom=False)
 
         script = '''
         DROP VIEW IF EXISTS meta2;
@@ -251,7 +232,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         self.conn.commit()
 
         self.book_on_device_func = None
-        self.data    = ResultCache(self.FIELD_MAP, self.custom_column_label_map)
+        self.data    = ResultCache(self.FIELD_MAP, self.custom_column_label_map,
+                                   self.tag_browser_categories)
         self.search  = self.data.search
         self.refresh = functools.partial(self.data.refresh, self)
         self.sort    = self.data.sort
@@ -671,14 +653,20 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         self.books_list_filter.change([] if not ids else ids)
 
         categories = {}
+        if icon_map is not None and type(icon_map) != TagsIcons:
+            raise TypeError('icon_map passed to get_categories must be of type TagIcons')
 
         #### First, build the standard and custom-column categories ####
-        for category in self.tag_browser_categories.keys():
+        tb_cats = self.tag_browser_categories
-            tn = self.tag_browser_categories[category]['table']
+        for category in tb_cats.keys():
+            cat = tb_cats[category]
+            if cat['kind'] == 'not_cat':
+                continue
+            tn = cat['table']
             categories[category] = []   #reserve the position in the ordered list
             if tn is None:              # Nothing to do for the moment
                 continue
-            cn = self.tag_browser_categories[category]['column']
+            cn = cat['column']
             if ids is None:
                 query = 'SELECT id, {0}, count FROM tag_browser_{1}'.format(cn, tn)
             else:
@@ -692,16 +680,17 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
             # icon_map is not None if get_categories is to store an icon and
             # possibly a tooltip in the tag structure.
             icon, tooltip = None, ''
+            label = tb_cats.get_field_label(category)
             if icon_map:
-                if self.tag_browser_categories[category]['kind'] == 'standard':
+                if not tb_cats.is_custom_field(category):
                     if category in icon_map:
-                        icon = icon_map[category]
+                        icon = icon_map[label]
-                elif self.tag_browser_categories[category]['kind'] == 'custom':
+                else:
                     icon = icon_map[':custom']
                     icon_map[category] = icon
-                    tooltip = self.custom_column_label_map[category]['name']
+                    tooltip = self.custom_column_label_map[label]['name']
 
-            datatype = self.tag_browser_categories[category]['type']
+            datatype = cat['datatype']
             if datatype == 'rating':
                 item_not_zero_func = (lambda x: x[1] > 0 and x[2] > 0)
                 formatter = (lambda x:u'\u2605'*int(round(x/2.)))
@@ -711,7 +700,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                 formatter = (lambda x: x.replace('|', ','))
             else:
                 item_not_zero_func = (lambda x: x[2] > 0)
-                formatter = (lambda x:x)
+                formatter = (lambda x:unicode(x))
 
             categories[category] = [Tag(formatter(r[1]), count=r[2], id=r[0],
                                         icon=icon, tooltip = tooltip)
@@ -750,9 +739,9 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
 
         # remove all user categories from tag_browser_categories. They can
         # easily come and go. We will add all the existing ones in below.
-        for k in self.tag_browser_categories.keys():
+        for k in tb_cats.keys():
-            if self.tag_browser_categories[k]['kind'] in ['user', 'search']:
+            if tb_cats[k]['kind'] in ['user', 'search']:
-                del self.tag_browser_categories[k]
+                del tb_cats[k]
 
         # We want to use same node in the user category as in the source
         # category. To do that, we need to find the original Tag node. There is
@@ -771,10 +760,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                 # else: do nothing, to not include nodes w zero counts
             if len(items):
                 cat_name = user_cat+':' # add the ':' to avoid name collision
-                self.tag_browser_categories[cat_name] = {
+                tb_cats.add_user_category(label=cat_name, name=user_cat)
-                                'table':None,  'column':None,
-                                'type':None,   'is_multiple':False,
-                                'kind':'user', 'name':user_cat}
                 # Not a problem if we accumulate entries in the icon map
                 if icon_map is not None:
                     icon_map[cat_name] = icon_map[':user']
@@ -793,10 +779,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         for srch in saved_searches.names():
             items.append(Tag(srch, tooltip=saved_searches.lookup(srch), icon=icon))
         if len(items):
-            self.tag_browser_categories['search'] = {
+            tb_cats.add_search_category(label='search', name=_('Searches'))
-                            'table':None,    'column':None,
-                            'type':None,     'is_multiple':False,
-                            'kind':'search', 'name':_('Searches')}
             if icon_map is not None:
                 icon_map['search'] = icon_map['search']
             categories['search'] = items

src/calibre/library/field_metadata.py

@@ -0,0 +1,257 @@
+'''
+Created on 25 May 2010
+
+@author: charles
+'''
+
+from UserDict import DictMixin
+from calibre.utils.ordered_dict import OrderedDict
+
+class TagsIcons(dict):
+    '''
+    If the client wants icons to be in the tag structure, this class must be
+    instantiated and filled in with real icons. If this class is instantiated
+    and passed to get_categories, All items must be given a value not None
+    '''
+
+    category_icons = ['authors', 'series', 'formats', 'publisher', 'rating',
+                      'news',    'tags',   ':custom', ':user',     'search',]
+    def __init__(self, icon_dict):
+        for a in self.category_icons:
+            if a not in icon_dict:
+                raise ValueError('Missing category icon [%s]'%a)
+            self[a] = icon_dict[a]
+
+class FieldMetadata(dict, DictMixin):
+
+    # kind == standard: is tag category. May be a search label. Is db col
+    #                   or is specially handled (e.g., news)
+    # kind == not_cat: Is not a tag category. May be a search label. Is db col
+    # kind == user: user-defined tag category
+    # kind == search: saved-searches category
+    # For 'standard', the order below is the order that the categories will
+    # appear in the tags pane.
+    #
+    # label is the column label. key is either the label or in the case of
+    # custom fields, the label prefixed with 'x'. Because of the prefixing,
+    # there cannot be a name clash between standard and custom fields, so key
+    # can be used as the metadata dictionary key.
+
+    category_items_ = [
+            ('authors',   {'table':'authors', 'column':'name',
+                           'datatype':'text', 'is_multiple':False,
+                           'kind':'standard', 'name':_('Authors'),
+                           'search_labels':['authors', 'author'],
+                           'is_custom':False}),
+            ('series',    {'table':'series', 'column':'name',
+                           'datatype':'text', 'is_multiple':False,
+                           'kind':'standard', 'name':_('Series'),
+                           'search_labels':['series'],
+                           'is_custom':False}),
+            ('formats',   {'table':None, 'column':None,
+                           'datatype':'text', 'is_multiple':False, # must think what type this is!
+                           'kind':'standard', 'name':_('Formats'),
+                           'search_labels':['formats', 'format'],
+                           'is_custom':False}),
+            ('publisher', {'table':'publishers', 'column':'name',
+                           'datatype':'text', 'is_multiple':False,
+                           'kind':'standard', 'name':_('Publishers'),
+                           'search_labels':['publisher'],
+                           'is_custom':False}),
+            ('rating',    {'table':'ratings', 'column':'rating',
+                           'datatype':'rating', 'is_multiple':False,
+                           'kind':'standard', 'name':_('Ratings'),
+                           'search_labels':['rating'],
+                           'is_custom':False}),
+            ('news',      {'table':'news', 'column':'name',
+                           'datatype':None, 'is_multiple':False,
+                           'kind':'standard', 'name':_('News'),
+                           'search_labels':[],
+                           'is_custom':False}),
+            ('tags',      {'table':'tags', 'column':'name',
+                           'datatype':'text', 'is_multiple':True,
+                           'kind':'standard', 'name':_('Tags'),
+                           'search_labels':['tags', 'tag'],
+                           'is_custom':False}),
+            ('author_sort',{'table':None, 'column':None, 'datatype':'text',
+                           'is_multiple':False, 'kind':'not_cat', 'name':None,
+                           'search_labels':[], 'is_custom':False}),
+            ('comments',  {'table':None, 'column':None, 'datatype':'text',
+                           'is_multiple':False, 'kind':'not_cat', 'name':None,
+                           'search_labels':['comments', 'comment'], 'is_custom':False}),
+            ('cover',     {'table':None, 'column':None, 'datatype':None,
+                           'is_multiple':False, 'kind':'not_cat', 'name':None,
+                           'search_labels':['cover'], 'is_custom':False}),
+            ('flags',     {'table':None, 'column':None, 'datatype':'text',
+                           'is_multiple':False, 'kind':'not_cat', 'name':None,
+                           'search_labels':[], 'is_custom':False}),
+            ('id',        {'table':None, 'column':None, 'datatype':'int',
+                           'is_multiple':False, 'kind':'not_cat', 'name':None,
+                           'search_labels':[], 'is_custom':False}),
+            ('isbn',      {'table':None, 'column':None, 'datatype':'text',
+                           'is_multiple':False, 'kind':'not_cat', 'name':None,
+                           'search_labels':['isbn'], 'is_custom':False}),
+            ('lccn',      {'table':None, 'column':None, 'datatype':'text',
+                           'is_multiple':False, 'kind':'not_cat', 'name':None,
+                           'search_labels':[], 'is_custom':False}),
+            ('ondevice',  {'table':None, 'column':None, 'datatype':'bool',
+                           'is_multiple':False, 'kind':'not_cat', 'name':None,
+                           'search_labels':[], 'is_custom':False}),
+            ('path',      {'table':None, 'column':None, 'datatype':'text',
+                           'is_multiple':False, 'kind':'not_cat', 'name':None,
+                           'search_labels':[], 'is_custom':False}),
+            ('pubdate',   {'table':None, 'column':None, 'datatype':'datetime',
+                           'is_multiple':False, 'kind':'not_cat', 'name':None,
+                           'search_labels':['pubdate'], 'is_custom':False}),
+            ('series_index',{'table':None, 'column':None, 'datatype':'float',
+                           'is_multiple':False, 'kind':'not_cat', 'name':None,
+                           'search_labels':[], 'is_custom':False}),
+            ('sort',      {'table':None, 'column':None, 'datatype':'text',
+                           'is_multiple':False, 'kind':'not_cat', 'name':None,
+                           'search_labels':[], 'is_custom':False}),
+            ('size',      {'table':None, 'column':None, 'datatype':'float',
+                           'is_multiple':False, 'kind':'not_cat', 'name':None,
+                           'search_labels':[], 'is_custom':False}),
+            ('timestamp', {'table':None, 'column':None, 'datatype':'datetime',
+                           'is_multiple':False, 'kind':'not_cat', 'name':None,
+                           'search_labels':['date'], 'is_custom':False}),
+            ('title',     {'table':None, 'column':None, 'datatype':'text',
+                           'is_multiple':False, 'kind':'not_cat', 'name':None,
+                           'search_labels':['title'], 'is_custom':False}),
+            ('uuid',      {'table':None, 'column':None, 'datatype':'text',
+                           'is_multiple':False, 'kind':'not_cat', 'name':None,
+                           'search_labels':[], 'is_custom':False}),
+            ]
+
+    # search labels that are not db columns
+    search_items = [    'all',
+#                        'date',
+                        'search',
+                    ]
+
+    def __init__(self):
+        self._tb_cats = OrderedDict()
+        for k,v in self.category_items_:
+            self._tb_cats[k] = v
+        self.custom_field_prefix = '#'
+
+    def __getitem__(self, key):
+        return self._tb_cats[key]
+
+    def __setitem__(self, key, val):
+        raise AttributeError('Assigning to this object is forbidden')
+
+    def __delitem__(self, key):
+        del self._tb_cats[key]
+
+    def __iter__(self):
+        for key in self._tb_cats:
+            yield key
+
+    def keys(self):
+        return self._tb_cats.keys()
+
+    def iterkeys(self):
+        for key in self._tb_cats:
+            yield key
+
+    def iteritems(self):
+        for key in self._tb_cats:
+            yield (key, self._tb_cats[key])
+
+    def is_custom_field(self, key):
+        return key.startswith(self.custom_field_prefix)
+
+    def get_field_label(self, key):
+        if 'label' not in self._tb_cats[key]:
+            return key
+        return self._tb_cats[key]['label']
+
+    def get_search_label(self, label):
+        if 'label' in self._tb_cats:
+            return label
+        if self.is_custom_field(label):
+            return self.custom_field_prefix+label
+        raise ValueError('Unknown key [%s]'%(label))
+
+    def get_custom_fields(self):
+        return [l for l in self._tb_cats if self._tb_cats[l]['is_custom']]
+
+    def add_custom_field(self, label, table, column, datatype,
+                               is_multiple, colnum, name, searchable):
+        fn = self.custom_field_prefix + label
+        if fn in self._tb_cats:
+            raise ValueError('Duplicate custom field [%s]'%(label))
+        if searchable:
+            sl = [fn]
+            kind = 'standard'
+        else:
+            sl = []
+            kind = 'not_cat'
+        self._tb_cats[fn] = {'table':table,       'column':column,
+                             'datatype':datatype, 'is_multiple':is_multiple,
+                             'kind':kind,         'name':name,
+                             'search_labels':sl,  'label':label,
+                             'colnum':colnum,     'is_custom':True}
+
+    def set_field_record_index(self, label, index, prefer_custom=False):
+        if prefer_custom:
+            key = self.custom_field_prefix+label
+            if key not in self._tb_cats:
+                key = label
+        else:
+            if label in self._tb_cats:
+                key = label
+            else:
+                key = self.custom_field_prefix+label
+        self._tb_cats[key]['rec_index'] = index  # let the exception fly ...
+
+    def add_user_category(self, label, name):
+        if label in self._tb_cats:
+            raise ValueError('Duplicate user field [%s]'%(label))
+        self._tb_cats[label] = {'table':None,        'column':None,
+                                'datatype':None,     'is_multiple':False,
+                                'kind':'user',       'name':name,
+                                'search_labels':[],  'is_custom':False}
+
+    def add_search_category(self, label, name):
+        if label in self._tb_cats:
+            raise ValueError('Duplicate user field [%s]'%(label))
+        self._tb_cats[label] = {'table':None,        'column':None,
+                                'datatype':None,     'is_multiple':False,
+                                'kind':'search',     'name':name,
+                                'search_labels':[],  'is_custom':False}
+
+#    DEFAULT_LOCATIONS = frozenset([
+#        'all',
+#        'author',       # compatibility
+#        'authors',
+#        'comment',      # compatibility
+#        'comments',
+#        'cover',
+#        'date',
+#        'format',       # compatibility
+#        'formats',
+#        'isbn',
+#        'ondevice',
+#        'pubdate',
+#        'publisher',
+#        'search',
+#        'series',
+#        'rating',
+#        'tag',          # compatibility
+#        'tags',
+#        'title',
+#                 ])
+
+
+    def get_search_labels(self):
+        s_labels = []
+        for v in self._tb_cats.itervalues():
+            map((lambda x:s_labels.append(x)), v['search_labels'])
+        for v in self.search_items:
+            s_labels.append(v)
+#        if set(s_labels) != self.DEFAULT_LOCATIONS:
+#            print 'search labels and default_locations do not match:'
+#            print set(s_labels) ^ self.DEFAULT_LOCATIONS
+        return s_labels

src/calibre/utils/search_query_parser.py

@@ -22,7 +22,6 @@ from calibre.utils.pyparsing import Keyword, Group, Forward, CharsNotIn, Suppres
                       OneOrMore, oneOf, CaselessLiteral, Optional, NoMatch, ParseException
 from calibre.constants import preferred_encoding
 from calibre.utils.config import prefs
-from calibre.ebooks.metadata.book import RESERVED_METADATA_FIELDS
 
 '''
 This class manages access to the preference holding the saved search queries.
@@ -86,27 +85,6 @@ class SearchQueryParser(object):
       * `(author:Asimov or author:Hardy) and not tag:read` [search for unread books by Asimov or Hardy]
     '''
 
-    DEFAULT_LOCATIONS = [
-        'all',
-        'author',       # compatibility
-        'authors',
-        'comment',      # compatibility
-        'comments',
-        'cover',
-        'date',
-        'format',       # compatibility
-        'formats',
-        'isbn',
-        'ondevice',
-        'pubdate',
-        'publisher',
-        'search',
-        'series',
-        'rating',
-        'tag',          # compatibility
-        'tags',
-        'title',
-                 ]
 
     @staticmethod
     def run_tests(parser, result, tests):
@@ -121,12 +99,7 @@ class SearchQueryParser(object):
                 failed.append(test[0])
         return failed
 
-    def __init__(self, locations=None, test=False):
+    def __init__(self, locations, test=False):
-        for k in self.DEFAULT_LOCATIONS:
-            if k not in RESERVED_METADATA_FIELDS:
-                raise ValueError('Search location [%s] is not a reserved word.' %(k))
-        if locations is None:
-            locations = self.DEFAULT_LOCATIONS
         self._tests_failed = False
         # Define a token
         standard_locations = map(lambda x : CaselessLiteral(x)+Suppress(':'),