KG beta updates

resources/recipes/corriere_della_sera_it.recipe

@@ -9,22 +9,25 @@ __description__ = 'Italian daily newspaper'
 '''
 http://www.corriere.it/
 '''
-
+import time
 from calibre.web.feeds.news import BasicNewsRecipe
 
 class ilCorriere(BasicNewsRecipe):
-    __author__     = 'Lorenzo Vigentini, based on Darko Miletic'
+    __author__     = 'Lorenzo Vigentini, based on Darko Miletic, Gabriele Marini'
     description    = 'Italian daily newspaper'
 
-    cover_url      = 'http://images.corriereobjects.it/images/static/common/logo_home.gif?v=200709121520'
+#    cover_url      = 'http://images.corriereobjects.it/images/static/common/logo_home.gif?v=200709121520
-    title          = u'Il Corriere della sera '
+
+
+    title          = u'Il Corriere della sera'
     publisher      = 'RCS Digital'
     category       = 'News, politics, culture, economy, general interest'
 
+    encoding       = 'cp1252'
     language       = 'it'
     timefmt        = '[%a, %d %b, %Y]'
 
-    oldest_article = 1
+    oldest_article = 10
     max_articles_per_feed = 100
     use_embedded_content  = False
     recursion             = 10
@@ -51,9 +54,25 @@ class ilCorriere(BasicNewsRecipe):
 
     remove_tags_after = dict(name='p', attrs={'class':'footnotes'})
 
+    def get_cover_url(self):
+        cover = None
+        st = time.localtime()
+        year = str(st.tm_year)
+        month = "%.2d" % st.tm_mon
+        day = "%.2d" % st.tm_mday
+		#http://images.corriere.it/primapagina/storico/2010_05_17/images/prima_pagina_grande.png
+        cover='http://images.corriere.it/primapagina/storico/'+ year + '_' +  month +'_' + day +'/images/prima_pagina_grande.png'
+        br = BasicNewsRecipe.get_browser()
+        try:
+            br.open(cover)
+        except:
+            self.log("\nCover unavailable")
+            cover ='http://images.corriereobjects.it/images/static/common/logo_home.gif?v=200709121520'
+        return cover
+
     feeds = [
              (u'Ultimora'   ,  u'http://www.corriere.it/rss/ultimora.xml'  ),
-             (u'Editoriali', u'http://www.corriere.it/rss/editoriali.xml'),
+             (u'Editoriali' ,  u'http://www.corriere.it/rss/editoriali.xml'),
              (u'Cronache'   ,  u'http://www.corriere.it/rss/cronache.xml'  ),
              (u'Politica'   ,  u'http://www.corriere.it/rss/politica.xml'  ),
              (u'Esteri'     ,  u'http://www.corriere.it/rss/esteri.xml'    ),
@@ -61,7 +80,9 @@ class ilCorriere(BasicNewsRecipe):
              (u'Cultura'    ,  u'http://www.corriere.it/rss/cultura.xml'   ),
              (u'Scienze'    ,  u'http://www.corriere.it/rss/scienze.xml'   ),
              (u'Salute'     ,  u'http://www.corriere.it/rss/salute.xml'    ),
-             (u'Spettacolo', u'http://www.corriere.it/rss/spettacoli.xml'),
+             (u'Spettacolo' ,  u'http://www.corriere.it/rss/spettacoli.xml'),
              (u'Cinema e TV',  u'http://www.corriere.it/rss/cinema.xml'    ),
-             (u'Sport'     , u'http://www.corriere.it/rss/sport.xml'     )
+             (u'Sport'      ,  u'http://www.corriere.it/rss/sport.xml'     ),
+             (u'Roma'      ,   u'http://www.corriere.it/rss/homepage_roma.xml'),
+             (u'Milano'      , u'http://www.corriere.it/rss/homepage_milano.xml')
             ]

resources/recipes/darknet.recipe

@@ -23,7 +23,8 @@ class darknet(BasicNewsRecipe):
 
     remove_tags = [dict(id='navi_top'),
 		   dict(id='navi_bottom'),
-		   dict(id='logo'),
+		   dict(id='nav'),
+		   dict(id='top-ad'),
 		   dict(id='login_suche'),
 		   dict(id='navi_login'),
 		   dict(id='breadcrumb'),
@@ -32,13 +33,14 @@ class darknet(BasicNewsRecipe):
 		   dict(name='span', attrs={'class':'rsaquo'}),
 		   dict(name='span', attrs={'class':'next'}),
 		   dict(name='span', attrs={'class':'prev'}),
+		   dict(name='span', attrs={'class':'comments'}),
 		   dict(name='div', attrs={'class':'news_logo'}),
 		   dict(name='div', attrs={'class':'nextprev'}),
+		   dict(name='div', attrs={'class':'tags'}),
+		   dict(name='div', attrs={'class':'Nav'}),
 		   dict(name='p', attrs={'class':'news_option'}),
 		   dict(name='p', attrs={'class':'news_foren'})]
-    remove_tags_after = [dict(name='div', attrs={'class':'entrybody'})]
+    remove_tags_after = [dict(name='div', attrs={'class':'meta-footer'})]
 
     feeds =  [ ('darknet', 'http://feedproxy.google.com/darknethackers') ]
 
-
-

resources/recipes/leggo_it.recipe

@@ -54,12 +54,16 @@ class LeggoIT(BasicNewsRecipe):
         day = "%.2d" % st.tm_mday
         cover='http://www.leggo.it/'+ year +  month + day + '/jpeg/LEGGO_ROMA_1.jpg'
         br = BasicNewsRecipe.get_browser()
+        try:
+            br.open(cover)
+        except:
+            cover='http://www.leggo.it/'+ year +  month + day + '/jpeg/LEGGO_ROMA_3.jpg'
+            br = BasicNewsRecipe.get_browser()
             try:
                 br.open(cover)
             except:
                 self.log("\nCover unavailable")
                 cover = 'http://www.leggo.it/img/logo-leggo2.gif'
-
         return cover
 
 

src/calibre/devices/__init__.py

@@ -5,7 +5,7 @@ __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 Device drivers.
 '''
 
-import sys, time, pprint
+import sys, time, pprint, operator
 from functools import partial
 from StringIO import StringIO
 
@@ -82,7 +82,9 @@ def debug(ioreg_to_tmp=False, buf=None):
         if iswindows:
             drives = win_pnp_drives(debug=True)
             out('Drives detected:')
-            out(pprint.pformat(drives))
+            for drive in sorted(drives.keys(),
+                    key=operator.attrgetter('order')):
+                prints(u'\t(%d)'%drive.order, drive, '~', drives[drive])
 
         ioreg = None
         if isosx:

src/calibre/devices/apple/driver.py

@@ -22,6 +22,7 @@ if isosx:
     import appscript, osax
 
 if iswindows:
+    print "ITUNES: Running under windows"
     import win32com.client
 
 class UserInteractionRequired(Exception):

src/calibre/devices/binatone/driver.py

@@ -35,15 +35,6 @@ class README(USBMS):
 
     SUPPORTS_SUB_DIRS = True
 
-    def windows_sort_drives(self, drives):
-        main = drives.get('main', None)
-        card = drives.get('carda', None)
-        if card and main and card < main:
-            drives['main'] = card
-            drives['carda'] = main
-
-        return drives
-
     def linux_swap_drives(self, drives):
         if len(drives) < 2: return drives
         drives = list(drives)

src/calibre/devices/eb600/driver.py

@@ -48,15 +48,6 @@ class EB600(USBMS):
     EBOOK_DIR_CARD_A = ''
     SUPPORTS_SUB_DIRS = True
 
-    def windows_sort_drives(self, drives):
-        main = drives.get('main', None)
-        card = drives.get('carda', None)
-        if card and main and card < main:
-            drives['main'] = card
-            drives['carda'] = main
-
-        return drives
-
 
 class COOL_ER(EB600):
 

src/calibre/devices/edge/driver.py

@@ -36,12 +36,4 @@ class EDGE(USBMS):
     EBOOK_DIR_MAIN = 'download'
     SUPPORTS_SUB_DIRS = True
 
-    def windows_sort_drives(self, drives):
-        main = drives.get('main', None)
-        card = drives.get('carda', None)
-        if card and main and card < main:
-            drives['main'] = card
-            drives['carda'] = main
-
-        return drives
 

src/calibre/devices/eslick/driver.py

@@ -36,12 +36,4 @@ class ESLICK(USBMS):
 
     SUPPORTS_SUB_DIRS = True
 
-    def windows_sort_drives(self, drives):
-        main = drives.get('main', None)
-        card = drives.get('carda', None)
-        if card and main and card < main:
-            drives['main'] = card
-            drives['carda'] = main
-
-        return drives
 

src/calibre/devices/hanlin/driver.py

@@ -39,23 +39,6 @@ class HANLINV3(USBMS):
 
     SUPPORTS_SUB_DIRS = True
 
-    def windows_sort_drives(self, drives):
-        main = drives.get('main', None)
-        card = drives.get('carda', None)
-        if card and main and card > main:
-            drives['main'] = card
-            drives['carda'] = main
-
-        if card and not main:
-            drives['main'] = card
-            drives['carda'] = None
-
-        return drives
-
-    def windows_open_callback(self, drives):
-        if 'main' not in drives and 'carda' in drives:
-            drives['main'] = drives.pop('carda')
-        return drives
 
     def osx_sort_names(self, names):
         main = names.get('main', None)
@@ -129,13 +112,4 @@ class BOOX(HANLINV3):
     EBOOK_DIR_CARD_A = 'MyBooks'
 
 
-    def windows_sort_drives(self, drives):
-        main = drives.get('main', None)
-        card = drives.get('carda', None)
-        if card and main and card < main:
-            drives['main'] = card
-            drives['carda'] = main
-
-        return drives
-
 

src/calibre/devices/iriver/driver.py

@@ -36,12 +36,4 @@ class IRIVER_STORY(USBMS):
 
     SUPPORTS_SUB_DIRS = True
 
-    def windows_open_callback(self, drives):
-        main = drives.get('main', None)
-        card = drives.get('carda', None)
-        if card and main and card < main:
-            drives['main'] = card
-            drives['carda'] = main
-
-        return drives
 

src/calibre/devices/jetbook/driver.py

@@ -80,11 +80,3 @@ class JETBOOK(USBMS):
 
         return mi
 
-    def windows_sort_drives(self, drives):
-        main = drives.get('main', None)
-        card = drives.get('carda', None)
-        if card and main and card < main:
-            drives['main'] = card
-            drives['carda'] = main
-
-        return drives

src/calibre/devices/misc.py

@@ -45,7 +45,7 @@ class KOBO(USBMS):
     BCD         = [0x0110]
 
     VENDOR_NAME = 'KOBO_INC'
-    WINDOWS_MAIN_MEM = '.KOBOEREADER'
+    WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = '.KOBOEREADER'
 
     EBOOK_DIR_MAIN = ''
     SUPPORTS_SUB_DIRS = True

src/calibre/devices/nook/driver.py

@@ -77,14 +77,6 @@ class NOOK(USBMS):
         with open('%s.jpg' % os.path.join(path, filename), 'wb') as coverfile:
             coverfile.write(coverdata)
 
-    def windows_sort_drives(self, drives):
-        main = drives.get('main', None)
-        card = drives.get('carda', None)
-        if card and main and card < main:
-            drives['main'] = card
-            drives['carda'] = main
-
-        return drives
 
     def sanitize_path_components(self, components):
         return [x.replace('#', '_') for x in components]

src/calibre/devices/scanner.py

@@ -5,7 +5,7 @@ Device scanner that fetches list of devices on system ina  platform dependent
 manner.
 '''
 
-import sys, os
+import sys, os, re
 from threading import RLock
 
 from calibre import iswindows, isosx, plugins, islinux
@@ -23,6 +23,14 @@ elif isosx:
     except:
         raise RuntimeError('Failed to load the usbobserver plugin: %s'%plugins['usbobserver'][1])
 
+class Drive(str):
+
+    def __new__(self, val, order=0):
+        typ = str.__new__(self, val)
+        typ.order = order
+        return typ
+
+
 class WinPNPScanner(object):
 
     def __init__(self):
@@ -45,6 +53,13 @@ class WinPNPScanner(object):
             finally:
                 win32api.SetErrorMode(oldError)
 
+    def drive_order(self, pnp_id):
+        order = 0
+        match = re.search(r'REV_.*?&(\d+)', pnp_id)
+        if match is not None:
+            order = int(match.group(1))
+        return order
+
     def __call__(self, debug=False):
         if self.scanner is None:
             return {}
@@ -66,7 +81,7 @@ class WinPNPScanner(object):
             val = [x.upper() for x in val]
             val = [x for x in val if 'USBSTOR' in x]
             if val:
-                ans[key+':\\'] = val[-1]
+                ans[Drive(key+':\\', order=self.drive_order(val[-1]))] = val[-1]
         return ans
 
 win_pnp_drives = WinPNPScanner()

src/calibre/devices/teclast/driver.py

@@ -30,14 +30,6 @@ class TECLAST_K3(USBMS):
     EBOOK_DIR_CARD_A = ''
     SUPPORTS_SUB_DIRS = True
 
-    def windows_sort_drives(self, drives):
-        main = drives.get('main', None)
-        card = drives.get('carda', None)
-        if card and main and card < main:
-            drives['main'] = card
-            drives['carda'] = main
-
-        return drives
 
 class NEWSMY(TECLAST_K3):
     name = 'Newsmy device interface'
@@ -50,9 +42,6 @@ class NEWSMY(TECLAST_K3):
     WINDOWS_MAIN_MEM = 'NEWSMY'
     WINDOWS_CARD_A_MEM = 'USBDISK____SD'
 
-    def windows_sort_drives(self, drives):
-        return drives
-
 class IPAPYRUS(TECLAST_K3):
 
     name = 'iPapyrus device interface'

src/calibre/devices/usbms/device.py

@@ -11,13 +11,7 @@ intended to be subclassed with the relevant parts implemented for a particular
 device. This class handles device detection.
 '''
 
-import os
+import os, subprocess, time, re, sys, glob, operator
-import subprocess
-import time
-import re
-import sys
-import glob
-
 from itertools import repeat
 
 from calibre.devices.interface import DevicePlugin
@@ -62,6 +56,8 @@ class Device(DeviceConfig, DevicePlugin):
     BCD         = None
 
     VENDOR_NAME = None
+
+    # These can be None, string, list of strings or compiled regex
     WINDOWS_MAIN_MEM = None
     WINDOWS_CARD_A_MEM = None
     WINDOWS_CARD_B_MEM = None
@@ -246,21 +242,26 @@ class Device(DeviceConfig, DevicePlugin):
                 drives.get('main', None) is None:
             drives['main'] = drives.pop('carda')
 
-        drives = self.windows_open_callback(drives)
-
         if drives.get('main', None) is None:
             raise DeviceError(
                 _('Unable to detect the %s disk drive. Try rebooting.') %
                 self.__class__.__name__)
 
+        # Sort drives by their PNP drive numbers if the CARD and MAIN
+        # MEM strings are identical
+        if self.WINDOWS_MAIN_MEM in (self.WINDOWS_CARD_A_MEM,
+                self.WINDOWS_CARD_B_MEM) or \
+                self.WINDOWS_CARD_A_MEM == self.WINDOWS_CARD_B_MEM:
+            letters = sorted(drives.values(), key=operator.attrgetter('order'))
+            drives = {}
+            for which, letter in zip(['main', 'carda', 'cardb'], letters):
+                drives[which] = letter
+
         drives = self.windows_sort_drives(drives)
         self._main_prefix = drives.get('main')
         self._card_a_prefix = drives.get('carda', None)
         self._card_b_prefix = drives.get('cardb', None)
 
-    def windows_open_callback(self, drives):
-        return drives
-
     @classmethod
     def run_ioreg(cls, raw=None):
         if raw is not None:

src/calibre/ebooks/epub/output.py

@@ -121,11 +121,27 @@ class EPUBOutput(OutputFormatPlugin):
                 if not pre.text and len(pre) == 0:
                     pre.tag = 'div'
 
+    def upshift_markup(self):
+        'Upgrade markup to comply with XHTML 1.1 where possible'
+        from calibre.ebooks.oeb.base import XPath
+        for x in self.oeb.spine:
+            root = x.data
+            body = XPath('//h:body')(root)
+            if body:
+                body = body[0]
+
+            if not hasattr(body, 'xpath'):
+                continue
+            for u in XPath('//h:u')(root):
+                u.tag = 'span'
+                u.set('style', 'text-decoration:underline')
+
     def convert(self, oeb, output_path, input_plugin, opts, log):
         self.log, self.opts, self.oeb = log, opts, oeb
 
         self.workaround_ade_quirks()
         self.workaround_webkit_quirks()
+        self.upshift_markup()
         from calibre.ebooks.oeb.transforms.rescale import RescaleImages
         RescaleImages()(oeb, opts)
 

src/calibre/ebooks/oeb/transforms/cover.py

@@ -5,10 +5,15 @@ __license__   = 'GPL v3'
 __copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import textwrap
+import textwrap, cStringIO
 from urllib import unquote
 
 from lxml import etree
+try:
+    from PIL import Image as PILImage
+    PILImage
+except ImportError:
+    import Image as PILImage
 
 from calibre import __appname__, __version__, guess_type
 
@@ -28,9 +33,9 @@ class CoverManager(object):
             <body>
                 <svg version="1.1" xmlns="http://www.w3.org/2000/svg"
                     xmlns:xlink="http://www.w3.org/1999/xlink"
-                    width="100%%" height="100%%" viewBox="0 0 600 800"
+                    width="100%%" height="100%%" viewBox="__viewbox__"
                     preserveAspectRatio="__ar__">
-                    <image width="600" height="800" xlink:href="%s"/>
+                    <image width="__width__" height="__height__" xlink:href="%s"/>
                 </svg>
             </body>
         </html>
@@ -93,7 +98,6 @@ class CoverManager(object):
         title = unicode(m.title[0])
         authors = [unicode(x) for x in m.creator if x.role == 'aut']
 
-        import cStringIO
         cover_file = cStringIO.StringIO()
         try:
             try:
@@ -142,6 +146,18 @@ class CoverManager(object):
             self.log.exception('Failed to generate default cover')
         return None
 
+    def inspect_cover(self, href):
+        from calibre.ebooks.oeb.base import urlnormalize
+        for x in self.oeb.manifest:
+            if x.href == urlnormalize(href):
+                try:
+                    raw = x.data
+                    f = cStringIO.StringIO(raw)
+                    im = PILImage.open(f)
+                    return im.size
+                except:
+                    self.log.exception('Failed to read image dimensions')
+        return None, None
 
     def insert_cover(self):
         from calibre.ebooks.oeb.base import urldefrag
@@ -152,6 +168,19 @@ class CoverManager(object):
                 href = g['cover'].href
             else:
                 href = self.default_cover()
+            width, height = self.inspect_cover(href)
+            if width is None or height is None:
+                self.log.warning('Failed to read cover dimensions')
+                width, height = 600, 800
+            if self.preserve_aspect_ratio:
+                width, height = 600, 800
+            self.svg_template = self.svg_template.replace('__viewbox__',
+                    '0 0 %d %d'%(width, height))
+            self.svg_template = self.svg_template.replace('__width__',
+                    str(width))
+            self.svg_template = self.svg_template.replace('__height__',
+                    str(height))
+
             if href is not None:
                 templ = self.non_svg_template if self.no_svg_cover \
                         else self.svg_template

src/calibre/gui2/__init__.py

@@ -530,6 +530,7 @@ class Application(QApplication):
                         border-radius: 10px;
                         opacity: 200;
                         background-color: #e1e1ff;
+                        color: black;
                     }
             ''')
 

src/calibre/gui2/dialogs/config/__init__.py

@@ -371,7 +371,7 @@ class ConfigDialog(ResizableDialog, Ui_Dialog):
         hidden_cols = state['hidden_columns']
         positions = state['column_positions']
         colmap.sort(cmp=lambda x,y: cmp(positions[x], positions[y]))
-        self.custcols = copy.deepcopy(self.db.custom_column_label_map)
+        self.custcols = copy.deepcopy(self.db.field_metadata.get_custom_field_metadata())
         for col in colmap:
             item = QListWidgetItem(self.model.headers[col], self.columns)
             item.setData(Qt.UserRole, QVariant(col))
@@ -713,20 +713,20 @@ class ConfigDialog(ResizableDialog, Ui_Dialog):
 
         must_restart = False
         for c in self.custcols:
-            if self.custcols[c]['num'] is None:
+            if self.custcols[c]['colnum'] is None:
                 self.db.create_custom_column(
-                                label=c,
+                                label=self.custcols[c]['label'],
                                 name=self.custcols[c]['name'],
                                 datatype=self.custcols[c]['datatype'],
                                 is_multiple=self.custcols[c]['is_multiple'],
                                 display = self.custcols[c]['display'])
                 must_restart = True
             elif '*deleteme' in self.custcols[c]:
-                self.db.delete_custom_column(label=c)
+                self.db.delete_custom_column(label=self.custcols[c]['label'])
                 must_restart = True
             elif '*edited' in self.custcols[c]:
                 cc = self.custcols[c]
-                self.db.set_custom_column_metadata(cc['num'], name=cc['name'],
+                self.db.set_custom_column_metadata(cc['colnum'], name=cc['name'],
                                                    label=cc['label'],
                                                    display = self.custcols[c]['display'])
                 if '*must_restart' in self.custcols[c]:

src/calibre/gui2/dialogs/config/create_custom_column.py

@@ -69,12 +69,13 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
         self.column_name_box.setText(c['label'])
         self.column_heading_box.setText(c['name'])
         ct = c['datatype'] if not c['is_multiple'] else '*text'
-        self.orig_column_number = c['num']
+        self.orig_column_number = c['colnum']
         self.orig_column_name = col
         column_numbers = dict(map(lambda x:(self.column_types[x]['datatype'], x), self.column_types))
         self.column_type_box.setCurrentIndex(column_numbers[ct])
         self.column_type_box.setEnabled(False)
         if ct == 'datetime':
+            if c['display'].get('date_format', None):
                 self.date_format_box.setText(c['display'].get('date_format', ''))
         self.datatype_changed()
         self.exec_()
@@ -90,7 +91,11 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
 
 
     def accept(self):
-        col = unicode(self.column_name_box.text())
+        col = unicode(self.column_name_box.text()).lower()
+        if not col:
+            return self.simple_error('', _('No lookup name was provided'))
+        if not col.isalnum() or not col[0].isalpha():
+            return self.simple_error('', _('The label must contain only letters and digits, and start with a letter'))
         col_heading = unicode(self.column_heading_box.text())
         col_type = self.column_types[self.column_type_box.currentIndex()]['datatype']
         if col_type == '*text':
@@ -98,20 +103,18 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
             is_multiple = True
         else:
             is_multiple = False
-        if not col:
-            return self.simple_error('', _('No lookup name was provided'))
         if not col_heading:
             return self.simple_error('', _('No column heading was provided'))
         bad_col = False
         if col in self.parent.custcols:
-            if not self.editing_col or self.parent.custcols[col]['num'] != self.orig_column_number:
+            if not self.editing_col or self.parent.custcols[col]['colnum'] != self.orig_column_number:
                 bad_col = True
         if bad_col:
             return self.simple_error('', _('The lookup name %s is already used')%col)
         bad_head = False
         for t in self.parent.custcols:
             if self.parent.custcols[t]['name'] == col_heading:
-                if not self.editing_col or self.parent.custcols[t]['num'] != self.orig_column_number:
+                if not self.editing_col or self.parent.custcols[t]['colnum'] != self.orig_column_number:
                     bad_head = True
         for t in self.standard_colheads:
             if self.standard_colheads[t] == col_heading:
@@ -128,25 +131,27 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn):
             else:
                 date_format = {'date_format': None}
 
+        key = self.parent.db.field_metadata.custom_field_prefix+col
         if not self.editing_col:
-            self.parent.custcols[col] = {
+            self.parent.db.field_metadata
+            self.parent.custcols[key] = {
                     'label':col,
                     'name':col_heading,
                     'datatype':col_type,
                     'editable':True,
                     'display':date_format,
                     'normalized':None,
-                    'num':None,
+                    'colnum':None,
                     'is_multiple':is_multiple,
                 }
             item = QListWidgetItem(col_heading, self.parent.columns)
-            item.setData(Qt.UserRole, QVariant(col))
+            item.setData(Qt.UserRole, QVariant(key))
             item.setFlags(Qt.ItemIsEnabled|Qt.ItemIsUserCheckable|Qt.ItemIsSelectable)
             item.setCheckState(Qt.Checked)
         else:
             idx = self.parent.columns.currentRow()
             item = self.parent.columns.item(idx)
-            item.setData(Qt.UserRole, QVariant(col))
+            item.setData(Qt.UserRole, QVariant(key))
             item.setText(col_heading)
             self.parent.custcols[self.orig_column_name]['label'] = col
             self.parent.custcols[self.orig_column_name]['name'] = col_heading

src/calibre/gui2/dialogs/config/create_custom_column.ui

@@ -65,7 +65,7 @@
           </size>
          </property>
          <property name="toolTip">
-          <string>Used for searching the column. Must be lower case and not contain spaces or colons.</string>
+          <string>Used for searching the column. Must contain only digits and lower case letters.</string>
          </property>
         </widget>
        </item>

src/calibre/gui2/dialogs/tag_categories.py

@@ -49,7 +49,7 @@ class TagCategories(QDialog, Ui_TagCategories):
         cc_map = self.db.custom_column_label_map
         for cc in cc_map:
             if cc_map[cc]['datatype'] == 'text':
-                self.category_labels.append(db.tag_browser_categories.get_search_label(cc))
+                self.category_labels.append(db.field_metadata.label_to_key(cc))
                 category_icons.append(cc_icon)
                 category_values.append(lambda col=cc: self.db.all_custom(label=col))
                 category_names.append(cc_map[cc]['name'])

src/calibre/gui2/dialogs/tag_editor.py

@@ -9,6 +9,9 @@ from calibre.constants import islinux
 
 class TagEditor(QDialog, Ui_TagEditor):
 
+    def tag_cmp(self, x, y):
+        return cmp(x.lower(), y.lower())
+
     def __init__(self, window, db, index=None):
         QDialog.__init__(self, window)
         Ui_TagEditor.__init__(self)
@@ -22,7 +25,7 @@ class TagEditor(QDialog, Ui_TagEditor):
             tags = []
         if tags:
             tags = [tag.strip() for tag in tags.split(',') if tag.strip()]
-            tags.sort()
+            tags.sort(cmp=self.tag_cmp)
             for tag in tags:
                 self.applied_tags.addItem(tag)
         else:
@@ -32,7 +35,7 @@ class TagEditor(QDialog, Ui_TagEditor):
 
         all_tags = [tag for tag in self.db.all_tags()]
         all_tags = list(set(all_tags))
-        all_tags.sort()
+        all_tags.sort(cmp=self.tag_cmp)
         for tag in all_tags:
             if tag not in tags:
                 self.available_tags.addItem(tag)
@@ -79,7 +82,7 @@ class TagEditor(QDialog, Ui_TagEditor):
             self.tags.append(tag)
             self.available_tags.takeItem(self.available_tags.row(item))
 
-        self.tags.sort()
+        self.tags.sort(cmp=self.tag_cmp)
         self.applied_tags.clear()
         for tag in self.tags:
             self.applied_tags.addItem(tag)
@@ -93,12 +96,17 @@ class TagEditor(QDialog, Ui_TagEditor):
             self.tags.remove(tag)
             self.available_tags.addItem(tag)
 
-        self.tags.sort()
+        self.tags.sort(cmp=self.tag_cmp)
         self.applied_tags.clear()
         for tag in self.tags:
             self.applied_tags.addItem(tag)
 
-        self.available_tags.sortItems()
+        items = [unicode(self.available_tags.item(x).text()) for x in
+                range(self.available_tags.count())]
+        items.sort(cmp=self.tag_cmp)
+        self.available_tags.clear()
+        for item in items:
+            self.available_tags.addItem(item)
 
     def add_tag(self):
         tags = unicode(self.add_tag_input.text()).split(',')
@@ -109,7 +117,7 @@ class TagEditor(QDialog, Ui_TagEditor):
             if tag not in self.tags:
                 self.tags.append(tag)
 
-        self.tags.sort()
+        self.tags.sort(cmp=self.tag_cmp)
         self.applied_tags.clear()
         for tag in self.tags:
             self.applied_tags.addItem(tag)

src/calibre/gui2/library/delegates.py

@@ -171,7 +171,8 @@ class TagsDelegate(QStyledItemDelegate): # {{{
             if not index.model().is_custom_column(col):
                 editor = TagsLineEdit(parent, self.db.all_tags())
             else:
-                editor = TagsLineEdit(parent, sorted(list(self.db.all_custom(label=col))))
+                editor = TagsLineEdit(parent,
+                        sorted(list(self.db.all_custom(label=self.db.field_metadata.key_to_label(col)))))
                 return editor
         else:
             editor = EnLineEdit(parent)
@@ -209,7 +210,7 @@ class CcDateDelegate(QStyledItemDelegate): # {{{
         m = index.model()
         # db col is not named for the field, but for the table number. To get it,
         # gui column -> column label -> table number -> db column
-        val = m.db.data[index.row()][m.db.FIELD_MAP[m.custom_columns[m.column_map[index.column()]]['num']]]
+        val = m.db.data[index.row()][m.custom_columns[m.column_map[index.column()]]['rec_index']]
         if val is None:
             val = now()
         editor.setDate(val)
@@ -243,7 +244,7 @@ class CcTextDelegate(QStyledItemDelegate): # {{{
             editor.setDecimals(2)
         else:
             editor = EnLineEdit(parent)
-            complete_items = sorted(list(m.db.all_custom(label=col)))
+            complete_items = sorted(list(m.db.all_custom(label=m.db.field_metadata.key_to_label(col))))
             completer = QCompleter(complete_items, self)
             completer.setCaseSensitivity(Qt.CaseInsensitive)
             completer.setCompletionMode(QCompleter.PopupCompletion)
@@ -260,9 +261,7 @@ class CcCommentsDelegate(QStyledItemDelegate): # {{{
     def createEditor(self, parent, option, index):
         m = index.model()
         col = m.column_map[index.column()]
-        # db col is not named for the field, but for the table number. To get it,
+        text = m.db.data[index.row()][m.custom_columns[col]['rec_index']]
-        # gui column -> column label -> table number -> db column
-        text = m.db.data[index.row()][m.db.FIELD_MAP[m.custom_columns[col]['num']]]
         editor = CommentsDialog(parent, text)
         d = editor.exec_()
         if d:
@@ -297,9 +296,7 @@ class CcBoolDelegate(QStyledItemDelegate): # {{{
 
     def setEditorData(self, editor, index):
         m = index.model()
-        # db col is not named for the field, but for the table number. To get it,
+        val = m.db.data[index.row()][m.custom_columns[m.column_map[index.column()]]['rec_index']]
-        # gui column -> column label -> table number -> db column
-        val = m.db.data[index.row()][m.db.FIELD_MAP[m.custom_columns[m.column_map[index.column()]]['num']]]
         if tweaks['bool_custom_columns_are_tristate'] == 'no':
             val = 1 if not val else 0
         else:

src/calibre/gui2/library/models.py

@@ -111,15 +111,15 @@ class BooksModel(QAbstractTableModel): # {{{
 
     def set_database(self, db):
         self.db = db
-        self.custom_columns = self.db.custom_column_label_map
+        self.custom_columns = self.db.field_metadata.get_custom_field_metadata()
         self.column_map = list(self.orig_headers.keys()) + \
                           list(self.custom_columns)
         def col_idx(name):
             if name == 'ondevice':
                 return -1
-            if name not in self.db.FIELD_MAP:
+            if name not in self.db.field_metadata:
                 return 100000
-            return self.db.FIELD_MAP[name]
+            return self.db.field_metadata[name]['rec_index']
 
         self.column_map.sort(cmp=lambda x,y: cmp(col_idx(x), col_idx(y)))
         for col in self.column_map:
@@ -232,11 +232,12 @@ class BooksModel(QAbstractTableModel): # {{{
             return
         self.about_to_be_sorted.emit(self.db.id)
         ascending = order == Qt.AscendingOrder
-        self.db.sort(self.column_map[col], ascending)
+        label = self.column_map[col]
+        self.db.sort(label, ascending)
         if reset:
             self.clear_caches()
             self.reset()
-        self.sorted_on = (self.column_map[col], order)
+        self.sorted_on = (label, order)
         self.sort_history.insert(0, self.sorted_on)
         self.sorting_done.emit(self.db.index)
 
@@ -551,36 +552,36 @@ class BooksModel(QAbstractTableModel): # {{{
 
         self.dc = {
                    'title'    : functools.partial(text_type,
-                                idx=self.db.FIELD_MAP['title'], mult=False),
+                                idx=self.db.field_metadata['title']['rec_index'], mult=False),
                    'authors'  : functools.partial(authors,
-                                idx=self.db.FIELD_MAP['authors']),
+                                idx=self.db.field_metadata['authors']['rec_index']),
                    'size'     : functools.partial(size,
-                                idx=self.db.FIELD_MAP['size']),
+                                idx=self.db.field_metadata['size']['rec_index']),
                    'timestamp': functools.partial(datetime_type,
-                                idx=self.db.FIELD_MAP['timestamp']),
+                                idx=self.db.field_metadata['timestamp']['rec_index']),
                    'pubdate'  : functools.partial(datetime_type,
-                                idx=self.db.FIELD_MAP['pubdate']),
+                                idx=self.db.field_metadata['pubdate']['rec_index']),
                    'rating'   : functools.partial(rating_type,
-                                idx=self.db.FIELD_MAP['rating']),
+                                idx=self.db.field_metadata['rating']['rec_index']),
                    'publisher': functools.partial(text_type,
-                                idx=self.db.FIELD_MAP['publisher'], mult=False),
+                                idx=self.db.field_metadata['publisher']['rec_index'], mult=False),
                    'tags'     : functools.partial(tags,
-                                idx=self.db.FIELD_MAP['tags']),
+                                idx=self.db.field_metadata['tags']['rec_index']),
                    'series'   : functools.partial(series,
-                                idx=self.db.FIELD_MAP['series'],
+                                idx=self.db.field_metadata['series']['rec_index'],
-                                siix=self.db.FIELD_MAP['series_index']),
+                                siix=self.db.field_metadata['series_index']['rec_index']),
                    'ondevice' : functools.partial(text_type,
-                                idx=self.db.FIELD_MAP['ondevice'], mult=False),
+                                idx=self.db.field_metadata['ondevice']['rec_index'], mult=False),
                    }
 
         self.dc_decorator = {
                 'ondevice':functools.partial(ondevice_decorator,
-                    idx=self.db.FIELD_MAP['ondevice']),
+                    idx=self.db.field_metadata['ondevice']['rec_index']),
                     }
 
         # Add the custom columns to the data converters
         for col in self.custom_columns:
-            idx = self.db.FIELD_MAP[self.custom_columns[col]['num']]
+            idx = self.custom_columns[col]['rec_index']
             datatype = self.custom_columns[col]['datatype']
             if datatype in ('text', 'comments'):
                 self.dc[col] = functools.partial(text_type, idx=idx, mult=self.custom_columns[col]['is_multiple'])
@@ -632,8 +633,6 @@ class BooksModel(QAbstractTableModel): # {{{
                 return None
             if role == Qt.ToolTipRole:
                 ht = self.column_map[section]
-                if self.is_custom_column(self.column_map[section]):
-                    ht = self.db.tag_browser_categories.custom_field_prefix + ht
                 if ht == 'timestamp': # change help text because users know this field as 'date'
                     ht = 'date'
                 return QVariant(_('The lookup/search name is "{0}"').format(ht))
@@ -652,7 +651,7 @@ class BooksModel(QAbstractTableModel): # {{{
             if colhead in self.editable_cols:
                 flags |= Qt.ItemIsEditable
             elif self.is_custom_column(colhead):
-                if self.custom_columns[colhead]['editable']:
+                if self.custom_columns[colhead]['is_editable']:
                     flags |= Qt.ItemIsEditable
         return flags
 
@@ -679,7 +678,9 @@ class BooksModel(QAbstractTableModel): # {{{
                 if not val.isValid():
                     return False
                 val = qt_to_dt(val, as_utc=False)
-        self.db.set_custom(self.db.id(row), val, label=colhead, num=None, append=False, notify=True)
+        self.db.set_custom(self.db.id(row), val,
+                           label=self.db.field_metadata.key_to_label(colhead),
+                           num=None, append=False, notify=True)
         return True
 
     def setData(self, index, value, role):

src/calibre/gui2/tag_view.py

@@ -15,6 +15,7 @@ from PyQt4.Qt import Qt, QTreeView, QApplication, pyqtSignal, \
 from calibre.gui2 import config, NONE
 from calibre.utils.config import prefs
 from calibre.library.field_metadata import TagsIcons
+from calibre.utils.search_query_parser import saved_searches
 
 class TagsView(QTreeView): # {{{
 
@@ -221,10 +222,22 @@ class TagsModel(QAbstractItemModel): # {{{
         self.db = db
         self.search_restriction = ''
         self.ignore_next_search = 0
+
+        # Reconstruct the user categories, putting them into metadata
+        tb_cats = self.db.field_metadata
+        for k in tb_cats.keys():
+            if tb_cats[k]['kind'] in ['user', 'search']:
+                del tb_cats[k]
+        for user_cat in sorted(prefs['user_categories'].keys()):
+            cat_name = user_cat+':' # add the ':' to avoid name collision
+            tb_cats.add_user_category(label=cat_name, name=user_cat)
+        if len(saved_searches.names()):
+            tb_cats.add_search_category(label='search', name=_('Searches'))
+
         data = self.get_node_tree(config['sort_by_popularity'])
         self.root_item = TagTreeItem()
         for i, r in enumerate(self.row_map):
-            if self.db.get_tag_browser_categories()[r]['kind'] != 'user':
+            if self.db.field_metadata[r]['kind'] != 'user':
                 tt = _('The lookup/search name is "{0}"').format(r)
             else:
                 tt = ''
@@ -248,7 +261,7 @@ class TagsModel(QAbstractItemModel): # {{{
         else:
             data = self.db.get_categories(sort_on_count=sort, icon_map=self.category_icon_map)
 
-        tb_categories = self.db.get_tag_browser_categories()
+        tb_categories = self.db.field_metadata
         for category in tb_categories:
             if category in data: # They should always be there, but ...
                 self.row_map.append(category)

src/calibre/library/caches.py

@@ -150,14 +150,13 @@ class ResultCache(SearchQueryParser):
     '''
     Stores sorted and filtered metadata in memory.
     '''
-    def __init__(self, FIELD_MAP, cc_label_map, tag_browser_categories):
+    def __init__(self, FIELD_MAP, field_metadata):
         self.FIELD_MAP = FIELD_MAP
-        self.custom_column_label_map = cc_label_map
         self._map = self._map_filtered = self._data = []
         self.first_sort = True
         self.search_restriction = ''
-        self.tag_browser_categories = tag_browser_categories
+        self.field_metadata = field_metadata
-        self.all_search_locations = tag_browser_categories.get_search_labels()
+        self.all_search_locations = field_metadata.get_search_terms()
         SearchQueryParser.__init__(self, self.all_search_locations)
         self.build_date_relop_dict()
         self.build_numeric_relop_dict()
@@ -249,10 +248,10 @@ class ResultCache(SearchQueryParser):
                 query = query[p:]
         if relop is None:
                 (p, relop) = self.date_search_relops['=']
-        if location in self.custom_column_label_map:
+
-            loc = self.FIELD_MAP[self.custom_column_label_map[location]['num']]
+        if location == 'date':
-        else:
+            location = 'timestamp'
-            loc = self.FIELD_MAP[{'date':'timestamp', 'pubdate':'pubdate'}[location]]
+        loc = self.field_metadata[location]['rec_index']
 
         if query == _('today'):
             qd = now()
@@ -310,9 +309,9 @@ class ResultCache(SearchQueryParser):
                 query = query[p:]
         if relop is None:
                 (p, relop) = self.numeric_search_relops['=']
-        if location in self.custom_column_label_map:
+
-            loc = self.FIELD_MAP[self.custom_column_label_map[location]['num']]
+        loc = self.field_metadata[location]['rec_index']
-            dt = self.custom_column_label_map[location]['datatype']
+        dt = self.field_metadata[location]['datatype']
         if dt == 'int':
             cast = (lambda x: int (x))
             adjust = lambda x: x
@@ -322,10 +321,6 @@ class ResultCache(SearchQueryParser):
         elif dt == 'float':
             cast = lambda x : float (x)
             adjust = lambda x: x
-        else:
-            loc = self.FIELD_MAP['rating']
-            cast = (lambda x: int (x))
-            adjust = lambda x: x/2
 
         try:
             q = cast(query)
@@ -346,22 +341,21 @@ class ResultCache(SearchQueryParser):
     def get_matches(self, location, query):
         matches = set([])
         if query and query.strip():
-            location = location.lower().strip()
+            # get metadata key associated with the search term. Eliminates
+            # dealing with plurals and other aliases
+            location = self.field_metadata.search_term_to_key(location.lower().strip())
 
-            ### take care of dates special case
+            # take care of dates special case
-            if (location in ('pubdate', 'date')) or \
+            if location in self.field_metadata and \
-                    ((location in self.custom_column_label_map) and \
+                     self.field_metadata[location]['datatype'] == 'datetime':
-                     self.custom_column_label_map[location]['datatype'] == 'datetime'):
                 return self.get_dates_matches(location, query.lower())
 
-            ### take care of numerics special case
+            # take care of numbers special case
-            if location == 'rating' or \
+            if location in self.field_metadata and \
-                    (location in self.custom_column_label_map and
+                    self.field_metadata[location]['datatype'] in ('rating', 'int', 'float'):
-                     self.custom_column_label_map[location]['datatype'] in
-                                ('rating', 'int', 'float')):
                 return self.get_numeric_matches(location, query.lower())
 
-            ### everything else
+            # everything else, or 'all' matches
             matchkind = CONTAINS_MATCH
             if (len(query) > 1):
                 if query.startswith('\\'):
@@ -372,57 +366,41 @@ class ResultCache(SearchQueryParser):
                 elif query.startswith('~'):
                     matchkind = REGEXP_MATCH
                     query = query[1:]
-            if matchkind != REGEXP_MATCH: ### leave case in regexps because it can be significant e.g. \S \W \D
+            if matchkind != REGEXP_MATCH:
+                # leave case in regexps because it can be significant e.g. \S \W \D
                 query = query.lower()
 
             if not isinstance(query, unicode):
                 query = query.decode('utf-8')
-            if location in ('tag', 'author', 'format', 'comment'):
-                location += 's'
 
-            MAP = {}
+            db_col = {}
-            # Fields not used when matching against text contents. These are
+            exclude_fields = [] # fields to not check when matching against text.
-            # the non-text fields
+            col_datatype = []
-            EXCLUDE_FIELDS = []
+            is_multiple_cols = {}
-
-            # get the db columns for the standard searchables
-            for x in self.tag_browser_categories:
-                if len(self.tag_browser_categories[x]['search_labels']) and \
-                         not self.tag_browser_categories.is_custom_field(x):
-                    MAP[x] = self.tag_browser_categories[x]['rec_index']
-                    if self.tag_browser_categories[x]['datatype'] != 'text':
-                        EXCLUDE_FIELDS.append(MAP[x])
-
-            # add custom columns to MAP. Put the column's type into IS_CUSTOM
-            IS_CUSTOM = []
             for x in range(len(self.FIELD_MAP)):
-                IS_CUSTOM.append('')
+                col_datatype.append('')
-            # normal and custom ratings columns use the same code
+            for x in self.field_metadata:
-            IS_CUSTOM[self.FIELD_MAP['rating']] = 'rating'
+                if len(self.field_metadata[x]['search_terms']):
-            for x in self.tag_browser_categories.get_custom_fields():
+                    db_col[x] = self.field_metadata[x]['rec_index']
-                if self.tag_browser_categories[x]['datatype'] != "datetime":
+                    if self.field_metadata[x]['datatype'] not in ['text', 'comments']:
-                    MAP[x] = self.FIELD_MAP[self.tag_browser_categories[x]['colnum']]
+                        exclude_fields.append(db_col[x])
-                    IS_CUSTOM[MAP[x]] = self.tag_browser_categories[x]['datatype']
+                    col_datatype[db_col[x]] = self.field_metadata[x]['datatype']
-
+                    is_multiple_cols[db_col[x]] = self.field_metadata[x]['is_multiple']
-            SPLITABLE_FIELDS = [MAP['authors'], MAP['tags'], MAP['formats']]
-            for x in self.tag_browser_categories.get_custom_fields():
-                if self.tag_browser_categories[x]['is_multiple']:
-                    SPLITABLE_FIELDS.append(MAP[x])
 
             try:
                 rating_query = int(query) * 2
             except:
                 rating_query = None
 
-            location = [location] if location != 'all' else list(MAP.keys())
+            location = [location] if location != 'all' else list(db_col.keys())
             for i, loc in enumerate(location):
-                location[i] = MAP[loc]
+                location[i] = db_col[loc]
 
             # get the tweak here so that the string lookup and compare aren't in the loop
             bools_are_tristate = tweaks['bool_custom_columns_are_tristate'] == 'yes'
 
-            for loc in location:
+            for loc in location: # location is now an array of field indices
-                if loc == MAP['authors']:
+                if loc == db_col['authors']:
                     ### DB stores authors with commas changed to bars, so change query
                     q = query.replace(',', '|');
                 else:
@@ -431,7 +409,7 @@ class ResultCache(SearchQueryParser):
                 for item in self._data:
                     if item is None: continue
 
-                    if IS_CUSTOM[loc] == 'bool': # complexity caused by the two-/three-value tweak
+                    if col_datatype[loc] == 'bool': # complexity caused by the two-/three-value tweak
                         v = item[loc]
                         if not bools_are_tristate:
                             if v is None or not v: # item is None or set to false
@@ -466,18 +444,18 @@ class ResultCache(SearchQueryParser):
                         matches.add(item[0])
                         continue
 
-                    if IS_CUSTOM[loc] == 'rating': # get here if 'all' query
+                    if col_datatype[loc] == 'rating': # get here if 'all' query
                         if rating_query and rating_query == int(item[loc]):
                             matches.add(item[0])
                         continue
 
                     try: # a conversion below might fail
-                        # relationals not supported in 'all' queries
+                        # relationals are not supported in 'all' queries
-                        if IS_CUSTOM[loc] == 'float':
+                        if col_datatype[loc] == 'float':
                             if float(query) == item[loc]:
                                 matches.add(item[0])
                             continue
-                        if IS_CUSTOM[loc] == 'int':
+                        if col_datatype[loc] == 'int':
                             if int(query) == item[loc]:
                                 matches.add(item[0])
                             continue
@@ -486,12 +464,9 @@ class ResultCache(SearchQueryParser):
                         # no further match is possible
                         continue
 
-                    if loc not in EXCLUDE_FIELDS:
+                    if loc not in exclude_fields: # time for text matching
-                        if loc in SPLITABLE_FIELDS:
+                        if is_multiple_cols[loc] is not None:
-                            if IS_CUSTOM[loc]:
+                            vals = item[loc].split(is_multiple_cols[loc])
-                                vals = item[loc].split('|')
-                            else:
-                                vals = item[loc].split(',')
                         else:
                             vals = [item[loc]] ### make into list to make _match happy
                         if _match(q, vals, matchkind):
@@ -622,9 +597,9 @@ class ResultCache(SearchQueryParser):
         elif field == 'title': field = 'sort'
         elif field == 'authors': field = 'author_sort'
         as_string = field not in ('size', 'rating', 'timestamp')
-        if field in self.custom_column_label_map:
+        if self.field_metadata[field]['is_custom']:
-            as_string = self.custom_column_label_map[field]['datatype'] in ('comments', 'text')
+            as_string = self.field_metadata[field]['datatype'] in ('comments', 'text')
-            field = self.custom_column_label_map[field]['num']
+            field = self.field_metadata[field]['colnum']
 
         if self.first_sort:
             subsort = True

src/calibre/library/custom_columns.py

@@ -144,14 +144,19 @@ class CustomColumns(object):
         for k in sorted(self.custom_column_label_map.keys()):
             v = self.custom_column_label_map[k]
             if v['normalized']:
-                searchable = True
+                is_category = True
             else:
-                searchable = False
+                is_category = False
+            if v['is_multiple']:
+                is_m = '|'
+            else:
+                is_m = None
             tn = 'custom_column_{0}'.format(v['num'])
-            self.tag_browser_categories.add_custom_field(label=v['label'],
+            self.field_metadata.add_custom_field(label=v['label'],
                     table=tn, column='value', datatype=v['datatype'],
-                    is_multiple=v['is_multiple'], colnum=v['num'],
+                    colnum=v['num'], name=v['name'], display=v['display'],
-                    name=v['name'], searchable=searchable)
+                    is_multiple=is_m, is_category=is_category,
+                    is_editable=v['editable'])
 
     def get_custom(self, idx, label=None, num=None, index_is_id=False):
         if label is not None:
@@ -257,7 +262,7 @@ class CustomColumns(object):
                             'SELECT id FROM %s WHERE value=?'%table, (ex,), all=False)
                     if ex != x:
                         self.conn.execute(
-                                'UPDATE %s SET value=? WHERE id=?', (x, xid))
+                                'UPDATE %s SET value=? WHERE id=?'%table, (x, xid))
                 else:
                     xid = self.conn.execute(
                             'INSERT INTO %s(value) VALUES(?)'%table, (x,)).lastrowid

src/calibre/library/database2.py

@@ -116,7 +116,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         self.books_list_filter = self.conn.create_dynamic_filter('books_list_filter')
 
     def __init__(self, library_path, row_factory=False):
-        self.tag_browser_categories = FieldMetadata() #.get_tag_browser_categories()
+        self.field_metadata = FieldMetadata()
         if not os.path.exists(library_path):
             os.makedirs(library_path)
         self.listeners = set([])
@@ -206,20 +206,20 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
              'lccn':16, 'pubdate':17, 'flags':18, 'uuid':19}
 
         for k,v in self.FIELD_MAP.iteritems():
-            self.tag_browser_categories.set_field_record_index(k, v, prefer_custom=False)
+            self.field_metadata.set_field_record_index(k, v, prefer_custom=False)
 
         base = max(self.FIELD_MAP.values())
         for col in custom_cols:
             self.FIELD_MAP[col] = base = base+1
-            self.tag_browser_categories.set_field_record_index(
+            self.field_metadata.set_field_record_index(
                                         self.custom_column_num_map[col]['label'],
                                         base,
                                         prefer_custom=True)
 
         self.FIELD_MAP['cover'] = base+1
-        self.tag_browser_categories.set_field_record_index('cover', base+1, prefer_custom=False)
+        self.field_metadata.set_field_record_index('cover', base+1, prefer_custom=False)
         self.FIELD_MAP['ondevice'] = base+2
-        self.tag_browser_categories.set_field_record_index('ondevice', base+2, prefer_custom=False)
+        self.field_metadata.set_field_record_index('ondevice', base+2, prefer_custom=False)
 
         script = '''
         DROP VIEW IF EXISTS meta2;
@@ -231,9 +231,20 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         self.conn.executescript(script)
         self.conn.commit()
 
+        # Reconstruct the user categories, putting them into field_metadata
+        # Assumption is that someone else will fix them if they change.
+        tb_cats = self.field_metadata
+        for k in tb_cats.keys():
+            if tb_cats[k]['kind'] in ['user', 'search']:
+                del tb_cats[k]
+        for user_cat in sorted(prefs['user_categories'].keys()):
+            cat_name = user_cat+':' # add the ':' to avoid name collision
+            tb_cats.add_user_category(label=cat_name, name=user_cat)
+        if len(saved_searches.names()):
+            tb_cats.add_search_category(label='search', name=_('Searches'))
+
         self.book_on_device_func = None
-        self.data    = ResultCache(self.FIELD_MAP, self.custom_column_label_map,
+        self.data    = ResultCache(self.FIELD_MAP, self.field_metadata)
-                                   self.tag_browser_categories)
         self.search  = self.data.search
         self.refresh = functools.partial(self.data.refresh, self)
         self.sort    = self.data.sort
@@ -646,9 +657,6 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
     def get_recipe(self, id):
         return self.conn.get('SELECT script FROM feeds WHERE id=?', (id,), all=False)
 
-    def get_tag_browser_categories(self):
-        return self.tag_browser_categories
-
     def get_categories(self, sort_on_count=False, ids=None, icon_map=None):
         self.books_list_filter.change([] if not ids else ids)
 
@@ -656,11 +664,11 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         if icon_map is not None and type(icon_map) != TagsIcons:
             raise TypeError('icon_map passed to get_categories must be of type TagIcons')
 
+        tb_cats = self.field_metadata
         #### First, build the standard and custom-column categories ####
-        tb_cats = self.tag_browser_categories
         for category in tb_cats.keys():
             cat = tb_cats[category]
-            if cat['kind'] == 'not_cat':
+            if not cat['is_category'] or cat['kind'] in ['user', 'search']:
                 continue
             tn = cat['table']
             categories[category] = []   #reserve the position in the ordered list
@@ -680,7 +688,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
             # icon_map is not None if get_categories is to store an icon and
             # possibly a tooltip in the tag structure.
             icon, tooltip = None, ''
-            label = tb_cats.get_field_label(category)
+            label = tb_cats.key_to_label(category)
             if icon_map:
                 if not tb_cats.is_custom_field(category):
                     if category in icon_map:
@@ -737,12 +745,6 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         #### Now do the user-defined categories. ####
         user_categories = prefs['user_categories']
 
-        # remove all user categories from tag_browser_categories. They can
-        # easily come and go. We will add all the existing ones in below.
-        for k in tb_cats.keys():
-            if tb_cats[k]['kind'] in ['user', 'search']:
-                del tb_cats[k]
-
         # We want to use same node in the user category as in the source
         # category. To do that, we need to find the original Tag node. There is
         # a time/space tradeoff here. By converting the tags into a map, we can
@@ -760,7 +762,6 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                 # else: do nothing, to not include nodes w zero counts
             if len(items):
                 cat_name = user_cat+':' # add the ':' to avoid name collision
-                tb_cats.add_user_category(label=cat_name, name=user_cat)
                 # Not a problem if we accumulate entries in the icon map
                 if icon_map is not None:
                     icon_map[cat_name] = icon_map[':user']
@@ -779,7 +780,6 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         for srch in saved_searches.names():
             items.append(Tag(srch, tooltip=saved_searches.lookup(srch), icon=icon))
         if len(items):
-            tb_cats.add_search_category(label='search', name=_('Searches'))
             if icon_map is not None:
                 icon_map['search'] = icon_map['search']
             categories['search'] = items

src/calibre/library/field_metadata.py

@@ -4,7 +4,6 @@ Created on 25 May 2010
 @author: charles
 '''
 
-from UserDict import DictMixin
 from calibre.utils.ordered_dict import OrderedDict
 
 class TagsIcons(dict):
@@ -22,105 +21,259 @@ class TagsIcons(dict):
                 raise ValueError('Missing category icon [%s]'%a)
             self[a] = icon_dict[a]
 
-class FieldMetadata(dict, DictMixin):
+class FieldMetadata(dict):
+    '''
+    key: the key to the dictionary is:
+    - for standard fields, the metadata field name.
+    - for custom fields, the metadata field name prefixed by '#'
+    This is done to create two 'namespaces' so the names don't clash
 
-    # kind == standard: is tag category. May be a search label. Is db col
+    label: the actual column label. No prefixing.
-    #                   or is specially handled (e.g., news)
-    # kind == not_cat: Is not a tag category. May be a search label. Is db col
-    # kind == user: user-defined tag category
-    # kind == search: saved-searches category
-    # For 'standard', the order below is the order that the categories will
-    # appear in the tags pane.
-    #
-    # label is the column label. key is either the label or in the case of
-    # custom fields, the label prefixed with 'x'. Because of the prefixing,
-    # there cannot be a name clash between standard and custom fields, so key
-    # can be used as the metadata dictionary key.
 
-    category_items_ = [
+    datatype: the type of the information in the field. Valid values are float,
-            ('authors',   {'table':'authors', 'column':'name',
+    int, rating, bool, comments, datetime, text.
-                           'datatype':'text', 'is_multiple':False,
+    is_multiple: valid for the text datatype. If None, the field is to be
-                           'kind':'standard', 'name':_('Authors'),
+    treated as a single term. If not None, it contains a string, and the field
-                           'search_labels':['authors', 'author'],
+    is assumed to contain a list of terms separated by that string
-                           'is_custom':False}),
+
-            ('series',    {'table':'series', 'column':'name',
+    kind == standard: is a db field.
-                           'datatype':'text', 'is_multiple':False,
+    kind == category: standard tag category that isn't a field. see news.
-                           'kind':'standard', 'name':_('Series'),
+    kind == user: user-defined tag category.
-                           'search_labels':['series'],
+    kind == search: saved-searches category.
-                           'is_custom':False}),
+
-            ('formats',   {'table':None, 'column':None,
+    is_category: is a tag browser category. If true, then:
-                           'datatype':'text', 'is_multiple':False, # must think what type this is!
+       table: name of the db table used to construct item list
-                           'kind':'standard', 'name':_('Formats'),
+       column: name of the column in the normalized table to join on
-                           'search_labels':['formats', 'format'],
+       link_column: name of the column in the connection table to join on
-                           'is_custom':False}),
+       If these are None, then the category constructor must know how
-            ('publisher', {'table':'publishers', 'column':'name',
+       to build the item list (e.g., formats).
-                           'datatype':'text', 'is_multiple':False,
+       The order below is the order that the categories will
-                           'kind':'standard', 'name':_('Publishers'),
+       appear in the tags pane.
-                           'search_labels':['publisher'],
+
-                           'is_custom':False}),
+    name: the text that is to be used when displaying the field. Column headings
-            ('rating',    {'table':'ratings', 'column':'rating',
+    in the GUI, etc.
-                           'datatype':'rating', 'is_multiple':False,
+
-                           'kind':'standard', 'name':_('Ratings'),
+    search_terms: the terms that can be used to identify the field when
-                           'search_labels':['rating'],
+    searching. They can be thought of as aliases for metadata keys, but are only
-                           'is_custom':False}),
+    valid when passed to search().
-            ('news',      {'table':'news', 'column':'name',
+
-                           'datatype':None, 'is_multiple':False,
+    is_custom: the field has been added by the user.
-                           'kind':'standard', 'name':_('News'),
+
-                           'search_labels':[],
+    rec_index: the index of the field in the db metadata record.
-                           'is_custom':False}),
+
-            ('tags',      {'table':'tags', 'column':'name',
+    '''
-                           'datatype':'text', 'is_multiple':True,
+    _field_metadata = [
-                           'kind':'standard', 'name':_('Tags'),
+            ('authors',   {'table':'authors',
-                           'search_labels':['tags', 'tag'],
+                           'column':'name',
-                           'is_custom':False}),
+                           'link_column':'author',
-            ('author_sort',{'table':None, 'column':None, 'datatype':'text',
+                           'datatype':'text',
-                           'is_multiple':False, 'kind':'not_cat', 'name':None,
+                           'is_multiple':',',
-                           'search_labels':[], 'is_custom':False}),
+                           'kind':'field',
-            ('comments',  {'table':None, 'column':None, 'datatype':'text',
+                           'name':_('Authors'),
-                           'is_multiple':False, 'kind':'not_cat', 'name':None,
+                           'search_terms':['authors', 'author'],
-                           'search_labels':['comments', 'comment'], 'is_custom':False}),
+                           'is_custom':False,
-            ('cover',     {'table':None, 'column':None, 'datatype':None,
+                           'is_category':True}),
-                           'is_multiple':False, 'kind':'not_cat', 'name':None,
+            ('series',    {'table':'series',
-                           'search_labels':['cover'], 'is_custom':False}),
+                           'column':'name',
-            ('flags',     {'table':None, 'column':None, 'datatype':'text',
+                           'link_column':'series',
-                           'is_multiple':False, 'kind':'not_cat', 'name':None,
+                           'datatype':'text',
-                           'search_labels':[], 'is_custom':False}),
+                           'is_multiple':None,
-            ('id',        {'table':None, 'column':None, 'datatype':'int',
+                           'kind':'field',
-                           'is_multiple':False, 'kind':'not_cat', 'name':None,
+                           'name':_('Series'),
-                           'search_labels':[], 'is_custom':False}),
+                           'search_terms':['series'],
-            ('isbn',      {'table':None, 'column':None, 'datatype':'text',
+                           'is_custom':False,
-                           'is_multiple':False, 'kind':'not_cat', 'name':None,
+                           'is_category':True}),
-                           'search_labels':['isbn'], 'is_custom':False}),
+            ('formats',   {'table':None,
-            ('lccn',      {'table':None, 'column':None, 'datatype':'text',
+                           'column':None,
-                           'is_multiple':False, 'kind':'not_cat', 'name':None,
+                           'datatype':'text',
-                           'search_labels':[], 'is_custom':False}),
+                           'is_multiple':',',
-            ('ondevice',  {'table':None, 'column':None, 'datatype':'bool',
+                           'kind':'field',
-                           'is_multiple':False, 'kind':'not_cat', 'name':None,
+                           'name':_('Formats'),
-                           'search_labels':[], 'is_custom':False}),
+                           'search_terms':['formats', 'format'],
-            ('path',      {'table':None, 'column':None, 'datatype':'text',
+                           'is_custom':False,
-                           'is_multiple':False, 'kind':'not_cat', 'name':None,
+                           'is_category':True}),
-                           'search_labels':[], 'is_custom':False}),
+            ('publisher', {'table':'publishers',
-            ('pubdate',   {'table':None, 'column':None, 'datatype':'datetime',
+                           'column':'name',
-                           'is_multiple':False, 'kind':'not_cat', 'name':None,
+                           'link_column':'publisher',
-                           'search_labels':['pubdate'], 'is_custom':False}),
+                           'datatype':'text',
-            ('series_index',{'table':None, 'column':None, 'datatype':'float',
+                           'is_multiple':None,
-                           'is_multiple':False, 'kind':'not_cat', 'name':None,
+                           'kind':'field',
-                           'search_labels':[], 'is_custom':False}),
+                           'name':_('Publishers'),
-            ('sort',      {'table':None, 'column':None, 'datatype':'text',
+                           'search_terms':['publisher'],
-                           'is_multiple':False, 'kind':'not_cat', 'name':None,
+                           'is_custom':False,
-                           'search_labels':[], 'is_custom':False}),
+                           'is_category':True}),
-            ('size',      {'table':None, 'column':None, 'datatype':'float',
+            ('rating',    {'table':'ratings',
-                           'is_multiple':False, 'kind':'not_cat', 'name':None,
+                           'column':'rating',
-                           'search_labels':[], 'is_custom':False}),
+                           'link_column':'rating',
-            ('timestamp', {'table':None, 'column':None, 'datatype':'datetime',
+                           'datatype':'rating',
-                           'is_multiple':False, 'kind':'not_cat', 'name':None,
+                           'is_multiple':None,
-                           'search_labels':['date'], 'is_custom':False}),
+                           'kind':'field',
-            ('title',     {'table':None, 'column':None, 'datatype':'text',
+                           'name':_('Ratings'),
-                           'is_multiple':False, 'kind':'not_cat', 'name':None,
+                           'search_terms':['rating'],
-                           'search_labels':['title'], 'is_custom':False}),
+                           'is_custom':False,
-            ('uuid',      {'table':None, 'column':None, 'datatype':'text',
+                           'is_category':True}),
-                           'is_multiple':False, 'kind':'not_cat', 'name':None,
+            ('news',      {'table':'news',
-                           'search_labels':[], 'is_custom':False}),
+                           'column':'name',
+                           'datatype':None,
+                           'is_multiple':None,
+                           'kind':'category',
+                           'name':_('News'),
+                           'search_terms':[],
+                           'is_custom':False,
+                           'is_category':True}),
+            ('tags',      {'table':'tags',
+                           'column':'name',
+                           'link_column': 'tag',
+                           'datatype':'text',
+                           'is_multiple':',',
+                           'kind':'field',
+                           'name':_('Tags'),
+                           'search_terms':['tags', 'tag'],
+                           'is_custom':False,
+                           'is_category':True}),
+            ('author_sort',{'table':None,
+                            'column':None,
+                            'datatype':'text',
+                           'is_multiple':None,
+                           'kind':'field',
+                           'name':None,
+                           'search_terms':[],
+                           'is_custom':False,
+                           'is_category':False}),
+            ('comments',  {'table':None,
+                           'column':None,
+                           'datatype':'text',
+                           'is_multiple':None,
+                           'kind':'field',
+                           'name':None,
+                           'search_terms':['comments', 'comment'],
+                           'is_custom':False, 'is_category':False}),
+            ('cover',     {'table':None,
+                           'column':None,
+                           'datatype':None,
+                           'is_multiple':None,
+                           'kind':'field',
+                           'name':None,
+                           'search_terms':['cover'],
+                           'is_custom':False,
+                           'is_category':False}),
+            ('flags',     {'table':None,
+                           'column':None,
+                           'datatype':'text',
+                           'is_multiple':None,
+                           'kind':'field',
+                           'name':None,
+                           'search_terms':[],
+                           'is_custom':False,
+                           'is_category':False}),
+            ('id',        {'table':None,
+                           'column':None,
+                           'datatype':'int',
+                           'is_multiple':None,
+                           'kind':'field',
+                           'name':None,
+                           'search_terms':[],
+                           'is_custom':False,
+                           'is_category':False}),
+            ('isbn',      {'table':None,
+                           'column':None,
+                           'datatype':'text',
+                           'is_multiple':None,
+                           'kind':'field',
+                           'name':None,
+                           'search_terms':['isbn'],
+                           'is_custom':False,
+                           'is_category':False}),
+            ('lccn',      {'table':None,
+                           'column':None,
+                           'datatype':'text',
+                           'is_multiple':None,
+                           'kind':'field',
+                           'name':None,
+                           'search_terms':[],
+                           'is_custom':False,
+                           'is_category':False}),
+            ('ondevice',  {'table':None,
+                           'column':None,
+                           'datatype':'bool',
+                           'is_multiple':None,
+                           'kind':'field',
+                           'name':None,
+                           'search_terms':[],
+                           'is_custom':False,
+                           'is_category':False}),
+            ('path',      {'table':None,
+                           'column':None,
+                           'datatype':'text',
+                           'is_multiple':None,
+                           'kind':'field',
+                           'name':None,
+                           'search_terms':[],
+                           'is_custom':False,
+                           'is_category':False}),
+            ('pubdate',   {'table':None,
+                           'column':None,
+                           'datatype':'datetime',
+                           'is_multiple':None,
+                           'kind':'field',
+                           'name':None,
+                           'search_terms':['pubdate'],
+                           'is_custom':False,
+                           'is_category':False}),
+            ('series_index',{'table':None,
+                             'column':None,
+                             'datatype':'float',
+                             'is_multiple':None,
+                             'kind':'field',
+                             'name':None,
+                             'search_terms':[],
+                             'is_custom':False,
+                             'is_category':False}),
+            ('sort',      {'table':None,
+                           'column':None,
+                           'datatype':'text',
+                           'is_multiple':None,
+                           'kind':'field',
+                           'name':None,
+                           'search_terms':[],
+                           'is_custom':False,
+                           'is_category':False}),
+            ('size',      {'table':None,
+                           'column':None,
+                           'datatype':'float',
+                           'is_multiple':None,
+                           'kind':'field',
+                           'name':None,
+                           'search_terms':[],
+                           'is_custom':False,
+                           'is_category':False}),
+            ('timestamp', {'table':None,
+                           'column':None,
+                           'datatype':'datetime',
+                           'is_multiple':None,
+                           'kind':'field',
+                           'name':None,
+                           'search_terms':['date'],
+                           'is_custom':False,
+                           'is_category':False}),
+            ('title',     {'table':None,
+                           'column':None,
+                           'datatype':'text',
+                           'is_multiple':None,
+                           'kind':'field',
+                           'name':None,
+                           'search_terms':['title'],
+                           'is_custom':False,
+                           'is_category':False}),
+            ('uuid',      {'table':None,
+                           'column':None,
+                           'datatype':'text',
+                           'is_multiple':None,
+                           'kind':'field',
+                           'name':None,
+                           'search_terms':[],
+                           'is_custom':False,
+                           'is_category':False}),
             ]
 
     # search labels that are not db columns
@@ -131,10 +284,15 @@ class FieldMetadata(dict, DictMixin):
 
     def __init__(self):
         self._tb_cats = OrderedDict()
-        for k,v in self.category_items_:
+        self._search_term_map = {}
+        self.custom_label_to_key_map = {}
+        for k,v in self._field_metadata:
             self._tb_cats[k] = v
+            self._tb_cats[k]['label'] = k
+            self._tb_cats[k]['display'] = {}
+            self._tb_cats[k]['is_editable'] = True
+            self._add_search_terms_to_map(k, self._tb_cats[k]['search_terms'])
         self.custom_field_prefix = '#'
-
         self.get = self._tb_cats.get
 
     def __getitem__(self, key):
@@ -150,6 +308,12 @@ class FieldMetadata(dict, DictMixin):
         for key in self._tb_cats:
             yield key
 
+    def __contains__(self, key):
+        return self.has_key(key)
+
+    def has_key(self, key):
+        return key in self._tb_cats
+
     def keys(self):
         return self._tb_cats.keys()
 
@@ -157,44 +321,80 @@ class FieldMetadata(dict, DictMixin):
         for key in self._tb_cats:
             yield key
 
+    def itervalues(self):
+        return self._tb_cats.itervalues()
+
+    def values(self):
+        return self._tb_cats.values()
+
     def iteritems(self):
         for key in self._tb_cats:
             yield (key, self._tb_cats[key])
 
+    def items(self):
+        return list(self.iteritems())
+
     def is_custom_field(self, key):
         return key.startswith(self.custom_field_prefix)
 
-    def get_field_label(self, key):
+    def key_to_label(self, key):
         if 'label' not in self._tb_cats[key]:
             return key
         return self._tb_cats[key]['label']
 
-    def get_search_label(self, label):
+    def label_to_key(self, label, prefer_custom=False):
+        if prefer_custom:
+            if label in self.custom_label_to_key_map:
+                return self.custom_label_to_key_map[label]
         if 'label' in self._tb_cats:
             return label
-        if self.is_custom_field(label):
+        if not prefer_custom:
-            return self.custom_field_prefix+label
+            if label in self.custom_label_to_key_map:
+                return self.custom_label_to_key_map[label]
         raise ValueError('Unknown key [%s]'%(label))
 
     def get_custom_fields(self):
         return [l for l in self._tb_cats if self._tb_cats[l]['is_custom']]
 
-    def add_custom_field(self, label, table, column, datatype,
+    def get_custom_field_metadata(self):
-                               is_multiple, colnum, name, searchable):
+        l = {}
-        fn = self.custom_field_prefix + label
+        for k in self._tb_cats:
-        if fn in self._tb_cats:
+            if self._tb_cats[k]['is_custom']:
+                l[k] = self._tb_cats[k]
+        return l
+
+    def add_custom_field(self, label, table, column, datatype, colnum, name,
+                               display, is_editable, is_multiple, is_category):
+        key = self.custom_field_prefix + label
+        if key in self._tb_cats:
             raise ValueError('Duplicate custom field [%s]'%(label))
-        if searchable:
+        self._tb_cats[key] = {'table':table,       'column':column,
-            sl = [fn]
-            kind = 'standard'
-        else:
-            sl = []
-            kind = 'not_cat'
-        self._tb_cats[fn] = {'table':table,       'column':column,
                              'datatype':datatype,  'is_multiple':is_multiple,
-                             'kind':kind,         'name':name,
+                             'kind':'field',       'name':name,
-                             'search_labels':sl,  'label':label,
+                             'search_terms':[key], 'label':label,
-                             'colnum':colnum,     'is_custom':True}
+                             'colnum':colnum,      'display':display,
+                             'is_custom':True,     'is_category':is_category,
+                             'is_editable': is_editable,}
+        self._add_search_terms_to_map(key, [key])
+        self.custom_label_to_key_map[label] = key
+
+    def add_user_category(self, label, name):
+        if label in self._tb_cats:
+            raise ValueError('Duplicate user field [%s]'%(label))
+        self._tb_cats[label] = {'table':None,        'column':None,
+                                'datatype':None,     'is_multiple':None,
+                                'kind':'user',       'name':name,
+                                'search_terms':[],    'is_custom':False,
+                                'is_category':True}
+
+    def add_search_category(self, label, name):
+        if label in self._tb_cats:
+            raise ValueError('Duplicate user field [%s]'%(label))
+        self._tb_cats[label] = {'table':None,        'column':None,
+                                'datatype':None,     'is_multiple':None,
+                                'kind':'search',     'name':name,
+                                'search_terms':[],    'is_custom':False,
+                                'is_category':True}
 
     def set_field_record_index(self, label, index, prefer_custom=False):
         if prefer_custom:
@@ -208,21 +408,6 @@ class FieldMetadata(dict, DictMixin):
                 key = self.custom_field_prefix+label
         self._tb_cats[key]['rec_index'] = index  # let the exception fly ...
 
-    def add_user_category(self, label, name):
-        if label in self._tb_cats:
-            raise ValueError('Duplicate user field [%s]'%(label))
-        self._tb_cats[label] = {'table':None,        'column':None,
-                                'datatype':None,     'is_multiple':False,
-                                'kind':'user',       'name':name,
-                                'search_labels':[],  'is_custom':False}
-
-    def add_search_category(self, label, name):
-        if label in self._tb_cats:
-            raise ValueError('Duplicate user field [%s]'%(label))
-        self._tb_cats[label] = {'table':None,        'column':None,
-                                'datatype':None,     'is_multiple':False,
-                                'kind':'search',     'name':name,
-                                'search_labels':[],  'is_custom':False}
 
 #    DEFAULT_LOCATIONS = frozenset([
 #        'all',
@@ -246,14 +431,23 @@ class FieldMetadata(dict, DictMixin):
 #        'title',
 #                 ])
 
-
+    def get_search_terms(self):
-    def get_search_labels(self):
+        s_keys = []
-        s_labels = []
         for v in self._tb_cats.itervalues():
-            map((lambda x:s_labels.append(x)), v['search_labels'])
+            map((lambda x:s_keys.append(x)), v['search_terms'])
         for v in self.search_items:
-            s_labels.append(v)
+            s_keys.append(v)
-#        if set(s_labels) != self.DEFAULT_LOCATIONS:
+#        if set(s_keys) != self.DEFAULT_LOCATIONS:
 #            print 'search labels and default_locations do not match:'
-#            print set(s_labels) ^ self.DEFAULT_LOCATIONS
+#            print set(s_keys) ^ self.DEFAULT_LOCATIONS
-        return s_labels
+        return s_keys
+
+    def _add_search_terms_to_map(self, key, terms):
+        if terms is not None:
+            for t in terms:
+                self._search_term_map[t] = key
+
+    def search_term_to_key(self, term):
+        if term in self._search_term_map:
+            return  self._search_term_map[term]
+        return term

src/calibre/library/schema_upgrades.py

@@ -289,6 +289,6 @@ class SchemaUpgrade(object):
                 '''.format(tn=table_name, cn=column_name, vcn=view_column_name))
             self.conn.executescript(script)
 
-        for tn, cn in self.tag_browser_categories.items():
+        for field in self.field_metadata.itervalues():
-            if tn != 'news':
+            if field['is_category'] and not field['is_custom'] and 'link_column' in field:
-                create_tag_browser_view(tn, cn[0], cn[1])
+                create_tag_browser_view(field['table'], field['link_column'], field['column'])

src/calibre/library/server/opds.py

@@ -322,16 +322,24 @@ class OPDSServer(object):
             return self.get_opds_all_books(which, page_url, up_url,
                     version=version, offset=offset)
         elif type_ == 'N':
-            return self.get_opds_navcatalog(which)
+            return self.get_opds_navcatalog(which, version=version, offset=offset)
         raise cherrypy.HTTPError(404, 'Not found')
 
+    def get_opds_navcatalog(self, which, version=0, offset=0):
+        categories = self.categories_cache(
+                self.get_opds_allowed_ids_for_version(version))
+        if which not in categories:
+            raise cherrypy.HTTPError(404, 'Category %r not found'%which)
+
+
+
     def opds(self, version=0):
         version = int(version)
         if version not in BASE_HREFS:
             raise cherrypy.HTTPError(404, 'Not found')
         categories = self.categories_cache(
                 self.get_opds_allowed_ids_for_version(version))
-        category_meta = self.db.get_tag_browser_categories()
+        category_meta = self.db.field_metadata
         cats = [
                 (_('Newest'), _('Date'), 'Onewest'),
                 (_('Title'), _('Title'), 'Otitle'),