1) Move all tag category code to DB2.

2) Fix bug where opening preferences resets the folder device menus even when connected

src/calibre/ebooks/metadata/book/__init__.py

@@ -88,17 +88,25 @@ CALIBRE_METADATA_FIELDS = frozenset([
     ]
 )
 
+CALIBRE_RESERVED_LABELS = frozenset([
+    # reserved for saved searches
+    'search',
+    ]
+)
+
 RESERVED_METADATA_FIELDS = SOCIAL_METADATA_FIELDS.union(
                            PUBLICATION_METADATA_FIELDS).union(
                            BOOK_STRUCTURE_FIELDS).union(
                            USER_METADATA_FIELDS).union(
                            DEVICE_METADATA_FIELDS).union(
-                           CALIBRE_METADATA_FIELDS)
+                           CALIBRE_METADATA_FIELDS).union(
+                           CALIBRE_RESERVED_LABELS)
 
 assert len(RESERVED_METADATA_FIELDS) == sum(map(len, (
     SOCIAL_METADATA_FIELDS, PUBLICATION_METADATA_FIELDS,
     BOOK_STRUCTURE_FIELDS, USER_METADATA_FIELDS,
     DEVICE_METADATA_FIELDS, CALIBRE_METADATA_FIELDS,
+    CALIBRE_RESERVED_LABELS
     )))
 
 SERIALIZABLE_FIELDS = SOCIAL_METADATA_FIELDS.union(

src/calibre/gui2/__init__.py

@@ -97,8 +97,6 @@ def _config():
             help=_('Overwrite author and title with new metadata'))
     c.add_opt('enforce_cpu_limit', default=True,
             help=_('Limit max simultaneous jobs to number of CPUs'))
-    c.add_opt('user_categories', default={},
-            help=_('User-created tag browser categories'))
 
     return ConfigProxy(c)
 

src/calibre/gui2/dialogs/tag_categories.py

@@ -7,7 +7,7 @@ from PyQt4.QtCore import SIGNAL, Qt
 from PyQt4.QtGui import QDialog, QIcon, QListWidgetItem
 
 from calibre.gui2.dialogs.tag_categories_ui import Ui_TagCategories
-from calibre.gui2 import config
+from calibre.utils.config import prefs
 from calibre.gui2.dialogs.confirm_delete import confirm
 from calibre.constants import islinux
 
@@ -22,7 +22,7 @@ class Item:
         return 'name=%s, label=%s, index=%s, exists='%(self.name, self.label, self.index, self.exists)
 
 class TagCategories(QDialog, Ui_TagCategories):
-    category_labels_orig =   ['', 'author', 'series', 'publisher', 'tag']
+    category_labels_orig =   ['', 'authors', 'series', 'publishers', 'tags']
 
     def __init__(self, window, db, index=None):
         QDialog.__init__(self, window)
@@ -64,7 +64,7 @@ class TagCategories(QDialog, Ui_TagCategories):
                 self.all_items.append(t)
                 self.all_items_dict[label+':'+n] = t
 
-        self.categories = dict.copy(config['user_categories'])
+        self.categories = dict.copy(prefs['user_categories'])
         if self.categories is None:
             self.categories = {}
         for cat in self.categories:
@@ -181,7 +181,7 @@ class TagCategories(QDialog, Ui_TagCategories):
 
     def accept(self):
         self.save_category()
-        config['user_categories'] = self.categories
+        prefs['user_categories'] = self.categories
         QDialog.accept(self)
 
     def save_category(self):

src/calibre/gui2/tag_view.py

@@ -201,29 +201,34 @@ class TagsModel(QAbstractItemModel): # {{{
                        _('Ratings'), _('News'), _('Tags')]
     row_map_orig    = ['authors', 'series', 'formats', 'publishers', 'ratings',
                        'news', 'tags']
-    tags_categories_start= 7
     search_keys=['search', _('Searches')]
 
+
     def __init__(self, db, parent=None):
         QAbstractItemModel.__init__(self, parent)
-        self.cat_icon_map_orig = list(map(QIcon, [I('user_profile.svg'),
+
-                I('series.svg'), I('book.svg'), I('publisher.png'), I('star.png'),
+        # must do this here because 'QPixmap: Must construct a QApplication
-                I('news.svg'), I('tags.svg')]))
+        # before a QPaintDevice'
+        self.category_icon_map = {'authors': QIcon(I('user_profile.svg')),
+                    'series': QIcon(I('series.svg')),
+                    'formats':QIcon(I('book.svg')),
+                    'publishers': QIcon(I('publisher.png')),
+                    'ratings':QIcon(I('star.png')),
+                    'news':QIcon(I('news.svg')),
+                    'tags':QIcon(I('tags.svg')),
+                    '*custom':QIcon(I('column.svg')),
+                    '*user':QIcon(I('drawer.svg')),
+                    'search':QIcon(I('search.svg'))}
         self.icon_state_map = [None, QIcon(I('plus.svg')), QIcon(I('minus.svg'))]
-        self.custcol_icon = QIcon(I('column.svg'))
-        self.search_icon = QIcon(I('search.svg'))
-        self.usercat_icon = QIcon(I('drawer.svg'))
-        self.label_to_icon_map = dict(map(None, self.row_map_orig, self.cat_icon_map_orig))
-        self.label_to_icon_map['*custom'] = self.custcol_icon
         self.db = db
         self.search_restriction = ''
-        self.user_categories = {}
         self.ignore_next_search = 0
         data = self.get_node_tree(config['sort_by_popularity'])
         self.root_item = TagTreeItem()
         for i, r in enumerate(self.row_map):
             c = TagTreeItem(parent=self.root_item,
-                    data=self.categories[i], category_icon=self.cat_icon_map[i])
+                    data=self.categories[i],
+                    category_icon=self.category_icon_map[r])
             for tag in data[r]:
                 TagTreeItem(parent=c, data=tag, icon_map=self.icon_state_map)
 
@@ -233,66 +238,19 @@ class TagsModel(QAbstractItemModel): # {{{
     def get_node_tree(self, sort):
         self.row_map = []
         self.categories = []
-        # strip the icons after the 'standard' categories. We will put them back later
-        if self.tags_categories_start < len(self.row_map_orig):
-            self.cat_icon_map = self.cat_icon_map_orig[:self.tags_categories_start-len(self.row_map_orig)]
-        else:
-            self.cat_icon_map = self.cat_icon_map_orig[:]
 
-        self.user_categories = dict.copy(config['user_categories'])
-        column_map = config['column_map']
-
-        for i in range(0, self.tags_categories_start): # First the standard categories
-            self.row_map.append(self.row_map_orig[i])
-            self.categories.append(self.categories_orig[i])
         if len(self.search_restriction):
-            data = self.db.get_categories(sort_on_count=sort, icon_map=self.label_to_icon_map,
+            data = self.db.get_categories(sort_on_count=sort, icon_map=self.category_icon_map,
                         ids=self.db.search(self.search_restriction, return_matches=True))
         else:
-            data = self.db.get_categories(sort_on_count=sort, icon_map=self.label_to_icon_map)
+            data = self.db.get_categories(sort_on_count=sort, icon_map=self.category_icon_map)
 
-        for c in data:  # now the custom columns
+        tb_categories = self.db.get_tag_browser_categories()
-            if c not in self.row_map_orig and c in column_map:
+        for category in tb_categories.iterkeys():
-                self.row_map.append(c)
+            if category in data: # They should always be there, but ...
-                self.categories.append(self.db.custom_column_label_map[c]['name'])
+                self.row_map.append(category)
-                self.cat_icon_map.append(self.custcol_icon)
+                self.categories.append(tb_categories[category]['name'])
 
-        # Now the rest of the normal tag categories
-        for i in range(self.tags_categories_start, len(self.row_map_orig)):
-            self.row_map.append(self.row_map_orig[i])
-            self.categories.append(self.categories_orig[i])
-            self.cat_icon_map.append(self.cat_icon_map_orig[i])
-
-        # Clean up the author's tags, getting rid of the '|' characters
-        if data['authors'] is not None:
-            for t in data['authors']:
-                t.name = t.name.replace('|', ',')
-
-        # Now do the user-defined categories. There is a time/space tradeoff here.
-        # By converting the tags into a map, we can do the verification in the category
-        # loop much faster, at the cost of duplicating the categories lists.
-        taglist = {}
-        for c in self.row_map:
-            taglist[c] = dict(map(lambda t:(t.name, t), data[c]))
-
-        for c in self.user_categories:
-            l = []
-            for (name,label,ign) in self.user_categories[c]:
-                if label in taglist and name in taglist[label]: # use same node as the complete category
-                    l.append(taglist[label][name])
-                # else: do nothing, to eliminate nodes that have zero counts
-            if config['sort_by_popularity']:
-                data[c+'*'] = sorted(l, cmp=(lambda x, y: cmp(x.count, y.count)))
-            else:
-                data[c+'*'] = sorted(l, cmp=(lambda x, y: cmp(x.name.lower(), y.name.lower())))
-            self.row_map.append(c+'*')
-            self.categories.append(c)
-            self.cat_icon_map.append(self.usercat_icon)
-
-        data['search'] = self.get_search_nodes(self.search_icon)  # Add the search category
-        self.row_map.append(self.search_keys[0])
-        self.categories.append(self.search_keys[1])
-        self.cat_icon_map.append(self.search_icon)
         return data
 
     def get_search_nodes(self, icon):

src/calibre/gui2/ui.py

@@ -183,7 +183,7 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI):
                 _('Error communicating with device'), ' ')
         self.device_error_dialog.setModal(Qt.NonModal)
         self.tb_wrapper = textwrap.TextWrapper(width=40)
-        self.device_connected = False
+        self.device_connected = None
         self.viewers = collections.deque()
         self.content_server = None
         self.system_tray_icon = SystemTrayIcon(QIcon(I('library.png')), self)
@@ -675,6 +675,15 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI):
         self._sync_menu.fetch_annotations.connect(self.fetch_annotations)
         self._sync_menu.connect_to_folder.connect(self.connect_to_folder)
         self._sync_menu.disconnect_from_folder.connect(self.disconnect_from_folder)
+        if self.device_connected:
+            self._sync_menu.connect_to_folder_action.setEnabled(False)
+            if self.device_connected == 'folder':
+                self._sync_menu.disconnect_from_folder_action.setEnabled(True)
+            else:
+                self._sync_menu.disconnect_from_folder_action.setEnabled(False)
+        else:
+            self._sync_menu.connect_to_folder_action.setEnabled(True)
+            self._sync_menu.disconnect_from_folder_action.setEnabled(False)
 
     def add_spare_server(self, *args):
         self.spare_servers.append(Server(limit=int(config['worker_limit']/2.0)))
@@ -944,7 +953,7 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI):
             self.status_bar.showMessage(_('Device: ')+\
                 self.device_manager.device.__class__.get_gui_name()+\
                         _(' detected.'), 3000)
-            self.device_connected = True
+            self.device_connected = 'device' if not is_folder_device else 'folder'
             self._sync_menu.enable_device_actions(True,
                     self.device_manager.device.card_prefix(),
                     self.device_manager.device)
@@ -955,7 +964,7 @@ class Main(MainWindow, Ui_MainWindow, DeviceGUI):
             self._sync_menu.connect_to_folder_action.setEnabled(True)
             self._sync_menu.disconnect_from_folder_action.setEnabled(False)
             self.save_device_view_settings()
-            self.device_connected = False
+            self.device_connected = None
             self._sync_menu.enable_device_actions(False)
             self.location_view.model().update_devices()
             self.vanity.setText(self.vanity_template%\

src/calibre/library/custom_columns.py

@@ -141,11 +141,15 @@ class CustomColumns(object):
         }
 
         # Create Tag Browser categories for custom columns
-        for i, v in self.custom_column_num_map.items():
+        for k in sorted(self.custom_column_label_map.keys()):
+            v = self.custom_column_label_map[k]
             if v['normalized']:
-                tn = 'custom_column_{0}'.format(i)
+                tn = 'custom_column_{0}'.format(v['num'])
-                self.tag_browser_categories[v['label']] = {'table':tn, 'column':'value', 'type':v['datatype'], 'name':v['name']}
+                self.tag_browser_categories[v['label']] = {
-                #self.tag_browser_datatype[v['label']] = v['datatype']
+                        'table':tn, 'column':'value',
+                        'type':v['datatype'], 'is_multiple':v['is_multiple'],
+                        'kind':'custom', 'name':v['name']
+                    }
 
     def get_custom(self, idx, label=None, num=None, index_is_id=False):
         if label is not None:

src/calibre/library/database2.py

@@ -34,6 +34,8 @@ from calibre.customize.ui import run_plugins_on_import
 from calibre.utils.filenames import ascii_filename
 from calibre.utils.date import utcnow, now as nowf, utcfromtimestamp
 from calibre.utils.ordered_dict import OrderedDict
+from calibre.utils.config import prefs
+from calibre.utils.search_query_parser import saved_searches
 from calibre.ebooks import BOOK_EXTENSIONS, check_ebook_format
 
 if iswindows:
@@ -125,26 +127,32 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
             self.dbpath = self.dbpath.encode(filesystem_encoding)
 
         # Order as has been customary in the tags pane.
-        self.tag_browser_categories = OrderedDict([
+        tag_browser_categories_items = [
-                ('authors',   {'table':'authors', 'column':'name', 'type':'text', 'name':_('Authors')}),
+                ('authors',   {'table':'authors', 'column':'name',
-                ('series',    {'table':'series', 'column':'name', 'type':None, 'name':_('Series')}),
+                               'type':'text', 'is_multiple':False,
-                ('formats',   {'table':None, 'column':None, 'type':None, 'name':_('Formats')}),
+                               'kind':'standard', 'name':_('Authors')}),
-                ('publishers',{'table':'publishers', 'column':'name', 'type':'text', 'name':_('Publishers')}),
+                ('series',    {'table':'series', 'column':'name',
-                ('ratings',   {'table':'ratings', 'column':'rating', 'type':'rating', 'name':_('Ratings')}),
+                               'type':None, 'is_multiple':False,
-                ('news',      {'table':'news', 'column':'name', 'type':None, 'name':_('News')}),
+                               'kind':'standard', 'name':_('Series')}),
-                ('tags',      {'table':'tags', 'column':'name', 'type':'textmult', 'name':_('Tags')}),
+                ('formats',   {'table':None, 'column':None,
-        ])
+                               'type':None, 'is_multiple':False,
-
+                               'kind':'standard', 'name':_('Formats')}),
-#        self.tag_browser_datatype = {
+                ('publishers',{'table':'publishers', 'column':'name',
-#                'tag'       : 'textmult',
+                               'type':'text', 'is_multiple':False,
-#                'series'    : None,
+                               'kind':'standard', 'name':_('Publishers')}),
-#                'publisher' : 'text',
+                ('ratings',   {'table':'ratings', 'column':'rating',
-#                'author'    : 'text',
+                               'type':'rating', 'is_multiple':False,
-#                'news'      : None,
+                               'kind':'standard', 'name':_('Ratings')}),
-#                'rating'    : 'rating',
+                ('news',      {'table':'news', 'column':'name',
-#        }
+                               'type':None, 'is_multiple':False,
-
+                               'kind':'standard', 'name':_('News')}),
-        self.tag_browser_formatters = {'rating': lambda x:u'\u2605'*int(round(x/2.))}
+                ('tags',      {'table':'tags', 'column':'name',
+                               'type':'text', 'is_multiple':True,
+                               'kind':'standard', 'name':_('Tags')}),
+        ]
+        self.tag_browser_categories = OrderedDict()
+        for k,v in tag_browser_categories_items:
+            self.tag_browser_categories[k] = v
 
         self.connect()
         self.is_case_sensitive = not iswindows and not isosx and \
@@ -653,14 +661,19 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
     def get_recipe(self, id):
         return self.conn.get('SELECT script FROM feeds WHERE id=?', (id,), all=False)
 
+    def get_tag_browser_categories(self):
+        return self.tag_browser_categories
+
     def get_categories(self, sort_on_count=False, ids=None, icon_map=None):
         self.books_list_filter.change([] if not ids else ids)
 
         categories = {}
+
+        #### First, build the standard and custom-column categories ####
         for category in self.tag_browser_categories.keys():
             tn = self.tag_browser_categories[category]['table']
             categories[category] = []   #reserve the position in the ordered list
-            if tn is None:
+            if tn is None:              # Nothing to do for the moment
                 continue
             cn = self.tag_browser_categories[category]['column']
             if ids is None:
@@ -672,22 +685,41 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
             else:
                 query += ' ORDER BY {0} ASC'.format(cn)
             data = self.conn.get(query)
-            # category = cn[0]
+
+            # icon_map is not None if get_categories is to store an icon and
+            # possibly a tooltip in the tag structure.
             icon, tooltip = None, ''
             if icon_map:
+                if self.tag_browser_categories[category]['kind'] == 'standard':
                     if category in icon_map:
                         icon = icon_map[category]
-                else:
+                elif self.tag_browser_categories[category]['kind'] == 'custom':
                     icon = icon_map['*custom']
+                    icon_map[category] = icon_map['*custom']
                     tooltip = self.custom_column_label_map[category]['name']
+
             datatype = self.tag_browser_categories[category]['type']
-            formatter = self.tag_browser_formatters.get(datatype, lambda x: x)
+            if datatype == 'rating':
+                item_zero_func = (lambda x: len(formatter(r[1])) > 0)
+                formatter = (lambda x:u'\u2605'*int(round(x/2.)))
+            elif category == 'authors':
+                item_zero_func = (lambda x: x[2] > 0)
+                # Clean up the authors strings to human-readable form
+                formatter = (lambda x: x.replace('|', ','))
+            else:
+                item_zero_func = (lambda x: x[2] > 0)
+                formatter = (lambda x:x)
+
             categories[category] = [Tag(formatter(r[1]), count=r[2], id=r[0],
                                         icon=icon, tooltip = tooltip)
-                                    for r in data
+                                    for r in data if item_zero_func(r)]
-                                        if r[2] > 0 and
+
-                                          (datatype != 'rating' or len(formatter(r[1])) > 0)]
+        # We delayed computing the standard formats category because it does not
+        # use a view, but is computed dynamically
         categories['formats'] = []
+        icon = None
+        if icon_map and 'formats' in icon_map:
+                icon = icon_map['formats']
         for fmt in self.conn.get('SELECT DISTINCT format FROM data'):
             fmt = fmt[0]
             if ids is not None:
@@ -702,13 +734,70 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                                        WHERE format="%s"'''%fmt,
                                        all=False)
             if count > 0:
-                categories['formats'].append(Tag(fmt, count=count))
+                categories['formats'].append(Tag(fmt, count=count, icon=icon))
 
         if sort_on_count:
             categories['formats'].sort(cmp=lambda x,y:cmp(x.count, y.count),
                     reverse=True)
         else:
             categories['formats'].sort(cmp=lambda x,y:cmp(x.name, y.name))
+
+        #### Now do the user-defined categories. ####
+        user_categories = dict.copy(prefs['user_categories'])
+
+        # remove all user categories from tag_browser_categories. They can
+        # easily come and go. We will add all the existing ones in below.
+        for k in self.tag_browser_categories.keys():
+            if self.tag_browser_categories[k]['kind'] in ['user', 'search']:
+                del self.tag_browser_categories[k]
+
+        # We want to use same node in the user category as in the source
+        # category. To do that, we need to find the original Tag node. There is
+        # a time/space tradeoff here. By converting the tags into a map, we can
+        # do the verification in the category loop much faster, at the cost of
+        # temporarily duplicating the categories lists.
+        taglist = {}
+        for c in categories.keys():
+            taglist[c] = dict(map(lambda t:(t.name, t), categories[c]))
+
+        for user_cat in sorted(user_categories.keys()):
+            items = []
+            for (name,label,ign) in user_categories[user_cat]:
+                if label in taglist and name in taglist[label]:
+                    items.append(taglist[label][name])
+                # else: do nothing, to not include nodes w zero counts
+            if len(items):
+                cat_name = user_cat+'*' # add the * to avoid name collision
+                self.tag_browser_categories[cat_name] = {
+                                'table':None,  'column':None,
+                                'type':None,   'is_multiple':False,
+                                'kind':'user', 'name':user_cat}
+                # Not a problem if we accumulate entries in the icon map
+                if icon_map is not None:
+                    icon_map[cat_name] = icon_map['*user']
+                if sort_on_count:
+                    categories[cat_name] = \
+                        sorted(items, cmp=(lambda x, y: cmp(y.count, x.count)))
+                else:
+                    categories[cat_name] = \
+                        sorted(items, cmp=(lambda x, y: cmp(x.name.lower(), y.name.lower())))
+
+        #### Finally, the saved searches category ####
+        items = []
+        icon = None
+        if icon_map and 'search' in icon_map:
+                icon = icon_map['search']
+        for srch in saved_searches.names():
+            items.append(Tag(srch, tooltip=saved_searches.lookup(srch), icon=icon))
+        if len(items):
+            self.tag_browser_categories['search'] = {
+                            'table':None,    'column':None,
+                            'type':None,     'is_multiple':False,
+                            'kind':'search', 'name':_('Searches')}
+            if icon_map is not None:
+                icon_map['search'] = icon_map['search']
+            categories['search'] = items
+
         return categories
 
     def tags_older_than(self, tag, delta):

src/calibre/utils/config.py

@@ -694,8 +694,10 @@ def _prefs():
             help=_('Add new formats to existing book records'))
     c.add_opt('installation_uuid', default=None, help='Installation UUID')
 
-    # this is here instead of the gui preferences because calibredb can execute searches
+    # these are here instead of the gui preferences because calibredb and
+    # calibre server can execute searches
     c.add_opt('saved_searches', default={}, help=_('List of named saved searches'))
+    c.add_opt('user_categories', default={}, help=_('User-created tag browser categories'))
 
     c.add_opt('migrated', default=False, help='For Internal use. Don\'t modify.')
     return c