First iteration of refactoring tag category and search term management

This commit is contained in:
Charles Haley 2010-05-26 12:19:53 +01:00
parent 0fab678b39
commit 53fb1ef462
7 changed files with 254 additions and 119 deletions

View File

@ -89,17 +89,6 @@ CALIBRE_METADATA_FIELDS = frozenset([
)
CALIBRE_RESERVED_LABELS = frozenset([
'all', # search term
'date', # search term
'formats', # search term
'inlibrary', # search term
'news', # search term
'ondevice', # search term
'search', # search term
'format', # The next four are here for backwards compatibility
'tag', # with searching. The terms can be used without the
'author', # trailing 's'.
'comment', # Sigh ...
]
)

View File

@ -14,7 +14,7 @@ from PyQt4.Qt import Qt, QTreeView, QApplication, pyqtSignal, \
QAbstractItemModel, QVariant, QModelIndex
from calibre.gui2 import config, NONE
from calibre.utils.config import prefs
from calibre.ebooks.metadata.book import RESERVED_METADATA_FIELDS
from calibre.library.tag_categories import TagsIcons
class TagsView(QTreeView): # {{{
@ -205,7 +205,7 @@ class TagsModel(QAbstractItemModel): # {{{
# must do this here because 'QPixmap: Must construct a QApplication
# before a QPaintDevice'. The ':' in front avoids polluting either the
# user-defined categories (':' at end) or columns namespaces (no ':').
self.category_icon_map = {
self.category_icon_map = TagsIcons({
'authors' : QIcon(I('user_profile.svg')),
'series' : QIcon(I('series.svg')),
'formats' : QIcon(I('book.svg')),
@ -215,10 +215,8 @@ class TagsModel(QAbstractItemModel): # {{{
'tags' : QIcon(I('tags.svg')),
':custom' : QIcon(I('column.svg')),
':user' : QIcon(I('drawer.svg')),
'search' : QIcon(I('search.svg'))}
for k in self.category_icon_map.keys():
if not k.startswith(':') and k not in RESERVED_METADATA_FIELDS:
raise ValueError('Tag category [%s] is not a reserved word.' %(k))
'search' : QIcon(I('search.svg'))})
self.icon_state_map = [None, QIcon(I('plus.svg')), QIcon(I('minus.svg'))]
self.db = db
self.search_restriction = ''
@ -247,7 +245,7 @@ class TagsModel(QAbstractItemModel): # {{{
data = self.db.get_categories(sort_on_count=sort, icon_map=self.category_icon_map)
tb_categories = self.db.get_tag_browser_categories()
for category in tb_categories.iterkeys():
for category in tb_categories:
if category in data: # They should always be there, but ...
self.row_map.append(category)
self.categories.append(tb_categories[category]['name'])

View File

@ -17,6 +17,7 @@ from calibre.utils.config import tweaks
from calibre.utils.date import parse_date, now, UNDEFINED_DATE
from calibre.utils.search_query_parser import SearchQueryParser
from calibre.utils.pyparsing import ParseException
from calibre.library.tag_categories import TagsMetadata
class CoverCache(QThread):
@ -149,15 +150,15 @@ class ResultCache(SearchQueryParser):
'''
Stores sorted and filtered metadata in memory.
'''
def __init__(self, FIELD_MAP, cc_label_map):
def __init__(self, FIELD_MAP, cc_label_map, tag_browser_categories):
self.FIELD_MAP = FIELD_MAP
self.custom_column_label_map = cc_label_map
self._map = self._map_filtered = self._data = []
self.first_sort = True
self.search_restriction = ''
SearchQueryParser.__init__(self,
locations=SearchQueryParser.DEFAULT_LOCATIONS +
[c for c in cc_label_map])
self.tag_browser_categories = tag_browser_categories
self.all_search_locations = tag_browser_categories.get_search_labels()
SearchQueryParser.__init__(self, self.all_search_locations)
self.build_date_relop_dict()
self.build_numeric_relop_dict()
@ -379,25 +380,33 @@ class ResultCache(SearchQueryParser):
if location in ('tag', 'author', 'format', 'comment'):
location += 's'
all = ('title', 'authors', 'publisher', 'tags', 'comments', 'series',
'formats', 'isbn', 'rating', 'cover', 'ondevice')
# all = ('title', 'authors', 'publisher', 'tags', 'comments', 'series',
# 'formats', 'isbn', 'rating', 'cover', 'ondevice')
MAP = {}
for x in all: # get the db columns for the standard searchables
MAP[x] = self.FIELD_MAP[x]
# get the db columns for the standard searchables
for x in self.tag_browser_categories:
if (len(self.tag_browser_categories[x]['search_labels']) and \
self.tag_browser_categories[x]['kind'] in ['standard', 'not_cat']):
# self.tag_browser_categories[x]['kind'] == 'standard') \
# or self.tag_browser_categories[x]['kind'] == 'not_cat':
MAP[x] = self.FIELD_MAP[self.tag_browser_categories.get_label(x)]
IS_CUSTOM = []
for x in range(len(self.FIELD_MAP)): # build a list containing '' the size of FIELD_MAP
for x in range(len(self.FIELD_MAP)):
IS_CUSTOM.append('')
IS_CUSTOM[self.FIELD_MAP['rating']] = 'rating' # normal and custom ratings columns use the same code
for x in self.custom_column_label_map: # add custom columns to MAP. Put the column's type into IS_CUSTOM
if self.custom_column_label_map[x]['datatype'] != "datetime":
MAP[x] = self.FIELD_MAP[self.custom_column_label_map[x]['num']]
IS_CUSTOM[MAP[x]] = self.custom_column_label_map[x]['datatype']
# add custom columns to MAP. Put the column's type into IS_CUSTOM
for x in self.tag_browser_categories.get_custom_fields():
if self.tag_browser_categories[x]['datatype'] != "datetime":
MAP[x] = self.FIELD_MAP[self.tag_browser_categories[x]['colnum']]
IS_CUSTOM[MAP[x]] = self.tag_browser_categories[x]['datatype']
EXCLUDE_FIELDS = [MAP['rating'], MAP['cover']]
SPLITABLE_FIELDS = [MAP['authors'], MAP['tags'], MAP['formats']]
for x in self.custom_column_label_map:
if self.custom_column_label_map[x]['is_multiple']:
for x in self.tag_browser_categories.get_custom_fields():
if self.tag_browser_categories[x]['is_multiple']:
SPLITABLE_FIELDS.append(MAP[x])
try:

View File

@ -145,11 +145,10 @@ class CustomColumns(object):
v = self.custom_column_label_map[k]
if v['normalized']:
tn = 'custom_column_{0}'.format(v['num'])
self.tag_browser_categories[v['label']] = {
'table':tn, 'column':'value',
'type':v['datatype'], 'is_multiple':v['is_multiple'],
'kind':'custom', 'name':v['name']
}
self.tag_browser_categories.add_custom_field(
field_name = v['label'], table = tn, column='value',
datatype=v['datatype'], is_multiple=v['is_multiple'],
number=v['num'], name=v['name'])
def get_custom(self, idx, label=None, num=None, index_is_id=False):
if label is not None:

View File

@ -20,6 +20,7 @@ from PyQt4.QtGui import QImage
from calibre.ebooks.metadata import title_sort
from calibre.library.database import LibraryDatabase
from calibre.library.tag_categories import TagsMetadata, TagsIcons
from calibre.library.schema_upgrades import SchemaUpgrade
from calibre.library.caches import ResultCache
from calibre.library.custom_columns import CustomColumns
@ -33,11 +34,10 @@ from calibre.customize.ui import run_plugins_on_import
from calibre.utils.filenames import ascii_filename
from calibre.utils.date import utcnow, now as nowf, utcfromtimestamp
from calibre.utils.ordered_dict import OrderedDict
from calibre.utils.config import prefs
from calibre.utils.search_query_parser import saved_searches
from calibre.ebooks import BOOK_EXTENSIONS, check_ebook_format
from calibre.ebooks.metadata.book import RESERVED_METADATA_FIELDS
if iswindows:
import calibre.utils.winshell as winshell
@ -116,6 +116,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
self.books_list_filter = self.conn.create_dynamic_filter('books_list_filter')
def __init__(self, library_path, row_factory=False):
self.tag_browser_categories = TagsMetadata() #.get_tag_browser_categories()
if not os.path.exists(library_path):
os.makedirs(library_path)
self.listeners = set([])
@ -127,36 +128,6 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
if isinstance(self.dbpath, unicode):
self.dbpath = self.dbpath.encode(filesystem_encoding)
# Order as has been customary in the tags pane.
tag_browser_categories_items = [
('authors', {'table':'authors', 'column':'name',
'type':'text', 'is_multiple':False,
'kind':'standard', 'name':_('Authors')}),
('series', {'table':'series', 'column':'name',
'type':None, 'is_multiple':False,
'kind':'standard', 'name':_('Series')}),
('formats', {'table':None, 'column':None,
'type':None, 'is_multiple':False,
'kind':'standard', 'name':_('Formats')}),
('publisher', {'table':'publishers', 'column':'name',
'type':'text', 'is_multiple':False,
'kind':'standard', 'name':_('Publishers')}),
('rating', {'table':'ratings', 'column':'rating',
'type':'rating', 'is_multiple':False,
'kind':'standard', 'name':_('Ratings')}),
('news', {'table':'news', 'column':'name',
'type':None, 'is_multiple':False,
'kind':'standard', 'name':_('News')}),
('tags', {'table':'tags', 'column':'name',
'type':'text', 'is_multiple':True,
'kind':'standard', 'name':_('Tags')}),
]
self.tag_browser_categories = OrderedDict()
for k,v in tag_browser_categories_items:
if k not in RESERVED_METADATA_FIELDS:
raise ValueError('Tag category [%s] is not a reserved word.' %(k))
self.tag_browser_categories[k] = v
self.connect()
self.is_case_sensitive = not iswindows and not isosx and \
not os.path.exists(self.dbpath.replace('metadata.db', 'MeTAdAtA.dB'))
@ -251,7 +222,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
self.conn.commit()
self.book_on_device_func = None
self.data = ResultCache(self.FIELD_MAP, self.custom_column_label_map)
self.data = ResultCache(self.FIELD_MAP, self.custom_column_label_map,
self.tag_browser_categories)
self.search = self.data.search
self.refresh = functools.partial(self.data.refresh, self)
self.sort = self.data.sort
@ -671,14 +643,20 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
self.books_list_filter.change([] if not ids else ids)
categories = {}
if icon_map is not None and type(icon_map) != TagsIcons:
raise TypeError('icon_map passed to get_categories must be of type TagIcons')
#### First, build the standard and custom-column categories ####
for category in self.tag_browser_categories.keys():
tn = self.tag_browser_categories[category]['table']
tb_cats = self.tag_browser_categories
for category in tb_cats.keys():
cat = tb_cats[category]
if cat['kind'] == 'not_cat':
continue
tn = cat['table']
categories[category] = [] #reserve the position in the ordered list
if tn is None: # Nothing to do for the moment
continue
cn = self.tag_browser_categories[category]['column']
cn = cat['column']
if ids is None:
query = 'SELECT id, {0}, count FROM tag_browser_{1}'.format(cn, tn)
else:
@ -692,16 +670,17 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
# icon_map is not None if get_categories is to store an icon and
# possibly a tooltip in the tag structure.
icon, tooltip = None, ''
label = tb_cats.get_label(category)
if icon_map:
if self.tag_browser_categories[category]['kind'] == 'standard':
if cat['kind'] == 'standard':
if category in icon_map:
icon = icon_map[category]
elif self.tag_browser_categories[category]['kind'] == 'custom':
icon = icon_map[label]
elif cat['kind'] == 'custom':
icon = icon_map[':custom']
icon_map[category] = icon
tooltip = self.custom_column_label_map[category]['name']
tooltip = self.custom_column_label_map[label]['name']
datatype = self.tag_browser_categories[category]['type']
datatype = cat['datatype']
if datatype == 'rating':
item_not_zero_func = (lambda x: x[1] > 0 and x[2] > 0)
formatter = (lambda x:u'\u2605'*int(round(x/2.)))
@ -711,7 +690,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
formatter = (lambda x: x.replace('|', ','))
else:
item_not_zero_func = (lambda x: x[2] > 0)
formatter = (lambda x:x)
formatter = (lambda x:unicode(x))
categories[category] = [Tag(formatter(r[1]), count=r[2], id=r[0],
icon=icon, tooltip = tooltip)
@ -750,9 +729,9 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
# remove all user categories from tag_browser_categories. They can
# easily come and go. We will add all the existing ones in below.
for k in self.tag_browser_categories.keys():
if self.tag_browser_categories[k]['kind'] in ['user', 'search']:
del self.tag_browser_categories[k]
for k in tb_cats.keys():
if tb_cats[k]['kind'] in ['user', 'search']:
del tb_cats[k]
# We want to use same node in the user category as in the source
# category. To do that, we need to find the original Tag node. There is
@ -771,10 +750,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
# else: do nothing, to not include nodes w zero counts
if len(items):
cat_name = user_cat+':' # add the ':' to avoid name collision
self.tag_browser_categories[cat_name] = {
'table':None, 'column':None,
'type':None, 'is_multiple':False,
'kind':'user', 'name':user_cat}
tb_cats.add_user_category(field_name=cat_name, name=user_cat)
# Not a problem if we accumulate entries in the icon map
if icon_map is not None:
icon_map[cat_name] = icon_map[':user']
@ -793,10 +769,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
for srch in saved_searches.names():
items.append(Tag(srch, tooltip=saved_searches.lookup(srch), icon=icon))
if len(items):
self.tag_browser_categories['search'] = {
'table':None, 'column':None,
'type':None, 'is_multiple':False,
'kind':'search', 'name':_('Searches')}
tb_cats.add_user_category(field_name='search', name=_('Searches'))
if icon_map is not None:
icon_map['search'] = icon_map['search']
categories['search'] = items

View File

@ -0,0 +1,194 @@
'''
Created on 25 May 2010
@author: charles
'''
from UserDict import DictMixin
from calibre.utils.ordered_dict import OrderedDict
class TagsIcons(dict):
'''
If the client wants icons to be in the tag structure, this class must be
instantiated and filled in with real icons. If this class is instantiated
and passed to get_categories, All items must be given a value not None
'''
category_icons = ['authors', 'series', 'formats', 'publisher', 'rating',
'news', 'tags', ':custom', ':user', 'search',]
def __init__(self, icon_dict):
for a in self.category_icons:
if a not in icon_dict:
raise ValueError('Missing category icon [%s]'%a)
self[a] = icon_dict[a]
class TagsMetadata(dict, DictMixin):
# kind == standard: is tag category. May be a search label. Is db col
# or is specially handled (e.g., news)
# kind == not_cat: Is not a tag category. Should be a search label. Is db col
# kind == user: user-defined tag category
# kind == search: saved-searches category
# Order as has been customary in the tags pane.
category_items_ = [
('authors', {'table':'authors', 'column':'name',
'datatype':'text', 'is_multiple':False,
'kind':'standard', 'name':_('Authors'),
'search_labels':['authors', 'author']}),
('series', {'table':'series', 'column':'name',
'datatype':None, 'is_multiple':False,
'kind':'standard', 'name':_('Series'),
'search_labels':['series']}),
('formats', {'table':None, 'column':None,
'datatype':None, 'is_multiple':False,
'kind':'standard', 'name':_('Formats'),
'search_labels':['formats', 'format']}),
('publisher', {'table':'publishers', 'column':'name',
'datatype':'text', 'is_multiple':False,
'kind':'standard', 'name':_('Publishers'),
'search_labels':['publisher']}),
('rating', {'table':'ratings', 'column':'rating',
'datatype':'rating', 'is_multiple':False,
'kind':'standard', 'name':_('Ratings'),
'search_labels':['rating']}),
('news', {'table':'news', 'column':'name',
'datatype':None, 'is_multiple':False,
'kind':'standard', 'name':_('News'),
'search_labels':[]}),
('tags', {'table':'tags', 'column':'name',
'datatype':'text', 'is_multiple':True,
'kind':'standard', 'name':_('Tags'),
'search_labels':['tags', 'tag']}),
('comments', {'table':None, 'column':None,
'datatype':None, 'is_multiple':False,
'kind':'not_cat', 'name':None,
'search_labels':['comments', 'comment']}),
('cover', {'table':None, 'column':None,
'datatype':None, 'is_multiple':False,
'kind':'not_cat', 'name':None,
'search_labels':['cover']}),
('isbn', {'table':None, 'column':None,
'datatype':None, 'is_multiple':False,
'kind':'not_cat', 'name':None,
'search_labels':['isbn']}),
('pubdate', {'table':None, 'column':None,
'datatype':None, 'is_multiple':False,
'kind':'not_cat', 'name':None,
'search_labels':['pubdate']}),
('title', {'table':None, 'column':None,
'datatype':None, 'is_multiple':False,
'kind':'not_cat', 'name':None,
'search_labels':['title']}),
]
# search labels that are not db columns
search_items = [ 'all',
'date',
'search',
]
def __init__(self):
self.tb_cats_ = OrderedDict()
for k,v in self.category_items_:
self.tb_cats_[k] = v
def __getattr__(self, name):
if name in self.tb_cats_:
return self.tb_cats_[name]
return None
# def __setattr__(self, name, val):
# dict.__setattr__(self, name, val)
def __getitem__(self, key):
return self.tb_cats_[key]
# def __setitem__(self, key, val):
# print 'setitem', key, val
# self.tb_cats_[key] = val
def __delitem__(self, key):
del self.tb_cats_[key]
def __iter__(self):
for key in self.tb_cats_:
yield key
def keys(self):
return self.tb_cats_.keys()
def iterkeys(self):
for key in self.tb_cats_:
yield key
def iteritems(self):
for key in self.tb_cats_:
yield (key, self.tb_cats_[key])
def get_label(self, key):
if 'label' not in self.tb_cats_[key]:
return key
return self.tb_cats_[key]['label']
def get_custom_fields(self):
return [l for l in self.tb_cats_ if self.tb_cats_[l]['kind'] == 'custom']
def add_custom_field(self, field_name, table, column, datatype, is_multiple, number, name):
fn = '#' + field_name
if fn in self.tb_cats_:
raise ValueError('Duplicate custom field [%s]'%(field_name))
self.tb_cats_[fn] = {'table':table, 'column':column,
'datatype':datatype, 'is_multiple':is_multiple,
'kind':'custom', 'name':name,
'search_labels':[fn],'label':field_name,
'colnum':number}
def add_user_category(self, field_name, name):
if field_name in self.tb_cats_:
raise ValueError('Duplicate user field [%s]'%(field_name))
self.tb_cats_[field_name] = {'table':None, 'column':None,
'datatype':None, 'is_multiple':False,
'kind':'user', 'name':name,
'search_labels':[]}
def add_search_category(self, field_name, name):
if field_name in self.tb_cats_:
raise ValueError('Duplicate user field [%s]'%(field_name))
self.tb_cats_[field_name] = {'table':None, 'column':None,
'datatype':None, 'is_multiple':False,
'kind':'search', 'name':name,
'search_labels':[]}
# DEFAULT_LOCATIONS = frozenset([
# 'all',
# 'author', # compatibility
# 'authors',
# 'comment', # compatibility
# 'comments',
# 'cover',
# 'date',
# 'format', # compatibility
# 'formats',
# 'isbn',
# 'ondevice',
# 'pubdate',
# 'publisher',
# 'search',
# 'series',
# 'rating',
# 'tag', # compatibility
# 'tags',
# 'title',
# ])
def get_search_labels(self):
s_labels = []
for v in self.tb_cats_.itervalues():
map((lambda x:s_labels.append(x)), v['search_labels'])
for v in self.search_items:
s_labels.append(v)
# if set(s_labels) != self.DEFAULT_LOCATIONS:
# print 'search labels and default_locations do not match:'
# print set(s_labels) ^ self.DEFAULT_LOCATIONS
return s_labels

View File

@ -22,7 +22,6 @@ from calibre.utils.pyparsing import Keyword, Group, Forward, CharsNotIn, Suppres
OneOrMore, oneOf, CaselessLiteral, Optional, NoMatch, ParseException
from calibre.constants import preferred_encoding
from calibre.utils.config import prefs
from calibre.ebooks.metadata.book import RESERVED_METADATA_FIELDS
'''
This class manages access to the preference holding the saved search queries.
@ -86,27 +85,6 @@ class SearchQueryParser(object):
* `(author:Asimov or author:Hardy) and not tag:read` [search for unread books by Asimov or Hardy]
'''
DEFAULT_LOCATIONS = [
'all',
'author', # compatibility
'authors',
'comment', # compatibility
'comments',
'cover',
'date',
'format', # compatibility
'formats',
'isbn',
'ondevice',
'pubdate',
'publisher',
'search',
'series',
'rating',
'tag', # compatibility
'tags',
'title',
]
@staticmethod
def run_tests(parser, result, tests):
@ -121,12 +99,7 @@ class SearchQueryParser(object):
failed.append(test[0])
return failed
def __init__(self, locations=None, test=False):
for k in self.DEFAULT_LOCATIONS:
if k not in RESERVED_METADATA_FIELDS:
raise ValueError('Search location [%s] is not a reserved word.' %(k))
if locations is None:
locations = self.DEFAULT_LOCATIONS
def __init__(self, locations, test=False):
self._tests_failed = False
# Define a token
standard_locations = map(lambda x : CaselessLiteral(x)+Suppress(':'),