From 3e926bb612c8a833fd81f3d3b4876b1001678e1f Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Tue, 25 May 2010 14:29:51 +0100 Subject: [PATCH 1/3] 1) Ensure that Calibres always uses the same names for tag and search categories. 2) Add code to verify the names are reserved. 3) add compatibility names to the reserved words list 4) fix model to show the correct search label 'data' for the timestamp column 5) change internal naming of icons and user categories to avoid namespace collisions 6) fix get_categories to do the right thing with rating fields --- src/calibre/ebooks/metadata/book/__init__.py | 17 ++++++--- src/calibre/gui2/library/models.py | 15 +++++--- src/calibre/gui2/tag_view.py | 38 +++++++++++--------- src/calibre/library/database2.py | 23 ++++++------ src/calibre/utils/search_query_parser.py | 33 ++++++++++------- 5 files changed, 76 insertions(+), 50 deletions(-) diff --git a/src/calibre/ebooks/metadata/book/__init__.py b/src/calibre/ebooks/metadata/book/__init__.py index 0edf08c405..c106582bfa 100644 --- a/src/calibre/ebooks/metadata/book/__init__.py +++ b/src/calibre/ebooks/metadata/book/__init__.py @@ -89,11 +89,18 @@ CALIBRE_METADATA_FIELDS = frozenset([ ) CALIBRE_RESERVED_LABELS = frozenset([ - 'search', # reserved for saved searches - 'date', - 'all', - 'ondevice', - 'inlibrary', + 'all', # search term + 'author_sort', # can appear in device collection customization + 'date', # search term + 'formats', # search term + 'inlibrary', # search term + 'news', # search term + 'ondevice', # search term + 'search', # search term + 'format', # The next four are here for backwards compatibility + 'tag', # with searching. The terms can be used without the + 'author', # trailing 's'. + 'comment', # Sigh ... ] ) diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py index bc0367b766..18af6d8560 100644 --- a/src/calibre/gui2/library/models.py +++ b/src/calibre/gui2/library/models.py @@ -631,7 +631,10 @@ class BooksModel(QAbstractTableModel): # {{{ if section >= len(self.column_map): # same problem as in data, the column_map can be wrong return None if role == Qt.ToolTipRole: - return QVariant(_('The lookup/search name is "{0}"').format(self.column_map[section])) + ht = self.column_map[section] + if ht == 'timestamp': # change help text because users know this field as 'date' + ht = 'date' + return QVariant(_('The lookup/search name is "{0}"').format(ht)) if role == Qt.DisplayRole: return QVariant(self.headers[self.column_map[section]]) return NONE @@ -730,11 +733,13 @@ class BooksModel(QAbstractTableModel): # {{{ class OnDeviceSearch(SearchQueryParser): # {{{ USABLE_LOCATIONS = [ - 'collections', - 'title', - 'author', - 'format', 'all', + 'author', + 'authors', + 'collections', + 'format', + 'formats', + 'title', ] diff --git a/src/calibre/gui2/tag_view.py b/src/calibre/gui2/tag_view.py index 6d1bf5ab28..8bbdc69c62 100644 --- a/src/calibre/gui2/tag_view.py +++ b/src/calibre/gui2/tag_view.py @@ -14,8 +14,7 @@ from PyQt4.Qt import Qt, QTreeView, QApplication, pyqtSignal, \ QAbstractItemModel, QVariant, QModelIndex from calibre.gui2 import config, NONE from calibre.utils.config import prefs -from calibre.utils.search_query_parser import saved_searches -from calibre.library.database2 import Tag +from calibre.ebooks.metadata.book import RESERVED_METADATA_FIELDS class TagsView(QTreeView): # {{{ @@ -204,17 +203,22 @@ class TagsModel(QAbstractItemModel): # {{{ QAbstractItemModel.__init__(self, parent) # must do this here because 'QPixmap: Must construct a QApplication - # before a QPaintDevice' - self.category_icon_map = {'authors': QIcon(I('user_profile.svg')), - 'series': QIcon(I('series.svg')), - 'formats':QIcon(I('book.svg')), - 'publishers': QIcon(I('publisher.png')), - 'ratings':QIcon(I('star.png')), - 'news':QIcon(I('news.svg')), - 'tags':QIcon(I('tags.svg')), - '*custom':QIcon(I('column.svg')), - '*user':QIcon(I('drawer.svg')), - 'search':QIcon(I('search.svg'))} + # before a QPaintDevice'. The ':' in front avoids polluting either the + # user-defined categories (':' at end) or columns namespaces (no ':'). + self.category_icon_map = { + 'authors' : QIcon(I('user_profile.svg')), + 'series' : QIcon(I('series.svg')), + 'formats' : QIcon(I('book.svg')), + 'publisher' : QIcon(I('publisher.png')), + 'rating' : QIcon(I('star.png')), + 'news' : QIcon(I('news.svg')), + 'tags' : QIcon(I('tags.svg')), + ':custom' : QIcon(I('column.svg')), + ':user' : QIcon(I('drawer.svg')), + 'search' : QIcon(I('search.svg'))} + for k in self.category_icon_map.keys(): + if not k.startswith(':') and k not in RESERVED_METADATA_FIELDS: + raise ValueError('Tag category [%s] is not a reserved word.' %(k)) self.icon_state_map = [None, QIcon(I('plus.svg')), QIcon(I('minus.svg'))] self.db = db self.search_restriction = '' @@ -381,9 +385,9 @@ class TagsModel(QAbstractItemModel): # {{{ def tokens(self): ans = [] - tags_seen = [] + tags_seen = set() for i, key in enumerate(self.row_map): - if key.endswith('*'): # User category, so skip it. The tag will be marked in its real category + if key.endswith(':'): # User category, so skip it. The tag will be marked in its real category continue category_item = self.root_item.children[i] for tag_item in category_item.children: @@ -394,10 +398,10 @@ class TagsModel(QAbstractItemModel): # {{{ if tag.name[0] == u'\u2605': # char is a star. Assume rating ans.append('%s%s:%s'%(prefix, category, len(tag.name))) else: - if category == 'tag': + if category == 'tags': if tag.name in tags_seen: continue - tags_seen.append(tag.name) + tags_seen.add(tag.name) ans.append('%s%s:"=%s"'%(prefix, category, tag.name)) return ans diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 8278386b8e..84124b6ce9 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -37,6 +37,7 @@ from calibre.utils.ordered_dict import OrderedDict from calibre.utils.config import prefs from calibre.utils.search_query_parser import saved_searches from calibre.ebooks import BOOK_EXTENSIONS, check_ebook_format +from calibre.ebooks.metadata.book import RESERVED_METADATA_FIELDS if iswindows: import calibre.utils.winshell as winshell @@ -137,10 +138,10 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): ('formats', {'table':None, 'column':None, 'type':None, 'is_multiple':False, 'kind':'standard', 'name':_('Formats')}), - ('publishers',{'table':'publishers', 'column':'name', + ('publisher', {'table':'publishers', 'column':'name', 'type':'text', 'is_multiple':False, 'kind':'standard', 'name':_('Publishers')}), - ('ratings', {'table':'ratings', 'column':'rating', + ('rating', {'table':'ratings', 'column':'rating', 'type':'rating', 'is_multiple':False, 'kind':'standard', 'name':_('Ratings')}), ('news', {'table':'news', 'column':'name', @@ -152,6 +153,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): ] self.tag_browser_categories = OrderedDict() for k,v in tag_browser_categories_items: + if k not in RESERVED_METADATA_FIELDS: + raise ValueError('Tag category [%s] is not a reserved word.' %(k)) self.tag_browser_categories[k] = v self.connect() @@ -694,25 +697,25 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): if category in icon_map: icon = icon_map[category] elif self.tag_browser_categories[category]['kind'] == 'custom': - icon = icon_map['*custom'] - icon_map[category] = icon_map['*custom'] + icon = icon_map[':custom'] + icon_map[category] = icon tooltip = self.custom_column_label_map[category]['name'] datatype = self.tag_browser_categories[category]['type'] if datatype == 'rating': - item_zero_func = (lambda x: len(formatter(r[1])) > 0) + item_not_zero_func = (lambda x: x[1] > 0 and x[2] > 0) formatter = (lambda x:u'\u2605'*int(round(x/2.))) elif category == 'authors': - item_zero_func = (lambda x: x[2] > 0) + item_not_zero_func = (lambda x: x[2] > 0) # Clean up the authors strings to human-readable form formatter = (lambda x: x.replace('|', ',')) else: - item_zero_func = (lambda x: x[2] > 0) + item_not_zero_func = (lambda x: x[2] > 0) formatter = (lambda x:x) categories[category] = [Tag(formatter(r[1]), count=r[2], id=r[0], icon=icon, tooltip = tooltip) - for r in data if item_zero_func(r)] + for r in data if item_not_zero_func(r)] # We delayed computing the standard formats category because it does not # use a view, but is computed dynamically @@ -767,14 +770,14 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): items.append(taglist[label][name]) # else: do nothing, to not include nodes w zero counts if len(items): - cat_name = user_cat+'*' # add the * to avoid name collision + cat_name = user_cat+':' # add the ':' to avoid name collision self.tag_browser_categories[cat_name] = { 'table':None, 'column':None, 'type':None, 'is_multiple':False, 'kind':'user', 'name':user_cat} # Not a problem if we accumulate entries in the icon map if icon_map is not None: - icon_map[cat_name] = icon_map['*user'] + icon_map[cat_name] = icon_map[':user'] if sort_on_count: categories[cat_name] = \ sorted(items, cmp=(lambda x, y: cmp(y.count, x.count))) diff --git a/src/calibre/utils/search_query_parser.py b/src/calibre/utils/search_query_parser.py index 11991727b7..5fe0a242f8 100644 --- a/src/calibre/utils/search_query_parser.py +++ b/src/calibre/utils/search_query_parser.py @@ -22,7 +22,7 @@ from calibre.utils.pyparsing import Keyword, Group, Forward, CharsNotIn, Suppres OneOrMore, oneOf, CaselessLiteral, Optional, NoMatch, ParseException from calibre.constants import preferred_encoding from calibre.utils.config import prefs - +from calibre.ebooks.metadata.book import RESERVED_METADATA_FIELDS ''' This class manages access to the preference holding the saved search queries. @@ -87,21 +87,25 @@ class SearchQueryParser(object): ''' DEFAULT_LOCATIONS = [ - 'tag', - 'title', - 'author', + 'all', + 'author', # compatibility + 'authors', + 'comment', # compatibility + 'comments', + 'cover', + 'date', + 'format', # compatibility + 'formats', + 'isbn', + 'ondevice', + 'pubdate', 'publisher', + 'search', 'series', 'rating', - 'cover', - 'comments', - 'format', - 'isbn', - 'search', - 'date', - 'pubdate', - 'ondevice', - 'all', + 'tag', # compatibility + 'tags', + 'title', ] @staticmethod @@ -118,6 +122,9 @@ class SearchQueryParser(object): return failed def __init__(self, locations=None, test=False): + for k in self.DEFAULT_LOCATIONS: + if k not in RESERVED_METADATA_FIELDS: + raise ValueError('Search location [%s] is not a reserved word.' %(k)) if locations is None: locations = self.DEFAULT_LOCATIONS self._tests_failed = False From e43f1b9fd2d11d9ccb42a3d6f312f6f9d469148f Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 25 May 2010 09:56:44 -0600 Subject: [PATCH 2/3] ... --- src/calibre/ebooks/metadata/book/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/calibre/ebooks/metadata/book/__init__.py b/src/calibre/ebooks/metadata/book/__init__.py index c106582bfa..39fb1920cd 100644 --- a/src/calibre/ebooks/metadata/book/__init__.py +++ b/src/calibre/ebooks/metadata/book/__init__.py @@ -90,7 +90,6 @@ CALIBRE_METADATA_FIELDS = frozenset([ CALIBRE_RESERVED_LABELS = frozenset([ 'all', # search term - 'author_sort', # can appear in device collection customization 'date', # search term 'formats', # search term 'inlibrary', # search term From b92c8f21dd4a881e62367fc59094b49635a680da Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 25 May 2010 14:06:10 -0600 Subject: [PATCH 3/3] Searching now works in the OPDS feeds --- src/calibre/library/server/cache.py | 8 ++- src/calibre/library/server/opds.py | 100 +++++++++++++++++++++++++--- 2 files changed, 96 insertions(+), 12 deletions(-) diff --git a/src/calibre/library/server/cache.py b/src/calibre/library/server/cache.py index 5c9be367d0..f8de28a735 100644 --- a/src/calibre/library/server/cache.py +++ b/src/calibre/library/server/cache.py @@ -17,10 +17,14 @@ class Cache(object): def search_cache(self, search): old = self._search_cache.get(search, None) if old is None or old[0] <= self.db.last_modified(): - matches = self.db.data.search(search) - self._search_cache[search] = frozenset(matches) + matches = self.db.data.search(search, return_matches=True, + ignore_search_restriction=True) + if not matches: + matches = [] + self._search_cache[search] = (utcnow(), frozenset(matches)) if len(self._search_cache) > 10: self._search_cache.popitem(last=False) + return self._search_cache[search][1] def categories_cache(self, restrict_to=frozenset([])): diff --git a/src/calibre/library/server/opds.py b/src/calibre/library/server/opds.py index d6702cbe75..149f12644c 100644 --- a/src/calibre/library/server/opds.py +++ b/src/calibre/library/server/opds.py @@ -7,18 +7,24 @@ __docformat__ = 'restructuredtext en' import hashlib, binascii from functools import partial +from itertools import repeat -from lxml import etree +from lxml import etree, html from lxml.builder import ElementMaker import cherrypy from calibre.constants import __appname__ +from calibre.ebooks.metadata import fmt_sidx +from calibre.library.comments import comments_to_html +from calibre import guess_type BASE_HREFS = { 0 : '/stanza', 1 : '/opds', } +STANZA_FORMATS = frozenset(['epub', 'pdb']) + # Vocabulary for building OPDS feeds {{{ E = ElementMaker(namespace='http://www.w3.org/2005/Atom', nsmap={ @@ -71,9 +77,72 @@ LAST_LINK = partial(NAVLINK, rel='last') NEXT_LINK = partial(NAVLINK, rel='next') PREVIOUS_LINK = partial(NAVLINK, rel='previous') +def html_to_lxml(raw): + raw = u'
%s
'%raw + root = html.fragment_fromstring(raw) + root.set('xmlns', "http://www.w3.org/1999/xhtml") + raw = etree.tostring(root, encoding=None) + return etree.fromstring(raw) + +def ACQUISITION_ENTRY(item, version, FM, updated): + title = item[FM['title']] + if not title: + title = _('Unknown') + authors = item[FM['authors']] + if not authors: + authors = _('Unknown') + authors = ' & '.join([i.replace('|', ',') for i in + authors.split(',')]) + extra = [] + rating = item[FM['rating']] + if rating > 0: + rating = u''.join(repeat(u'\u2605', int(rating/2.))) + extra.append(_('RATING: %s
')%rating) + tags = item[FM['tags']] + if tags: + extra.append(_('TAGS: %s
')%\ + ', '.join(tags.split(','))) + series = item[FM['series']] + if series: + extra.append(_('SERIES: %s [%s]
')%\ + (series, + fmt_sidx(float(item[FM['series_index']])))) + comments = item[FM['comments']] + if comments: + comments = comments_to_html(comments) + extra.append(comments) + if extra: + extra = html_to_lxml('\n'.join(extra)) + idm = 'calibre' if version == 0 else 'uuid' + id_ = 'urn:%s:%s'%(idm, item[FM['uuid']]) + ans = E.entry(TITLE(title), E.author(E.name(authors)), ID(id_), + UPDATED(updated)) + if extra: + ans.append(E.content(extra, type='xhtml')) + formats = item[FM['formats']] + if formats: + for fmt in formats.split(','): + fmt = fmt.lower() + mt = guess_type('a.'+fmt)[0] + href = '/get/%s/%s'%(fmt, item[FM['id']]) + if mt: + link = E.link(type=mt, href=href) + if version > 0: + link.set('rel', "http://opds-spec.org/acquisition") + ans.append(link) + ans.append(E.link(type='image/jpeg', href='/get/cover/%s'%item[FM['id']], + rel="x-stanza-cover-image" if version == 0 else + "http://opds-spec.org/cover")) + ans.append(E.link(type='image/jpeg', href='/get/thumb/%s'%item[FM['id']], + rel="x-stanza-cover-image-thumbnail" if version == 0 else + "http://opds-spec.org/thumbnail")) + + return ans + + # }}} -class Feed(object): +class Feed(object): # {{{ def __init__(self, id_, updated, version, subtitle=None, title=__appname__ + ' ' + _('Library'), @@ -106,6 +175,7 @@ class Feed(object): def __str__(self): return etree.tostring(self.root, pretty_print=True, encoding='utf-8', xml_declaration=True) + # }}} class TopLevel(Feed): # {{{ @@ -126,9 +196,9 @@ class TopLevel(Feed): # {{{ self.root.append(x) # }}} -class AcquisitionFeed(Feed): +class NavFeed(Feed): - def __init__(self, updated, id_, items, offsets, page_url, up_url, version): + def __init__(self, id_, updated, version, offsets, page_url, up_url): kwargs = {'up_link': up_url} kwargs['first_link'] = page_url kwargs['last_link'] = page_url+'?offset=%d'%offsets.last_offset @@ -140,7 +210,14 @@ class AcquisitionFeed(Feed): page_url+'?offset=%d'%offsets.next_offset Feed.__init__(self, id_, updated, version, **kwargs) -STANZA_FORMATS = frozenset(['epub', 'pdb']) +class AcquisitionFeed(NavFeed): + + def __init__(self, updated, id_, items, offsets, page_url, up_url, version, + FM): + NavFeed.__init__(self, id_, updated, version, offsets, page_url, up_url) + for item in items: + self.root.append(ACQUISITION_ENTRY(item, version, FM, updated)) + class OPDSOffsets(object): @@ -176,9 +253,10 @@ class OPDSServer(object): self.opds_search, version=version) def get_opds_allowed_ids_for_version(self, version): - search = '' if version > 0 else ' '.join(['format:='+x for x in + search = '' if version > 0 else ' or '.join(['format:='+x for x in STANZA_FORMATS]) - self.search_cache(search) + ids = self.search_cache(search) + return ids def get_opds_acquisition_feed(self, ids, offset, page_url, up_url, id_, sort_by='title', ascending=True, version=0): @@ -189,7 +267,8 @@ class OPDSServer(object): max_items = self.opts.max_opds_items offsets = OPDSOffsets(offset, max_items, len(items)) items = items[offsets.offset:offsets.next_offset] - return str(AcquisitionFeed(self.db.last_modified(), id_, items, offsets, page_url, up_url, version)) + return str(AcquisitionFeed(self.db.last_modified(), id_, items, offsets, + page_url, up_url, version, self.db.FIELD_MAP)) def opds_search(self, query=None, version=0, offset=0): try: @@ -203,8 +282,9 @@ class OPDSServer(object): ids = self.search_cache(query) except: raise cherrypy.HTTPError(404, 'Search: %r not understood'%query) - return self.get_opds_acquisition_feed(ids, - sort_by='title', version=version) + return self.get_opds_acquisition_feed(ids, offset, '/search/'+query, + BASE_HREFS[version], 'calibre-search:'+query, + version=version) def opds_navcatalog(self, which=None, version=0): version = int(version)