From f9edb21498b1a7028a530bd4091f1879da06bb4d Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 30 May 2010 22:49:50 -0600 Subject: [PATCH] Content server: All OPDS feeds now working --- src/calibre/library/database2.py | 12 ++ src/calibre/library/field_metadata.py | 1 + src/calibre/library/server/cache.py | 12 +- src/calibre/library/server/opds.py | 205 +++++++++++++++++++++++++- 4 files changed, 220 insertions(+), 10 deletions(-) diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index c8b2646144..4107d327ce 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -657,6 +657,18 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): def get_recipe(self, id): return self.conn.get('SELECT script FROM feeds WHERE id=?', (id,), all=False) + def get_books_for_category(self, category, id_): + ans = set([]) + + if category not in self.field_metadata: + return ans + + field = self.field_metadata[category] + ans = self.conn.get( + 'SELECT book FROM books_{tn}_link WHERE {col}=?'.format( + tn=field['table'], col=field['link_column']), (id_,)) + return set(x[0] for x in ans) + def get_categories(self, sort_on_count=False, ids=None, icon_map=None): self.books_list_filter.change([] if not ids else ids) diff --git a/src/calibre/library/field_metadata.py b/src/calibre/library/field_metadata.py index 9d5cd4edc1..07c90119bd 100644 --- a/src/calibre/library/field_metadata.py +++ b/src/calibre/library/field_metadata.py @@ -374,6 +374,7 @@ class FieldMetadata(dict): 'search_terms':[key], 'label':label, 'colnum':colnum, 'display':display, 'is_custom':True, 'is_category':is_category, + 'link_column':'value', 'is_editable': is_editable,} self._add_search_terms_to_map(key, [key]) self.custom_label_to_key_map[label] = key diff --git a/src/calibre/library/server/cache.py b/src/calibre/library/server/cache.py index f8de28a735..9fec2c2737 100644 --- a/src/calibre/library/server/cache.py +++ b/src/calibre/library/server/cache.py @@ -15,23 +15,27 @@ class Cache(object): self._search_cache = OrderedDict() def search_cache(self, search): - old = self._search_cache.get(search, None) + old = self._search_cache.pop(search, None) if old is None or old[0] <= self.db.last_modified(): matches = self.db.data.search(search, return_matches=True, ignore_search_restriction=True) if not matches: matches = [] self._search_cache[search] = (utcnow(), frozenset(matches)) - if len(self._search_cache) > 10: + if len(self._search_cache) > 50: self._search_cache.popitem(last=False) + else: + self._search_cache[search] = old return self._search_cache[search][1] def categories_cache(self, restrict_to=frozenset([])): - old = self._category_cache.get(frozenset(restrict_to), None) + old = self._category_cache.pop(frozenset(restrict_to), None) if old is None or old[0] <= self.db.last_modified(): categories = self.db.get_categories(ids=restrict_to) self._category_cache[restrict_to] = (utcnow(), categories) - if len(self._category_cache) > 10: + if len(self._category_cache) > 20: self._category_cache.popitem(last=False) + else: + self._category_cache[frozenset(restrict_to)] = old return self._category_cache[restrict_to][1] diff --git a/src/calibre/library/server/opds.py b/src/calibre/library/server/opds.py index ca635f9c28..2315b781aa 100644 --- a/src/calibre/library/server/opds.py +++ b/src/calibre/library/server/opds.py @@ -18,6 +18,7 @@ from calibre.constants import __appname__ from calibre.ebooks.metadata import fmt_sidx from calibre.library.comments import comments_to_html from calibre import guess_type +from calibre.utils.ordered_dict import OrderedDict BASE_HREFS = { 0 : '/stanza', @@ -31,6 +32,14 @@ def url_for(name, version, **kwargs): name += '_' return routes.url_for(name+str(version), **kwargs) +def hexlify(x): + if isinstance(x, unicode): + x = x.encode('utf-8') + return binascii.hexlify(x) + +def unhexlify(x): + return binascii.unhexlify(x).decode('utf-8') + # Vocabulary for building OPDS feeds {{{ E = ElementMaker(namespace='http://www.w3.org/2005/Atom', nsmap={ @@ -66,7 +75,7 @@ def AUTHOR(name, uri=None): SUBTITLE = E.subtitle def NAVCATALOG_ENTRY(base_href, updated, title, description, query, version=0): - href = base_href+'/navcatalog/'+binascii.hexlify(query) + href = base_href+'/navcatalog/'+hexlify(query) id_ = 'calibre-navcatalog:'+str(hashlib.sha1(href).hexdigest()) return E.entry( TITLE(title), @@ -90,6 +99,32 @@ def html_to_lxml(raw): raw = etree.tostring(root, encoding=None) return etree.fromstring(raw) +def CATALOG_ENTRY(item, base_href, version, updated): + id_ = 'calibre:category:'+item.name + iid = 'N' + item.name + if item.id is not None: + iid = 'I' + str(item.id) + link = NAVLINK(href = base_href + '/' + hexlify(iid)) + return E.entry( + TITLE(item.name), + ID(id_), + UPDATED(updated), + E.content(_('%d books')%item.count, type='text'), + link + ) + +def CATALOG_GROUP_ENTRY(item, category, base_href, version, updated): + id_ = 'calibre:category-group:'+category+':'+item.text + iid = item.text + link = NAVLINK(href = base_href + '/' + hexlify(iid)) + return E.entry( + TITLE(item.text), + ID(id_), + UPDATED(updated), + E.content(_('%d books')%item.count, type='text'), + link + ) + def ACQUISITION_ENTRY(item, version, FM, updated): title = item[FM['title']] if not title: @@ -225,6 +260,22 @@ class AcquisitionFeed(NavFeed): for item in items: self.root.append(ACQUISITION_ENTRY(item, version, FM, updated)) +class CategoryFeed(NavFeed): + + def __init__(self, items, which, id_, updated, version, offsets, page_url, up_url): + NavFeed.__init__(self, id_, updated, version, offsets, page_url, up_url) + base_href = self.base_href + '/category/' + hexlify(which) + for item in items: + self.root.append(CATALOG_ENTRY(item, base_href, version, updated)) + +class CategoryGroupFeed(NavFeed): + + def __init__(self, items, which, id_, updated, version, offsets, page_url, up_url): + NavFeed.__init__(self, id_, updated, version, offsets, page_url, up_url) + base_href = self.base_href + '/categorygroup/' + hexlify(which) + for item in items: + self.root.append(CATALOG_GROUP_ENTRY(item, which, base_href, version, updated)) + class OPDSOffsets(object): @@ -254,8 +305,13 @@ class OPDSServer(object): base_href = BASE_HREFS[version] ver = str(version) connect('opds_'+ver, base_href, self.opds, version=version) + connect('opdst_'+ver, base_href+'/', self.opds, version=version) connect('opdsnavcatalog_'+ver, base_href+'/navcatalog/{which}', self.opds_navcatalog, version=version) + connect('opdscategory_'+ver, base_href+'/category/{category}/{which}', + self.opds_category, version=version) + connect('opdscategorygroup_'+ver, base_href+'/categorygroup/{category}/{which}', + self.opds_category_group, version=version) connect('opdssearch_'+ver, base_href+'/search/{query}', self.opds_search, version=version) @@ -269,12 +325,17 @@ class OPDSServer(object): sort_by='title', ascending=True, version=0): idx = self.db.FIELD_MAP['id'] ids &= self.get_opds_allowed_ids_for_version(version) + if not ids: + raise cherrypy.HTTPError(404, 'No books found') items = [x for x in self.db.data.iterall() if x[idx] in ids] self.sort(items, sort_by, ascending) max_items = self.opts.max_opds_items offsets = OPDSOffsets(offset, max_items, len(items)) items = items[offsets.offset:offsets.offset+max_items] - return str(AcquisitionFeed(self.db.last_modified(), id_, items, offsets, + updated = self.db.last_modified() + cherrypy.response.headers['Last-Modified'] = self.last_modified(updated) + cherrypy.response.headers['Content-Type'] = 'text/xml' + return str(AcquisitionFeed(updated, id_, items, offsets, page_url, up_url, version, self.db.FIELD_MAP)) def opds_search(self, query=None, version=0, offset=0): @@ -309,28 +370,160 @@ class OPDSServer(object): id_='calibre-all:'+sort, sort_by=sort, ascending=ascending, version=version) + # Categories {{{ + + def opds_category_group(self, category=None, which=None, version=0, offset=0): + try: + offset = int(offset) + version = int(version) + except: + raise cherrypy.HTTPError(404, 'Not found') + + if not which or not category or version not in BASE_HREFS: + raise cherrypy.HTTPError(404, 'Not found') + + categories = self.categories_cache( + self.get_opds_allowed_ids_for_version(version)) + page_url = url_for('opdscategorygroup', version, category=category, which=which) + + category = unhexlify(category) + if category not in categories: + raise cherrypy.HTTPError(404, 'Category %r not found'%which) + which = unhexlify(which) + owhich = hexlify('N'+which) + up_url = url_for('opdsnavcatalog', version, which=owhich) + items = categories[category] + items = [x for x in items if x.name.startswith(which)] + if not items: + raise cherrypy.HTTPError(404, 'No items in group %r:%r'%(category, + which)) + updated = self.db.last_modified() + + id_ = 'calibre-category-group-feed:'+category+':'+which + + max_items = self.opts.max_opds_items + offsets = OPDSOffsets(offset, max_items, len(items)) + items = list(items)[offsets.offset:offsets.offset+max_items] + + cherrypy.response.headers['Last-Modified'] = self.last_modified(updated) + cherrypy.response.headers['Content-Type'] = 'text/xml' + + return str(CategoryFeed(items, category, id_, updated, version, offsets, + page_url, up_url)) + + def opds_navcatalog(self, which=None, version=0, offset=0): - version = int(version) + try: + offset = int(offset) + version = int(version) + except: + raise cherrypy.HTTPError(404, 'Not found') + if not which or version not in BASE_HREFS: raise cherrypy.HTTPError(404, 'Not found') page_url = url_for('opdsnavcatalog', version, which=which) up_url = url_for('opds', version) - which = binascii.unhexlify(which) + which = unhexlify(which) type_ = which[0] which = which[1:] if type_ == 'O': return self.get_opds_all_books(which, page_url, up_url, version=version, offset=offset) elif type_ == 'N': - return self.get_opds_navcatalog(which, version=version, offset=offset) + return self.get_opds_navcatalog(which, page_url, up_url, + version=version, offset=offset) raise cherrypy.HTTPError(404, 'Not found') - def get_opds_navcatalog(self, which, version=0, offset=0): + def get_opds_navcatalog(self, which, page_url, up_url, version=0, offset=0): categories = self.categories_cache( self.get_opds_allowed_ids_for_version(version)) if which not in categories: raise cherrypy.HTTPError(404, 'Category %r not found'%which) + items = categories[which] + updated = self.db.last_modified() + + id_ = 'calibre-category-feed:'+which + + MAX_ITEMS = 50 + + if len(items) <= MAX_ITEMS: + max_items = self.opts.max_opds_items + offsets = OPDSOffsets(offset, max_items, len(items)) + items = list(items)[offsets.offset:offsets.offset+max_items] + ans = CategoryFeed(items, which, id_, updated, version, offsets, + page_url, up_url) + else: + class Group: + def __init__(self, text, count): + self.text, self.count = text, count + + starts = set([x.name[0] for x in items]) + if len(starts) > MAX_ITEMS: + starts = set([x.name[:2] for x in items]) + category_groups = OrderedDict() + for x in sorted(starts, cmp=lambda x,y:cmp(x.lower(), y.lower())): + category_groups[x] = len([y for y in items if + y.name.startswith(x)]) + items = [Group(x, y) for x, y in category_groups.items()] + max_items = self.opts.max_opds_items + offsets = OPDSOffsets(offset, max_items, len(items)) + items = items[offsets.offset:offsets.offset+max_items] + ans = CategoryGroupFeed(items, which, id_, updated, version, offsets, + page_url, up_url) + + cherrypy.response.headers['Last-Modified'] = self.last_modified(updated) + cherrypy.response.headers['Content-Type'] = 'text/xml' + + return str(ans) + + def opds_category(self, category=None, which=None, version=0, offset=0): + try: + offset = int(offset) + version = int(version) + except: + raise cherrypy.HTTPError(404, 'Not found') + + if not which or not category or version not in BASE_HREFS: + raise cherrypy.HTTPError(404, 'Not found') + page_url = url_for('opdscategory', version, which=which, + category=category) + up_url = url_for('opdsnavcatalog', version, which=category) + + which, category = unhexlify(which), unhexlify(category) + type_ = which[0] + which = which[1:] + if type_ == 'I': + try: + which = int(which) + except: + raise cherrypy.HTTPError(404, 'Tag %r not found'%which) + + categories = self.categories_cache( + self.get_opds_allowed_ids_for_version(version)) + if category not in categories: + raise cherrypy.HTTPError(404, 'Category %r not found'%which) + + if category == 'search': + try: + ids = self.search_cache(which) + except: + raise cherrypy.HTTPError(404, 'Search: %r not understood'%which) + return self.get_opds_acquisition_feed(ids, offset, page_url, + up_url, 'calibre-search:'+which, + version=version) + + if type_ != 'I': + raise cherrypy.HTTPError(404, 'Non id categories not supported') + + ids = self.db.get_books_for_category(category, which) + sort_by = 'series' if category == 'series' else 'title' + + return self.get_opds_acquisition_feed(ids, offset, page_url, + up_url, 'calibre-category:'+category+':'+str(which), + version=version, sort_by=sort_by) + + # }}} def opds(self, version=0):