Content server: All OPDS feeds now working

src/calibre/library/database2.py

@@ -657,6 +657,18 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
     def get_recipe(self, id):
         return self.conn.get('SELECT script FROM feeds WHERE id=?', (id,), all=False)
 
+    def get_books_for_category(self, category, id_):
+        ans = set([])
+
+        if category not in self.field_metadata:
+            return ans
+
+        field = self.field_metadata[category]
+        ans = self.conn.get(
+                'SELECT book FROM books_{tn}_link WHERE {col}=?'.format(
+                    tn=field['table'], col=field['link_column']), (id_,))
+        return set(x[0] for x in ans)
+
     def get_categories(self, sort_on_count=False, ids=None, icon_map=None):
         self.books_list_filter.change([] if not ids else ids)
 

src/calibre/library/field_metadata.py

@@ -374,6 +374,7 @@ class FieldMetadata(dict):
                              'search_terms':[key], 'label':label,
                              'colnum':colnum,      'display':display,
                              'is_custom':True,     'is_category':is_category,
+                             'link_column':'value',
                              'is_editable': is_editable,}
         self._add_search_terms_to_map(key, [key])
         self.custom_label_to_key_map[label] = key

src/calibre/library/server/cache.py

@@ -15,23 +15,27 @@ class Cache(object):
         self._search_cache = OrderedDict()
 
     def search_cache(self, search):
-        old = self._search_cache.get(search, None)
+        old = self._search_cache.pop(search, None)
         if old is None or old[0] <= self.db.last_modified():
             matches = self.db.data.search(search, return_matches=True,
                     ignore_search_restriction=True)
             if not matches:
                 matches = []
             self._search_cache[search] = (utcnow(), frozenset(matches))
-            if len(self._search_cache) > 10:
+            if len(self._search_cache) > 50:
                 self._search_cache.popitem(last=False)
+        else:
+            self._search_cache[search] = old
         return self._search_cache[search][1]
 
 
     def categories_cache(self, restrict_to=frozenset([])):
-        old = self._category_cache.get(frozenset(restrict_to), None)
+        old = self._category_cache.pop(frozenset(restrict_to), None)
         if old is None or old[0] <= self.db.last_modified():
             categories = self.db.get_categories(ids=restrict_to)
             self._category_cache[restrict_to] = (utcnow(), categories)
-            if len(self._category_cache) > 10:
+            if len(self._category_cache) > 20:
                 self._category_cache.popitem(last=False)
+        else:
+            self._category_cache[frozenset(restrict_to)] = old
         return self._category_cache[restrict_to][1]

src/calibre/library/server/opds.py

@@ -18,6 +18,7 @@ from calibre.constants import __appname__
 from calibre.ebooks.metadata import fmt_sidx
 from calibre.library.comments import comments_to_html
 from calibre import guess_type
+from calibre.utils.ordered_dict import OrderedDict
 
 BASE_HREFS = {
         0 : '/stanza',
@@ -31,6 +32,14 @@ def url_for(name, version, **kwargs):
         name += '_'
     return routes.url_for(name+str(version), **kwargs)
 
+def hexlify(x):
+    if isinstance(x, unicode):
+        x = x.encode('utf-8')
+    return binascii.hexlify(x)
+
+def unhexlify(x):
+    return binascii.unhexlify(x).decode('utf-8')
+
 # Vocabulary for building OPDS feeds {{{
 E = ElementMaker(namespace='http://www.w3.org/2005/Atom',
                  nsmap={
@@ -66,7 +75,7 @@ def AUTHOR(name, uri=None):
 SUBTITLE = E.subtitle
 
 def NAVCATALOG_ENTRY(base_href, updated, title, description, query, version=0):
-    href = base_href+'/navcatalog/'+binascii.hexlify(query)
+    href = base_href+'/navcatalog/'+hexlify(query)
     id_ = 'calibre-navcatalog:'+str(hashlib.sha1(href).hexdigest())
     return E.entry(
         TITLE(title),
@@ -90,6 +99,32 @@ def html_to_lxml(raw):
     raw = etree.tostring(root, encoding=None)
     return etree.fromstring(raw)
 
+def CATALOG_ENTRY(item, base_href, version, updated):
+    id_ = 'calibre:category:'+item.name
+    iid = 'N' + item.name
+    if item.id is not None:
+        iid = 'I' + str(item.id)
+    link = NAVLINK(href = base_href + '/' + hexlify(iid))
+    return E.entry(
+            TITLE(item.name),
+            ID(id_),
+            UPDATED(updated),
+            E.content(_('%d books')%item.count, type='text'),
+            link
+            )
+
+def CATALOG_GROUP_ENTRY(item, category, base_href, version, updated):
+    id_ = 'calibre:category-group:'+category+':'+item.text
+    iid = item.text
+    link = NAVLINK(href = base_href + '/' + hexlify(iid))
+    return E.entry(
+            TITLE(item.text),
+            ID(id_),
+            UPDATED(updated),
+            E.content(_('%d books')%item.count, type='text'),
+            link
+            )
+
 def ACQUISITION_ENTRY(item, version, FM, updated):
     title = item[FM['title']]
     if not title:
@@ -225,6 +260,22 @@ class AcquisitionFeed(NavFeed):
         for item in items:
             self.root.append(ACQUISITION_ENTRY(item, version, FM, updated))
 
+class CategoryFeed(NavFeed):
+
+    def __init__(self, items, which, id_, updated, version, offsets, page_url, up_url):
+        NavFeed.__init__(self, id_, updated, version, offsets, page_url, up_url)
+        base_href = self.base_href + '/category/' + hexlify(which)
+        for item in items:
+            self.root.append(CATALOG_ENTRY(item, base_href, version, updated))
+
+class CategoryGroupFeed(NavFeed):
+
+    def __init__(self, items, which, id_, updated, version, offsets, page_url, up_url):
+        NavFeed.__init__(self, id_, updated, version, offsets, page_url, up_url)
+        base_href = self.base_href + '/categorygroup/' + hexlify(which)
+        for item in items:
+            self.root.append(CATALOG_GROUP_ENTRY(item, which, base_href, version, updated))
+
 
 class OPDSOffsets(object):
 
@@ -254,8 +305,13 @@ class OPDSServer(object):
             base_href = BASE_HREFS[version]
             ver = str(version)
             connect('opds_'+ver, base_href, self.opds, version=version)
+            connect('opdst_'+ver, base_href+'/', self.opds, version=version)
             connect('opdsnavcatalog_'+ver, base_href+'/navcatalog/{which}',
                     self.opds_navcatalog, version=version)
+            connect('opdscategory_'+ver, base_href+'/category/{category}/{which}',
+                    self.opds_category, version=version)
+            connect('opdscategorygroup_'+ver, base_href+'/categorygroup/{category}/{which}',
+                    self.opds_category_group, version=version)
             connect('opdssearch_'+ver, base_href+'/search/{query}',
                     self.opds_search, version=version)
 
@@ -269,12 +325,17 @@ class OPDSServer(object):
             sort_by='title', ascending=True, version=0):
         idx = self.db.FIELD_MAP['id']
         ids &= self.get_opds_allowed_ids_for_version(version)
+        if not ids:
+            raise cherrypy.HTTPError(404, 'No books found')
         items = [x for x in self.db.data.iterall() if x[idx] in ids]
         self.sort(items, sort_by, ascending)
         max_items = self.opts.max_opds_items
         offsets = OPDSOffsets(offset, max_items, len(items))
         items = items[offsets.offset:offsets.offset+max_items]
-        return str(AcquisitionFeed(self.db.last_modified(), id_, items, offsets,
+        updated = self.db.last_modified()
+        cherrypy.response.headers['Last-Modified'] = self.last_modified(updated)
+        cherrypy.response.headers['Content-Type'] = 'text/xml'
+        return str(AcquisitionFeed(updated, id_, items, offsets,
             page_url, up_url, version, self.db.FIELD_MAP))
 
     def opds_search(self, query=None, version=0, offset=0):
@@ -309,28 +370,160 @@ class OPDSServer(object):
                 id_='calibre-all:'+sort, sort_by=sort, ascending=ascending,
                 version=version)
 
+    # Categories {{{
+
+    def opds_category_group(self, category=None, which=None, version=0, offset=0):
+        try:
+            offset = int(offset)
+            version = int(version)
+        except:
+            raise cherrypy.HTTPError(404, 'Not found')
+
+        if not which or not category or version not in BASE_HREFS:
+            raise cherrypy.HTTPError(404, 'Not found')
+
+        categories = self.categories_cache(
+                self.get_opds_allowed_ids_for_version(version))
+        page_url = url_for('opdscategorygroup', version, category=category, which=which)
+
+        category = unhexlify(category)
+        if category not in categories:
+            raise cherrypy.HTTPError(404, 'Category %r not found'%which)
+        which = unhexlify(which)
+        owhich = hexlify('N'+which)
+        up_url = url_for('opdsnavcatalog', version, which=owhich)
+        items = categories[category]
+        items = [x for x in items if x.name.startswith(which)]
+        if not items:
+            raise cherrypy.HTTPError(404, 'No items in group %r:%r'%(category,
+                which))
+        updated = self.db.last_modified()
+
+        id_ = 'calibre-category-group-feed:'+category+':'+which
+
+        max_items = self.opts.max_opds_items
+        offsets = OPDSOffsets(offset, max_items, len(items))
+        items = list(items)[offsets.offset:offsets.offset+max_items]
+
+        cherrypy.response.headers['Last-Modified'] = self.last_modified(updated)
+        cherrypy.response.headers['Content-Type'] = 'text/xml'
+
+        return str(CategoryFeed(items, category, id_, updated, version, offsets,
+            page_url, up_url))
+
+
     def opds_navcatalog(self, which=None, version=0, offset=0):
-        version = int(version)
+        try:
+            offset = int(offset)
+            version = int(version)
+        except:
+            raise cherrypy.HTTPError(404, 'Not found')
+
         if not which or version not in BASE_HREFS:
             raise cherrypy.HTTPError(404, 'Not found')
         page_url = url_for('opdsnavcatalog', version, which=which)
         up_url = url_for('opds', version)
-        which = binascii.unhexlify(which)
+        which = unhexlify(which)
         type_ = which[0]
         which = which[1:]
         if type_ == 'O':
             return self.get_opds_all_books(which, page_url, up_url,
                     version=version, offset=offset)
         elif type_ == 'N':
-            return self.get_opds_navcatalog(which, version=version, offset=offset)
+            return self.get_opds_navcatalog(which, page_url, up_url,
+                    version=version, offset=offset)
         raise cherrypy.HTTPError(404, 'Not found')
 
-    def get_opds_navcatalog(self, which, version=0, offset=0):
+    def get_opds_navcatalog(self, which, page_url, up_url, version=0, offset=0):
         categories = self.categories_cache(
                 self.get_opds_allowed_ids_for_version(version))
         if which not in categories:
             raise cherrypy.HTTPError(404, 'Category %r not found'%which)
 
+        items = categories[which]
+        updated = self.db.last_modified()
+
+        id_ = 'calibre-category-feed:'+which
+
+        MAX_ITEMS = 50
+
+        if len(items) <= MAX_ITEMS:
+            max_items = self.opts.max_opds_items
+            offsets = OPDSOffsets(offset, max_items, len(items))
+            items = list(items)[offsets.offset:offsets.offset+max_items]
+            ans = CategoryFeed(items, which, id_, updated, version, offsets,
+                page_url, up_url)
+        else:
+            class Group:
+                def __init__(self, text, count):
+                    self.text, self.count = text, count
+
+            starts = set([x.name[0] for x in items])
+            if len(starts) > MAX_ITEMS:
+                starts = set([x.name[:2] for x in items])
+            category_groups = OrderedDict()
+            for x in sorted(starts, cmp=lambda x,y:cmp(x.lower(), y.lower())):
+                category_groups[x] = len([y for y in items if
+                    y.name.startswith(x)])
+            items = [Group(x, y) for x, y in category_groups.items()]
+            max_items = self.opts.max_opds_items
+            offsets = OPDSOffsets(offset, max_items, len(items))
+            items = items[offsets.offset:offsets.offset+max_items]
+            ans = CategoryGroupFeed(items, which, id_, updated, version, offsets,
+                page_url, up_url)
+
+        cherrypy.response.headers['Last-Modified'] = self.last_modified(updated)
+        cherrypy.response.headers['Content-Type'] = 'text/xml'
+
+        return str(ans)
+
+    def opds_category(self, category=None, which=None, version=0, offset=0):
+        try:
+            offset = int(offset)
+            version = int(version)
+        except:
+            raise cherrypy.HTTPError(404, 'Not found')
+
+        if not which or not category or version not in BASE_HREFS:
+            raise cherrypy.HTTPError(404, 'Not found')
+        page_url = url_for('opdscategory', version, which=which,
+                category=category)
+        up_url = url_for('opdsnavcatalog', version, which=category)
+
+        which, category = unhexlify(which), unhexlify(category)
+        type_ = which[0]
+        which = which[1:]
+        if type_ == 'I':
+            try:
+                which = int(which)
+            except:
+                raise cherrypy.HTTPError(404, 'Tag %r not found'%which)
+
+        categories = self.categories_cache(
+                self.get_opds_allowed_ids_for_version(version))
+        if category not in categories:
+            raise cherrypy.HTTPError(404, 'Category %r not found'%which)
+
+        if category == 'search':
+            try:
+                ids = self.search_cache(which)
+            except:
+                raise cherrypy.HTTPError(404, 'Search: %r not understood'%which)
+            return self.get_opds_acquisition_feed(ids, offset, page_url,
+                    up_url, 'calibre-search:'+which,
+                    version=version)
+
+        if type_ != 'I':
+            raise cherrypy.HTTPError(404, 'Non id categories not supported')
+
+        ids = self.db.get_books_for_category(category, which)
+        sort_by = 'series' if category == 'series' else 'title'
+
+        return self.get_opds_acquisition_feed(ids, offset, page_url,
+                up_url, 'calibre-category:'+category+':'+str(which),
+                version=version, sort_by=sort_by)
+
+    # }}}
 
 
     def opds(self, version=0):