diff --git a/src/calibre/srv/ajax.py b/src/calibre/srv/ajax.py
index 231eb584cc..a3d0fc2e6e 100644
--- a/src/calibre/srv/ajax.py
+++ b/src/calibre/srv/ajax.py
@@ -168,7 +168,7 @@ def book(ctx, rd, book_id, library_id):
                     book_id = None
             except Exception:
                 book_id = None
-        if book_id is None or book_id not in ctx.allowed_book_ids(rd, db):
+        if book_id is None or not db.has_id(book_id):
             raise HTTPNotFound('Book with id %r does not exist' % oid)
         category_urls = rd.query.get('category_urls', 'true').lower()
         device_compatible = rd.query.get('device_compatible', 'false').lower()
@@ -216,9 +216,8 @@ def books(ctx, rd, library_id):
         device_compatible = rd.query.get('device_compatible', 'false').lower() == 'true'
         device_for_template = rd.query.get('device_for_template', None)
         ans = {}
-        restricted_to = ctx.allowed_book_ids(rd, db)
         for book_id in ids:
-            if book_id not in restricted_to:
+            if not db.has_id(book_id):
                 ans[book_id] = None
                 continue
             data, lm = book_to_json(
@@ -251,7 +250,7 @@ def categories(ctx, rd, library_id):
     db = get_db(ctx, rd, library_id)
     with db.safe_read_lock:
         ans = {}
-        categories = ctx.get_categories(rd, db)
+        categories = ctx.get_categories(rd, db, vl=rd.query.get('vl') or '')
         category_meta = db.field_metadata
         library_id = db.server_library_id
 
@@ -484,7 +483,7 @@ def books_in(ctx, rd, encoded_category, encoded_item, library_id):
             raise HTTPNotFound('%s is not a valid sort field'%sort)
 
         if dname in ('allbooks', 'newest'):
-            ids = ctx.allowed_book_ids(rd, db)
+            ids = db.all_book_ids()
         elif dname == 'search':
             try:
                 ids = ctx.search(rd, db, 'search:"%s"'%ditem)
@@ -498,7 +497,7 @@ def books_in(ctx, rd, encoded_category, encoded_item, library_id):
 
             if dname == 'news':
                 dname = 'tags'
-            ids = db.get_books_for_category(dname, cid).intersection(ctx.allowed_book_ids(rd, db))
+            ids = db.get_books_for_category(dname, cid)
 
         ids = db.multisort(fields=[(sfield, sort_order == 'asc')], ids_to_sort=ids)
         total_num = len(ids)
@@ -528,7 +527,7 @@ def books_in(ctx, rd, encoded_category, encoded_item, library_id):
 # Search {{{
 
 
-def search_result(ctx, rd, db, query, num, offset, sort, sort_order):
+def search_result(ctx, rd, db, query, num, offset, sort, sort_order, vl=''):
     multisort = [(sanitize_sort_field_name(db.field_metadata, s), ensure_val(o, 'asc', 'desc') == 'asc')
                  for s, o in zip(sort.split(','), cycle(sort_order.split(',')))]
     skeys = db.field_metadata.sortable_field_keys()
@@ -536,10 +535,7 @@ def search_result(ctx, rd, db, query, num, offset, sort, sort_order):
         if sfield not in skeys:
             raise HTTPNotFound('%s is not a valid sort field'%sort)
 
-    if not query:
-        ids = ctx.allowed_book_ids(rd, db)
-    else:
-        ids = ctx.search(rd, db, query)
+    ids = ctx.search(rd, db, query, vl=vl)
     ids = db.multisort(fields=multisort, ids_to_sort=ids)
     total_num = len(ids)
     ids = ids[offset:offset+num]
@@ -558,13 +554,13 @@ def search(ctx, rd, library_id):
     '''
     Return the books (as list of ids) matching the specified search query.
 
-    Optional: ?num=100&offset=0&sort=title&sort_order=asc&query=
+    Optional: ?num=100&offset=0&sort=title&sort_order=asc&query=&vl=
     '''
     db = get_db(ctx, rd, library_id)
     query = rd.query.get('query')
     num, offset = get_pagination(rd.query)
     with db.safe_read_lock:
-        return search_result(ctx, rd, db, query, num, offset, rd.query.get('sort', 'title'), rd.query.get('sort_order', 'asc'))
+        return search_result(ctx, rd, db, query, num, offset, rd.query.get('sort', 'title'), rd.query.get('sort_order', 'asc'), rd.query.get('vl') or '')
 
 # }}}
 
diff --git a/src/calibre/srv/books.py b/src/calibre/srv/books.py
index 202624ec5a..70252688c2 100644
--- a/src/calibre/srv/books.py
+++ b/src/calibre/srv/books.py
@@ -130,7 +130,7 @@ def book_manifest(ctx, rd, book_id, fmt):
     force_reload = rd.query.get('force_reload') == '1'
     if plugin_for_input_format(fmt) is None:
         raise HTTPNotFound('The format %s cannot be viewed' % fmt.upper())
-    if book_id not in ctx.allowed_book_ids(rd, db):
+    if not db.has_id(book_id):
         raise HTTPNotFound('No book with id: %s in library: %s' % (book_id, library_id))
     with db.safe_read_lock:
         fm = db.format_metadata(book_id, fmt)
@@ -166,7 +166,7 @@ def book_manifest(ctx, rd, book_id, fmt):
 @endpoint('/book-file/{book_id}/{fmt}/{size}/{mtime}/{+name}', types={'book_id':int, 'size':int, 'mtime':int})
 def book_file(ctx, rd, book_id, fmt, size, mtime, name):
     db, library_id = get_library_data(ctx, rd)[:2]
-    if book_id not in ctx.allowed_book_ids(rd, db):
+    if not db.has_id(book_id):
         raise HTTPNotFound('No book with id: %s in library: %s' % (book_id, library_id))
     bhash = book_hash(db.library_id, book_id, fmt, size, mtime)
     base = abspath(os.path.join(books_cache_dir(), 'f'))
@@ -190,14 +190,13 @@ def get_last_read_position(ctx, rd, library_id, which):
     db = get_db(ctx, rd, library_id)
     user = rd.username or None
     ans = {}
-    allowed_book_ids = ctx.allowed_book_ids(rd, db)
     for item in which.split('_'):
         book_id, fmt = item.partition('-')[::2]
         try:
             book_id = int(book_id)
         except Exception:
             continue
-        if book_id not in allowed_book_ids:
+        if not db.has_id(book_id):
             continue
         key = '{}:{}'.format(book_id, fmt)
         ans[key] = db.get_last_read_positions(book_id, fmt, user)
@@ -208,8 +207,7 @@ def get_last_read_position(ctx, rd, library_id, which):
 def set_last_read_position(ctx, rd, library_id, book_id, fmt):
     db = get_db(ctx, rd, library_id)
     user = rd.username or None
-    allowed_book_ids = ctx.allowed_book_ids(rd, db)
-    if book_id not in allowed_book_ids:
+    if not db.has_id(book_id):
         raise HTTPNotFound('No book with id {} found'.format(book_id))
     try:
         data = jsonlib.load(rd.request_body_file)
diff --git a/src/calibre/srv/code.py b/src/calibre/srv/code.py
index 55f7f71b75..68739112b8 100644
--- a/src/calibre/srv/code.py
+++ b/src/calibre/srv/code.py
@@ -3,7 +3,7 @@
 # License: GPLv3 Copyright: 2015, Kovid Goyal <kovid at kovidgoyal.net>
 
 from __future__ import (unicode_literals, division, absolute_import, print_function)
-import hashlib, random, zipfile, shutil, sys
+import hashlib, random, zipfile, shutil, sys, cPickle
 from json import load as load_json_file
 
 from calibre import as_unicode
@@ -11,7 +11,7 @@ from calibre.customize.ui import available_input_formats
 from calibre.db.view import sanitize_sort_field_name
 from calibre.srv.ajax import search_result
 from calibre.srv.errors import HTTPNotFound, HTTPBadRequest
-from calibre.srv.metadata import book_as_json, categories_as_json, icon_map
+from calibre.srv.metadata import book_as_json, categories_as_json, icon_map, categories_settings
 from calibre.srv.routes import endpoint, json
 from calibre.srv.utils import get_library_data, get_use_roman
 from calibre.utils.config import prefs, tweaks
@@ -70,7 +70,7 @@ def get_basic_query_data(ctx, rd):
                 sorts.append(s), orders.append(o)
     if not sorts:
         sorts, orders = ['timestamp'], ['desc']
-    return library_id, db, sorts, orders
+    return library_id, db, sorts, orders, rd.query.get('vl') or ''
 
 
 _cached_translations = None
@@ -125,18 +125,18 @@ def update_interface_data(ctx, rd):
     return basic_interface_data(ctx, rd)
 
 
-def get_library_init_data(ctx, rd, db, num, sorts, orders):
+def get_library_init_data(ctx, rd, db, num, sorts, orders, vl):
     ans = {}
     with db.safe_read_lock:
         try:
             ans['search_result'] = search_result(
                 ctx, rd, db,
                 rd.query.get('search', ''), num, 0, ','.join(sorts),
-                ','.join(orders)
+                ','.join(orders), vl
             )
         except ParseException:
             ans['search_result'] = search_result(
-                ctx, rd, db, '', num, 0, ','.join(sorts), ','.join(orders)
+                ctx, rd, db, '', num, 0, ','.join(sorts), ','.join(orders), vl
             )
         sf = db.field_metadata.ui_sortable_field_keys()
         sf.pop('ondevice', None)
@@ -146,6 +146,7 @@ def get_library_init_data(ctx, rd, db, num, sorts, orders):
             key=lambda (field, name): sort_key(name)
         )
         ans['field_metadata'] = db.field_metadata.all_metadata()
+        ans['virtual_libraries'] = db._pref('virtual_libraries', {})
         mdata = ans['metadata'] = {}
         try:
             extra_books = set(
@@ -168,15 +169,15 @@ def books(ctx, rd):
     Get data to create list of books
 
     Optional: ?num=50&sort=timestamp.desc&library_id=<default library>
-              &search=''&extra_books=''
+              &search=''&extra_books=''&vl=''
     '''
     ans = {}
     try:
         num = int(rd.query.get('num', DEFAULT_NUMBER_OF_BOOKS))
     except Exception:
         raise HTTPNotFound('Invalid number of books: %r' % rd.query.get('num'))
-    library_id, db, sorts, orders = get_basic_query_data(ctx, rd)
-    ans = get_library_init_data(ctx, rd, db, num, sorts, orders)
+    library_id, db, sorts, orders, vl = get_basic_query_data(ctx, rd)
+    ans = get_library_init_data(ctx, rd, db, num, sorts, orders, vl)
     ans['library_id'] = library_id
     return ans
 
@@ -187,7 +188,7 @@ def interface_data(ctx, rd):
     Return the data needed to create the server UI as well as a list of books.
 
     Optional: ?num=50&sort=timestamp.desc&library_id=<default library>
-              &search=''&extra_books=''
+              &search=''&extra_books=''&vl=''
     '''
     ans = basic_interface_data(ctx, rd)
     ud = {}
@@ -201,13 +202,13 @@ def interface_data(ctx, rd):
         usort = ud.get('sort')
         if usort:
             rd.query.set('sort', usort)
-    ans['library_id'], db, sorts, orders = get_basic_query_data(ctx, rd)
+    ans['library_id'], db, sorts, orders, vl = get_basic_query_data(ctx, rd)
     ans['user_session_data'] = ud
     try:
         num = int(rd.query.get('num', DEFAULT_NUMBER_OF_BOOKS))
     except Exception:
         raise HTTPNotFound('Invalid number of books: %r' % rd.query.get('num'))
-    ans.update(get_library_init_data(ctx, rd, db, num, sorts, orders))
+    ans.update(get_library_init_data(ctx, rd, db, num, sorts, orders, vl))
     return ans
 
 
@@ -271,9 +272,9 @@ def get_books(ctx, rd):
     '''
     Get books for the specified query
 
-    Optional: ?library_id=<default library>&num=50&sort=timestamp.desc&search=''
+    Optional: ?library_id=<default library>&num=50&sort=timestamp.desc&search=''&vl=''
     '''
-    library_id, db, sorts, orders = get_basic_query_data(ctx, rd)
+    library_id, db, sorts, orders, vl = get_basic_query_data(ctx, rd)
     try:
         num = int(rd.query.get('num', DEFAULT_NUMBER_OF_BOOKS))
     except Exception:
@@ -285,7 +286,7 @@ def get_books(ctx, rd):
     with db.safe_read_lock:
         try:
             ans['search_result'] = search_result(
-                ctx, rd, db, searchq, num, 0, ','.join(sorts), ','.join(orders)
+                ctx, rd, db, searchq, num, 0, ','.join(sorts), ','.join(orders), vl
             )
         except ParseException as err:
             # This must not be translated as it is used by the front end to
@@ -306,16 +307,13 @@ def book_metadata(ctx, rd, book_id):
     Optional: ?library_id=<default library>
     '''
     library_id, db = get_basic_query_data(ctx, rd)[:2]
-    book_ids = ctx.allowed_book_ids(rd, db)
 
     def notfound():
-        raise HTTPNotFound(_('No book with id: %d in library') % book_id)
+        raise HTTPNotFound(_('No book with id: {} in library: {}').format(book_id, library_id))
 
-    if not book_ids:
-        notfound()
     if not book_id:
-        book_id = random.choice(tuple(book_ids))
-    elif book_id not in book_ids:
+        book_id = random.choice(tuple(db.all_book_ids()))
+    elif not db.has_id(book_id):
         notfound()
     data = book_as_json(db, book_id)
     if data is None:
@@ -329,14 +327,15 @@ def tag_browser(ctx, rd):
     '''
     Get the Tag Browser serialized as JSON
     Optional: ?library_id=<default library>&sort_tags_by=name&partition_method=first letter
-              &collapse_at=25&dont_collapse=&hide_empty_categories=
+              &collapse_at=25&dont_collapse=&hide_empty_categories=&vl=''
     '''
     db, library_id = get_library_data(ctx, rd)[:2]
-    etag = '%s||%s||%s' % (db.last_modified(), rd.username, library_id)
-    etag = hashlib.sha1(etag.encode('utf-8')).hexdigest()
+    opts = categories_settings(rd.query, db)
+    vl = rd.query.get('vl') or ''
+    etag = cPickle.dumps([db.last_modified().isoformat(), rd.username, library_id, vl, list(opts)], -1)
+    etag = hashlib.sha1(etag).hexdigest()
 
     def generate():
-        db, library_id = get_library_data(ctx, rd)[:2]
-        return json(ctx, rd, tag_browser, categories_as_json(ctx, rd, db))
+        return json(ctx, rd, tag_browser, categories_as_json(ctx, rd, db, opts, vl))
 
     return rd.etagged_dynamic_response(etag, generate)
diff --git a/src/calibre/srv/content.py b/src/calibre/srv/content.py
index 5d667ebd2a..a0c3e4b8a4 100644
--- a/src/calibre/srv/content.py
+++ b/src/calibre/srv/content.py
@@ -276,7 +276,7 @@ def get(ctx, rd, what, book_id, library_id):
     if db is None:
         raise HTTPNotFound('Library %r not found' % library_id)
     with db.safe_read_lock:
-        if book_id not in ctx.allowed_book_ids(rd, db):
+        if not db.has_id(book_id):
             raise HTTPNotFound('Book with id %r does not exist' % book_id)
         library_id = db.server_library_id  # in case library_id was None
         if what == 'thumb':
diff --git a/src/calibre/srv/handler.py b/src/calibre/srv/handler.py
index e0412ebb46..0f49dab70b 100644
--- a/src/calibre/srv/handler.py
+++ b/src/calibre/srv/handler.py
@@ -71,16 +71,9 @@ class Context(object):
             raise HTTPForbidden('The user {} is not allowed to access any libraries on this server'.format(data.username))
         return dict(allowed_libraries), next(allowed_libraries.iterkeys())
 
-    def allowed_book_ids(self, data, db):
-        with self.lock:
-            ans = data.allowed_book_ids.get(db.server_library_id)
-            if ans is None:
-                ans = data.allowed_book_ids[db.server_library_id] = db.all_book_ids()
-            return ans
-
-    def get_categories(self, data, db, restrict_to_ids=None, sort='name', first_letter_sort=True):
+    def get_categories(self, data, db, restrict_to_ids=None, sort='name', first_letter_sort=True, vl=''):
         if restrict_to_ids is None:
-            restrict_to_ids = self.allowed_book_ids(data, db)
+            restrict_to_ids = db.books_in_virtual_library(vl)
         key = (restrict_to_ids, sort, first_letter_sort)
         with self.lock:
             cache = self.library_broker.category_caches[db.server_library_id]
@@ -94,9 +87,9 @@ class Context(object):
                 cache[key] = old
             return old[1]
 
-    def get_tag_browser(self, data, db, opts, render, restrict_to_ids=None):
+    def get_tag_browser(self, data, db, opts, render, restrict_to_ids=None, vl=''):
         if restrict_to_ids is None:
-            restrict_to_ids = self.allowed_book_ids(data, db)
+            restrict_to_ids = db.books_in_virtual_library(vl)
         key = (restrict_to_ids, opts)
         with self.lock:
             cache = self.library_broker.category_caches[db.server_library_id]
@@ -113,15 +106,14 @@ class Context(object):
                 cache[key] = old
             return old[1]
 
-    def search(self, data, db, query, restrict_to_ids=None):
-        if restrict_to_ids is None:
-            restrict_to_ids = self.allowed_book_ids(data, db)
+    def search(self, data, db, query, restrict_to_ids=None, vl=''):
         with self.lock:
             cache = self.library_broker.search_caches[db.server_library_id]
-            key = (query, restrict_to_ids)
+            vl = db.pref('virtual_libraries', {}).get(vl) or ''
+            key = query, restrict_to_ids, vl
             old = cache.pop(key, None)
             if old is None or old[0] < db.clear_search_cache_count:
-                matches = db.search(query, book_ids=restrict_to_ids)
+                matches = db.search(query, restriction=vl, book_ids=restrict_to_ids)
                 cache[key] = old = (db.clear_search_cache_count, matches)
                 if len(cache) > self.SEARCH_CACHE_SIZE:
                     cache.popitem(last=False)
diff --git a/src/calibre/srv/http_response.py b/src/calibre/srv/http_response.py
index 5b87e9cc9e..5159b08cff 100644
--- a/src/calibre/srv/http_response.py
+++ b/src/calibre/srv/http_response.py
@@ -224,7 +224,6 @@ class RequestData(object):  # {{{
         self.lang_code = self.gettext_func = self.ngettext_func = None
         self.set_translator(self.get_preferred_language())
         self.tdir = tdir
-        self.allowed_book_ids = {}
 
     def generate_static_output(self, name, generator, content_type='text/html; charset=UTF-8'):
         ans = self.static_cache.get(name)
diff --git a/src/calibre/srv/metadata.py b/src/calibre/srv/metadata.py
index b1445853a9..d69c13513a 100644
--- a/src/calibre/srv/metadata.py
+++ b/src/calibre/srv/metadata.py
@@ -39,6 +39,7 @@ def encode_datetime(dateval):
         return None
     return isoformat(dateval)
 
+
 empty_val = ((), '', {})
 
 
@@ -84,6 +85,7 @@ def book_as_json(db, book_id):
             ans['lang_names'] = {l:calibre_langcode_to_name(l) for l in langs}
     return ans
 
+
 _include_fields = frozenset(Tag.__slots__) - frozenset({
     'state', 'is_editable', 'is_searchable', 'original_name', 'use_sort_as_name', 'is_hierarchical'
 })
@@ -133,6 +135,7 @@ def category_item_as_json(x, clear_rating=False):
         del ans['avg_rating']
     return ans
 
+
 CategoriesSettings = namedtuple(
     'CategoriesSettings', 'dont_collapse collapse_model collapse_at sort_by'
     ' template using_hierarchy grouped_search_terms hidden_categories hide_empty_categories')
@@ -162,6 +165,7 @@ class GroupedSearchTerms(object):
         except AttributeError:
             return False
 
+
 _icon_map = None
 _icon_map_lock = Lock()
 
@@ -289,6 +293,7 @@ def build_first_letter_list(category_items):
         cl_list[idx] = last_c
     return cl_list
 
+
 categories_with_ratings = {'authors', 'series', 'publisher', 'tags'}
 
 
@@ -528,9 +533,8 @@ def render_categories(opts, db, category_data):
     return {'root':root, 'item_map': items}
 
 
-def categories_as_json(ctx, rd, db):
-    opts = categories_settings(rd.query, db)
-    return ctx.get_tag_browser(rd, db, opts, partial(render_categories, opts))
+def categories_as_json(ctx, rd, db, opts, vl):
+    return ctx.get_tag_browser(rd, db, opts, partial(render_categories, opts), vl='')
 
 # Test tag browser {{{
 
diff --git a/src/calibre/srv/opds.py b/src/calibre/srv/opds.py
index 2af13c1dc8..944513e469 100644
--- a/src/calibre/srv/opds.py
+++ b/src/calibre/srv/opds.py
@@ -56,6 +56,7 @@ def format_tag_string(tags, sep, joinval=', '):
     tlist.sort(key=sort_key)
     return joinval.join(tlist) if tlist else ''
 
+
 # Vocabulary for building OPDS feeds {{{
 DC_NS = 'http://purl.org/dc/terms/'
 E = ElementMaker(namespace='http://www.w3.org/2005/Atom',
@@ -75,6 +76,7 @@ ICON    = E.icon
 def UPDATED(dt, *args, **kwargs):
     return E.updated(as_utc(dt).strftime('%Y-%m-%dT%H:%M:%S+00:00'), *args, **kwargs)
 
+
 LINK = partial(E.link, type='application/atom+xml')
 NAVLINK = partial(E.link,
         type='application/atom+xml;type=feed;profile=opds-catalog')
@@ -93,6 +95,7 @@ def AUTHOR(name, uri=None):
         args.append(E.uri(uri))
     return E.author(*args)
 
+
 SUBTITLE = E.subtitle
 
 
@@ -107,6 +110,7 @@ def NAVCATALOG_ENTRY(url_for, updated, title, description, query):
         NAVLINK(href=href)
     )
 
+
 START_LINK = partial(NAVLINK, rel='start')
 UP_LINK = partial(NAVLINK, rel='up')
 FIRST_LINK = partial(NAVLINK, rel='first')
@@ -363,8 +367,8 @@ class RequestContext(object):
         ans += '?' + urlencode(q)
         return ans
 
-    def allowed_book_ids(self):
-        return self.ctx.allowed_book_ids(self.rd, self.db)
+    def all_book_ids(self):
+        return self.db.all_book_ids()
 
     @property
     def outheaders(self):
@@ -410,7 +414,7 @@ def get_all_books(rc, which, page_url, up_url, offset=0):
     ascending = which == 'title'
     feed_title = {'newest':_('Newest'), 'title': _('Title')}.get(which, which)
     feed_title = default_feed_title + ' :: ' + _('By %s') % feed_title
-    ids = rc.allowed_book_ids()
+    ids = rc.all_book_ids()
     return get_acquisition_feed(rc, ids, offset, page_url, up_url,
             id_='calibre-all:'+sort, sort_by=sort, ascending=ascending,
             feed_title=feed_title)
diff --git a/src/calibre/srv/tests/ajax.py b/src/calibre/srv/tests/ajax.py
index 4d659056d3..732ac3c5d2 100644
--- a/src/calibre/srv/tests/ajax.py
+++ b/src/calibre/srv/tests/ajax.py
@@ -53,6 +53,7 @@ class ContentTest(LibraryBaseTest):
         'Test /ajax/categories and /ajax/search'
         with self.create_server() as server:
             db = server.handler.router.ctx.library_broker.get(None)
+            db.set_pref('virtual_libraries', {'1':'title:"=Title One"'})
             conn = server.connect()
             request = partial(make_request, conn)
 
@@ -74,4 +75,6 @@ class ContentTest(LibraryBaseTest):
             r, data = request('/search?' + urlencode({'query': 'tags:"=Tag One"'}))
             self.ae(r.status, httplib.OK)
             self.ae(set(data['book_ids']), {1, 2})
+            r, data = request('/search?' + urlencode({'query': 'tags:"=Tag One"', 'vl':'1'}))
+            self.ae(set(data['book_ids']), {2})
     # }}}