Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-09 03:04:10 -04:00
Code Issues Packages Projects Releases Wiki Activity

Remove the allowed_books_id API from the server

Browse Source 
Better performance and also allows implementing virtual library support
in the future.
This commit is contained in:
Kovid Goyal 2017-04-23 15:54:54 +05:30
parent 303a5cca6a
commit 147b16e4de
No known key found for this signature in database
GPG Key ID: 06BC317B515ACE7C
9 changed files with 64 additions and 69 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
src/calibre/srv/ajax.py
View File

@ -168,7 +168,7 @@ def book(ctx, rd, book_id, library_id):
book_id = None
except Exception:
book_id = None
if book_id is None or book_id not in ctx.allowed_book_ids(rd, db):
if book_id is None or not db.has_id(book_id):
raise HTTPNotFound('Book with id %r does not exist' % oid)
category_urls = rd.query.get('category_urls', 'true').lower()
device_compatible = rd.query.get('device_compatible', 'false').lower()
@ -216,9 +216,8 @@ def books(ctx, rd, library_id):
device_compatible = rd.query.get('device_compatible', 'false').lower() == 'true'
device_for_template = rd.query.get('device_for_template', None)
ans = {}
restricted_to = ctx.allowed_book_ids(rd, db)
for book_id in ids:
if book_id not in restricted_to:
if not db.has_id(book_id):
ans[book_id] = None
continue
data, lm = book_to_json(
@ -251,7 +250,7 @@ def categories(ctx, rd, library_id):
db = get_db(ctx, rd, library_id)
with db.safe_read_lock:
ans = {}
categories = ctx.get_categories(rd, db)
categories = ctx.get_categories(rd, db, vl=rd.query.get('vl') or '')
category_meta = db.field_metadata
library_id = db.server_library_id
@ -484,7 +483,7 @@ def books_in(ctx, rd, encoded_category, encoded_item, library_id):
raise HTTPNotFound('%s is not a valid sort field'%sort)
if dname in ('allbooks', 'newest'):
ids = ctx.allowed_book_ids(rd, db)
ids = db.all_book_ids()
elif dname == 'search':
try:
ids = ctx.search(rd, db, 'search:"%s"'%ditem)
@ -498,7 +497,7 @@ def books_in(ctx, rd, encoded_category, encoded_item, library_id):
if dname == 'news':
dname = 'tags'
ids = db.get_books_for_category(dname, cid).intersection(ctx.allowed_book_ids(rd, db))
ids = db.get_books_for_category(dname, cid)
ids = db.multisort(fields=[(sfield, sort_order == 'asc')], ids_to_sort=ids)
total_num = len(ids)
@ -528,7 +527,7 @@ def books_in(ctx, rd, encoded_category, encoded_item, library_id):
# Search {{{
def search_result(ctx, rd, db, query, num, offset, sort, sort_order):
def search_result(ctx, rd, db, query, num, offset, sort, sort_order, vl=''):
multisort = [(sanitize_sort_field_name(db.field_metadata, s), ensure_val(o, 'asc', 'desc') == 'asc')
for s, o in zip(sort.split(','), cycle(sort_order.split(',')))]
skeys = db.field_metadata.sortable_field_keys()
@ -536,10 +535,7 @@ def search_result(ctx, rd, db, query, num, offset, sort, sort_order):
if sfield not in skeys:
raise HTTPNotFound('%s is not a valid sort field'%sort)
if not query:
ids = ctx.allowed_book_ids(rd, db)
else:
ids = ctx.search(rd, db, query)
ids = ctx.search(rd, db, query, vl=vl)
ids = db.multisort(fields=multisort, ids_to_sort=ids)
total_num = len(ids)
ids = ids[offset:offset+num]
@ -558,13 +554,13 @@ def search(ctx, rd, library_id):
'''
Return the books (as list of ids) matching the specified search query.
Optional: ?num=100&offset=0&sort=title&sort_order=asc&query=
Optional: ?num=100&offset=0&sort=title&sort_order=asc&query=&vl=
'''
db = get_db(ctx, rd, library_id)
query = rd.query.get('query')
num, offset = get_pagination(rd.query)
with db.safe_read_lock:
return search_result(ctx, rd, db, query, num, offset, rd.query.get('sort', 'title'), rd.query.get('sort_order', 'asc'))
return search_result(ctx, rd, db, query, num, offset, rd.query.get('sort', 'title'), rd.query.get('sort_order', 'asc'), rd.query.get('vl') or '')
# }}}

10
src/calibre/srv/books.py
View File

@ -130,7 +130,7 @@ def book_manifest(ctx, rd, book_id, fmt):
force_reload = rd.query.get('force_reload') == '1'
if plugin_for_input_format(fmt) is None:
raise HTTPNotFound('The format %s cannot be viewed' % fmt.upper())
if book_id not in ctx.allowed_book_ids(rd, db):
if not db.has_id(book_id):
raise HTTPNotFound('No book with id: %s in library: %s' % (book_id, library_id))
with db.safe_read_lock:
fm = db.format_metadata(book_id, fmt)
@ -166,7 +166,7 @@ def book_manifest(ctx, rd, book_id, fmt):
@endpoint('/book-file/{book_id}/{fmt}/{size}/{mtime}/{+name}', types={'book_id':int, 'size':int, 'mtime':int})
def book_file(ctx, rd, book_id, fmt, size, mtime, name):
db, library_id = get_library_data(ctx, rd)[:2]
if book_id not in ctx.allowed_book_ids(rd, db):
if not db.has_id(book_id):
raise HTTPNotFound('No book with id: %s in library: %s' % (book_id, library_id))
bhash = book_hash(db.library_id, book_id, fmt, size, mtime)
base = abspath(os.path.join(books_cache_dir(), 'f'))
@ -190,14 +190,13 @@ def get_last_read_position(ctx, rd, library_id, which):
db = get_db(ctx, rd, library_id)
user = rd.username or None
ans = {}
allowed_book_ids = ctx.allowed_book_ids(rd, db)
for item in which.split('_'):
book_id, fmt = item.partition('-')[::2]
try:
book_id = int(book_id)
except Exception:
continue
if book_id not in allowed_book_ids:
if not db.has_id(book_id):
continue
key = '{}:{}'.format(book_id, fmt)
ans[key] = db.get_last_read_positions(book_id, fmt, user)
@ -208,8 +207,7 @@ def get_last_read_position(ctx, rd, library_id, which):
def set_last_read_position(ctx, rd, library_id, book_id, fmt):
db = get_db(ctx, rd, library_id)
user = rd.username or None
allowed_book_ids = ctx.allowed_book_ids(rd, db)
if book_id not in allowed_book_ids:
if not db.has_id(book_id):
raise HTTPNotFound('No book with id {} found'.format(book_id))
try:
data = jsonlib.load(rd.request_body_file)

51
src/calibre/srv/code.py
View File

@ -3,7 +3,7 @@
# License: GPLv3 Copyright: 2015, Kovid Goyal <kovid at kovidgoyal.net>
from __future__ import (unicode_literals, division, absolute_import, print_function)
import hashlib, random, zipfile, shutil, sys
import hashlib, random, zipfile, shutil, sys, cPickle
from json import load as load_json_file
from calibre import as_unicode
@ -11,7 +11,7 @@ from calibre.customize.ui import available_input_formats
from calibre.db.view import sanitize_sort_field_name
from calibre.srv.ajax import search_result
from calibre.srv.errors import HTTPNotFound, HTTPBadRequest
from calibre.srv.metadata import book_as_json, categories_as_json, icon_map
from calibre.srv.metadata import book_as_json, categories_as_json, icon_map, categories_settings
from calibre.srv.routes import endpoint, json
from calibre.srv.utils import get_library_data, get_use_roman
from calibre.utils.config import prefs, tweaks
@ -70,7 +70,7 @@ def get_basic_query_data(ctx, rd):
sorts.append(s), orders.append(o)
if not sorts:
sorts, orders = ['timestamp'], ['desc']
return library_id, db, sorts, orders
return library_id, db, sorts, orders, rd.query.get('vl') or ''
_cached_translations = None
@ -125,18 +125,18 @@ def update_interface_data(ctx, rd):
return basic_interface_data(ctx, rd)
def get_library_init_data(ctx, rd, db, num, sorts, orders):
def get_library_init_data(ctx, rd, db, num, sorts, orders, vl):
ans = {}
with db.safe_read_lock:
try:
ans['search_result'] = search_result(
ctx, rd, db,
rd.query.get('search', ''), num, 0, ','.join(sorts),
','.join(orders)
','.join(orders), vl
)
except ParseException:
ans['search_result'] = search_result(
ctx, rd, db, '', num, 0, ','.join(sorts), ','.join(orders)
ctx, rd, db, '', num, 0, ','.join(sorts), ','.join(orders), vl
)
sf = db.field_metadata.ui_sortable_field_keys()
sf.pop('ondevice', None)
@ -146,6 +146,7 @@ def get_library_init_data(ctx, rd, db, num, sorts, orders):
key=lambda (field, name): sort_key(name)
)
ans['field_metadata'] = db.field_metadata.all_metadata()
ans['virtual_libraries'] = db._pref('virtual_libraries', {})
mdata = ans['metadata'] = {}
try:
extra_books = set(
@ -168,15 +169,15 @@ def books(ctx, rd):
Get data to create list of books
Optional: ?num=50&sort=timestamp.desc&library_id=<default library>
&search=''&extra_books=''
&search=''&extra_books=''&vl=''
'''
ans = {}
try:
num = int(rd.query.get('num', DEFAULT_NUMBER_OF_BOOKS))
except Exception:
raise HTTPNotFound('Invalid number of books: %r' % rd.query.get('num'))
library_id, db, sorts, orders = get_basic_query_data(ctx, rd)
ans = get_library_init_data(ctx, rd, db, num, sorts, orders)
library_id, db, sorts, orders, vl = get_basic_query_data(ctx, rd)
ans = get_library_init_data(ctx, rd, db, num, sorts, orders, vl)
ans['library_id'] = library_id
return ans
@ -187,7 +188,7 @@ def interface_data(ctx, rd):
Return the data needed to create the server UI as well as a list of books.
Optional: ?num=50&sort=timestamp.desc&library_id=<default library>
&search=''&extra_books=''
&search=''&extra_books=''&vl=''
'''
ans = basic_interface_data(ctx, rd)
ud = {}
@ -201,13 +202,13 @@ def interface_data(ctx, rd):
usort = ud.get('sort')
if usort:
rd.query.set('sort', usort)
ans['library_id'], db, sorts, orders = get_basic_query_data(ctx, rd)
ans['library_id'], db, sorts, orders, vl = get_basic_query_data(ctx, rd)
ans['user_session_data'] = ud
try:
num = int(rd.query.get('num', DEFAULT_NUMBER_OF_BOOKS))
except Exception:
raise HTTPNotFound('Invalid number of books: %r' % rd.query.get('num'))
ans.update(get_library_init_data(ctx, rd, db, num, sorts, orders))
ans.update(get_library_init_data(ctx, rd, db, num, sorts, orders, vl))
return ans
@ -271,9 +272,9 @@ def get_books(ctx, rd):
'''
Get books for the specified query
Optional: ?library_id=<default library>&num=50&sort=timestamp.desc&search=''
Optional: ?library_id=<default library>&num=50&sort=timestamp.desc&search=''&vl=''
'''
library_id, db, sorts, orders = get_basic_query_data(ctx, rd)
library_id, db, sorts, orders, vl = get_basic_query_data(ctx, rd)
try:
num = int(rd.query.get('num', DEFAULT_NUMBER_OF_BOOKS))
except Exception:
@ -285,7 +286,7 @@ def get_books(ctx, rd):
with db.safe_read_lock:
try:
ans['search_result'] = search_result(
ctx, rd, db, searchq, num, 0, ','.join(sorts), ','.join(orders)
ctx, rd, db, searchq, num, 0, ','.join(sorts), ','.join(orders), vl
)
except ParseException as err:
# This must not be translated as it is used by the front end to
@ -306,16 +307,13 @@ def book_metadata(ctx, rd, book_id):
Optional: ?library_id=<default library>
'''
library_id, db = get_basic_query_data(ctx, rd)[:2]
book_ids = ctx.allowed_book_ids(rd, db)
def notfound():
raise HTTPNotFound(_('No book with id: %d in library') % book_id)
raise HTTPNotFound(_('No book with id: {} in library: {}').format(book_id, library_id))
if not book_ids:
notfound()
if not book_id:
book_id = random.choice(tuple(book_ids))
elif book_id not in book_ids:
book_id = random.choice(tuple(db.all_book_ids()))
elif not db.has_id(book_id):
notfound()
data = book_as_json(db, book_id)
if data is None:
@ -329,14 +327,15 @@ def tag_browser(ctx, rd):
'''
Get the Tag Browser serialized as JSON
Optional: ?library_id=<default library>&sort_tags_by=name&partition_method=first letter
&collapse_at=25&dont_collapse=&hide_empty_categories=
&collapse_at=25&dont_collapse=&hide_empty_categories=&vl=''
'''
db, library_id = get_library_data(ctx, rd)[:2]
etag = '%s||%s||%s' % (db.last_modified(), rd.username, library_id)
etag = hashlib.sha1(etag.encode('utf-8')).hexdigest()
opts = categories_settings(rd.query, db)
vl = rd.query.get('vl') or ''
etag = cPickle.dumps([db.last_modified().isoformat(), rd.username, library_id, vl, list(opts)], -1)
etag = hashlib.sha1(etag).hexdigest()
def generate():
db, library_id = get_library_data(ctx, rd)[:2]
return json(ctx, rd, tag_browser, categories_as_json(ctx, rd, db))
return json(ctx, rd, tag_browser, categories_as_json(ctx, rd, db, opts, vl))
return rd.etagged_dynamic_response(etag, generate)

2
src/calibre/srv/content.py
View File

@ -276,7 +276,7 @@ def get(ctx, rd, what, book_id, library_id):
if db is None:
raise HTTPNotFound('Library %r not found' % library_id)
with db.safe_read_lock:
if book_id not in ctx.allowed_book_ids(rd, db):
if not db.has_id(book_id):
raise HTTPNotFound('Book with id %r does not exist' % book_id)
library_id = db.server_library_id # in case library_id was None
if what == 'thumb':

24
src/calibre/srv/handler.py
View File

@ -71,16 +71,9 @@ class Context(object):
raise HTTPForbidden('The user {} is not allowed to access any libraries on this server'.format(data.username))
return dict(allowed_libraries), next(allowed_libraries.iterkeys())
def allowed_book_ids(self, data, db):
with self.lock:
ans = data.allowed_book_ids.get(db.server_library_id)
if ans is None:
ans = data.allowed_book_ids[db.server_library_id] = db.all_book_ids()
return ans
def get_categories(self, data, db, restrict_to_ids=None, sort='name', first_letter_sort=True):
def get_categories(self, data, db, restrict_to_ids=None, sort='name', first_letter_sort=True, vl=''):
if restrict_to_ids is None:
restrict_to_ids = self.allowed_book_ids(data, db)
restrict_to_ids = db.books_in_virtual_library(vl)
key = (restrict_to_ids, sort, first_letter_sort)
with self.lock:
cache = self.library_broker.category_caches[db.server_library_id]
@ -94,9 +87,9 @@ class Context(object):
cache[key] = old
return old[1]
def get_tag_browser(self, data, db, opts, render, restrict_to_ids=None):
def get_tag_browser(self, data, db, opts, render, restrict_to_ids=None, vl=''):
if restrict_to_ids is None:
restrict_to_ids = self.allowed_book_ids(data, db)
restrict_to_ids = db.books_in_virtual_library(vl)
key = (restrict_to_ids, opts)
with self.lock:
cache = self.library_broker.category_caches[db.server_library_id]
@ -113,15 +106,14 @@ class Context(object):
cache[key] = old
return old[1]
def search(self, data, db, query, restrict_to_ids=None):
if restrict_to_ids is None:
restrict_to_ids = self.allowed_book_ids(data, db)
def search(self, data, db, query, restrict_to_ids=None, vl=''):
with self.lock:
cache = self.library_broker.search_caches[db.server_library_id]
key = (query, restrict_to_ids)
vl = db.pref('virtual_libraries', {}).get(vl) or ''
key = query, restrict_to_ids, vl
old = cache.pop(key, None)
if old is None or old[0] < db.clear_search_cache_count:
matches = db.search(query, book_ids=restrict_to_ids)
matches = db.search(query, restriction=vl, book_ids=restrict_to_ids)
cache[key] = old = (db.clear_search_cache_count, matches)
if len(cache) > self.SEARCH_CACHE_SIZE:
cache.popitem(last=False)

1
src/calibre/srv/http_response.py
View File

@ -224,7 +224,6 @@ class RequestData(object): # {{{
self.lang_code = self.gettext_func = self.ngettext_func = None
self.set_translator(self.get_preferred_language())
self.tdir = tdir
self.allowed_book_ids = {}
def generate_static_output(self, name, generator, content_type='text/html; charset=UTF-8'):
ans = self.static_cache.get(name)

10
src/calibre/srv/metadata.py
View File

@ -39,6 +39,7 @@ def encode_datetime(dateval):
return None
return isoformat(dateval)
empty_val = ((), '', {})
@ -84,6 +85,7 @@ def book_as_json(db, book_id):
ans['lang_names'] = {l:calibre_langcode_to_name(l) for l in langs}
return ans
_include_fields = frozenset(Tag.__slots__) - frozenset({
'state', 'is_editable', 'is_searchable', 'original_name', 'use_sort_as_name', 'is_hierarchical'
})
@ -133,6 +135,7 @@ def category_item_as_json(x, clear_rating=False):
del ans['avg_rating']
return ans
CategoriesSettings = namedtuple(
'CategoriesSettings', 'dont_collapse collapse_model collapse_at sort_by'
' template using_hierarchy grouped_search_terms hidden_categories hide_empty_categories')
@ -162,6 +165,7 @@ class GroupedSearchTerms(object):
except AttributeError:
return False
_icon_map = None
_icon_map_lock = Lock()
@ -289,6 +293,7 @@ def build_first_letter_list(category_items):
cl_list[idx] = last_c
return cl_list
categories_with_ratings = {'authors', 'series', 'publisher', 'tags'}
@ -528,9 +533,8 @@ def render_categories(opts, db, category_data):
return {'root':root, 'item_map': items}
def categories_as_json(ctx, rd, db):
opts = categories_settings(rd.query, db)
return ctx.get_tag_browser(rd, db, opts, partial(render_categories, opts))
def categories_as_json(ctx, rd, db, opts, vl):
return ctx.get_tag_browser(rd, db, opts, partial(render_categories, opts), vl='')
# Test tag browser {{{

10
src/calibre/srv/opds.py
View File

@ -56,6 +56,7 @@ def format_tag_string(tags, sep, joinval=', '):
tlist.sort(key=sort_key)
return joinval.join(tlist) if tlist else ''
# Vocabulary for building OPDS feeds {{{
DC_NS = 'http://purl.org/dc/terms/'
E = ElementMaker(namespace='http://www.w3.org/2005/Atom',
@ -75,6 +76,7 @@ ICON = E.icon
def UPDATED(dt, *args, **kwargs):
return E.updated(as_utc(dt).strftime('%Y-%m-%dT%H:%M:%S+00:00'), *args, **kwargs)
LINK = partial(E.link, type='application/atom+xml')
NAVLINK = partial(E.link,
type='application/atom+xml;type=feed;profile=opds-catalog')
@ -93,6 +95,7 @@ def AUTHOR(name, uri=None):
args.append(E.uri(uri))
return E.author(*args)
SUBTITLE = E.subtitle
@ -107,6 +110,7 @@ def NAVCATALOG_ENTRY(url_for, updated, title, description, query):
NAVLINK(href=href)
)
START_LINK = partial(NAVLINK, rel='start')
UP_LINK = partial(NAVLINK, rel='up')
FIRST_LINK = partial(NAVLINK, rel='first')
@ -363,8 +367,8 @@ class RequestContext(object):
ans += '?' + urlencode(q)
return ans
def allowed_book_ids(self):
return self.ctx.allowed_book_ids(self.rd, self.db)
def all_book_ids(self):
return self.db.all_book_ids()
@property
def outheaders(self):
@ -410,7 +414,7 @@ def get_all_books(rc, which, page_url, up_url, offset=0):
ascending = which == 'title'
feed_title = {'newest':_('Newest'), 'title': _('Title')}.get(which, which)
feed_title = default_feed_title + ' :: ' + _('By %s') % feed_title
ids = rc.allowed_book_ids()
ids = rc.all_book_ids()
return get_acquisition_feed(rc, ids, offset, page_url, up_url,
id_='calibre-all:'+sort, sort_by=sort, ascending=ascending,
feed_title=feed_title)

3
src/calibre/srv/tests/ajax.py
View File

@ -53,6 +53,7 @@ class ContentTest(LibraryBaseTest):
'Test /ajax/categories and /ajax/search'
with self.create_server() as server:
db = server.handler.router.ctx.library_broker.get(None)
db.set_pref('virtual_libraries', {'1':'title:"=Title One"'})
conn = server.connect()
request = partial(make_request, conn)
@ -74,4 +75,6 @@ class ContentTest(LibraryBaseTest):
r, data = request('/search?' + urlencode({'query': 'tags:"=Tag One"'}))
self.ae(r.status, httplib.OK)
self.ae(set(data['book_ids']), {1, 2})
r, data = request('/search?' + urlencode({'query': 'tags:"=Tag One"', 'vl':'1'}))
self.ae(set(data['book_ids']), {2})
# }}}