From d2a5aab26dad03e290cd9f74fecbdc0ccb5865ef Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 29 Apr 2017 12:03:29 +0530 Subject: [PATCH] Wire up remote access for calibredb --- src/calibre/db/cli/__init__.py | 6 ++ src/calibre/db/cli/cmd_list.py | 2 +- src/calibre/db/cli/main.py | 108 +++++++++++++++++++++++++++++---- src/calibre/srv/cdb.py | 40 ++++++++++++ src/calibre/srv/handler.py | 8 ++- src/calibre/srv/routes.py | 23 ++++++- src/calibre/srv/utils.py | 6 +- src/calibre/utils/serialize.py | 50 +++++++++++++++ 8 files changed, 226 insertions(+), 17 deletions(-) create mode 100644 src/calibre/srv/cdb.py create mode 100644 src/calibre/utils/serialize.py diff --git a/src/calibre/db/cli/__init__.py b/src/calibre/db/cli/__init__.py index 7c9bf8b04c..8b0fd3c5e3 100644 --- a/src/calibre/db/cli/__init__.py +++ b/src/calibre/db/cli/__init__.py @@ -3,3 +3,9 @@ # License: GPLv3 Copyright: 2017, Kovid Goyal from __future__ import absolute_import, division, print_function, unicode_literals + +import importlib + + +def module_for_cmd(cmd): + return importlib.import_module('calibre.db.cli.cmd_' + cmd) diff --git a/src/calibre/db/cli/cmd_list.py b/src/calibre/db/cli/cmd_list.py index d2fdfdc0ff..2cfee3589e 100644 --- a/src/calibre/db/cli/cmd_list.py +++ b/src/calibre/db/cli/cmd_list.py @@ -150,7 +150,7 @@ def do_list( book_ids, data, metadata = ans['book_ids'], ans['data'], ans['metadata'] except TypeError: raise SystemExit(ans) - fields = ans['fields'] + fields = list(ans['fields']) try: fields.remove('id') except ValueError: diff --git a/src/calibre/db/cli/main.py b/src/calibre/db/cli/main.py index 64e5e2cb27..8554de08f6 100644 --- a/src/calibre/db/cli/main.py +++ b/src/calibre/db/cli/main.py @@ -4,12 +4,18 @@ from __future__ import absolute_import, division, print_function, unicode_literals -import importlib +import httplib +import json import os import sys +from urllib import urlencode +from urlparse import urlparse, urlunparse -from calibre import prints +from calibre import browser, prints +from calibre.constants import __appname__, __version__ +from calibre.db.cli import module_for_cmd from calibre.utils.config import OptionParser, prefs +from calibre.utils.serialize import MSGPACK_MIME COMMANDS = ( 'list', 'add', 'remove', 'add_format', 'remove_format', 'show_metadata', @@ -20,13 +26,11 @@ COMMANDS = ( ) -def module_for_cmd(cmd): - return importlib.import_module('calibre.db.cli.cmd_' + cmd) - - def option_parser_for(cmd): + def cmd_option_parser(): return module_for_cmd(cmd).option_parser(get_parser) + return cmd_option_parser @@ -59,6 +63,12 @@ def get_parser(usage): default=None, help=_( 'Path to the calibre library. Default is to use the path stored in the settings.' + ' You can also connect to a calibre Content server to perform actions on' + ' remote libraries. To do so use a URL of the form: http://hostname:port/#library_id' + ' for example, http://localhost:8080/#mylibrary. library_id is the library id' + ' of the library you want to connect to on the Content server. You can use' + ' the special library_id value of - to get a list of library ids available' + ' on the server.' ) ) go.add_option( @@ -78,6 +88,17 @@ def get_parser(usage): help=_("show program's version number and exit"), action='version' ) + go.add_option( + '--username', + help=_('Username for connecting to a calibre Content server') + ) + go.add_option( + '--password', + help=_('Password for connecting to a calibre Content server.' + ' To read the password from standard input, use the special value: {}.' + ' To read the password from a file, use: {}.)').format( + '', '') + ) return parser @@ -100,17 +121,44 @@ For help on an individual command: %%prog command --help return parser +def read_credetials(opts): + username = opts.username + pw = opts.password + if pw: + if pw == '': + import getpass + pw = getpass.getpass(_('Enter the password: ')) + elif pw.startswith(''): + with lopen(pw[3:-1], 'rb') as f: + pw = f.read().decode('utf-8') + return username, pw + + class DBCtx(object): def __init__(self, opts): self.library_path = opts.library_path or prefs['library_path'] self.url = None if self.library_path is None: - raise SystemExit('No saved library path, either run the GUI or use the' - ' --with-library option') + raise SystemExit( + 'No saved library path, either run the GUI or use the' + ' --with-library option' + ) if self.library_path.partition(':')[0] in ('http', 'https'): - self.url = self.library_path + parts = urlparse(self.library_path) + self.library_id = parts.fragment or None + self.url = urlunparse(parts._replace(fragment='')).rstrip('/') + self.br = browser(handle_refresh=False, user_agent='{} {}'.format(__appname__, __version__)) + self.br.addheaders += [('Accept', MSGPACK_MIME), ('Content-Type', MSGPACK_MIME)] self.is_remote = True + username, password = read_credetials(opts) + self.has_credentials = False + if username and password: + self.br.add_password(self.url, username, password) + self.has_credentials = True + if self.library_id == '-': + self.list_libraries() + raise SystemExit() else: self.library_path = os.path.expanduser(self.library_path) self._db = None @@ -124,11 +172,49 @@ class DBCtx(object): return self._db def run(self, name, *args): - if self.is_remote: - raise NotImplementedError() m = module_for_cmd(name) + if self.is_remote: + return self.remote_run(name, m, *args) return m.implementation(self.db, False, *args) + def interpret_http_error(self, err): + if err.code == httplib.UNAUTHORIZED: + raise SystemExit('A username and password is required to access this server') + if err.code == httplib.FORBIDDEN: + raise SystemExit('The username/password combination is incorrect') + if err.code == httplib.NOT_FOUND: + raise SystemExit(err.reason) + + def remote_run(self, name, m, *args): + from mechanize import HTTPError + from calibre.utils.serialize import msgpack_loads + url = self.url + '/cdb/run/' + name + if self.library_id: + url += '?' + urlencode({'library_id':self.library_id}) + try: + res = self.br.open_novisit(url, data=json.dumps(args)) + ans = msgpack_loads(res.read()) + except HTTPError as err: + self.interpret_http_error(err) + raise + if 'err' in ans: + prints(ans['tb']) + raise SystemExit(ans['err']) + return ans['result'] + + def list_libraries(self): + from mechanize import HTTPError + url = self.url + '/ajax/library-info' + try: + res = self.br.open_novisit(url) + ans = json.loads(res.read()) + except HTTPError as err: + self.interpret_http_error(err) + raise + library_map, default_library = ans['library_map'], ans['default_library'] + for lid in sorted(library_map, key=lambda lid: (lid != default_library, lid)): + prints(lid) + def main(args=sys.argv): parser = option_parser() diff --git a/src/calibre/srv/cdb.py b/src/calibre/srv/cdb.py new file mode 100644 index 0000000000..0c24d81405 --- /dev/null +++ b/src/calibre/srv/cdb.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python2 +# vim:fileencoding=utf-8 +# License: GPLv3 Copyright: 2017, Kovid Goyal + +from __future__ import absolute_import, division, print_function, unicode_literals + +from calibre import as_unicode +from calibre.db.cli import module_for_cmd +from calibre.srv.errors import HTTPNotFound, HTTPBadRequest +from calibre.srv.routes import endpoint, msgpack_or_json +from calibre.srv.utils import get_library_data +from calibre.utils.serialize import MSGPACK_MIME, msgpack_loads, json_loads + +receive_data_methods = {'GET', 'POST'} + + +@endpoint('/cdb/run/{which}', postprocess=msgpack_or_json, methods=receive_data_methods) +def cdb_run(ctx, rd, which): + try: + m = module_for_cmd(which) + except ImportError: + raise HTTPNotFound('No module named: {}'.format(which)) + if not getattr(m, 'readonly', False): + ctx.check_for_write_access(rd) + raw = rd.read() + ct = rd.inheaders.get('Content-Type', all=True) + try: + if MSGPACK_MIME in ct: + args = msgpack_loads(raw) + else: + args = json_loads(raw) + except Exception: + raise HTTPBadRequest('args are not valid encoded data') + db = get_library_data(ctx, rd, strict_library_id=True)[0] + try: + result = m.implementation(db, True, *args) + except Exception as err: + import traceback + return {'err': as_unicode(err), 'tb':traceback.format_stack()} + return {'result': result} diff --git a/src/calibre/srv/handler.py b/src/calibre/srv/handler.py index 0f49dab70b..c156619f6e 100644 --- a/src/calibre/srv/handler.py +++ b/src/calibre/srv/handler.py @@ -71,6 +71,12 @@ class Context(object): raise HTTPForbidden('The user {} is not allowed to access any libraries on this server'.format(data.username)) return dict(allowed_libraries), next(allowed_libraries.iterkeys()) + def check_for_write_access(self, data): + if not data.username: + raise HTTPForbidden('Anonymous users are not allowed to make changes') + if self.user_manager.is_readonly(data.username): + raise HTTPForbidden('The user {} does not have permission to make changes'.format(data.username)) + def get_categories(self, data, db, restrict_to_ids=None, sort='name', first_letter_sort=True, vl=''): if restrict_to_ids is None: restrict_to_ids = db.books_in_virtual_library(vl) @@ -132,7 +138,7 @@ class Handler(object): prefer_basic_auth = {'auto':has_ssl, 'basic':True}.get(opts.auth_mode, False) self.auth_controller = AuthController(user_credentials=ctx.user_manager, prefer_basic_auth=prefer_basic_auth) self.router = Router(ctx=ctx, url_prefix=opts.url_prefix, auth_controller=self.auth_controller) - for module in ('content', 'ajax', 'code', 'legacy', 'opds', 'books'): + for module in ('content', 'ajax', 'code', 'legacy', 'opds', 'books', 'cdb'): module = import_module('calibre.srv.' + module) self.router.load_routes(vars(module).itervalues()) self.router.finalize() diff --git a/src/calibre/srv/routes.py b/src/calibre/srv/routes.py index 4d7fb100cd..917b8caffd 100644 --- a/src/calibre/srv/routes.py +++ b/src/calibre/srv/routes.py @@ -13,6 +13,7 @@ from operator import attrgetter from calibre.srv.errors import HTTPSimpleResponse, HTTPNotFound, RouteError from calibre.srv.utils import http_date +from calibre.utils.serialize import msgpack_dumps, json_dumps, MSGPACK_MIME default_methods = frozenset(('HEAD', 'GET')) @@ -22,12 +23,28 @@ def json(ctx, rd, endpoint, output): if isinstance(output, bytes) or hasattr(output, 'fileno'): ans = output # Assume output is already UTF-8 encoded json else: - ans = jsonlib.dumps(output, ensure_ascii=False) - if not isinstance(ans, bytes): - ans = ans.encode('utf-8') + ans = json_dumps(output) return ans +def msgpack(ctx, rd, endpoint, output): + rd.outheaders.set('Content-Type', MSGPACK_MIME, replace_all=True) + if isinstance(output, bytes) or hasattr(output, 'fileno'): + ans = output # Assume output is already msgpack encoded + else: + ans = msgpack_dumps(output) + return ans + + +def msgpack_or_json(ctx, rd, endpoint, output): + accept = rd.inheaders.get('Accept', all=True) + func = msgpack if MSGPACK_MIME in accept else json + return func(ctx, rd, endpoint, output) + + +json.loads, json.dumps = jsonlib.loads, jsonlib.dumps + + def route_key(route): return route.partition('{')[0].rstrip('/') diff --git a/src/calibre/srv/utils.py b/src/calibre/srv/utils.py index f98d3ad69e..dc061467be 100644 --- a/src/calibre/srv/utils.py +++ b/src/calibre/srv/utils.py @@ -114,6 +114,7 @@ def error_codes(*errnames): ans.discard(None) return ans + socket_errors_eintr = error_codes("EINTR", "WSAEINTR") socket_errors_socket_closed = error_codes( # errors indicating a disconnected connection @@ -464,10 +465,12 @@ def get_db(ctx, rd, library_id): return db -def get_library_data(ctx, rd): +def get_library_data(ctx, rd, strict_library_id=False): library_id = rd.query.get('library_id') library_map, default_library = ctx.library_info(rd) if library_id not in library_map: + if strict_library_id and library_id: + raise HTTPNotFound('No library with id: {}'.format(library_id)) library_id = default_library db = get_db(ctx, rd, library_id) return db, library_id, library_map, default_library @@ -495,6 +498,7 @@ class Offsets(object): if self.last_offset < 0: self.last_offset = 0 + _use_roman = None diff --git a/src/calibre/utils/serialize.py b/src/calibre/utils/serialize.py new file mode 100644 index 0000000000..7921d614d8 --- /dev/null +++ b/src/calibre/utils/serialize.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python2 +# vim:fileencoding=utf-8 +# License: GPLv3 Copyright: 2017, Kovid Goyal + +from __future__ import absolute_import, division, print_function, unicode_literals + +import json +from datetime import datetime +from calibre.utils.iso8601 import parse_iso8601 + + +MSGPACK_MIME = 'application/x-msgpack' + + +def encoder(obj): + if isinstance(obj, datetime): + return {'__datetime__': obj.isoformat()} + return obj + + +def msgpack_dumps(data): + import msgpack + return msgpack.packb(data, use_bin_type=True, default=encoder) + + +def json_dumps(data, **kw): + kw['default'] = encoder + kw['ensure_ascii'] = False + ans = json.dumps(data, **kw) + if not isinstance(ans, bytes): + ans = ans.encode('utf-8') + return ans + + +def decoder(obj): + dt = obj.get('__datetime__') + if dt is not None: + obj = parse_iso8601(dt, assume_utc=True) + return obj + + +def msgpack_loads(data): + import msgpack + return msgpack.unpackb( + data, encoding='utf-8', use_list=False, object_hook=decoder + ) + + +def json_loads(data): + return json.loads(data, object_hook=decoder)