Wire up remote access for calibredb

src/calibre/db/cli/__init__.py

@@ -3,3 +3,9 @@
 # License: GPLv3 Copyright: 2017, Kovid Goyal <kovid at kovidgoyal.net>
 
 from __future__ import absolute_import, division, print_function, unicode_literals
+
+import importlib
+
+
+def module_for_cmd(cmd):
+    return importlib.import_module('calibre.db.cli.cmd_' + cmd)

src/calibre/db/cli/cmd_list.py

@@ -150,7 +150,7 @@ def do_list(
         book_ids, data, metadata = ans['book_ids'], ans['data'], ans['metadata']
     except TypeError:
         raise SystemExit(ans)
-    fields = ans['fields']
+    fields = list(ans['fields'])
     try:
         fields.remove('id')
     except ValueError:

src/calibre/db/cli/main.py

@@ -4,12 +4,18 @@
 
 from __future__ import absolute_import, division, print_function, unicode_literals
 
-import importlib
+import httplib
+import json
 import os
 import sys
+from urllib import urlencode
+from urlparse import urlparse, urlunparse
 
-from calibre import prints
+from calibre import browser, prints
+from calibre.constants import __appname__, __version__
+from calibre.db.cli import module_for_cmd
 from calibre.utils.config import OptionParser, prefs
+from calibre.utils.serialize import MSGPACK_MIME
 
 COMMANDS = (
     'list', 'add', 'remove', 'add_format', 'remove_format', 'show_metadata',
@@ -20,13 +26,11 @@ COMMANDS = (
 )
 
 
-def module_for_cmd(cmd):
-    return importlib.import_module('calibre.db.cli.cmd_' + cmd)
-
-
 def option_parser_for(cmd):
+
     def cmd_option_parser():
         return module_for_cmd(cmd).option_parser(get_parser)
+
     return cmd_option_parser
 
 
@@ -59,6 +63,12 @@ def get_parser(usage):
         default=None,
         help=_(
             'Path to the calibre library. Default is to use the path stored in the settings.'
+            ' You can also connect to a calibre Content server to perform actions on'
+            ' remote libraries. To do so use a URL of the form: http://hostname:port/#library_id'
+            ' for example, http://localhost:8080/#mylibrary. library_id is the library id'
+            ' of the library you want to connect to on the Content server. You can use'
+            ' the special library_id value of - to get a list of library ids available'
+            ' on the server.'
         )
     )
     go.add_option(
@@ -78,6 +88,17 @@ def get_parser(usage):
         help=_("show program's version number and exit"),
         action='version'
     )
+    go.add_option(
+        '--username',
+        help=_('Username for connecting to a calibre Content server')
+    )
+    go.add_option(
+        '--password',
+        help=_('Password for connecting to a calibre Content server.'
+               ' To read the password from standard input, use the special value: {}.'
+               ' To read the password from a file, use: {}.)').format(
+                   '<stdin>', '<f:/path/to/file>')
+    )
 
     return parser
 
@@ -100,17 +121,44 @@ For help on an individual command: %%prog command --help
     return parser
 
 
+def read_credetials(opts):
+    username = opts.username
+    pw = opts.password
+    if pw:
+        if pw == '<stdin>':
+            import getpass
+            pw = getpass.getpass(_('Enter the password: '))
+        elif pw.startswith('<f:') and pw.endswith('>'):
+            with lopen(pw[3:-1], 'rb') as f:
+                pw = f.read().decode('utf-8')
+    return username, pw
+
+
 class DBCtx(object):
 
     def __init__(self, opts):
         self.library_path = opts.library_path or prefs['library_path']
         self.url = None
         if self.library_path is None:
-            raise SystemExit('No saved library path, either run the GUI or use the'
+            raise SystemExit(
-                             ' --with-library option')
+                'No saved library path, either run the GUI or use the'
+                ' --with-library option'
+            )
         if self.library_path.partition(':')[0] in ('http', 'https'):
-            self.url = self.library_path
+            parts = urlparse(self.library_path)
+            self.library_id = parts.fragment or None
+            self.url = urlunparse(parts._replace(fragment='')).rstrip('/')
+            self.br = browser(handle_refresh=False, user_agent='{} {}'.format(__appname__, __version__))
+            self.br.addheaders += [('Accept', MSGPACK_MIME), ('Content-Type', MSGPACK_MIME)]
             self.is_remote = True
+            username, password = read_credetials(opts)
+            self.has_credentials = False
+            if username and password:
+                self.br.add_password(self.url, username, password)
+                self.has_credentials = True
+            if self.library_id == '-':
+                self.list_libraries()
+                raise SystemExit()
         else:
             self.library_path = os.path.expanduser(self.library_path)
             self._db = None
@@ -124,11 +172,49 @@ class DBCtx(object):
         return self._db
 
     def run(self, name, *args):
-        if self.is_remote:
-            raise NotImplementedError()
         m = module_for_cmd(name)
+        if self.is_remote:
+            return self.remote_run(name, m, *args)
         return m.implementation(self.db, False, *args)
 
+    def interpret_http_error(self, err):
+        if err.code == httplib.UNAUTHORIZED:
+            raise SystemExit('A username and password is required to access this server')
+        if err.code == httplib.FORBIDDEN:
+            raise SystemExit('The username/password combination is incorrect')
+        if err.code == httplib.NOT_FOUND:
+            raise SystemExit(err.reason)
+
+    def remote_run(self, name, m, *args):
+        from mechanize import HTTPError
+        from calibre.utils.serialize import msgpack_loads
+        url = self.url + '/cdb/run/' + name
+        if self.library_id:
+            url += '?' + urlencode({'library_id':self.library_id})
+        try:
+            res = self.br.open_novisit(url, data=json.dumps(args))
+            ans = msgpack_loads(res.read())
+        except HTTPError as err:
+            self.interpret_http_error(err)
+            raise
+        if 'err' in ans:
+            prints(ans['tb'])
+            raise SystemExit(ans['err'])
+        return ans['result']
+
+    def list_libraries(self):
+        from mechanize import HTTPError
+        url = self.url + '/ajax/library-info'
+        try:
+            res = self.br.open_novisit(url)
+            ans = json.loads(res.read())
+        except HTTPError as err:
+            self.interpret_http_error(err)
+            raise
+        library_map, default_library = ans['library_map'], ans['default_library']
+        for lid in sorted(library_map, key=lambda lid: (lid != default_library, lid)):
+            prints(lid)
+
 
 def main(args=sys.argv):
     parser = option_parser()

src/calibre/srv/cdb.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env python2
+# vim:fileencoding=utf-8
+# License: GPLv3 Copyright: 2017, Kovid Goyal <kovid at kovidgoyal.net>
+
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+from calibre import as_unicode
+from calibre.db.cli import module_for_cmd
+from calibre.srv.errors import HTTPNotFound, HTTPBadRequest
+from calibre.srv.routes import endpoint, msgpack_or_json
+from calibre.srv.utils import get_library_data
+from calibre.utils.serialize import MSGPACK_MIME, msgpack_loads, json_loads
+
+receive_data_methods = {'GET', 'POST'}
+
+
+@endpoint('/cdb/run/{which}', postprocess=msgpack_or_json, methods=receive_data_methods)
+def cdb_run(ctx, rd, which):
+    try:
+        m = module_for_cmd(which)
+    except ImportError:
+        raise HTTPNotFound('No module named: {}'.format(which))
+    if not getattr(m, 'readonly', False):
+        ctx.check_for_write_access(rd)
+    raw = rd.read()
+    ct = rd.inheaders.get('Content-Type', all=True)
+    try:
+        if MSGPACK_MIME in ct:
+            args = msgpack_loads(raw)
+        else:
+            args = json_loads(raw)
+    except Exception:
+        raise HTTPBadRequest('args are not valid encoded data')
+    db = get_library_data(ctx, rd, strict_library_id=True)[0]
+    try:
+        result = m.implementation(db, True, *args)
+    except Exception as err:
+        import traceback
+        return {'err': as_unicode(err), 'tb':traceback.format_stack()}
+    return {'result': result}

src/calibre/srv/handler.py

@@ -71,6 +71,12 @@ class Context(object):
             raise HTTPForbidden('The user {} is not allowed to access any libraries on this server'.format(data.username))
         return dict(allowed_libraries), next(allowed_libraries.iterkeys())
 
+    def check_for_write_access(self, data):
+        if not data.username:
+            raise HTTPForbidden('Anonymous users are not allowed to make changes')
+        if self.user_manager.is_readonly(data.username):
+            raise HTTPForbidden('The user {} does not have permission to make changes'.format(data.username))
+
     def get_categories(self, data, db, restrict_to_ids=None, sort='name', first_letter_sort=True, vl=''):
         if restrict_to_ids is None:
             restrict_to_ids = db.books_in_virtual_library(vl)
@@ -132,7 +138,7 @@ class Handler(object):
             prefer_basic_auth = {'auto':has_ssl, 'basic':True}.get(opts.auth_mode, False)
             self.auth_controller = AuthController(user_credentials=ctx.user_manager, prefer_basic_auth=prefer_basic_auth)
         self.router = Router(ctx=ctx, url_prefix=opts.url_prefix, auth_controller=self.auth_controller)
-        for module in ('content', 'ajax', 'code', 'legacy', 'opds', 'books'):
+        for module in ('content', 'ajax', 'code', 'legacy', 'opds', 'books', 'cdb'):
             module = import_module('calibre.srv.' + module)
             self.router.load_routes(vars(module).itervalues())
         self.router.finalize()

src/calibre/srv/routes.py

@@ -13,6 +13,7 @@ from operator import attrgetter
 
 from calibre.srv.errors import HTTPSimpleResponse, HTTPNotFound, RouteError
 from calibre.srv.utils import http_date
+from calibre.utils.serialize import msgpack_dumps, json_dumps, MSGPACK_MIME
 
 default_methods = frozenset(('HEAD', 'GET'))
 
@@ -22,12 +23,28 @@ def json(ctx, rd, endpoint, output):
     if isinstance(output, bytes) or hasattr(output, 'fileno'):
         ans = output  # Assume output is already UTF-8 encoded json
     else:
-        ans = jsonlib.dumps(output, ensure_ascii=False)
+        ans = json_dumps(output)
-        if not isinstance(ans, bytes):
-            ans = ans.encode('utf-8')
     return ans
 
 
+def msgpack(ctx, rd, endpoint, output):
+    rd.outheaders.set('Content-Type', MSGPACK_MIME, replace_all=True)
+    if isinstance(output, bytes) or hasattr(output, 'fileno'):
+        ans = output  # Assume output is already msgpack encoded
+    else:
+        ans = msgpack_dumps(output)
+    return ans
+
+
+def msgpack_or_json(ctx, rd, endpoint, output):
+    accept = rd.inheaders.get('Accept', all=True)
+    func = msgpack if MSGPACK_MIME in accept else json
+    return func(ctx, rd, endpoint, output)
+
+
+json.loads, json.dumps = jsonlib.loads, jsonlib.dumps
+
+
 def route_key(route):
     return route.partition('{')[0].rstrip('/')
 

src/calibre/srv/utils.py

@@ -114,6 +114,7 @@ def error_codes(*errnames):
     ans.discard(None)
     return ans
 
+
 socket_errors_eintr = error_codes("EINTR", "WSAEINTR")
 
 socket_errors_socket_closed = error_codes(  # errors indicating a disconnected connection
@@ -464,10 +465,12 @@ def get_db(ctx, rd, library_id):
     return db
 
 
-def get_library_data(ctx, rd):
+def get_library_data(ctx, rd, strict_library_id=False):
     library_id = rd.query.get('library_id')
     library_map, default_library = ctx.library_info(rd)
     if library_id not in library_map:
+        if strict_library_id and library_id:
+            raise HTTPNotFound('No library with id: {}'.format(library_id))
         library_id = default_library
     db = get_db(ctx, rd, library_id)
     return db, library_id, library_map, default_library
@@ -495,6 +498,7 @@ class Offsets(object):
         if self.last_offset < 0:
             self.last_offset = 0
 
+
 _use_roman = None
 
 

src/calibre/utils/serialize.py

@@ -0,0 +1,50 @@
+#!/usr/bin/env python2
+# vim:fileencoding=utf-8
+# License: GPLv3 Copyright: 2017, Kovid Goyal <kovid at kovidgoyal.net>
+
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+import json
+from datetime import datetime
+from calibre.utils.iso8601 import parse_iso8601
+
+
+MSGPACK_MIME = 'application/x-msgpack'
+
+
+def encoder(obj):
+    if isinstance(obj, datetime):
+        return {'__datetime__': obj.isoformat()}
+    return obj
+
+
+def msgpack_dumps(data):
+    import msgpack
+    return msgpack.packb(data, use_bin_type=True, default=encoder)
+
+
+def json_dumps(data, **kw):
+    kw['default'] = encoder
+    kw['ensure_ascii'] = False
+    ans = json.dumps(data, **kw)
+    if not isinstance(ans, bytes):
+        ans = ans.encode('utf-8')
+    return ans
+
+
+def decoder(obj):
+    dt = obj.get('__datetime__')
+    if dt is not None:
+        obj = parse_iso8601(dt, assume_utc=True)
+    return obj
+
+
+def msgpack_loads(data):
+    import msgpack
+    return msgpack.unpackb(
+        data, encoding='utf-8', use_list=False, object_hook=decoder
+    )
+
+
+def json_loads(data):
+    return json.loads(data, object_hook=decoder)