Start work on refactoring calibredb to make it work with remote libraries

src/calibre/db/cli/__init__.py

@@ -0,0 +1,5 @@
+#!/usr/bin/env python2
+# vim:fileencoding=utf-8
+# License: GPLv3 Copyright: 2017, Kovid Goyal <kovid at kovidgoyal.net>
+
+from __future__ import absolute_import, division, print_function, unicode_literals

src/calibre/db/cli/cmd_list.py

@@ -0,0 +1,330 @@
+#!/usr/bin/env python2
+# vim:fileencoding=utf-8
+# License: GPLv3 Copyright: 2017, Kovid Goyal <kovid at kovidgoyal.net>
+
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+import json
+import os
+import sys
+import unicodedata
+from textwrap import TextWrapper
+
+from calibre import prints
+from calibre.ebooks.metadata import authors_to_string
+from calibre.utils.date import isoformat
+
+readonly = True
+FIELDS = {
+    'title', 'authors', 'author_sort', 'publisher', 'rating', 'timestamp', 'size',
+    'tags', 'comments', 'series', 'series_index', 'formats', 'isbn', 'uuid',
+    'pubdate', 'cover', 'last_modified', 'identifiers', 'languages'
+}
+
+
+def formats(db, book_id):
+    for fmt in db.formats(book_id, verify_formats=False):
+        path = db.format_abspath(book_id, fmt)
+        if path:
+            yield path.replace(os.sep, '/')
+
+
+def cover(db, book_id):
+    return db.format_abspath(book_id, '__COVER_INTERNAL__')
+
+
+def implementation(
+    db, is_remote, fields, sort_by, ascending, search_text, limit
+):
+    with db.safe_read_lock:
+        fm = db.field_metadata
+        afields = set(FIELDS) | {'id'}
+        for k in fm.custom_field_keys():
+            afields.add('*' + k[1:])
+        if 'all' in fields:
+            fields = sorted(afields)
+        sort_by = sort_by or 'id'
+        if sort_by not in afields:
+            return 'Unknown sort field: {}'.format(sort_by)
+        if not set(fields).issubset(afields):
+            return 'Unknown fields: {}'.format(', '.join(set(fields) - afields))
+        if search_text:
+            book_ids = db.multisort([(sort_by, ascending)],
+                                    ids_to_sort=db.search(search_text))
+        else:
+            book_ids = db.multisort([(sort_by, ascending)])
+        if limit > -1:
+            book_ids = book_ids[:limit]
+        data = {}
+        metadata = {}
+        for field in fields:
+            if field in 'id':
+                continue
+            if field == 'isbn':
+                x = db.all_field_for('identifiers', book_ids, default_value={})
+                data[field] = {k: v.get('isbn') or '' for k, v in x.iteritems()}
+                continue
+            field = field.replace('*', '#')
+            metadata[field] = fm[field]
+            if not is_remote:
+                if field == 'formats':
+                    data[field] = {k: list(formats(db, k)) for k in book_ids}
+                    continue
+                if field == 'cover':
+                    data[field] = {k: cover(db, k) for k in book_ids}
+                    continue
+            data[field] = db.all_field_for(field, book_ids)
+    return {'book_ids': book_ids, "data": data, 'metadata': metadata, 'fields':fields}
+
+
+def stringify(data, metadata, for_machine):
+    for field, m in metadata.iteritems():
+        if field == 'authors':
+            data[field] = {
+                k: authors_to_string(v)
+                for k, v in data[field].iteritems()
+            }
+        else:
+            dt = m['datatype']
+            if dt == 'datetime':
+                data[field] = {
+                    k: isoformat(v, as_utc=for_machine) if v else 'None'
+                    for k, v in data[field].iteritems()
+                }
+            elif not for_machine:
+                ism = m['is_multiple']
+                if ism:
+                    data[field] = {
+                        k: ism['list_to_ui'].join(v)
+                        for k, v in data[field].iteritems()
+                    }
+                    if field == 'formats':
+                        data[field] = {
+                            k: '[' + v + ']'
+                            for k, v in data[field].iteritems()
+                        }
+
+
+def as_machine_data(book_ids, data, metadata):
+    for book_id in book_ids:
+        ans = {'id': book_id}
+        for field, val_map in data.iteritems():
+            val = val_map.get(book_id)
+            if val is not None:
+                ans[field.replace('#', '*')] = val
+        yield ans
+
+
+def prepare_output_table(fields, book_ids, data, metadata):
+    ans = []
+    u = type('')
+    for book_id in book_ids:
+        row = []
+        ans.append(row)
+        for field in fields:
+            if field == 'id':
+                row.append(u(book_id))
+                continue
+            val = data.get(field.replace('*', '#'), {}).get(book_id)
+            row.append(u(val).replace('\n', ' '))
+    return ans
+
+
+def do_list(
+    dbctx,
+    fields,
+    afields,
+    sort_by,
+    ascending,
+    search_text,
+    line_width,
+    separator,
+    prefix,
+    limit,
+    for_machine=False
+):
+    if sort_by is None:
+        ascending = True
+    ans = dbctx.run('list', fields, sort_by, ascending, search_text, limit)
+    try:
+        book_ids, data, metadata = ans['book_ids'], ans['data'], ans['metadata']
+    except TypeError:
+        raise SystemExit(ans)
+    fields = ans['fields']
+    try:
+        fields.remove('id')
+    except ValueError:
+        pass
+    fields = ['id'] + fields
+    stringify(data, metadata, for_machine)
+    if for_machine:
+        json.dump(
+            list(as_machine_data(book_ids, data, metadata)),
+            sys.stdout,
+            indent=2,
+            sort_keys=True
+        )
+        return
+    from calibre.utils.terminal import ColoredStream, geometry
+
+    output_table = prepare_output_table(fields, book_ids, data, metadata)
+    widths = list(map(lambda x: 0, fields))
+
+    def chr_width(x):
+        return 1 + unicodedata.east_asian_width(x).startswith('W')
+
+    def str_width(x):
+        return sum(map(chr_width, x))
+
+    for record in output_table:
+        for j in range(len(fields)):
+            widths[j] = max(widths[j], str_width(record[j]))
+
+    screen_width = geometry()[0] if line_width < 0 else line_width
+    if not screen_width:
+        screen_width = 80
+    field_width = screen_width // len(fields)
+    base_widths = map(lambda x: min(x + 1, field_width), widths)
+
+    while sum(base_widths) < screen_width:
+        adjusted = False
+        for i in range(len(widths)):
+            if base_widths[i] < widths[i]:
+                base_widths[i] += min(
+                    screen_width - sum(base_widths), widths[i] - base_widths[i]
+                )
+                adjusted = True
+                break
+        if not adjusted:
+            break
+
+    widths = list(base_widths)
+    titles = map(
+        lambda x, y: '%-*s%s' % (x - len(separator), y, separator), widths,
+        fields
+    )
+    with ColoredStream(sys.stdout, fg='green'):
+        prints(''.join(titles))
+
+    wrappers = [TextWrapper(x - 1).wrap if x > 1 else lambda y: y for x in widths]
+
+    for record in output_table:
+        text = [
+            wrappers[i](record[i]) for i, field in enumerate(fields)
+        ]
+        lines = max(map(len, text))
+        for l in range(lines):
+            for i, field in enumerate(text):
+                ft = text[i][l] if l < len(text[i]) else u''
+                sys.stdout.write(ft.encode('utf-8'))
+                if i < len(text) - 1:
+                    filler = (u'%*s' % (widths[i] - str_width(ft) - 1, u''))
+                    sys.stdout.write((filler + separator).encode('utf-8'))
+            print()
+
+
+def option_parser(get_parser):
+    parser = get_parser(
+        _(
+            '''\
+%prog list [options]
+
+List the books available in the calibre database.
+'''
+        )
+    )
+    parser.add_option(
+        '-f',
+        '--fields',
+        default='title,authors',
+        help=_(
+            'The fields to display when listing books in the'
+            ' database. Should be a comma separated list of'
+            ' fields.\nAvailable fields: %s\nDefault: %%default. The'
+            ' special field "all" can be used to select all fields.'
+            ' In addition to the builtin fields above, custom fields are'
+            ' also available as *field_name, for example, for a custom field'
+            ' #rating, use the name: *rating'
+        ) % ', '.join(sorted(FIELDS))
+    )
+    parser.add_option(
+        '--sort-by',
+        default=None,
+        help=_(
+            'The field by which to sort the results.\nAvailable fields: {0}\nDefault: {1}'
+        ).format(', '.join(sorted(FIELDS)), 'id')
+    )
+    parser.add_option(
+        '--ascending',
+        default=False,
+        action='store_true',
+        help=_('Sort results in ascending order')
+    )
+    parser.add_option(
+        '-s',
+        '--search',
+        default=None,
+        help=_(
+            'Filter the results by the search query. For the format of the search query,'
+            ' please see the search related documentation in the User Manual. Default is to do no filtering.'
+        )
+    )
+    parser.add_option(
+        '-w',
+        '--line-width',
+        default=-1,
+        type=int,
+        help=_(
+            'The maximum width of a single line in the output. Defaults to detecting screen size.'
+        )
+    )
+    parser.add_option(
+        '--separator',
+        default=' ',
+        help=_('The string used to separate fields. Default is a space.')
+    )
+    parser.add_option(
+        '--prefix',
+        default=None,
+        help=_(
+            'The prefix for all file paths. Default is the absolute path to the library folder.'
+        )
+    )
+    parser.add_option(
+        '--limit',
+        default=-1,
+        type=int,
+        help=_('The maximum number of results to display. Default: all')
+    )
+    parser.add_option(
+        '--for-machine',
+        default=False,
+        action='store_true',
+        help=_(
+            'Generate output in JSON format, which is more suitable for machine parsing. Causes the line width and separator options to be ignored.'
+        )
+    )
+    return parser
+
+
+def main(opts, args, dbctx):
+    afields = set(FIELDS) | {'id'}
+    if opts.fields.strip():
+        fields = [str(f.strip().lower()) for f in opts.fields.split(',')]
+    else:
+        fields = []
+
+    do_list(
+        dbctx,
+        fields,
+        afields,
+        opts.sort_by,
+        opts.ascending,
+        opts.search,
+        opts.line_width,
+        opts.separator,
+        opts.prefix,
+        opts.limit,
+        for_machine=opts.for_machine
+    )
+    return 0

src/calibre/db/cli/main.py

@@ -0,0 +1,152 @@
+#!/usr/bin/env python2
+# vim:fileencoding=utf-8
+# License: GPLv3 Copyright: 2017, Kovid Goyal <kovid at kovidgoyal.net>
+
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+import importlib
+import os
+import sys
+
+from calibre import prints
+from calibre.utils.config import OptionParser, prefs
+
+COMMANDS = (
+    'list', 'add', 'remove', 'add_format', 'remove_format', 'show_metadata',
+    'set_metadata', 'export', 'catalog', 'saved_searches', 'add_custom_column',
+    'custom_columns', 'remove_custom_column', 'set_custom', 'restore_database',
+    'check_library', 'list_categories', 'backup_metadata', 'clone', 'embed_metadata',
+    'search'
+)
+
+
+def module_for_cmd(cmd):
+    return importlib.import_module('calibre.db.cli.cmd_' + cmd)
+
+
+def option_parser_for(cmd):
+    def cmd_option_parser():
+        return module_for_cmd(cmd).option_parser(get_parser)
+    return cmd_option_parser
+
+
+def send_message(msg=''):
+    prints('Notifying calibre of the change')
+    from calibre.utils.ipc import RC
+    t = RC(print_error=False)
+    t.start()
+    t.join(3)
+    if t.done:
+        t.conn.send('refreshdb:' + msg)
+        t.conn.close()
+
+
+def run_cmd(cmd, opts, args, db_ctx):
+    m = module_for_cmd(cmd)
+    ret = m.main(opts, args, db_ctx)
+    if not opts.dont_notify_gui and not getattr(m, 'readonly', False):
+        send_message()
+    return ret
+
+
+def get_parser(usage):
+    parser = OptionParser(usage)
+    go = parser.add_option_group(_('GLOBAL OPTIONS'))
+    go.is_global_options = True
+    go.add_option(
+        '--library-path',
+        '--with-library',
+        default=None,
+        help=_(
+            'Path to the calibre library. Default is to use the path stored in the settings.'
+        )
+    )
+    go.add_option(
+        '--dont-notify-gui',
+        default=False,
+        action='store_true',
+        help=_(
+            'Do not notify the running calibre GUI (if any) that the database has'
+            ' changed. Use with care, as it can lead to database corruption!'
+        )
+    )
+    go.add_option(
+        '-h', '--help', help=_('show this help message and exit'), action='help'
+    )
+    go.add_option(
+        '--version',
+        help=_("show program's version number and exit"),
+        action='version'
+    )
+
+    return parser
+
+
+def option_parser():
+    parser = OptionParser(
+        _(
+            '''\
+%%prog command [options] [arguments]
+
+%%prog is the command line interface to the calibre books database.
+
+command is one of:
+  %s
+
+For help on an individual command: %%prog command --help
+'''
+        ) % '\n  '.join(COMMANDS)
+    )
+    return parser
+
+
+class DBCtx(object):
+
+    def __init__(self, opts):
+        self.library_path = opts.library_path or prefs['library_path']
+        self.url = None
+        if self.library_path is None:
+            raise SystemExit('No saved library path, either run the GUI or use the'
+                             ' --with-library option')
+        if self.library_path.partition(':')[0] in ('http', 'https'):
+            self.url = self.library_path
+            self.is_remote = True
+        else:
+            self.library_path = os.path.expanduser(self.library_path)
+            self._db = None
+            self.is_remote = False
+
+    @property
+    def db(self):
+        if self._db is None:
+            from calibre.db.legacy import LibraryDatabase
+            self._db = LibraryDatabase(self.library_path).new_api
+        return self._db
+
+    def run(self, name, *args):
+        if self.is_remote:
+            raise NotImplementedError()
+        m = module_for_cmd(name)
+        return m.implementation(self.db, False, *args)
+
+
+def main(args=sys.argv):
+    parser = option_parser()
+    if len(args) < 2:
+        parser.print_help()
+        return 1
+    cmd = args[1]
+    if cmd not in COMMANDS:
+        if cmd == '--version':
+            parser.print_version()
+            return 0
+        parser.print_help()
+        return 1
+
+    parser = option_parser_for(cmd)()
+    opts, args = parser.parse_args(args)
+    return run_cmd(cmd, opts, args[2:], DBCtx(opts))
+
+
+if __name__ == '__main__':
+    main()

src/calibre/db/cli/opts.py

@@ -0,0 +1,66 @@
+#!/usr/bin/env python2
+# vim:fileencoding=utf-8
+# License: GPLv3 Copyright: 2017, Kovid Goyal <kovid at kovidgoyal.net>
+
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+from calibre.utils.config import OptionParser
+
+COMMANDS = (
+    'list', 'add', 'remove', 'add_format', 'remove_format', 'show_metadata',
+    'set_metadata', 'export', 'catalog', 'saved_searches', 'add_custom_column',
+    'custom_columns', 'remove_custom_column', 'set_custom', 'restore_database',
+    'check_library', 'list_categories', 'backup_metadata', 'clone', 'embed_metadata',
+    'search'
+)
+
+
+def get_parser(usage):
+    parser = OptionParser(usage)
+    go = parser.add_option_group(_('GLOBAL OPTIONS'))
+    go.is_global_options = True
+    go.add_option(
+        '--library-path',
+        '--with-library',
+        default=None,
+        help=_(
+            'Path to the calibre library. Default is to use the path stored in the settings.'
+        )
+    )
+    go.add_option(
+        '--dont-notify-gui',
+        default=False,
+        action='store_true',
+        help=_(
+            'Do not notify the running calibre GUI (if any) that the database has'
+            ' changed. Use with care, as it can lead to database corruption!'
+        )
+    )
+    go.add_option(
+        '-h', '--help', help=_('show this help message and exit'), action='help'
+    )
+    go.add_option(
+        '--version',
+        help=_("show program's version number and exit"),
+        action='version'
+    )
+
+    return parser
+
+
+def option_parser():
+    parser = OptionParser(
+        _(
+            '''\
+%%prog command [options] [arguments]
+
+%%prog is the command line interface to the calibre books database.
+
+command is one of:
+  %s
+
+For help on an individual command: %%prog command --help
+'''
+        ) % '\n  '.join(COMMANDS)
+    )
+    return parser