ebook-convert: Make TAB completion (Linux only) faster

.bzrignore

@@ -11,6 +11,7 @@ resources/localization
 resources/images.qrc
 resources/recipes.pickle
 resources/scripts.pickle
+resources/ebook-convert-complete.pickle
 setup/installer/windows/calibre/build.log
 src/calibre/translations/.errors
 src/cssutils/.svn/

setup/install.py

@@ -11,7 +11,7 @@ import sys, os, textwrap, subprocess, shutil, tempfile, atexit
 from setup import Command, islinux, basenames, modules, functions, \
         __appname__, __version__
 
-TEMPLATE = '''\
+HEADER = '''\
 #!/usr/bin/env python
 
 """
@@ -20,6 +20,9 @@ Do not modify it unless you know what you are doing.
 """
 
 import sys
+'''
+
+TEMPLATE = HEADER+'''
 sys.path.insert(0, {path!r})
 
 sys.resources_location = {resources!r}
@@ -29,6 +32,18 @@ from {module} import {func!s}
 sys.exit({func!s}())
 '''
 
+COMPLETE_TEMPLATE = HEADER+'''
+import os
+sys.path.insert(0, {path!r})
+sys.path.insert(0, os.path.join({path!r}, 'calibre', 'utils'))
+import complete
+sys.path = sys.path[1:]
+
+sys.resources_location = {resources!r}
+sys.extensions_location = {extensions!r}
+sys.exit(complete.main())
+'''
+
 class Develop(Command):
 
     description = textwrap.dedent('''\
@@ -117,7 +132,8 @@ class Develop(Command):
             self.write_template(opts, 'calibre_postinstall', 'calibre.linux', 'main')
 
     def write_template(self, opts, name, mod, func):
-        script = TEMPLATE.format(
+        template = COMPLETE_TEMPLATE if name == 'calibre-complete' else TEMPLATE
+        script = template.format(
                 module=mod, func=func,
                 path=self.path, resources=self.resources,
                 extensions=self.extensions)

setup/resources.py

@@ -10,6 +10,18 @@ import os, cPickle
 
 from setup import Command, basenames
 
+def get_opts_from_parser(parser):
+    def do_opt(opt):
+        for x in opt._long_opts:
+            yield x
+        for x in opt._short_opts:
+            yield x
+    for o in parser.option_list:
+        for x in do_opt(o): yield x
+    for g in parser.option_groups:
+        for o in g.option_list:
+            for x in do_opt(o): yield x
+
 class Resources(Command):
 
     def get_recipes(self):
@@ -45,9 +57,42 @@ class Resources(Command):
             f = open(dest, 'wb')
             cPickle.dump(recipes, f, -1)
 
+        dest = self.j(self.RESOURCES, 'ebook-convert-complete.pickle')
+        files = []
+        for x in os.walk(self.j(self.SRC, 'calibre')):
+            for f in x[-1]:
+                if f.endswith('.py'):
+                    files.append(self.j(x[0], f))
+        if self.newer(dest, files):
+            self.info('\tCreating ebook-convert-complete.pickle')
+            complete = {}
+            from calibre.ebooks.conversion.plumber import supported_input_formats
+            complete['input_fmts'] = set(supported_input_formats())
+            from calibre.web.feeds.recipes import recipes
+            complete['input_recipes'] = [t.title+'.recipe ' for t in recipes]
+            from calibre.customize.ui import available_output_formats
+            complete['output'] = set(available_output_formats())
+            from calibre.ebooks.conversion.cli import create_option_parser
+            from calibre.utils.logging import Log
+            log = Log()
+            #log.outputs = []
+            for inf in supported_input_formats():
+                if inf in ('zip', 'rar', 'oebzip'):
+                    continue
+                for ouf in available_output_formats():
+                    of = ouf if ouf == 'oeb' else 'dummy.'+ouf
+                    p = create_option_parser(('ec', 'dummy1.'+inf, of, '-h'),
+                            log)[0]
+                    complete[(inf, ouf)] = [x+' 'for x in
+                            get_opts_from_parser(p)]
+
+            cPickle.dump(complete, open(dest, 'wb'), -1)
+
+
+
 
     def clean(self):
-        for x in ('scripts', 'recipes'):
+        for x in ('scripts', 'recipes', 'ebook-convert-complete'):
             x = self.j(self.RESOURCES, x+'.pickle')
             if os.path.exists(x):
                 os.remove(x)

src/calibre/utils/complete.py

@@ -12,12 +12,47 @@ BASH completion for calibre commands that are too complex for simple
 completion.
 '''
 
-import sys, os, shlex, glob, re
+import sys, os, shlex, glob, re, cPickle
-from functools import partial
+
+def prints(*args, **kwargs):
+    '''
+    Print unicode arguments safely by encoding them to preferred_encoding
+    Has the same signature as the print function from Python 3, except for the
+    additional keyword argument safe_encode, which if set to True will cause the
+    function to use repr when encoding fails.
+    '''
+    file = kwargs.get('file', sys.stdout)
+    sep  = kwargs.get('sep', ' ')
+    end  = kwargs.get('end', '\n')
+    enc = 'utf-8'
+    safe_encode = kwargs.get('safe_encode', False)
+    for i, arg in enumerate(args):
+        if isinstance(arg, unicode):
+            try:
+                arg = arg.encode(enc)
+            except UnicodeEncodeError:
+                if not safe_encode:
+                    raise
+                arg = repr(arg)
+        if not isinstance(arg, str):
+            try:
+                arg = str(arg)
+            except ValueError:
+                arg = unicode(arg)
+            if isinstance(arg, unicode):
+                try:
+                    arg = arg.encode(enc)
+                except UnicodeEncodeError:
+                    if not safe_encode:
+                        raise
+                    arg = repr(arg)
+
+        file.write(arg)
+        if i != len(args)-1:
+            file.write(sep)
+    file.write(end)
 
-from calibre import prints
 
-debug = partial(prints, file=sys.stderr)
 
 def split(src):
     try:
@@ -47,10 +82,10 @@ def get_opts_from_parser(parser, prefix):
             if x.startswith(prefix):
                 yield x
     for o in parser.option_list:
-        for x in do_opt(o): yield x
+        for x in do_opt(o): yield x+' '
     for g in parser.option_groups:
         for o in g.option_list:
-            for x in do_opt(o): yield x
+            for x in do_opt(o): yield x+' '
 
 def send(ans):
     pat = re.compile('([^0-9a-zA-Z_./-])')
@@ -74,6 +109,8 @@ class EbookConvert(object):
         self.words = words
         self.prefix = prefix
         self.previous = words[-2 if prefix else -1]
+        self.cache = cPickle.load(open(os.path.join(sys.resources_location,
+            'ebook-convert-complete.pickle'), 'rb'))
         self.complete(wc)
 
     def complete(self, wc):
@@ -82,41 +119,42 @@ class EbookConvert(object):
         elif wc == 3:
             self.complete_output()
         else:
-            from calibre.ebooks.conversion.cli import create_option_parser
+            q = list(self.words[1:3])
-            from calibre.utils.logging import Log
+            q = [os.path.splitext(x)[0 if x.startswith('.') else 1].partition('.')[-1].lower() for x in q]
-            log = Log()
+            if not q[1]:
-            log.outputs = []
+                q[1] = 'oeb'
-            ans = []
+            q = tuple(q)
-            if not self.prefix or self.prefix.startswith('-'):
+            if q in self.cache:
-                try:
+                ans = [x for x in self.cache[q] if x.startswith(self.prefix)]
-                    parser, _ = create_option_parser(self.words[:3], log)
+            else:
-                    ans += list(get_opts_from_parser(parser, self.prefix))
+                from calibre.ebooks.conversion.cli import create_option_parser
-                except:
+                from calibre.utils.logging import Log
-                    pass
+                log = Log()
+                log.outputs = []
+                ans = []
+                if not self.prefix or self.prefix.startswith('-'):
+                    try:
+                        parser, _ = create_option_parser(self.words[:3], log)
+                        ans += list(get_opts_from_parser(parser, self.prefix))
+                    except:
+                        pass
             if self.previous.startswith('-'):
                 ans += list(files_and_dirs(self.prefix, None))
             send(ans)
 
     def complete_input(self):
-        from calibre.ebooks.conversion.plumber import supported_input_formats
+        ans = list(files_and_dirs(self.prefix, self.cache['input_fmts']))
-        ans = list(files_and_dirs(self.prefix, supported_input_formats()))
+        ans += [t for t in self.cache['input_recipes'] if
-        from calibre.web.feeds.recipes import recipes
+                t.startswith(self.prefix)]
-        ans += [t.title+'.recipe ' for t in recipes if
-                (t.title+'.recipe').startswith(self.prefix)]
         send(ans)
 
     def complete_output(self):
-        from calibre.customize.ui import available_output_formats
+        fmts = self.cache['output']
-        fmts = available_output_formats()
         ans = list(files_and_dirs(self.prefix, fmts))
         ans += ['.'+x+' ' for x in fmts if ('.'+x).startswith(self.prefix)]
         send(ans)
 
 
-
-
-
-
 def main(args=sys.argv):
     comp_line, pos = os.environ['COMP_LINE'], int(os.environ['COMP_POINT'])
     module = split(comp_line)[0].split(os.sep)[-1]