COmmnad ot make testing extension modules easier

bypy/init_env.py

@@ -165,17 +165,20 @@ def run(*args, **extra_env):
     return subprocess.call(list(args), env=env, cwd=CALIBRE_DIR)
 
 
-def build_c_extensions(ext_dir):
+def build_c_extensions(ext_dir, args):
     bdir = os.path.join(build_dir(), 'calibre-extension-objects')
-    if run(
+    cmd = [
         PYTHON, 'setup.py', 'build',
         '--output-dir', ext_dir, '--build-dir', bdir,
-        COMPILER_CWD=bdir
-    ) != 0:
+    ]
+    if args.build_only:
+        cmd.extend(('--only', args.build_only))
+    if run(*cmd, COMPILER_CWD=bdir) != 0:
         print('Building of calibre C extensions failed', file=sys.stderr)
         os.chdir(CALIBRE_DIR)
         run_shell()
         raise SystemExit('Building of calibre C extensions failed')
+    return ext_dir
 
 
 def run_tests(path_to_calibre_debug, cwd_on_failure):

setup.py

@@ -88,6 +88,7 @@ def main(args=sys.argv):
         'Usage: python setup.py {} [options]\n\n'.format(args[1]) + command.description)
 
     opts, args = parser.parse_args(args)
+    opts.cli_args = args[2:]
 
     if opts.clean_backups:
         clean_backups()

setup/commands.py

@@ -14,7 +14,7 @@ __all__ = [
         'develop', 'install',
         'kakasi', 'rapydscript', 'cacerts', 'recent_uas', 'resources',
         'check', 'to3', 'unicode_check', 'iterators_check', 'test',
-        'sdist', 'bootstrap',
+        'sdist', 'bootstrap', 'extdev',
         'manual', 'tag_release',
         'upload_to_server',
         'upload_installers',
@@ -23,10 +23,11 @@ __all__ = [
         'linux', 'linux32', 'linux64', 'win', 'win32', 'win64', 'osx',
         ]
 
-from setup.installers import Linux, Win, OSX, Linux32, Linux64, Win32, Win64
+from setup.installers import Linux, Win, OSX, Linux32, Linux64, Win32, Win64, ExtDev
 linux, linux32, linux64 = Linux(), Linux32(), Linux64()
 win, win32, win64 = Win(), Win32(), Win64()
 osx = OSX()
+extdev = ExtDev()
 
 from setup.translations import POT, GetTranslations, Translations, ISO639, ISO3166
 pot = POT()

setup/installers.py

@@ -3,13 +3,15 @@
 # License: GPLv3 Copyright: 2016, Kovid Goyal <kovid at kovidgoyal.net>
 from __future__ import absolute_import, division, print_function, unicode_literals
 
-import os, sys, subprocess
+import os, sys, subprocess, binascii, json
 
 from setup import Command
 
 
-def build_single(which, bitness, shutdown=True):
 d = os.path.dirname
+
+
+def get_paths():
     base = d(d(os.path.abspath(__file__)))
     bypy = os.path.join(d(base), 'bypy')
     bypy = os.environ.get('BYPY_LOCATION', bypy)
@@ -17,7 +19,14 @@ def build_single(which, bitness, shutdown=True):
         raise SystemExit(
             'Cannot find the bypy code. Set the environment variable BYPY_LOCATION to point to it'
         )
-    exe = 'python3' if sys.version_info.major == 2 else sys.executable
+    return base, bypy
+
+
+def get_exe():
+    return 'python3' if sys.version_info.major == 2 else sys.executable
+
+
+def get_cmd(exe, bypy, which, bitness):
     cmd = [exe, bypy, which]
     if bitness and bitness == '32':
         cmd.append(bitness)
@@ -25,9 +34,10 @@ def build_single(which, bitness, shutdown=True):
     if not sys.stdout.isatty():
         cmd.append('--no-tty')
     cmd.append('--sign-installers')
-    ret = subprocess.Popen(cmd).wait()
-    if ret != 0:
-        raise SystemExit(ret)
+    return cmd
+
+
+def get_dist(base, which, bitness):
     dist = os.path.join(base, 'bypy', 'b', which)
     if bitness:
         dist = os.path.join(dist, bitness)
@@ -35,14 +45,44 @@ def build_single(which, bitness, shutdown=True):
         if os.path.exists(os.path.join(dist, q)):
             dist = os.path.join(dist, q)
             break
+    return dist
+
+
+def build_only(which, bitness, spec, shutdown=False):
+    base, bypy = get_paths()
+    exe = get_exe()
+    cmd = get_cmd(exe, bypy, which, bitness)
+    cmd.extend(['--build-only', spec])
+    ret = subprocess.Popen(cmd).wait()
+    if ret != 0:
+        raise SystemExit(ret)
+    dist = get_dist(base, which, bitness)
+    dist = os.path.join(dist, 'c-extensions')
+    if shutdown:
+        cmd = [exe, bypy, which, 'shutdown']
+        subprocess.Popen(cmd).wait()
+    return dist
+
+
+def build_single(which='windows', bitness='64', shutdown=True):
+    base, bypy = get_paths()
+    exe = get_exe()
+    cmd = get_cmd(exe, bypy, which, bitness)
+    ret = subprocess.Popen(cmd).wait()
+    if ret != 0:
+        raise SystemExit(ret)
+    dist = get_dist(base, which, bitness)
     for x in os.listdir(dist):
+        src = os.path.join(dist, x)
+        if os.path.isdir(src):
+            continue
         print(x)
         dest = os.path.join(base, 'dist', x)
         try:
             os.remove(dest)
         except EnvironmentError:
             pass
-        os.link(os.path.join(dist, x), dest)
+        os.link(src, dest)
     if shutdown:
         cmd = [exe, bypy, which, 'shutdown']
         subprocess.Popen(cmd).wait()
@@ -109,3 +149,51 @@ class Linux(BuildInstallers):
 
 class Win(BuildInstallers):
     OS = 'win'
+
+
+class ExtDev(Command):
+
+    description = 'Develop a single native extension conveniently'
+
+    def run(self, opts):
+        which, ext = opts.cli_args[:2]
+        cmd = opts.cli_args[2:]
+        bitness = '64' if which == 'windows' else ''
+        ext_dir = build_only(which, bitness, ext)
+        if which == 'windows':
+            host = 'win'
+            path = '/cygdrive/c/Program Files/Calibre2/app/bin/{}.pyd'
+            bin_dir = '/cygdrive/c/Program Files/Calibre2'
+        elif which == 'macos':
+            host = 'ox'
+            path = '/Applications/calibre.app/Contents/Frameworks/plugins/{}.so'
+            bin_dir = '/Applications/calibre.app/Contents/MacOS'
+        control_path = os.path.expanduser('~/.ssh/extdev-master-%C')
+        if subprocess.Popen([
+            'ssh', '-o', 'ControlMaster=auto', '-o', 'ControlPath=' + control_path, '-o', 'ControlPersist=yes', host,
+            'echo', 'ssh master running'
+        ]).wait() != 0:
+            raise SystemExit(1)
+        try:
+            path = path.format(ext)
+            src = os.path.join(ext_dir, os.path.basename(path))
+            subprocess.check_call(['ssh', '-S', control_path, host, 'chmod', '+w', '"{}"'.format(path)])
+            with open(src, 'rb') as f:
+                p = subprocess.Popen(['ssh', '-S', control_path, host, 'cat - > "{}"'.format(path)], stdin=subprocess.PIPE)
+                p.communicate(f.read())
+            if p.wait() != 0:
+                raise SystemExit(1)
+            subprocess.check_call(['ssh', '-S', control_path, host, './update-calibre'])
+            enc = json.dumps(cmd)
+            if not isinstance(enc, bytes):
+                enc = enc.encode('utf-8')
+            enc = binascii.hexlify(enc).decode('ascii')
+            cmd = ['"{}"'.format(os.path.join(bin_dir, 'calibre-debug')), '-c', '"'
+                    'import sys, json, binascii, os; cmd = json.loads(binascii.unhexlify(sys.argv[-1]));'
+                    '''os.environ['CALIBRE_DEVELOP_FROM'] = os.path.expanduser('~/calibre-src/src');'''
+                    'from calibre.debug import get_debug_executable; exe_dir = os.path.dirname(get_debug_executable());'
+                    'os.execv(os.path.join(exe_dir, cmd[0]), cmd)'
+                    '"', enc]
+            subprocess.check_call(['ssh', '-S', control_path, host] + cmd)
+        finally:
+            subprocess.Popen(['ssh', '-O', 'exit', '-S', control_path, host])