From ab6b7248350921027eb01bd4f78cc8b4d85cd3f7 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 4 Sep 2019 19:52:33 +0530 Subject: [PATCH] COmmnad ot make testing extension modules easier --- bypy/init_env.py | 11 +++-- setup.py | 1 + setup/commands.py | 5 ++- setup/installers.py | 104 ++++++++++++++++++++++++++++++++++++++++---- 4 files changed, 107 insertions(+), 14 deletions(-) diff --git a/bypy/init_env.py b/bypy/init_env.py index 193d7bd87e..99a39554fb 100644 --- a/bypy/init_env.py +++ b/bypy/init_env.py @@ -165,17 +165,20 @@ def run(*args, **extra_env): return subprocess.call(list(args), env=env, cwd=CALIBRE_DIR) -def build_c_extensions(ext_dir): +def build_c_extensions(ext_dir, args): bdir = os.path.join(build_dir(), 'calibre-extension-objects') - if run( + cmd = [ PYTHON, 'setup.py', 'build', '--output-dir', ext_dir, '--build-dir', bdir, - COMPILER_CWD=bdir - ) != 0: + ] + if args.build_only: + cmd.extend(('--only', args.build_only)) + if run(*cmd, COMPILER_CWD=bdir) != 0: print('Building of calibre C extensions failed', file=sys.stderr) os.chdir(CALIBRE_DIR) run_shell() raise SystemExit('Building of calibre C extensions failed') + return ext_dir def run_tests(path_to_calibre_debug, cwd_on_failure): diff --git a/setup.py b/setup.py index fb8b0991c4..97e029fe38 100755 --- a/setup.py +++ b/setup.py @@ -88,6 +88,7 @@ def main(args=sys.argv): 'Usage: python setup.py {} [options]\n\n'.format(args[1]) + command.description) opts, args = parser.parse_args(args) + opts.cli_args = args[2:] if opts.clean_backups: clean_backups() diff --git a/setup/commands.py b/setup/commands.py index d7681584a2..a3c67cd99a 100644 --- a/setup/commands.py +++ b/setup/commands.py @@ -14,7 +14,7 @@ __all__ = [ 'develop', 'install', 'kakasi', 'rapydscript', 'cacerts', 'recent_uas', 'resources', 'check', 'to3', 'unicode_check', 'iterators_check', 'test', - 'sdist', 'bootstrap', + 'sdist', 'bootstrap', 'extdev', 'manual', 'tag_release', 'upload_to_server', 'upload_installers', @@ -23,10 +23,11 @@ __all__ = [ 'linux', 'linux32', 'linux64', 'win', 'win32', 'win64', 'osx', ] -from setup.installers import Linux, Win, OSX, Linux32, Linux64, Win32, Win64 +from setup.installers import Linux, Win, OSX, Linux32, Linux64, Win32, Win64, ExtDev linux, linux32, linux64 = Linux(), Linux32(), Linux64() win, win32, win64 = Win(), Win32(), Win64() osx = OSX() +extdev = ExtDev() from setup.translations import POT, GetTranslations, Translations, ISO639, ISO3166 pot = POT() diff --git a/setup/installers.py b/setup/installers.py index ea14d9bd08..4c63307209 100644 --- a/setup/installers.py +++ b/setup/installers.py @@ -3,13 +3,15 @@ # License: GPLv3 Copyright: 2016, Kovid Goyal from __future__ import absolute_import, division, print_function, unicode_literals -import os, sys, subprocess +import os, sys, subprocess, binascii, json from setup import Command -def build_single(which, bitness, shutdown=True): - d = os.path.dirname +d = os.path.dirname + + +def get_paths(): base = d(d(os.path.abspath(__file__))) bypy = os.path.join(d(base), 'bypy') bypy = os.environ.get('BYPY_LOCATION', bypy) @@ -17,7 +19,14 @@ def build_single(which, bitness, shutdown=True): raise SystemExit( 'Cannot find the bypy code. Set the environment variable BYPY_LOCATION to point to it' ) - exe = 'python3' if sys.version_info.major == 2 else sys.executable + return base, bypy + + +def get_exe(): + return 'python3' if sys.version_info.major == 2 else sys.executable + + +def get_cmd(exe, bypy, which, bitness): cmd = [exe, bypy, which] if bitness and bitness == '32': cmd.append(bitness) @@ -25,9 +34,10 @@ def build_single(which, bitness, shutdown=True): if not sys.stdout.isatty(): cmd.append('--no-tty') cmd.append('--sign-installers') - ret = subprocess.Popen(cmd).wait() - if ret != 0: - raise SystemExit(ret) + return cmd + + +def get_dist(base, which, bitness): dist = os.path.join(base, 'bypy', 'b', which) if bitness: dist = os.path.join(dist, bitness) @@ -35,14 +45,44 @@ def build_single(which, bitness, shutdown=True): if os.path.exists(os.path.join(dist, q)): dist = os.path.join(dist, q) break + return dist + + +def build_only(which, bitness, spec, shutdown=False): + base, bypy = get_paths() + exe = get_exe() + cmd = get_cmd(exe, bypy, which, bitness) + cmd.extend(['--build-only', spec]) + ret = subprocess.Popen(cmd).wait() + if ret != 0: + raise SystemExit(ret) + dist = get_dist(base, which, bitness) + dist = os.path.join(dist, 'c-extensions') + if shutdown: + cmd = [exe, bypy, which, 'shutdown'] + subprocess.Popen(cmd).wait() + return dist + + +def build_single(which='windows', bitness='64', shutdown=True): + base, bypy = get_paths() + exe = get_exe() + cmd = get_cmd(exe, bypy, which, bitness) + ret = subprocess.Popen(cmd).wait() + if ret != 0: + raise SystemExit(ret) + dist = get_dist(base, which, bitness) for x in os.listdir(dist): + src = os.path.join(dist, x) + if os.path.isdir(src): + continue print(x) dest = os.path.join(base, 'dist', x) try: os.remove(dest) except EnvironmentError: pass - os.link(os.path.join(dist, x), dest) + os.link(src, dest) if shutdown: cmd = [exe, bypy, which, 'shutdown'] subprocess.Popen(cmd).wait() @@ -109,3 +149,51 @@ class Linux(BuildInstallers): class Win(BuildInstallers): OS = 'win' + + +class ExtDev(Command): + + description = 'Develop a single native extension conveniently' + + def run(self, opts): + which, ext = opts.cli_args[:2] + cmd = opts.cli_args[2:] + bitness = '64' if which == 'windows' else '' + ext_dir = build_only(which, bitness, ext) + if which == 'windows': + host = 'win' + path = '/cygdrive/c/Program Files/Calibre2/app/bin/{}.pyd' + bin_dir = '/cygdrive/c/Program Files/Calibre2' + elif which == 'macos': + host = 'ox' + path = '/Applications/calibre.app/Contents/Frameworks/plugins/{}.so' + bin_dir = '/Applications/calibre.app/Contents/MacOS' + control_path = os.path.expanduser('~/.ssh/extdev-master-%C') + if subprocess.Popen([ + 'ssh', '-o', 'ControlMaster=auto', '-o', 'ControlPath=' + control_path, '-o', 'ControlPersist=yes', host, + 'echo', 'ssh master running' + ]).wait() != 0: + raise SystemExit(1) + try: + path = path.format(ext) + src = os.path.join(ext_dir, os.path.basename(path)) + subprocess.check_call(['ssh', '-S', control_path, host, 'chmod', '+w', '"{}"'.format(path)]) + with open(src, 'rb') as f: + p = subprocess.Popen(['ssh', '-S', control_path, host, 'cat - > "{}"'.format(path)], stdin=subprocess.PIPE) + p.communicate(f.read()) + if p.wait() != 0: + raise SystemExit(1) + subprocess.check_call(['ssh', '-S', control_path, host, './update-calibre']) + enc = json.dumps(cmd) + if not isinstance(enc, bytes): + enc = enc.encode('utf-8') + enc = binascii.hexlify(enc).decode('ascii') + cmd = ['"{}"'.format(os.path.join(bin_dir, 'calibre-debug')), '-c', '"' + 'import sys, json, binascii, os; cmd = json.loads(binascii.unhexlify(sys.argv[-1]));' + '''os.environ['CALIBRE_DEVELOP_FROM'] = os.path.expanduser('~/calibre-src/src');''' + 'from calibre.debug import get_debug_executable; exe_dir = os.path.dirname(get_debug_executable());' + 'os.execv(os.path.join(exe_dir, cmd[0]), cmd)' + '"', enc] + subprocess.check_call(['ssh', '-S', control_path, host] + cmd) + finally: + subprocess.Popen(['ssh', '-O', 'exit', '-S', control_path, host])