From 4488703e1e11c5caff6cb5c2c60d1c3761c34ae1 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 18 Jul 2016 13:32:07 +0530 Subject: [PATCH] Start work on setting up CO testing on windows with AppVeyor --- appveyor.yml | 4 + setup/build_environment.py | 28 +---- setup/win-ci.py | 203 +++++++++++++++++++++++++++++++++++++ 3 files changed, 208 insertions(+), 27 deletions(-) create mode 100644 setup/win-ci.py diff --git a/appveyor.yml b/appveyor.yml index 9efa153e3a..a282516a36 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -8,6 +8,10 @@ branches: - vs2015 cache: +<<<<<<< HEAD +======= + - node_modules +>>>>>>> Start work on setting up CO testing on windows with AppVeyor - .build-cache environment: diff --git a/setup/build_environment.py b/setup/build_environment.py index 50382ea5b3..8f9383f97f 100644 --- a/setup/build_environment.py +++ b/setup/build_environment.py @@ -84,7 +84,7 @@ pyqt['sip_bin'] = os.environ.get('SIP_BIN', 'sip') from PyQt5.QtCore import PYQT_CONFIGURATION pyqt['sip_flags'] = PYQT_CONFIGURATION['sip_flags'] def get_sip_dir(): - q = os.environ.get('SIP_DIR', sys.prefix if iswindows else os.path.join(sys.prefix, 'share', 'sip')) + q = os.environ.get('SIP_DIR', os.path.join(sys.prefix, 'share', 'sip') if iswindows else os.path.join(sys.prefix, 'share', 'sip')) for x in ('', 'Py2-PyQt5', 'PyQt5', 'sip/PyQt5'): base = os.path.join(q, x) if os.path.exists(os.path.join(base, 'QtWidgets')): @@ -112,26 +112,7 @@ openssl_inc_dirs, openssl_lib_dirs = [], [] icu_libs = ['icudata', 'icui18n', 'icuuc', 'icuio'] ICU = sw = '' -QT_DLLS = ['Qt5' + x for x in ( -'Core', 'Gui', 'OpenGL', 'Network', 'PrintSupport', 'Positioning', 'Sensors', 'Sql', 'Svg', -'WebKit', 'WebKitWidgets', 'Widgets', 'Multimedia', 'MultimediaWidgets', 'Xml', # 'XmlPatterns', -)] -QT_PLUGINS = ('imageformats', 'audio', 'iconengines', 'mediaservice', 'platforms', 'playlistformats', 'printsupport', 'sqldrivers') -if islinux or ishaiku: - # platformthemes cause crashes in Ubuntu - QT_PLUGINS += ('platforminputcontexts', 'generic',) - -PYQT_MODULES = ('Qt', 'QtCore', 'QtGui', 'QtNetwork', # 'QtMultimedia', 'QtMultimediaWidgets', - 'QtPrintSupport', 'QtSensors', 'QtSvg', 'QtWebKit', 'QtWebKitWidgets', 'QtWidgets') -QT_FRAMEWORKS = [] - - if iswindows: - icu_libs = ['icudt', 'icuin', 'icuuc', 'icuio'] - QT_DLLS += ['Qt5WinExtras'] - QT_DLLS = {x + '.dll' for x in QT_DLLS} - PYQT_MODULES += ('QtWinExtras',) - PYQT_MODULES = {x + '.pyd' for x in PYQT_MODULES} prefix = sw = os.environ.get('SW', r'C:\cygwin64\home\kovid\sw') sw_inc_dir = os.path.join(prefix, 'include') sw_lib_dir = os.path.join(prefix, 'lib') @@ -150,9 +131,6 @@ if iswindows: podofo_inc = os.path.join(sw_inc_dir, 'podofo') podofo_lib = sw_lib_dir elif isosx: - QT_DLLS += ['Qt5DBus', 'Qt5MacExtras'] - PYQT_MODULES += ('QtMacExtras',) - QT_FRAMEWORKS = [x.replace('5', '') for x in QT_DLLS] sw = os.environ.get('SW', os.path.expanduser('~/sw')) podofo_inc = os.path.join(sw, 'include', 'podofo') podofo_lib = os.path.join(sw, 'lib') @@ -164,8 +142,6 @@ elif isosx: openssl_inc_dirs = [os.path.join(SSL, 'include')] openssl_lib_dirs = [os.path.join(SSL, 'lib')] else: - QT_DLLS += ['Qt5DBus', 'Qt5XcbQpa', 'Qt5X11Extras'] - PYQT_MODULES += ('QtX11Extras',) ft_inc_dirs = pkgconfig_include_dirs('freetype2', 'FT_INC_DIR', '/usr/include/freetype2') ft_lib_dirs = pkgconfig_lib_dirs('freetype2', 'FT_LIB_DIR', '/usr/lib') @@ -188,5 +164,3 @@ podofo_inc = [podofo_inc, os.path.dirname(podofo_inc)] BUILD_HOST='192.168.81.1' PROJECT=os.path.basename(os.path.abspath('.')) - - diff --git a/setup/win-ci.py b/setup/win-ci.py new file mode 100644 index 0000000000..aba072ac6f --- /dev/null +++ b/setup/win-ci.py @@ -0,0 +1,203 @@ +#!/usr/bin/env python2 +# vim:fileencoding=utf-8 +# License: GPLv3 Copyright: 2016, Kovid Goyal + +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +import os +import io +import sys +import subprocess +import tarfile + +def vcvars(is64bit=True): # {{{ + import _winreg as winreg + RegOpenKeyEx = winreg.OpenKeyEx + RegEnumValue = winreg.EnumValue + RegError = winreg.error + + HKEYS = (winreg.HKEY_USERS, + winreg.HKEY_CURRENT_USER, + winreg.HKEY_LOCAL_MACHINE, + winreg.HKEY_CLASSES_ROOT) + VS_BASE = r"Software\Wow6432Node\Microsoft\VisualStudio\%0.1f" + + def get_reg_value(path, key): + for base in HKEYS: + d = read_values(base, path) + if d and key in d: + return d[key] + raise KeyError(key) + + def convert_mbcs(s): + dec = getattr(s, "decode", None) + if dec is not None: + try: + s = dec("mbcs") + except UnicodeError: + pass + return s + + def read_values(base, key): + """Return dict of registry keys and values. + + All names are converted to lowercase. + """ + try: + handle = RegOpenKeyEx(base, key) + except RegError: + return None + d = {} + i = 0 + while True: + try: + name, value, type = RegEnumValue(handle, i) + except RegError: + break + name = name.lower() + d[convert_mbcs(name)] = convert_mbcs(value) + i += 1 + return d + + def find_vcvarsall(version=14.0): + vsbase = VS_BASE % version + try: + productdir = get_reg_value(r"%s\Setup\VC" % vsbase, "productdir") + except KeyError: + raise SystemExit("Unable to find Visual Studio product directory in the registry") + + if not productdir: + raise SystemExit("No productdir found") + vcvarsall = os.path.join(productdir, "vcvarsall.bat") + if os.path.isfile(vcvarsall): + return vcvarsall + raise SystemExit("Unable to find vcvarsall.bat in productdir: " + productdir) + + def remove_dups(variable): + old_list = variable.split(os.pathsep) + new_list = [] + for i in old_list: + if i not in new_list: + new_list.append(i) + return os.pathsep.join(new_list) + + def query_process(cmd): + if is64bit and 'PROGRAMFILES(x86)' not in os.environ: + os.environ['PROGRAMFILES(x86)'] = os.environ['PROGRAMFILES'] + ' (x86)' + result = {} + popen = subprocess.Popen(cmd, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + try: + stdout, stderr = popen.communicate() + if popen.wait() != 0: + raise RuntimeError(stderr.decode("mbcs")) + + stdout = stdout.decode("mbcs") + for line in stdout.splitlines(): + if '=' not in line: + continue + line = line.strip() + key, value = line.split('=', 1) + key = key.lower() + if key == 'path': + if value.endswith(os.pathsep): + value = value[:-1] + value = remove_dups(value) + result[key] = value + + finally: + popen.stdout.close() + popen.stderr.close() + return result + + def query_vcvarsall(): + plat = 'amd64' if is64bit else 'x86' + vcvarsall = find_vcvarsall() + env = query_process('"%s" %s & set' % (vcvarsall, plat)) + + def g(k): + try: + return env[k] + except KeyError: + return env[k.lower()] + + # We have to insert the correct path to MSBuild.exe so that the one + # from the .net frameworks is not used. + paths = g('PATH').split(os.pathsep) + for i, p in enumerate(tuple(paths)): + if os.path.exists(os.path.join(p, 'MSBuild.exe')): + if '.net' in p.lower(): + paths.insert(i, r'C:\Program Files (x86)\MSBuild\14.0\bin' + (r'\amd64' if is64bit else '')) + env["PATH"] = os.pathsep.join(paths) + break + + return {k: g(k) for k in 'PATH LIB INCLUDE LIBPATH WINDOWSSDKDIR VS140COMNTOOLS UCRTVERSION UNIVERSALCRTSDKDIR'.split()} + + return query_vcvarsall() +# }}} + + +def printf(*args, **kw): + print(*args, **kw) + sys.stdout.flush() + +def sw(): + sw=os.environ['SW'] + os.makedirs(sw) + os.chdir(sw) + url = 'https://download.calibre-ebook.com/travis/win-64.tar.xz' + printf('Downloading', url) + tarball = subprocess.check_output(['curl.exe', '-fsSL', url]) + with tarfile.open(fileobj=io.BytesIO(tarball)) as tf: + tf.extractall() + printf('Download complete') + + +def sanitize_path(): + needed_paths = [] + executables = 'git.exe curl.exe'.split() + for p in os.environ['PATH'].split(os.pathsep): + for x in tuple(executables): + if os.path.exists(os.path.join(p, x)): + needed_paths.append(p) + executables.remove(x) + sw = os.environ['SW'] + paths = r'{0}\private\python\DLLs {0}\private\python\Lib\site-packages\pywin32_system32 {0}\bin {0}\qt\bin C:\Windows\System32'.format( + sw).split() + needed_paths + os.environ[b'PATH'] = os.pathsep.join(paths).encode('ascii') + +def vcenv(): + env = os.environ.copy() + env.update(vcvars()) + return {str(k):str(v) for k, v in env.items()} + +def build(): + sanitize_path() + cmd = [sys.executable, 'setup.py', 'bootstrap', '--ephemeral'] + printf(*cmd) + p = subprocess.Popen(cmd, env=vcenv()) + raise SystemExit(p.wait()) + +def test(): + sanitize_path() + cmd = [sys.executable, 'setup.py', 'test'] + printf(*cmd) + p = subprocess.Popen(cmd) + raise SystemExit(p.wait()) + +def main(): + q = sys.argv[-1] + if q == 'build': + build() + elif q == 'test': + test() + elif q == 'sw': + sw() + else: + if len(sys.argv) == 1: + raise SystemExit('Usage: win-ci.py sw|build|test') + raise SystemExit('%r is not a valid action' % sys.argv[-1]) + +if __name__ == '__main__': + main()