Start work on setting up CO testing on windows with AppVeyor

appveyor.yml

@@ -8,6 +8,10 @@ branches:
     - vs2015
 
 cache:
+<<<<<<< HEAD
+=======
+    - node_modules
+>>>>>>> Start work on setting up CO testing on windows with AppVeyor
     - .build-cache
 
 environment:

setup/build_environment.py

@@ -84,7 +84,7 @@ pyqt['sip_bin'] = os.environ.get('SIP_BIN', 'sip')
 from PyQt5.QtCore import PYQT_CONFIGURATION
 pyqt['sip_flags'] = PYQT_CONFIGURATION['sip_flags']
 def get_sip_dir():
-    q = os.environ.get('SIP_DIR', sys.prefix if iswindows else os.path.join(sys.prefix, 'share', 'sip'))
+    q = os.environ.get('SIP_DIR', os.path.join(sys.prefix, 'share', 'sip') if iswindows else os.path.join(sys.prefix, 'share', 'sip'))
     for x in ('', 'Py2-PyQt5', 'PyQt5', 'sip/PyQt5'):
         base = os.path.join(q, x)
         if os.path.exists(os.path.join(base, 'QtWidgets')):
@@ -112,26 +112,7 @@ openssl_inc_dirs, openssl_lib_dirs = [], []
 icu_libs = ['icudata', 'icui18n', 'icuuc', 'icuio']
 ICU = sw = ''
 
-QT_DLLS = ['Qt5' + x for x in (
-'Core', 'Gui',  'OpenGL', 'Network', 'PrintSupport', 'Positioning', 'Sensors', 'Sql', 'Svg',
-'WebKit', 'WebKitWidgets', 'Widgets',  'Multimedia', 'MultimediaWidgets', 'Xml',  # 'XmlPatterns',
-)]
-QT_PLUGINS = ('imageformats', 'audio', 'iconengines', 'mediaservice', 'platforms', 'playlistformats', 'printsupport', 'sqldrivers')
-if islinux or ishaiku:
-    # platformthemes cause crashes in Ubuntu
-    QT_PLUGINS += ('platforminputcontexts', 'generic',)
-
-PYQT_MODULES = ('Qt', 'QtCore', 'QtGui', 'QtNetwork',  # 'QtMultimedia', 'QtMultimediaWidgets',
-                'QtPrintSupport', 'QtSensors', 'QtSvg', 'QtWebKit', 'QtWebKitWidgets', 'QtWidgets')
-QT_FRAMEWORKS = []
-
-
 if iswindows:
-    icu_libs = ['icudt', 'icuin', 'icuuc', 'icuio']
-    QT_DLLS += ['Qt5WinExtras']
-    QT_DLLS = {x + '.dll' for x in QT_DLLS}
-    PYQT_MODULES += ('QtWinExtras',)
-    PYQT_MODULES = {x + '.pyd' for x in PYQT_MODULES}
     prefix  = sw = os.environ.get('SW', r'C:\cygwin64\home\kovid\sw')
     sw_inc_dir  = os.path.join(prefix, 'include')
     sw_lib_dir  = os.path.join(prefix, 'lib')
@@ -150,9 +131,6 @@ if iswindows:
     podofo_inc = os.path.join(sw_inc_dir, 'podofo')
     podofo_lib = sw_lib_dir
 elif isosx:
-    QT_DLLS += ['Qt5DBus', 'Qt5MacExtras']
-    PYQT_MODULES += ('QtMacExtras',)
-    QT_FRAMEWORKS = [x.replace('5', '') for x in QT_DLLS]
     sw = os.environ.get('SW', os.path.expanduser('~/sw'))
     podofo_inc = os.path.join(sw, 'include', 'podofo')
     podofo_lib = os.path.join(sw, 'lib')
@@ -164,8 +142,6 @@ elif isosx:
     openssl_inc_dirs = [os.path.join(SSL, 'include')]
     openssl_lib_dirs = [os.path.join(SSL, 'lib')]
 else:
-    QT_DLLS += ['Qt5DBus', 'Qt5XcbQpa', 'Qt5X11Extras']
-    PYQT_MODULES += ('QtX11Extras',)
     ft_inc_dirs = pkgconfig_include_dirs('freetype2', 'FT_INC_DIR',
             '/usr/include/freetype2')
     ft_lib_dirs = pkgconfig_lib_dirs('freetype2', 'FT_LIB_DIR', '/usr/lib')
@@ -188,5 +164,3 @@ podofo_inc = [podofo_inc, os.path.dirname(podofo_inc)]
 
 BUILD_HOST='192.168.81.1'
 PROJECT=os.path.basename(os.path.abspath('.'))
-
-

setup/win-ci.py

@@ -0,0 +1,203 @@
+#!/usr/bin/env python2
+# vim:fileencoding=utf-8
+# License: GPLv3 Copyright: 2016, Kovid Goyal <kovid at kovidgoyal.net>
+
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+import os
+import io
+import sys
+import subprocess
+import tarfile
+
+def vcvars(is64bit=True):  # {{{
+    import _winreg as winreg
+    RegOpenKeyEx = winreg.OpenKeyEx
+    RegEnumValue = winreg.EnumValue
+    RegError = winreg.error
+
+    HKEYS = (winreg.HKEY_USERS,
+            winreg.HKEY_CURRENT_USER,
+            winreg.HKEY_LOCAL_MACHINE,
+            winreg.HKEY_CLASSES_ROOT)
+    VS_BASE = r"Software\Wow6432Node\Microsoft\VisualStudio\%0.1f"
+
+    def get_reg_value(path, key):
+        for base in HKEYS:
+            d = read_values(base, path)
+            if d and key in d:
+                return d[key]
+        raise KeyError(key)
+
+    def convert_mbcs(s):
+        dec = getattr(s, "decode", None)
+        if dec is not None:
+            try:
+                s = dec("mbcs")
+            except UnicodeError:
+                pass
+        return s
+
+    def read_values(base, key):
+        """Return dict of registry keys and values.
+
+        All names are converted to lowercase.
+        """
+        try:
+            handle = RegOpenKeyEx(base, key)
+        except RegError:
+            return None
+        d = {}
+        i = 0
+        while True:
+            try:
+                name, value, type = RegEnumValue(handle, i)
+            except RegError:
+                break
+            name = name.lower()
+            d[convert_mbcs(name)] = convert_mbcs(value)
+            i += 1
+        return d
+
+    def find_vcvarsall(version=14.0):
+        vsbase = VS_BASE % version
+        try:
+            productdir = get_reg_value(r"%s\Setup\VC" % vsbase, "productdir")
+        except KeyError:
+            raise SystemExit("Unable to find Visual Studio product directory in the registry")
+
+        if not productdir:
+            raise SystemExit("No productdir found")
+        vcvarsall = os.path.join(productdir, "vcvarsall.bat")
+        if os.path.isfile(vcvarsall):
+            return vcvarsall
+        raise SystemExit("Unable to find vcvarsall.bat in productdir: " + productdir)
+
+    def remove_dups(variable):
+        old_list = variable.split(os.pathsep)
+        new_list = []
+        for i in old_list:
+            if i not in new_list:
+                new_list.append(i)
+        return os.pathsep.join(new_list)
+
+    def query_process(cmd):
+        if is64bit and 'PROGRAMFILES(x86)' not in os.environ:
+            os.environ['PROGRAMFILES(x86)'] = os.environ['PROGRAMFILES'] + ' (x86)'
+        result = {}
+        popen = subprocess.Popen(cmd, stdout=subprocess.PIPE,
+                                stderr=subprocess.PIPE)
+        try:
+            stdout, stderr = popen.communicate()
+            if popen.wait() != 0:
+                raise RuntimeError(stderr.decode("mbcs"))
+
+            stdout = stdout.decode("mbcs")
+            for line in stdout.splitlines():
+                if '=' not in line:
+                    continue
+                line = line.strip()
+                key, value = line.split('=', 1)
+                key = key.lower()
+                if key == 'path':
+                    if value.endswith(os.pathsep):
+                        value = value[:-1]
+                    value = remove_dups(value)
+                result[key] = value
+
+        finally:
+            popen.stdout.close()
+            popen.stderr.close()
+        return result
+
+    def query_vcvarsall():
+        plat = 'amd64' if is64bit else 'x86'
+        vcvarsall = find_vcvarsall()
+        env = query_process('"%s" %s & set' % (vcvarsall, plat))
+
+        def g(k):
+            try:
+                return env[k]
+            except KeyError:
+                return env[k.lower()]
+
+        # We have to insert the correct path to MSBuild.exe so that the one
+        # from the .net frameworks is not used.
+        paths = g('PATH').split(os.pathsep)
+        for i, p in enumerate(tuple(paths)):
+            if os.path.exists(os.path.join(p, 'MSBuild.exe')):
+                if '.net' in p.lower():
+                    paths.insert(i, r'C:\Program Files (x86)\MSBuild\14.0\bin' + (r'\amd64' if is64bit else ''))
+                    env["PATH"] = os.pathsep.join(paths)
+                break
+
+        return {k: g(k) for k in 'PATH LIB INCLUDE LIBPATH WINDOWSSDKDIR VS140COMNTOOLS UCRTVERSION UNIVERSALCRTSDKDIR'.split()}
+
+    return query_vcvarsall()
+# }}}
+
+
+def printf(*args, **kw):
+    print(*args, **kw)
+    sys.stdout.flush()
+
+def sw():
+    sw=os.environ['SW']
+    os.makedirs(sw)
+    os.chdir(sw)
+    url =  'https://download.calibre-ebook.com/travis/win-64.tar.xz'
+    printf('Downloading', url)
+    tarball = subprocess.check_output(['curl.exe', '-fsSL', url])
+    with tarfile.open(fileobj=io.BytesIO(tarball)) as tf:
+        tf.extractall()
+    printf('Download complete')
+
+
+def sanitize_path():
+    needed_paths = []
+    executables = 'git.exe curl.exe'.split()
+    for p in os.environ['PATH'].split(os.pathsep):
+        for x in tuple(executables):
+            if os.path.exists(os.path.join(p, x)):
+                needed_paths.append(p)
+                executables.remove(x)
+    sw = os.environ['SW']
+    paths = r'{0}\private\python\DLLs {0}\private\python\Lib\site-packages\pywin32_system32 {0}\bin {0}\qt\bin C:\Windows\System32'.format(
+        sw).split() + needed_paths
+    os.environ[b'PATH'] = os.pathsep.join(paths).encode('ascii')
+
+def vcenv():
+    env = os.environ.copy()
+    env.update(vcvars())
+    return {str(k):str(v) for k, v in env.items()}
+
+def build():
+    sanitize_path()
+    cmd = [sys.executable, 'setup.py', 'bootstrap', '--ephemeral']
+    printf(*cmd)
+    p = subprocess.Popen(cmd, env=vcenv())
+    raise SystemExit(p.wait())
+
+def test():
+    sanitize_path()
+    cmd = [sys.executable, 'setup.py', 'test']
+    printf(*cmd)
+    p = subprocess.Popen(cmd)
+    raise SystemExit(p.wait())
+
+def main():
+    q = sys.argv[-1]
+    if q == 'build':
+        build()
+    elif q == 'test':
+        test()
+    elif q == 'sw':
+        sw()
+    else:
+        if len(sys.argv) == 1:
+            raise SystemExit('Usage: win-ci.py sw|build|test')
+        raise SystemExit('%r is not a valid action' % sys.argv[-1])
+
+if __name__ == '__main__':
+    main()