Work on windows CI

README.md

@@ -10,7 +10,7 @@ reading. It is cross platform, running on Linux, Windows and macOS.
 
 For more information, see the [calibre About page](https://calibre-ebook.com/about)
 
-[![Build Status](https://dev.azure.com/divok/calibre/_apis/build/status/kovidgoyal.calibre?branchName=engine)](https://dev.azure.com/divok/calibre/_build/latest?definitionId=1&branchName=engine)
+[![Build Status](https://dev.azure.com/divok/calibre/_apis/build/status/kovidgoyal.calibre?branchName=master)](https://dev.azure.com/divok/calibre/_build/latest?definitionId=1&branchName=master)
 
 ## Screenshots  
 

azure-pipelines.yml

@@ -61,3 +61,28 @@ jobs:
   - script: |
       python3 setup/unix-ci.py test
     displayName: 'Test calibre'
+
+- job: 'Windows'
+  pool:
+    vmImage: 'vs2017-win2016'
+
+  steps:
+  - checkout: self
+    fetchDepth: 1
+
+  - task: UsePythonVersion@0
+    inputs:
+      versionSpec: '3.7'
+      architecture: 'x64'
+
+  - script: |
+      python setup/win-ci.py install
+    displayName: 'Install calibre dependencies'
+
+  - script: |
+      python setup/win-ci.py bootstrap
+    displayName: 'Bootstrap calibre'
+
+  - script: |
+      python setup/win-ci.py test
+    displayName: 'Test calibre'

setup/win-ci.py

@@ -11,152 +11,6 @@ import sys
 import tarfile
 import time
 
-if sys.version_info.major > 2:
-    import winreg
-else:
-    try:
-        import _winreg as winreg
-    except ImportError:
-        import winreg
-is64bit = os.environ.get('PLATFORM') != 'x86'
-
-
-def vcvars():  # {{{
-    RegOpenKeyEx = winreg.OpenKeyEx
-    RegEnumValue = winreg.EnumValue
-    RegError = winreg.error
-
-    HKEYS = (
-        winreg.HKEY_USERS, winreg.HKEY_CURRENT_USER, winreg.HKEY_LOCAL_MACHINE,
-        winreg.HKEY_CLASSES_ROOT
-    )
-    VS_BASE = r"Software\Wow6432Node\Microsoft\VisualStudio\%0.1f"
-
-    def get_reg_value(path, key):
-        for base in HKEYS:
-            d = read_values(base, path)
-            if d and key in d:
-                return d[key]
-        raise KeyError(key)
-
-    def convert_mbcs(s):
-        dec = getattr(s, "decode", None)
-        if dec is not None:
-            try:
-                s = dec("mbcs")
-            except UnicodeError:
-                pass
-        return s
-
-    def read_values(base, key):
-        """Return dict of registry keys and values.
-
-        All names are converted to lowercase.
-        """
-        try:
-            handle = RegOpenKeyEx(base, key)
-        except RegError:
-            return None
-        d = {}
-        i = 0
-        while True:
-            try:
-                name, value, type = RegEnumValue(handle, i)
-            except RegError:
-                break
-            name = name.lower()
-            d[convert_mbcs(name)] = convert_mbcs(value)
-            i += 1
-        return d
-
-    def find_vcvarsall(version=14.0):
-        vsbase = VS_BASE % version
-        try:
-            productdir = get_reg_value(r"%s\Setup\VC" % vsbase, "productdir")
-        except KeyError:
-            raise SystemExit(
-                "Unable to find Visual Studio product directory in the registry"
-            )
-
-        if not productdir:
-            raise SystemExit("No productdir found")
-        vcvarsall = os.path.join(productdir, "vcvarsall.bat")
-        if os.path.isfile(vcvarsall):
-            return vcvarsall
-        raise SystemExit("Unable to find vcvarsall.bat in productdir: " + productdir)
-
-    def remove_dups(variable):
-        old_list = variable.split(os.pathsep)
-        new_list = []
-        for i in old_list:
-            if i not in new_list:
-                new_list.append(i)
-        return os.pathsep.join(new_list)
-
-    def query_process(cmd):
-        if is64bit and 'PROGRAMFILES(x86)' not in os.environ:
-            os.environ['PROGRAMFILES(x86)'] = os.environ['PROGRAMFILES'] + ' (x86)'
-        result = {}
-        popen = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-        try:
-            stdout, stderr = popen.communicate()
-            if popen.wait() != 0:
-                raise RuntimeError(stderr.decode("mbcs"))
-
-            stdout = stdout.decode("mbcs")
-            for line in stdout.splitlines():
-                if '=' not in line:
-                    continue
-                line = line.strip()
-                key, value = line.split('=', 1)
-                key = key.lower()
-                if key == 'path':
-                    if value.endswith(os.pathsep):
-                        value = value[:-1]
-                    value = remove_dups(value)
-                result[key] = value
-
-        finally:
-            popen.stdout.close()
-            popen.stderr.close()
-        return result
-
-    def query_vcvarsall():
-        plat = 'amd64' if is64bit else 'x86'
-        vcvarsall = find_vcvarsall()
-        env = query_process('"%s" %s & set' % (vcvarsall, plat))
-
-        def g(k):
-            try:
-                return env[k]
-            except KeyError:
-                return env[k.lower()]
-
-        # We have to insert the correct path to MSBuild.exe so that the one
-        # from the .net frameworks is not used.
-        paths = g('PATH').split(os.pathsep)
-        for i, p in enumerate(tuple(paths)):
-            if os.path.exists(os.path.join(p, 'MSBuild.exe')):
-                if '.net' in p.lower():
-                    paths.insert(
-                        i, r'C:\Program Files (x86)\MSBuild\14.0\bin' +
-                        (r'\amd64' if is64bit else '')
-                    )
-                    env["PATH"] = os.pathsep.join(paths)
-                break
-
-        return {
-            k: g(k)
-            for k in
-            'PATH LIB INCLUDE LIBPATH WINDOWSSDKDIR VS140COMNTOOLS UCRTVERSION UNIVERSALCRTSDKDIR'.
-            split()
-        }
-
-    return query_vcvarsall()
-
-
-# }}}
-
 
 def printf(*args, **kw):
     print(*args, **kw)
@@ -164,21 +18,24 @@ def printf(*args, **kw):
 
 
 def download_file(url):
-    for i in range(5):
+    from urllib.request import urlopen
+    count = 5
+    while count > 0:
+        count -= 1
         try:
             printf('Downloading', url)
-            return subprocess.check_output(['curl.exe', '-fSL', url])
+            return urlopen(url).read()
-        except subprocess.CalledProcessError:
+        except Exception:
+            if count <= 0:
+                raise
+            print('Download failed retrying...')
             time.sleep(1)
-    raise SystemExit('Failed to download: {}'.format(url))
 
 
 def sw():
     sw = os.environ['SW']
-    os.makedirs(sw)
     os.chdir(sw)
-    url = 'https://download.calibre-ebook.com/travis/win-{}.tar.xz'.format(
+    url = 'https://download.calibre-ebook.com/ci/calibre/windows-64.tar.xz'
-        '64' if is64bit else '32')
     tarball = download_file(url)
     with tarfile.open(fileobj=io.BytesIO(tarball)) as tf:
         tf.extractall()
@@ -201,35 +58,42 @@ def sanitize_path():
     print('PATH:', os.environ['PATH'])
 
 
-def vcenv():
+def python_exe():
-    env = os.environ.copy()
+    return os.path.join(os.environ['SW'], 'private', 'python', 'python.exe')
-    env.update(vcvars())
-    return {str(k): str(v) for k, v in env.items()}
 
 
 def build():
     sanitize_path()
-    cmd = [sys.executable, 'setup.py', 'bootstrap', '--ephemeral']
+    cmd = [python_exe(), 'setup.py', 'bootstrap', '--ephemeral']
-    printf(*cmd)
-    p = subprocess.Popen(cmd, env=vcenv())
-    raise SystemExit(p.wait())
-
-
-def test():
-    sanitize_path()
-    cmd = [sys.executable, 'setup.py', 'test']
     printf(*cmd)
     p = subprocess.Popen(cmd)
     raise SystemExit(p.wait())
 
 
+def test():
+    sanitize_path()
+    cmd = [python_exe(), 'setup.py', 'test']
+    printf(*cmd)
+    p = subprocess.Popen(cmd)
+    raise SystemExit(p.wait())
+
+
+def setup_env():
+    os.environ['SW'] = SW = r'C:\r\sw64\sw'
+    os.makedirs(SW, exist_ok=True)
+    os.environ['QMAKE'] = os.path.join(SW, r'qt\bin\qmake')
+    os.environ['CALIBRE_QT_PREFIX'] = os.path.join(SW, r'qt')
+    os.environ['CI'] = 'true'
+
+
 def main():
     q = sys.argv[-1]
-    if q == 'build':
+    setup_env()
+    if q == 'bootstrap':
         build()
     elif q == 'test':
         test()
-    elif q == 'sw':
+    elif q == 'install':
         sw()
     else:
         if len(sys.argv) == 1:

src/calibre/test_build.py

@@ -91,14 +91,6 @@ class BuildTest(unittest.TestCase):
 
     def test_plugins(self):
         exclusions = set()
-        if is_ci:
-            if isosx:
-                # The compiler version on OS X is different between the
-                # machine on which the dependencies are built and the
-                # machine on which the calibre modules are built, which causes
-                # C++ name mangling incompatibilities preventing some modules
-                # from loading
-                exclusions.update(set('podofo'.split()))
         if islinux and (not os.path.exists('/dev/bus/usb') and not os.path.exists('/proc/bus/usb')):
             # libusb fails to initialize in containers without USB subsystems
             exclusions.update(set('libusb libmtp'.split()))
@@ -299,7 +291,6 @@ class BuildTest(unittest.TestCase):
         import psutil
         psutil.Process(os.getpid())
 
-    @unittest.skipIf(is_ci and isosx, 'Currently there is a C++ ABI incompatibility until the osx-build machine is moved to OS X 10.9')
     def test_podofo(self):
         from calibre.utils.podofo import test_podofo as dotest
         dotest()