diff --git a/README.md b/README.md index fcb2671ba3..954e4824e2 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ reading. It is cross platform, running on Linux, Windows and macOS. For more information, see the [calibre About page](https://calibre-ebook.com/about) -[![Build Status](https://dev.azure.com/divok/calibre/_apis/build/status/kovidgoyal.calibre?branchName=engine)](https://dev.azure.com/divok/calibre/_build/latest?definitionId=1&branchName=engine) +[![Build Status](https://dev.azure.com/divok/calibre/_apis/build/status/kovidgoyal.calibre?branchName=master)](https://dev.azure.com/divok/calibre/_build/latest?definitionId=1&branchName=master) ## Screenshots diff --git a/azure-pipelines.yml b/azure-pipelines.yml index a009398056..99cd541456 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -61,3 +61,28 @@ jobs: - script: | python3 setup/unix-ci.py test displayName: 'Test calibre' + +- job: 'Windows' + pool: + vmImage: 'vs2017-win2016' + + steps: + - checkout: self + fetchDepth: 1 + + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.7' + architecture: 'x64' + + - script: | + python setup/win-ci.py install + displayName: 'Install calibre dependencies' + + - script: | + python setup/win-ci.py bootstrap + displayName: 'Bootstrap calibre' + + - script: | + python setup/win-ci.py test + displayName: 'Test calibre' diff --git a/setup/win-ci.py b/setup/win-ci.py index 347fd174cc..e677703236 100644 --- a/setup/win-ci.py +++ b/setup/win-ci.py @@ -11,152 +11,6 @@ import sys import tarfile import time -if sys.version_info.major > 2: - import winreg -else: - try: - import _winreg as winreg - except ImportError: - import winreg -is64bit = os.environ.get('PLATFORM') != 'x86' - - -def vcvars(): # {{{ - RegOpenKeyEx = winreg.OpenKeyEx - RegEnumValue = winreg.EnumValue - RegError = winreg.error - - HKEYS = ( - winreg.HKEY_USERS, winreg.HKEY_CURRENT_USER, winreg.HKEY_LOCAL_MACHINE, - winreg.HKEY_CLASSES_ROOT - ) - VS_BASE = r"Software\Wow6432Node\Microsoft\VisualStudio\%0.1f" - - def get_reg_value(path, key): - for base in HKEYS: - d = read_values(base, path) - if d and key in d: - return d[key] - raise KeyError(key) - - def convert_mbcs(s): - dec = getattr(s, "decode", None) - if dec is not None: - try: - s = dec("mbcs") - except UnicodeError: - pass - return s - - def read_values(base, key): - """Return dict of registry keys and values. - - All names are converted to lowercase. - """ - try: - handle = RegOpenKeyEx(base, key) - except RegError: - return None - d = {} - i = 0 - while True: - try: - name, value, type = RegEnumValue(handle, i) - except RegError: - break - name = name.lower() - d[convert_mbcs(name)] = convert_mbcs(value) - i += 1 - return d - - def find_vcvarsall(version=14.0): - vsbase = VS_BASE % version - try: - productdir = get_reg_value(r"%s\Setup\VC" % vsbase, "productdir") - except KeyError: - raise SystemExit( - "Unable to find Visual Studio product directory in the registry" - ) - - if not productdir: - raise SystemExit("No productdir found") - vcvarsall = os.path.join(productdir, "vcvarsall.bat") - if os.path.isfile(vcvarsall): - return vcvarsall - raise SystemExit("Unable to find vcvarsall.bat in productdir: " + productdir) - - def remove_dups(variable): - old_list = variable.split(os.pathsep) - new_list = [] - for i in old_list: - if i not in new_list: - new_list.append(i) - return os.pathsep.join(new_list) - - def query_process(cmd): - if is64bit and 'PROGRAMFILES(x86)' not in os.environ: - os.environ['PROGRAMFILES(x86)'] = os.environ['PROGRAMFILES'] + ' (x86)' - result = {} - popen = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - try: - stdout, stderr = popen.communicate() - if popen.wait() != 0: - raise RuntimeError(stderr.decode("mbcs")) - - stdout = stdout.decode("mbcs") - for line in stdout.splitlines(): - if '=' not in line: - continue - line = line.strip() - key, value = line.split('=', 1) - key = key.lower() - if key == 'path': - if value.endswith(os.pathsep): - value = value[:-1] - value = remove_dups(value) - result[key] = value - - finally: - popen.stdout.close() - popen.stderr.close() - return result - - def query_vcvarsall(): - plat = 'amd64' if is64bit else 'x86' - vcvarsall = find_vcvarsall() - env = query_process('"%s" %s & set' % (vcvarsall, plat)) - - def g(k): - try: - return env[k] - except KeyError: - return env[k.lower()] - - # We have to insert the correct path to MSBuild.exe so that the one - # from the .net frameworks is not used. - paths = g('PATH').split(os.pathsep) - for i, p in enumerate(tuple(paths)): - if os.path.exists(os.path.join(p, 'MSBuild.exe')): - if '.net' in p.lower(): - paths.insert( - i, r'C:\Program Files (x86)\MSBuild\14.0\bin' + - (r'\amd64' if is64bit else '') - ) - env["PATH"] = os.pathsep.join(paths) - break - - return { - k: g(k) - for k in - 'PATH LIB INCLUDE LIBPATH WINDOWSSDKDIR VS140COMNTOOLS UCRTVERSION UNIVERSALCRTSDKDIR'. - split() - } - - return query_vcvarsall() - - -# }}} - def printf(*args, **kw): print(*args, **kw) @@ -164,21 +18,24 @@ def printf(*args, **kw): def download_file(url): - for i in range(5): + from urllib.request import urlopen + count = 5 + while count > 0: + count -= 1 try: printf('Downloading', url) - return subprocess.check_output(['curl.exe', '-fSL', url]) - except subprocess.CalledProcessError: + return urlopen(url).read() + except Exception: + if count <= 0: + raise + print('Download failed retrying...') time.sleep(1) - raise SystemExit('Failed to download: {}'.format(url)) def sw(): sw = os.environ['SW'] - os.makedirs(sw) os.chdir(sw) - url = 'https://download.calibre-ebook.com/travis/win-{}.tar.xz'.format( - '64' if is64bit else '32') + url = 'https://download.calibre-ebook.com/ci/calibre/windows-64.tar.xz' tarball = download_file(url) with tarfile.open(fileobj=io.BytesIO(tarball)) as tf: tf.extractall() @@ -201,35 +58,42 @@ def sanitize_path(): print('PATH:', os.environ['PATH']) -def vcenv(): - env = os.environ.copy() - env.update(vcvars()) - return {str(k): str(v) for k, v in env.items()} +def python_exe(): + return os.path.join(os.environ['SW'], 'private', 'python', 'python.exe') def build(): sanitize_path() - cmd = [sys.executable, 'setup.py', 'bootstrap', '--ephemeral'] - printf(*cmd) - p = subprocess.Popen(cmd, env=vcenv()) - raise SystemExit(p.wait()) - - -def test(): - sanitize_path() - cmd = [sys.executable, 'setup.py', 'test'] + cmd = [python_exe(), 'setup.py', 'bootstrap', '--ephemeral'] printf(*cmd) p = subprocess.Popen(cmd) raise SystemExit(p.wait()) +def test(): + sanitize_path() + cmd = [python_exe(), 'setup.py', 'test'] + printf(*cmd) + p = subprocess.Popen(cmd) + raise SystemExit(p.wait()) + + +def setup_env(): + os.environ['SW'] = SW = r'C:\r\sw64\sw' + os.makedirs(SW, exist_ok=True) + os.environ['QMAKE'] = os.path.join(SW, r'qt\bin\qmake') + os.environ['CALIBRE_QT_PREFIX'] = os.path.join(SW, r'qt') + os.environ['CI'] = 'true' + + def main(): q = sys.argv[-1] - if q == 'build': + setup_env() + if q == 'bootstrap': build() elif q == 'test': test() - elif q == 'sw': + elif q == 'install': sw() else: if len(sys.argv) == 1: diff --git a/src/calibre/test_build.py b/src/calibre/test_build.py index f5faae5535..ab4b623d02 100644 --- a/src/calibre/test_build.py +++ b/src/calibre/test_build.py @@ -91,14 +91,6 @@ class BuildTest(unittest.TestCase): def test_plugins(self): exclusions = set() - if is_ci: - if isosx: - # The compiler version on OS X is different between the - # machine on which the dependencies are built and the - # machine on which the calibre modules are built, which causes - # C++ name mangling incompatibilities preventing some modules - # from loading - exclusions.update(set('podofo'.split())) if islinux and (not os.path.exists('/dev/bus/usb') and not os.path.exists('/proc/bus/usb')): # libusb fails to initialize in containers without USB subsystems exclusions.update(set('libusb libmtp'.split())) @@ -299,7 +291,6 @@ class BuildTest(unittest.TestCase): import psutil psutil.Process(os.getpid()) - @unittest.skipIf(is_ci and isosx, 'Currently there is a C++ ABI incompatibility until the osx-build machine is moved to OS X 10.9') def test_podofo(self): from calibre.utils.podofo import test_podofo as dotest dotest()