Use the new bypy freezing code for windows build

bypy/windows/__main__.py

@@ -3,6 +3,7 @@
 # License: GPLv3 Copyright: 2016, Kovid Goyal <kovid at kovidgoyal.net>
 
 
+import contextlib
 import errno
 import glob
 import os
@@ -18,7 +19,11 @@ from bypy.constants import (
     CL, LINK, MT, PREFIX, RC, SIGNTOOL, SRC as CALIBRE_DIR, SW, build_dir, is64bit,
     python_major_minor_version, worker_env
 )
-from bypy.utils import py_compile, run, walk
+from bypy.freeze import (
+    cleanup_site_packages, extract_extension_modules, freeze_python,
+    path_to_freeze_dir, save_importer_src_to_header
+)
+from bypy.utils import mkdtemp, py_compile, run, walk
 
 iv = globals()['init_env']
 calibre_constants = iv['calibre_constants']
@@ -120,14 +125,7 @@ def initbase(env):
     os.mkdir(env.dist)
 
 
-def add_plugins(env, ext_dir):
+def freeze(env, ext_dir, incdir):
-    printf('Adding plugins...')
-    tgt = env.dll_dir
-    for f in glob.glob(j(ext_dir, '*.pyd')):
-        shutil.copy2(f, tgt)
-
-
-def freeze(env, ext_dir):
     shutil.copy2(j(env.src_root, 'LICENSE'), env.base)
 
     printf('Adding resources...')
@@ -140,11 +138,8 @@ def freeze(env, ext_dir):
 
     def copybin(x):
         shutil.copy2(x, env.dll_dir)
-        try:
+        with contextlib.suppress(FileNotFoundError):
             shutil.copy2(x + '.manifest', env.dll_dir)
-        except EnvironmentError as err:
-            if err.errno != errno.ENOENT:
-                raise
 
     bindir = os.path.join(PREFIX, 'bin')
     for x in ('pdftohtml', 'pdfinfo', 'pdftoppm', 'jpegtran-calibre', 'cjpeg-calibre', 'optipng-calibre', 'JXRDecApp-calibre'):
@@ -155,13 +150,14 @@ def freeze(env, ext_dir):
 
     copybin(os.path.join(env.python_base, 'python%s.dll' % env.py_ver.replace('.', '')))
     copybin(os.path.join(env.python_base, 'python%s.dll' % env.py_ver[0]))
-    for x in glob.glob(os.path.join(env.python_base, 'DLLs', '*')):  # python pyd modules and dlls
+    for x in glob.glob(os.path.join(env.python_base, 'DLLs', '*.dll')):  # dlls needed by python
         copybin(x)
     for f in walk(os.path.join(env.python_base, 'Lib')):
         q = f.lower()
         if q.endswith('.dll') and 'scintilla' not in q and 'pyqtbuild' not in q:
             copybin(f)
-    add_plugins(env, ext_dir)
+    ext_map = extract_extension_modules(ext_dir, env.dll_dir)
+    ext_map.update(extract_extension_modules(j(env.python_base, 'DLLs'), env.dll_dir, move=False))
 
     printf('Adding Qt...')
     for x in QT_DLLS:
@@ -196,22 +192,7 @@ def freeze(env, ext_dir):
 
     shutil.copytree(r'%s\Lib' % env.python_base, env.lib_dir, ignore=ignore_lib)
     install_site_py(env)
-
-    # Fix win32com
     sp_dir = j(env.lib_dir, 'site-packages')
-    comext = j(sp_dir, 'win32comext')
-    shutil.copytree(j(comext, 'shell'), j(sp_dir, 'win32com', 'shell'))
-    shutil.rmtree(comext)
-
-    # Fix pycryptodome
-    with open(j(sp_dir, 'Crypto', 'Util', '_file_system.py'), 'w') as fspy:
-        fspy.write('''
-import os, sys
-def pycryptodome_filename(dir_comps, filename):
-    base = os.path.join(sys.app_dir, 'app', 'bin')
-    path = os.path.join(base, '.'.join(dir_comps + [filename]))
-    return path
-''')
 
     printf('Adding calibre sources...')
     for x in glob.glob(j(CALIBRE_DIR, 'src', '*')):
@@ -221,24 +202,20 @@ def pycryptodome_filename(dir_comps, filename):
         else:
             shutil.copy(x, j(sp_dir, b(x)))
 
-    for x in (r'calibre\manual', r'calibre\plugins', 'pythonwin'):
+    ext_map.update(cleanup_site_packages(sp_dir))
-        deld = j(sp_dir, x)
+    for x in os.listdir(sp_dir):
-        if os.path.exists(deld):
+        os.rename(j(sp_dir, x), j(env.lib_dir, x))
-            shutil.rmtree(deld)
+    os.rmdir(sp_dir)
-
+    printf('Extracting extension modules from', env.lib_dir, 'to', env.dll_dir)
-    for x in os.walk(j(sp_dir, 'calibre')):
+    ext_map.update(extract_extension_modules(env.lib_dir, env.dll_dir))
-        for f in x[-1]:
-            if not f.endswith('.py'):
-                os.remove(j(x[0], f))
-
-    extract_pyd_modules(env, sp_dir)
 
     printf('Byte-compiling all python modules...')
-    for x in ('test', 'lib2to3'):
-        x = j(env.lib_dir, x)
-        if os.path.exists(x):
-            shutil.rmtree(x)
     py_compile(env.lib_dir.replace(os.sep, '/'))
+    # from bypy.utils import run_shell
+    # run_shell(cwd=env.lib_dir)
+    freeze_python(env.lib_dir, env.dll_dir, incdir)
+    shutil.rmtree(env.lib_dir)
+    save_importer_src_to_header(incdir, ext_map, develop_mode_env_var='CALIBRE_DEVELOP_FROM')
 
 
 def embed_manifests(env):
@@ -255,58 +232,6 @@ def embed_manifests(env):
         os.remove(manifest)
 
 
-def extract_pyd_modules(env, site_packages_dir):
-    printf('\nExtracting .pyd modules from site-packages...')
-
-    def extract_pyd(path, root):
-        fullname = os.path.relpath(path, root).replace(os.sep, '/').replace('/', '.')
-        dest = os.path.join(env.dll_dir, fullname)
-        if os.path.exists(dest):
-            raise ValueError('Cannot extract %s into DLLs as it already exists' % fullname)
-        os.rename(path, dest)
-        bpy = dest[:-1]
-        if os.path.exists(bpy):
-            with open(bpy, 'rb') as f:
-                raw = f.read().strip().decode('utf-8')
-            if (not raw.startswith('def __bootstrap__') or not raw.endswith('__bootstrap__()')):
-                raise ValueError('The file %r has non bootstrap code' % bpy)
-        for ext in ('', 'c', 'o'):
-            try:
-                os.remove(bpy + ext)
-            except EnvironmentError as err:
-                if err.errno != errno.ENOENT:
-                    raise
-
-    def find_pyds(base):
-        for dirpath, dirnames, filenames in os.walk(base):
-            for fname in filenames:
-                if fname.lower().endswith('.pyd'):
-                    yield os.path.join(dirpath, fname)
-
-    def process_root(root, base=None):
-        for path in find_pyds(root):
-            extract_pyd(path, base or root)
-
-    def absp(x):
-        return os.path.normcase(os.path.abspath(os.path.join(site_packages_dir, x)))
-
-    roots = set()
-    for pth in glob.glob(os.path.join(site_packages_dir, '*.pth')):
-        for line in open(pth).readlines():
-            line = line.strip()
-            if line and not line.startswith('#') and os.path.exists(os.path.join(site_packages_dir, line)):
-                roots.add(absp(line))
-
-    for x in os.listdir(site_packages_dir):
-        x = absp(x)
-        if x in roots:
-            process_root(x)
-        elif os.path.isdir(x):
-            process_root(x, site_packages_dir)
-        elif x.lower().endswith('.pyd'):
-            extract_pyd(x, site_packages_dir)
-
-
 def embed_resources(env, module, desc=None, extra_data=None, product_description=None):
     icon_base = j(env.src_root, 'icons')
     icon_map = {
@@ -526,7 +451,7 @@ def build_utils(env):
     build(j(base, 'eject.c'), 'calibre-eject.exe')
 
 
-def build_launchers(env, debug=False):
+def build_launchers(env, incdir, debug=False):
     if not os.path.exists(env.obj_dir):
         os.makedirs(env.obj_dir)
     dflags = (['/Zi'] if debug else [])
@@ -536,6 +461,7 @@ def build_launchers(env, debug=False):
     objects = [j(env.obj_dir, b(x) + '.obj') for x in sources]
     cflags = '/c /EHsc /W3 /Ox /nologo /D_UNICODE'.split()
     cflags += ['/DPYDLL="python%s.dll"' % env.py_ver.replace('.', ''), '/I%s/include' % env.python_base]
+    cflags += [f'/I{path_to_freeze_dir()}', f'/I{incdir}']
     for src, obj in zip(sources, objects):
         cmd = [CL] + cflags + dflags + ['/MD', '/Fo' + obj, '/Tc' + src]
         run_compiler(env, *cmd)
@@ -586,80 +512,6 @@ def build_launchers(env, debug=False):
             run(*cmd)
 
 
-def add_to_zipfile(zf, name, base, zf_names):
-    if '__pycache__' in name:
-        return
-    abspath = j(base, name)
-    name = name.replace(os.sep, '/')
-    if name in zf_names:
-        raise ValueError('Already added %r to zipfile [%r]' % (name, abspath))
-    zinfo = zipfile.ZipInfo(filename=name, date_time=(1980, 1, 1, 0, 0, 0))
-
-    if os.path.isdir(abspath):
-        if not os.listdir(abspath):
-            return
-        zinfo.external_attr = 0o700 << 16
-        zf.writestr(zinfo, '')
-        for x in os.listdir(abspath):
-            add_to_zipfile(zf, name + os.sep + x, base, zf_names)
-    else:
-        ext = os.path.splitext(name)[1].lower()
-        if ext in ('.dll',):
-            raise ValueError('Cannot add %r to zipfile' % abspath)
-        zinfo.external_attr = 0o600 << 16
-        if ext in ('.py', '.pyc', '.pyo'):
-            with open(abspath, 'rb') as f:
-                zf.writestr(zinfo, f.read())
-
-    zf_names.add(name)
-
-
-def archive_lib_dir(env):
-    printf('Putting all python code into a zip file for performance')
-    zf_names = set()
-    with zipfile.ZipFile(env.pylib, 'w', zipfile.ZIP_STORED) as zf:
-        # Add everything in Lib except site-packages to the zip file
-        for x in os.listdir(env.lib_dir):
-            if x == 'site-packages':
-                continue
-            add_to_zipfile(zf, x, env.lib_dir, zf_names)
-
-        sp = j(env.lib_dir, 'site-packages')
-        # Special handling for pywin32
-        handled = {'pywin32.pth', 'win32'}
-        base = j(sp, 'win32', 'lib')
-        for x in os.listdir(base):
-            if os.path.splitext(x)[1] not in ('.exe',) and x != '__pycache__':
-                add_to_zipfile(zf, x, base, zf_names)
-        base = os.path.dirname(base)
-        for x in os.listdir(base):
-            if not os.path.isdir(j(base, x)):
-                if os.path.splitext(x)[1] not in ('.exe',):
-                    add_to_zipfile(zf, x, base, zf_names)
-
-        # We dont want the site.py (if any) from site-packages
-        handled.add('site.pyo')
-        handled.add('site.pyc')
-        handled.add('site.py')
-        handled.add('sitecustomize.pyo')
-        handled.add('sitecustomize.pyc')
-        handled.add('sitecustomize.py')
-
-        # The rest of site-packages
-        for x in os.listdir(sp):
-            if x in handled or x.endswith('.egg-info') or x.endswith('.dist-info'):
-                continue
-            absp = j(sp, x)
-            if os.path.isdir(absp):
-                if not os.listdir(absp):
-                    continue
-                add_to_zipfile(zf, x, sp, zf_names)
-            else:
-                add_to_zipfile(zf, x, sp, zf_names)
-
-    shutil.rmtree(env.lib_dir)
-
-
 def copy_crt_and_d3d(env):
     printf('Copying CRT and D3D...')
     plat = ('x64' if is64bit else 'x86')
@@ -718,13 +570,13 @@ def main():
     args = globals()['args']
     run_tests = iv['run_tests']
     env = Env(build_dir())
+    incdir = mkdtemp('include')
     initbase(env)
-    build_launchers(env)
+    freeze(env, ext_dir, incdir)
+    build_launchers(env, incdir)
     build_utils(env)
-    freeze(env, ext_dir)
     embed_manifests(env)
     copy_crt_and_d3d(env)
-    archive_lib_dir(env)
     if not args.skip_tests:
         run_tests(os.path.join(env.base, 'calibre-debug.exe'), env.base)
     if args.sign_installers:

bypy/windows/util.c

@@ -17,57 +17,9 @@
 #include <fcntl.h>
 #include "../run-python.h"
 
-static int GUI_APP = 0;
 static char python_dll[] = PYDLL;
 
-static int _show_error(const wchar_t *preamble, const wchar_t *msg, const int code) {
-    static wchar_t buf[4096];
-	static char utf8_buf[4096] = {0};
-	int n = WideCharToMultiByte(CP_UTF8, 0, preamble, -1, utf8_buf, sizeof(utf8_buf) - 1, NULL, NULL);
-	if (n > 0) fprintf(stderr, "%s\r\n  ", utf8_buf);
-	n = WideCharToMultiByte(CP_UTF8, 0, msg, -1, utf8_buf, sizeof(utf8_buf) - 1, NULL, NULL);
-	if (n > 0) fprintf(stderr, "%s (Error Code: %d)\r\n ", utf8_buf, code);
-    fflush(stderr);
 
-    if (GUI_APP) {
-        _snwprintf_s(buf, arraysz(buf), _TRUNCATE, L"%ls\r\n  %ls (Error Code: %d)\r\n", preamble, msg, code);
-        MessageBeep(MB_ICONERROR);
-        MessageBox(NULL, buf, NULL, MB_OK|MB_ICONERROR);
-    }
-    return code;
-}
-
-
-
-int show_last_error_crt(wchar_t *preamble) {
-    wchar_t buf[1000];
-    int err = 0;
-
-    _get_errno(&err);
-    _wcserror_s(buf, 1000, err);
-    return _show_error(preamble, buf, err);
-}
-
-int show_last_error(wchar_t *preamble) {
-    wchar_t *msg = NULL;
-    DWORD dw = GetLastError();
-    int ret;
-
-    FormatMessage(
-        FORMAT_MESSAGE_ALLOCATE_BUFFER |
-        FORMAT_MESSAGE_FROM_SYSTEM |
-        FORMAT_MESSAGE_IGNORE_INSERTS,
-        NULL,
-        dw,
-        MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),
-        (LPWSTR)&msg,
-        0,
-        NULL );
-
-    ret = _show_error(preamble, msg, (int)dw);
-    if (msg != NULL) LocalFree(msg);
-    return ret;
-}
 
 static wchar_t qt_prefix_dir[MAX_PATH] = {0};
 
@@ -97,8 +49,7 @@ get_install_locations(void) {
 static void
 load_python_dll() {
     get_install_locations();
-    if (FAILED(__HrLoadAllImportsForDll(python_dll)))
+	bypy_setup_python(python_dll);
-        ExitProcess(_show_error(L"Failed to delay load the python dll", L"", 1));
 }
 
 const static wchar_t out_of_memory[] = L"Out of memory";
@@ -115,42 +66,6 @@ redirect_out_stream(FILE *stream) {
     }
 }
 
-static PyObject*
-gui_error_message(PyObject *self, PyObject *args) {
-    PyObject *pt, *pm;
-    if (!PyArg_ParseTuple(args, "UU", &pt, &pm)) return NULL;
-    wchar_t title[256] = {0}, text[4096] = {0};
-    PyUnicode_AsWideChar(pt, title, arraysz(title)-1);
-    PyUnicode_AsWideChar(pm, text, arraysz(text)-1);
-    MessageBoxW(NULL, text, title, MB_ICONERROR|MB_OK);
-    return PyBool_FromLong(1);
-}
-
-static PyMethodDef methods[] = {
-    {"gui_error_message", (PyCFunction)gui_error_message, METH_VARARGS,
-     "gui_error_message(title, msg) -> Show a GUI based error message."
-    },
-    {NULL}  /* Sentinel */
-};
-
-
-static struct PyModuleDef module = {
-    /* m_base     */ PyModuleDef_HEAD_INIT,
-    /* m_name     */ "calibre_os_module",
-    /* m_doc      */ "Integration with OS facilities during startup",
-    /* m_size     */ -1,
-    /* m_methods  */ methods,
-    /* m_slots    */ 0,
-    /* m_traverse */ 0,
-    /* m_clear    */ 0,
-    /* m_free     */ 0,
-};
-
-PyObject*
-calibre_os_module(void) {
-    return PyModule_Create(&module);
-}
-
 static void
 null_invalid_parameter_handler(
    const wchar_t * expression,
@@ -187,7 +102,6 @@ execute_python_entrypoint(const wchar_t *basename, const wchar_t *module, const
     interpreter_data.argv = CommandLineToArgvW(GetCommandLineW(), &interpreter_data.argc);
     if (interpreter_data.argv == NULL) ExitProcess(show_last_error(L"Failed to get command line"));
     interpreter_data.basename = basename; interpreter_data.module = module; interpreter_data.function = function;
-    interpreter_data.calibre_os_module = calibre_os_module;
     load_python_dll();
     pre_initialize_interpreter(is_gui_app);
 	run_interpreter();

src/calibre/test_build.py

@@ -32,6 +32,8 @@ class BuildTest(unittest.TestCase):
                     self.assertTrue(False, 'Failed to load DLL %s with error: %s' % (x, err))
         from Crypto.Cipher import AES
         del AES
+        from pywintypes import error
+        del error
 
     @unittest.skipUnless(islinux, 'DBUS only used on linux')
     def test_dbus(self):