mirror of
https://github.com/kovidgoyal/calibre.git
synced 2025-07-08 10:44:09 -04:00
Use the new bypy freezing code for windows build
This commit is contained in:
Kovid Goyal
parent
4e5fda27a8
commit
fe478a2cee
@ -3,6 +3,7 @@
|
||||
# License: GPLv3 Copyright: 2016, Kovid Goyal <kovid at kovidgoyal.net>
|
||||
|
||||
|
||||
import contextlib
|
||||
import errno
|
||||
import glob
|
||||
import os
|
||||
@ -18,7 +19,11 @@ from bypy.constants import (
|
||||
CL, LINK, MT, PREFIX, RC, SIGNTOOL, SRC as CALIBRE_DIR, SW, build_dir, is64bit,
|
||||
python_major_minor_version, worker_env
|
||||
)
|
||||
from bypy.utils import py_compile, run, walk
|
||||
from bypy.freeze import (
|
||||
cleanup_site_packages, extract_extension_modules, freeze_python,
|
||||
path_to_freeze_dir, save_importer_src_to_header
|
||||
)
|
||||
from bypy.utils import mkdtemp, py_compile, run, walk
|
||||
|
||||
iv = globals()['init_env']
|
||||
calibre_constants = iv['calibre_constants']
|
||||
@ -120,14 +125,7 @@ def initbase(env):
|
||||
os.mkdir(env.dist)
|
||||
|
||||
|
||||
def add_plugins(env, ext_dir):
|
||||
printf('Adding plugins...')
|
||||
tgt = env.dll_dir
|
||||
for f in glob.glob(j(ext_dir, '*.pyd')):
|
||||
shutil.copy2(f, tgt)
|
||||
|
||||
|
||||
def freeze(env, ext_dir):
|
||||
def freeze(env, ext_dir, incdir):
|
||||
shutil.copy2(j(env.src_root, 'LICENSE'), env.base)
|
||||
|
||||
printf('Adding resources...')
|
||||
@ -140,11 +138,8 @@ def freeze(env, ext_dir):
|
||||
|
||||
def copybin(x):
|
||||
shutil.copy2(x, env.dll_dir)
|
||||
try:
|
||||
with contextlib.suppress(FileNotFoundError):
|
||||
shutil.copy2(x + '.manifest', env.dll_dir)
|
||||
except EnvironmentError as err:
|
||||
if err.errno != errno.ENOENT:
|
||||
raise
|
||||
|
||||
bindir = os.path.join(PREFIX, 'bin')
|
||||
for x in ('pdftohtml', 'pdfinfo', 'pdftoppm', 'jpegtran-calibre', 'cjpeg-calibre', 'optipng-calibre', 'JXRDecApp-calibre'):
|
||||
@ -155,13 +150,14 @@ def freeze(env, ext_dir):
|
||||
|
||||
copybin(os.path.join(env.python_base, 'python%s.dll' % env.py_ver.replace('.', '')))
|
||||
copybin(os.path.join(env.python_base, 'python%s.dll' % env.py_ver[0]))
|
||||
for x in glob.glob(os.path.join(env.python_base, 'DLLs', '*')): # python pyd modules and dlls
|
||||
for x in glob.glob(os.path.join(env.python_base, 'DLLs', '*.dll')): # dlls needed by python
|
||||
copybin(x)
|
||||
for f in walk(os.path.join(env.python_base, 'Lib')):
|
||||
q = f.lower()
|
||||
if q.endswith('.dll') and 'scintilla' not in q and 'pyqtbuild' not in q:
|
||||
copybin(f)
|
||||
add_plugins(env, ext_dir)
|
||||
ext_map = extract_extension_modules(ext_dir, env.dll_dir)
|
||||
ext_map.update(extract_extension_modules(j(env.python_base, 'DLLs'), env.dll_dir, move=False))
|
||||
|
||||
printf('Adding Qt...')
|
||||
for x in QT_DLLS:
|
||||
@ -196,22 +192,7 @@ def freeze(env, ext_dir):
|
||||
|
||||
shutil.copytree(r'%s\Lib' % env.python_base, env.lib_dir, ignore=ignore_lib)
|
||||
install_site_py(env)
|
||||
|
||||
# Fix win32com
|
||||
sp_dir = j(env.lib_dir, 'site-packages')
|
||||
comext = j(sp_dir, 'win32comext')
|
||||
shutil.copytree(j(comext, 'shell'), j(sp_dir, 'win32com', 'shell'))
|
||||
shutil.rmtree(comext)
|
||||
|
||||
# Fix pycryptodome
|
||||
with open(j(sp_dir, 'Crypto', 'Util', '_file_system.py'), 'w') as fspy:
|
||||
fspy.write('''
|
||||
import os, sys
|
||||
def pycryptodome_filename(dir_comps, filename):
|
||||
base = os.path.join(sys.app_dir, 'app', 'bin')
|
||||
path = os.path.join(base, '.'.join(dir_comps + [filename]))
|
||||
return path
|
||||
''')
|
||||
|
||||
printf('Adding calibre sources...')
|
||||
for x in glob.glob(j(CALIBRE_DIR, 'src', '*')):
|
||||
@ -221,24 +202,20 @@ def pycryptodome_filename(dir_comps, filename):
|
||||
else:
|
||||
shutil.copy(x, j(sp_dir, b(x)))
|
||||
|
||||
for x in (r'calibre\manual', r'calibre\plugins', 'pythonwin'):
|
||||
deld = j(sp_dir, x)
|
||||
if os.path.exists(deld):
|
||||
shutil.rmtree(deld)
|
||||
|
||||
for x in os.walk(j(sp_dir, 'calibre')):
|
||||
for f in x[-1]:
|
||||
if not f.endswith('.py'):
|
||||
os.remove(j(x[0], f))
|
||||
|
||||
extract_pyd_modules(env, sp_dir)
|
||||
ext_map.update(cleanup_site_packages(sp_dir))
|
||||
for x in os.listdir(sp_dir):
|
||||
os.rename(j(sp_dir, x), j(env.lib_dir, x))
|
||||
os.rmdir(sp_dir)
|
||||
printf('Extracting extension modules from', env.lib_dir, 'to', env.dll_dir)
|
||||
ext_map.update(extract_extension_modules(env.lib_dir, env.dll_dir))
|
||||
|
||||
printf('Byte-compiling all python modules...')
|
||||
for x in ('test', 'lib2to3'):
|
||||
x = j(env.lib_dir, x)
|
||||
if os.path.exists(x):
|
||||
shutil.rmtree(x)
|
||||
py_compile(env.lib_dir.replace(os.sep, '/'))
|
||||
# from bypy.utils import run_shell
|
||||
# run_shell(cwd=env.lib_dir)
|
||||
freeze_python(env.lib_dir, env.dll_dir, incdir)
|
||||
shutil.rmtree(env.lib_dir)
|
||||
save_importer_src_to_header(incdir, ext_map, develop_mode_env_var='CALIBRE_DEVELOP_FROM')
|
||||
|
||||
|
||||
def embed_manifests(env):
|
||||
@ -255,58 +232,6 @@ def embed_manifests(env):
|
||||
os.remove(manifest)
|
||||
|
||||
|
||||
def extract_pyd_modules(env, site_packages_dir):
|
||||
printf('\nExtracting .pyd modules from site-packages...')
|
||||
|
||||
def extract_pyd(path, root):
|
||||
fullname = os.path.relpath(path, root).replace(os.sep, '/').replace('/', '.')
|
||||
dest = os.path.join(env.dll_dir, fullname)
|
||||
if os.path.exists(dest):
|
||||
raise ValueError('Cannot extract %s into DLLs as it already exists' % fullname)
|
||||
os.rename(path, dest)
|
||||
bpy = dest[:-1]
|
||||
if os.path.exists(bpy):
|
||||
with open(bpy, 'rb') as f:
|
||||
raw = f.read().strip().decode('utf-8')
|
||||
if (not raw.startswith('def __bootstrap__') or not raw.endswith('__bootstrap__()')):
|
||||
raise ValueError('The file %r has non bootstrap code' % bpy)
|
||||
for ext in ('', 'c', 'o'):
|
||||
try:
|
||||
os.remove(bpy + ext)
|
||||
except EnvironmentError as err:
|
||||
if err.errno != errno.ENOENT:
|
||||
raise
|
||||
|
||||
def find_pyds(base):
|
||||
for dirpath, dirnames, filenames in os.walk(base):
|
||||
for fname in filenames:
|
||||
if fname.lower().endswith('.pyd'):
|
||||
yield os.path.join(dirpath, fname)
|
||||
|
||||
def process_root(root, base=None):
|
||||
for path in find_pyds(root):
|
||||
extract_pyd(path, base or root)
|
||||
|
||||
def absp(x):
|
||||
return os.path.normcase(os.path.abspath(os.path.join(site_packages_dir, x)))
|
||||
|
||||
roots = set()
|
||||
for pth in glob.glob(os.path.join(site_packages_dir, '*.pth')):
|
||||
for line in open(pth).readlines():
|
||||
line = line.strip()
|
||||
if line and not line.startswith('#') and os.path.exists(os.path.join(site_packages_dir, line)):
|
||||
roots.add(absp(line))
|
||||
|
||||
for x in os.listdir(site_packages_dir):
|
||||
x = absp(x)
|
||||
if x in roots:
|
||||
process_root(x)
|
||||
elif os.path.isdir(x):
|
||||
process_root(x, site_packages_dir)
|
||||
elif x.lower().endswith('.pyd'):
|
||||
extract_pyd(x, site_packages_dir)
|
||||
|
||||
|
||||
def embed_resources(env, module, desc=None, extra_data=None, product_description=None):
|
||||
icon_base = j(env.src_root, 'icons')
|
||||
icon_map = {
|
||||
@ -526,7 +451,7 @@ def build_utils(env):
|
||||
build(j(base, 'eject.c'), 'calibre-eject.exe')
|
||||
|
||||
|
||||
def build_launchers(env, debug=False):
|
||||
def build_launchers(env, incdir, debug=False):
|
||||
if not os.path.exists(env.obj_dir):
|
||||
os.makedirs(env.obj_dir)
|
||||
dflags = (['/Zi'] if debug else [])
|
||||
@ -536,6 +461,7 @@ def build_launchers(env, debug=False):
|
||||
objects = [j(env.obj_dir, b(x) + '.obj') for x in sources]
|
||||
cflags = '/c /EHsc /W3 /Ox /nologo /D_UNICODE'.split()
|
||||
cflags += ['/DPYDLL="python%s.dll"' % env.py_ver.replace('.', ''), '/I%s/include' % env.python_base]
|
||||
cflags += [f'/I{path_to_freeze_dir()}', f'/I{incdir}']
|
||||
for src, obj in zip(sources, objects):
|
||||
cmd = [CL] + cflags + dflags + ['/MD', '/Fo' + obj, '/Tc' + src]
|
||||
run_compiler(env, *cmd)
|
||||
@ -586,80 +512,6 @@ def build_launchers(env, debug=False):
|
||||
run(*cmd)
|
||||
|
||||
|
||||
def add_to_zipfile(zf, name, base, zf_names):
|
||||
if '__pycache__' in name:
|
||||
return
|
||||
abspath = j(base, name)
|
||||
name = name.replace(os.sep, '/')
|
||||
if name in zf_names:
|
||||
raise ValueError('Already added %r to zipfile [%r]' % (name, abspath))
|
||||
zinfo = zipfile.ZipInfo(filename=name, date_time=(1980, 1, 1, 0, 0, 0))
|
||||
|
||||
if os.path.isdir(abspath):
|
||||
if not os.listdir(abspath):
|
||||
return
|
||||
zinfo.external_attr = 0o700 << 16
|
||||
zf.writestr(zinfo, '')
|
||||
for x in os.listdir(abspath):
|
||||
add_to_zipfile(zf, name + os.sep + x, base, zf_names)
|
||||
else:
|
||||
ext = os.path.splitext(name)[1].lower()
|
||||
if ext in ('.dll',):
|
||||
raise ValueError('Cannot add %r to zipfile' % abspath)
|
||||
zinfo.external_attr = 0o600 << 16
|
||||
if ext in ('.py', '.pyc', '.pyo'):
|
||||
with open(abspath, 'rb') as f:
|
||||
zf.writestr(zinfo, f.read())
|
||||
|
||||
zf_names.add(name)
|
||||
|
||||
|
||||
def archive_lib_dir(env):
|
||||
printf('Putting all python code into a zip file for performance')
|
||||
zf_names = set()
|
||||
with zipfile.ZipFile(env.pylib, 'w', zipfile.ZIP_STORED) as zf:
|
||||
# Add everything in Lib except site-packages to the zip file
|
||||
for x in os.listdir(env.lib_dir):
|
||||
if x == 'site-packages':
|
||||
continue
|
||||
add_to_zipfile(zf, x, env.lib_dir, zf_names)
|
||||
|
||||
sp = j(env.lib_dir, 'site-packages')
|
||||
# Special handling for pywin32
|
||||
handled = {'pywin32.pth', 'win32'}
|
||||
base = j(sp, 'win32', 'lib')
|
||||
for x in os.listdir(base):
|
||||
if os.path.splitext(x)[1] not in ('.exe',) and x != '__pycache__':
|
||||
add_to_zipfile(zf, x, base, zf_names)
|
||||
base = os.path.dirname(base)
|
||||
for x in os.listdir(base):
|
||||
if not os.path.isdir(j(base, x)):
|
||||
if os.path.splitext(x)[1] not in ('.exe',):
|
||||
add_to_zipfile(zf, x, base, zf_names)
|
||||
|
||||
# We dont want the site.py (if any) from site-packages
|
||||
handled.add('site.pyo')
|
||||
handled.add('site.pyc')
|
||||
handled.add('site.py')
|
||||
handled.add('sitecustomize.pyo')
|
||||
handled.add('sitecustomize.pyc')
|
||||
handled.add('sitecustomize.py')
|
||||
|
||||
# The rest of site-packages
|
||||
for x in os.listdir(sp):
|
||||
if x in handled or x.endswith('.egg-info') or x.endswith('.dist-info'):
|
||||
continue
|
||||
absp = j(sp, x)
|
||||
if os.path.isdir(absp):
|
||||
if not os.listdir(absp):
|
||||
continue
|
||||
add_to_zipfile(zf, x, sp, zf_names)
|
||||
else:
|
||||
add_to_zipfile(zf, x, sp, zf_names)
|
||||
|
||||
shutil.rmtree(env.lib_dir)
|
||||
|
||||
|
||||
def copy_crt_and_d3d(env):
|
||||
printf('Copying CRT and D3D...')
|
||||
plat = ('x64' if is64bit else 'x86')
|
||||
@ -718,13 +570,13 @@ def main():
|
||||
args = globals()['args']
|
||||
run_tests = iv['run_tests']
|
||||
env = Env(build_dir())
|
||||
incdir = mkdtemp('include')
|
||||
initbase(env)
|
||||
build_launchers(env)
|
||||
freeze(env, ext_dir, incdir)
|
||||
build_launchers(env, incdir)
|
||||
build_utils(env)
|
||||
freeze(env, ext_dir)
|
||||
embed_manifests(env)
|
||||
copy_crt_and_d3d(env)
|
||||
archive_lib_dir(env)
|
||||
if not args.skip_tests:
|
||||
run_tests(os.path.join(env.base, 'calibre-debug.exe'), env.base)
|
||||
if args.sign_installers:
|
||||
|
||||
@ -17,57 +17,9 @@
|
||||
#include <fcntl.h>
|
||||
#include "../run-python.h"
|
||||
|
||||
static int GUI_APP = 0;
|
||||
static char python_dll[] = PYDLL;
|
||||
|
||||
static int _show_error(const wchar_t *preamble, const wchar_t *msg, const int code) {
|
||||
static wchar_t buf[4096];
|
||||
static char utf8_buf[4096] = {0};
|
||||
int n = WideCharToMultiByte(CP_UTF8, 0, preamble, -1, utf8_buf, sizeof(utf8_buf) - 1, NULL, NULL);
|
||||
if (n > 0) fprintf(stderr, "%s\r\n ", utf8_buf);
|
||||
n = WideCharToMultiByte(CP_UTF8, 0, msg, -1, utf8_buf, sizeof(utf8_buf) - 1, NULL, NULL);
|
||||
if (n > 0) fprintf(stderr, "%s (Error Code: %d)\r\n ", utf8_buf, code);
|
||||
fflush(stderr);
|
||||
|
||||
if (GUI_APP) {
|
||||
_snwprintf_s(buf, arraysz(buf), _TRUNCATE, L"%ls\r\n %ls (Error Code: %d)\r\n", preamble, msg, code);
|
||||
MessageBeep(MB_ICONERROR);
|
||||
MessageBox(NULL, buf, NULL, MB_OK|MB_ICONERROR);
|
||||
}
|
||||
return code;
|
||||
}
|
||||
|
||||
|
||||
|
||||
int show_last_error_crt(wchar_t *preamble) {
|
||||
wchar_t buf[1000];
|
||||
int err = 0;
|
||||
|
||||
_get_errno(&err);
|
||||
_wcserror_s(buf, 1000, err);
|
||||
return _show_error(preamble, buf, err);
|
||||
}
|
||||
|
||||
int show_last_error(wchar_t *preamble) {
|
||||
wchar_t *msg = NULL;
|
||||
DWORD dw = GetLastError();
|
||||
int ret;
|
||||
|
||||
FormatMessage(
|
||||
FORMAT_MESSAGE_ALLOCATE_BUFFER |
|
||||
FORMAT_MESSAGE_FROM_SYSTEM |
|
||||
FORMAT_MESSAGE_IGNORE_INSERTS,
|
||||
NULL,
|
||||
dw,
|
||||
MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),
|
||||
(LPWSTR)&msg,
|
||||
0,
|
||||
NULL );
|
||||
|
||||
ret = _show_error(preamble, msg, (int)dw);
|
||||
if (msg != NULL) LocalFree(msg);
|
||||
return ret;
|
||||
}
|
||||
|
||||
static wchar_t qt_prefix_dir[MAX_PATH] = {0};
|
||||
|
||||
@ -97,8 +49,7 @@ get_install_locations(void) {
|
||||
static void
|
||||
load_python_dll() {
|
||||
get_install_locations();
|
||||
if (FAILED(__HrLoadAllImportsForDll(python_dll)))
|
||||
ExitProcess(_show_error(L"Failed to delay load the python dll", L"", 1));
|
||||
bypy_setup_python(python_dll);
|
||||
}
|
||||
|
||||
const static wchar_t out_of_memory[] = L"Out of memory";
|
||||
@ -115,42 +66,6 @@ redirect_out_stream(FILE *stream) {
|
||||
}
|
||||
}
|
||||
|
||||
static PyObject*
|
||||
gui_error_message(PyObject *self, PyObject *args) {
|
||||
PyObject *pt, *pm;
|
||||
if (!PyArg_ParseTuple(args, "UU", &pt, &pm)) return NULL;
|
||||
wchar_t title[256] = {0}, text[4096] = {0};
|
||||
PyUnicode_AsWideChar(pt, title, arraysz(title)-1);
|
||||
PyUnicode_AsWideChar(pm, text, arraysz(text)-1);
|
||||
MessageBoxW(NULL, text, title, MB_ICONERROR|MB_OK);
|
||||
return PyBool_FromLong(1);
|
||||
}
|
||||
|
||||
static PyMethodDef methods[] = {
|
||||
{"gui_error_message", (PyCFunction)gui_error_message, METH_VARARGS,
|
||||
"gui_error_message(title, msg) -> Show a GUI based error message."
|
||||
},
|
||||
{NULL} /* Sentinel */
|
||||
};
|
||||
|
||||
|
||||
static struct PyModuleDef module = {
|
||||
/* m_base */ PyModuleDef_HEAD_INIT,
|
||||
/* m_name */ "calibre_os_module",
|
||||
/* m_doc */ "Integration with OS facilities during startup",
|
||||
/* m_size */ -1,
|
||||
/* m_methods */ methods,
|
||||
/* m_slots */ 0,
|
||||
/* m_traverse */ 0,
|
||||
/* m_clear */ 0,
|
||||
/* m_free */ 0,
|
||||
};
|
||||
|
||||
PyObject*
|
||||
calibre_os_module(void) {
|
||||
return PyModule_Create(&module);
|
||||
}
|
||||
|
||||
static void
|
||||
null_invalid_parameter_handler(
|
||||
const wchar_t * expression,
|
||||
@ -187,7 +102,6 @@ execute_python_entrypoint(const wchar_t *basename, const wchar_t *module, const
|
||||
interpreter_data.argv = CommandLineToArgvW(GetCommandLineW(), &interpreter_data.argc);
|
||||
if (interpreter_data.argv == NULL) ExitProcess(show_last_error(L"Failed to get command line"));
|
||||
interpreter_data.basename = basename; interpreter_data.module = module; interpreter_data.function = function;
|
||||
interpreter_data.calibre_os_module = calibre_os_module;
|
||||
load_python_dll();
|
||||
pre_initialize_interpreter(is_gui_app);
|
||||
run_interpreter();
|
||||
|
||||
@ -32,6 +32,8 @@ class BuildTest(unittest.TestCase):
|
||||
self.assertTrue(False, 'Failed to load DLL %s with error: %s' % (x, err))
|
||||
from Crypto.Cipher import AES
|
||||
del AES
|
||||
from pywintypes import error
|
||||
del error
|
||||
|
||||
@unittest.skipUnless(islinux, 'DBUS only used on linux')
|
||||
def test_dbus(self):
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user