From 77a0558cf2246058c9b13bb1816f3a55bd0e27b3 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 6 Dec 2015 18:19:03 +0530 Subject: [PATCH] Initial stab at replacing use of MemoryModule --- setup/installer/windows/MemoryModule.c | 488 ------------------------- setup/installer/windows/MemoryModule.h | 48 --- setup/installer/windows/freeze.py | 236 +++++------- setup/installer/windows/site.py | 88 ++--- setup/installer/windows/util.c | 104 ------ 5 files changed, 110 insertions(+), 854 deletions(-) delete mode 100644 setup/installer/windows/MemoryModule.c delete mode 100644 setup/installer/windows/MemoryModule.h diff --git a/setup/installer/windows/MemoryModule.c b/setup/installer/windows/MemoryModule.c deleted file mode 100644 index a0fdecb6d4..0000000000 --- a/setup/installer/windows/MemoryModule.c +++ /dev/null @@ -1,488 +0,0 @@ -/* - * Memory DLL loading code - * Version 0.0.3 - * - * Copyright (c) 2004-2012 by Joachim Bauch / mail@joachim-bauch.de - * http://www.joachim-bauch.de - * - * The contents of this file are subject to the Mozilla Public License Version - * 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is MemoryModule.c - * - * The Initial Developer of the Original Code is Joachim Bauch. - * - * Portions created by Joachim Bauch are Copyright (C) 2004-2012 - * Joachim Bauch. All Rights Reserved. - * - */ - -#ifndef __GNUC__ -// disable warnings about pointer <-> DWORD conversions -#pragma warning( disable : 4311 4312 ) -#endif - -#ifdef _WIN64 -#define POINTER_TYPE ULONGLONG -#else -#define POINTER_TYPE DWORD -#endif - -#include -#include -#ifdef DEBUG_OUTPUT -#include -#endif - -#ifndef IMAGE_SIZEOF_BASE_RELOCATION -// Vista SDKs no longer define IMAGE_SIZEOF_BASE_RELOCATION!? -#define IMAGE_SIZEOF_BASE_RELOCATION (sizeof(IMAGE_BASE_RELOCATION)) -#endif - -#include "MemoryModule.h" - -typedef struct { - PIMAGE_NT_HEADERS headers; - unsigned char *codeBase; - HMODULE *modules; - int numModules; - int initialized; -} MEMORYMODULE, *PMEMORYMODULE; - -typedef BOOL (WINAPI *DllEntryProc)(HINSTANCE hinstDLL, DWORD fdwReason, LPVOID lpReserved); - -#define GET_HEADER_DICTIONARY(module, idx) &(module)->headers->OptionalHeader.DataDirectory[idx] - -#ifdef DEBUG_OUTPUT -static void -OutputLastError(const char *msg) -{ - LPVOID tmp; - char *tmpmsg; - FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, - NULL, GetLastError(), MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), (LPTSTR)&tmp, 0, NULL); - tmpmsg = (char *)LocalAlloc(LPTR, strlen(msg) + strlen(tmp) + 3); - sprintf(tmpmsg, "%s: %s", msg, tmp); - OutputDebugString(tmpmsg); - LocalFree(tmpmsg); - LocalFree(tmp); -} -#endif - -static void -CopySections(const unsigned char *data, PIMAGE_NT_HEADERS old_headers, PMEMORYMODULE module) -{ - int i, size; - unsigned char *codeBase = module->codeBase; - unsigned char *dest; - PIMAGE_SECTION_HEADER section = IMAGE_FIRST_SECTION(module->headers); - for (i=0; iheaders->FileHeader.NumberOfSections; i++, section++) { - if (section->SizeOfRawData == 0) { - // section doesn't contain data in the dll itself, but may define - // uninitialized data - size = old_headers->OptionalHeader.SectionAlignment; - if (size > 0) { - dest = (unsigned char *)VirtualAlloc(codeBase + section->VirtualAddress, - size, - MEM_COMMIT, - PAGE_READWRITE); - - section->Misc.PhysicalAddress = (DWORD)dest; - memset(dest, 0, size); - } - - // section is empty - continue; - } - - // commit memory block and copy data from dll - dest = (unsigned char *)VirtualAlloc(codeBase + section->VirtualAddress, - section->SizeOfRawData, - MEM_COMMIT, - PAGE_READWRITE); - memcpy(dest, data + section->PointerToRawData, section->SizeOfRawData); - section->Misc.PhysicalAddress = (DWORD)dest; - } -} - -// Protection flags for memory pages (Executable, Readable, Writeable) -static int ProtectionFlags[2][2][2] = { - { - // not executable - {PAGE_NOACCESS, PAGE_WRITECOPY}, - {PAGE_READONLY, PAGE_READWRITE}, - }, { - // executable - {PAGE_EXECUTE, PAGE_EXECUTE_WRITECOPY}, - {PAGE_EXECUTE_READ, PAGE_EXECUTE_READWRITE}, - }, -}; - -static void -FinalizeSections(PMEMORYMODULE module) -{ - int i; - PIMAGE_SECTION_HEADER section = IMAGE_FIRST_SECTION(module->headers); -#ifdef _WIN64 - POINTER_TYPE imageOffset = (module->headers->OptionalHeader.ImageBase & 0xffffffff00000000); -#else - #define imageOffset 0 -#endif - - // loop through all sections and change access flags - for (i=0; iheaders->FileHeader.NumberOfSections; i++, section++) { - DWORD protect, oldProtect, size; - int executable = (section->Characteristics & IMAGE_SCN_MEM_EXECUTE) != 0; - int readable = (section->Characteristics & IMAGE_SCN_MEM_READ) != 0; - int writeable = (section->Characteristics & IMAGE_SCN_MEM_WRITE) != 0; - - if (section->Characteristics & IMAGE_SCN_MEM_DISCARDABLE) { - // section is not needed any more and can safely be freed - VirtualFree((LPVOID)((POINTER_TYPE)section->Misc.PhysicalAddress | imageOffset), section->SizeOfRawData, MEM_DECOMMIT); - continue; - } - - // determine protection flags based on characteristics - protect = ProtectionFlags[executable][readable][writeable]; - if (section->Characteristics & IMAGE_SCN_MEM_NOT_CACHED) { - protect |= PAGE_NOCACHE; - } - - // determine size of region - size = section->SizeOfRawData; - if (size == 0) { - if (section->Characteristics & IMAGE_SCN_CNT_INITIALIZED_DATA) { - size = module->headers->OptionalHeader.SizeOfInitializedData; - } else if (section->Characteristics & IMAGE_SCN_CNT_UNINITIALIZED_DATA) { - size = module->headers->OptionalHeader.SizeOfUninitializedData; - } - } - - if (size > 0) { - // change memory access flags - if (VirtualProtect((LPVOID)((POINTER_TYPE)section->Misc.PhysicalAddress | imageOffset), size, protect, &oldProtect) == 0) -#ifdef DEBUG_OUTPUT - OutputLastError("Error protecting memory page") -#endif - ; - } - } -#ifndef _WIN64 -#undef imageOffset -#endif -} - -static void -PerformBaseRelocation(PMEMORYMODULE module, SIZE_T delta) -{ - DWORD i; - unsigned char *codeBase = module->codeBase; - - PIMAGE_DATA_DIRECTORY directory = GET_HEADER_DICTIONARY(module, IMAGE_DIRECTORY_ENTRY_BASERELOC); - if (directory->Size > 0) { - PIMAGE_BASE_RELOCATION relocation = (PIMAGE_BASE_RELOCATION) (codeBase + directory->VirtualAddress); - for (; relocation->VirtualAddress > 0; ) { - unsigned char *dest = codeBase + relocation->VirtualAddress; - unsigned short *relInfo = (unsigned short *)((unsigned char *)relocation + IMAGE_SIZEOF_BASE_RELOCATION); - for (i=0; i<((relocation->SizeOfBlock-IMAGE_SIZEOF_BASE_RELOCATION) / 2); i++, relInfo++) { - DWORD *patchAddrHL; -#ifdef _WIN64 - ULONGLONG *patchAddr64; -#endif - int type, offset; - - // the upper 4 bits define the type of relocation - type = *relInfo >> 12; - // the lower 12 bits define the offset - offset = *relInfo & 0xfff; - - switch (type) - { - case IMAGE_REL_BASED_ABSOLUTE: - // skip relocation - break; - - case IMAGE_REL_BASED_HIGHLOW: - // change complete 32 bit address - patchAddrHL = (DWORD *) (dest + offset); - *patchAddrHL += (DWORD)delta; - break; - -#ifdef _WIN64 - case IMAGE_REL_BASED_DIR64: - patchAddr64 = (ULONGLONG *) (dest + offset); - *patchAddr64 += delta; - break; -#endif - - default: - //printf("Unknown relocation: %d\n", type); - break; - } - } - - // advance to next relocation block - relocation = (PIMAGE_BASE_RELOCATION) (((char *) relocation) + relocation->SizeOfBlock); - } - } -} - -static int -BuildImportTable(PMEMORYMODULE module) -{ - int result=1; - unsigned char *codeBase = module->codeBase; - - PIMAGE_DATA_DIRECTORY directory = GET_HEADER_DICTIONARY(module, IMAGE_DIRECTORY_ENTRY_IMPORT); - if (directory->Size > 0) { - PIMAGE_IMPORT_DESCRIPTOR importDesc = (PIMAGE_IMPORT_DESCRIPTOR) (codeBase + directory->VirtualAddress); - for (; !IsBadReadPtr(importDesc, sizeof(IMAGE_IMPORT_DESCRIPTOR)) && importDesc->Name; importDesc++) { - POINTER_TYPE *thunkRef; - FARPROC *funcRef; - HMODULE handle = LoadLibrary((LPCSTR) (codeBase + importDesc->Name)); - if (handle == NULL) { -#if DEBUG_OUTPUT - OutputLastError("Can't load library"); -#endif - result = 0; - break; - } - - module->modules = (HMODULE *)realloc(module->modules, (module->numModules+1)*(sizeof(HMODULE))); - if (module->modules == NULL) { - result = 0; - break; - } - - module->modules[module->numModules++] = handle; - if (importDesc->OriginalFirstThunk) { - thunkRef = (POINTER_TYPE *) (codeBase + importDesc->OriginalFirstThunk); - funcRef = (FARPROC *) (codeBase + importDesc->FirstThunk); - } else { - // no hint table - thunkRef = (POINTER_TYPE *) (codeBase + importDesc->FirstThunk); - funcRef = (FARPROC *) (codeBase + importDesc->FirstThunk); - } - for (; *thunkRef; thunkRef++, funcRef++) { - if (IMAGE_SNAP_BY_ORDINAL(*thunkRef)) { - *funcRef = (FARPROC)GetProcAddress(handle, (LPCSTR)IMAGE_ORDINAL(*thunkRef)); - } else { - PIMAGE_IMPORT_BY_NAME thunkData = (PIMAGE_IMPORT_BY_NAME) (codeBase + (*thunkRef)); - *funcRef = (FARPROC)GetProcAddress(handle, (LPCSTR)&thunkData->Name); - } - if (*funcRef == 0) { - result = 0; - break; - } - } - - if (!result) { - break; - } - } - } - - return result; -} - -HMEMORYMODULE MemoryLoadLibrary(const void *data) -{ - PMEMORYMODULE result; - PIMAGE_DOS_HEADER dos_header; - PIMAGE_NT_HEADERS old_header; - unsigned char *code, *headers; - SIZE_T locationDelta; - DllEntryProc DllEntry; - BOOL successfull; - - dos_header = (PIMAGE_DOS_HEADER)data; - if (dos_header->e_magic != IMAGE_DOS_SIGNATURE) { -#if DEBUG_OUTPUT - OutputDebugString("Not a valid executable file.\n"); -#endif - return NULL; - } - - old_header = (PIMAGE_NT_HEADERS)&((const unsigned char *)(data))[dos_header->e_lfanew]; - if (old_header->Signature != IMAGE_NT_SIGNATURE) { -#if DEBUG_OUTPUT - OutputDebugString("No PE header found.\n"); -#endif - return NULL; - } - - // reserve memory for image of library - code = (unsigned char *)VirtualAlloc((LPVOID)(old_header->OptionalHeader.ImageBase), - old_header->OptionalHeader.SizeOfImage, - MEM_RESERVE, - PAGE_READWRITE); - - if (code == NULL) { - // try to allocate memory at arbitrary position - code = (unsigned char *)VirtualAlloc(NULL, - old_header->OptionalHeader.SizeOfImage, - MEM_RESERVE, - PAGE_READWRITE); - if (code == NULL) { -#if DEBUG_OUTPUT - OutputLastError("Can't reserve memory"); -#endif - return NULL; - } - } - - result = (PMEMORYMODULE)HeapAlloc(GetProcessHeap(), 0, sizeof(MEMORYMODULE)); - result->codeBase = code; - result->numModules = 0; - result->modules = NULL; - result->initialized = 0; - - // XXX: is it correct to commit the complete memory region at once? - // calling DllEntry raises an exception if we don't... - VirtualAlloc(code, - old_header->OptionalHeader.SizeOfImage, - MEM_COMMIT, - PAGE_READWRITE); - - // commit memory for headers - headers = (unsigned char *)VirtualAlloc(code, - old_header->OptionalHeader.SizeOfHeaders, - MEM_COMMIT, - PAGE_READWRITE); - - // copy PE header to code - memcpy(headers, dos_header, dos_header->e_lfanew + old_header->OptionalHeader.SizeOfHeaders); - result->headers = (PIMAGE_NT_HEADERS)&((const unsigned char *)(headers))[dos_header->e_lfanew]; - - // update position - result->headers->OptionalHeader.ImageBase = (POINTER_TYPE)code; - - // copy sections from DLL file block to new memory location - CopySections(data, old_header, result); - - // adjust base address of imported data - locationDelta = (SIZE_T)(code - old_header->OptionalHeader.ImageBase); - if (locationDelta != 0) { - PerformBaseRelocation(result, locationDelta); - } - - // load required dlls and adjust function table of imports - if (!BuildImportTable(result)) { - goto error; - } - - // mark memory pages depending on section headers and release - // sections that are marked as "discardable" - FinalizeSections(result); - - // get entry point of loaded library - if (result->headers->OptionalHeader.AddressOfEntryPoint != 0) { - DllEntry = (DllEntryProc) (code + result->headers->OptionalHeader.AddressOfEntryPoint); - if (DllEntry == 0) { -#if DEBUG_OUTPUT - OutputDebugString("Library has no entry point.\n"); -#endif - goto error; - } - - // notify library about attaching to process - successfull = (*DllEntry)((HINSTANCE)code, DLL_PROCESS_ATTACH, 0); - if (!successfull) { -#if DEBUG_OUTPUT - OutputDebugString("Can't attach library.\n"); -#endif - goto error; - } - result->initialized = 1; - } - - return (HMEMORYMODULE)result; - -error: - // cleanup - MemoryFreeLibrary(result); - return NULL; -} - -FARPROC MemoryGetProcAddress(HMEMORYMODULE module, const char *name) -{ - unsigned char *codeBase = ((PMEMORYMODULE)module)->codeBase; - int idx=-1; - DWORD i, *nameRef; - WORD *ordinal; - PIMAGE_EXPORT_DIRECTORY exports; - PIMAGE_DATA_DIRECTORY directory = GET_HEADER_DICTIONARY((PMEMORYMODULE)module, IMAGE_DIRECTORY_ENTRY_EXPORT); - if (directory->Size == 0) { - // no export table found - return NULL; - } - - exports = (PIMAGE_EXPORT_DIRECTORY) (codeBase + directory->VirtualAddress); - if (exports->NumberOfNames == 0 || exports->NumberOfFunctions == 0) { - // DLL doesn't export anything - return NULL; - } - - // search function name in list of exported names - nameRef = (DWORD *) (codeBase + exports->AddressOfNames); - ordinal = (WORD *) (codeBase + exports->AddressOfNameOrdinals); - for (i=0; iNumberOfNames; i++, nameRef++, ordinal++) { - if (_stricmp(name, (const char *) (codeBase + (*nameRef))) == 0) { - idx = *ordinal; - break; - } - } - - if (idx == -1) { - // exported symbol not found - return NULL; - } - - if ((DWORD)idx > exports->NumberOfFunctions) { - // name <-> ordinal number don't match - return NULL; - } - - // AddressOfFunctions contains the RVAs to the "real" functions - return (FARPROC) (codeBase + (*(DWORD *) (codeBase + exports->AddressOfFunctions + (idx*4)))); -} - -void MemoryFreeLibrary(HMEMORYMODULE mod) -{ - int i; - PMEMORYMODULE module = (PMEMORYMODULE)mod; - - if (module != NULL) { - if (module->initialized != 0) { - // notify library about detaching from process - DllEntryProc DllEntry = (DllEntryProc) (module->codeBase + module->headers->OptionalHeader.AddressOfEntryPoint); - (*DllEntry)((HINSTANCE)module->codeBase, DLL_PROCESS_DETACH, 0); - module->initialized = 0; - } - - if (module->modules != NULL) { - // free previously opened libraries - for (i=0; inumModules; i++) { - if (module->modules[i] != INVALID_HANDLE_VALUE) { - FreeLibrary(module->modules[i]); - } - } - - free(module->modules); - } - - if (module->codeBase != NULL) { - // release memory of library - VirtualFree(module->codeBase, 0, MEM_RELEASE); - } - - HeapFree(GetProcessHeap(), 0, module); - } -} diff --git a/setup/installer/windows/MemoryModule.h b/setup/installer/windows/MemoryModule.h deleted file mode 100644 index 1757c7ed98..0000000000 --- a/setup/installer/windows/MemoryModule.h +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Memory DLL loading code - * Version 0.0.3 - * - * Copyright (c) 2004-2012 by Joachim Bauch / mail@joachim-bauch.de - * http://www.joachim-bauch.de - * - * The contents of this file are subject to the Mozilla Public License Version - * 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License - * for the specific language governing rights and limitations under the - * License. - * - * The Original Code is MemoryModule.h - * - * The Initial Developer of the Original Code is Joachim Bauch. - * - * Portions created by Joachim Bauch are Copyright (C) 2004-2012 - * Joachim Bauch. All Rights Reserved. - * - */ - -#ifndef __MEMORY_MODULE_HEADER -#define __MEMORY_MODULE_HEADER - -#include - -typedef void *HMEMORYMODULE; - -#ifdef __cplusplus -extern "C" { -#endif - -HMEMORYMODULE MemoryLoadLibrary(const void *); - -FARPROC MemoryGetProcAddress(HMEMORYMODULE, const char *); - -void MemoryFreeLibrary(HMEMORYMODULE); - -#ifdef __cplusplus -} -#endif - -#endif // __MEMORY_MODULE_HEADER diff --git a/setup/installer/windows/freeze.py b/setup/installer/windows/freeze.py index 4d173b064b..30297b71e2 100644 --- a/setup/installer/windows/freeze.py +++ b/setup/installer/windows/freeze.py @@ -6,8 +6,7 @@ __license__ = 'GPL v3' __copyright__ = '2009, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import sys, os, shutil, glob, py_compile, subprocess, re, zipfile, time, textwrap -from itertools import chain +import sys, os, shutil, glob, py_compile, subprocess, re, zipfile, time, textwrap, errno from setup import (Command, modules, functions, basenames, __version__, __appname__) @@ -16,8 +15,7 @@ from setup.build_environment import ( from setup.installer.windows.wix import WixMixIn OPENSSL_DIR = os.environ.get('OPENSSL_DIR', os.path.join(SW, 'private', 'openssl')) -SW = r'C:\cygwin64\home\kovid\sw' -CRT = r'C:\Microsoft.VC90.CRT' +IMAGEMAGICK = os.path.join(SW, 'build', 'ImageMagick-*\\VisualMagick\\bin') LZMA = os.path.join(SW, *('private/easylzma/build/easylzma-0.0.8'.split('/'))) QT_DIR = subprocess.check_output([QMAKE, '-query', 'QT_INSTALL_PREFIX']).decode('utf-8').strip() @@ -50,51 +48,6 @@ def walk(dir): for f in record[-1]: yield os.path.join(record[0], f) -# Remove CRT dep from manifests {{{ -def get_manifest_from_dll(dll): - import win32api, pywintypes - LOAD_LIBRARY_AS_DATAFILE = 2 - d = win32api.LoadLibraryEx(os.path.abspath(dll), 0, LOAD_LIBRARY_AS_DATAFILE) - try: - resources = win32api.EnumResourceNames(d, 24) - except pywintypes.error as err: - if err.winerror == 1812: - return None, None # no resource section (probably a .pyd file) - raise - if resources: - return resources[0], win32api.LoadResource(d, 24, resources[0]) - return None, None - -def update_manifest(dll, rnum, manifest): - import win32api - h = win32api.BeginUpdateResource(dll, 0) - win32api.UpdateResource(h, 24, rnum, manifest) - win32api.EndUpdateResource(h, 0) - -_crt_pat = re.compile(r'Microsoft\.VC\d+\.CRT') - -def remove_CRT_from_manifest(dll, log=print): - from lxml import etree - rnum, manifest = get_manifest_from_dll(dll) - if manifest is None: - return - root = etree.fromstring(manifest) - found = False - for ai in root.xpath('//*[local-name()="assemblyIdentity" and @name]'): - name = ai.get('name') - if _crt_pat.match(name): - p = ai.getparent() - pp = p.getparent() - pp.remove(p) - if len(pp) == 0: - pp.getparent().remove(pp) - found = True - if found: - manifest = etree.tostring(root, pretty_print=True) - update_manifest(dll, rnum, manifest) - log('\t', os.path.basename(dll)) -# }}} - class Win32Freeze(Command, WixMixIn): description = 'Freeze windows calibre installation' @@ -114,7 +67,7 @@ class Win32Freeze(Command, WixMixIn): action='store_true', help='Dont strip the generated binaries (no-op on windows)') def run(self, opts): - self.SW = SW + self.python_base = os.path.join(SW, 'private', 'python') self.portable_uncompressed_size = 0 self.opts = opts self.src_root = self.d(self.SRC) @@ -124,36 +77,21 @@ class Win32Freeze(Command, WixMixIn): self.lib_dir = self.j(self.base, 'Lib') self.pylib = self.j(self.base, 'pylib.zip') self.dll_dir = self.j(self.base, 'DLLs') - self.plugins_dir = os.path.join(self.base, 'plugins2') self.portable_base = self.j(self.d(self.base), 'Calibre Portable') self.obj_dir = self.j(self.src_root, 'build', 'launcher') self.initbase() self.build_launchers() self.build_utils() - self.add_plugins() self.freeze() self.embed_manifests() self.install_site_py() self.archive_lib_dir() - self.remove_CRT_from_manifests() - self.create_installer() + # self.create_installer() if not is64bit: self.build_portable() self.build_portable_installer() - self.sign_installers() - - def remove_CRT_from_manifests(self): - ''' - The dependency on the CRT is removed from the manifests of all DLLs. - This allows the CRT loaded by the .exe files to be used instead. - ''' - self.info('Removing CRT dependency from manifests of:') - for dll in chain(walk(self.dll_dir), walk(self.plugins_dir)): - bn = self.b(dll) - if bn.lower().rpartition('.')[-1] not in {'dll', 'pyd'}: - continue - remove_CRT_from_manifest(dll, self.info) + # self.sign_installers() def initbase(self): if self.e(self.base): @@ -162,38 +100,16 @@ class Win32Freeze(Command, WixMixIn): def add_plugins(self): self.info('Adding plugins...') - tgt = self.plugins_dir - if os.path.exists(tgt): - shutil.rmtree(tgt) - os.mkdir(tgt) + tgt = self.dll_dir base = self.j(self.SRC, 'calibre', 'plugins') for f in glob.glob(self.j(base, '*.pyd')): - # We dont want the manifests as the manifest in the exe will be - # used instead shutil.copy2(f, tgt) - def fix_pyd_bootstraps_in(self, folder): - for dirpath, dirnames, filenames in os.walk(folder): - for f in filenames: - name, ext = os.path.splitext(f) - bpy = self.j(dirpath, name + '.py') - if ext == '.pyd' and os.path.exists(bpy): - with open(bpy, 'rb') as f: - raw = f.read().strip() - if (not raw.startswith('def __bootstrap__') or not - raw.endswith('__bootstrap__()')): - raise Exception('The file %r has non' - ' bootstrap code'%self.j(dirpath, f)) - for ext in ('.py', '.pyc', '.pyo'): - x = self.j(dirpath, name+ext) - if os.path.exists(x): - os.remove(x) - def freeze(self): shutil.copy2(self.j(self.src_root, 'LICENSE'), self.base) - self.info('Adding CRT') - shutil.copytree(CRT, self.j(self.base, os.path.basename(CRT))) + # self.info('Adding CRT') + # shutil.copytree(CRT, self.j(self.base, os.path.basename(CRT))) self.info('Adding resources...') tgt = self.j(self.base, 'resources') @@ -201,8 +117,8 @@ class Win32Freeze(Command, WixMixIn): shutil.rmtree(tgt) shutil.copytree(self.j(self.src_root, 'resources'), tgt) - self.info('Adding Qt and python...') - shutil.copytree(r'C:\Python%s\DLLs'%self.py_ver, self.dll_dir, + self.info('Adding Qt...') + shutil.copytree(os.path.join(self.python_base, 'DLLs') , self.dll_dir, ignore=shutil.ignore_patterns('msvc*.dll', 'Microsoft.*')) for x in glob.glob(self.j(OPENSSL_DIR, 'bin', '*.dll')): shutil.copy2(x, self.dll_dir) @@ -211,30 +127,27 @@ class Win32Freeze(Command, WixMixIn): for x in QT_DLLS: shutil.copy2(os.path.join(QT_DIR, 'bin', x), self.dll_dir) - shutil.copy2(r'C:\windows\system32\python%s.dll'%self.py_ver, - self.dll_dir) - for dirpath, dirnames, filenames in os.walk(r'C:\Python%s\Lib'%self.py_ver): + shutil.copy2(os.path.join(self.python_base, 'python%s.dll'%self.py_ver), self.dll_dir) + for dirpath, dirnames, filenames in os.walk(r'%s\Lib'%self.python_base): if os.path.basename(dirpath) == 'pythonwin': continue for f in filenames: if f.lower().endswith('.dll'): f = self.j(dirpath, f) shutil.copy2(f, self.dll_dir) - shutil.copy2( - r'C:\Python%(v)s\Lib\site-packages\pywin32_system32\pywintypes%(v)s.dll' - % dict(v=self.py_ver), self.dll_dir) + self.add_plugins() def ignore_lib(root, items): ans = [] for x in items: ext = os.path.splitext(x)[1] - if (not ext and (x in ('demos', 'tests'))) or \ + if (not ext and (x in ('demos', 'tests', 'test'))) or \ (ext in ('.dll', '.chm', '.htm', '.txt')): ans.append(x) return ans - shutil.copytree(r'C:\Python%s\Lib'%self.py_ver, self.lib_dir, - ignore=ignore_lib) + self.info('Adding python...') + shutil.copytree(r'%s\Lib'%self.python_base, self.lib_dir, ignore=ignore_lib) # Fix win32com sp_dir = self.j(self.lib_dir, 'site-packages') @@ -242,13 +155,6 @@ class Win32Freeze(Command, WixMixIn): shutil.copytree(self.j(comext, 'shell'), self.j(sp_dir, 'win32com', 'shell')) shutil.rmtree(comext) - # Fix PyCrypto and Pillow, removing the bootstrap .py modules that load - # the .pyd modules, since they do not work when in a zip file - for folder in os.listdir(sp_dir): - folder = self.j(sp_dir, folder) - if os.path.isdir(folder): - self.fix_pyd_bootstraps_in(folder) - for pat in (r'PyQt5\uic\port_v3', ): x = glob.glob(self.j(self.lib_dir, 'site-packages', pat))[0] shutil.rmtree(x) @@ -259,11 +165,12 @@ class Win32Freeze(Command, WixMixIn): self.info('Adding calibre sources...') for x in glob.glob(self.j(self.SRC, '*')): if os.path.isdir(x): - shutil.copytree(x, self.j(sp_dir, self.b(x))) + if os.path.exists(os.path.join(x, '__init__.py')): + shutil.copytree(x, self.j(sp_dir, self.b(x))) else: shutil.copy(x, self.j(sp_dir, self.b(x))) - for x in (r'calibre\manual', r'calibre\trac', 'pythonwin'): + for x in (r'calibre\manual', r'calibre\plugins', 'pythonwin'): deld = self.j(sp_dir, x) if os.path.exists(deld): shutil.rmtree(deld) @@ -273,9 +180,13 @@ class Win32Freeze(Command, WixMixIn): if not f.endswith('.py'): os.remove(self.j(x[0], f)) + self.extract_pyd_modules(sp_dir) + self.info('Byte-compiling all python modules...') for x in ('test', 'lib2to3', 'distutils'): - shutil.rmtree(self.j(self.lib_dir, x)) + x = self.j(self.lib_dir, x) + if os.path.exists(x): + shutil.rmtree(x) for x in os.walk(self.lib_dir): root = x[0] for f in x[-1]: @@ -315,14 +226,8 @@ class Win32Freeze(Command, WixMixIn): self.info('\tAdding misc binary deps') bindir = os.path.join(SW, 'bin') - for x in ('pdftohtml', 'pdfinfo', 'pdftoppm', 'jpegtran-calibre', 'cjpeg-calibre'): + for x in ('pdftohtml', 'pdfinfo', 'pdftoppm', 'jpegtran-calibre', 'cjpeg-calibre', 'optipng-calibre'): shutil.copy2(os.path.join(bindir, x+'.exe'), self.base) - for x in ('', '.manifest'): - fname = 'optipng.exe' + x - src = os.path.join(bindir, fname) - shutil.copy2(src, self.base) - src = os.path.join(self.base, fname) - os.rename(src, src.replace('.exe', '-calibre.exe')) for pat in ('*.dll',): for f in glob.glob(os.path.join(bindir, pat)): ok = True @@ -355,6 +260,57 @@ class Win32Freeze(Command, WixMixIn): '-outputresource:%s;%d'%(dll,res)]) os.remove(manifest) + def extract_pyd_modules(self, site_packages_dir): + self.info('\nExtracting .pyd modules from site-packages...') + + def extract_pyd(path, root): + fullname = os.path.relpath(path, root).replace(os.sep, '/').replace('/', '.') + dest = os.path.join(self.dll_dir, fullname) + if os.path.exists(dest): + raise ValueError('Cannot extract %s into DLLs as it already exists' % fullname) + os.rename(path, dest) + bpy = dest[:-1] + if os.path.exists(bpy): + with open(bpy, 'rb') as f: + raw = f.read().strip() + if (not raw.startswith('def __bootstrap__') or not raw.endswith('__bootstrap__()')): + raise ValueError('The file %r has non bootstrap code'%bpy) + for ext in ('', 'c', 'o'): + try: + os.remove(bpy + ext) + except EnvironmentError as err: + if err.errno != errno.ENOENT: + raise + + def find_pyds(base): + for dirpath, dirnames, filenames in os.walk(base): + for fname in filenames: + if fname.lower().endswith('.pyd'): + yield os.path.join(dirpath, fname) + + def process_root(root, base=None): + for path in find_pyds(root): + extract_pyd(path, base or root) + + def absp(x): + return os.path.normcase(os.path.abspath(os.path.join(site_packages_dir, x))) + + roots = set() + for pth in glob.glob(os.path.join(site_packages_dir, '*.pth')): + for line in open(pth, 'rb').readlines(): + line = line.strip() + if line and not line.startswith('#') and os.path.exists(os.path.join(site_packages_dir, line)): + roots.add(absp(line)) + + for x in os.listdir(site_packages_dir): + x = absp(x) + if x in roots: + process_root(x) + elif os.path.isdir(x): + process_root(x, site_packages_dir) + elif x.lower().endswith('.pyd'): + extract_pyd(x, site_packages_dir) + def compress(self): self.info('Compressing app dir using 7-zip') subprocess.check_call([r'C:\Program Files\7-Zip\7z.exe', 'a', '-r', @@ -604,11 +560,11 @@ class Win32Freeze(Command, WixMixIn): dflags = (['/Zi'] if debug else []) dlflags = (['/DEBUG'] if debug else ['/INCREMENTAL:NO']) base = self.j(self.src_root, 'setup', 'installer', 'windows') - sources = [self.j(base, x) for x in ['util.c', 'MemoryModule.c']] - headers = [self.j(base, x) for x in ['util.h', 'MemoryModule.h']] + sources = [self.j(base, x) for x in ['util.c',]] + headers = [self.j(base, x) for x in ['util.h',]] objects = [self.j(self.obj_dir, self.b(x)+'.obj') for x in sources] cflags = '/c /EHsc /MD /W3 /Ox /nologo /D_UNICODE'.split() - cflags += ['/DPYDLL="python%s.dll"'%self.py_ver, '/IC:/Python%s/include'%self.py_ver] + cflags += ['/DPYDLL="python%s.dll"'%self.py_ver, '/I%s/include'%self.python_base] for src, obj in zip(sources, objects): if not self.newer(obj, headers+[src]): continue @@ -621,7 +577,7 @@ class Win32Freeze(Command, WixMixIn): cmd = [msvc.linker, '/DLL', '/VERSION:'+ver, '/OUT:'+dll, '/nologo', '/MACHINE:'+machine] + dlflags + objects + \ [self.embed_resources(dll), - '/LIBPATH:C:/Python%s/libs'%self.py_ver, + '/LIBPATH:%s/libs'%self.python_base, 'python%s.lib'%self.py_ver, '/delayload:python%s.dll'%self.py_ver] self.info('Linking calibre-launcher.dll') @@ -651,7 +607,7 @@ class Win32Freeze(Command, WixMixIn): self.info('Linking', bname) cmd = [msvc.linker] + ['/MACHINE:'+machine, '/LIBPATH:'+self.obj_dir, '/SUBSYSTEM:'+subsys, - '/LIBPATH:C:/Python%s/libs'%self.py_ver, '/RELEASE', + '/LIBPATH:%s/libs'%self.python_base, '/RELEASE', '/OUT:'+exe] + dlflags + [self.embed_resources(exe), dest, lib] self.run_builder(cmd) @@ -661,27 +617,6 @@ class Win32Freeze(Command, WixMixIn): self.zf_timestamp = time.localtime(time.time())[:6] self.zf_names = set() with zipfile.ZipFile(self.pylib, 'w', zipfile.ZIP_STORED) as zf: - # Add the .pyds from python and calibre to the zip file - for x in (self.plugins_dir, self.dll_dir): - for pyd in os.listdir(x): - if pyd.endswith('.pyd') and pyd not in { - # sqlite_custom has to be a file for - # sqlite_load_extension to work - 'sqlite_custom.pyd', - # calibre_style has to be loaded by Qt therefore it - # must be a file - 'calibre_style.pyd', - # Because of https://github.com/fancycode/MemoryModule/issues/4 - # any extensions that use C++ exceptions must be loaded - # from files - 'unrar.pyd', 'wpd.pyd', 'podofo.pyd', 'imageops.pyd', - 'progress_indicator.pyd', 'hunspell.pyd', - # dupypy crashes when loaded from the zip file - 'dukpy.pyd', - }: - self.add_to_zipfile(zf, pyd, x) - os.remove(self.j(x, pyd)) - # Add everything in Lib except site-packages to the zip file for x in os.listdir(self.lib_dir): if x == 'site-packages': @@ -690,12 +625,7 @@ class Win32Freeze(Command, WixMixIn): sp = self.j(self.lib_dir, 'site-packages') # Special handling for PIL and pywin32 - handled = set(['pywin32.pth', 'win32']) - pil_dir = glob.glob(self.j(sp, 'Pillow*', 'PIL'))[-1] - if is64bit: - # PIL can raise exceptions, which cause crashes on 64bit - shutil.copytree(pil_dir, self.j(self.dll_dir, 'PIL')) - handled.add(self.b(self.d(pil_dir))) + handled = {'pywin32.pth', 'win32'} base = self.j(sp, 'win32', 'lib') for x in os.listdir(base): if os.path.splitext(x)[1] not in ('.exe',): @@ -766,7 +696,7 @@ class Win32Freeze(Command, WixMixIn): if ext in ('.dll',): raise ValueError('Cannot add %r to zipfile'%abspath) zinfo.external_attr = 0o600 << 16 - if ext in ('.py', '.pyc', '.pyo', '.pyd'): + if ext in ('.py', '.pyc', '.pyo'): with open(abspath, 'rb') as f: zf.writestr(zinfo, f.read()) diff --git a/setup/installer/windows/site.py b/setup/installer/windows/site.py index aec3b41a90..693d7610ea 100644 --- a/setup/installer/windows/site.py +++ b/setup/installer/windows/site.py @@ -7,67 +7,39 @@ __docformat__ = 'restructuredtext en' import sys import os -import zipimport -import _memimporter +import imp -DEBUG_ZIPIMPORT = False +class PydImporter(object): -class ZipExtensionImporter(zipimport.zipimporter): - ''' - Taken, with thanks, from the py2exe source code - ''' + __slots__ = ('items', 'description') - def __init__(self, *args, **kwargs): - zipimport.zipimporter.__init__(self, *args, **kwargs) - # We know there are no dlls in the zip file, so dont set findproc - # (performance optimization) - #_memimporter.set_find_proc(self.locate_dll_image) + def __init__(self): + self.items = None + self.description = ('.pyd', 'rb', imp.C_EXTENSION) def find_module(self, fullname, path=None): - result = zipimport.zipimporter.find_module(self, fullname, path) - if result: - return result - fullname = fullname.replace(".", "\\") - if (fullname + '.pyd') in self._files: - return self - return None - - def locate_dll_image(self, name): - # A callback function for_memimporter.import_module. Tries to - # locate additional dlls. Returns the image as Python string, - # or None if not found. - if name in self._files: - return self.get_data(name) - return None + if self.items is None: + dlls_dir = os.path.join(sys.app_dir, 'DLLs') + items = self.items = {} + for x in os.listdir(dlls_dir): + lx = x.lower() + if lx.endswith(b'.pyd'): + items[lx[:-4]] = os.path.abspath(os.path.join(dlls_dir, x)) + return self if fullname.lower() in self.items else None def load_module(self, fullname): - if sys.modules.has_key(fullname): - mod = sys.modules[fullname] - if DEBUG_ZIPIMPORT: - sys.stderr.write("import %s # previously loaded from zipfile %s\n" % (fullname, self.archive)) - return mod + m = sys.modules.get(fullname) + if m is not None: + return m try: - return zipimport.zipimporter.load_module(self, fullname) - except zipimport.ZipImportError: - pass - initname = "init" + fullname.split(".")[-1] # name of initfunction - filename = fullname.replace(".", "\\") - path = filename + '.pyd' - if path in self._files: - if DEBUG_ZIPIMPORT: - sys.stderr.write("# found %s in zipfile %s\n" % (path, self.archive)) - code = self.get_data(path) - mod = _memimporter.import_module(code, initname, fullname, path) - mod.__file__ = "%s\\%s" % (self.archive, path) - mod.__loader__ = self - if DEBUG_ZIPIMPORT: - sys.stderr.write("import %s # loaded from zipfile %s\n" % (fullname, mod.__file__)) - return mod - raise zipimport.ZipImportError, "can't find module %s" % fullname - - def __repr__(self): - return "<%s object %r>" % (self.__class__.__name__, self.archive) - + path = self.items[fullname.lower()] + except KeyError: + raise ImportError('The native code module %s seems to have disappeared from self.items' % fullname) + package, name = fullname.rpartition(b'.')[::2] + m = imp.load_module(fullname, None, path, self.description) # This inserts the module into sys.modules itself + m.__loader__ = self + m.__package__ = package or None + return m def abs__file__(): """Set all module __file__ attribute to an absolute path""" @@ -92,16 +64,12 @@ def aliasmbcs(): def add_calibre_vars(): sys.resources_location = os.path.join(sys.app_dir, 'resources') - sys.extensions_location = os.path.join(sys.app_dir, 'plugins2') + sys.extensions_location = os.path.join(sys.app_dir, 'DLLs') dv = os.environ.get('CALIBRE_DEVELOP_FROM', None) if dv and os.path.exists(dv): sys.path.insert(0, os.path.abspath(dv)) -def makepath(*paths): - dir = os.path.abspath(os.path.join(*paths)) - return dir, os.path.normcase(dir) - def run_entry_point(): bname, mod, func = sys.calibre_basename, sys.calibre_module, sys.calibre_function sys.argv[0] = bname+'.exe' @@ -113,7 +81,7 @@ def main(): sys.setdefaultencoding('utf-8') aliasmbcs() - sys.path_hooks.insert(0, ZipExtensionImporter) + sys.meta_path.insert(0, PydImporter()) sys.path_importer_cache.clear() import linecache @@ -130,5 +98,3 @@ def main(): sys.path.append(os.path.join(sys.app_dir, 'DLLs')) return run_entry_point() - - diff --git a/setup/installer/windows/util.c b/setup/installer/windows/util.c index ce22d6b47b..225de8d720 100644 --- a/setup/installer/windows/util.c +++ b/setup/installer/windows/util.c @@ -1,6 +1,5 @@ /* * Copyright 2009 Kovid Goyal - * The memimporter code is taken from the py2exe project */ #include "util.h" @@ -18,102 +17,6 @@ char is_gui_app() { return GUI_APP; } int calibre_show_python_error(const wchar_t *preamble, int code); -// memimporter {{{ - -#include "MemoryModule.h" - -static char **DLL_Py_PackageContext = NULL; -static PyObject **DLL_ImportError = NULL; -static char module_doc[] = -"Importer which can load extension modules from memory"; - - -static void *memdup(void *ptr, Py_ssize_t size) -{ - void *p = malloc(size); - if (p == NULL) - return NULL; - memcpy(p, ptr, size); - return p; -} - -/* - Be sure to detect errors in FindLibrary - undetected errors lead to - very strange behaviour. -*/ -static void* FindLibrary(char *name, PyObject *callback) -{ - PyObject *result; - char *p; - Py_ssize_t size; - - if (callback == NULL) - return NULL; - result = PyObject_CallFunction(callback, "s", name); - if (result == NULL) { - PyErr_Clear(); - return NULL; - } - if (-1 == PyString_AsStringAndSize(result, &p, &size)) { - PyErr_Clear(); - Py_DECREF(result); - return NULL; - } - p = memdup(p, size); - Py_DECREF(result); - return p; -} - -static PyObject * -import_module(PyObject *self, PyObject *args) -{ - char *data; - int size; - char *initfuncname; - char *modname; - char *pathname; - HMEMORYMODULE hmem; - FARPROC do_init; - - char *oldcontext; - - /* code, initfuncname, fqmodulename, path */ - if (!PyArg_ParseTuple(args, "s#sss:import_module", - &data, &size, - &initfuncname, &modname, &pathname)) - return NULL; - hmem = MemoryLoadLibrary(data); - if (!hmem) { - PyErr_Format(*DLL_ImportError, - "MemoryLoadLibrary() failed loading %s", pathname); - return NULL; - } - do_init = MemoryGetProcAddress(hmem, initfuncname); - if (!do_init) { - MemoryFreeLibrary(hmem); - PyErr_Format(*DLL_ImportError, - "Could not find function %s in memory loaded pyd", initfuncname); - return NULL; - } - - oldcontext = *DLL_Py_PackageContext; - *DLL_Py_PackageContext = modname; - do_init(); - *DLL_Py_PackageContext = oldcontext; - if (PyErr_Occurred()) - return NULL; - /* Retrieve from sys.modules */ - return PyImport_ImportModule(modname); -} - -static PyMethodDef methods[] = { - { "import_module", import_module, METH_VARARGS, - "import_module(code, initfunc, dllname[, finder]) -> module" }, - { NULL, NULL }, /* Sentinel */ -}; - -// }}} - static int _show_error(const wchar_t *preamble, const wchar_t *msg, const int code) { wchar_t *buf; char *cbuf; @@ -324,11 +227,6 @@ void initialize_interpreter(const char *basename, const char *module, const char if (!flag) ExitProcess(_show_error(L"Failed to get debug flag", L"", 1)); //*flag = 1; - DLL_Py_PackageContext = (char**)GetProcAddress(dll, "_Py_PackageContext"); - if (!DLL_Py_PackageContext) ExitProcess(_show_error(L"Failed to load _Py_PackageContext from dll", L"", 1)); - DLL_ImportError = (PyObject**)GetProcAddress(dll, "PyExc_ImportError"); - if (!DLL_ImportError) ExitProcess(_show_error(L"Failed to load PyExc_ImportError from dll", L"", 1)); - Py_SetProgramName(program_name); Py_SetPythonHome(python_home); @@ -359,7 +257,6 @@ void initialize_interpreter(const char *basename, const char *module, const char } PySys_SetObject("argv", argv); - Py_InitModule3("_memimporter", methods, module_doc); } @@ -502,7 +399,6 @@ wchar_t* get_temp_filename(const wchar_t *prefix) { void redirect_out_stream(FILE *stream) { FILE *f = NULL; - wchar_t *temp_file; errno_t err; err = freopen_s(&f, "NUL", "wt", stream);