Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-09 03:04:10 -04:00
Code Issues Packages Projects Releases Wiki Activity

Initial stab at replacing use of MemoryModule

Browse Source
This commit is contained in:
Kovid Goyal 2015-12-06 18:19:03 +05:30
parent 923048a387
commit 77a0558cf2
No known key found for this signature in database
GPG Key ID: 06BC317B515ACE7C
5 changed files with 110 additions and 854 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

488
setup/installer/windows/MemoryModule.c
View File

@ -1,488 +0,0 @@
/*
* Memory DLL loading code
* Version 0.0.3
*
* Copyright (c) 2004-2012 by Joachim Bauch / mail@joachim-bauch.de
* http://www.joachim-bauch.de
*
* The contents of this file are subject to the Mozilla Public License Version
* 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
* http://www.mozilla.org/MPL/
*
* Software distributed under the License is distributed on an "AS IS" basis,
* WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
* for the specific language governing rights and limitations under the
* License.
*
* The Original Code is MemoryModule.c
*
* The Initial Developer of the Original Code is Joachim Bauch.
*
* Portions created by Joachim Bauch are Copyright (C) 2004-2012
* Joachim Bauch. All Rights Reserved.
*
*/
#ifndef __GNUC__
// disable warnings about pointer <-> DWORD conversions
#pragma warning( disable : 4311 4312 )
#endif
#ifdef _WIN64
#define POINTER_TYPE ULONGLONG
#else
#define POINTER_TYPE DWORD
#endif
#include <Windows.h>
#include <winnt.h>
#ifdef DEBUG_OUTPUT
#include <stdio.h>
#endif
#ifndef IMAGE_SIZEOF_BASE_RELOCATION
// Vista SDKs no longer define IMAGE_SIZEOF_BASE_RELOCATION!?
#define IMAGE_SIZEOF_BASE_RELOCATION (sizeof(IMAGE_BASE_RELOCATION))
#endif
#include "MemoryModule.h"
typedef struct {
PIMAGE_NT_HEADERS headers;
unsigned char *codeBase;
HMODULE *modules;
int numModules;
int initialized;
} MEMORYMODULE, *PMEMORYMODULE;
typedef BOOL (WINAPI *DllEntryProc)(HINSTANCE hinstDLL, DWORD fdwReason, LPVOID lpReserved);
#define GET_HEADER_DICTIONARY(module, idx) &(module)->headers->OptionalHeader.DataDirectory[idx]
#ifdef DEBUG_OUTPUT
static void
OutputLastError(const char *msg)
{
LPVOID tmp;
char *tmpmsg;
FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS,
NULL, GetLastError(), MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), (LPTSTR)&tmp, 0, NULL);
tmpmsg = (char *)LocalAlloc(LPTR, strlen(msg) + strlen(tmp) + 3);
sprintf(tmpmsg, "%s: %s", msg, tmp);
OutputDebugString(tmpmsg);
LocalFree(tmpmsg);
LocalFree(tmp);
}
#endif
static void
CopySections(const unsigned char *data, PIMAGE_NT_HEADERS old_headers, PMEMORYMODULE module)
{
int i, size;
unsigned char *codeBase = module->codeBase;
unsigned char *dest;
PIMAGE_SECTION_HEADER section = IMAGE_FIRST_SECTION(module->headers);
for (i=0; i<module->headers->FileHeader.NumberOfSections; i++, section++) {
if (section->SizeOfRawData == 0) {
// section doesn't contain data in the dll itself, but may define
// uninitialized data
size = old_headers->OptionalHeader.SectionAlignment;
if (size > 0) {
dest = (unsigned char *)VirtualAlloc(codeBase + section->VirtualAddress,
size,
MEM_COMMIT,
PAGE_READWRITE);
section->Misc.PhysicalAddress = (DWORD)dest;
memset(dest, 0, size);
}
// section is empty
continue;
}
// commit memory block and copy data from dll
dest = (unsigned char *)VirtualAlloc(codeBase + section->VirtualAddress,
section->SizeOfRawData,
MEM_COMMIT,
PAGE_READWRITE);
memcpy(dest, data + section->PointerToRawData, section->SizeOfRawData);
section->Misc.PhysicalAddress = (DWORD)dest;
}
}
// Protection flags for memory pages (Executable, Readable, Writeable)
static int ProtectionFlags[2][2][2] = {
{
// not executable
{PAGE_NOACCESS, PAGE_WRITECOPY},
{PAGE_READONLY, PAGE_READWRITE},
}, {
// executable
{PAGE_EXECUTE, PAGE_EXECUTE_WRITECOPY},
{PAGE_EXECUTE_READ, PAGE_EXECUTE_READWRITE},
},
};
static void
FinalizeSections(PMEMORYMODULE module)
{
int i;
PIMAGE_SECTION_HEADER section = IMAGE_FIRST_SECTION(module->headers);
#ifdef _WIN64
POINTER_TYPE imageOffset = (module->headers->OptionalHeader.ImageBase & 0xffffffff00000000);
#else
#define imageOffset 0
#endif
// loop through all sections and change access flags
for (i=0; i<module->headers->FileHeader.NumberOfSections; i++, section++) {
DWORD protect, oldProtect, size;
int executable = (section->Characteristics & IMAGE_SCN_MEM_EXECUTE) != 0;
int readable = (section->Characteristics & IMAGE_SCN_MEM_READ) != 0;
int writeable = (section->Characteristics & IMAGE_SCN_MEM_WRITE) != 0;
if (section->Characteristics & IMAGE_SCN_MEM_DISCARDABLE) {
// section is not needed any more and can safely be freed
VirtualFree((LPVOID)((POINTER_TYPE)section->Misc.PhysicalAddress | imageOffset), section->SizeOfRawData, MEM_DECOMMIT);
continue;
}
// determine protection flags based on characteristics
protect = ProtectionFlags[executable][readable][writeable];
if (section->Characteristics & IMAGE_SCN_MEM_NOT_CACHED) {
protect |= PAGE_NOCACHE;
}
// determine size of region
size = section->SizeOfRawData;
if (size == 0) {
if (section->Characteristics & IMAGE_SCN_CNT_INITIALIZED_DATA) {
size = module->headers->OptionalHeader.SizeOfInitializedData;
} else if (section->Characteristics & IMAGE_SCN_CNT_UNINITIALIZED_DATA) {
size = module->headers->OptionalHeader.SizeOfUninitializedData;
}
}
if (size > 0) {
// change memory access flags
if (VirtualProtect((LPVOID)((POINTER_TYPE)section->Misc.PhysicalAddress | imageOffset), size, protect, &oldProtect) == 0)
#ifdef DEBUG_OUTPUT
OutputLastError("Error protecting memory page")
#endif
;
}
}
#ifndef _WIN64
#undef imageOffset
#endif
}
static void
PerformBaseRelocation(PMEMORYMODULE module, SIZE_T delta)
{
DWORD i;
unsigned char *codeBase = module->codeBase;
PIMAGE_DATA_DIRECTORY directory = GET_HEADER_DICTIONARY(module, IMAGE_DIRECTORY_ENTRY_BASERELOC);
if (directory->Size > 0) {
PIMAGE_BASE_RELOCATION relocation = (PIMAGE_BASE_RELOCATION) (codeBase + directory->VirtualAddress);
for (; relocation->VirtualAddress > 0; ) {
unsigned char *dest = codeBase + relocation->VirtualAddress;
unsigned short *relInfo = (unsigned short *)((unsigned char *)relocation + IMAGE_SIZEOF_BASE_RELOCATION);
for (i=0; i<((relocation->SizeOfBlock-IMAGE_SIZEOF_BASE_RELOCATION) / 2); i++, relInfo++) {
DWORD *patchAddrHL;
#ifdef _WIN64
ULONGLONG *patchAddr64;
#endif
int type, offset;
// the upper 4 bits define the type of relocation
type = *relInfo >> 12;
// the lower 12 bits define the offset
offset = *relInfo & 0xfff;
switch (type)
{
case IMAGE_REL_BASED_ABSOLUTE:
// skip relocation
break;
case IMAGE_REL_BASED_HIGHLOW:
// change complete 32 bit address
patchAddrHL = (DWORD *) (dest + offset);
*patchAddrHL += (DWORD)delta;
break;
#ifdef _WIN64
case IMAGE_REL_BASED_DIR64:
patchAddr64 = (ULONGLONG *) (dest + offset);
*patchAddr64 += delta;
break;
#endif
default:
//printf("Unknown relocation: %d\n", type);
break;
}
}
// advance to next relocation block
relocation = (PIMAGE_BASE_RELOCATION) (((char *) relocation) + relocation->SizeOfBlock);
}
}
}
static int
BuildImportTable(PMEMORYMODULE module)
{
int result=1;
unsigned char *codeBase = module->codeBase;
PIMAGE_DATA_DIRECTORY directory = GET_HEADER_DICTIONARY(module, IMAGE_DIRECTORY_ENTRY_IMPORT);
if (directory->Size > 0) {
PIMAGE_IMPORT_DESCRIPTOR importDesc = (PIMAGE_IMPORT_DESCRIPTOR) (codeBase + directory->VirtualAddress);
for (; !IsBadReadPtr(importDesc, sizeof(IMAGE_IMPORT_DESCRIPTOR)) && importDesc->Name; importDesc++) {
POINTER_TYPE *thunkRef;
FARPROC *funcRef;
HMODULE handle = LoadLibrary((LPCSTR) (codeBase + importDesc->Name));
if (handle == NULL) {
#if DEBUG_OUTPUT
OutputLastError("Can't load library");
#endif
result = 0;
break;
}
module->modules = (HMODULE *)realloc(module->modules, (module->numModules+1)*(sizeof(HMODULE)));
if (module->modules == NULL) {
result = 0;
break;
}
module->modules[module->numModules++] = handle;
if (importDesc->OriginalFirstThunk) {
thunkRef = (POINTER_TYPE *) (codeBase + importDesc->OriginalFirstThunk);
funcRef = (FARPROC *) (codeBase + importDesc->FirstThunk);
} else {
// no hint table
thunkRef = (POINTER_TYPE *) (codeBase + importDesc->FirstThunk);
funcRef = (FARPROC *) (codeBase + importDesc->FirstThunk);
}
for (; *thunkRef; thunkRef++, funcRef++) {
if (IMAGE_SNAP_BY_ORDINAL(*thunkRef)) {
*funcRef = (FARPROC)GetProcAddress(handle, (LPCSTR)IMAGE_ORDINAL(*thunkRef));
} else {
PIMAGE_IMPORT_BY_NAME thunkData = (PIMAGE_IMPORT_BY_NAME) (codeBase + (*thunkRef));
*funcRef = (FARPROC)GetProcAddress(handle, (LPCSTR)&thunkData->Name);
}
if (*funcRef == 0) {
result = 0;
break;
}
}
if (!result) {
break;
}
}
}
return result;
}
HMEMORYMODULE MemoryLoadLibrary(const void *data)
{
PMEMORYMODULE result;
PIMAGE_DOS_HEADER dos_header;
PIMAGE_NT_HEADERS old_header;
unsigned char *code, *headers;
SIZE_T locationDelta;
DllEntryProc DllEntry;
BOOL successfull;
dos_header = (PIMAGE_DOS_HEADER)data;
if (dos_header->e_magic != IMAGE_DOS_SIGNATURE) {
#if DEBUG_OUTPUT
OutputDebugString("Not a valid executable file.\n");
#endif
return NULL;
}
old_header = (PIMAGE_NT_HEADERS)&((const unsigned char *)(data))[dos_header->e_lfanew];
if (old_header->Signature != IMAGE_NT_SIGNATURE) {
#if DEBUG_OUTPUT
OutputDebugString("No PE header found.\n");
#endif
return NULL;
}
// reserve memory for image of library
code = (unsigned char *)VirtualAlloc((LPVOID)(old_header->OptionalHeader.ImageBase),
old_header->OptionalHeader.SizeOfImage,
MEM_RESERVE,
PAGE_READWRITE);
if (code == NULL) {
// try to allocate memory at arbitrary position
code = (unsigned char *)VirtualAlloc(NULL,
old_header->OptionalHeader.SizeOfImage,
MEM_RESERVE,
PAGE_READWRITE);
if (code == NULL) {
#if DEBUG_OUTPUT
OutputLastError("Can't reserve memory");
#endif
return NULL;
}
}
result = (PMEMORYMODULE)HeapAlloc(GetProcessHeap(), 0, sizeof(MEMORYMODULE));
result->codeBase = code;
result->numModules = 0;
result->modules = NULL;
result->initialized = 0;
// XXX: is it correct to commit the complete memory region at once?
// calling DllEntry raises an exception if we don't...
VirtualAlloc(code,
old_header->OptionalHeader.SizeOfImage,
MEM_COMMIT,
PAGE_READWRITE);
// commit memory for headers
headers = (unsigned char *)VirtualAlloc(code,
old_header->OptionalHeader.SizeOfHeaders,
MEM_COMMIT,
PAGE_READWRITE);
// copy PE header to code
memcpy(headers, dos_header, dos_header->e_lfanew + old_header->OptionalHeader.SizeOfHeaders);
result->headers = (PIMAGE_NT_HEADERS)&((const unsigned char *)(headers))[dos_header->e_lfanew];
// update position
result->headers->OptionalHeader.ImageBase = (POINTER_TYPE)code;
// copy sections from DLL file block to new memory location
CopySections(data, old_header, result);
// adjust base address of imported data
locationDelta = (SIZE_T)(code - old_header->OptionalHeader.ImageBase);
if (locationDelta != 0) {
PerformBaseRelocation(result, locationDelta);
}
// load required dlls and adjust function table of imports
if (!BuildImportTable(result)) {
goto error;
}
// mark memory pages depending on section headers and release
// sections that are marked as "discardable"
FinalizeSections(result);
// get entry point of loaded library
if (result->headers->OptionalHeader.AddressOfEntryPoint != 0) {
DllEntry = (DllEntryProc) (code + result->headers->OptionalHeader.AddressOfEntryPoint);
if (DllEntry == 0) {
#if DEBUG_OUTPUT
OutputDebugString("Library has no entry point.\n");
#endif
goto error;
}
// notify library about attaching to process
successfull = (*DllEntry)((HINSTANCE)code, DLL_PROCESS_ATTACH, 0);
if (!successfull) {
#if DEBUG_OUTPUT
OutputDebugString("Can't attach library.\n");
#endif
goto error;
}
result->initialized = 1;
}
return (HMEMORYMODULE)result;
error:
// cleanup
MemoryFreeLibrary(result);
return NULL;
}
FARPROC MemoryGetProcAddress(HMEMORYMODULE module, const char *name)
{
unsigned char *codeBase = ((PMEMORYMODULE)module)->codeBase;
int idx=-1;
DWORD i, *nameRef;
WORD *ordinal;
PIMAGE_EXPORT_DIRECTORY exports;
PIMAGE_DATA_DIRECTORY directory = GET_HEADER_DICTIONARY((PMEMORYMODULE)module, IMAGE_DIRECTORY_ENTRY_EXPORT);
if (directory->Size == 0) {
// no export table found
return NULL;
}
exports = (PIMAGE_EXPORT_DIRECTORY) (codeBase + directory->VirtualAddress);
if (exports->NumberOfNames == 0 || exports->NumberOfFunctions == 0) {
// DLL doesn't export anything
return NULL;
}
// search function name in list of exported names
nameRef = (DWORD *) (codeBase + exports->AddressOfNames);
ordinal = (WORD *) (codeBase + exports->AddressOfNameOrdinals);
for (i=0; i<exports->NumberOfNames; i++, nameRef++, ordinal++) {
if (_stricmp(name, (const char *) (codeBase + (*nameRef))) == 0) {
idx = *ordinal;
break;
}
}
if (idx == -1) {
// exported symbol not found
return NULL;
}
if ((DWORD)idx > exports->NumberOfFunctions) {
// name <-> ordinal number don't match
return NULL;
}
// AddressOfFunctions contains the RVAs to the "real" functions
return (FARPROC) (codeBase + (*(DWORD *) (codeBase + exports->AddressOfFunctions + (idx*4))));
}
void MemoryFreeLibrary(HMEMORYMODULE mod)
{
int i;
PMEMORYMODULE module = (PMEMORYMODULE)mod;
if (module != NULL) {
if (module->initialized != 0) {
// notify library about detaching from process
DllEntryProc DllEntry = (DllEntryProc) (module->codeBase + module->headers->OptionalHeader.AddressOfEntryPoint);
(*DllEntry)((HINSTANCE)module->codeBase, DLL_PROCESS_DETACH, 0);
module->initialized = 0;
}
if (module->modules != NULL) {
// free previously opened libraries
for (i=0; i<module->numModules; i++) {
if (module->modules[i] != INVALID_HANDLE_VALUE) {
FreeLibrary(module->modules[i]);
}
}
free(module->modules);
}
if (module->codeBase != NULL) {
// release memory of library
VirtualFree(module->codeBase, 0, MEM_RELEASE);
}
HeapFree(GetProcessHeap(), 0, module);
}
}

48
setup/installer/windows/MemoryModule.h
View File

@ -1,48 +0,0 @@
/*
* Memory DLL loading code
* Version 0.0.3
*
* Copyright (c) 2004-2012 by Joachim Bauch / mail@joachim-bauch.de
* http://www.joachim-bauch.de
*
* The contents of this file are subject to the Mozilla Public License Version
* 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
* http://www.mozilla.org/MPL/
*
* Software distributed under the License is distributed on an "AS IS" basis,
* WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
* for the specific language governing rights and limitations under the
* License.
*
* The Original Code is MemoryModule.h
*
* The Initial Developer of the Original Code is Joachim Bauch.
*
* Portions created by Joachim Bauch are Copyright (C) 2004-2012
* Joachim Bauch. All Rights Reserved.
*
*/
#ifndef __MEMORY_MODULE_HEADER
#define __MEMORY_MODULE_HEADER
#include <Windows.h>
typedef void *HMEMORYMODULE;
#ifdef __cplusplus
extern "C" {
#endif
HMEMORYMODULE MemoryLoadLibrary(const void *);
FARPROC MemoryGetProcAddress(HMEMORYMODULE, const char *);
void MemoryFreeLibrary(HMEMORYMODULE);
#ifdef __cplusplus
}
#endif
#endif // __MEMORY_MODULE_HEADER

236
setup/installer/windows/freeze.py
View File

@ -6,8 +6,7 @@ __license__ = 'GPL v3'
__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
__docformat__ = 'restructuredtext en'
import sys, os, shutil, glob, py_compile, subprocess, re, zipfile, time, textwrap
from itertools import chain
import sys, os, shutil, glob, py_compile, subprocess, re, zipfile, time, textwrap, errno
from setup import (Command, modules, functions, basenames, __version__,
__appname__)
@ -16,8 +15,7 @@ from setup.build_environment import (
from setup.installer.windows.wix import WixMixIn
OPENSSL_DIR = os.environ.get('OPENSSL_DIR', os.path.join(SW, 'private', 'openssl'))
SW = r'C:\cygwin64\home\kovid\sw'
CRT = r'C:\Microsoft.VC90.CRT'
IMAGEMAGICK = os.path.join(SW, 'build', 'ImageMagick-*\\VisualMagick\\bin')
LZMA = os.path.join(SW, *('private/easylzma/build/easylzma-0.0.8'.split('/')))
QT_DIR = subprocess.check_output([QMAKE, '-query', 'QT_INSTALL_PREFIX']).decode('utf-8').strip()
@ -50,51 +48,6 @@ def walk(dir):
for f in record[-1]:
yield os.path.join(record[0], f)
# Remove CRT dep from manifests {{{
def get_manifest_from_dll(dll):
import win32api, pywintypes
LOAD_LIBRARY_AS_DATAFILE = 2
d = win32api.LoadLibraryEx(os.path.abspath(dll), 0, LOAD_LIBRARY_AS_DATAFILE)
try:
resources = win32api.EnumResourceNames(d, 24)
except pywintypes.error as err:
if err.winerror == 1812:
return None, None # no resource section (probably a .pyd file)
raise
if resources:
return resources[0], win32api.LoadResource(d, 24, resources[0])
return None, None
def update_manifest(dll, rnum, manifest):
import win32api
h = win32api.BeginUpdateResource(dll, 0)
win32api.UpdateResource(h, 24, rnum, manifest)
win32api.EndUpdateResource(h, 0)
_crt_pat = re.compile(r'Microsoft\.VC\d+\.CRT')
def remove_CRT_from_manifest(dll, log=print):
from lxml import etree
rnum, manifest = get_manifest_from_dll(dll)
if manifest is None:
return
root = etree.fromstring(manifest)
found = False
for ai in root.xpath('//*[local-name()="assemblyIdentity" and @name]'):
name = ai.get('name')
if _crt_pat.match(name):
p = ai.getparent()
pp = p.getparent()
pp.remove(p)
if len(pp) == 0:
pp.getparent().remove(pp)
found = True
if found:
manifest = etree.tostring(root, pretty_print=True)
update_manifest(dll, rnum, manifest)
log('\t', os.path.basename(dll))
# }}}
class Win32Freeze(Command, WixMixIn):
description = 'Freeze windows calibre installation'
@ -114,7 +67,7 @@ class Win32Freeze(Command, WixMixIn):
action='store_true', help='Dont strip the generated binaries (no-op on windows)')
def run(self, opts):
self.SW = SW
self.python_base = os.path.join(SW, 'private', 'python')
self.portable_uncompressed_size = 0
self.opts = opts
self.src_root = self.d(self.SRC)
@ -124,36 +77,21 @@ class Win32Freeze(Command, WixMixIn):
self.lib_dir = self.j(self.base, 'Lib')
self.pylib = self.j(self.base, 'pylib.zip')
self.dll_dir = self.j(self.base, 'DLLs')
self.plugins_dir = os.path.join(self.base, 'plugins2')
self.portable_base = self.j(self.d(self.base), 'Calibre Portable')
self.obj_dir = self.j(self.src_root, 'build', 'launcher')
self.initbase()
self.build_launchers()
self.build_utils()
self.add_plugins()
self.freeze()
self.embed_manifests()
self.install_site_py()
self.archive_lib_dir()
self.remove_CRT_from_manifests()
self.create_installer()
# self.create_installer()
if not is64bit:
self.build_portable()
self.build_portable_installer()
self.sign_installers()
def remove_CRT_from_manifests(self):
'''
The dependency on the CRT is removed from the manifests of all DLLs.
This allows the CRT loaded by the .exe files to be used instead.
'''
self.info('Removing CRT dependency from manifests of:')
for dll in chain(walk(self.dll_dir), walk(self.plugins_dir)):
bn = self.b(dll)
if bn.lower().rpartition('.')[-1] not in {'dll', 'pyd'}:
continue
remove_CRT_from_manifest(dll, self.info)
# self.sign_installers()
def initbase(self):
if self.e(self.base):
@ -162,38 +100,16 @@ class Win32Freeze(Command, WixMixIn):
def add_plugins(self):
self.info('Adding plugins...')
tgt = self.plugins_dir
if os.path.exists(tgt):
shutil.rmtree(tgt)
os.mkdir(tgt)
tgt = self.dll_dir
base = self.j(self.SRC, 'calibre', 'plugins')
for f in glob.glob(self.j(base, '*.pyd')):
# We dont want the manifests as the manifest in the exe will be
# used instead
shutil.copy2(f, tgt)
def fix_pyd_bootstraps_in(self, folder):
for dirpath, dirnames, filenames in os.walk(folder):
for f in filenames:
name, ext = os.path.splitext(f)
bpy = self.j(dirpath, name + '.py')
if ext == '.pyd' and os.path.exists(bpy):
with open(bpy, 'rb') as f:
raw = f.read().strip()
if (not raw.startswith('def __bootstrap__') or not
raw.endswith('__bootstrap__()')):
raise Exception('The file %r has non'
' bootstrap code'%self.j(dirpath, f))
for ext in ('.py', '.pyc', '.pyo'):
x = self.j(dirpath, name+ext)
if os.path.exists(x):
os.remove(x)
def freeze(self):
shutil.copy2(self.j(self.src_root, 'LICENSE'), self.base)
self.info('Adding CRT')
shutil.copytree(CRT, self.j(self.base, os.path.basename(CRT)))
# self.info('Adding CRT')
# shutil.copytree(CRT, self.j(self.base, os.path.basename(CRT)))
self.info('Adding resources...')
tgt = self.j(self.base, 'resources')
@ -201,8 +117,8 @@ class Win32Freeze(Command, WixMixIn):
shutil.rmtree(tgt)
shutil.copytree(self.j(self.src_root, 'resources'), tgt)
self.info('Adding Qt and python...')
shutil.copytree(r'C:\Python%s\DLLs'%self.py_ver, self.dll_dir,
self.info('Adding Qt...')
shutil.copytree(os.path.join(self.python_base, 'DLLs') , self.dll_dir,
ignore=shutil.ignore_patterns('msvc*.dll', 'Microsoft.*'))
for x in glob.glob(self.j(OPENSSL_DIR, 'bin', '*.dll')):
shutil.copy2(x, self.dll_dir)
@ -211,30 +127,27 @@ class Win32Freeze(Command, WixMixIn):
for x in QT_DLLS:
shutil.copy2(os.path.join(QT_DIR, 'bin', x), self.dll_dir)
shutil.copy2(r'C:\windows\system32\python%s.dll'%self.py_ver,
self.dll_dir)
for dirpath, dirnames, filenames in os.walk(r'C:\Python%s\Lib'%self.py_ver):
shutil.copy2(os.path.join(self.python_base, 'python%s.dll'%self.py_ver), self.dll_dir)
for dirpath, dirnames, filenames in os.walk(r'%s\Lib'%self.python_base):
if os.path.basename(dirpath) == 'pythonwin':
continue
for f in filenames:
if f.lower().endswith('.dll'):
f = self.j(dirpath, f)
shutil.copy2(f, self.dll_dir)
shutil.copy2(
r'C:\Python%(v)s\Lib\site-packages\pywin32_system32\pywintypes%(v)s.dll'
% dict(v=self.py_ver), self.dll_dir)
self.add_plugins()
def ignore_lib(root, items):
ans = []
for x in items:
ext = os.path.splitext(x)[1]
if (not ext and (x in ('demos', 'tests'))) or \
if (not ext and (x in ('demos', 'tests', 'test'))) or \
(ext in ('.dll', '.chm', '.htm', '.txt')):
ans.append(x)
return ans
shutil.copytree(r'C:\Python%s\Lib'%self.py_ver, self.lib_dir,
ignore=ignore_lib)
self.info('Adding python...')
shutil.copytree(r'%s\Lib'%self.python_base, self.lib_dir, ignore=ignore_lib)
# Fix win32com
sp_dir = self.j(self.lib_dir, 'site-packages')
@ -242,13 +155,6 @@ class Win32Freeze(Command, WixMixIn):
shutil.copytree(self.j(comext, 'shell'), self.j(sp_dir, 'win32com', 'shell'))
shutil.rmtree(comext)
# Fix PyCrypto and Pillow, removing the bootstrap .py modules that load
# the .pyd modules, since they do not work when in a zip file
for folder in os.listdir(sp_dir):
folder = self.j(sp_dir, folder)
if os.path.isdir(folder):
self.fix_pyd_bootstraps_in(folder)
for pat in (r'PyQt5\uic\port_v3', ):
x = glob.glob(self.j(self.lib_dir, 'site-packages', pat))[0]
shutil.rmtree(x)
@ -259,11 +165,12 @@ class Win32Freeze(Command, WixMixIn):
self.info('Adding calibre sources...')
for x in glob.glob(self.j(self.SRC, '*')):
if os.path.isdir(x):
shutil.copytree(x, self.j(sp_dir, self.b(x)))
if os.path.exists(os.path.join(x, '__init__.py')):
shutil.copytree(x, self.j(sp_dir, self.b(x)))
else:
shutil.copy(x, self.j(sp_dir, self.b(x)))
for x in (r'calibre\manual', r'calibre\trac', 'pythonwin'):
for x in (r'calibre\manual', r'calibre\plugins', 'pythonwin'):
deld = self.j(sp_dir, x)
if os.path.exists(deld):
shutil.rmtree(deld)
@ -273,9 +180,13 @@ class Win32Freeze(Command, WixMixIn):
if not f.endswith('.py'):
os.remove(self.j(x[0], f))
self.extract_pyd_modules(sp_dir)
self.info('Byte-compiling all python modules...')
for x in ('test', 'lib2to3', 'distutils'):
shutil.rmtree(self.j(self.lib_dir, x))
x = self.j(self.lib_dir, x)
if os.path.exists(x):
shutil.rmtree(x)
for x in os.walk(self.lib_dir):
root = x[0]
for f in x[-1]:
@ -315,14 +226,8 @@ class Win32Freeze(Command, WixMixIn):
self.info('\tAdding misc binary deps')
bindir = os.path.join(SW, 'bin')
for x in ('pdftohtml', 'pdfinfo', 'pdftoppm', 'jpegtran-calibre', 'cjpeg-calibre'):
for x in ('pdftohtml', 'pdfinfo', 'pdftoppm', 'jpegtran-calibre', 'cjpeg-calibre', 'optipng-calibre'):
shutil.copy2(os.path.join(bindir, x+'.exe'), self.base)
for x in ('', '.manifest'):
fname = 'optipng.exe' + x
src = os.path.join(bindir, fname)
shutil.copy2(src, self.base)
src = os.path.join(self.base, fname)
os.rename(src, src.replace('.exe', '-calibre.exe'))
for pat in ('*.dll',):
for f in glob.glob(os.path.join(bindir, pat)):
ok = True
@ -355,6 +260,57 @@ class Win32Freeze(Command, WixMixIn):
'-outputresource:%s;%d'%(dll,res)])
os.remove(manifest)
def extract_pyd_modules(self, site_packages_dir):
self.info('\nExtracting .pyd modules from site-packages...')
def extract_pyd(path, root):
fullname = os.path.relpath(path, root).replace(os.sep, '/').replace('/', '.')
dest = os.path.join(self.dll_dir, fullname)
if os.path.exists(dest):
raise ValueError('Cannot extract %s into DLLs as it already exists' % fullname)
os.rename(path, dest)
bpy = dest[:-1]
if os.path.exists(bpy):
with open(bpy, 'rb') as f:
raw = f.read().strip()
if (not raw.startswith('def __bootstrap__') or not raw.endswith('__bootstrap__()')):
raise ValueError('The file %r has non bootstrap code'%bpy)
for ext in ('', 'c', 'o'):
try:
os.remove(bpy + ext)
except EnvironmentError as err:
if err.errno != errno.ENOENT:
raise
def find_pyds(base):
for dirpath, dirnames, filenames in os.walk(base):
for fname in filenames:
if fname.lower().endswith('.pyd'):
yield os.path.join(dirpath, fname)
def process_root(root, base=None):
for path in find_pyds(root):
extract_pyd(path, base or root)
def absp(x):
return os.path.normcase(os.path.abspath(os.path.join(site_packages_dir, x)))
roots = set()
for pth in glob.glob(os.path.join(site_packages_dir, '*.pth')):
for line in open(pth, 'rb').readlines():
line = line.strip()
if line and not line.startswith('#') and os.path.exists(os.path.join(site_packages_dir, line)):
roots.add(absp(line))
for x in os.listdir(site_packages_dir):
x = absp(x)
if x in roots:
process_root(x)
elif os.path.isdir(x):
process_root(x, site_packages_dir)
elif x.lower().endswith('.pyd'):
extract_pyd(x, site_packages_dir)
def compress(self):
self.info('Compressing app dir using 7-zip')
subprocess.check_call([r'C:\Program Files\7-Zip\7z.exe', 'a', '-r',
@ -604,11 +560,11 @@ class Win32Freeze(Command, WixMixIn):
dflags = (['/Zi'] if debug else [])
dlflags = (['/DEBUG'] if debug else ['/INCREMENTAL:NO'])
base = self.j(self.src_root, 'setup', 'installer', 'windows')
sources = [self.j(base, x) for x in ['util.c', 'MemoryModule.c']]
headers = [self.j(base, x) for x in ['util.h', 'MemoryModule.h']]
sources = [self.j(base, x) for x in ['util.c',]]
headers = [self.j(base, x) for x in ['util.h',]]
objects = [self.j(self.obj_dir, self.b(x)+'.obj') for x in sources]
cflags = '/c /EHsc /MD /W3 /Ox /nologo /D_UNICODE'.split()
cflags += ['/DPYDLL="python%s.dll"'%self.py_ver, '/IC:/Python%s/include'%self.py_ver]
cflags += ['/DPYDLL="python%s.dll"'%self.py_ver, '/I%s/include'%self.python_base]
for src, obj in zip(sources, objects):
if not self.newer(obj, headers+[src]):
continue
@ -621,7 +577,7 @@ class Win32Freeze(Command, WixMixIn):
cmd = [msvc.linker, '/DLL', '/VERSION:'+ver, '/OUT:'+dll,
'/nologo', '/MACHINE:'+machine] + dlflags + objects + \
[self.embed_resources(dll),
'/LIBPATH:C:/Python%s/libs'%self.py_ver,
'/LIBPATH:%s/libs'%self.python_base,
'python%s.lib'%self.py_ver,
'/delayload:python%s.dll'%self.py_ver]
self.info('Linking calibre-launcher.dll')
@ -651,7 +607,7 @@ class Win32Freeze(Command, WixMixIn):
self.info('Linking', bname)
cmd = [msvc.linker] + ['/MACHINE:'+machine,
'/LIBPATH:'+self.obj_dir, '/SUBSYSTEM:'+subsys,
'/LIBPATH:C:/Python%s/libs'%self.py_ver, '/RELEASE',
'/LIBPATH:%s/libs'%self.python_base, '/RELEASE',
'/OUT:'+exe] + dlflags + [self.embed_resources(exe),
dest, lib]
self.run_builder(cmd)
@ -661,27 +617,6 @@ class Win32Freeze(Command, WixMixIn):
self.zf_timestamp = time.localtime(time.time())[:6]
self.zf_names = set()
with zipfile.ZipFile(self.pylib, 'w', zipfile.ZIP_STORED) as zf:
# Add the .pyds from python and calibre to the zip file
for x in (self.plugins_dir, self.dll_dir):
for pyd in os.listdir(x):
if pyd.endswith('.pyd') and pyd not in {
# sqlite_custom has to be a file for
# sqlite_load_extension to work
'sqlite_custom.pyd',
# calibre_style has to be loaded by Qt therefore it
# must be a file
'calibre_style.pyd',
# Because of https://github.com/fancycode/MemoryModule/issues/4
# any extensions that use C++ exceptions must be loaded
# from files
'unrar.pyd', 'wpd.pyd', 'podofo.pyd', 'imageops.pyd',
'progress_indicator.pyd', 'hunspell.pyd',
# dupypy crashes when loaded from the zip file
'dukpy.pyd',
}:
self.add_to_zipfile(zf, pyd, x)
os.remove(self.j(x, pyd))
# Add everything in Lib except site-packages to the zip file
for x in os.listdir(self.lib_dir):
if x == 'site-packages':
@ -690,12 +625,7 @@ class Win32Freeze(Command, WixMixIn):
sp = self.j(self.lib_dir, 'site-packages')
# Special handling for PIL and pywin32
handled = set(['pywin32.pth', 'win32'])
pil_dir = glob.glob(self.j(sp, 'Pillow*', 'PIL'))[-1]
if is64bit:
# PIL can raise exceptions, which cause crashes on 64bit
shutil.copytree(pil_dir, self.j(self.dll_dir, 'PIL'))
handled.add(self.b(self.d(pil_dir)))
handled = {'pywin32.pth', 'win32'}
base = self.j(sp, 'win32', 'lib')
for x in os.listdir(base):
if os.path.splitext(x)[1] not in ('.exe',):
@ -766,7 +696,7 @@ class Win32Freeze(Command, WixMixIn):
if ext in ('.dll',):
raise ValueError('Cannot add %r to zipfile'%abspath)
zinfo.external_attr = 0o600 << 16
if ext in ('.py', '.pyc', '.pyo', '.pyd'):
if ext in ('.py', '.pyc', '.pyo'):
with open(abspath, 'rb') as f:
zf.writestr(zinfo, f.read())

88
setup/installer/windows/site.py
View File

@ -7,67 +7,39 @@ __docformat__ = 'restructuredtext en'
import sys
import os
import zipimport
import _memimporter
import imp
DEBUG_ZIPIMPORT = False
class PydImporter(object):
class ZipExtensionImporter(zipimport.zipimporter):
'''
Taken, with thanks, from the py2exe source code
'''
__slots__ = ('items', 'description')
def __init__(self, *args, **kwargs):
zipimport.zipimporter.__init__(self, *args, **kwargs)
# We know there are no dlls in the zip file, so dont set findproc
# (performance optimization)
#_memimporter.set_find_proc(self.locate_dll_image)
def __init__(self):
self.items = None
self.description = ('.pyd', 'rb', imp.C_EXTENSION)
def find_module(self, fullname, path=None):
result = zipimport.zipimporter.find_module(self, fullname, path)
if result:
return result
fullname = fullname.replace(".", "\\")
if (fullname + '.pyd') in self._files:
return self
return None
def locate_dll_image(self, name):
# A callback function for_memimporter.import_module. Tries to
# locate additional dlls. Returns the image as Python string,
# or None if not found.
if name in self._files:
return self.get_data(name)
return None
if self.items is None:
dlls_dir = os.path.join(sys.app_dir, 'DLLs')
items = self.items = {}
for x in os.listdir(dlls_dir):
lx = x.lower()
if lx.endswith(b'.pyd'):
items[lx[:-4]] = os.path.abspath(os.path.join(dlls_dir, x))
return self if fullname.lower() in self.items else None
def load_module(self, fullname):
if sys.modules.has_key(fullname):
mod = sys.modules[fullname]
if DEBUG_ZIPIMPORT:
sys.stderr.write("import %s # previously loaded from zipfile %s\n" % (fullname, self.archive))
return mod
m = sys.modules.get(fullname)
if m is not None:
return m
try:
return zipimport.zipimporter.load_module(self, fullname)
except zipimport.ZipImportError:
pass
initname = "init" + fullname.split(".")[-1] # name of initfunction
filename = fullname.replace(".", "\\")
path = filename + '.pyd'
if path in self._files:
if DEBUG_ZIPIMPORT:
sys.stderr.write("# found %s in zipfile %s\n" % (path, self.archive))
code = self.get_data(path)
mod = _memimporter.import_module(code, initname, fullname, path)
mod.__file__ = "%s\\%s" % (self.archive, path)
mod.__loader__ = self
if DEBUG_ZIPIMPORT:
sys.stderr.write("import %s # loaded from zipfile %s\n" % (fullname, mod.__file__))
return mod
raise zipimport.ZipImportError, "can't find module %s" % fullname
def __repr__(self):
return "<%s object %r>" % (self.__class__.__name__, self.archive)
path = self.items[fullname.lower()]
except KeyError:
raise ImportError('The native code module %s seems to have disappeared from self.items' % fullname)
package, name = fullname.rpartition(b'.')[::2]
m = imp.load_module(fullname, None, path, self.description) # This inserts the module into sys.modules itself
m.__loader__ = self
m.__package__ = package or None
return m
def abs__file__():
"""Set all module __file__ attribute to an absolute path"""
@ -92,16 +64,12 @@ def aliasmbcs():
def add_calibre_vars():
sys.resources_location = os.path.join(sys.app_dir, 'resources')
sys.extensions_location = os.path.join(sys.app_dir, 'plugins2')
sys.extensions_location = os.path.join(sys.app_dir, 'DLLs')
dv = os.environ.get('CALIBRE_DEVELOP_FROM', None)
if dv and os.path.exists(dv):
sys.path.insert(0, os.path.abspath(dv))
def makepath(*paths):
dir = os.path.abspath(os.path.join(*paths))
return dir, os.path.normcase(dir)
def run_entry_point():
bname, mod, func = sys.calibre_basename, sys.calibre_module, sys.calibre_function
sys.argv[0] = bname+'.exe'
@ -113,7 +81,7 @@ def main():
sys.setdefaultencoding('utf-8')
aliasmbcs()
sys.path_hooks.insert(0, ZipExtensionImporter)
sys.meta_path.insert(0, PydImporter())
sys.path_importer_cache.clear()
import linecache
@ -130,5 +98,3 @@ def main():
sys.path.append(os.path.join(sys.app_dir, 'DLLs'))
return run_entry_point()

104
setup/installer/windows/util.c
View File

@ -1,6 +1,5 @@
/*
* Copyright 2009 Kovid Goyal
* The memimporter code is taken from the py2exe project
*/
#include "util.h"
@ -18,102 +17,6 @@ char is_gui_app() { return GUI_APP; }
int calibre_show_python_error(const wchar_t *preamble, int code);
// memimporter {{{
#include "MemoryModule.h"
static char **DLL_Py_PackageContext = NULL;
static PyObject **DLL_ImportError = NULL;
static char module_doc[] =
"Importer which can load extension modules from memory";
static void *memdup(void *ptr, Py_ssize_t size)
{
void *p = malloc(size);
if (p == NULL)
return NULL;
memcpy(p, ptr, size);
return p;
}
/*
Be sure to detect errors in FindLibrary - undetected errors lead to
very strange behaviour.
*/
static void* FindLibrary(char *name, PyObject *callback)
{
PyObject *result;
char *p;
Py_ssize_t size;
if (callback == NULL)
return NULL;
result = PyObject_CallFunction(callback, "s", name);
if (result == NULL) {
PyErr_Clear();
return NULL;
}
if (-1 == PyString_AsStringAndSize(result, &p, &size)) {
PyErr_Clear();
Py_DECREF(result);
return NULL;
}
p = memdup(p, size);
Py_DECREF(result);
return p;
}
static PyObject *
import_module(PyObject *self, PyObject *args)
{
char *data;
int size;
char *initfuncname;
char *modname;
char *pathname;
HMEMORYMODULE hmem;
FARPROC do_init;
char *oldcontext;
/* code, initfuncname, fqmodulename, path */
if (!PyArg_ParseTuple(args, "s#sss:import_module",
&data, &size,
&initfuncname, &modname, &pathname))
return NULL;
hmem = MemoryLoadLibrary(data);
if (!hmem) {
PyErr_Format(*DLL_ImportError,
"MemoryLoadLibrary() failed loading %s", pathname);
return NULL;
}
do_init = MemoryGetProcAddress(hmem, initfuncname);
if (!do_init) {
MemoryFreeLibrary(hmem);
PyErr_Format(*DLL_ImportError,
"Could not find function %s in memory loaded pyd", initfuncname);
return NULL;
}
oldcontext = *DLL_Py_PackageContext;
*DLL_Py_PackageContext = modname;
do_init();
*DLL_Py_PackageContext = oldcontext;
if (PyErr_Occurred())
return NULL;
/* Retrieve from sys.modules */
return PyImport_ImportModule(modname);
}
static PyMethodDef methods[] = {
{ "import_module", import_module, METH_VARARGS,
"import_module(code, initfunc, dllname[, finder]) -> module" },
{ NULL, NULL }, /* Sentinel */
};
// }}}
static int _show_error(const wchar_t *preamble, const wchar_t *msg, const int code) {
wchar_t *buf;
char *cbuf;
@ -324,11 +227,6 @@ void initialize_interpreter(const char *basename, const char *module, const char
if (!flag) ExitProcess(_show_error(L"Failed to get debug flag", L"", 1));
//*flag = 1;
DLL_Py_PackageContext = (char**)GetProcAddress(dll, "_Py_PackageContext");
if (!DLL_Py_PackageContext) ExitProcess(_show_error(L"Failed to load _Py_PackageContext from dll", L"", 1));
DLL_ImportError = (PyObject**)GetProcAddress(dll, "PyExc_ImportError");
if (!DLL_ImportError) ExitProcess(_show_error(L"Failed to load PyExc_ImportError from dll", L"", 1));
Py_SetProgramName(program_name);
Py_SetPythonHome(python_home);
@ -359,7 +257,6 @@ void initialize_interpreter(const char *basename, const char *module, const char
}
PySys_SetObject("argv", argv);
Py_InitModule3("_memimporter", methods, module_doc);
}
@ -502,7 +399,6 @@ wchar_t* get_temp_filename(const wchar_t *prefix) {
void redirect_out_stream(FILE *stream) {
FILE *f = NULL;
wchar_t *temp_file;
errno_t err;
err = freopen_s(&f, "NUL", "wt", stream);