Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-09 03:04:10 -04:00
Code Issues Packages Projects Releases Wiki Activity

Windows build: Add code to load .pyd python extensions from a zip file. This allows many more files in the calibre installation to be zipped up, speeding up the installer.

Browse Source
This commit is contained in:
Kovid Goyal 2011-06-05 20:21:09 -06:00
parent 9ea27629c2
commit b22d5ac5fb
8 changed files with 1089 additions and 97 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

689
setup/installer/windows/MemoryModule.c Normal file
View File

@ -0,0 +1,689 @@
/*
* Memory DLL loading code
* Version 0.0.2 with additions from Thomas Heller
*
* Copyright (c) 2004-2005 by Joachim Bauch / mail@joachim-bauch.de
* http://www.joachim-bauch.de
*
* The contents of this file are subject to the Mozilla Public License Version
* 1.1 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
* http://www.mozilla.org/MPL/
*
* Software distributed under the License is distributed on an "AS IS" basis,
* WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
* for the specific language governing rights and limitations under the
* License.
*
* The Original Code is MemoryModule.c
*
* The Initial Developer of the Original Code is Joachim Bauch.
*
* Portions created by Joachim Bauch are Copyright (C) 2004-2005
* Joachim Bauch. All Rights Reserved.
*
* Portions Copyright (C) 2005 Thomas Heller.
*
*/
// disable warnings about pointer <-> DWORD conversions
#pragma warning( disable : 4311 4312 )
#include <Windows.h>
#include <winnt.h>
#if DEBUG_OUTPUT
#include <stdio.h>
#endif
#ifndef IMAGE_SIZEOF_BASE_RELOCATION
// Vista SDKs no longer define IMAGE_SIZEOF_BASE_RELOCATION!?
# define IMAGE_SIZEOF_BASE_RELOCATION (sizeof(IMAGE_BASE_RELOCATION))
#endif
#include "MemoryModule.h"
/*
XXX We need to protect at least walking the 'loaded' linked list with a lock!
*/
/******************************************************************/
FINDPROC findproc;
void *findproc_data = NULL;
struct NAME_TABLE {
char *name;
DWORD ordinal;
};
typedef struct tagMEMORYMODULE {
PIMAGE_NT_HEADERS headers;
unsigned char *codeBase;
HMODULE *modules;
int numModules;
int initialized;
struct NAME_TABLE *name_table;
char *name;
int refcount;
struct tagMEMORYMODULE *next, *prev;
} MEMORYMODULE, *PMEMORYMODULE;
typedef BOOL (WINAPI *DllEntryProc)(HINSTANCE hinstDLL, DWORD fdwReason, LPVOID lpReserved);
#define GET_HEADER_DICTIONARY(module, idx) &(module)->headers->OptionalHeader.DataDirectory[idx]
MEMORYMODULE *loaded; /* linked list of loaded memory modules */
/* private - insert a loaded library in a linked list */
static void _Register(char *name, MEMORYMODULE *module)
{
module->next = loaded;
if (loaded)
loaded->prev = module;
module->prev = NULL;
loaded = module;
}
/* private - remove a loaded library from a linked list */
static void _Unregister(MEMORYMODULE *module)
{
free(module->name);
if (module->prev)
module->prev->next = module->next;
if (module->next)
module->next->prev = module->prev;
if (module == loaded)
loaded = module->next;
}
/* public - replacement for GetModuleHandle() */
HMODULE MyGetModuleHandle(LPCTSTR lpModuleName)
{
MEMORYMODULE *p = loaded;
while (p) {
// If already loaded, only increment the reference count
if (0 == stricmp(lpModuleName, p->name)) {
return (HMODULE)p;
}
p = p->next;
}
return GetModuleHandle(lpModuleName);
}
/* public - replacement for LoadLibrary, but searches FIRST for memory
libraries, then for normal libraries. So, it will load libraries AS memory
module if they are found by findproc().
*/
HMODULE MyLoadLibrary(char *lpFileName)
{
MEMORYMODULE *p = loaded;
HMODULE hMod;
while (p) {
// If already loaded, only increment the reference count
if (0 == stricmp(lpFileName, p->name)) {
p->refcount++;
return (HMODULE)p;
}
p = p->next;
}
if (findproc && findproc_data) {
void *pdata = findproc(lpFileName, findproc_data);
if (pdata) {
hMod = MemoryLoadLibrary(lpFileName, pdata);
free(p);
return hMod;
}
}
hMod = LoadLibrary(lpFileName);
return hMod;
}
/* public - replacement for GetProcAddress() */
FARPROC MyGetProcAddress(HMODULE hModule, LPCSTR lpProcName)
{
MEMORYMODULE *p = loaded;
while (p) {
if ((HMODULE)p == hModule)
return MemoryGetProcAddress(p, lpProcName);
p = p->next;
}
return GetProcAddress(hModule, lpProcName);
}
/* public - replacement for FreeLibrary() */
BOOL MyFreeLibrary(HMODULE hModule)
{
MEMORYMODULE *p = loaded;
while (p) {
if ((HMODULE)p == hModule) {
if (--p->refcount == 0) {
_Unregister(p);
MemoryFreeLibrary(p);
}
return TRUE;
}
p = p->next;
}
return FreeLibrary(hModule);
}
#if DEBUG_OUTPUT
static void
OutputLastError(const char *msg)
{
LPVOID tmp;
char *tmpmsg;
FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS,
NULL, GetLastError(), MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), (LPTSTR)&tmp, 0, NULL);
tmpmsg = (char *)LocalAlloc(LPTR, strlen(msg) + strlen(tmp) + 3);
sprintf(tmpmsg, "%s: %s", msg, tmp);
OutputDebugString(tmpmsg);
LocalFree(tmpmsg);
LocalFree(tmp);
}
#endif
/*
static int dprintf(char *fmt, ...)
{
char Buffer[4096];
va_list marker;
int result;
va_start(marker, fmt);
result = vsprintf(Buffer, fmt, marker);
OutputDebugString(Buffer);
return result;
}
*/
static void
CopySections(const unsigned char *data, PIMAGE_NT_HEADERS old_headers, PMEMORYMODULE module)
{
int i, size;
unsigned char *codeBase = module->codeBase;
unsigned char *dest;
PIMAGE_SECTION_HEADER section = IMAGE_FIRST_SECTION(module->headers);
for (i=0; i<module->headers->FileHeader.NumberOfSections; i++, section++)
{
if (section->SizeOfRawData == 0)
{
// section doesn't contain data in the dll itself, but may define
// uninitialized data
size = old_headers->OptionalHeader.SectionAlignment;
if (size > 0)
{
dest = (unsigned char *)VirtualAlloc(codeBase + section->VirtualAddress,
size,
MEM_COMMIT,
PAGE_READWRITE);
section->Misc.PhysicalAddress = (DWORD)dest;
memset(dest, 0, size);
}
// section is empty
continue;
}
// commit memory block and copy data from dll
dest = (unsigned char *)VirtualAlloc(codeBase + section->VirtualAddress,
section->SizeOfRawData,
MEM_COMMIT,
PAGE_READWRITE);
memcpy(dest, data + section->PointerToRawData, section->SizeOfRawData);
section->Misc.PhysicalAddress = (DWORD)dest;
}
}
// Protection flags for memory pages (Executable, Readable, Writeable)
static int ProtectionFlags[2][2][2] = {
{
// not executable
{PAGE_NOACCESS, PAGE_WRITECOPY},
{PAGE_READONLY, PAGE_READWRITE},
}, {
// executable
{PAGE_EXECUTE, PAGE_EXECUTE_WRITECOPY},
{PAGE_EXECUTE_READ, PAGE_EXECUTE_READWRITE},
},
};
static void
FinalizeSections(PMEMORYMODULE module)
{
int i;
PIMAGE_SECTION_HEADER section = IMAGE_FIRST_SECTION(module->headers);
// loop through all sections and change access flags
for (i=0; i<module->headers->FileHeader.NumberOfSections; i++, section++)
{
DWORD protect, oldProtect, size;
int executable = (section->Characteristics & IMAGE_SCN_MEM_EXECUTE) != 0;
int readable = (section->Characteristics & IMAGE_SCN_MEM_READ) != 0;
int writeable = (section->Characteristics & IMAGE_SCN_MEM_WRITE) != 0;
if (section->Characteristics & IMAGE_SCN_MEM_DISCARDABLE)
{
// section is not needed any more and can safely be freed
VirtualFree((LPVOID)section->Misc.PhysicalAddress, section->SizeOfRawData, MEM_DECOMMIT);
continue;
}
// determine protection flags based on characteristics
protect = ProtectionFlags[executable][readable][writeable];
if (section->Characteristics & IMAGE_SCN_MEM_NOT_CACHED)
protect |= PAGE_NOCACHE;
// determine size of region
size = section->SizeOfRawData;
if (size == 0)
{
if (section->Characteristics & IMAGE_SCN_CNT_INITIALIZED_DATA)
size = module->headers->OptionalHeader.SizeOfInitializedData;
else if (section->Characteristics & IMAGE_SCN_CNT_UNINITIALIZED_DATA)
size = module->headers->OptionalHeader.SizeOfUninitializedData;
}
if (size > 0)
{
// change memory access flags
if (VirtualProtect((LPVOID)section->Misc.PhysicalAddress, section->SizeOfRawData, protect, &oldProtect) == 0)
#if DEBUG_OUTPUT
OutputLastError("Error protecting memory page")
#endif
;
}
}
}
static void
PerformBaseRelocation(PMEMORYMODULE module, DWORD delta)
{
DWORD i;
unsigned char *codeBase = module->codeBase;
PIMAGE_DATA_DIRECTORY directory = GET_HEADER_DICTIONARY(module, IMAGE_DIRECTORY_ENTRY_BASERELOC);
if (directory->Size > 0)
{
PIMAGE_BASE_RELOCATION relocation = (PIMAGE_BASE_RELOCATION)(codeBase + directory->VirtualAddress);
for (; relocation->VirtualAddress > 0; )
{
unsigned char *dest = (unsigned char *)(codeBase + relocation->VirtualAddress);
unsigned short *relInfo = (unsigned short *)((unsigned char *)relocation + IMAGE_SIZEOF_BASE_RELOCATION);
for (i=0; i<((relocation->SizeOfBlock-IMAGE_SIZEOF_BASE_RELOCATION) / 2); i++, relInfo++)
{
DWORD *patchAddrHL;
int type, offset;
// the upper 4 bits define the type of relocation
type = *relInfo >> 12;
// the lower 12 bits define the offset
offset = *relInfo & 0xfff;
switch (type)
{
case IMAGE_REL_BASED_ABSOLUTE:
// skip relocation
break;
case IMAGE_REL_BASED_HIGHLOW:
// change complete 32 bit address
patchAddrHL = (DWORD *)(dest + offset);
*patchAddrHL += delta;
break;
default:
//printf("Unknown relocation: %d\n", type);
break;
}
}
// advance to next relocation block
relocation = (PIMAGE_BASE_RELOCATION)(((DWORD)relocation) + relocation->SizeOfBlock);
}
}
}
static int
BuildImportTable(PMEMORYMODULE module)
{
int result=1;
unsigned char *codeBase = module->codeBase;
PIMAGE_DATA_DIRECTORY directory = GET_HEADER_DICTIONARY(module, IMAGE_DIRECTORY_ENTRY_IMPORT);
if (directory->Size > 0)
{
PIMAGE_IMPORT_DESCRIPTOR importDesc = (PIMAGE_IMPORT_DESCRIPTOR)(codeBase + directory->VirtualAddress);
for (; !IsBadReadPtr(importDesc, sizeof(IMAGE_IMPORT_DESCRIPTOR)) && importDesc->Name; importDesc++)
{
DWORD *thunkRef, *funcRef;
HMODULE handle;
handle = MyLoadLibrary(codeBase + importDesc->Name);
if (handle == INVALID_HANDLE_VALUE)
{
//LastError should already be set
#if DEBUG_OUTPUT
OutputLastError("Can't load library");
#endif
result = 0;
break;
}
module->modules = (HMODULE *)realloc(module->modules, (module->numModules+1)*(sizeof(HMODULE)));
if (module->modules == NULL)
{
SetLastError(ERROR_NOT_ENOUGH_MEMORY);
result = 0;
break;
}
module->modules[module->numModules++] = handle;
if (importDesc->OriginalFirstThunk)
{
thunkRef = (DWORD *)(codeBase + importDesc->OriginalFirstThunk);
funcRef = (DWORD *)(codeBase + importDesc->FirstThunk);
} else {
// no hint table
thunkRef = (DWORD *)(codeBase + importDesc->FirstThunk);
funcRef = (DWORD *)(codeBase + importDesc->FirstThunk);
}
for (; *thunkRef; thunkRef++, funcRef++)
{
if IMAGE_SNAP_BY_ORDINAL(*thunkRef) {
*funcRef = (DWORD)MyGetProcAddress(handle, (LPCSTR)IMAGE_ORDINAL(*thunkRef));
} else {
PIMAGE_IMPORT_BY_NAME thunkData = (PIMAGE_IMPORT_BY_NAME)(codeBase + *thunkRef);
*funcRef = (DWORD)MyGetProcAddress(handle, (LPCSTR)&thunkData->Name);
}
if (*funcRef == 0)
{
SetLastError(ERROR_PROC_NOT_FOUND);
result = 0;
break;
}
}
if (!result)
break;
}
}
return result;
}
/*
MemoryLoadLibrary - load a library AS MEMORY MODULE, or return
existing MEMORY MODULE with increased refcount.
This allows to load a library AGAIN as memory module which is
already loaded as HMODULE!
*/
HMEMORYMODULE MemoryLoadLibrary(char *name, const void *data)
{
PMEMORYMODULE result;
PIMAGE_DOS_HEADER dos_header;
PIMAGE_NT_HEADERS old_header;
unsigned char *code, *headers;
DWORD locationDelta;
DllEntryProc DllEntry;
BOOL successfull;
MEMORYMODULE *p = loaded;
while (p) {
// If already loaded, only increment the reference count
if (0 == stricmp(name, p->name)) {
p->refcount++;
return (HMODULE)p;
}
p = p->next;
}
/* Do NOT check for GetModuleHandle here! */
dos_header = (PIMAGE_DOS_HEADER)data;
if (dos_header->e_magic != IMAGE_DOS_SIGNATURE)
{
SetLastError(ERROR_BAD_FORMAT);
#if DEBUG_OUTPUT
OutputDebugString("Not a valid executable file.\n");
#endif
return NULL;
}
old_header = (PIMAGE_NT_HEADERS)&((const unsigned char *)(data))[dos_header->e_lfanew];
if (old_header->Signature != IMAGE_NT_SIGNATURE)
{
SetLastError(ERROR_BAD_FORMAT);
#if DEBUG_OUTPUT
OutputDebugString("No PE header found.\n");
#endif
return NULL;
}
// reserve memory for image of library
code = (unsigned char *)VirtualAlloc((LPVOID)(old_header->OptionalHeader.ImageBase),
old_header->OptionalHeader.SizeOfImage,
MEM_RESERVE,
PAGE_READWRITE);
if (code == NULL)
// try to allocate memory at arbitrary position
code = (unsigned char *)VirtualAlloc(NULL,
old_header->OptionalHeader.SizeOfImage,
MEM_RESERVE,
PAGE_READWRITE);
if (code == NULL)
{
SetLastError(ERROR_NOT_ENOUGH_MEMORY);
#if DEBUG_OUTPUT
OutputLastError("Can't reserve memory");
#endif
return NULL;
}
result = (PMEMORYMODULE)HeapAlloc(GetProcessHeap(), 0, sizeof(MEMORYMODULE));
result->codeBase = code;
result->numModules = 0;
result->modules = NULL;
result->initialized = 0;
result->next = result->prev = NULL;
result->refcount = 1;
result->name = strdup(name);
result->name_table = NULL;
// XXX: is it correct to commit the complete memory region at once?
// calling DllEntry raises an exception if we don't...
VirtualAlloc(code,
old_header->OptionalHeader.SizeOfImage,
MEM_COMMIT,
PAGE_READWRITE);
// commit memory for headers
headers = (unsigned char *)VirtualAlloc(code,
old_header->OptionalHeader.SizeOfHeaders,
MEM_COMMIT,
PAGE_READWRITE);
// copy PE header to code
memcpy(headers, dos_header, dos_header->e_lfanew + old_header->OptionalHeader.SizeOfHeaders);
result->headers = (PIMAGE_NT_HEADERS)&((const unsigned char *)(headers))[dos_header->e_lfanew];
// update position
result->headers->OptionalHeader.ImageBase = (DWORD)code;
// copy sections from DLL file block to new memory location
CopySections(data, old_header, result);
// adjust base address of imported data
locationDelta = (DWORD)(code - old_header->OptionalHeader.ImageBase);
if (locationDelta != 0)
PerformBaseRelocation(result, locationDelta);
// load required dlls and adjust function table of imports
if (!BuildImportTable(result))
goto error;
// mark memory pages depending on section headers and release
// sections that are marked as "discardable"
FinalizeSections(result);
// get entry point of loaded library
if (result->headers->OptionalHeader.AddressOfEntryPoint != 0)
{
DllEntry = (DllEntryProc)(code + result->headers->OptionalHeader.AddressOfEntryPoint);
if (DllEntry == 0)
{
SetLastError(ERROR_BAD_FORMAT); /* XXX ? */
#if DEBUG_OUTPUT
OutputDebugString("Library has no entry point.\n");
#endif
goto error;
}
// notify library about attaching to process
successfull = (*DllEntry)((HINSTANCE)code, DLL_PROCESS_ATTACH, 0);
if (!successfull)
{
#if DEBUG_OUTPUT
OutputDebugString("Can't attach library.\n");
#endif
goto error;
}
result->initialized = 1;
}
_Register(name, result);
return (HMEMORYMODULE)result;
error:
// cleanup
free(result->name);
MemoryFreeLibrary(result);
return NULL;
}
int _compare(const struct NAME_TABLE *p1, const struct NAME_TABLE *p2)
{
return stricmp(p1->name, p2->name);
}
int _find(const char **name, const struct NAME_TABLE *p)
{
return stricmp(*name, p->name);
}
struct NAME_TABLE *GetNameTable(PMEMORYMODULE module)
{
unsigned char *codeBase;
PIMAGE_EXPORT_DIRECTORY exports;
PIMAGE_DATA_DIRECTORY directory;
DWORD i, *nameRef;
WORD *ordinal;
struct NAME_TABLE *p, *ptab;
if (module->name_table)
return module->name_table;
codeBase = module->codeBase;
directory = GET_HEADER_DICTIONARY(module, IMAGE_DIRECTORY_ENTRY_EXPORT);
exports = (PIMAGE_EXPORT_DIRECTORY)(codeBase + directory->VirtualAddress);
nameRef = (DWORD *)(codeBase + exports->AddressOfNames);
ordinal = (WORD *)(codeBase + exports->AddressOfNameOrdinals);
p = ((PMEMORYMODULE)module)->name_table = (struct NAME_TABLE *)malloc(sizeof(struct NAME_TABLE)
* exports->NumberOfNames);
if (p == NULL)
return NULL;
ptab = p;
for (i=0; i<exports->NumberOfNames; ++i) {
p->name = (char *)(codeBase + *nameRef++);
p->ordinal = *ordinal++;
++p;
}
qsort(ptab, exports->NumberOfNames, sizeof(struct NAME_TABLE), _compare);
return ptab;
}
FARPROC MemoryGetProcAddress(HMEMORYMODULE module, const char *name)
{
unsigned char *codeBase = ((PMEMORYMODULE)module)->codeBase;
int idx=-1;
PIMAGE_EXPORT_DIRECTORY exports;
PIMAGE_DATA_DIRECTORY directory = GET_HEADER_DICTIONARY((PMEMORYMODULE)module, IMAGE_DIRECTORY_ENTRY_EXPORT);
if (directory->Size == 0)
// no export table found
return NULL;
exports = (PIMAGE_EXPORT_DIRECTORY)(codeBase + directory->VirtualAddress);
if (exports->NumberOfNames == 0 || exports->NumberOfFunctions == 0)
// DLL doesn't export anything
return NULL;
if (HIWORD(name)) {
struct NAME_TABLE *ptab;
struct NAME_TABLE *found;
ptab = GetNameTable((PMEMORYMODULE)module);
if (ptab == NULL)
// some failure
return NULL;
found = bsearch(&name, ptab, exports->NumberOfNames, sizeof(struct NAME_TABLE), _find);
if (found == NULL)
// exported symbol not found
return NULL;
idx = found->ordinal;
}
else
idx = LOWORD(name) - exports->Base;
if ((DWORD)idx > exports->NumberOfFunctions)
// name <-> ordinal number don't match
return NULL;
// AddressOfFunctions contains the RVAs to the "real" functions
return (FARPROC)(codeBase + *(DWORD *)(codeBase + exports->AddressOfFunctions + (idx*4)));
}
void MemoryFreeLibrary(HMEMORYMODULE mod)
{
int i;
PMEMORYMODULE module = (PMEMORYMODULE)mod;
if (module != NULL)
{
if (module->initialized != 0)
{
// notify library about detaching from process
DllEntryProc DllEntry = (DllEntryProc)(module->codeBase + module->headers->OptionalHeader.AddressOfEntryPoint);
(*DllEntry)((HINSTANCE)module->codeBase, DLL_PROCESS_DETACH, 0);
module->initialized = 0;
}
if (module->modules != NULL)
{
// free previously opened libraries
for (i=0; i<module->numModules; i++)
if (module->modules[i] != INVALID_HANDLE_VALUE)
MyFreeLibrary(module->modules[i]);
free(module->modules);
}
if (module->codeBase != NULL)
// release memory of library
VirtualFree(module->codeBase, 0, MEM_RELEASE);
if (module->name_table != NULL)
free(module->name_table);
HeapFree(GetProcessHeap(), 0, module);
}
}

58
setup/installer/windows/MemoryModule.h Normal file
View File

@ -0,0 +1,58 @@
/*
* Memory DLL loading code
* Version 0.0.2
*
* Copyright (c) 2004-2005 by Joachim Bauch / mail@joachim-bauch.de
* http://www.joachim-bauch.de
*
* The contents of this file are subject to the Mozilla Public License Version
* 1.1 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
* http://www.mozilla.org/MPL/
*
* Software distributed under the License is distributed on an "AS IS" basis,
* WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
* for the specific language governing rights and limitations under the
* License.
*
* The Original Code is MemoryModule.h
*
* The Initial Developer of the Original Code is Joachim Bauch.
*
* Portions created by Joachim Bauch are Copyright (C) 2004-2005
* Joachim Bauch. All Rights Reserved.
*
*/
#ifndef __MEMORY_MODULE_HEADER
#define __MEMORY_MODULE_HEADER
#include <Windows.h>
typedef void *HMEMORYMODULE;
#ifdef __cplusplus
extern "C" {
#endif
typedef void *(*FINDPROC)();
extern FINDPROC findproc;
extern void *findproc_data;
HMEMORYMODULE MemoryLoadLibrary(char *, const void *);
FARPROC MemoryGetProcAddress(HMEMORYMODULE, const char *);
void MemoryFreeLibrary(HMEMORYMODULE);
BOOL MyFreeLibrary(HMODULE hModule);
HMODULE MyLoadLibrary(char *lpFileName);
FARPROC MyGetProcAddress(HMODULE hModule, LPCSTR lpProcName);
HMODULE MyGetModuleHandle(LPCTSTR lpModuleName);
#ifdef __cplusplus
}
#endif
#endif // __MEMORY_MODULE_HEADER

92
setup/installer/windows/freeze.py
View File

@ -71,11 +71,13 @@ class Win32Freeze(Command, WixMixIn):
self.rc_template = self.j(self.d(self.a(__file__)), 'template.rc')
self.py_ver = ''.join(map(str, sys.version_info[:2]))
self.lib_dir = self.j(self.base, 'Lib')
self.pydlib = self.j(self.base, 'pydlib')
self.pylib = self.j(self.base, 'pylib.zip')
self.dll_dir = self.j(self.base, 'DLLs')
self.plugins_dir = os.path.join(self.base, 'plugins')
self.initbase()
self.build_launchers()
self.add_plugins()
self.freeze()
self.embed_manifests()
self.install_site_py()
@ -87,18 +89,20 @@ class Win32Freeze(Command, WixMixIn):
shutil.rmtree(self.base)
os.makedirs(self.base)
def freeze(self):
shutil.copy2(self.j(self.src_root, 'LICENSE'), self.base)
def add_plugins(self):
self.info('Adding plugins...')
tgt = os.path.join(self.base, 'plugins')
if not os.path.exists(tgt):
os.mkdir(tgt)
tgt = self.plugins_dir
if os.path.exists(tgt):
shutil.rmtree(tgt)
os.mkdir(tgt)
base = self.j(self.SRC, 'calibre', 'plugins')
for pat in ('*.pyd', '*.manifest'):
for f in glob.glob(self.j(base, pat)):
shutil.copy2(f, tgt)
def freeze(self):
shutil.copy2(self.j(self.src_root, 'LICENSE'), self.base)
self.info('Adding resources...')
tgt = self.j(self.base, 'resources')
if os.path.exists(tgt):
@ -106,7 +110,6 @@ class Win32Freeze(Command, WixMixIn):
shutil.copytree(self.j(self.src_root, 'resources'), tgt)
self.info('Adding Qt and python...')
self.dll_dir = self.j(self.base, 'DLLs')
shutil.copytree(r'C:\Python%s\DLLs'%self.py_ver, self.dll_dir,
ignore=shutil.ignore_patterns('msvc*.dll', 'Microsoft.*'))
for x in glob.glob(self.j(OPENSSL_DIR, 'bin', '*.dll')):
@ -318,8 +321,8 @@ class Win32Freeze(Command, WixMixIn):
if not os.path.exists(self.obj_dir):
os.makedirs(self.obj_dir)
base = self.j(self.src_root, 'setup', 'installer', 'windows')
sources = [self.j(base, x) for x in ['util.c']]
headers = [self.j(base, x) for x in ['util.h']]
sources = [self.j(base, x) for x in ['util.c', 'MemoryModule.c']]
headers = [self.j(base, x) for x in ['util.h', 'MemoryModule.h']]
objects = [self.j(self.obj_dir, self.b(x)+'.obj') for x in sources]
cflags = '/c /EHsc /MD /W3 /Ox /nologo /D_UNICODE'.split()
cflags += ['/DPYDLL="python%s.dll"'%self.py_ver, '/IC:/Python%s/include'%self.py_ver]
@ -371,43 +374,49 @@ class Win32Freeze(Command, WixMixIn):
def archive_lib_dir(self):
self.info('Putting all python code into a zip file for performance')
if os.path.exists(self.pydlib):
shutil.rmtree(self.pydlib)
os.makedirs(self.pydlib)
self.zf_timestamp = time.localtime(time.time())[:6]
self.zf_names = set()
with zipfile.ZipFile(self.pylib, 'w', zipfile.ZIP_STORED) as zf:
# Add the .pyds from python and calibre to the zip file
for x in (self.plugins_dir, self.dll_dir):
for pyd in os.listdir(x):
if pyd.endswith('.pyd') and pyd != 'sqlite_custom.pyd':
# sqlite_custom has to be a file for
# sqlite_load_extension to work
self.add_to_zipfile(zf, pyd, x)
os.remove(self.j(x, pyd))
# Add everything in Lib except site-packages to the zip file
for x in os.listdir(self.lib_dir):
if x == 'site-packages':
continue
self.add_to_zipfile(zf, x, self.lib_dir)
sp = self.j(self.lib_dir, 'site-packages')
handled = set(['site.pyo'])
for pth in ('PIL.pth', 'pywin32.pth'):
handled.add(pth)
shutil.copyfile(self.j(sp, pth), self.j(self.pydlib, pth))
for d in self.get_pth_dirs(self.j(sp, pth)):
shutil.copytree(d, self.j(self.pydlib, self.b(d)), True)
handled.add(self.b(d))
# Special handling for PIL and pywin32
handled = set(['PIL.pth', 'pywin32.pth', 'PIL', 'win32'])
self.add_to_zipfile(zf, 'PIL', sp)
base = self.j(sp, 'win32', 'lib')
for x in os.listdir(base):
if os.path.splitext(x)[1] not in ('.exe',):
self.add_to_zipfile(zf, x, base)
base = self.d(base)
for x in os.listdir(base):
if not os.path.isdir(self.j(base, x)):
if os.path.splitext(x)[1] not in ('.exe',):
self.add_to_zipfile(zf, x, base)
handled.add('easy-install.pth')
for d in self.get_pth_dirs(self.j(sp, 'easy-install.pth')):
handled.add(self.b(d))
zip_safe = self.is_zip_safe(d)
for x in os.listdir(d):
if x == 'EGG-INFO':
continue
if zip_safe:
self.add_to_zipfile(zf, x, d)
else:
absp = self.j(d, x)
dest = self.j(self.pydlib, x)
if os.path.isdir(absp):
shutil.copytree(absp, dest, True)
else:
shutil.copy2(absp, dest)
self.add_to_zipfile(zf, x, d)
# The rest of site-packages
# We dont want the site.py from site-packages
handled.add('site.pyo')
for x in os.listdir(sp):
if x in handled or x.endswith('.egg-info'):
continue
@ -415,33 +424,18 @@ class Win32Freeze(Command, WixMixIn):
if os.path.isdir(absp):
if not os.listdir(absp):
continue
if self.is_zip_safe(absp):
self.add_to_zipfile(zf, x, sp)
else:
shutil.copytree(absp, self.j(self.pydlib, x), True)
self.add_to_zipfile(zf, x, sp)
else:
if x.endswith('.pyd'):
shutil.copy2(absp, self.j(self.pydlib, x))
else:
self.add_to_zipfile(zf, x, sp)
self.add_to_zipfile(zf, x, sp)
shutil.rmtree(self.lib_dir)
def is_zip_safe(self, path):
for f in walk(path):
ext = os.path.splitext(f)[1].lower()
if ext in ('.pyd', '.dll', '.exe'):
return False
return True
def get_pth_dirs(self, pth):
base = os.path.dirname(pth)
for line in open(pth).readlines():
line = line.strip()
if not line or line.startswith('#') or line.startswith('import'):
continue
if line == 'win32\\lib':
continue
candidate = self.j(base, line)
if os.path.exists(candidate):
yield candidate
@ -463,10 +457,10 @@ class Win32Freeze(Command, WixMixIn):
self.add_to_zipfile(zf, name + os.sep + x, base)
else:
ext = os.path.splitext(name)[1].lower()
if ext in ('.pyd', '.dll', '.exe'):
if ext in ('.dll',):
raise ValueError('Cannot add %r to zipfile'%abspath)
zinfo.external_attr = 0600 << 16
if ext in ('.py', '.pyc', '.pyo'):
if ext in ('.py', '.pyc', '.pyo', '.pyd'):
with open(abspath, 'rb') as f:
zf.writestr(zinfo, f.read())

109
setup/installer/windows/site.py
View File

@ -1,12 +1,72 @@
#!/usr/bin/env python
# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
from __future__ import with_statement
__license__ = 'GPL v3'
__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
__docformat__ = 'restructuredtext en'
import sys, os, linecache
import sys
import os
import zipimport
import _memimporter
DEBUG_ZIPIMPORT = False
class ZipExtensionImporter(zipimport.zipimporter):
'''
Taken, with thanks, from the py2exe source code
'''
def __init__(self, *args, **kwargs):
zipimport.zipimporter.__init__(self, *args, **kwargs)
# We know there are no dlls in the zip file, so dont set findproc
# (performance optimization)
#_memimporter.set_find_proc(self.locate_dll_image)
def find_module(self, fullname, path=None):
result = zipimport.zipimporter.find_module(self, fullname, path)
if result:
return result
fullname = fullname.replace(".", "\\")
if (fullname + '.pyd') in self._files:
return self
return None
def locate_dll_image(self, name):
# A callback function for_memimporter.import_module. Tries to
# locate additional dlls. Returns the image as Python string,
# or None if not found.
if name in self._files:
return self.get_data(name)
return None
def load_module(self, fullname):
if sys.modules.has_key(fullname):
mod = sys.modules[fullname]
if DEBUG_ZIPIMPORT:
sys.stderr.write("import %s # previously loaded from zipfile %s\n" % (fullname, self.archive))
return mod
try:
return zipimport.zipimporter.load_module(self, fullname)
except zipimport.ZipImportError:
pass
initname = "init" + fullname.split(".")[-1] # name of initfunction
filename = fullname.replace(".", "\\")
path = filename + '.pyd'
if path in self._files:
if DEBUG_ZIPIMPORT:
sys.stderr.write("# found %s in zipfile %s\n" % (path, self.archive))
code = self.get_data(path)
mod = _memimporter.import_module(code, initname, fullname, path)
mod.__file__ = "%s\\%s" % (self.archive, path)
mod.__loader__ = self
if DEBUG_ZIPIMPORT:
sys.stderr.write("import %s # loaded from zipfile %s\n" % (fullname, mod.__file__))
return mod
raise zipimport.ZipImportError, "can't find module %s" % fullname
def __repr__(self):
return "<%s object %r>" % (self.__class__.__name__, self.archive)
def abs__file__():
@ -42,42 +102,6 @@ def makepath(*paths):
dir = os.path.abspath(os.path.join(*paths))
return dir, os.path.normcase(dir)
def addpackage(sitedir, name):
"""Process a .pth file within the site-packages directory:
For each line in the file, either combine it with sitedir to a path,
or execute it if it starts with 'import '.
"""
fullname = os.path.join(sitedir, name)
try:
f = open(fullname, "rU")
except IOError:
return
with f:
for line in f:
if line.startswith("#"):
continue
if line.startswith(("import ", "import\t")):
exec line
continue
line = line.rstrip()
dir, dircase = makepath(sitedir, line)
if os.path.exists(dir):
sys.path.append(dir)
def addsitedir(sitedir):
"""Add 'sitedir' argument to sys.path if missing and handle .pth files in
'sitedir'"""
sitedir, sitedircase = makepath(sitedir)
try:
names = os.listdir(sitedir)
except os.error:
return
dotpth = os.extsep + "pth"
names = [name for name in names if name.endswith(dotpth)]
for name in sorted(names):
addpackage(sitedir, name)
def run_entry_point():
bname, mod, func = sys.calibre_basename, sys.calibre_module, sys.calibre_function
sys.argv[0] = bname+'.exe'
@ -89,6 +113,10 @@ def main():
sys.setdefaultencoding('utf-8')
aliasmbcs()
sys.path_hooks.insert(0, ZipExtensionImporter)
sys.path_importer_cache.clear()
import linecache
def fake_getline(filename, lineno, module_globals=None):
return ''
linecache.orig_getline = linecache.getline
@ -96,10 +124,11 @@ def main():
abs__file__()
addsitedir(os.path.join(sys.app_dir, 'pydlib'))
add_calibre_vars()
# Needed for pywintypes to be able to load its DLL
sys.path.append(os.path.join(sys.app_dir, 'DLLs'))
return run_entry_point()

126
setup/installer/windows/util.c
View File

@ -1,18 +1,130 @@
/*
* Copyright 2009 Kovid Goyal
* The memimporter code is taken from the py2exe project
*/
#include "util.h"
#include <delayimp.h>
#include <io.h>
#include <fcntl.h>
static char GUI_APP = 0;
static char python_dll[] = PYDLL;
void set_gui_app(char yes) { GUI_APP = yes; }
char is_gui_app() { return GUI_APP; }
// memimporter {{{
#include "MemoryModule.h"
static char **DLL_Py_PackageContext = NULL;
static PyObject **DLL_ImportError = NULL;
static char module_doc[] =
"Importer which can load extension modules from memory";
static void *memdup(void *ptr, Py_ssize_t size)
{
void *p = malloc(size);
if (p == NULL)
return NULL;
memcpy(p, ptr, size);
return p;
}
/*
Be sure to detect errors in FindLibrary - undetected errors lead to
very strange behaviour.
*/
static void* FindLibrary(char *name, PyObject *callback)
{
PyObject *result;
char *p;
Py_ssize_t size;
if (callback == NULL)
return NULL;
result = PyObject_CallFunction(callback, "s", name);
if (result == NULL) {
PyErr_Clear();
return NULL;
}
if (-1 == PyString_AsStringAndSize(result, &p, &size)) {
PyErr_Clear();
Py_DECREF(result);
return NULL;
}
p = memdup(p, size);
Py_DECREF(result);
return p;
}
static PyObject *set_find_proc(PyObject *self, PyObject *args)
{
PyObject *callback = NULL;
if (!PyArg_ParseTuple(args, "|O:set_find_proc", &callback))
return NULL;
Py_DECREF((PyObject *)findproc_data);
Py_INCREF(callback);
findproc_data = (void *)callback;
return Py_BuildValue("i", 1);
}
static PyObject *
import_module(PyObject *self, PyObject *args)
{
char *data;
int size;
char *initfuncname;
char *modname;
char *pathname;
HMEMORYMODULE hmem;
FARPROC do_init;
char *oldcontext;
/* code, initfuncname, fqmodulename, path */
if (!PyArg_ParseTuple(args, "s#sss:import_module",
&data, &size,
&initfuncname, &modname, &pathname))
return NULL;
hmem = MemoryLoadLibrary(pathname, data);
if (!hmem) {
PyErr_Format(*DLL_ImportError,
"MemoryLoadLibrary() failed loading %s", pathname);
return NULL;
}
do_init = MemoryGetProcAddress(hmem, initfuncname);
if (!do_init) {
MemoryFreeLibrary(hmem);
PyErr_Format(*DLL_ImportError,
"Could not find function %s in memory loaded pyd", initfuncname);
return NULL;
}
oldcontext = *DLL_Py_PackageContext;
*DLL_Py_PackageContext = modname;
do_init();
*DLL_Py_PackageContext = oldcontext;
if (PyErr_Occurred())
return NULL;
/* Retrieve from sys.modules */
return PyImport_ImportModule(modname);
}
static PyMethodDef methods[] = {
{ "import_module", import_module, METH_VARARGS,
"import_module(code, initfunc, dllname[, finder]) -> module" },
{ "set_find_proc", set_find_proc, METH_VARARGS },
{ NULL, NULL }, /* Sentinel */
};
// }}}
static int _show_error(const wchar_t *preamble, const wchar_t *msg, const int code) {
wchar_t *buf, *cbuf;
buf = (wchar_t*)LocalAlloc(LMEM_ZEROINIT, sizeof(wchar_t)*
@ -185,7 +297,7 @@ void initialize_interpreter(wchar_t *outr, wchar_t *errr,
char *dummy_argv[1] = {""};
buf = (char*)calloc(MAX_PATH, sizeof(char));
path = (char*)calloc(3*MAX_PATH, sizeof(char));
path = (char*)calloc(MAX_PATH, sizeof(char));
if (!buf || !path) ExitProcess(_show_error(L"Out of memory", L"", 1));
sz = GetModuleFileNameA(NULL, buf, MAX_PATH);
@ -198,8 +310,7 @@ void initialize_interpreter(wchar_t *outr, wchar_t *errr,
buf[strlen(buf)-1] = '\0';
_snprintf_s(python_home, MAX_PATH, _TRUNCATE, "%s", buf);
_snprintf_s(path, 3*MAX_PATH, _TRUNCATE, "%s\\pylib.zip;%s\\pydlib;%s\\DLLs",
buf, buf, buf);
_snprintf_s(path, MAX_PATH, _TRUNCATE, "%s\\pylib.zip", buf);
free(buf);
@ -227,7 +338,10 @@ void initialize_interpreter(wchar_t *outr, wchar_t *errr,
if (!flag) ExitProcess(_show_error(L"Failed to get debug flag", L"", 1));
//*flag = 1;
DLL_Py_PackageContext = (char**)GetProcAddress(dll, "_Py_PackageContext");
if (!DLL_Py_PackageContext) ExitProcess(_show_error(L"Failed to load _Py_PackageContext from dll", L"", 1));
DLL_ImportError = (PyObject**)GetProcAddress(dll, "PyExc_ImportError");
if (!DLL_ImportError) ExitProcess(_show_error(L"Failed to load PyExc_ImportError from dll", L"", 1));
Py_SetProgramName(program_name);
Py_SetPythonHome(python_home);
@ -263,6 +377,10 @@ void initialize_interpreter(wchar_t *outr, wchar_t *errr,
PyList_SetItem(argv, i, v);
}
PySys_SetObject("argv", argv);
findproc = FindLibrary;
Py_InitModule3("_memimporter", methods, module_doc);
}

4
setup/installer/windows/wix-template.xml
View File

@ -164,10 +164,6 @@
<CustomAction Id="LaunchApplication" BinaryKey="WixCA"
DllEntry="WixShellExec" Impersonate="yes"/>
<InstallUISequence>
<FileCost Suppress="yes" />
</InstallUISequence>
</Product>
</Wix>

5
src/calibre/debug.py
View File

@ -53,6 +53,8 @@ Run an embedded python interpreter.
default=False, action='store_true')
parser.add_option('-m', '--inspect-mobi',
help='Inspect the MOBI file at the specified path', default=None)
parser.add_option('--test-build', help='Test binary modules in build',
action='store_true', default=False)
return parser
@ -232,6 +234,9 @@ def main(args=sys.argv):
elif opts.inspect_mobi is not None:
from calibre.ebooks.mobi.debug import inspect_mobi
inspect_mobi(opts.inspect_mobi)
elif opts.test_build:
from calibre.test_build import test
test()
else:
from calibre import ipython
ipython()

103
src/calibre/test_build.py Normal file
View File

@ -0,0 +1,103 @@
#!/usr/bin/env python
# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
from __future__ import (unicode_literals, division, absolute_import,
print_function)
from future_builtins import map
__license__ = 'GPL v3'
__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
__docformat__ = 'restructuredtext en'
'''
Test a binary calibre build to ensure that all needed binary images/libraries have loaded.
'''
import cStringIO
from calibre.constants import plugins, iswindows
def test_plugins():
for name in plugins:
mod, err = plugins[name]
if err or not mod:
raise RuntimeError('Plugin %s failed to load with error: %s' %
(name, err))
print (mod, 'loaded')
def test_lxml():
from lxml import etree
raw = '<a/>'
root = etree.fromstring(raw)
if etree.tostring(root) == raw:
print ('lxml OK!')
else:
raise RuntimeError('lxml failed')
def test_fontconfig():
from calibre.utils.fonts import fontconfig
families = fontconfig.find_font_families()
num = len(families)
if num < 10:
raise RuntimeError('Fontconfig found only %d font families'%num)
print ('Fontconfig OK! (%d families)'%num)
def test_winutil():
from calibre.devices.scanner import win_pnp_drives
matches = win_pnp_drives.scanner()
if len(matches) < 1:
raise RuntimeError('win_pnp_drives returned no drives')
print ('win_pnp_drives OK!')
def test_win32():
from calibre.utils.winshell import desktop
d = desktop()
if not d:
raise RuntimeError('winshell failed')
print ('winshell OK! (%s is the desktop)'%d)
def test_sqlite():
import sqlite3
conn = sqlite3.connect(':memory:')
from calibre.library.sqlite import load_c_extensions
if not load_c_extensions(conn, True):
raise RuntimeError('Failed to load sqlite extension')
print ('sqlite OK!')
def test_qt():
from PyQt4.Qt import (QWebView, QDialog, QImageReader, QNetworkAccessManager)
fmts = set(map(unicode, QImageReader.supportedImageFormats()))
if 'jpg' not in fmts or 'png' not in fmts:
raise RuntimeError(
"Qt doesn't seem to be able to load its image plugins")
QWebView, QDialog
na = QNetworkAccessManager()
if not hasattr(na, 'sslErrors'):
raise RuntimeError('Qt not compiled with openssl')
print ('Qt OK!')
def test_imaging():
from calibre.utils.magick.draw import create_canvas, Image
im = create_canvas(20, 20, '#ffffff')
jpg = im.export('jpg')
Image().load(jpg)
im.export('png')
print ('ImageMagick OK!')
from PIL import Image
i = Image.open(cStringIO.StringIO(jpg))
if i.size != (20, 20):
raise RuntimeError('PIL choked!')
print ('PIL OK!')
def test():
test_plugins()
test_lxml()
test_fontconfig()
test_sqlite()
if iswindows:
test_winutil()
test_win32()
test_qt()
test_imaging()
if __name__ == '__main__':
test()