diff --git a/setup/installer/windows/MemoryModule.c b/setup/installer/windows/MemoryModule.c new file mode 100644 index 0000000000..253c8d7d9f --- /dev/null +++ b/setup/installer/windows/MemoryModule.c @@ -0,0 +1,689 @@ +/* + * Memory DLL loading code + * Version 0.0.2 with additions from Thomas Heller + * + * Copyright (c) 2004-2005 by Joachim Bauch / mail@joachim-bauch.de + * http://www.joachim-bauch.de + * + * The contents of this file are subject to the Mozilla Public License Version + * 1.1 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * http://www.mozilla.org/MPL/ + * + * Software distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License + * for the specific language governing rights and limitations under the + * License. + * + * The Original Code is MemoryModule.c + * + * The Initial Developer of the Original Code is Joachim Bauch. + * + * Portions created by Joachim Bauch are Copyright (C) 2004-2005 + * Joachim Bauch. All Rights Reserved. + * + * Portions Copyright (C) 2005 Thomas Heller. + * + */ + +// disable warnings about pointer <-> DWORD conversions +#pragma warning( disable : 4311 4312 ) + +#include +#include +#if DEBUG_OUTPUT +#include +#endif + +#ifndef IMAGE_SIZEOF_BASE_RELOCATION +// Vista SDKs no longer define IMAGE_SIZEOF_BASE_RELOCATION!? +# define IMAGE_SIZEOF_BASE_RELOCATION (sizeof(IMAGE_BASE_RELOCATION)) +#endif +#include "MemoryModule.h" + +/* + XXX We need to protect at least walking the 'loaded' linked list with a lock! +*/ + +/******************************************************************/ +FINDPROC findproc; +void *findproc_data = NULL; + +struct NAME_TABLE { + char *name; + DWORD ordinal; +}; + +typedef struct tagMEMORYMODULE { + PIMAGE_NT_HEADERS headers; + unsigned char *codeBase; + HMODULE *modules; + int numModules; + int initialized; + + struct NAME_TABLE *name_table; + + char *name; + int refcount; + struct tagMEMORYMODULE *next, *prev; +} MEMORYMODULE, *PMEMORYMODULE; + +typedef BOOL (WINAPI *DllEntryProc)(HINSTANCE hinstDLL, DWORD fdwReason, LPVOID lpReserved); + +#define GET_HEADER_DICTIONARY(module, idx) &(module)->headers->OptionalHeader.DataDirectory[idx] + +MEMORYMODULE *loaded; /* linked list of loaded memory modules */ + +/* private - insert a loaded library in a linked list */ +static void _Register(char *name, MEMORYMODULE *module) +{ + module->next = loaded; + if (loaded) + loaded->prev = module; + module->prev = NULL; + loaded = module; +} + +/* private - remove a loaded library from a linked list */ +static void _Unregister(MEMORYMODULE *module) +{ + free(module->name); + if (module->prev) + module->prev->next = module->next; + if (module->next) + module->next->prev = module->prev; + if (module == loaded) + loaded = module->next; +} + +/* public - replacement for GetModuleHandle() */ +HMODULE MyGetModuleHandle(LPCTSTR lpModuleName) +{ + MEMORYMODULE *p = loaded; + while (p) { + // If already loaded, only increment the reference count + if (0 == stricmp(lpModuleName, p->name)) { + return (HMODULE)p; + } + p = p->next; + } + return GetModuleHandle(lpModuleName); +} + +/* public - replacement for LoadLibrary, but searches FIRST for memory + libraries, then for normal libraries. So, it will load libraries AS memory + module if they are found by findproc(). +*/ +HMODULE MyLoadLibrary(char *lpFileName) +{ + MEMORYMODULE *p = loaded; + HMODULE hMod; + + while (p) { + // If already loaded, only increment the reference count + if (0 == stricmp(lpFileName, p->name)) { + p->refcount++; + return (HMODULE)p; + } + p = p->next; + } + if (findproc && findproc_data) { + void *pdata = findproc(lpFileName, findproc_data); + if (pdata) { + hMod = MemoryLoadLibrary(lpFileName, pdata); + free(p); + return hMod; + } + } + hMod = LoadLibrary(lpFileName); + return hMod; +} + +/* public - replacement for GetProcAddress() */ +FARPROC MyGetProcAddress(HMODULE hModule, LPCSTR lpProcName) +{ + MEMORYMODULE *p = loaded; + while (p) { + if ((HMODULE)p == hModule) + return MemoryGetProcAddress(p, lpProcName); + p = p->next; + } + return GetProcAddress(hModule, lpProcName); +} + +/* public - replacement for FreeLibrary() */ +BOOL MyFreeLibrary(HMODULE hModule) +{ + MEMORYMODULE *p = loaded; + while (p) { + if ((HMODULE)p == hModule) { + if (--p->refcount == 0) { + _Unregister(p); + MemoryFreeLibrary(p); + } + return TRUE; + } + p = p->next; + } + return FreeLibrary(hModule); +} + +#if DEBUG_OUTPUT +static void +OutputLastError(const char *msg) +{ + LPVOID tmp; + char *tmpmsg; + FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, + NULL, GetLastError(), MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), (LPTSTR)&tmp, 0, NULL); + tmpmsg = (char *)LocalAlloc(LPTR, strlen(msg) + strlen(tmp) + 3); + sprintf(tmpmsg, "%s: %s", msg, tmp); + OutputDebugString(tmpmsg); + LocalFree(tmpmsg); + LocalFree(tmp); +} +#endif + +/* +static int dprintf(char *fmt, ...) +{ + char Buffer[4096]; + va_list marker; + int result; + + va_start(marker, fmt); + result = vsprintf(Buffer, fmt, marker); + OutputDebugString(Buffer); + return result; +} +*/ + +static void +CopySections(const unsigned char *data, PIMAGE_NT_HEADERS old_headers, PMEMORYMODULE module) +{ + int i, size; + unsigned char *codeBase = module->codeBase; + unsigned char *dest; + PIMAGE_SECTION_HEADER section = IMAGE_FIRST_SECTION(module->headers); + for (i=0; iheaders->FileHeader.NumberOfSections; i++, section++) + { + if (section->SizeOfRawData == 0) + { + // section doesn't contain data in the dll itself, but may define + // uninitialized data + size = old_headers->OptionalHeader.SectionAlignment; + if (size > 0) + { + dest = (unsigned char *)VirtualAlloc(codeBase + section->VirtualAddress, + size, + MEM_COMMIT, + PAGE_READWRITE); + + section->Misc.PhysicalAddress = (DWORD)dest; + memset(dest, 0, size); + } + + // section is empty + continue; + } + + // commit memory block and copy data from dll + dest = (unsigned char *)VirtualAlloc(codeBase + section->VirtualAddress, + section->SizeOfRawData, + MEM_COMMIT, + PAGE_READWRITE); + memcpy(dest, data + section->PointerToRawData, section->SizeOfRawData); + section->Misc.PhysicalAddress = (DWORD)dest; + } +} + +// Protection flags for memory pages (Executable, Readable, Writeable) +static int ProtectionFlags[2][2][2] = { + { + // not executable + {PAGE_NOACCESS, PAGE_WRITECOPY}, + {PAGE_READONLY, PAGE_READWRITE}, + }, { + // executable + {PAGE_EXECUTE, PAGE_EXECUTE_WRITECOPY}, + {PAGE_EXECUTE_READ, PAGE_EXECUTE_READWRITE}, + }, +}; + +static void +FinalizeSections(PMEMORYMODULE module) +{ + int i; + PIMAGE_SECTION_HEADER section = IMAGE_FIRST_SECTION(module->headers); + + // loop through all sections and change access flags + for (i=0; iheaders->FileHeader.NumberOfSections; i++, section++) + { + DWORD protect, oldProtect, size; + int executable = (section->Characteristics & IMAGE_SCN_MEM_EXECUTE) != 0; + int readable = (section->Characteristics & IMAGE_SCN_MEM_READ) != 0; + int writeable = (section->Characteristics & IMAGE_SCN_MEM_WRITE) != 0; + + if (section->Characteristics & IMAGE_SCN_MEM_DISCARDABLE) + { + // section is not needed any more and can safely be freed + VirtualFree((LPVOID)section->Misc.PhysicalAddress, section->SizeOfRawData, MEM_DECOMMIT); + continue; + } + + // determine protection flags based on characteristics + protect = ProtectionFlags[executable][readable][writeable]; + if (section->Characteristics & IMAGE_SCN_MEM_NOT_CACHED) + protect |= PAGE_NOCACHE; + + // determine size of region + size = section->SizeOfRawData; + if (size == 0) + { + if (section->Characteristics & IMAGE_SCN_CNT_INITIALIZED_DATA) + size = module->headers->OptionalHeader.SizeOfInitializedData; + else if (section->Characteristics & IMAGE_SCN_CNT_UNINITIALIZED_DATA) + size = module->headers->OptionalHeader.SizeOfUninitializedData; + } + + if (size > 0) + { + // change memory access flags + if (VirtualProtect((LPVOID)section->Misc.PhysicalAddress, section->SizeOfRawData, protect, &oldProtect) == 0) +#if DEBUG_OUTPUT + OutputLastError("Error protecting memory page") +#endif + ; + } + } +} + +static void +PerformBaseRelocation(PMEMORYMODULE module, DWORD delta) +{ + DWORD i; + unsigned char *codeBase = module->codeBase; + + PIMAGE_DATA_DIRECTORY directory = GET_HEADER_DICTIONARY(module, IMAGE_DIRECTORY_ENTRY_BASERELOC); + if (directory->Size > 0) + { + PIMAGE_BASE_RELOCATION relocation = (PIMAGE_BASE_RELOCATION)(codeBase + directory->VirtualAddress); + for (; relocation->VirtualAddress > 0; ) + { + unsigned char *dest = (unsigned char *)(codeBase + relocation->VirtualAddress); + unsigned short *relInfo = (unsigned short *)((unsigned char *)relocation + IMAGE_SIZEOF_BASE_RELOCATION); + for (i=0; i<((relocation->SizeOfBlock-IMAGE_SIZEOF_BASE_RELOCATION) / 2); i++, relInfo++) + { + DWORD *patchAddrHL; + int type, offset; + + // the upper 4 bits define the type of relocation + type = *relInfo >> 12; + // the lower 12 bits define the offset + offset = *relInfo & 0xfff; + + switch (type) + { + case IMAGE_REL_BASED_ABSOLUTE: + // skip relocation + break; + + case IMAGE_REL_BASED_HIGHLOW: + // change complete 32 bit address + patchAddrHL = (DWORD *)(dest + offset); + *patchAddrHL += delta; + break; + + default: + //printf("Unknown relocation: %d\n", type); + break; + } + } + + // advance to next relocation block + relocation = (PIMAGE_BASE_RELOCATION)(((DWORD)relocation) + relocation->SizeOfBlock); + } + } +} + +static int +BuildImportTable(PMEMORYMODULE module) +{ + int result=1; + unsigned char *codeBase = module->codeBase; + + PIMAGE_DATA_DIRECTORY directory = GET_HEADER_DICTIONARY(module, IMAGE_DIRECTORY_ENTRY_IMPORT); + if (directory->Size > 0) + { + PIMAGE_IMPORT_DESCRIPTOR importDesc = (PIMAGE_IMPORT_DESCRIPTOR)(codeBase + directory->VirtualAddress); + for (; !IsBadReadPtr(importDesc, sizeof(IMAGE_IMPORT_DESCRIPTOR)) && importDesc->Name; importDesc++) + { + DWORD *thunkRef, *funcRef; + HMODULE handle; + + handle = MyLoadLibrary(codeBase + importDesc->Name); + if (handle == INVALID_HANDLE_VALUE) + { + //LastError should already be set +#if DEBUG_OUTPUT + OutputLastError("Can't load library"); +#endif + result = 0; + break; + } + + module->modules = (HMODULE *)realloc(module->modules, (module->numModules+1)*(sizeof(HMODULE))); + if (module->modules == NULL) + { + SetLastError(ERROR_NOT_ENOUGH_MEMORY); + result = 0; + break; + } + + module->modules[module->numModules++] = handle; + if (importDesc->OriginalFirstThunk) + { + thunkRef = (DWORD *)(codeBase + importDesc->OriginalFirstThunk); + funcRef = (DWORD *)(codeBase + importDesc->FirstThunk); + } else { + // no hint table + thunkRef = (DWORD *)(codeBase + importDesc->FirstThunk); + funcRef = (DWORD *)(codeBase + importDesc->FirstThunk); + } + for (; *thunkRef; thunkRef++, funcRef++) + { + if IMAGE_SNAP_BY_ORDINAL(*thunkRef) { + *funcRef = (DWORD)MyGetProcAddress(handle, (LPCSTR)IMAGE_ORDINAL(*thunkRef)); + } else { + PIMAGE_IMPORT_BY_NAME thunkData = (PIMAGE_IMPORT_BY_NAME)(codeBase + *thunkRef); + *funcRef = (DWORD)MyGetProcAddress(handle, (LPCSTR)&thunkData->Name); + } + if (*funcRef == 0) + { + SetLastError(ERROR_PROC_NOT_FOUND); + result = 0; + break; + } + } + + if (!result) + break; + } + } + + return result; +} + +/* + MemoryLoadLibrary - load a library AS MEMORY MODULE, or return + existing MEMORY MODULE with increased refcount. + + This allows to load a library AGAIN as memory module which is + already loaded as HMODULE! + +*/ +HMEMORYMODULE MemoryLoadLibrary(char *name, const void *data) +{ + PMEMORYMODULE result; + PIMAGE_DOS_HEADER dos_header; + PIMAGE_NT_HEADERS old_header; + unsigned char *code, *headers; + DWORD locationDelta; + DllEntryProc DllEntry; + BOOL successfull; + MEMORYMODULE *p = loaded; + + while (p) { + // If already loaded, only increment the reference count + if (0 == stricmp(name, p->name)) { + p->refcount++; + return (HMODULE)p; + } + p = p->next; + } + + /* Do NOT check for GetModuleHandle here! */ + + dos_header = (PIMAGE_DOS_HEADER)data; + if (dos_header->e_magic != IMAGE_DOS_SIGNATURE) + { + SetLastError(ERROR_BAD_FORMAT); +#if DEBUG_OUTPUT + OutputDebugString("Not a valid executable file.\n"); +#endif + return NULL; + } + + old_header = (PIMAGE_NT_HEADERS)&((const unsigned char *)(data))[dos_header->e_lfanew]; + if (old_header->Signature != IMAGE_NT_SIGNATURE) + { + SetLastError(ERROR_BAD_FORMAT); +#if DEBUG_OUTPUT + OutputDebugString("No PE header found.\n"); +#endif + return NULL; + } + + // reserve memory for image of library + code = (unsigned char *)VirtualAlloc((LPVOID)(old_header->OptionalHeader.ImageBase), + old_header->OptionalHeader.SizeOfImage, + MEM_RESERVE, + PAGE_READWRITE); + + if (code == NULL) + // try to allocate memory at arbitrary position + code = (unsigned char *)VirtualAlloc(NULL, + old_header->OptionalHeader.SizeOfImage, + MEM_RESERVE, + PAGE_READWRITE); + + if (code == NULL) + { + SetLastError(ERROR_NOT_ENOUGH_MEMORY); +#if DEBUG_OUTPUT + OutputLastError("Can't reserve memory"); +#endif + return NULL; + } + + result = (PMEMORYMODULE)HeapAlloc(GetProcessHeap(), 0, sizeof(MEMORYMODULE)); + result->codeBase = code; + result->numModules = 0; + result->modules = NULL; + result->initialized = 0; + result->next = result->prev = NULL; + result->refcount = 1; + result->name = strdup(name); + result->name_table = NULL; + + // XXX: is it correct to commit the complete memory region at once? + // calling DllEntry raises an exception if we don't... + VirtualAlloc(code, + old_header->OptionalHeader.SizeOfImage, + MEM_COMMIT, + PAGE_READWRITE); + + // commit memory for headers + headers = (unsigned char *)VirtualAlloc(code, + old_header->OptionalHeader.SizeOfHeaders, + MEM_COMMIT, + PAGE_READWRITE); + + // copy PE header to code + memcpy(headers, dos_header, dos_header->e_lfanew + old_header->OptionalHeader.SizeOfHeaders); + result->headers = (PIMAGE_NT_HEADERS)&((const unsigned char *)(headers))[dos_header->e_lfanew]; + + // update position + result->headers->OptionalHeader.ImageBase = (DWORD)code; + + // copy sections from DLL file block to new memory location + CopySections(data, old_header, result); + + // adjust base address of imported data + locationDelta = (DWORD)(code - old_header->OptionalHeader.ImageBase); + if (locationDelta != 0) + PerformBaseRelocation(result, locationDelta); + + // load required dlls and adjust function table of imports + if (!BuildImportTable(result)) + goto error; + + // mark memory pages depending on section headers and release + // sections that are marked as "discardable" + FinalizeSections(result); + + // get entry point of loaded library + if (result->headers->OptionalHeader.AddressOfEntryPoint != 0) + { + DllEntry = (DllEntryProc)(code + result->headers->OptionalHeader.AddressOfEntryPoint); + if (DllEntry == 0) + { + SetLastError(ERROR_BAD_FORMAT); /* XXX ? */ +#if DEBUG_OUTPUT + OutputDebugString("Library has no entry point.\n"); +#endif + goto error; + } + + // notify library about attaching to process + successfull = (*DllEntry)((HINSTANCE)code, DLL_PROCESS_ATTACH, 0); + if (!successfull) + { +#if DEBUG_OUTPUT + OutputDebugString("Can't attach library.\n"); +#endif + goto error; + } + result->initialized = 1; + } + + _Register(name, result); + + return (HMEMORYMODULE)result; + +error: + // cleanup + free(result->name); + MemoryFreeLibrary(result); + return NULL; +} + +int _compare(const struct NAME_TABLE *p1, const struct NAME_TABLE *p2) +{ + return stricmp(p1->name, p2->name); +} + +int _find(const char **name, const struct NAME_TABLE *p) +{ + return stricmp(*name, p->name); +} + +struct NAME_TABLE *GetNameTable(PMEMORYMODULE module) +{ + unsigned char *codeBase; + PIMAGE_EXPORT_DIRECTORY exports; + PIMAGE_DATA_DIRECTORY directory; + DWORD i, *nameRef; + WORD *ordinal; + struct NAME_TABLE *p, *ptab; + + if (module->name_table) + return module->name_table; + + codeBase = module->codeBase; + directory = GET_HEADER_DICTIONARY(module, IMAGE_DIRECTORY_ENTRY_EXPORT); + exports = (PIMAGE_EXPORT_DIRECTORY)(codeBase + directory->VirtualAddress); + + nameRef = (DWORD *)(codeBase + exports->AddressOfNames); + ordinal = (WORD *)(codeBase + exports->AddressOfNameOrdinals); + + p = ((PMEMORYMODULE)module)->name_table = (struct NAME_TABLE *)malloc(sizeof(struct NAME_TABLE) + * exports->NumberOfNames); + if (p == NULL) + return NULL; + ptab = p; + for (i=0; iNumberOfNames; ++i) { + p->name = (char *)(codeBase + *nameRef++); + p->ordinal = *ordinal++; + ++p; + } + qsort(ptab, exports->NumberOfNames, sizeof(struct NAME_TABLE), _compare); + return ptab; +} + +FARPROC MemoryGetProcAddress(HMEMORYMODULE module, const char *name) +{ + unsigned char *codeBase = ((PMEMORYMODULE)module)->codeBase; + int idx=-1; + PIMAGE_EXPORT_DIRECTORY exports; + PIMAGE_DATA_DIRECTORY directory = GET_HEADER_DICTIONARY((PMEMORYMODULE)module, IMAGE_DIRECTORY_ENTRY_EXPORT); + + if (directory->Size == 0) + // no export table found + return NULL; + + exports = (PIMAGE_EXPORT_DIRECTORY)(codeBase + directory->VirtualAddress); + if (exports->NumberOfNames == 0 || exports->NumberOfFunctions == 0) + // DLL doesn't export anything + return NULL; + + if (HIWORD(name)) { + struct NAME_TABLE *ptab; + struct NAME_TABLE *found; + ptab = GetNameTable((PMEMORYMODULE)module); + if (ptab == NULL) + // some failure + return NULL; + found = bsearch(&name, ptab, exports->NumberOfNames, sizeof(struct NAME_TABLE), _find); + if (found == NULL) + // exported symbol not found + return NULL; + + idx = found->ordinal; + } + else + idx = LOWORD(name) - exports->Base; + + if ((DWORD)idx > exports->NumberOfFunctions) + // name <-> ordinal number don't match + return NULL; + + // AddressOfFunctions contains the RVAs to the "real" functions + return (FARPROC)(codeBase + *(DWORD *)(codeBase + exports->AddressOfFunctions + (idx*4))); +} + +void MemoryFreeLibrary(HMEMORYMODULE mod) +{ + int i; + PMEMORYMODULE module = (PMEMORYMODULE)mod; + + if (module != NULL) + { + if (module->initialized != 0) + { + // notify library about detaching from process + DllEntryProc DllEntry = (DllEntryProc)(module->codeBase + module->headers->OptionalHeader.AddressOfEntryPoint); + (*DllEntry)((HINSTANCE)module->codeBase, DLL_PROCESS_DETACH, 0); + module->initialized = 0; + } + + if (module->modules != NULL) + { + // free previously opened libraries + for (i=0; inumModules; i++) + if (module->modules[i] != INVALID_HANDLE_VALUE) + MyFreeLibrary(module->modules[i]); + + free(module->modules); + } + + if (module->codeBase != NULL) + // release memory of library + VirtualFree(module->codeBase, 0, MEM_RELEASE); + + if (module->name_table != NULL) + free(module->name_table); + + HeapFree(GetProcessHeap(), 0, module); + } +} diff --git a/setup/installer/windows/MemoryModule.h b/setup/installer/windows/MemoryModule.h new file mode 100644 index 0000000000..601d4c50df --- /dev/null +++ b/setup/installer/windows/MemoryModule.h @@ -0,0 +1,58 @@ +/* + * Memory DLL loading code + * Version 0.0.2 + * + * Copyright (c) 2004-2005 by Joachim Bauch / mail@joachim-bauch.de + * http://www.joachim-bauch.de + * + * The contents of this file are subject to the Mozilla Public License Version + * 1.1 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * http://www.mozilla.org/MPL/ + * + * Software distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License + * for the specific language governing rights and limitations under the + * License. + * + * The Original Code is MemoryModule.h + * + * The Initial Developer of the Original Code is Joachim Bauch. + * + * Portions created by Joachim Bauch are Copyright (C) 2004-2005 + * Joachim Bauch. All Rights Reserved. + * + */ + +#ifndef __MEMORY_MODULE_HEADER +#define __MEMORY_MODULE_HEADER + +#include + +typedef void *HMEMORYMODULE; + +#ifdef __cplusplus +extern "C" { +#endif + +typedef void *(*FINDPROC)(); + +extern FINDPROC findproc; +extern void *findproc_data; + +HMEMORYMODULE MemoryLoadLibrary(char *, const void *); + +FARPROC MemoryGetProcAddress(HMEMORYMODULE, const char *); + +void MemoryFreeLibrary(HMEMORYMODULE); + +BOOL MyFreeLibrary(HMODULE hModule); +HMODULE MyLoadLibrary(char *lpFileName); +FARPROC MyGetProcAddress(HMODULE hModule, LPCSTR lpProcName); +HMODULE MyGetModuleHandle(LPCTSTR lpModuleName); + +#ifdef __cplusplus +} +#endif + +#endif // __MEMORY_MODULE_HEADER diff --git a/setup/installer/windows/freeze.py b/setup/installer/windows/freeze.py index 7fb60968e7..0fe494e831 100644 --- a/setup/installer/windows/freeze.py +++ b/setup/installer/windows/freeze.py @@ -16,7 +16,6 @@ from setup.installer.windows.wix import WixMixIn OPENSSL_DIR = r'Q:\openssl' QT_DIR = 'Q:\\Qt\\4.7.3' QT_DLLS = ['Core', 'Gui', 'Network', 'Svg', 'WebKit', 'Xml', 'XmlPatterns'] -LIBUSB_DIR = 'C:\\libusb' LIBUNRAR = 'C:\\Program Files\\UnrarDLL\\unrar.dll' SW = r'C:\cygwin\home\kovid\sw' IMAGEMAGICK = os.path.join(SW, 'build', 'ImageMagick-6.6.6', @@ -71,11 +70,13 @@ class Win32Freeze(Command, WixMixIn): self.rc_template = self.j(self.d(self.a(__file__)), 'template.rc') self.py_ver = ''.join(map(str, sys.version_info[:2])) self.lib_dir = self.j(self.base, 'Lib') - self.pydlib = self.j(self.base, 'pydlib') self.pylib = self.j(self.base, 'pylib.zip') + self.dll_dir = self.j(self.base, 'DLLs') + self.plugins_dir = os.path.join(self.base, 'plugins') self.initbase() self.build_launchers() + self.add_plugins() self.freeze() self.embed_manifests() self.install_site_py() @@ -87,18 +88,21 @@ class Win32Freeze(Command, WixMixIn): shutil.rmtree(self.base) os.makedirs(self.base) + def add_plugins(self): + self.info('Adding plugins...') + tgt = self.plugins_dir + if os.path.exists(tgt): + shutil.rmtree(tgt) + os.mkdir(tgt) + base = self.j(self.SRC, 'calibre', 'plugins') + for f in glob.glob(self.j(base, '*.pyd')): + # We dont want the manifests as the manifest in the exe will be + # used instead + shutil.copy2(f, tgt) + def freeze(self): shutil.copy2(self.j(self.src_root, 'LICENSE'), self.base) - self.info('Adding plugins...') - tgt = os.path.join(self.base, 'plugins') - if not os.path.exists(tgt): - os.mkdir(tgt) - base = self.j(self.SRC, 'calibre', 'plugins') - for pat in ('*.pyd', '*.manifest'): - for f in glob.glob(self.j(base, pat)): - shutil.copy2(f, tgt) - self.info('Adding resources...') tgt = self.j(self.base, 'resources') if os.path.exists(tgt): @@ -106,7 +110,6 @@ class Win32Freeze(Command, WixMixIn): shutil.copytree(self.j(self.src_root, 'resources'), tgt) self.info('Adding Qt and python...') - self.dll_dir = self.j(self.base, 'DLLs') shutil.copytree(r'C:\Python%s\DLLs'%self.py_ver, self.dll_dir, ignore=shutil.ignore_patterns('msvc*.dll', 'Microsoft.*')) for x in glob.glob(self.j(OPENSSL_DIR, 'bin', '*.dll')): @@ -197,11 +200,6 @@ class Win32Freeze(Command, WixMixIn): print print 'Adding third party dependencies' - tdir = os.path.join(self.base, 'driver') - os.makedirs(tdir) - for pat in ('*.dll', '*.sys', '*.cat', '*.inf'): - for f in glob.glob(os.path.join(LIBUSB_DIR, pat)): - shutil.copyfile(f, os.path.join(tdir, os.path.basename(f))) print '\tAdding unrar' shutil.copyfile(LIBUNRAR, os.path.join(self.dll_dir, os.path.basename(LIBUNRAR))) @@ -318,8 +316,8 @@ class Win32Freeze(Command, WixMixIn): if not os.path.exists(self.obj_dir): os.makedirs(self.obj_dir) base = self.j(self.src_root, 'setup', 'installer', 'windows') - sources = [self.j(base, x) for x in ['util.c']] - headers = [self.j(base, x) for x in ['util.h']] + sources = [self.j(base, x) for x in ['util.c', 'MemoryModule.c']] + headers = [self.j(base, x) for x in ['util.h', 'MemoryModule.h']] objects = [self.j(self.obj_dir, self.b(x)+'.obj') for x in sources] cflags = '/c /EHsc /MD /W3 /Ox /nologo /D_UNICODE'.split() cflags += ['/DPYDLL="python%s.dll"'%self.py_ver, '/IC:/Python%s/include'%self.py_ver] @@ -371,43 +369,49 @@ class Win32Freeze(Command, WixMixIn): def archive_lib_dir(self): self.info('Putting all python code into a zip file for performance') - if os.path.exists(self.pydlib): - shutil.rmtree(self.pydlib) - os.makedirs(self.pydlib) self.zf_timestamp = time.localtime(time.time())[:6] self.zf_names = set() with zipfile.ZipFile(self.pylib, 'w', zipfile.ZIP_STORED) as zf: + # Add the .pyds from python and calibre to the zip file + for x in (self.plugins_dir, self.dll_dir): + for pyd in os.listdir(x): + if pyd.endswith('.pyd') and pyd != 'sqlite_custom.pyd': + # sqlite_custom has to be a file for + # sqlite_load_extension to work + self.add_to_zipfile(zf, pyd, x) + os.remove(self.j(x, pyd)) + + # Add everything in Lib except site-packages to the zip file for x in os.listdir(self.lib_dir): if x == 'site-packages': continue self.add_to_zipfile(zf, x, self.lib_dir) sp = self.j(self.lib_dir, 'site-packages') - handled = set(['site.pyo']) - for pth in ('PIL.pth', 'pywin32.pth'): - handled.add(pth) - shutil.copyfile(self.j(sp, pth), self.j(self.pydlib, pth)) - for d in self.get_pth_dirs(self.j(sp, pth)): - shutil.copytree(d, self.j(self.pydlib, self.b(d)), True) - handled.add(self.b(d)) + # Special handling for PIL and pywin32 + handled = set(['PIL.pth', 'pywin32.pth', 'PIL', 'win32']) + self.add_to_zipfile(zf, 'PIL', sp) + base = self.j(sp, 'win32', 'lib') + for x in os.listdir(base): + if os.path.splitext(x)[1] not in ('.exe',): + self.add_to_zipfile(zf, x, base) + base = self.d(base) + for x in os.listdir(base): + if not os.path.isdir(self.j(base, x)): + if os.path.splitext(x)[1] not in ('.exe',): + self.add_to_zipfile(zf, x, base) handled.add('easy-install.pth') for d in self.get_pth_dirs(self.j(sp, 'easy-install.pth')): handled.add(self.b(d)) - zip_safe = self.is_zip_safe(d) for x in os.listdir(d): if x == 'EGG-INFO': continue - if zip_safe: - self.add_to_zipfile(zf, x, d) - else: - absp = self.j(d, x) - dest = self.j(self.pydlib, x) - if os.path.isdir(absp): - shutil.copytree(absp, dest, True) - else: - shutil.copy2(absp, dest) + self.add_to_zipfile(zf, x, d) + # The rest of site-packages + # We dont want the site.py from site-packages + handled.add('site.pyo') for x in os.listdir(sp): if x in handled or x.endswith('.egg-info'): continue @@ -415,33 +419,18 @@ class Win32Freeze(Command, WixMixIn): if os.path.isdir(absp): if not os.listdir(absp): continue - if self.is_zip_safe(absp): - self.add_to_zipfile(zf, x, sp) - else: - shutil.copytree(absp, self.j(self.pydlib, x), True) + self.add_to_zipfile(zf, x, sp) else: - if x.endswith('.pyd'): - shutil.copy2(absp, self.j(self.pydlib, x)) - else: - self.add_to_zipfile(zf, x, sp) + self.add_to_zipfile(zf, x, sp) shutil.rmtree(self.lib_dir) - def is_zip_safe(self, path): - for f in walk(path): - ext = os.path.splitext(f)[1].lower() - if ext in ('.pyd', '.dll', '.exe'): - return False - return True - def get_pth_dirs(self, pth): base = os.path.dirname(pth) for line in open(pth).readlines(): line = line.strip() if not line or line.startswith('#') or line.startswith('import'): continue - if line == 'win32\\lib': - continue candidate = self.j(base, line) if os.path.exists(candidate): yield candidate @@ -463,10 +452,10 @@ class Win32Freeze(Command, WixMixIn): self.add_to_zipfile(zf, name + os.sep + x, base) else: ext = os.path.splitext(name)[1].lower() - if ext in ('.pyd', '.dll', '.exe'): + if ext in ('.dll',): raise ValueError('Cannot add %r to zipfile'%abspath) zinfo.external_attr = 0600 << 16 - if ext in ('.py', '.pyc', '.pyo'): + if ext in ('.py', '.pyc', '.pyo', '.pyd'): with open(abspath, 'rb') as f: zf.writestr(zinfo, f.read()) diff --git a/setup/installer/windows/notes.rst b/setup/installer/windows/notes.rst index 11b5bccf79..0bb8b7b15b 100644 --- a/setup/installer/windows/notes.rst +++ b/setup/installer/windows/notes.rst @@ -88,7 +88,9 @@ Qt uses its own routine to locate and load "system libraries" including the open Now, run configure and make:: - configure -opensource -release -qt-zlib -qt-gif -qt-libmng -qt-libpng -qt-libtiff -qt-libjpeg -release -platform win32-msvc2008 -no-qt3support -webkit -xmlpatterns -no-phonon -no-style-plastique -no-style-cleanlooks -no-style-motif -no-style-cde -no-declarative -no-scripttools -no-audio-backend -no-multimedia -no-dbus -no-openvg -no-opengl -no-qt3support -confirm-license -nomake examples -nomake demos -nomake docs -openssl -I Q:\openssl\include -L Q:\openssl\lib && nmake +-no-plugin-manifests is needed so that loading the plugins does not fail looking for the CRT assembly + + configure -opensource -release -qt-zlib -qt-gif -qt-libmng -qt-libpng -qt-libtiff -qt-libjpeg -release -platform win32-msvc2008 -no-qt3support -webkit -xmlpatterns -no-phonon -no-style-plastique -no-style-cleanlooks -no-style-motif -no-style-cde -no-declarative -no-scripttools -no-audio-backend -no-multimedia -no-dbus -no-openvg -no-opengl -no-qt3support -confirm-license -nomake examples -nomake demos -nomake docs -no-plugin-manifests -openssl -I Q:\openssl\include -L Q:\openssl\lib && nmake SIP ----- diff --git a/setup/installer/windows/site.py b/setup/installer/windows/site.py index 5610ff197e..33f2e63585 100644 --- a/setup/installer/windows/site.py +++ b/setup/installer/windows/site.py @@ -1,12 +1,72 @@ #!/usr/bin/env python # vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai -from __future__ import with_statement __license__ = 'GPL v3' __copyright__ = '2009, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import sys, os, linecache +import sys +import os +import zipimport +import _memimporter + +DEBUG_ZIPIMPORT = False + +class ZipExtensionImporter(zipimport.zipimporter): + ''' + Taken, with thanks, from the py2exe source code + ''' + + def __init__(self, *args, **kwargs): + zipimport.zipimporter.__init__(self, *args, **kwargs) + # We know there are no dlls in the zip file, so dont set findproc + # (performance optimization) + #_memimporter.set_find_proc(self.locate_dll_image) + + def find_module(self, fullname, path=None): + result = zipimport.zipimporter.find_module(self, fullname, path) + if result: + return result + fullname = fullname.replace(".", "\\") + if (fullname + '.pyd') in self._files: + return self + return None + + def locate_dll_image(self, name): + # A callback function for_memimporter.import_module. Tries to + # locate additional dlls. Returns the image as Python string, + # or None if not found. + if name in self._files: + return self.get_data(name) + return None + + def load_module(self, fullname): + if sys.modules.has_key(fullname): + mod = sys.modules[fullname] + if DEBUG_ZIPIMPORT: + sys.stderr.write("import %s # previously loaded from zipfile %s\n" % (fullname, self.archive)) + return mod + try: + return zipimport.zipimporter.load_module(self, fullname) + except zipimport.ZipImportError: + pass + initname = "init" + fullname.split(".")[-1] # name of initfunction + filename = fullname.replace(".", "\\") + path = filename + '.pyd' + if path in self._files: + if DEBUG_ZIPIMPORT: + sys.stderr.write("# found %s in zipfile %s\n" % (path, self.archive)) + code = self.get_data(path) + mod = _memimporter.import_module(code, initname, fullname, path) + mod.__file__ = "%s\\%s" % (self.archive, path) + mod.__loader__ = self + if DEBUG_ZIPIMPORT: + sys.stderr.write("import %s # loaded from zipfile %s\n" % (fullname, mod.__file__)) + return mod + raise zipimport.ZipImportError, "can't find module %s" % fullname + + def __repr__(self): + return "<%s object %r>" % (self.__class__.__name__, self.archive) def abs__file__(): @@ -42,42 +102,6 @@ def makepath(*paths): dir = os.path.abspath(os.path.join(*paths)) return dir, os.path.normcase(dir) -def addpackage(sitedir, name): - """Process a .pth file within the site-packages directory: - For each line in the file, either combine it with sitedir to a path, - or execute it if it starts with 'import '. - """ - fullname = os.path.join(sitedir, name) - try: - f = open(fullname, "rU") - except IOError: - return - with f: - for line in f: - if line.startswith("#"): - continue - if line.startswith(("import ", "import\t")): - exec line - continue - line = line.rstrip() - dir, dircase = makepath(sitedir, line) - if os.path.exists(dir): - sys.path.append(dir) - - -def addsitedir(sitedir): - """Add 'sitedir' argument to sys.path if missing and handle .pth files in - 'sitedir'""" - sitedir, sitedircase = makepath(sitedir) - try: - names = os.listdir(sitedir) - except os.error: - return - dotpth = os.extsep + "pth" - names = [name for name in names if name.endswith(dotpth)] - for name in sorted(names): - addpackage(sitedir, name) - def run_entry_point(): bname, mod, func = sys.calibre_basename, sys.calibre_module, sys.calibre_function sys.argv[0] = bname+'.exe' @@ -89,6 +113,10 @@ def main(): sys.setdefaultencoding('utf-8') aliasmbcs() + sys.path_hooks.insert(0, ZipExtensionImporter) + sys.path_importer_cache.clear() + + import linecache def fake_getline(filename, lineno, module_globals=None): return '' linecache.orig_getline = linecache.getline @@ -96,10 +124,11 @@ def main(): abs__file__() - addsitedir(os.path.join(sys.app_dir, 'pydlib')) - add_calibre_vars() + # Needed for pywintypes to be able to load its DLL + sys.path.append(os.path.join(sys.app_dir, 'DLLs')) + return run_entry_point() diff --git a/setup/installer/windows/util.c b/setup/installer/windows/util.c index 329e3bf8c3..4075d7e123 100644 --- a/setup/installer/windows/util.c +++ b/setup/installer/windows/util.c @@ -1,18 +1,130 @@ /* * Copyright 2009 Kovid Goyal + * The memimporter code is taken from the py2exe project */ #include "util.h" + #include #include #include + static char GUI_APP = 0; static char python_dll[] = PYDLL; void set_gui_app(char yes) { GUI_APP = yes; } char is_gui_app() { return GUI_APP; } + +// memimporter {{{ + +#include "MemoryModule.h" + +static char **DLL_Py_PackageContext = NULL; +static PyObject **DLL_ImportError = NULL; +static char module_doc[] = +"Importer which can load extension modules from memory"; + + +static void *memdup(void *ptr, Py_ssize_t size) +{ + void *p = malloc(size); + if (p == NULL) + return NULL; + memcpy(p, ptr, size); + return p; +} + +/* + Be sure to detect errors in FindLibrary - undetected errors lead to + very strange behaviour. +*/ +static void* FindLibrary(char *name, PyObject *callback) +{ + PyObject *result; + char *p; + Py_ssize_t size; + + if (callback == NULL) + return NULL; + result = PyObject_CallFunction(callback, "s", name); + if (result == NULL) { + PyErr_Clear(); + return NULL; + } + if (-1 == PyString_AsStringAndSize(result, &p, &size)) { + PyErr_Clear(); + Py_DECREF(result); + return NULL; + } + p = memdup(p, size); + Py_DECREF(result); + return p; +} + +static PyObject *set_find_proc(PyObject *self, PyObject *args) +{ + PyObject *callback = NULL; + if (!PyArg_ParseTuple(args, "|O:set_find_proc", &callback)) + return NULL; + Py_DECREF((PyObject *)findproc_data); + Py_INCREF(callback); + findproc_data = (void *)callback; + return Py_BuildValue("i", 1); +} + +static PyObject * +import_module(PyObject *self, PyObject *args) +{ + char *data; + int size; + char *initfuncname; + char *modname; + char *pathname; + HMEMORYMODULE hmem; + FARPROC do_init; + + char *oldcontext; + + /* code, initfuncname, fqmodulename, path */ + if (!PyArg_ParseTuple(args, "s#sss:import_module", + &data, &size, + &initfuncname, &modname, &pathname)) + return NULL; + hmem = MemoryLoadLibrary(pathname, data); + if (!hmem) { + PyErr_Format(*DLL_ImportError, + "MemoryLoadLibrary() failed loading %s", pathname); + return NULL; + } + do_init = MemoryGetProcAddress(hmem, initfuncname); + if (!do_init) { + MemoryFreeLibrary(hmem); + PyErr_Format(*DLL_ImportError, + "Could not find function %s in memory loaded pyd", initfuncname); + return NULL; + } + + oldcontext = *DLL_Py_PackageContext; + *DLL_Py_PackageContext = modname; + do_init(); + *DLL_Py_PackageContext = oldcontext; + if (PyErr_Occurred()) + return NULL; + /* Retrieve from sys.modules */ + return PyImport_ImportModule(modname); +} + +static PyMethodDef methods[] = { + { "import_module", import_module, METH_VARARGS, + "import_module(code, initfunc, dllname[, finder]) -> module" }, + { "set_find_proc", set_find_proc, METH_VARARGS }, + { NULL, NULL }, /* Sentinel */ +}; + +// }}} + static int _show_error(const wchar_t *preamble, const wchar_t *msg, const int code) { wchar_t *buf, *cbuf; buf = (wchar_t*)LocalAlloc(LMEM_ZEROINIT, sizeof(wchar_t)* @@ -185,7 +297,7 @@ void initialize_interpreter(wchar_t *outr, wchar_t *errr, char *dummy_argv[1] = {""}; buf = (char*)calloc(MAX_PATH, sizeof(char)); - path = (char*)calloc(3*MAX_PATH, sizeof(char)); + path = (char*)calloc(MAX_PATH, sizeof(char)); if (!buf || !path) ExitProcess(_show_error(L"Out of memory", L"", 1)); sz = GetModuleFileNameA(NULL, buf, MAX_PATH); @@ -198,8 +310,7 @@ void initialize_interpreter(wchar_t *outr, wchar_t *errr, buf[strlen(buf)-1] = '\0'; _snprintf_s(python_home, MAX_PATH, _TRUNCATE, "%s", buf); - _snprintf_s(path, 3*MAX_PATH, _TRUNCATE, "%s\\pylib.zip;%s\\pydlib;%s\\DLLs", - buf, buf, buf); + _snprintf_s(path, MAX_PATH, _TRUNCATE, "%s\\pylib.zip", buf); free(buf); @@ -227,7 +338,10 @@ void initialize_interpreter(wchar_t *outr, wchar_t *errr, if (!flag) ExitProcess(_show_error(L"Failed to get debug flag", L"", 1)); //*flag = 1; - + DLL_Py_PackageContext = (char**)GetProcAddress(dll, "_Py_PackageContext"); + if (!DLL_Py_PackageContext) ExitProcess(_show_error(L"Failed to load _Py_PackageContext from dll", L"", 1)); + DLL_ImportError = (PyObject**)GetProcAddress(dll, "PyExc_ImportError"); + if (!DLL_ImportError) ExitProcess(_show_error(L"Failed to load PyExc_ImportError from dll", L"", 1)); Py_SetProgramName(program_name); Py_SetPythonHome(python_home); @@ -263,6 +377,10 @@ void initialize_interpreter(wchar_t *outr, wchar_t *errr, PyList_SetItem(argv, i, v); } PySys_SetObject("argv", argv); + + findproc = FindLibrary; + Py_InitModule3("_memimporter", methods, module_doc); + } diff --git a/setup/installer/windows/wix-template.xml b/setup/installer/windows/wix-template.xml index 0a85b6fb81..3ebe0882e0 100644 --- a/setup/installer/windows/wix-template.xml +++ b/setup/installer/windows/wix-template.xml @@ -164,10 +164,6 @@ - - - - diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py index 4c7e03e89d..75e25b8453 100644 --- a/src/calibre/customize/builtins.py +++ b/src/calibre/customize/builtins.py @@ -608,9 +608,9 @@ from calibre.devices.edge.driver import EDGE from calibre.devices.teclast.driver import TECLAST_K3, NEWSMY, IPAPYRUS, \ SOVOS, PICO, SUNSTECH_EB700, ARCHOS7O, STASH, WEXLER from calibre.devices.sne.driver import SNE -from calibre.devices.misc import PALMPRE, AVANT, SWEEX, PDNOVEL, \ - GEMEI, VELOCITYMICRO, PDNOVEL_KOBO, LUMIREAD, ALURATEK_COLOR, \ - TREKSTOR, EEEREADER, NEXTBOOK +from calibre.devices.misc import (PALMPRE, AVANT, SWEEX, PDNOVEL, + GEMEI, VELOCITYMICRO, PDNOVEL_KOBO, LUMIREAD, ALURATEK_COLOR, + TREKSTOR, EEEREADER, NEXTBOOK, ADAM) from calibre.devices.folder_device.driver import FOLDER_DEVICE_FOR_CONFIG from calibre.devices.kobo.driver import KOBO from calibre.devices.bambook.driver import BAMBOOK @@ -744,6 +744,7 @@ plugins += [ TREKSTOR, EEEREADER, NEXTBOOK, + ADAM, ITUNES, BOEYE_BEX, BOEYE_BDX, @@ -1231,7 +1232,7 @@ class StoreEpubBudStore(StoreBase): name = 'ePub Bud' description = 'Well, it\'s pretty much just "YouTube for Children\'s eBooks. A not-for-profit organization devoted to brining self published childrens books to the world.' actual_plugin = 'calibre.gui2.store.epubbud_plugin:EpubBudStore' - + drm_free_only = True headquarters = 'US' formats = ['EPUB'] diff --git a/src/calibre/debug.py b/src/calibre/debug.py index 8d65c37bbf..79110d9585 100644 --- a/src/calibre/debug.py +++ b/src/calibre/debug.py @@ -53,6 +53,8 @@ Run an embedded python interpreter. default=False, action='store_true') parser.add_option('-m', '--inspect-mobi', help='Inspect the MOBI file at the specified path', default=None) + parser.add_option('--test-build', help='Test binary modules in build', + action='store_true', default=False) return parser @@ -232,6 +234,9 @@ def main(args=sys.argv): elif opts.inspect_mobi is not None: from calibre.ebooks.mobi.debug import inspect_mobi inspect_mobi(opts.inspect_mobi) + elif opts.test_build: + from calibre.test_build import test + test() else: from calibre import ipython ipython() diff --git a/src/calibre/devices/misc.py b/src/calibre/devices/misc.py index 936faeb32d..2a6a76719d 100644 --- a/src/calibre/devices/misc.py +++ b/src/calibre/devices/misc.py @@ -255,6 +255,28 @@ class EEEREADER(USBMS): VENDOR_NAME = 'LINUX' WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = 'FILE-STOR_GADGET' +class ADAM(USBMS): + + name = 'Notion Ink Adam device interface' + gui_name = 'Adam' + + description = _('Communicate with the Adam tablet') + author = 'Kovid Goyal' + supported_platforms = ['windows', 'osx', 'linux'] + + # Ordered list of supported formats + FORMATS = ['epub', 'pdf', 'doc'] + + VENDOR_ID = [0x0955] + PRODUCT_ID = [0x7100] + BCD = [0x9999] + + EBOOK_DIR_MAIN = 'eBooks' + + VENDOR_NAME = 'NI' + WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = ['ADAM'] + SUPPORTS_SUB_DIRS = True + class NEXTBOOK(USBMS): name = 'Nextbook device interface' diff --git a/src/calibre/gui2/dialogs/tweak_epub.py b/src/calibre/gui2/dialogs/tweak_epub.py index 732d74b77d..e0be9fa1e9 100755 --- a/src/calibre/gui2/dialogs/tweak_epub.py +++ b/src/calibre/gui2/dialogs/tweak_epub.py @@ -7,7 +7,7 @@ __copyright__ = '2010, Kovid Goyal ' __docformat__ = 'restructuredtext en' import os, shutil -from zipfile import ZipFile, ZIP_DEFLATED, ZIP_STORED +from calibre.utils.zipfile import ZipFile, ZIP_DEFLATED, ZIP_STORED from PyQt4.Qt import QDialog diff --git a/src/calibre/gui2/preferences/toolbar.ui b/src/calibre/gui2/preferences/toolbar.ui index 0e601f74a2..51819b0df2 100644 --- a/src/calibre/gui2/preferences/toolbar.ui +++ b/src/calibre/gui2/preferences/toolbar.ui @@ -16,13 +16,28 @@ + + + 75 + true + + - Customize the actions in: + Choose the &toolbar to customize: + + + what + + + 75 + true + + QComboBox::AdjustToMinimumContentsLengthWithIcon diff --git a/src/calibre/gui2/store/amazon_de_plugin.py b/src/calibre/gui2/store/amazon_de_plugin.py index f7b17a2e83..88ccbdbded 100644 --- a/src/calibre/gui2/store/amazon_de_plugin.py +++ b/src/calibre/gui2/store/amazon_de_plugin.py @@ -6,21 +6,23 @@ __license__ = 'GPL 3' __copyright__ = '2011, John Schember ' __docformat__ = 'restructuredtext en' +import re, urllib +from contextlib import closing + +from lxml import html + from PyQt4.Qt import QUrl +from calibre import browser from calibre.gui2 import open_url -from calibre.gui2.store.amazon_plugin import AmazonKindleStore +from calibre.gui2.store import StorePlugin +from calibre.gui2.store.search_result import SearchResult -class AmazonDEKindleStore(AmazonKindleStore): +class AmazonDEKindleStore(StorePlugin): ''' For comments on the implementation, please see amazon_plugin.py ''' - search_url = 'http://www.amazon.de/s/?url=search-alias%3Ddigital-text&field-keywords=' - details_url = 'http://amazon.de/dp/' - drm_search_text = u'Gleichzeitige Verwendung von Geräten' - drm_free_text = u'Keine Einschränkung' - def open(self, parent=None, detail_item=None, external=False): aff_id = {'tag': 'charhale0a-21'} store_link = ('http://www.amazon.de/gp/redirect.html?ie=UTF8&site-redirect=de' @@ -32,3 +34,94 @@ class AmazonDEKindleStore(AmazonKindleStore): '&location=http://www.amazon.de/dp/%(asin)s&site-redirect=de' '&tag=%(tag)s&linkCode=ur2&camp=1638&creative=6742') % aff_id open_url(QUrl(store_link)) + + def search(self, query, max_results=10, timeout=60): + search_url = 'http://www.amazon.de/s/?url=search-alias%3Ddigital-text&field-keywords=' + url = search_url + urllib.quote_plus(query) + br = browser() + + counter = max_results + with closing(br.open(url, timeout=timeout)) as f: + doc = html.fromstring(f.read()) + + # Amazon has two results pages. + is_shot = doc.xpath('boolean(//div[@id="shotgunMainResults"])') + # Horizontal grid of books. + if is_shot: + data_xpath = '//div[contains(@class, "result")]' + format_xpath = './/div[@class="productTitle"]/text()' + cover_xpath = './/div[@class="productTitle"]//img/@src' + # Vertical list of books. + else: + data_xpath = '//div[@class="productData"]' + format_xpath = './/span[@class="format"]/text()' + cover_xpath = '../div[@class="productImage"]/a/img/@src' + + for data in doc.xpath(data_xpath): + if counter <= 0: + break + + # Even though we are searching digital-text only Amazon will still + # put in results for non Kindle books (author pages). Se we need + # to explicitly check if the item is a Kindle book and ignore it + # if it isn't. + format = ''.join(data.xpath(format_xpath)) + if 'kindle' not in format.lower(): + continue + + # We must have an asin otherwise we can't easily reference the + # book later. + asin_href = None + asin_a = data.xpath('.//div[@class="productTitle"]/a[1]') + if asin_a: + asin_href = asin_a[0].get('href', '') + m = re.search(r'/dp/(?P.+?)(/|$)', asin_href) + if m: + asin = m.group('asin') + else: + continue + else: + continue + + cover_url = ''.join(data.xpath(cover_xpath)) + + title = ''.join(data.xpath('.//div[@class="productTitle"]/a/text()')) + price = ''.join(data.xpath('.//div[@class="newPrice"]/span/text()')) + + if is_shot: + author = format.split(' von ')[-1] + else: + author = ''.join(data.xpath('.//div[@class="productTitle"]/span[@class="ptBrand"]/text()')) + author = author.split(' von ')[-1] + + counter -= 1 + + s = SearchResult() + s.cover_url = cover_url.strip() + s.title = title.strip() + s.author = author.strip() + s.price = price.strip() + s.detail_item = asin.strip() + s.formats = 'Kindle' + + yield s + + def get_details(self, search_result, timeout): + drm_search_text = u'Gleichzeitige Verwendung von Geräten' + drm_free_text = u'Keine Einschränkung' + url = 'http://amazon.de/dp/' + + br = browser() + with closing(br.open(url + search_result.detail_item, timeout=timeout)) as nf: + idata = html.fromstring(nf.read()) + if idata.xpath('boolean(//div[@class="content"]//li/b[contains(text(), "' + + drm_search_text + '")])'): + if idata.xpath('boolean(//div[@class="content"]//li[contains(., "' + + drm_free_text + '") and contains(b, "' + + drm_search_text + '")])'): + search_result.drm = SearchResult.DRM_UNLOCKED + else: + search_result.drm = SearchResult.DRM_UNKNOWN + else: + search_result.drm = SearchResult.DRM_LOCKED + return True diff --git a/src/calibre/gui2/store/amazon_uk_plugin.py b/src/calibre/gui2/store/amazon_uk_plugin.py index a922f0516b..f8686d19fe 100644 --- a/src/calibre/gui2/store/amazon_uk_plugin.py +++ b/src/calibre/gui2/store/amazon_uk_plugin.py @@ -15,17 +15,14 @@ from PyQt4.Qt import QUrl from calibre import browser from calibre.gui2 import open_url -from calibre.gui2.store.amazon_plugin import AmazonKindleStore +from calibre.gui2.store import StorePlugin from calibre.gui2.store.search_result import SearchResult -class AmazonUKKindleStore(AmazonKindleStore): +class AmazonUKKindleStore(StorePlugin): ''' For comments on the implementation, please see amazon_plugin.py ''' - search_url = 'http://www.amazon.co.uk/s/?url=search-alias%3Ddigital-text&field-keywords=' - details_url = 'http://amazon.co.uk/dp/' - def open(self, parent=None, detail_item=None, external=False): aff_id = {'tag': 'calcharles-21'} store_link = 'http://www.amazon.co.uk/gp/redirect.html?ie=UTF8&location=http://www.amazon.co.uk/Kindle-eBooks/b?ie=UTF8&node=341689031&ref_=sa_menu_kbo2&tag=%(tag)s&linkCode=ur2&camp=1634&creative=19450' % aff_id @@ -36,7 +33,8 @@ class AmazonUKKindleStore(AmazonKindleStore): open_url(QUrl(store_link)) def search(self, query, max_results=10, timeout=60): - url = self.search_url + urllib.quote_plus(query) + search_url = 'http://www.amazon.co.uk/s/?url=search-alias%3Ddigital-text&field-keywords=' + url = search_url + urllib.quote_plus(query) br = browser() counter = max_results @@ -95,7 +93,9 @@ class AmazonUKKindleStore(AmazonKindleStore): if search_result.drm: return - url = self.details_url + url = 'http://amazon.co.uk/dp/' + drm_search_text = u'Simultaneous Device Usage' + drm_free_text = u'Unlimited' br = browser() with closing(br.open(url + search_result.detail_item, timeout=timeout)) as nf: @@ -106,10 +106,10 @@ class AmazonUKKindleStore(AmazonKindleStore): if is_kindle: search_result.formats = 'Kindle' if idata.xpath('boolean(//div[@class="content"]//li/b[contains(text(), "' + - self.drm_search_text + '")])'): + drm_search_text + '")])'): if idata.xpath('boolean(//div[@class="content"]//li[contains(., "' + - self.drm_free_text + '") and contains(b, "' + - self.drm_search_text + '")])'): + drm_free_text + '") and contains(b, "' + + drm_search_text + '")])'): search_result.drm = SearchResult.DRM_UNLOCKED else: search_result.drm = SearchResult.DRM_UNKNOWN diff --git a/src/calibre/manual/faq.rst b/src/calibre/manual/faq.rst index 99c53e5a37..c1aa4e5614 100644 --- a/src/calibre/manual/faq.rst +++ b/src/calibre/manual/faq.rst @@ -562,6 +562,16 @@ You have two choices: 1. Create a patch by hacking on |app| and send it to me for review and inclusion. See `Development `_. 2. `Open a ticket `_ (you have to register and login first). Remember that |app| development is done by volunteers, so if you get no response to your feature request, it means no one feels like implementing it. +Why doesn't |app| have an automatic update? +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +For many reasons: + + * *There is no need to update every week*. If you are happy with how |app| works turn off the update notification and be on your merry way. Check back to see if you want to update once a year or so. + * Pre downloading the updates for all users in the background would mean require about 80TB of bandwidth *every week*. That costs thousands of dollars a month. And |app| is currently growing at 300,000 new users every month. + * If I implement a dialog that downloads the update and launches it, instead of going to the website as it does now, that would save the most ardent |app| updater, *at most five clicks a week*. There are far higher priority things to do in |app| development. + * If you really, really hate downloading |app| every week but still want to be upto the latest, I encourage you to run from source, which makes updating trivial. Instructions are :ref:`here `. + How is |app| licensed? ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ |app| is licensed under the GNU General Public License v3 (an open source license). This means that you are free to redistribute |app| as long as you make the source code available. So if you want to put |app| on a CD with your product, you must also put the |app| source code on the CD. The source code is available for download `from googlecode `_. You are free to use the results of conversions from |app| however you want. You cannot use code, libraries from |app| in your software without making your software open source. For details, see `The GNU GPL v3 `_. diff --git a/src/calibre/test_build.py b/src/calibre/test_build.py new file mode 100644 index 0000000000..0610f01805 --- /dev/null +++ b/src/calibre/test_build.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +from __future__ import (unicode_literals, division, absolute_import, + print_function) +from future_builtins import map + +__license__ = 'GPL v3' +__copyright__ = '2011, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + +''' +Test a binary calibre build to ensure that all needed binary images/libraries have loaded. +''' + +import cStringIO +from calibre.constants import plugins, iswindows + +def test_plugins(): + for name in plugins: + mod, err = plugins[name] + if err or not mod: + raise RuntimeError('Plugin %s failed to load with error: %s' % + (name, err)) + print (mod, 'loaded') + +def test_lxml(): + from lxml import etree + raw = '' + root = etree.fromstring(raw) + if etree.tostring(root) == raw: + print ('lxml OK!') + else: + raise RuntimeError('lxml failed') + +def test_fontconfig(): + from calibre.utils.fonts import fontconfig + families = fontconfig.find_font_families() + num = len(families) + if num < 10: + raise RuntimeError('Fontconfig found only %d font families'%num) + print ('Fontconfig OK! (%d families)'%num) + +def test_winutil(): + from calibre.devices.scanner import win_pnp_drives + matches = win_pnp_drives.scanner() + if len(matches) < 1: + raise RuntimeError('win_pnp_drives returned no drives') + print ('win_pnp_drives OK!') + +def test_win32(): + from calibre.utils.winshell import desktop + d = desktop() + if not d: + raise RuntimeError('winshell failed') + print ('winshell OK! (%s is the desktop)'%d) + +def test_sqlite(): + import sqlite3 + conn = sqlite3.connect(':memory:') + from calibre.library.sqlite import load_c_extensions + if not load_c_extensions(conn, True): + raise RuntimeError('Failed to load sqlite extension') + print ('sqlite OK!') + +def test_qt(): + from PyQt4.Qt import (QWebView, QDialog, QImageReader, QNetworkAccessManager) + fmts = set(map(unicode, QImageReader.supportedImageFormats())) + if 'jpg' not in fmts or 'png' not in fmts: + raise RuntimeError( + "Qt doesn't seem to be able to load its image plugins") + QWebView, QDialog + na = QNetworkAccessManager() + if not hasattr(na, 'sslErrors'): + raise RuntimeError('Qt not compiled with openssl') + print ('Qt OK!') + +def test_imaging(): + from calibre.utils.magick.draw import create_canvas, Image + im = create_canvas(20, 20, '#ffffff') + jpg = im.export('jpg') + Image().load(jpg) + im.export('png') + print ('ImageMagick OK!') + from PIL import Image + i = Image.open(cStringIO.StringIO(jpg)) + if i.size != (20, 20): + raise RuntimeError('PIL choked!') + print ('PIL OK!') + +def test(): + test_plugins() + test_lxml() + test_fontconfig() + test_sqlite() + if iswindows: + test_winutil() + test_win32() + test_qt() + test_imaging() + +if __name__ == '__main__': + test() + diff --git a/src/calibre/utils/zipfile.py b/src/calibre/utils/zipfile.py index 394b667bbf..868445d15a 100644 --- a/src/calibre/utils/zipfile.py +++ b/src/calibre/utils/zipfile.py @@ -148,6 +148,12 @@ def decode_arcname(name): name = name.decode('utf-8', 'replace') return name +# Added by Kovid to reset timestamp to default if it overflows the DOS +# limits +def fixtimevar(val): + if val < 0 or val > 0xffff: + val = 0 + return val def _check_zipfile(fp): try: @@ -340,13 +346,7 @@ class ZipInfo (object): """Return the per-file header as a string.""" dt = self.date_time dosdate = (dt[0] - 1980) << 9 | dt[1] << 5 | dt[2] - # Added by Kovid to reset timestamp to default if it overflows the DOS - # limits - if dosdate > 0xffff or dosdate < 0: - dosdate = 0 dostime = dt[3] << 11 | dt[4] << 5 | (dt[5] // 2) - if dostime > 0xffff or dostime < 0: - dostime = 0 if self.flag_bits & 0x08: # Set these to zero because we write them after the file data @@ -372,7 +372,7 @@ class ZipInfo (object): filename, flag_bits = self._encodeFilenameFlags() header = struct.pack(structFileHeader, stringFileHeader, self.extract_version, self.reserved, flag_bits, - self.compress_type, dostime, dosdate, CRC, + self.compress_type, fixtimevar(dostime), fixtimevar(dosdate), CRC, compress_size, file_size, len(filename), len(extra)) return header + filename + extra @@ -1328,8 +1328,8 @@ class ZipFile: for zinfo in self.filelist: # write central directory count = count + 1 dt = zinfo.date_time - dosdate = (dt[0] - 1980) << 9 | dt[1] << 5 | dt[2] - dostime = dt[3] << 11 | dt[4] << 5 | (dt[5] // 2) + dosdate = fixtimevar((dt[0] - 1980) << 9 | dt[1] << 5 | dt[2]) + dostime = fixtimevar(dt[3] << 11 | dt[4] << 5 | (dt[5] // 2)) extra = [] if zinfo.file_size > ZIP64_LIMIT \ or zinfo.compress_size > ZIP64_LIMIT: