diff --git a/recipes/brigitte_de.recipe b/recipes/brigitte_de.recipe new file mode 100644 index 0000000000..860d5176ac --- /dev/null +++ b/recipes/brigitte_de.recipe @@ -0,0 +1,36 @@ +from calibre.web.feeds.news import BasicNewsRecipe + +class AdvancedUserRecipe(BasicNewsRecipe): + + title = u'Brigitte.de' + __author__ = 'schuster' + oldest_article = 14 + max_articles_per_feed = 100 + no_stylesheets = True + use_embedded_content = False + language = 'de' + remove_javascript = True + remove_empty_feeds = True + timeout = 10 + cover_url = 'http://www.medienmilch.de/typo3temp/pics/Brigitte-Logo_d5feb4a6e4.jpg' + masthead_url = 'http://www.medienmilch.de/typo3temp/pics/Brigitte-Logo_d5feb4a6e4.jpg' + + + remove_tags = [dict(attrs={'class':['linklist', 'head', 'indent right relatedContent', 'artikel-meta segment', 'segment', 'comment commentFormWrapper segment borderBG', 'segment borderBG comments', 'segment borderBG box', 'center', 'segment nextPageLink', 'inCar']}), + dict(id=['header', 'artTools', 'context', 'interact', 'footer-navigation', 'bwNet', 'copy', 'keyboardNavigationHint']), + dict(name=['hjtrs', 'kud'])] + + feeds = [(u'Mode', u'http://www.brigitte.de/mode/feed.rss'), + (u'Beauty', u'http://www.brigitte.de/beauty/feed.rss'), + (u'Luxus', u'http://www.brigitte.de/luxus/feed.rss'), + (u'Figur', u'http://www.brigitte.de/figur/feed.rss'), + (u'Gesundheit', u'http://www.brigitte.de/gesundheit/feed.rss'), + (u'Liebe&Sex', u'http://www.brigitte.de/liebe-sex/feed.rss'), + (u'Gesellschaft', u'http://www.brigitte.de/gesellschaft/feed.rss'), + (u'Kultur', u'http://www.brigitte.de/kultur/feed.rss'), + (u'Reise', u'http://www.brigitte.de/reise/feed.rss'), + (u'Kochen', u'http://www.brigitte.de/kochen/feed.rss'), + (u'Wohnen', u'http://www.brigitte.de/wohnen/feed.rss'), + (u'Job', u'http://www.brigitte.de/job/feed.rss'), + (u'Erfahrungen', u'http://www.brigitte.de/erfahrungen/feed.rss'), +] diff --git a/recipes/express_de.recipe b/recipes/express_de.recipe index 255538b08e..10595b9d92 100644 --- a/recipes/express_de.recipe +++ b/recipes/express_de.recipe @@ -1,5 +1,4 @@ from calibre.web.feeds.news import BasicNewsRecipe - class AdvancedUserRecipe1303841067(BasicNewsRecipe): title = u'Express.de' @@ -12,7 +11,6 @@ class AdvancedUserRecipe1303841067(BasicNewsRecipe): extra_css = ''' h2{font-family:Arial,Helvetica,sans-serif; font-size: x-small;} h1{ font-family:Arial,Helvetica,sans-serif; font-size:x-large; font-weight:bold;} - ''' remove_javascript = True remove_tags_befor = [dict(name='div', attrs={'class':'Datum'})] @@ -25,6 +23,7 @@ class AdvancedUserRecipe1303841067(BasicNewsRecipe): dict(id='Logo'), dict(id='MainLinkSpacer'), dict(id='MainLinks'), + dict(id='ContainerPfad'), #neu dict(title='Diese Seite Bookmarken'), dict(name='span'), @@ -44,7 +43,8 @@ class AdvancedUserRecipe1303841067(BasicNewsRecipe): dict(name='div', attrs={'class':'HeaderSearch'}), dict(name='div', attrs={'class':'sbutton'}), dict(name='div', attrs={'class':'active'}), - + dict(name='div', attrs={'class':'MoreNews'}), #neu + dict(name='div', attrs={'class':'ContentBoxSubline'}) #neu ] @@ -68,7 +68,5 @@ class AdvancedUserRecipe1303841067(BasicNewsRecipe): (u'Fortuna D~Dorf', u'http://www.express.de/sport/fussball/fortuna/-/3292/3292/-/view/asFeed/-/index.xml'), (u'Basketball News', u'http://www.express.de/sport/basketball/-/3190/3190/-/view/asFeed/-/index.xml'), (u'Big Brother', u'http://www.express.de/news/promi-show/big-brother/-/2402/2402/-/view/asFeed/-/index.xml'), + ] - - -] diff --git a/recipes/heise_online.recipe b/recipes/heise_online.recipe new file mode 100644 index 0000000000..f83ff8126b --- /dev/null +++ b/recipes/heise_online.recipe @@ -0,0 +1,52 @@ +from calibre.web.feeds.news import BasicNewsRecipe +class AdvancedUserRecipe(BasicNewsRecipe): + + title = 'Heise-online' + description = 'News vom Heise-Verlag' + __author__ = 'schuster' + use_embedded_content = False + language = 'de' + oldest_article = 2 + max_articles_per_feed = 35 + rescale_images = True + remove_empty_feeds = True + timeout = 5 + no_stylesheets = True + + + remove_tags_after = dict(name ='p', attrs={'class':'editor'}) + remove_tags = [dict(id='navi_top_container'), + dict(id='navi_bottom'), + dict(id='mitte_rechts'), + dict(id='navigation'), + dict(id='subnavi'), + dict(id='social_bookmarks'), + dict(id='permalink'), + dict(id='content_foren'), + dict(id='seiten_navi'), + dict(id='adbottom'), + dict(id='sitemap')] + + feeds = [ + ('Newsticker', 'http://www.heise.de/newsticker/heise.rdf'), + ('Auto', 'http://www.heise.de/autos/rss/news.rdf'), + ('Foto ', 'http://www.heise.de/foto/rss/news-atom.xml'), + ('Mac&i', 'http://www.heise.de/mac-and-i/news.rdf'), + ('Mobile ', 'http://www.heise.de/mobil/newsticker/heise-atom.xml'), + ('Netz ', 'http://www.heise.de/netze/rss/netze-atom.xml'), + ('Open ', 'http://www.heise.de/open/news/news-atom.xml'), + ('Resale ', 'http://www.heise.de/resale/rss/resale.rdf'), + ('Security ', 'http://www.heise.de/security/news/news-atom.xml'), + ('C`t', 'http://www.heise.de/ct/rss/artikel-atom.xml'), + ('iX', 'http://www.heise.de/ix/news/news.rdf'), + ('Mach-flott', 'http://www.heise.de/mach-flott/rss/mach-flott-atom.xml'), + ('Blog: Babel-Bulletin', 'http://www.heise.de/developer/rss/babel-bulletin/blog.rdf'), + ('Blog: Der Dotnet-Doktor', 'http://www.heise.de/developer/rss/dotnet-doktor/blog.rdf'), + ('Blog: Bernds Management-Welt', 'http://www.heise.de/developer/rss/bernds-management-welt/blog.rdf'), + ('Blog: IT conversation', 'http://www.heise.de/developer/rss/world-of-it/blog.rdf'), + ('Blog: Kais bewegtes Web', 'http://www.heise.de/developer/rss/kais-bewegtes-web/blog.rdf') +] + + def print_version(self, url): + return url + '?view=print' + diff --git a/recipes/max_planck.recipe b/recipes/max_planck.recipe index e9bf62008a..cf778a7374 100644 --- a/recipes/max_planck.recipe +++ b/recipes/max_planck.recipe @@ -3,9 +3,6 @@ class AdvancedUserRecipe1303841067(BasicNewsRecipe): title = u'Max-Planck-Inst.' __author__ = 'schuster' - remove_tags = [dict(attrs={'class':['clearfix', 'lens', 'col2_box_list', 'col2_box_teaser group_ext no_print', 'dotted_line', 'col2_box_teaser', 'box_image small', 'bold', 'col2_box_teaser no_print', 'print_kontakt']}), - dict(id=['ie_clearing', 'col2', 'col2_content']), - dict(name=['script', 'noscript', 'style'])] oldest_article = 30 max_articles_per_feed = 100 no_stylesheets = True @@ -13,6 +10,11 @@ class AdvancedUserRecipe1303841067(BasicNewsRecipe): language = 'de' remove_javascript = True + remove_tags = [dict(attrs={'class':['box_url', 'print_kontakt']}), + dict(id=['skiplinks'])] + + + def print_version(self, url): split_url = url.split("/") print_url = 'http://www.mpg.de/print/' + split_url[3] diff --git a/recipes/newsweek.recipe b/recipes/newsweek.recipe index a31706e257..0cae4275b0 100644 --- a/recipes/newsweek.recipe +++ b/recipes/newsweek.recipe @@ -69,7 +69,11 @@ class Newsweek(BasicNewsRecipe): for section, shref in self.newsweek_sections(): self.log('Processing section', section, shref) articles = [] - soups = [self.index_to_soup(shref)] + try: + soups = [self.index_to_soup(shref)] + except: + self.log.warn('Section %s not found, skipping'%section) + continue na = soups[0].find('a', rel='next') if na: soups.append(self.index_to_soup(self.BASE_URL+na['href'])) diff --git a/recipes/polizeipress_de.recipe b/recipes/polizeipress_de.recipe new file mode 100644 index 0000000000..15114881ea --- /dev/null +++ b/recipes/polizeipress_de.recipe @@ -0,0 +1,35 @@ +from calibre.web.feeds.news import BasicNewsRecipe + +class AdvancedUserRecipe(BasicNewsRecipe): + + title = u'Polizeipresse - Deutschland' + __author__ = 'schuster' + description = 'Tagesaktuelle "Polizeiberichte" aus ganz Deutschland (bis auf Ortsebene).' 'Um deinen Ort/Stadt/Kreis usw. einzubinden, gehe auf "http://www.presseportal.de/polizeipresse/" und suche im oberen "Suchfeld" nach dem Namen.' 'Oberhalb der Suchergebnisse (Folgen:) auf den üblichen link zu den RSS-Feeds klicken und den RSS-link im Rezept unter "feeds" eintragen wie üblich.' 'Die Auswahl von Orten kann vereinfacht werden wenn man den Suchbegriff wie folgt eingibt:' '"Stadt-Ort".' + oldest_article = 21 + max_articles_per_feed = 100 + no_stylesheets = True + use_embedded_content = False + language = 'de' + remove_javascript = True + masthead_url = 'http://www.alt-heliservice.de/images/34_BPOL_Logo_4C_g_schutzbereich.jpg' + cover_url = 'http://berlinstadtservice.de/buerger/Bundespolizei-Logo.png' + + remove_tags = [ + dict(name='div', attrs={'id':'logo'}), + dict(name='div', attrs={'id':'origin'}), + dict(name='pre', attrs={'class':'xml_contact'})] + + def print_version(self,url): + segments = url.split('/') + printURL = 'http://www.presseportal.de/print.htx?nr=' + '/'.join(segments[5:6]) + '&type=polizei' + return printURL + + feeds = [(u'Frimmerdorf', u'http://www.presseportal.de/rss/rss2_vts.htx?q=Grevenbroich-frimmersdorf&w=public_service'), + (u'Neurath', u'http://www.presseportal.de/rss/rss2_vts.htx?q=Grevenbroich-neurath&w=public_service'), + (u'Gustorf', u'http://www.presseportal.de/rss/rss2_vts.htx?q=Grevenbroich-gustorf&w=public_service'), + (u'Neuenhausen', u'http://www.presseportal.de/rss/rss2_vts.htx?q=Grevenbroich-neuenhausen&w=public_service'), + (u'Wevelinghoven', u'http://www.presseportal.de/rss/rss2_vts.htx?q=Grevenbroich-Wevelinghoven&w=public_service'), + (u'Grevenbroich ges.', u'http://www.presseportal.de/rss/rss2_vts.htx?q=grevenbroich&w=public_service'), + (u'Kreis Neuss ges.', u'http://www.presseportal.de/rss/rss2_vts.htx?q=Rhein-Kreis+Neuss&w=public_service'), + ] + diff --git a/setup/installer/windows/MemoryModule.c b/setup/installer/windows/MemoryModule.c new file mode 100644 index 0000000000..253c8d7d9f --- /dev/null +++ b/setup/installer/windows/MemoryModule.c @@ -0,0 +1,689 @@ +/* + * Memory DLL loading code + * Version 0.0.2 with additions from Thomas Heller + * + * Copyright (c) 2004-2005 by Joachim Bauch / mail@joachim-bauch.de + * http://www.joachim-bauch.de + * + * The contents of this file are subject to the Mozilla Public License Version + * 1.1 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * http://www.mozilla.org/MPL/ + * + * Software distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License + * for the specific language governing rights and limitations under the + * License. + * + * The Original Code is MemoryModule.c + * + * The Initial Developer of the Original Code is Joachim Bauch. + * + * Portions created by Joachim Bauch are Copyright (C) 2004-2005 + * Joachim Bauch. All Rights Reserved. + * + * Portions Copyright (C) 2005 Thomas Heller. + * + */ + +// disable warnings about pointer <-> DWORD conversions +#pragma warning( disable : 4311 4312 ) + +#include +#include +#if DEBUG_OUTPUT +#include +#endif + +#ifndef IMAGE_SIZEOF_BASE_RELOCATION +// Vista SDKs no longer define IMAGE_SIZEOF_BASE_RELOCATION!? +# define IMAGE_SIZEOF_BASE_RELOCATION (sizeof(IMAGE_BASE_RELOCATION)) +#endif +#include "MemoryModule.h" + +/* + XXX We need to protect at least walking the 'loaded' linked list with a lock! +*/ + +/******************************************************************/ +FINDPROC findproc; +void *findproc_data = NULL; + +struct NAME_TABLE { + char *name; + DWORD ordinal; +}; + +typedef struct tagMEMORYMODULE { + PIMAGE_NT_HEADERS headers; + unsigned char *codeBase; + HMODULE *modules; + int numModules; + int initialized; + + struct NAME_TABLE *name_table; + + char *name; + int refcount; + struct tagMEMORYMODULE *next, *prev; +} MEMORYMODULE, *PMEMORYMODULE; + +typedef BOOL (WINAPI *DllEntryProc)(HINSTANCE hinstDLL, DWORD fdwReason, LPVOID lpReserved); + +#define GET_HEADER_DICTIONARY(module, idx) &(module)->headers->OptionalHeader.DataDirectory[idx] + +MEMORYMODULE *loaded; /* linked list of loaded memory modules */ + +/* private - insert a loaded library in a linked list */ +static void _Register(char *name, MEMORYMODULE *module) +{ + module->next = loaded; + if (loaded) + loaded->prev = module; + module->prev = NULL; + loaded = module; +} + +/* private - remove a loaded library from a linked list */ +static void _Unregister(MEMORYMODULE *module) +{ + free(module->name); + if (module->prev) + module->prev->next = module->next; + if (module->next) + module->next->prev = module->prev; + if (module == loaded) + loaded = module->next; +} + +/* public - replacement for GetModuleHandle() */ +HMODULE MyGetModuleHandle(LPCTSTR lpModuleName) +{ + MEMORYMODULE *p = loaded; + while (p) { + // If already loaded, only increment the reference count + if (0 == stricmp(lpModuleName, p->name)) { + return (HMODULE)p; + } + p = p->next; + } + return GetModuleHandle(lpModuleName); +} + +/* public - replacement for LoadLibrary, but searches FIRST for memory + libraries, then for normal libraries. So, it will load libraries AS memory + module if they are found by findproc(). +*/ +HMODULE MyLoadLibrary(char *lpFileName) +{ + MEMORYMODULE *p = loaded; + HMODULE hMod; + + while (p) { + // If already loaded, only increment the reference count + if (0 == stricmp(lpFileName, p->name)) { + p->refcount++; + return (HMODULE)p; + } + p = p->next; + } + if (findproc && findproc_data) { + void *pdata = findproc(lpFileName, findproc_data); + if (pdata) { + hMod = MemoryLoadLibrary(lpFileName, pdata); + free(p); + return hMod; + } + } + hMod = LoadLibrary(lpFileName); + return hMod; +} + +/* public - replacement for GetProcAddress() */ +FARPROC MyGetProcAddress(HMODULE hModule, LPCSTR lpProcName) +{ + MEMORYMODULE *p = loaded; + while (p) { + if ((HMODULE)p == hModule) + return MemoryGetProcAddress(p, lpProcName); + p = p->next; + } + return GetProcAddress(hModule, lpProcName); +} + +/* public - replacement for FreeLibrary() */ +BOOL MyFreeLibrary(HMODULE hModule) +{ + MEMORYMODULE *p = loaded; + while (p) { + if ((HMODULE)p == hModule) { + if (--p->refcount == 0) { + _Unregister(p); + MemoryFreeLibrary(p); + } + return TRUE; + } + p = p->next; + } + return FreeLibrary(hModule); +} + +#if DEBUG_OUTPUT +static void +OutputLastError(const char *msg) +{ + LPVOID tmp; + char *tmpmsg; + FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, + NULL, GetLastError(), MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), (LPTSTR)&tmp, 0, NULL); + tmpmsg = (char *)LocalAlloc(LPTR, strlen(msg) + strlen(tmp) + 3); + sprintf(tmpmsg, "%s: %s", msg, tmp); + OutputDebugString(tmpmsg); + LocalFree(tmpmsg); + LocalFree(tmp); +} +#endif + +/* +static int dprintf(char *fmt, ...) +{ + char Buffer[4096]; + va_list marker; + int result; + + va_start(marker, fmt); + result = vsprintf(Buffer, fmt, marker); + OutputDebugString(Buffer); + return result; +} +*/ + +static void +CopySections(const unsigned char *data, PIMAGE_NT_HEADERS old_headers, PMEMORYMODULE module) +{ + int i, size; + unsigned char *codeBase = module->codeBase; + unsigned char *dest; + PIMAGE_SECTION_HEADER section = IMAGE_FIRST_SECTION(module->headers); + for (i=0; iheaders->FileHeader.NumberOfSections; i++, section++) + { + if (section->SizeOfRawData == 0) + { + // section doesn't contain data in the dll itself, but may define + // uninitialized data + size = old_headers->OptionalHeader.SectionAlignment; + if (size > 0) + { + dest = (unsigned char *)VirtualAlloc(codeBase + section->VirtualAddress, + size, + MEM_COMMIT, + PAGE_READWRITE); + + section->Misc.PhysicalAddress = (DWORD)dest; + memset(dest, 0, size); + } + + // section is empty + continue; + } + + // commit memory block and copy data from dll + dest = (unsigned char *)VirtualAlloc(codeBase + section->VirtualAddress, + section->SizeOfRawData, + MEM_COMMIT, + PAGE_READWRITE); + memcpy(dest, data + section->PointerToRawData, section->SizeOfRawData); + section->Misc.PhysicalAddress = (DWORD)dest; + } +} + +// Protection flags for memory pages (Executable, Readable, Writeable) +static int ProtectionFlags[2][2][2] = { + { + // not executable + {PAGE_NOACCESS, PAGE_WRITECOPY}, + {PAGE_READONLY, PAGE_READWRITE}, + }, { + // executable + {PAGE_EXECUTE, PAGE_EXECUTE_WRITECOPY}, + {PAGE_EXECUTE_READ, PAGE_EXECUTE_READWRITE}, + }, +}; + +static void +FinalizeSections(PMEMORYMODULE module) +{ + int i; + PIMAGE_SECTION_HEADER section = IMAGE_FIRST_SECTION(module->headers); + + // loop through all sections and change access flags + for (i=0; iheaders->FileHeader.NumberOfSections; i++, section++) + { + DWORD protect, oldProtect, size; + int executable = (section->Characteristics & IMAGE_SCN_MEM_EXECUTE) != 0; + int readable = (section->Characteristics & IMAGE_SCN_MEM_READ) != 0; + int writeable = (section->Characteristics & IMAGE_SCN_MEM_WRITE) != 0; + + if (section->Characteristics & IMAGE_SCN_MEM_DISCARDABLE) + { + // section is not needed any more and can safely be freed + VirtualFree((LPVOID)section->Misc.PhysicalAddress, section->SizeOfRawData, MEM_DECOMMIT); + continue; + } + + // determine protection flags based on characteristics + protect = ProtectionFlags[executable][readable][writeable]; + if (section->Characteristics & IMAGE_SCN_MEM_NOT_CACHED) + protect |= PAGE_NOCACHE; + + // determine size of region + size = section->SizeOfRawData; + if (size == 0) + { + if (section->Characteristics & IMAGE_SCN_CNT_INITIALIZED_DATA) + size = module->headers->OptionalHeader.SizeOfInitializedData; + else if (section->Characteristics & IMAGE_SCN_CNT_UNINITIALIZED_DATA) + size = module->headers->OptionalHeader.SizeOfUninitializedData; + } + + if (size > 0) + { + // change memory access flags + if (VirtualProtect((LPVOID)section->Misc.PhysicalAddress, section->SizeOfRawData, protect, &oldProtect) == 0) +#if DEBUG_OUTPUT + OutputLastError("Error protecting memory page") +#endif + ; + } + } +} + +static void +PerformBaseRelocation(PMEMORYMODULE module, DWORD delta) +{ + DWORD i; + unsigned char *codeBase = module->codeBase; + + PIMAGE_DATA_DIRECTORY directory = GET_HEADER_DICTIONARY(module, IMAGE_DIRECTORY_ENTRY_BASERELOC); + if (directory->Size > 0) + { + PIMAGE_BASE_RELOCATION relocation = (PIMAGE_BASE_RELOCATION)(codeBase + directory->VirtualAddress); + for (; relocation->VirtualAddress > 0; ) + { + unsigned char *dest = (unsigned char *)(codeBase + relocation->VirtualAddress); + unsigned short *relInfo = (unsigned short *)((unsigned char *)relocation + IMAGE_SIZEOF_BASE_RELOCATION); + for (i=0; i<((relocation->SizeOfBlock-IMAGE_SIZEOF_BASE_RELOCATION) / 2); i++, relInfo++) + { + DWORD *patchAddrHL; + int type, offset; + + // the upper 4 bits define the type of relocation + type = *relInfo >> 12; + // the lower 12 bits define the offset + offset = *relInfo & 0xfff; + + switch (type) + { + case IMAGE_REL_BASED_ABSOLUTE: + // skip relocation + break; + + case IMAGE_REL_BASED_HIGHLOW: + // change complete 32 bit address + patchAddrHL = (DWORD *)(dest + offset); + *patchAddrHL += delta; + break; + + default: + //printf("Unknown relocation: %d\n", type); + break; + } + } + + // advance to next relocation block + relocation = (PIMAGE_BASE_RELOCATION)(((DWORD)relocation) + relocation->SizeOfBlock); + } + } +} + +static int +BuildImportTable(PMEMORYMODULE module) +{ + int result=1; + unsigned char *codeBase = module->codeBase; + + PIMAGE_DATA_DIRECTORY directory = GET_HEADER_DICTIONARY(module, IMAGE_DIRECTORY_ENTRY_IMPORT); + if (directory->Size > 0) + { + PIMAGE_IMPORT_DESCRIPTOR importDesc = (PIMAGE_IMPORT_DESCRIPTOR)(codeBase + directory->VirtualAddress); + for (; !IsBadReadPtr(importDesc, sizeof(IMAGE_IMPORT_DESCRIPTOR)) && importDesc->Name; importDesc++) + { + DWORD *thunkRef, *funcRef; + HMODULE handle; + + handle = MyLoadLibrary(codeBase + importDesc->Name); + if (handle == INVALID_HANDLE_VALUE) + { + //LastError should already be set +#if DEBUG_OUTPUT + OutputLastError("Can't load library"); +#endif + result = 0; + break; + } + + module->modules = (HMODULE *)realloc(module->modules, (module->numModules+1)*(sizeof(HMODULE))); + if (module->modules == NULL) + { + SetLastError(ERROR_NOT_ENOUGH_MEMORY); + result = 0; + break; + } + + module->modules[module->numModules++] = handle; + if (importDesc->OriginalFirstThunk) + { + thunkRef = (DWORD *)(codeBase + importDesc->OriginalFirstThunk); + funcRef = (DWORD *)(codeBase + importDesc->FirstThunk); + } else { + // no hint table + thunkRef = (DWORD *)(codeBase + importDesc->FirstThunk); + funcRef = (DWORD *)(codeBase + importDesc->FirstThunk); + } + for (; *thunkRef; thunkRef++, funcRef++) + { + if IMAGE_SNAP_BY_ORDINAL(*thunkRef) { + *funcRef = (DWORD)MyGetProcAddress(handle, (LPCSTR)IMAGE_ORDINAL(*thunkRef)); + } else { + PIMAGE_IMPORT_BY_NAME thunkData = (PIMAGE_IMPORT_BY_NAME)(codeBase + *thunkRef); + *funcRef = (DWORD)MyGetProcAddress(handle, (LPCSTR)&thunkData->Name); + } + if (*funcRef == 0) + { + SetLastError(ERROR_PROC_NOT_FOUND); + result = 0; + break; + } + } + + if (!result) + break; + } + } + + return result; +} + +/* + MemoryLoadLibrary - load a library AS MEMORY MODULE, or return + existing MEMORY MODULE with increased refcount. + + This allows to load a library AGAIN as memory module which is + already loaded as HMODULE! + +*/ +HMEMORYMODULE MemoryLoadLibrary(char *name, const void *data) +{ + PMEMORYMODULE result; + PIMAGE_DOS_HEADER dos_header; + PIMAGE_NT_HEADERS old_header; + unsigned char *code, *headers; + DWORD locationDelta; + DllEntryProc DllEntry; + BOOL successfull; + MEMORYMODULE *p = loaded; + + while (p) { + // If already loaded, only increment the reference count + if (0 == stricmp(name, p->name)) { + p->refcount++; + return (HMODULE)p; + } + p = p->next; + } + + /* Do NOT check for GetModuleHandle here! */ + + dos_header = (PIMAGE_DOS_HEADER)data; + if (dos_header->e_magic != IMAGE_DOS_SIGNATURE) + { + SetLastError(ERROR_BAD_FORMAT); +#if DEBUG_OUTPUT + OutputDebugString("Not a valid executable file.\n"); +#endif + return NULL; + } + + old_header = (PIMAGE_NT_HEADERS)&((const unsigned char *)(data))[dos_header->e_lfanew]; + if (old_header->Signature != IMAGE_NT_SIGNATURE) + { + SetLastError(ERROR_BAD_FORMAT); +#if DEBUG_OUTPUT + OutputDebugString("No PE header found.\n"); +#endif + return NULL; + } + + // reserve memory for image of library + code = (unsigned char *)VirtualAlloc((LPVOID)(old_header->OptionalHeader.ImageBase), + old_header->OptionalHeader.SizeOfImage, + MEM_RESERVE, + PAGE_READWRITE); + + if (code == NULL) + // try to allocate memory at arbitrary position + code = (unsigned char *)VirtualAlloc(NULL, + old_header->OptionalHeader.SizeOfImage, + MEM_RESERVE, + PAGE_READWRITE); + + if (code == NULL) + { + SetLastError(ERROR_NOT_ENOUGH_MEMORY); +#if DEBUG_OUTPUT + OutputLastError("Can't reserve memory"); +#endif + return NULL; + } + + result = (PMEMORYMODULE)HeapAlloc(GetProcessHeap(), 0, sizeof(MEMORYMODULE)); + result->codeBase = code; + result->numModules = 0; + result->modules = NULL; + result->initialized = 0; + result->next = result->prev = NULL; + result->refcount = 1; + result->name = strdup(name); + result->name_table = NULL; + + // XXX: is it correct to commit the complete memory region at once? + // calling DllEntry raises an exception if we don't... + VirtualAlloc(code, + old_header->OptionalHeader.SizeOfImage, + MEM_COMMIT, + PAGE_READWRITE); + + // commit memory for headers + headers = (unsigned char *)VirtualAlloc(code, + old_header->OptionalHeader.SizeOfHeaders, + MEM_COMMIT, + PAGE_READWRITE); + + // copy PE header to code + memcpy(headers, dos_header, dos_header->e_lfanew + old_header->OptionalHeader.SizeOfHeaders); + result->headers = (PIMAGE_NT_HEADERS)&((const unsigned char *)(headers))[dos_header->e_lfanew]; + + // update position + result->headers->OptionalHeader.ImageBase = (DWORD)code; + + // copy sections from DLL file block to new memory location + CopySections(data, old_header, result); + + // adjust base address of imported data + locationDelta = (DWORD)(code - old_header->OptionalHeader.ImageBase); + if (locationDelta != 0) + PerformBaseRelocation(result, locationDelta); + + // load required dlls and adjust function table of imports + if (!BuildImportTable(result)) + goto error; + + // mark memory pages depending on section headers and release + // sections that are marked as "discardable" + FinalizeSections(result); + + // get entry point of loaded library + if (result->headers->OptionalHeader.AddressOfEntryPoint != 0) + { + DllEntry = (DllEntryProc)(code + result->headers->OptionalHeader.AddressOfEntryPoint); + if (DllEntry == 0) + { + SetLastError(ERROR_BAD_FORMAT); /* XXX ? */ +#if DEBUG_OUTPUT + OutputDebugString("Library has no entry point.\n"); +#endif + goto error; + } + + // notify library about attaching to process + successfull = (*DllEntry)((HINSTANCE)code, DLL_PROCESS_ATTACH, 0); + if (!successfull) + { +#if DEBUG_OUTPUT + OutputDebugString("Can't attach library.\n"); +#endif + goto error; + } + result->initialized = 1; + } + + _Register(name, result); + + return (HMEMORYMODULE)result; + +error: + // cleanup + free(result->name); + MemoryFreeLibrary(result); + return NULL; +} + +int _compare(const struct NAME_TABLE *p1, const struct NAME_TABLE *p2) +{ + return stricmp(p1->name, p2->name); +} + +int _find(const char **name, const struct NAME_TABLE *p) +{ + return stricmp(*name, p->name); +} + +struct NAME_TABLE *GetNameTable(PMEMORYMODULE module) +{ + unsigned char *codeBase; + PIMAGE_EXPORT_DIRECTORY exports; + PIMAGE_DATA_DIRECTORY directory; + DWORD i, *nameRef; + WORD *ordinal; + struct NAME_TABLE *p, *ptab; + + if (module->name_table) + return module->name_table; + + codeBase = module->codeBase; + directory = GET_HEADER_DICTIONARY(module, IMAGE_DIRECTORY_ENTRY_EXPORT); + exports = (PIMAGE_EXPORT_DIRECTORY)(codeBase + directory->VirtualAddress); + + nameRef = (DWORD *)(codeBase + exports->AddressOfNames); + ordinal = (WORD *)(codeBase + exports->AddressOfNameOrdinals); + + p = ((PMEMORYMODULE)module)->name_table = (struct NAME_TABLE *)malloc(sizeof(struct NAME_TABLE) + * exports->NumberOfNames); + if (p == NULL) + return NULL; + ptab = p; + for (i=0; iNumberOfNames; ++i) { + p->name = (char *)(codeBase + *nameRef++); + p->ordinal = *ordinal++; + ++p; + } + qsort(ptab, exports->NumberOfNames, sizeof(struct NAME_TABLE), _compare); + return ptab; +} + +FARPROC MemoryGetProcAddress(HMEMORYMODULE module, const char *name) +{ + unsigned char *codeBase = ((PMEMORYMODULE)module)->codeBase; + int idx=-1; + PIMAGE_EXPORT_DIRECTORY exports; + PIMAGE_DATA_DIRECTORY directory = GET_HEADER_DICTIONARY((PMEMORYMODULE)module, IMAGE_DIRECTORY_ENTRY_EXPORT); + + if (directory->Size == 0) + // no export table found + return NULL; + + exports = (PIMAGE_EXPORT_DIRECTORY)(codeBase + directory->VirtualAddress); + if (exports->NumberOfNames == 0 || exports->NumberOfFunctions == 0) + // DLL doesn't export anything + return NULL; + + if (HIWORD(name)) { + struct NAME_TABLE *ptab; + struct NAME_TABLE *found; + ptab = GetNameTable((PMEMORYMODULE)module); + if (ptab == NULL) + // some failure + return NULL; + found = bsearch(&name, ptab, exports->NumberOfNames, sizeof(struct NAME_TABLE), _find); + if (found == NULL) + // exported symbol not found + return NULL; + + idx = found->ordinal; + } + else + idx = LOWORD(name) - exports->Base; + + if ((DWORD)idx > exports->NumberOfFunctions) + // name <-> ordinal number don't match + return NULL; + + // AddressOfFunctions contains the RVAs to the "real" functions + return (FARPROC)(codeBase + *(DWORD *)(codeBase + exports->AddressOfFunctions + (idx*4))); +} + +void MemoryFreeLibrary(HMEMORYMODULE mod) +{ + int i; + PMEMORYMODULE module = (PMEMORYMODULE)mod; + + if (module != NULL) + { + if (module->initialized != 0) + { + // notify library about detaching from process + DllEntryProc DllEntry = (DllEntryProc)(module->codeBase + module->headers->OptionalHeader.AddressOfEntryPoint); + (*DllEntry)((HINSTANCE)module->codeBase, DLL_PROCESS_DETACH, 0); + module->initialized = 0; + } + + if (module->modules != NULL) + { + // free previously opened libraries + for (i=0; inumModules; i++) + if (module->modules[i] != INVALID_HANDLE_VALUE) + MyFreeLibrary(module->modules[i]); + + free(module->modules); + } + + if (module->codeBase != NULL) + // release memory of library + VirtualFree(module->codeBase, 0, MEM_RELEASE); + + if (module->name_table != NULL) + free(module->name_table); + + HeapFree(GetProcessHeap(), 0, module); + } +} diff --git a/setup/installer/windows/MemoryModule.h b/setup/installer/windows/MemoryModule.h new file mode 100644 index 0000000000..601d4c50df --- /dev/null +++ b/setup/installer/windows/MemoryModule.h @@ -0,0 +1,58 @@ +/* + * Memory DLL loading code + * Version 0.0.2 + * + * Copyright (c) 2004-2005 by Joachim Bauch / mail@joachim-bauch.de + * http://www.joachim-bauch.de + * + * The contents of this file are subject to the Mozilla Public License Version + * 1.1 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * http://www.mozilla.org/MPL/ + * + * Software distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License + * for the specific language governing rights and limitations under the + * License. + * + * The Original Code is MemoryModule.h + * + * The Initial Developer of the Original Code is Joachim Bauch. + * + * Portions created by Joachim Bauch are Copyright (C) 2004-2005 + * Joachim Bauch. All Rights Reserved. + * + */ + +#ifndef __MEMORY_MODULE_HEADER +#define __MEMORY_MODULE_HEADER + +#include + +typedef void *HMEMORYMODULE; + +#ifdef __cplusplus +extern "C" { +#endif + +typedef void *(*FINDPROC)(); + +extern FINDPROC findproc; +extern void *findproc_data; + +HMEMORYMODULE MemoryLoadLibrary(char *, const void *); + +FARPROC MemoryGetProcAddress(HMEMORYMODULE, const char *); + +void MemoryFreeLibrary(HMEMORYMODULE); + +BOOL MyFreeLibrary(HMODULE hModule); +HMODULE MyLoadLibrary(char *lpFileName); +FARPROC MyGetProcAddress(HMODULE hModule, LPCSTR lpProcName); +HMODULE MyGetModuleHandle(LPCTSTR lpModuleName); + +#ifdef __cplusplus +} +#endif + +#endif // __MEMORY_MODULE_HEADER diff --git a/setup/installer/windows/freeze.py b/setup/installer/windows/freeze.py index 7fb60968e7..0fe494e831 100644 --- a/setup/installer/windows/freeze.py +++ b/setup/installer/windows/freeze.py @@ -16,7 +16,6 @@ from setup.installer.windows.wix import WixMixIn OPENSSL_DIR = r'Q:\openssl' QT_DIR = 'Q:\\Qt\\4.7.3' QT_DLLS = ['Core', 'Gui', 'Network', 'Svg', 'WebKit', 'Xml', 'XmlPatterns'] -LIBUSB_DIR = 'C:\\libusb' LIBUNRAR = 'C:\\Program Files\\UnrarDLL\\unrar.dll' SW = r'C:\cygwin\home\kovid\sw' IMAGEMAGICK = os.path.join(SW, 'build', 'ImageMagick-6.6.6', @@ -71,11 +70,13 @@ class Win32Freeze(Command, WixMixIn): self.rc_template = self.j(self.d(self.a(__file__)), 'template.rc') self.py_ver = ''.join(map(str, sys.version_info[:2])) self.lib_dir = self.j(self.base, 'Lib') - self.pydlib = self.j(self.base, 'pydlib') self.pylib = self.j(self.base, 'pylib.zip') + self.dll_dir = self.j(self.base, 'DLLs') + self.plugins_dir = os.path.join(self.base, 'plugins') self.initbase() self.build_launchers() + self.add_plugins() self.freeze() self.embed_manifests() self.install_site_py() @@ -87,18 +88,21 @@ class Win32Freeze(Command, WixMixIn): shutil.rmtree(self.base) os.makedirs(self.base) + def add_plugins(self): + self.info('Adding plugins...') + tgt = self.plugins_dir + if os.path.exists(tgt): + shutil.rmtree(tgt) + os.mkdir(tgt) + base = self.j(self.SRC, 'calibre', 'plugins') + for f in glob.glob(self.j(base, '*.pyd')): + # We dont want the manifests as the manifest in the exe will be + # used instead + shutil.copy2(f, tgt) + def freeze(self): shutil.copy2(self.j(self.src_root, 'LICENSE'), self.base) - self.info('Adding plugins...') - tgt = os.path.join(self.base, 'plugins') - if not os.path.exists(tgt): - os.mkdir(tgt) - base = self.j(self.SRC, 'calibre', 'plugins') - for pat in ('*.pyd', '*.manifest'): - for f in glob.glob(self.j(base, pat)): - shutil.copy2(f, tgt) - self.info('Adding resources...') tgt = self.j(self.base, 'resources') if os.path.exists(tgt): @@ -106,7 +110,6 @@ class Win32Freeze(Command, WixMixIn): shutil.copytree(self.j(self.src_root, 'resources'), tgt) self.info('Adding Qt and python...') - self.dll_dir = self.j(self.base, 'DLLs') shutil.copytree(r'C:\Python%s\DLLs'%self.py_ver, self.dll_dir, ignore=shutil.ignore_patterns('msvc*.dll', 'Microsoft.*')) for x in glob.glob(self.j(OPENSSL_DIR, 'bin', '*.dll')): @@ -197,11 +200,6 @@ class Win32Freeze(Command, WixMixIn): print print 'Adding third party dependencies' - tdir = os.path.join(self.base, 'driver') - os.makedirs(tdir) - for pat in ('*.dll', '*.sys', '*.cat', '*.inf'): - for f in glob.glob(os.path.join(LIBUSB_DIR, pat)): - shutil.copyfile(f, os.path.join(tdir, os.path.basename(f))) print '\tAdding unrar' shutil.copyfile(LIBUNRAR, os.path.join(self.dll_dir, os.path.basename(LIBUNRAR))) @@ -318,8 +316,8 @@ class Win32Freeze(Command, WixMixIn): if not os.path.exists(self.obj_dir): os.makedirs(self.obj_dir) base = self.j(self.src_root, 'setup', 'installer', 'windows') - sources = [self.j(base, x) for x in ['util.c']] - headers = [self.j(base, x) for x in ['util.h']] + sources = [self.j(base, x) for x in ['util.c', 'MemoryModule.c']] + headers = [self.j(base, x) for x in ['util.h', 'MemoryModule.h']] objects = [self.j(self.obj_dir, self.b(x)+'.obj') for x in sources] cflags = '/c /EHsc /MD /W3 /Ox /nologo /D_UNICODE'.split() cflags += ['/DPYDLL="python%s.dll"'%self.py_ver, '/IC:/Python%s/include'%self.py_ver] @@ -371,43 +369,49 @@ class Win32Freeze(Command, WixMixIn): def archive_lib_dir(self): self.info('Putting all python code into a zip file for performance') - if os.path.exists(self.pydlib): - shutil.rmtree(self.pydlib) - os.makedirs(self.pydlib) self.zf_timestamp = time.localtime(time.time())[:6] self.zf_names = set() with zipfile.ZipFile(self.pylib, 'w', zipfile.ZIP_STORED) as zf: + # Add the .pyds from python and calibre to the zip file + for x in (self.plugins_dir, self.dll_dir): + for pyd in os.listdir(x): + if pyd.endswith('.pyd') and pyd != 'sqlite_custom.pyd': + # sqlite_custom has to be a file for + # sqlite_load_extension to work + self.add_to_zipfile(zf, pyd, x) + os.remove(self.j(x, pyd)) + + # Add everything in Lib except site-packages to the zip file for x in os.listdir(self.lib_dir): if x == 'site-packages': continue self.add_to_zipfile(zf, x, self.lib_dir) sp = self.j(self.lib_dir, 'site-packages') - handled = set(['site.pyo']) - for pth in ('PIL.pth', 'pywin32.pth'): - handled.add(pth) - shutil.copyfile(self.j(sp, pth), self.j(self.pydlib, pth)) - for d in self.get_pth_dirs(self.j(sp, pth)): - shutil.copytree(d, self.j(self.pydlib, self.b(d)), True) - handled.add(self.b(d)) + # Special handling for PIL and pywin32 + handled = set(['PIL.pth', 'pywin32.pth', 'PIL', 'win32']) + self.add_to_zipfile(zf, 'PIL', sp) + base = self.j(sp, 'win32', 'lib') + for x in os.listdir(base): + if os.path.splitext(x)[1] not in ('.exe',): + self.add_to_zipfile(zf, x, base) + base = self.d(base) + for x in os.listdir(base): + if not os.path.isdir(self.j(base, x)): + if os.path.splitext(x)[1] not in ('.exe',): + self.add_to_zipfile(zf, x, base) handled.add('easy-install.pth') for d in self.get_pth_dirs(self.j(sp, 'easy-install.pth')): handled.add(self.b(d)) - zip_safe = self.is_zip_safe(d) for x in os.listdir(d): if x == 'EGG-INFO': continue - if zip_safe: - self.add_to_zipfile(zf, x, d) - else: - absp = self.j(d, x) - dest = self.j(self.pydlib, x) - if os.path.isdir(absp): - shutil.copytree(absp, dest, True) - else: - shutil.copy2(absp, dest) + self.add_to_zipfile(zf, x, d) + # The rest of site-packages + # We dont want the site.py from site-packages + handled.add('site.pyo') for x in os.listdir(sp): if x in handled or x.endswith('.egg-info'): continue @@ -415,33 +419,18 @@ class Win32Freeze(Command, WixMixIn): if os.path.isdir(absp): if not os.listdir(absp): continue - if self.is_zip_safe(absp): - self.add_to_zipfile(zf, x, sp) - else: - shutil.copytree(absp, self.j(self.pydlib, x), True) + self.add_to_zipfile(zf, x, sp) else: - if x.endswith('.pyd'): - shutil.copy2(absp, self.j(self.pydlib, x)) - else: - self.add_to_zipfile(zf, x, sp) + self.add_to_zipfile(zf, x, sp) shutil.rmtree(self.lib_dir) - def is_zip_safe(self, path): - for f in walk(path): - ext = os.path.splitext(f)[1].lower() - if ext in ('.pyd', '.dll', '.exe'): - return False - return True - def get_pth_dirs(self, pth): base = os.path.dirname(pth) for line in open(pth).readlines(): line = line.strip() if not line or line.startswith('#') or line.startswith('import'): continue - if line == 'win32\\lib': - continue candidate = self.j(base, line) if os.path.exists(candidate): yield candidate @@ -463,10 +452,10 @@ class Win32Freeze(Command, WixMixIn): self.add_to_zipfile(zf, name + os.sep + x, base) else: ext = os.path.splitext(name)[1].lower() - if ext in ('.pyd', '.dll', '.exe'): + if ext in ('.dll',): raise ValueError('Cannot add %r to zipfile'%abspath) zinfo.external_attr = 0600 << 16 - if ext in ('.py', '.pyc', '.pyo'): + if ext in ('.py', '.pyc', '.pyo', '.pyd'): with open(abspath, 'rb') as f: zf.writestr(zinfo, f.read()) diff --git a/setup/installer/windows/notes.rst b/setup/installer/windows/notes.rst index 11b5bccf79..0bb8b7b15b 100644 --- a/setup/installer/windows/notes.rst +++ b/setup/installer/windows/notes.rst @@ -88,7 +88,9 @@ Qt uses its own routine to locate and load "system libraries" including the open Now, run configure and make:: - configure -opensource -release -qt-zlib -qt-gif -qt-libmng -qt-libpng -qt-libtiff -qt-libjpeg -release -platform win32-msvc2008 -no-qt3support -webkit -xmlpatterns -no-phonon -no-style-plastique -no-style-cleanlooks -no-style-motif -no-style-cde -no-declarative -no-scripttools -no-audio-backend -no-multimedia -no-dbus -no-openvg -no-opengl -no-qt3support -confirm-license -nomake examples -nomake demos -nomake docs -openssl -I Q:\openssl\include -L Q:\openssl\lib && nmake +-no-plugin-manifests is needed so that loading the plugins does not fail looking for the CRT assembly + + configure -opensource -release -qt-zlib -qt-gif -qt-libmng -qt-libpng -qt-libtiff -qt-libjpeg -release -platform win32-msvc2008 -no-qt3support -webkit -xmlpatterns -no-phonon -no-style-plastique -no-style-cleanlooks -no-style-motif -no-style-cde -no-declarative -no-scripttools -no-audio-backend -no-multimedia -no-dbus -no-openvg -no-opengl -no-qt3support -confirm-license -nomake examples -nomake demos -nomake docs -no-plugin-manifests -openssl -I Q:\openssl\include -L Q:\openssl\lib && nmake SIP ----- diff --git a/setup/installer/windows/site.py b/setup/installer/windows/site.py index 5610ff197e..33f2e63585 100644 --- a/setup/installer/windows/site.py +++ b/setup/installer/windows/site.py @@ -1,12 +1,72 @@ #!/usr/bin/env python # vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai -from __future__ import with_statement __license__ = 'GPL v3' __copyright__ = '2009, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import sys, os, linecache +import sys +import os +import zipimport +import _memimporter + +DEBUG_ZIPIMPORT = False + +class ZipExtensionImporter(zipimport.zipimporter): + ''' + Taken, with thanks, from the py2exe source code + ''' + + def __init__(self, *args, **kwargs): + zipimport.zipimporter.__init__(self, *args, **kwargs) + # We know there are no dlls in the zip file, so dont set findproc + # (performance optimization) + #_memimporter.set_find_proc(self.locate_dll_image) + + def find_module(self, fullname, path=None): + result = zipimport.zipimporter.find_module(self, fullname, path) + if result: + return result + fullname = fullname.replace(".", "\\") + if (fullname + '.pyd') in self._files: + return self + return None + + def locate_dll_image(self, name): + # A callback function for_memimporter.import_module. Tries to + # locate additional dlls. Returns the image as Python string, + # or None if not found. + if name in self._files: + return self.get_data(name) + return None + + def load_module(self, fullname): + if sys.modules.has_key(fullname): + mod = sys.modules[fullname] + if DEBUG_ZIPIMPORT: + sys.stderr.write("import %s # previously loaded from zipfile %s\n" % (fullname, self.archive)) + return mod + try: + return zipimport.zipimporter.load_module(self, fullname) + except zipimport.ZipImportError: + pass + initname = "init" + fullname.split(".")[-1] # name of initfunction + filename = fullname.replace(".", "\\") + path = filename + '.pyd' + if path in self._files: + if DEBUG_ZIPIMPORT: + sys.stderr.write("# found %s in zipfile %s\n" % (path, self.archive)) + code = self.get_data(path) + mod = _memimporter.import_module(code, initname, fullname, path) + mod.__file__ = "%s\\%s" % (self.archive, path) + mod.__loader__ = self + if DEBUG_ZIPIMPORT: + sys.stderr.write("import %s # loaded from zipfile %s\n" % (fullname, mod.__file__)) + return mod + raise zipimport.ZipImportError, "can't find module %s" % fullname + + def __repr__(self): + return "<%s object %r>" % (self.__class__.__name__, self.archive) def abs__file__(): @@ -42,42 +102,6 @@ def makepath(*paths): dir = os.path.abspath(os.path.join(*paths)) return dir, os.path.normcase(dir) -def addpackage(sitedir, name): - """Process a .pth file within the site-packages directory: - For each line in the file, either combine it with sitedir to a path, - or execute it if it starts with 'import '. - """ - fullname = os.path.join(sitedir, name) - try: - f = open(fullname, "rU") - except IOError: - return - with f: - for line in f: - if line.startswith("#"): - continue - if line.startswith(("import ", "import\t")): - exec line - continue - line = line.rstrip() - dir, dircase = makepath(sitedir, line) - if os.path.exists(dir): - sys.path.append(dir) - - -def addsitedir(sitedir): - """Add 'sitedir' argument to sys.path if missing and handle .pth files in - 'sitedir'""" - sitedir, sitedircase = makepath(sitedir) - try: - names = os.listdir(sitedir) - except os.error: - return - dotpth = os.extsep + "pth" - names = [name for name in names if name.endswith(dotpth)] - for name in sorted(names): - addpackage(sitedir, name) - def run_entry_point(): bname, mod, func = sys.calibre_basename, sys.calibre_module, sys.calibre_function sys.argv[0] = bname+'.exe' @@ -89,6 +113,10 @@ def main(): sys.setdefaultencoding('utf-8') aliasmbcs() + sys.path_hooks.insert(0, ZipExtensionImporter) + sys.path_importer_cache.clear() + + import linecache def fake_getline(filename, lineno, module_globals=None): return '' linecache.orig_getline = linecache.getline @@ -96,10 +124,11 @@ def main(): abs__file__() - addsitedir(os.path.join(sys.app_dir, 'pydlib')) - add_calibre_vars() + # Needed for pywintypes to be able to load its DLL + sys.path.append(os.path.join(sys.app_dir, 'DLLs')) + return run_entry_point() diff --git a/setup/installer/windows/util.c b/setup/installer/windows/util.c index 329e3bf8c3..4075d7e123 100644 --- a/setup/installer/windows/util.c +++ b/setup/installer/windows/util.c @@ -1,18 +1,130 @@ /* * Copyright 2009 Kovid Goyal + * The memimporter code is taken from the py2exe project */ #include "util.h" + #include #include #include + static char GUI_APP = 0; static char python_dll[] = PYDLL; void set_gui_app(char yes) { GUI_APP = yes; } char is_gui_app() { return GUI_APP; } + +// memimporter {{{ + +#include "MemoryModule.h" + +static char **DLL_Py_PackageContext = NULL; +static PyObject **DLL_ImportError = NULL; +static char module_doc[] = +"Importer which can load extension modules from memory"; + + +static void *memdup(void *ptr, Py_ssize_t size) +{ + void *p = malloc(size); + if (p == NULL) + return NULL; + memcpy(p, ptr, size); + return p; +} + +/* + Be sure to detect errors in FindLibrary - undetected errors lead to + very strange behaviour. +*/ +static void* FindLibrary(char *name, PyObject *callback) +{ + PyObject *result; + char *p; + Py_ssize_t size; + + if (callback == NULL) + return NULL; + result = PyObject_CallFunction(callback, "s", name); + if (result == NULL) { + PyErr_Clear(); + return NULL; + } + if (-1 == PyString_AsStringAndSize(result, &p, &size)) { + PyErr_Clear(); + Py_DECREF(result); + return NULL; + } + p = memdup(p, size); + Py_DECREF(result); + return p; +} + +static PyObject *set_find_proc(PyObject *self, PyObject *args) +{ + PyObject *callback = NULL; + if (!PyArg_ParseTuple(args, "|O:set_find_proc", &callback)) + return NULL; + Py_DECREF((PyObject *)findproc_data); + Py_INCREF(callback); + findproc_data = (void *)callback; + return Py_BuildValue("i", 1); +} + +static PyObject * +import_module(PyObject *self, PyObject *args) +{ + char *data; + int size; + char *initfuncname; + char *modname; + char *pathname; + HMEMORYMODULE hmem; + FARPROC do_init; + + char *oldcontext; + + /* code, initfuncname, fqmodulename, path */ + if (!PyArg_ParseTuple(args, "s#sss:import_module", + &data, &size, + &initfuncname, &modname, &pathname)) + return NULL; + hmem = MemoryLoadLibrary(pathname, data); + if (!hmem) { + PyErr_Format(*DLL_ImportError, + "MemoryLoadLibrary() failed loading %s", pathname); + return NULL; + } + do_init = MemoryGetProcAddress(hmem, initfuncname); + if (!do_init) { + MemoryFreeLibrary(hmem); + PyErr_Format(*DLL_ImportError, + "Could not find function %s in memory loaded pyd", initfuncname); + return NULL; + } + + oldcontext = *DLL_Py_PackageContext; + *DLL_Py_PackageContext = modname; + do_init(); + *DLL_Py_PackageContext = oldcontext; + if (PyErr_Occurred()) + return NULL; + /* Retrieve from sys.modules */ + return PyImport_ImportModule(modname); +} + +static PyMethodDef methods[] = { + { "import_module", import_module, METH_VARARGS, + "import_module(code, initfunc, dllname[, finder]) -> module" }, + { "set_find_proc", set_find_proc, METH_VARARGS }, + { NULL, NULL }, /* Sentinel */ +}; + +// }}} + static int _show_error(const wchar_t *preamble, const wchar_t *msg, const int code) { wchar_t *buf, *cbuf; buf = (wchar_t*)LocalAlloc(LMEM_ZEROINIT, sizeof(wchar_t)* @@ -185,7 +297,7 @@ void initialize_interpreter(wchar_t *outr, wchar_t *errr, char *dummy_argv[1] = {""}; buf = (char*)calloc(MAX_PATH, sizeof(char)); - path = (char*)calloc(3*MAX_PATH, sizeof(char)); + path = (char*)calloc(MAX_PATH, sizeof(char)); if (!buf || !path) ExitProcess(_show_error(L"Out of memory", L"", 1)); sz = GetModuleFileNameA(NULL, buf, MAX_PATH); @@ -198,8 +310,7 @@ void initialize_interpreter(wchar_t *outr, wchar_t *errr, buf[strlen(buf)-1] = '\0'; _snprintf_s(python_home, MAX_PATH, _TRUNCATE, "%s", buf); - _snprintf_s(path, 3*MAX_PATH, _TRUNCATE, "%s\\pylib.zip;%s\\pydlib;%s\\DLLs", - buf, buf, buf); + _snprintf_s(path, MAX_PATH, _TRUNCATE, "%s\\pylib.zip", buf); free(buf); @@ -227,7 +338,10 @@ void initialize_interpreter(wchar_t *outr, wchar_t *errr, if (!flag) ExitProcess(_show_error(L"Failed to get debug flag", L"", 1)); //*flag = 1; - + DLL_Py_PackageContext = (char**)GetProcAddress(dll, "_Py_PackageContext"); + if (!DLL_Py_PackageContext) ExitProcess(_show_error(L"Failed to load _Py_PackageContext from dll", L"", 1)); + DLL_ImportError = (PyObject**)GetProcAddress(dll, "PyExc_ImportError"); + if (!DLL_ImportError) ExitProcess(_show_error(L"Failed to load PyExc_ImportError from dll", L"", 1)); Py_SetProgramName(program_name); Py_SetPythonHome(python_home); @@ -263,6 +377,10 @@ void initialize_interpreter(wchar_t *outr, wchar_t *errr, PyList_SetItem(argv, i, v); } PySys_SetObject("argv", argv); + + findproc = FindLibrary; + Py_InitModule3("_memimporter", methods, module_doc); + } diff --git a/setup/installer/windows/wix-template.xml b/setup/installer/windows/wix-template.xml index 0a85b6fb81..3ebe0882e0 100644 --- a/setup/installer/windows/wix-template.xml +++ b/setup/installer/windows/wix-template.xml @@ -164,10 +164,6 @@ - - - - diff --git a/setup/translations.py b/setup/translations.py index 1f026555ec..3a33b7bcc4 100644 --- a/setup/translations.py +++ b/setup/translations.py @@ -85,7 +85,7 @@ class Translations(POT): def mo_file(self, po_file): locale = os.path.splitext(os.path.basename(po_file))[0] - return locale, os.path.join(self.DEST, locale, 'LC_MESSAGES', 'messages.mo') + return locale, os.path.join(self.DEST, locale, 'messages.mo') def run(self, opts): @@ -94,9 +94,8 @@ class Translations(POT): base = os.path.dirname(dest) if not os.path.exists(base): os.makedirs(base) - if self.newer(dest, f): - self.info('\tCompiling translations for', locale) - subprocess.check_call(['msgfmt', '-o', dest, f]) + self.info('\tCompiling translations for', locale) + subprocess.check_call(['msgfmt', '-o', dest, f]) if locale in ('en_GB', 'nds', 'te', 'yi'): continue pycountry = self.j(sysconfig.get_python_lib(), 'pycountry', @@ -123,6 +122,16 @@ class Translations(POT): shutil.copy2(f, dest) self.write_stats() + self.freeze_locales() + + def freeze_locales(self): + zf = self.DEST + '.zip' + from calibre import CurrentDir + from calibre.utils.zipfile import ZipFile, ZIP_DEFLATED + with ZipFile(zf, 'w', ZIP_DEFLATED) as zf: + with CurrentDir(self.DEST): + zf.add_dir('.') + shutil.rmtree(self.DEST) @property def stats(self): diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py index 9ebec5e7e8..33685ea254 100644 --- a/src/calibre/customize/builtins.py +++ b/src/calibre/customize/builtins.py @@ -608,9 +608,9 @@ from calibre.devices.edge.driver import EDGE from calibre.devices.teclast.driver import TECLAST_K3, NEWSMY, IPAPYRUS, \ SOVOS, PICO, SUNSTECH_EB700, ARCHOS7O, STASH, WEXLER from calibre.devices.sne.driver import SNE -from calibre.devices.misc import PALMPRE, AVANT, SWEEX, PDNOVEL, \ - GEMEI, VELOCITYMICRO, PDNOVEL_KOBO, LUMIREAD, ALURATEK_COLOR, \ - TREKSTOR, EEEREADER, NEXTBOOK +from calibre.devices.misc import (PALMPRE, AVANT, SWEEX, PDNOVEL, + GEMEI, VELOCITYMICRO, PDNOVEL_KOBO, LUMIREAD, ALURATEK_COLOR, + TREKSTOR, EEEREADER, NEXTBOOK, ADAM) from calibre.devices.folder_device.driver import FOLDER_DEVICE_FOR_CONFIG from calibre.devices.kobo.driver import KOBO from calibre.devices.bambook.driver import BAMBOOK @@ -744,6 +744,7 @@ plugins += [ TREKSTOR, EEEREADER, NEXTBOOK, + ADAM, ITUNES, BOEYE_BEX, BOEYE_BDX, @@ -1231,7 +1232,7 @@ class StoreEpubBudStore(StoreBase): name = 'ePub Bud' description = 'Well, it\'s pretty much just "YouTube for Children\'s eBooks. A not-for-profit organization devoted to brining self published childrens books to the world.' actual_plugin = 'calibre.gui2.store.epubbud_plugin:EpubBudStore' - + drm_free_only = True headquarters = 'US' formats = ['EPUB'] diff --git a/src/calibre/debug.py b/src/calibre/debug.py index 8d65c37bbf..79110d9585 100644 --- a/src/calibre/debug.py +++ b/src/calibre/debug.py @@ -53,6 +53,8 @@ Run an embedded python interpreter. default=False, action='store_true') parser.add_option('-m', '--inspect-mobi', help='Inspect the MOBI file at the specified path', default=None) + parser.add_option('--test-build', help='Test binary modules in build', + action='store_true', default=False) return parser @@ -232,6 +234,9 @@ def main(args=sys.argv): elif opts.inspect_mobi is not None: from calibre.ebooks.mobi.debug import inspect_mobi inspect_mobi(opts.inspect_mobi) + elif opts.test_build: + from calibre.test_build import test + test() else: from calibre import ipython ipython() diff --git a/src/calibre/devices/android/driver.py b/src/calibre/devices/android/driver.py index b557ac3526..554ea3c698 100644 --- a/src/calibre/devices/android/driver.py +++ b/src/calibre/devices/android/driver.py @@ -53,6 +53,7 @@ class ANDROID(USBMS): 0x681c : [0x0222, 0x0224, 0x0400], 0x6640 : [0x0100], 0x685e : [0x0400], + 0x6860 : [0x0400], 0x6877 : [0x0400], }, diff --git a/src/calibre/devices/misc.py b/src/calibre/devices/misc.py index 936faeb32d..2a6a76719d 100644 --- a/src/calibre/devices/misc.py +++ b/src/calibre/devices/misc.py @@ -255,6 +255,28 @@ class EEEREADER(USBMS): VENDOR_NAME = 'LINUX' WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = 'FILE-STOR_GADGET' +class ADAM(USBMS): + + name = 'Notion Ink Adam device interface' + gui_name = 'Adam' + + description = _('Communicate with the Adam tablet') + author = 'Kovid Goyal' + supported_platforms = ['windows', 'osx', 'linux'] + + # Ordered list of supported formats + FORMATS = ['epub', 'pdf', 'doc'] + + VENDOR_ID = [0x0955] + PRODUCT_ID = [0x7100] + BCD = [0x9999] + + EBOOK_DIR_MAIN = 'eBooks' + + VENDOR_NAME = 'NI' + WINDOWS_MAIN_MEM = WINDOWS_CARD_A_MEM = ['ADAM'] + SUPPORTS_SUB_DIRS = True + class NEXTBOOK(USBMS): name = 'Nextbook device interface' diff --git a/src/calibre/devices/nook/driver.py b/src/calibre/devices/nook/driver.py index 22264c3458..72f2e02508 100644 --- a/src/calibre/devices/nook/driver.py +++ b/src/calibre/devices/nook/driver.py @@ -107,6 +107,9 @@ class NOOK_COLOR(NOOK): return filepath + def upload_cover(self, path, filename, metadata, filepath): + pass + class NOOK_TSR(NOOK): gui_name = _('Nook Simple') description = _('Communicate with the Nook TSR eBook reader.') diff --git a/src/calibre/devices/usbms/books.py b/src/calibre/devices/usbms/books.py index cfebe796a3..731d3e2b49 100644 --- a/src/calibre/devices/usbms/books.py +++ b/src/calibre/devices/usbms/books.py @@ -204,7 +204,8 @@ class CollectionsBookList(BookList): elif fm['datatype'] == 'text' and fm['is_multiple']: val = orig_val elif fm['datatype'] == 'composite' and fm['is_multiple']: - val = [v.strip() for v in val.split(fm['is_multiple'])] + val = [v.strip() for v in + val.split(fm['is_multiple']['ui_to_list'])] else: val = [val] diff --git a/src/calibre/ebooks/metadata/book/base.py b/src/calibre/ebooks/metadata/book/base.py index 378d4ab5f0..382cb6c5a2 100644 --- a/src/calibre/ebooks/metadata/book/base.py +++ b/src/calibre/ebooks/metadata/book/base.py @@ -621,10 +621,7 @@ class Metadata(object): orig_res = res datatype = cmeta['datatype'] if datatype == 'text' and cmeta['is_multiple']: - if cmeta['display'].get('is_names', False): - res = u' & '.join(res) - else: - res = u', '.join(sorted(res, key=sort_key)) + res = cmeta['is_multiple']['list_to_ui'].join(res) elif datatype == 'series' and series_with_index: if self.get_extra(key) is not None: res = res + \ @@ -668,7 +665,7 @@ class Metadata(object): elif datatype == 'text' and fmeta['is_multiple']: if isinstance(res, dict): res = [k + ':' + v for k,v in res.items()] - res = u', '.join(sorted(res, key=sort_key)) + res = fmeta['is_multiple']['list_to_ui'].join(sorted(res, key=sort_key)) elif datatype == 'series' and series_with_index: res = res + ' [%s]'%self.format_series_index() elif datatype == 'datetime': diff --git a/src/calibre/ebooks/metadata/book/json_codec.py b/src/calibre/ebooks/metadata/book/json_codec.py index 1d93b5dece..28bf3178ef 100644 --- a/src/calibre/ebooks/metadata/book/json_codec.py +++ b/src/calibre/ebooks/metadata/book/json_codec.py @@ -5,8 +5,7 @@ Created on 4 Jun 2010 ''' from base64 import b64encode, b64decode -import json -import traceback +import json, traceback from calibre.ebooks.metadata.book import SERIALIZABLE_FIELDS from calibre.constants import filesystem_encoding, preferred_encoding @@ -69,6 +68,40 @@ def object_to_unicode(obj, enc=preferred_encoding): return ans return obj +def encode_is_multiple(fm): + if fm.get('is_multiple', None): + # migrate is_multiple back to a character + fm['is_multiple2'] = fm.get('is_multiple', {}) + dt = fm.get('datatype', None) + if dt == 'composite': + fm['is_multiple'] = ',' + else: + fm['is_multiple'] = '|' + else: + fm['is_multiple'] = None + fm['is_multiple2'] = {} + +def decode_is_multiple(fm): + im = fm.get('is_multiple2', None) + if im: + fm['is_multiple'] = im + del fm['is_multiple2'] + else: + # Must migrate the is_multiple from char to dict + im = fm.get('is_multiple', {}) + if im: + dt = fm.get('datatype', None) + if dt == 'composite': + im = {'cache_to_list': ',', 'ui_to_list': ',', + 'list_to_ui': ', '} + elif fm.get('display', {}).get('is_names', False): + im = {'cache_to_list': '|', 'ui_to_list': '&', + 'list_to_ui': ', '} + else: + im = {'cache_to_list': '|', 'ui_to_list': ',', + 'list_to_ui': ', '} + fm['is_multiple'] = im + class JsonCodec(object): def __init__(self): @@ -93,9 +126,10 @@ class JsonCodec(object): def encode_metadata_attr(self, book, key): if key == 'user_metadata': meta = book.get_all_user_metadata(make_copy=True) - for k in meta: - if meta[k]['datatype'] == 'datetime': - meta[k]['#value#'] = datetime_to_string(meta[k]['#value#']) + for fm in meta.itervalues(): + if fm['datatype'] == 'datetime': + fm['#value#'] = datetime_to_string(fm['#value#']) + encode_is_multiple(fm) return meta if key in self.field_metadata: datatype = self.field_metadata[key]['datatype'] @@ -135,9 +169,10 @@ class JsonCodec(object): if key == 'classifiers': key = 'identifiers' if key == 'user_metadata': - for k in value: - if value[k]['datatype'] == 'datetime': - value[k]['#value#'] = string_to_datetime(value[k]['#value#']) + for fm in value.itervalues(): + if fm['datatype'] == 'datetime': + fm['#value#'] = string_to_datetime(fm['#value#']) + decode_is_multiple(fm) return value elif key in self.field_metadata: if self.field_metadata[key]['datatype'] == 'datetime': diff --git a/src/calibre/ebooks/metadata/opf2.py b/src/calibre/ebooks/metadata/opf2.py index 1d91236757..80fb84633b 100644 --- a/src/calibre/ebooks/metadata/opf2.py +++ b/src/calibre/ebooks/metadata/opf2.py @@ -7,7 +7,7 @@ __docformat__ = 'restructuredtext en' lxml based OPF parser. ''' -import re, sys, unittest, functools, os, uuid, glob, cStringIO, json +import re, sys, unittest, functools, os, uuid, glob, cStringIO, json, copy from urllib import unquote from urlparse import urlparse @@ -453,10 +453,13 @@ class TitleSortField(MetadataField): def serialize_user_metadata(metadata_elem, all_user_metadata, tail='\n'+(' '*8)): from calibre.utils.config import to_json - from calibre.ebooks.metadata.book.json_codec import object_to_unicode + from calibre.ebooks.metadata.book.json_codec import (object_to_unicode, + encode_is_multiple) for name, fm in all_user_metadata.items(): try: + fm = copy.copy(fm) + encode_is_multiple(fm) fm = object_to_unicode(fm) fm = json.dumps(fm, default=to_json, ensure_ascii=False) except: @@ -575,6 +578,7 @@ class OPF(object): # {{{ self._user_metadata_ = {} temp = Metadata('x', ['x']) from calibre.utils.config import from_json + from calibre.ebooks.metadata.book.json_codec import decode_is_multiple elems = self.root.xpath('//*[name() = "meta" and starts-with(@name,' '"calibre:user_metadata:") and @content]') for elem in elems: @@ -585,6 +589,7 @@ class OPF(object): # {{{ fm = elem.get('content') try: fm = json.loads(fm, object_hook=from_json) + decode_is_multiple(fm) temp.set_user_metadata(name, fm) except: prints('Failed to read user metadata:', name) diff --git a/src/calibre/ebooks/metadata/sources/amazon.py b/src/calibre/ebooks/metadata/sources/amazon.py index 7da37ce5af..6220f29020 100644 --- a/src/calibre/ebooks/metadata/sources/amazon.py +++ b/src/calibre/ebooks/metadata/sources/amazon.py @@ -42,6 +42,7 @@ class Worker(Thread): # Get details {{{ months = { 'de': { 1 : ['jän'], + 2 : ['februar'], 3 : ['märz'], 5 : ['mai'], 6 : ['juni'], diff --git a/src/calibre/ebooks/oeb/stylizer.py b/src/calibre/ebooks/oeb/stylizer.py index dc73862022..fc7a27b5cd 100644 --- a/src/calibre/ebooks/oeb/stylizer.py +++ b/src/calibre/ebooks/oeb/stylizer.py @@ -13,7 +13,13 @@ from weakref import WeakKeyDictionary from xml.dom import SyntaxErr as CSSSyntaxError import cssutils from cssutils.css import (CSSStyleRule, CSSPageRule, CSSStyleDeclaration, - CSSValueList, CSSFontFaceRule, cssproperties) + CSSFontFaceRule, cssproperties) +try: + from cssutils.css import CSSValueList + CSSValueList +except ImportError: + # cssutils >= 0.9.8 + from cssutils.css import PropertyValue as CSSValueList from cssutils import profile as cssprofiles from lxml import etree from lxml.cssselect import css_to_xpath, ExpressionError, SelectorSyntaxError diff --git a/src/calibre/gui2/actions/delete.py b/src/calibre/gui2/actions/delete.py index 43465512e0..bef456033b 100644 --- a/src/calibre/gui2/actions/delete.py +++ b/src/calibre/gui2/actions/delete.py @@ -94,6 +94,9 @@ class DeleteAction(InterfaceAction): self.delete_menu.addAction( _('Remove all formats from selected books, except...'), self.delete_all_but_selected_formats) + self.delete_menu.addAction( + _('Remove all formats from selected books'), + self.delete_all_formats) self.delete_menu.addAction( _('Remove covers from selected books'), self.delete_covers) self.delete_menu.addSeparator() @@ -174,6 +177,28 @@ class DeleteAction(InterfaceAction): if ids: self.gui.tags_view.recount() + def delete_all_formats(self, *args): + ids = self._get_selected_ids() + if not ids: + return + if not confirm('

'+_('All formats for the selected books will ' + 'be deleted from your library.
' + 'The book metadata will be kept. Are you sure?') + +'

', 'delete_all_formats', self.gui): + return + db = self.gui.library_view.model().db + for id in ids: + fmts = db.formats(id, index_is_id=True, verify_formats=False) + if fmts: + for fmt in fmts.split(','): + self.gui.library_view.model().db.remove_format(id, fmt, + index_is_id=True, notify=False) + self.gui.library_view.model().refresh_ids(ids) + self.gui.library_view.model().current_changed(self.gui.library_view.currentIndex(), + self.gui.library_view.currentIndex()) + if ids: + self.gui.tags_view.recount() + def remove_matching_books_from_device(self, *args): if not self.gui.device_manager.is_device_connected: d = error_dialog(self.gui, _('Cannot delete books'), diff --git a/src/calibre/gui2/custom_column_widgets.py b/src/calibre/gui2/custom_column_widgets.py index c94913ea2c..4706cce4c9 100644 --- a/src/calibre/gui2/custom_column_widgets.py +++ b/src/calibre/gui2/custom_column_widgets.py @@ -226,16 +226,14 @@ class Comments(Base): class Text(Base): def setup_ui(self, parent): - if self.col_metadata['display'].get('is_names', False): - self.sep = u' & ' - else: - self.sep = u', ' + self.sep = self.col_metadata['multiple_seps'] values = self.all_values = list(self.db.all_custom(num=self.col_id)) values.sort(key=sort_key) + if self.col_metadata['is_multiple']: w = MultiCompleteLineEdit(parent) - w.set_separator(self.sep.strip()) - if self.sep == u' & ': + w.set_separator(self.sep['ui_to_list']) + if self.sep['ui_to_list'] == '&': w.set_space_before_sep(True) w.set_add_separator(tweaks['authors_completer_append_separator']) w.update_items_cache(values) @@ -269,12 +267,12 @@ class Text(Base): if self.col_metadata['is_multiple']: if not val: val = [] - self.widgets[1].setText(self.sep.join(val)) + self.widgets[1].setText(self.sep['list_to_ui'].join(val)) def getter(self): if self.col_metadata['is_multiple']: val = unicode(self.widgets[1].text()).strip() - ans = [x.strip() for x in val.split(self.sep.strip()) if x.strip()] + ans = [x.strip() for x in val.split(self.sep['ui_to_list']) if x.strip()] if not ans: ans = None return ans @@ -899,9 +897,10 @@ class BulkText(BulkBase): if not self.a_c_checkbox.isChecked(): return if self.col_metadata['is_multiple']: + ism = self.col_metadata['multiple_seps'] if self.col_metadata['display'].get('is_names', False): val = self.gui_val - add = [v.strip() for v in val.split('&') if v.strip()] + add = [v.strip() for v in val.split(ism['ui_to_list']) if v.strip()] self.db.set_custom_bulk(book_ids, add, num=self.col_id) else: remove_all, adding, rtext = self.gui_val @@ -911,10 +910,10 @@ class BulkText(BulkBase): else: txt = rtext if txt: - remove = set([v.strip() for v in txt.split(',')]) + remove = set([v.strip() for v in txt.split(ism['ui_to_list'])]) txt = adding if txt: - add = set([v.strip() for v in txt.split(',')]) + add = set([v.strip() for v in txt.split(ism['ui_to_list'])]) else: add = set() self.db.set_custom_bulk_multiple(book_ids, add=add, diff --git a/src/calibre/gui2/dialogs/metadata_bulk.py b/src/calibre/gui2/dialogs/metadata_bulk.py index 66cf55a9b2..8829dc97c0 100644 --- a/src/calibre/gui2/dialogs/metadata_bulk.py +++ b/src/calibre/gui2/dialogs/metadata_bulk.py @@ -520,7 +520,7 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog): elif not fm['is_multiple']: val = [val] elif fm['datatype'] == 'composite': - val = [v.strip() for v in val.split(fm['is_multiple'])] + val = [v.strip() for v in val.split(fm['is_multiple']['ui_to_list'])] elif field == 'authors': val = [v.replace('|', ',') for v in val] else: @@ -655,19 +655,10 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog): if self.destination_field_fm['is_multiple']: if self.comma_separated.isChecked(): - if dest == 'authors' or \ - (self.destination_field_fm['is_custom'] and - self.destination_field_fm['datatype'] == 'text' and - self.destination_field_fm['display'].get('is_names', False)): - splitter = ' & ' - else: - splitter = ',' - + splitter = self.destination_field_fm['is_multiple']['ui_to_list'] res = [] for v in val: - for x in v.split(splitter): - if x.strip(): - res.append(x.strip()) + res.extend([x.strip() for x in v.split(splitter) if x.strip()]) val = res else: val = [v.replace(',', '') for v in val] diff --git a/src/calibre/gui2/dialogs/template_dialog.py b/src/calibre/gui2/dialogs/template_dialog.py index 852bbcc221..f78e7a7383 100644 --- a/src/calibre/gui2/dialogs/template_dialog.py +++ b/src/calibre/gui2/dialogs/template_dialog.py @@ -254,6 +254,15 @@ class TemplateDialog(QDialog, Ui_TemplateDialog): self.textbox_changed() self.rule = (None, '') + tt = _('Template language tutorial') + self.template_tutorial.setText( + '' + '%s'%tt) + tt = _('Template function reference') + self.template_func_reference.setText( + '' + '%s'%tt) + def textbox_changed(self): cur_text = unicode(self.textbox.toPlainText()) if self.last_text != cur_text: @@ -299,4 +308,4 @@ class TemplateDialog(QDialog, Ui_TemplateDialog): return self.rule = (unicode(self.colored_field.currentText()), txt) - QDialog.accept(self) \ No newline at end of file + QDialog.accept(self) diff --git a/src/calibre/gui2/dialogs/template_dialog.ui b/src/calibre/gui2/dialogs/template_dialog.ui index 13586e7049..674100fe04 100644 --- a/src/calibre/gui2/dialogs/template_dialog.ui +++ b/src/calibre/gui2/dialogs/template_dialog.ui @@ -125,6 +125,20 @@ + + + + true + + + + + + + true + + + diff --git a/src/calibre/gui2/dialogs/tweak_epub.py b/src/calibre/gui2/dialogs/tweak_epub.py index 732d74b77d..e0be9fa1e9 100755 --- a/src/calibre/gui2/dialogs/tweak_epub.py +++ b/src/calibre/gui2/dialogs/tweak_epub.py @@ -7,7 +7,7 @@ __copyright__ = '2010, Kovid Goyal ' __docformat__ = 'restructuredtext en' import os, shutil -from zipfile import ZipFile, ZIP_DEFLATED, ZIP_STORED +from calibre.utils.zipfile import ZipFile, ZIP_DEFLATED, ZIP_STORED from PyQt4.Qt import QDialog diff --git a/src/calibre/gui2/library/delegates.py b/src/calibre/gui2/library/delegates.py index b97cb3074a..02f7452694 100644 --- a/src/calibre/gui2/library/delegates.py +++ b/src/calibre/gui2/library/delegates.py @@ -288,6 +288,8 @@ class CcNumberDelegate(QStyledItemDelegate): # {{{ def setEditorData(self, editor, index): m = index.model() val = m.db.data[index.row()][m.custom_columns[m.column_map[index.column()]]['rec_index']] + if val is None: + val = 0 editor.setValue(val) # }}} diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py index 72c8e0629f..72655afd12 100644 --- a/src/calibre/gui2/library/models.py +++ b/src/calibre/gui2/library/models.py @@ -608,10 +608,11 @@ class BooksModel(QAbstractTableModel): # {{{ def text_type(r, mult=None, idx=-1): text = self.db.data[r][idx] - if text and mult is not None: - if mult: - return QVariant(u' & '.join(text.split('|'))) - return QVariant(u', '.join(sorted(text.split('|'),key=sort_key))) + if text and mult: + jv = mult['list_to_ui'] + sv = mult['cache_to_list'] + return QVariant(jv.join( + sorted([t.strip() for t in text.split(sv)], key=sort_key))) return QVariant(text) def decorated_text_type(r, idx=-1): @@ -665,8 +666,6 @@ class BooksModel(QAbstractTableModel): # {{{ datatype = self.custom_columns[col]['datatype'] if datatype in ('text', 'comments', 'composite', 'enumeration'): mult=self.custom_columns[col]['is_multiple'] - if mult is not None: - mult = self.custom_columns[col]['display'].get('is_names', False) self.dc[col] = functools.partial(text_type, idx=idx, mult=mult) if datatype in ['text', 'composite', 'enumeration'] and not mult: if self.custom_columns[col]['display'].get('use_decorations', False): @@ -722,9 +721,9 @@ class BooksModel(QAbstractTableModel): # {{{ if id_ in self.color_cache: if key in self.color_cache[id_]: return self.color_cache[id_][key] - if mi is None: - mi = self.db.get_metadata(id_, index_is_id=True) try: + if mi is None: + mi = self.db.get_metadata(id_, index_is_id=True) color = composite_formatter.safe_format(fmt, mi, '', mi) if color in self.colors: color = QColor(color) diff --git a/src/calibre/gui2/preferences/coloring.py b/src/calibre/gui2/preferences/coloring.py index 0fca30695b..f8376e9b84 100644 --- a/src/calibre/gui2/preferences/coloring.py +++ b/src/calibre/gui2/preferences/coloring.py @@ -159,21 +159,22 @@ class ConditionEditor(QWidget): # {{{ self.action_box.clear() self.action_box.addItem('', '') col = self.current_col - m = self.fm[col] - dt = m['datatype'] - if dt in self.action_map: - actions = self.action_map[dt] - else: - if col == 'ondevice': - k = 'ondevice' - elif col == 'identifiers': - k = 'identifiers' + if col: + m = self.fm[col] + dt = m['datatype'] + if dt in self.action_map: + actions = self.action_map[dt] else: - k = 'multiple' if m['is_multiple'] else 'single' - actions = self.action_map[k] + if col == 'ondevice': + k = 'ondevice' + elif col == 'identifiers': + k = 'identifiers' + else: + k = 'multiple' if m['is_multiple'] else 'single' + actions = self.action_map[k] - for text, key in actions: - self.action_box.addItem(text, key) + for text, key in actions: + self.action_box.addItem(text, key) self.action_box.setCurrentIndex(0) self.action_box.blockSignals(False) self.init_value_box() @@ -184,11 +185,15 @@ class ConditionEditor(QWidget): # {{{ self.value_box.setInputMask('') self.value_box.setValidator(None) col = self.current_col + if not col: + return m = self.fm[col] dt = m['datatype'] action = self.current_action - if not col or not action: + if not action: return + m = self.fm[col] + dt = m['datatype'] tt = '' if col == 'identifiers': tt = _('Enter either an identifier type or an ' @@ -206,7 +211,7 @@ class ConditionEditor(QWidget): # {{{ tt = _('Enter a regular expression') elif m.get('is_multiple', False): tt += '\n' + _('You can match multiple values by separating' - ' them with %s')%m['is_multiple'] + ' them with %s')%m['is_multiple']['ui_to_list'] self.value_box.setToolTip(tt) if action in ('is set', 'is not set', 'is true', 'is false', 'is undefined'): diff --git a/src/calibre/gui2/preferences/create_custom_column.py b/src/calibre/gui2/preferences/create_custom_column.py index 8eaa2dd7d9..d2f1786ab0 100644 --- a/src/calibre/gui2/preferences/create_custom_column.py +++ b/src/calibre/gui2/preferences/create_custom_column.py @@ -13,6 +13,9 @@ from calibre.gui2 import error_dialog class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn): + # Note: in this class, we are treating is_multiple as the boolean that + # custom_columns expects to find in its structure. It does not use the dict + column_types = { 0:{'datatype':'text', 'text':_('Text, column shown in the tag browser'), diff --git a/src/calibre/gui2/preferences/toolbar.ui b/src/calibre/gui2/preferences/toolbar.ui index 0e601f74a2..51819b0df2 100644 --- a/src/calibre/gui2/preferences/toolbar.ui +++ b/src/calibre/gui2/preferences/toolbar.ui @@ -16,13 +16,28 @@ + + + 75 + true + + - Customize the actions in: + Choose the &toolbar to customize: + + + what + + + 75 + true + + QComboBox::AdjustToMinimumContentsLengthWithIcon diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py index 470bbcdfa8..601071a2ce 100644 --- a/src/calibre/library/caches.py +++ b/src/calibre/library/caches.py @@ -509,7 +509,8 @@ class ResultCache(SearchQueryParser): # {{{ valq_mkind, valq = self._matchkind(query) loc = self.field_metadata[location]['rec_index'] - split_char = self.field_metadata[location]['is_multiple'] + split_char = self.field_metadata[location]['is_multiple'].get( + 'cache_to_list', ',') for id_ in candidates: item = self._data[id_] if item is None: @@ -665,7 +666,8 @@ class ResultCache(SearchQueryParser): # {{{ if fm['is_multiple'] and \ len(query) > 1 and query.startswith('#') and \ query[1:1] in '=<>!': - vf = lambda item, loc=fm['rec_index'], ms=fm['is_multiple']:\ + vf = lambda item, loc=fm['rec_index'], \ + ms=fm['is_multiple']['cache_to_list']:\ len(item[loc].split(ms)) if item[loc] is not None else 0 return self.get_numeric_matches(location, query[1:], candidates, val_func=vf) @@ -703,7 +705,8 @@ class ResultCache(SearchQueryParser): # {{{ ['composite', 'text', 'comments', 'series', 'enumeration']: exclude_fields.append(db_col[x]) col_datatype[db_col[x]] = self.field_metadata[x]['datatype'] - is_multiple_cols[db_col[x]] = self.field_metadata[x]['is_multiple'] + is_multiple_cols[db_col[x]] = \ + self.field_metadata[x]['is_multiple'].get('cache_to_list', None) try: rating_query = int(query) * 2 @@ -1045,13 +1048,14 @@ class SortKeyGenerator(object): elif dt in ('text', 'comments', 'composite', 'enumeration'): if val: - sep = fm['is_multiple'] - if sep: - if fm['display'].get('is_names', False): - val = sep.join( - [author_to_author_sort(v) for v in val.split(sep)]) + if fm['is_multiple']: + jv = fm['is_multiple']['list_to_ui'] + sv = fm['is_multiple']['cache_to_list'] + if '&' in jv: + val = jv.join( + [author_to_author_sort(v) for v in val.split(sv)]) else: - val = sep.join(sorted(val.split(sep), + val = jv.join(sorted(val.split(sv), key=self.string_sort_key)) val = self.string_sort_key(val) diff --git a/src/calibre/library/coloring.py b/src/calibre/library/coloring.py index f458b9c04f..584cb01e54 100644 --- a/src/calibre/library/coloring.py +++ b/src/calibre/library/coloring.py @@ -79,16 +79,19 @@ class Rule(object): # {{{ if dt == 'bool': return self.bool_condition(col, action, val) - if dt in ('int', 'float', 'rating'): + if dt in ('int', 'float'): return self.number_condition(col, action, val) + if dt == 'rating': + return self.rating_condition(col, action, val) + if dt == 'datetime': return self.date_condition(col, action, val) if dt in ('comments', 'series', 'text', 'enumeration', 'composite'): ism = m.get('is_multiple', False) if ism: - return self.multiple_condition(col, action, val, ',' if ism == '|' else ism) + return self.multiple_condition(col, action, val, ism['ui_to_list']) return self.text_condition(col, action, val) def identifiers_condition(self, col, action, val): @@ -114,9 +117,16 @@ class Rule(object): # {{{ 'lt': ('1', '', ''), 'gt': ('', '', '1') }[action] - lt, eq, gt = '', '1', '' return "cmp(raw_field('%s'), %s, '%s', '%s', '%s')" % (col, val, lt, eq, gt) + def rating_condition(self, col, action, val): + lt, eq, gt = { + 'eq': ('', '1', ''), + 'lt': ('1', '', ''), + 'gt': ('', '', '1') + }[action] + return "cmp(field('%s'), %s, '%s', '%s', '%s')" % (col, val, lt, eq, gt) + def date_condition(self, col, action, val): lt, eq, gt = { 'eq': ('', '1', ''), diff --git a/src/calibre/library/custom_columns.py b/src/calibre/library/custom_columns.py index 187d718a39..9a2a27aecc 100644 --- a/src/calibre/library/custom_columns.py +++ b/src/calibre/library/custom_columns.py @@ -78,6 +78,18 @@ class CustomColumns(object): } if data['display'] is None: data['display'] = {} + # set up the is_multiple separator dict + if data['is_multiple']: + if data['display'].get('is_names', False): + seps = {'cache_to_list': '|', 'ui_to_list': '&', 'list_to_ui': ' & '} + elif data['datatype'] == 'composite': + seps = {'cache_to_list': ',', 'ui_to_list': ',', 'list_to_ui': ', '} + else: + seps = {'cache_to_list': '|', 'ui_to_list': ',', 'list_to_ui': ', '} + else: + seps = {} + data['multiple_seps'] = seps + table, lt = self.custom_table_names(data['num']) if table not in custom_tables or (data['normalized'] and lt not in custom_tables): @@ -119,7 +131,7 @@ class CustomColumns(object): if x is None: return [] if isinstance(x, (str, unicode, bytes)): - x = x.split('&' if d['display'].get('is_names', False) else',') + x = x.split(d['multiple_seps']['ui_to_list']) x = [y.strip() for y in x if y.strip()] x = [y.decode(preferred_encoding, 'replace') if not isinstance(y, unicode) else y for y in x] @@ -181,10 +193,7 @@ class CustomColumns(object): is_category = True else: is_category = False - if v['is_multiple']: - is_m = ',' if v['datatype'] == 'composite' else '|' - else: - is_m = None + is_m = v['multiple_seps'] tn = 'custom_column_{0}'.format(v['num']) self.field_metadata.add_custom_field(label=v['label'], table=tn, column='value', datatype=v['datatype'], @@ -200,7 +209,7 @@ class CustomColumns(object): row = self.data._data[idx] if index_is_id else self.data[idx] ans = row[self.FIELD_MAP[data['num']]] if data['is_multiple'] and data['datatype'] == 'text': - ans = ans.split('|') if ans else [] + ans = ans.split(data['multiple_seps']['cache_to_list']) if ans else [] if data['display'].get('sort_alpha', False): ans.sort(cmp=lambda x,y:cmp(x.lower(), y.lower())) return ans @@ -566,14 +575,21 @@ class CustomColumns(object): def custom_columns_in_meta(self): lines = {} for data in self.custom_column_label_map.values(): - display = data['display'] table, lt = self.custom_table_names(data['num']) if data['normalized']: query = '%s.value' if data['is_multiple']: - query = 'group_concat(%s.value, "|")' - if not display.get('sort_alpha', False): - query = 'sort_concat(link.id, %s.value)' +# query = 'group_concat(%s.value, "{0}")'.format( +# data['multiple_seps']['cache_to_list']) +# if not display.get('sort_alpha', False): + if data['multiple_seps']['cache_to_list'] == '|': + query = 'sortconcat_bar(link.id, %s.value)' + elif data['multiple_seps']['cache_to_list'] == '&': + query = 'sortconcat_amper(link.id, %s.value)' + else: + prints('WARNING: unknown value in multiple_seps', + data['multiple_seps']['cache_to_list']) + query = 'sortconcat_bar(link.id, %s.value)' line = '''(SELECT {query} FROM {lt} AS link INNER JOIN {table} ON(link.value={table}.id) WHERE link.book=books.id) custom_{num} diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 3a151166e7..9c4c3eb004 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -1250,7 +1250,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): dex = field['rec_index'] for book in self.data.iterall(): if field['is_multiple']: - vals = [v.strip() for v in book[dex].split(field['is_multiple']) + vals = [v.strip() for v in + book[dex].split(field['is_multiple']['cache_to_list']) if v.strip()] if id_ in vals: ans.add(book[0]) @@ -1378,7 +1379,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): tcategories[category] = {} # create a list of category/field_index for the books scan to use. # This saves iterating through field_metadata for each book - md.append((category, cat['rec_index'], cat['is_multiple'], False)) + md.append((category, cat['rec_index'], + cat['is_multiple'].get('cache_to_list', None), False)) for category in tb_cats.iterkeys(): cat = tb_cats[category] @@ -1386,7 +1388,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): cat['display'].get('make_category', False): tids[category] = {} tcategories[category] = {} - md.append((category, cat['rec_index'], cat['is_multiple'], + md.append((category, cat['rec_index'], + cat['is_multiple'].get('cache_to_list', None), cat['datatype'] == 'composite')) #print 'end phase "collection":', time.clock() - last, 'seconds' #last = time.clock() diff --git a/src/calibre/library/field_metadata.py b/src/calibre/library/field_metadata.py index c884542241..231af23038 100644 --- a/src/calibre/library/field_metadata.py +++ b/src/calibre/library/field_metadata.py @@ -50,9 +50,16 @@ class FieldMetadata(dict): datatype: the type of information in the field. Valid values are listed in VALID_DATA_TYPES below. - is_multiple: valid for the text datatype. If None, the field is to be - treated as a single term. If not None, it contains a string, and the field - is assumed to contain a list of terms separated by that string + is_multiple: valid for the text datatype. If {}, the field is to be + treated as a single term. If not None, it contains a dict of the form + {'cache_to_list': ',', + 'ui_to_list': ',', + 'list_to_ui': ', '} + where the cache_to_list contains the character used to split the value in + the meta2 table, ui_to_list contains the character used to create a list + from a value shown in the ui (each resulting value must be strip()ed and + empty values removed), and list_to_ui contains the string used in join() + to create a displayable string from the list. kind == field: is a db field. kind == category: standard tag category that isn't a field. see news. @@ -97,7 +104,9 @@ class FieldMetadata(dict): 'link_column':'author', 'category_sort':'sort', 'datatype':'text', - 'is_multiple':',', + 'is_multiple':{'cache_to_list': ',', + 'ui_to_list': '&', + 'list_to_ui': ' & '}, 'kind':'field', 'name':_('Authors'), 'search_terms':['authors', 'author'], @@ -109,7 +118,7 @@ class FieldMetadata(dict): 'link_column':'series', 'category_sort':'(title_sort(name))', 'datatype':'series', - 'is_multiple':None, + 'is_multiple':{}, 'kind':'field', 'name':_('Series'), 'search_terms':['series'], @@ -119,7 +128,9 @@ class FieldMetadata(dict): ('formats', {'table':None, 'column':None, 'datatype':'text', - 'is_multiple':',', + 'is_multiple':{'cache_to_list': ',', + 'ui_to_list': ',', + 'list_to_ui': ', '}, 'kind':'field', 'name':_('Formats'), 'search_terms':['formats', 'format'], @@ -131,7 +142,7 @@ class FieldMetadata(dict): 'link_column':'publisher', 'category_sort':'name', 'datatype':'text', - 'is_multiple':None, + 'is_multiple':{}, 'kind':'field', 'name':_('Publishers'), 'search_terms':['publisher'], @@ -143,7 +154,7 @@ class FieldMetadata(dict): 'link_column':'rating', 'category_sort':'rating', 'datatype':'rating', - 'is_multiple':None, + 'is_multiple':{}, 'kind':'field', 'name':_('Ratings'), 'search_terms':['rating'], @@ -154,7 +165,7 @@ class FieldMetadata(dict): 'column':'name', 'category_sort':'name', 'datatype':None, - 'is_multiple':None, + 'is_multiple':{}, 'kind':'category', 'name':_('News'), 'search_terms':[], @@ -166,7 +177,9 @@ class FieldMetadata(dict): 'link_column': 'tag', 'category_sort':'name', 'datatype':'text', - 'is_multiple':',', + 'is_multiple':{'cache_to_list': ',', + 'ui_to_list': ',', + 'list_to_ui': ', '}, 'kind':'field', 'name':_('Tags'), 'search_terms':['tags', 'tag'], @@ -176,7 +189,9 @@ class FieldMetadata(dict): ('identifiers', {'table':None, 'column':None, 'datatype':'text', - 'is_multiple':',', + 'is_multiple':{'cache_to_list': ',', + 'ui_to_list': ',', + 'list_to_ui': ', '}, 'kind':'field', 'name':_('Identifiers'), 'search_terms':['identifiers', 'identifier', 'isbn'], @@ -186,7 +201,7 @@ class FieldMetadata(dict): ('author_sort',{'table':None, 'column':None, 'datatype':'text', - 'is_multiple':None, + 'is_multiple':{}, 'kind':'field', 'name':_('Author Sort'), 'search_terms':['author_sort'], @@ -196,7 +211,9 @@ class FieldMetadata(dict): ('au_map', {'table':None, 'column':None, 'datatype':'text', - 'is_multiple':',', + 'is_multiple':{'cache_to_list': ',', + 'ui_to_list': None, + 'list_to_ui': None}, 'kind':'field', 'name':None, 'search_terms':[], @@ -206,7 +223,7 @@ class FieldMetadata(dict): ('comments', {'table':None, 'column':None, 'datatype':'text', - 'is_multiple':None, + 'is_multiple':{}, 'kind':'field', 'name':_('Comments'), 'search_terms':['comments', 'comment'], @@ -216,7 +233,7 @@ class FieldMetadata(dict): ('cover', {'table':None, 'column':None, 'datatype':'int', - 'is_multiple':None, + 'is_multiple':{}, 'kind':'field', 'name':None, 'search_terms':['cover'], @@ -226,7 +243,7 @@ class FieldMetadata(dict): ('id', {'table':None, 'column':None, 'datatype':'int', - 'is_multiple':None, + 'is_multiple':{}, 'kind':'field', 'name':None, 'search_terms':[], @@ -236,7 +253,7 @@ class FieldMetadata(dict): ('last_modified', {'table':None, 'column':None, 'datatype':'datetime', - 'is_multiple':None, + 'is_multiple':{}, 'kind':'field', 'name':_('Modified'), 'search_terms':['last_modified'], @@ -246,7 +263,7 @@ class FieldMetadata(dict): ('ondevice', {'table':None, 'column':None, 'datatype':'text', - 'is_multiple':None, + 'is_multiple':{}, 'kind':'field', 'name':_('On Device'), 'search_terms':['ondevice'], @@ -256,7 +273,7 @@ class FieldMetadata(dict): ('path', {'table':None, 'column':None, 'datatype':'text', - 'is_multiple':None, + 'is_multiple':{}, 'kind':'field', 'name':_('Path'), 'search_terms':[], @@ -266,7 +283,7 @@ class FieldMetadata(dict): ('pubdate', {'table':None, 'column':None, 'datatype':'datetime', - 'is_multiple':None, + 'is_multiple':{}, 'kind':'field', 'name':_('Published'), 'search_terms':['pubdate'], @@ -276,7 +293,7 @@ class FieldMetadata(dict): ('marked', {'table':None, 'column':None, 'datatype':'text', - 'is_multiple':None, + 'is_multiple':{}, 'kind':'field', 'name': None, 'search_terms':['marked'], @@ -286,7 +303,7 @@ class FieldMetadata(dict): ('series_index',{'table':None, 'column':None, 'datatype':'float', - 'is_multiple':None, + 'is_multiple':{}, 'kind':'field', 'name':None, 'search_terms':['series_index'], @@ -296,7 +313,7 @@ class FieldMetadata(dict): ('sort', {'table':None, 'column':None, 'datatype':'text', - 'is_multiple':None, + 'is_multiple':{}, 'kind':'field', 'name':_('Title Sort'), 'search_terms':['title_sort'], @@ -306,7 +323,7 @@ class FieldMetadata(dict): ('size', {'table':None, 'column':None, 'datatype':'float', - 'is_multiple':None, + 'is_multiple':{}, 'kind':'field', 'name':_('Size'), 'search_terms':['size'], @@ -316,7 +333,7 @@ class FieldMetadata(dict): ('timestamp', {'table':None, 'column':None, 'datatype':'datetime', - 'is_multiple':None, + 'is_multiple':{}, 'kind':'field', 'name':_('Date'), 'search_terms':['date'], @@ -326,7 +343,7 @@ class FieldMetadata(dict): ('title', {'table':None, 'column':None, 'datatype':'text', - 'is_multiple':None, + 'is_multiple':{}, 'kind':'field', 'name':_('Title'), 'search_terms':['title'], @@ -336,7 +353,7 @@ class FieldMetadata(dict): ('uuid', {'table':None, 'column':None, 'datatype':'text', - 'is_multiple':None, + 'is_multiple':{}, 'kind':'field', 'name':None, 'search_terms':[], @@ -508,7 +525,7 @@ class FieldMetadata(dict): if datatype == 'series': key += '_index' self._tb_cats[key] = {'table':None, 'column':None, - 'datatype':'float', 'is_multiple':None, + 'datatype':'float', 'is_multiple':{}, 'kind':'field', 'name':'', 'search_terms':[key], 'label':label+'_index', 'colnum':None, 'display':{}, @@ -560,7 +577,7 @@ class FieldMetadata(dict): if icu_lower(label) != label: st.append(icu_lower(label)) self._tb_cats[label] = {'table':None, 'column':None, - 'datatype':None, 'is_multiple':None, + 'datatype':None, 'is_multiple':{}, 'kind':'user', 'name':name, 'search_terms':st, 'is_custom':False, 'is_category':True, 'is_csp': False} @@ -570,7 +587,7 @@ class FieldMetadata(dict): if label in self._tb_cats: raise ValueError('Duplicate user field [%s]'%(label)) self._tb_cats[label] = {'table':None, 'column':None, - 'datatype':None, 'is_multiple':None, + 'datatype':None, 'is_multiple':{}, 'kind':'search', 'name':name, 'search_terms':[], 'is_custom':False, 'is_category':True, 'is_csp': False} diff --git a/src/calibre/library/restore.py b/src/calibre/library/restore.py index e03edd449a..20065309aa 100644 --- a/src/calibre/library/restore.py +++ b/src/calibre/library/restore.py @@ -171,7 +171,7 @@ class Restore(Thread): for x in fields: if x in cfm: if x == 'is_multiple': - args.append(cfm[x] is not None) + args.append(bool(cfm[x])) else: args.append(cfm[x]) if len(args) == len(fields): diff --git a/src/calibre/library/server/mobile.py b/src/calibre/library/server/mobile.py index 1bf9f549bc..ad5ee4af96 100644 --- a/src/calibre/library/server/mobile.py +++ b/src/calibre/library/server/mobile.py @@ -231,7 +231,8 @@ class MobileServer(object): book['size'] = human_readable(book['size']) aus = record[FM['authors']] if record[FM['authors']] else __builtin__._('Unknown') - authors = '|'.join([i.replace('|', ',') for i in aus.split(',')]) + aut_is = CFM['authors']['is_multiple'] + authors = aut_is['list_to_ui'].join([i.replace('|', ',') for i in aus.split(',')]) book['authors'] = authors book['series_index'] = fmt_sidx(float(record[FM['series_index']])) book['series'] = record[FM['series']] @@ -254,8 +255,10 @@ class MobileServer(object): continue if datatype == 'text' and CFM[key]['is_multiple']: book[key] = concat(name, - format_tag_string(val, ',', - no_tag_count=True)) + format_tag_string(val, + CFM[key]['is_multiple']['ui_to_list'], + no_tag_count=True, + joinval=CFM[key]['is_multiple']['list_to_ui'])) else: book[key] = concat(name, val) diff --git a/src/calibre/library/server/opds.py b/src/calibre/library/server/opds.py index 5f6180e68a..04300ea0e3 100644 --- a/src/calibre/library/server/opds.py +++ b/src/calibre/library/server/opds.py @@ -180,9 +180,12 @@ def ACQUISITION_ENTRY(item, version, db, updated, CFM, CKEYS, prefix): if val: datatype = CFM[key]['datatype'] if datatype == 'text' and CFM[key]['is_multiple']: - extra.append('%s: %s
'%(xml(name), xml(format_tag_string(val, ',', - ignore_max=True, - no_tag_count=True)))) + extra.append('%s: %s
'% + (xml(name), + xml(format_tag_string(val, + CFM[key]['is_multiple']['ui_to_list'], + ignore_max=True, no_tag_count=True, + joinval=CFM[key]['is_multiple']['list_to_ui'])))) elif datatype == 'comments': extra.append('%s: %s
'%(xml(name), comments_to_html(unicode(val)))) else: diff --git a/src/calibre/library/server/utils.py b/src/calibre/library/server/utils.py index e58dd2f19b..53c6cdbd9d 100644 --- a/src/calibre/library/server/utils.py +++ b/src/calibre/library/server/utils.py @@ -68,7 +68,7 @@ def strftime(fmt='%Y/%m/%d %H:%M:%S', dt=None): except: return _strftime(fmt, nowf().timetuple()) -def format_tag_string(tags, sep, ignore_max=False, no_tag_count=False): +def format_tag_string(tags, sep, ignore_max=False, no_tag_count=False, joinval=', '): MAX = sys.maxint if ignore_max else tweaks['max_content_server_tags_shown'] if tags: tlist = [t.strip() for t in tags.split(sep)] @@ -78,10 +78,10 @@ def format_tag_string(tags, sep, ignore_max=False, no_tag_count=False): if len(tlist) > MAX: tlist = tlist[:MAX]+['...'] if no_tag_count: - return ', '.join(tlist) if tlist else '' + return joinval.join(tlist) if tlist else '' else: return u'%s:&:%s'%(tweaks['max_content_server_tags_shown'], - ', '.join(tlist)) if tlist else '' + joinval.join(tlist)) if tlist else '' def quote(s): if isinstance(s, unicode): diff --git a/src/calibre/library/server/xml.py b/src/calibre/library/server/xml.py index 14955dc541..18ddf6bb43 100644 --- a/src/calibre/library/server/xml.py +++ b/src/calibre/library/server/xml.py @@ -121,8 +121,12 @@ class XMLServer(object): name = CFM[key]['name'] custcols.append(k) if datatype == 'text' and CFM[key]['is_multiple']: - kwargs[k] = concat('#T#'+name, format_tag_string(val,',', - ignore_max=True)) + kwargs[k] = \ + concat('#T#'+name, + format_tag_string(val, + CFM[key]['is_multiple']['ui_to_list'], + ignore_max=True, + joinval=CFM[key]['is_multiple']['list_to_ui'])) else: kwargs[k] = concat(name, val) kwargs['custcols'] = ','.join(custcols) diff --git a/src/calibre/library/sqlite.py b/src/calibre/library/sqlite.py index 511106fe7b..96874d2c27 100644 --- a/src/calibre/library/sqlite.py +++ b/src/calibre/library/sqlite.py @@ -121,9 +121,12 @@ class SortedConcatenate(object): return None return self.sep.join(map(self.ans.get, sorted(self.ans.keys()))) -class SafeSortedConcatenate(SortedConcatenate): +class SortedConcatenateBar(SortedConcatenate): sep = '|' +class SortedConcatenateAmper(SortedConcatenate): + sep = '&' + class IdentifiersConcat(object): '''String concatenation aggregator for the identifiers map''' def __init__(self): @@ -220,7 +223,8 @@ class DBThread(Thread): self.conn.execute('pragma cache_size=5000') encoding = self.conn.execute('pragma encoding').fetchone()[0] self.conn.create_aggregate('sortconcat', 2, SortedConcatenate) - self.conn.create_aggregate('sort_concat', 2, SafeSortedConcatenate) + self.conn.create_aggregate('sortconcat_bar', 2, SortedConcatenateBar) + self.conn.create_aggregate('sortconcat_amper', 2, SortedConcatenateAmper) self.conn.create_aggregate('identifiers_concat', 2, IdentifiersConcat) load_c_extensions(self.conn) self.conn.row_factory = sqlite.Row if self.row_factory else lambda cursor, row : list(row) diff --git a/src/calibre/library/sqlite_custom.c b/src/calibre/library/sqlite_custom.c index dee17c79d4..52f0be4575 100644 --- a/src/calibre/library/sqlite_custom.c +++ b/src/calibre/library/sqlite_custom.c @@ -141,6 +141,22 @@ static void sort_concat_finalize2(sqlite3_context *context) { } +static void sort_concat_finalize3(sqlite3_context *context) { + SortConcatList *list; + unsigned char *ans; + + list = (SortConcatList*) sqlite3_aggregate_context(context, sizeof(*list)); + + if (list != NULL && list->vals != NULL && list->count > 0) { + qsort(list->vals, list->count, sizeof(list->vals[0]), sort_concat_cmp); + ans = sort_concat_do_finalize(list, '&'); + if (ans != NULL) sqlite3_result_text(context, (char*)ans, -1, SQLITE_TRANSIENT); + free(ans); + sort_concat_free(list); + } + +} + // }}} // identifiers_concat {{{ @@ -237,7 +253,8 @@ MYEXPORT int sqlite3_extension_init( sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi){ SQLITE_EXTENSION_INIT2(pApi); sqlite3_create_function(db, "sortconcat", 2, SQLITE_UTF8, NULL, NULL, sort_concat_step, sort_concat_finalize); - sqlite3_create_function(db, "sort_concat", 2, SQLITE_UTF8, NULL, NULL, sort_concat_step, sort_concat_finalize2); + sqlite3_create_function(db, "sortconcat_bar", 2, SQLITE_UTF8, NULL, NULL, sort_concat_step, sort_concat_finalize2); + sqlite3_create_function(db, "sortconcat_amper", 2, SQLITE_UTF8, NULL, NULL, sort_concat_step, sort_concat_finalize3); sqlite3_create_function(db, "identifiers_concat", 2, SQLITE_UTF8, NULL, NULL, identifiers_concat_step, identifiers_concat_finalize); return 0; } diff --git a/src/calibre/linux.py b/src/calibre/linux.py index 9e58d4f638..1686f66b22 100644 --- a/src/calibre/linux.py +++ b/src/calibre/linux.py @@ -23,7 +23,6 @@ entry_points = { 'calibre-server = calibre.library.server.main:main', 'lrf2lrs = calibre.ebooks.lrf.lrfparser:main', 'lrs2lrf = calibre.ebooks.lrf.lrs.convert_from:main', - 'librarything = calibre.ebooks.metadata.library_thing:main', 'calibre-debug = calibre.debug:main', 'calibredb = calibre.library.cli:main', 'calibre-parallel = calibre.utils.ipc.worker:main', diff --git a/src/calibre/manual/faq.rst b/src/calibre/manual/faq.rst index 99c53e5a37..c1aa4e5614 100644 --- a/src/calibre/manual/faq.rst +++ b/src/calibre/manual/faq.rst @@ -562,6 +562,16 @@ You have two choices: 1. Create a patch by hacking on |app| and send it to me for review and inclusion. See `Development `_. 2. `Open a ticket `_ (you have to register and login first). Remember that |app| development is done by volunteers, so if you get no response to your feature request, it means no one feels like implementing it. +Why doesn't |app| have an automatic update? +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +For many reasons: + + * *There is no need to update every week*. If you are happy with how |app| works turn off the update notification and be on your merry way. Check back to see if you want to update once a year or so. + * Pre downloading the updates for all users in the background would mean require about 80TB of bandwidth *every week*. That costs thousands of dollars a month. And |app| is currently growing at 300,000 new users every month. + * If I implement a dialog that downloads the update and launches it, instead of going to the website as it does now, that would save the most ardent |app| updater, *at most five clicks a week*. There are far higher priority things to do in |app| development. + * If you really, really hate downloading |app| every week but still want to be upto the latest, I encourage you to run from source, which makes updating trivial. Instructions are :ref:`here `. + How is |app| licensed? ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ |app| is licensed under the GNU General Public License v3 (an open source license). This means that you are free to redistribute |app| as long as you make the source code available. So if you want to put |app| on a CD with your product, you must also put the |app| source code on the CD. The source code is available for download `from googlecode `_. You are free to use the results of conversions from |app| however you want. You cannot use code, libraries from |app| in your software without making your software open source. For details, see `The GNU GPL v3 `_. diff --git a/src/calibre/test_build.py b/src/calibre/test_build.py new file mode 100644 index 0000000000..0610f01805 --- /dev/null +++ b/src/calibre/test_build.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +from __future__ import (unicode_literals, division, absolute_import, + print_function) +from future_builtins import map + +__license__ = 'GPL v3' +__copyright__ = '2011, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + +''' +Test a binary calibre build to ensure that all needed binary images/libraries have loaded. +''' + +import cStringIO +from calibre.constants import plugins, iswindows + +def test_plugins(): + for name in plugins: + mod, err = plugins[name] + if err or not mod: + raise RuntimeError('Plugin %s failed to load with error: %s' % + (name, err)) + print (mod, 'loaded') + +def test_lxml(): + from lxml import etree + raw = '' + root = etree.fromstring(raw) + if etree.tostring(root) == raw: + print ('lxml OK!') + else: + raise RuntimeError('lxml failed') + +def test_fontconfig(): + from calibre.utils.fonts import fontconfig + families = fontconfig.find_font_families() + num = len(families) + if num < 10: + raise RuntimeError('Fontconfig found only %d font families'%num) + print ('Fontconfig OK! (%d families)'%num) + +def test_winutil(): + from calibre.devices.scanner import win_pnp_drives + matches = win_pnp_drives.scanner() + if len(matches) < 1: + raise RuntimeError('win_pnp_drives returned no drives') + print ('win_pnp_drives OK!') + +def test_win32(): + from calibre.utils.winshell import desktop + d = desktop() + if not d: + raise RuntimeError('winshell failed') + print ('winshell OK! (%s is the desktop)'%d) + +def test_sqlite(): + import sqlite3 + conn = sqlite3.connect(':memory:') + from calibre.library.sqlite import load_c_extensions + if not load_c_extensions(conn, True): + raise RuntimeError('Failed to load sqlite extension') + print ('sqlite OK!') + +def test_qt(): + from PyQt4.Qt import (QWebView, QDialog, QImageReader, QNetworkAccessManager) + fmts = set(map(unicode, QImageReader.supportedImageFormats())) + if 'jpg' not in fmts or 'png' not in fmts: + raise RuntimeError( + "Qt doesn't seem to be able to load its image plugins") + QWebView, QDialog + na = QNetworkAccessManager() + if not hasattr(na, 'sslErrors'): + raise RuntimeError('Qt not compiled with openssl') + print ('Qt OK!') + +def test_imaging(): + from calibre.utils.magick.draw import create_canvas, Image + im = create_canvas(20, 20, '#ffffff') + jpg = im.export('jpg') + Image().load(jpg) + im.export('png') + print ('ImageMagick OK!') + from PIL import Image + i = Image.open(cStringIO.StringIO(jpg)) + if i.size != (20, 20): + raise RuntimeError('PIL choked!') + print ('PIL OK!') + +def test(): + test_plugins() + test_lxml() + test_fontconfig() + test_sqlite() + if iswindows: + test_winutil() + test_win32() + test_qt() + test_imaging() + +if __name__ == '__main__': + test() + diff --git a/src/calibre/translations/dynamic.py b/src/calibre/translations/dynamic.py index c1f368ff5a..4d65475ac0 100644 --- a/src/calibre/translations/dynamic.py +++ b/src/calibre/translations/dynamic.py @@ -5,10 +5,9 @@ Dynamic language lookup of translations for user-visible strings. __license__ = 'GPL v3' __copyright__ = '2008, Marshall T. Vandegrift ' -import os - +import cStringIO from gettext import GNUTranslations -from calibre.utils.localization import get_lc_messages_path +from calibre.utils.localization import get_lc_messages_path, ZipFile __all__ = ['translate'] @@ -21,10 +20,15 @@ def translate(lang, text): else: mpath = get_lc_messages_path(lang) if mpath is not None: - p = os.path.join(mpath, 'messages.mo') - if os.path.exists(p): - trans = GNUTranslations(open(p, 'rb')) - _CACHE[lang] = trans + with ZipFile(P('localization/locales.zip', + allow_user_override=False), 'r') as zf: + try: + buf = cStringIO.StringIO(zf.read(mpath + '/messages.mo')) + except: + pass + else: + trans = GNUTranslations(buf) + _CACHE[lang] = trans if trans is None: return getattr(__builtins__, '_', lambda x: x)(text) return trans.ugettext(text) diff --git a/src/calibre/utils/config_base.py b/src/calibre/utils/config_base.py index 7660370353..345caae384 100644 --- a/src/calibre/utils/config_base.py +++ b/src/calibre/utils/config_base.py @@ -223,8 +223,7 @@ class OptionSet(object): if val is val is True or val is False or val is None or \ isinstance(val, (int, float, long, basestring)): return repr(val) - from PyQt4.QtCore import QString - if isinstance(val, QString): + if val.__class__.__name__ == 'QString': return repr(unicode(val)) pickle = cPickle.dumps(val, -1) return 'cPickle.loads(%s)'%repr(pickle) diff --git a/src/calibre/utils/formatter_functions.py b/src/calibre/utils/formatter_functions.py index 1a8867b44e..4c1cec6462 100644 --- a/src/calibre/utils/formatter_functions.py +++ b/src/calibre/utils/formatter_functions.py @@ -727,13 +727,8 @@ class BuiltinNot(BuiltinFormatterFunction): 'returns the empty string. This function works well with test or ' 'first_non_empty. You can have as many values as you want.') - def evaluate(self, formatter, kwargs, mi, locals, *args): - i = 0 - while i < len(args): - if args[i]: - return '1' - i += 1 - return '' + def evaluate(self, formatter, kwargs, mi, locals, val): + return '' if val else '1' class BuiltinMergeLists(BuiltinFormatterFunction): name = 'merge_lists' diff --git a/src/calibre/utils/localization.py b/src/calibre/utils/localization.py index 92e6ea9b5e..a8285ad5a2 100644 --- a/src/calibre/utils/localization.py +++ b/src/calibre/utils/localization.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai -from __future__ import with_statement +from __future__ import absolute_import __license__ = 'GPL v3' __copyright__ = '2009, Kovid Goyal ' @@ -8,13 +8,14 @@ __docformat__ = 'restructuredtext en' import os, locale, re, cStringIO, cPickle from gettext import GNUTranslations +from zipfile import ZipFile _available_translations = None def available_translations(): global _available_translations if _available_translations is None: - stats = P('localization/stats.pickle') + stats = P('localization/stats.pickle', allow_user_override=False) if os.path.exists(stats): stats = cPickle.load(open(stats, 'rb')) else: @@ -49,21 +50,20 @@ def get_lang(): lang = 'en' return lang -def messages_path(lang): - return P('localization/locales/%s/LC_MESSAGES'%lang) - def get_lc_messages_path(lang): hlang = None - if lang in available_translations(): - hlang = lang - else: - xlang = lang.split('_')[0] - if xlang in available_translations(): - hlang = xlang - if hlang is not None: - return messages_path(hlang) - return None + if zf_exists(): + if lang in available_translations(): + hlang = lang + else: + xlang = lang.split('_')[0] + if xlang in available_translations(): + hlang = xlang + return hlang +def zf_exists(): + return os.path.exists(P('localization/locales.zip', + allow_user_override=False)) def set_translators(): # To test different translations invoke as @@ -79,12 +79,17 @@ def set_translators(): mpath = get_lc_messages_path(lang) if mpath is not None: - if buf is None: - buf = open(os.path.join(mpath, 'messages.mo'), 'rb') - mpath = mpath.replace(os.sep+'nds'+os.sep, os.sep+'de'+os.sep) - isof = os.path.join(mpath, 'iso639.mo') - if os.path.exists(isof): - iso639 = open(isof, 'rb') + with ZipFile(P('localization/locales.zip', + allow_user_override=False), 'r') as zf: + if buf is None: + buf = cStringIO.StringIO(zf.read(mpath + '/messages.mo')) + if mpath == 'nds': + mpath = 'de' + isof = mpath + '/iso639.mo' + try: + iso639 = cStringIO.StringIO(zf.read(isof)) + except: + pass # No iso639 translations for this lang if buf is not None: t = GNUTranslations(buf) diff --git a/src/calibre/utils/zipfile.py b/src/calibre/utils/zipfile.py index fa15bff4b4..868445d15a 100644 --- a/src/calibre/utils/zipfile.py +++ b/src/calibre/utils/zipfile.py @@ -148,6 +148,12 @@ def decode_arcname(name): name = name.decode('utf-8', 'replace') return name +# Added by Kovid to reset timestamp to default if it overflows the DOS +# limits +def fixtimevar(val): + if val < 0 or val > 0xffff: + val = 0 + return val def _check_zipfile(fp): try: @@ -341,6 +347,7 @@ class ZipInfo (object): dt = self.date_time dosdate = (dt[0] - 1980) << 9 | dt[1] << 5 | dt[2] dostime = dt[3] << 11 | dt[4] << 5 | (dt[5] // 2) + if self.flag_bits & 0x08: # Set these to zero because we write them after the file data CRC = compress_size = file_size = 0 @@ -365,7 +372,7 @@ class ZipInfo (object): filename, flag_bits = self._encodeFilenameFlags() header = struct.pack(structFileHeader, stringFileHeader, self.extract_version, self.reserved, flag_bits, - self.compress_type, dostime, dosdate, CRC, + self.compress_type, fixtimevar(dostime), fixtimevar(dosdate), CRC, compress_size, file_size, len(filename), len(extra)) return header + filename + extra @@ -1321,8 +1328,8 @@ class ZipFile: for zinfo in self.filelist: # write central directory count = count + 1 dt = zinfo.date_time - dosdate = (dt[0] - 1980) << 9 | dt[1] << 5 | dt[2] - dostime = dt[3] << 11 | dt[4] << 5 | (dt[5] // 2) + dosdate = fixtimevar((dt[0] - 1980) << 9 | dt[1] << 5 | dt[2]) + dostime = fixtimevar(dt[3] << 11 | dt[4] << 5 | (dt[5] // 2)) extra = [] if zinfo.file_size > ZIP64_LIMIT \ or zinfo.compress_size > ZIP64_LIMIT: