From 606ee6958278272144f9b15f02762d74c17a13e6 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 8 Mar 2014 07:24:35 +0530 Subject: [PATCH 001/122] ... --- recipes/der_spiegel.recipe | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/recipes/der_spiegel.recipe b/recipes/der_spiegel.recipe index 9ea4be6201..2405b75427 100644 --- a/recipes/der_spiegel.recipe +++ b/recipes/der_spiegel.recipe @@ -44,9 +44,9 @@ class DerSpiegel(BasicNewsRecipe): br = BasicNewsRecipe.get_browser(self) if self.username is not None and self.password is not None: - br.open(self.PREFIX + '/meinspiegel/login.html') + br.open(self.PREFIX + '/meinspiegel/login.html?backUrl=' + self.PREFIX + '/spiegel/print') br.select_form(predicate=has_login_name) - br['f.loginName' ] = self.username + br['f.loginName'] = self.username br['f.password'] = self.password br.submit() return br @@ -80,4 +80,4 @@ class DerSpiegel(BasicNewsRecipe): url = self.PREFIX + link['href'] articles.append({'title' : title, 'date' : strftime(self.timefmt), 'url' : url}) feeds.append((section_title,articles)) - return feeds; + return feeds From a33265cf46bf269a1f015ace04f1111b8f0095e6 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 8 Mar 2014 08:08:46 +0530 Subject: [PATCH 002/122] Content server: Fix (maybe) an error on some windows computers with a non-standard default encoding See http://www.mobileread.com/forums/showthread.php?t=235366 --- src/calibre/library/server/base.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/calibre/library/server/base.py b/src/calibre/library/server/base.py index e64319e88b..69d187dfa3 100644 --- a/src/calibre/library/server/base.py +++ b/src/calibre/library/server/base.py @@ -47,6 +47,10 @@ class DispatchController(object): # {{{ aw = kwargs.pop('android_workaround', False) if route != '/': route = self.prefix + route + if isinstance(route, unicode): + # Apparently the routes package chokes on unicode routes, see + # http://www.mobileread.com/forums/showthread.php?t=235366 + route = route.encode('utf-8') elif self.prefix: self.dispatcher.connect(name+'prefix_extra', self.prefix, self, **kwargs) From 5816c4aaeb9a0fb7e0cac09ed2692670e34eba7f Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 8 Mar 2014 08:27:27 +0530 Subject: [PATCH 003/122] ... --- recipes/american_thinker.recipe | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipes/american_thinker.recipe b/recipes/american_thinker.recipe index c4dc7328f7..6ecca8549c 100644 --- a/recipes/american_thinker.recipe +++ b/recipes/american_thinker.recipe @@ -15,6 +15,7 @@ class AmericanThinker(BasicNewsRecipe): max_articles_per_feed = 50 summary_length = 150 language = 'en' + ignore_duplicate_articles = {'title', 'url'} remove_javascript = True no_stylesheets = True @@ -34,4 +35,3 @@ class AmericanThinker(BasicNewsRecipe): def print_version(self, url): return 'http://www.americanthinker.com/assets/3rd_party/printpage/?url=' + url - return 'http://www.americanthinker.com/printpage/?url=' + url From 37ac52ad32ef1cbd87d3bc6264e7fa4433b0a21b Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 8 Mar 2014 09:27:40 +0530 Subject: [PATCH 004/122] Allow capsule based access to the ICU collator --- src/calibre/utils/icu.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/calibre/utils/icu.c b/src/calibre/utils/icu.c index db6bdea876..93d66a20a2 100644 --- a/src/calibre/utils/icu.c +++ b/src/calibre/utils/icu.c @@ -131,6 +131,12 @@ icu_Collator_actual_locale(icu_Collator *self, void *closure) { // }}} +// Collator.capsule {{{ +static PyObject * +icu_Collator_capsule(icu_Collator *self, void *closure) { + return PyCapsule_New(self->collator, NULL, NULL); +} // }}} + // Collator.sort_key {{{ static PyObject * icu_Collator_sort_key(icu_Collator *self, PyObject *args, PyObject *kwargs) { @@ -411,6 +417,11 @@ static PyGetSetDef icu_Collator_getsetters[] = { (char *)"Actual locale used by this collator.", NULL}, + {(char *)"capsule", + (getter)icu_Collator_capsule, NULL, + (char *)"A capsule enclosing the pointer to the ICU collator struct", + NULL}, + {(char *)"display_name", (getter)icu_Collator_display_name, NULL, (char *)"Display name of this collator in English. The name reflects the actual data source used.", From 6e9afc0398fc60cb7a20c8d5b4198c1dbf443df3 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 8 Mar 2014 10:56:31 +0530 Subject: [PATCH 005/122] ... --- setup/extensions.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/setup/extensions.py b/setup/extensions.py index 7a22836ebe..c3a7fe4550 100644 --- a/setup/extensions.py +++ b/setup/extensions.py @@ -303,9 +303,10 @@ if islinux or isosx: if isunix: cc = os.environ.get('CC', 'gcc') cxx = os.environ.get('CXX', 'g++') + debug = '' + # debug = '-ggdb' cflags = os.environ.get('OVERRIDE_CFLAGS', - # '-Wall -DNDEBUG -ggdb -fno-strict-aliasing -pipe') - '-Wall -DNDEBUG -fno-strict-aliasing -pipe') + '-Wall -DNDEBUG %s -fno-strict-aliasing -pipe' % debug) cflags = shlex.split(cflags) + ['-fPIC'] ldflags = os.environ.get('OVERRIDE_LDFLAGS', '-Wall') ldflags = shlex.split(ldflags) From b672f4ed119646a66c80dd67db708f466d1bcc54 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 8 Mar 2014 10:59:19 +0530 Subject: [PATCH 006/122] Subsequence matcher: Use primary collation --- src/calibre/gui2/tweak_book/matcher.c | 72 +++++++++++++++++++++----- src/calibre/gui2/tweak_book/matcher.py | 69 ++++++++++++++++++++---- 2 files changed, 116 insertions(+), 25 deletions(-) diff --git a/src/calibre/gui2/tweak_book/matcher.c b/src/calibre/gui2/tweak_book/matcher.c index e9c773a0c3..6ea1062499 100644 --- a/src/calibre/gui2/tweak_book/matcher.c +++ b/src/calibre/gui2/tweak_book/matcher.c @@ -171,12 +171,13 @@ static void convert_positions(int32_t *positions, int32_t *final_positions, UCha } } -static double process_item(MatchInfo *m, Stack *stack, int32_t *final_positions) { - UChar32 nc, hc, lc; - UChar *p; +static double process_item(MatchInfo *m, Stack *stack, int32_t *final_positions, UStringSearch **searches) { + UChar32 hc, lc; double final_score = 0.0, score = 0.0, score_for_char = 0.0; int32_t pos, i, j, hidx, nidx, last_idx, distance, *positions = final_positions + m->needle_len; MemoryItem mem = {0}; + UStringSearch *search = NULL; + UErrorCode status = U_ZERO_ERROR; stack_push(stack, 0, 0, 0, 0.0, final_positions); @@ -187,11 +188,14 @@ static double process_item(MatchInfo *m, Stack *stack, int32_t *final_positions) // No memoized result, calculate the score for (i = nidx; i < m->needle_len;) { nidx = i; - U16_NEXT(m->needle, i, m->needle_len, nc); // i now points to next char in needle - if (m->haystack_len - hidx < m->needle_len - nidx) { score = 0.0; break; } - p = u_strchr32(m->haystack + hidx, nc); // TODO: Use primary collation for the find - if (p == NULL) { score = 0.0; break; } - pos = (int32_t)(p - m->haystack); + U16_FWD_1(m->needle, i, m->needle_len);// i now points to next char in needle + search = searches[nidx]; + if (search == NULL || m->haystack_len - hidx < m->needle_len - nidx) { score = 0.0; break; } + status = U_ZERO_ERROR; // We ignore any errors as we already know that hidx is correct + usearch_setOffset(search, hidx, &status); + status = U_ZERO_ERROR; + pos = usearch_next(search, &status); + if (pos == USEARCH_DONE) { score = 0.0; break; } // No matches found distance = u_countChar32(m->haystack + last_idx, pos - last_idx); if (distance <= 1) score_for_char = m->max_score_per_char; else { @@ -222,8 +226,30 @@ static double process_item(MatchInfo *m, Stack *stack, int32_t *final_positions) return final_score; } +static bool create_searches(UStringSearch **searches, UChar *haystack, int32_t haystack_len, UChar *needle, int32_t needle_len, UCollator *collator) { + int32_t i = 0, pos = 0; + UErrorCode status = U_ZERO_ERROR; -static bool match(UChar **items, int32_t *item_lengths, uint32_t item_count, UChar *needle, Match *match_results, int32_t *final_positions, int32_t needle_char_len, UChar *level1, UChar *level2, UChar *level3) { + while (i < needle_len) { + pos = i; + U16_FWD_1(needle, i, needle_len); + if (pos == i) break; + searches[pos] = usearch_openFromCollator(needle + pos, i - pos, haystack, haystack_len, collator, NULL, &status); + if (U_FAILURE(status)) { PyErr_SetString(PyExc_ValueError, u_errorName(status)); searches[pos] = NULL; return FALSE; } + } + + return TRUE; +} + +static void free_searches(UStringSearch **searches, int32_t count) { + int32_t i = 0; + for (i = 0; i < count; i++) { + if (searches[i] != NULL) usearch_close(searches[i]); + searches[i] = NULL; + } +} + +static bool match(UChar **items, int32_t *item_lengths, uint32_t item_count, UChar *needle, Match *match_results, int32_t *final_positions, int32_t needle_char_len, UCollator *collator, UChar *level1, UChar *level2, UChar *level3) { Stack stack = {0}; int32_t i = 0, maxhl = 0; int32_t r = 0, *positions = NULL; @@ -231,6 +257,7 @@ static bool match(UChar **items, int32_t *item_lengths, uint32_t item_count, UCh bool ok = FALSE; MemoryItem ***memo = NULL; int32_t needle_len = u_strlen(needle); + UStringSearch **searches = NULL; if (needle_len <= 0 || item_count <= 0) { for (i = 0; i < (int32_t)item_count; i++) match_results[i].score = 0.0; @@ -240,7 +267,8 @@ static bool match(UChar **items, int32_t *item_lengths, uint32_t item_count, UCh matches = (MatchInfo*)calloc(item_count, sizeof(MatchInfo)); positions = (int32_t*)calloc(2*needle_len, sizeof(int32_t)); // One set of positions is the final answer and one set is working space - if (matches == NULL || positions == NULL) {PyErr_NoMemory(); goto end;} + searches = (UStringSearch**) calloc(needle_len, sizeof(UStringSearch*)); + if (matches == NULL || positions == NULL || searches == NULL) {PyErr_NoMemory(); goto end;} for (i = 0; i < (int32_t)item_count; i++) { matches[i].haystack = items[i]; @@ -270,8 +298,10 @@ static bool match(UChar **items, int32_t *item_lengths, uint32_t item_count, UCh } stack_clear(&stack); clear_memory(memo, needle_len, matches[i].haystack_len); + free_searches(searches, needle_len); + if (!create_searches(searches, matches[i].haystack, matches[i].haystack_len, needle, needle_len, collator)) goto end; matches[i].memo = memo; - match_results[i].score = process_item(&matches[i], &stack, positions); + match_results[i].score = process_item(&matches[i], &stack, positions, searches); convert_positions(positions, final_positions + i, matches[i].haystack, needle_char_len, needle_len, match_results[i].score); } @@ -281,6 +311,7 @@ end: nullfree(stack.items); nullfree(matches); nullfree(memo); + if (searches != NULL) { free_searches(searches, needle_len); nullfree(searches); } return ok; } @@ -296,6 +327,7 @@ typedef struct { UChar *level1; UChar *level2; UChar *level3; + UCollator *collator; } Matcher; @@ -308,6 +340,7 @@ static void free_matcher(Matcher *self) { } nullfree(self->items); nullfree(self->item_lengths); nullfree(self->level1); nullfree(self->level2); nullfree(self->level3); + if (self->collator != NULL) ucol_close(self->collator); self->collator = NULL; } static void Matcher_dealloc(Matcher* self) @@ -320,10 +353,21 @@ Matcher_dealloc(Matcher* self) static int Matcher_init(Matcher *self, PyObject *args, PyObject *kwds) { - PyObject *items = NULL, *p = NULL, *py_items = NULL, *level1 = NULL, *level2 = NULL, *level3 = NULL; + PyObject *items = NULL, *p = NULL, *py_items = NULL, *level1 = NULL, *level2 = NULL, *level3 = NULL, *collator = NULL; int32_t i = 0; + UErrorCode status = U_ZERO_ERROR; + UCollator *col = NULL; + + if (!PyArg_ParseTuple(args, "OOOOO", &items, &collator, &level1, &level2, &level3)) return -1; + + // Clone the passed in collator (cloning is needed as collators are not thread safe) + if (!PyCapsule_CheckExact(collator)) { PyErr_SetString(PyExc_TypeError, "Collator must be a capsule"); return -1; } + col = (UCollator*)PyCapsule_GetPointer(collator, NULL); + if (col == NULL) return -1; + self->collator = ucol_safeClone(col, NULL, NULL, &status); + col = NULL; + if (U_FAILURE(status)) { self->collator = NULL; PyErr_SetString(PyExc_ValueError, u_errorName(status)); return -1; } - if (!PyArg_ParseTuple(args, "OOOO", &items, &level1, &level2, &level3)) return -1; py_items = PySequence_Fast(items, "Must pass in two sequence objects"); if (py_items == NULL) goto end; self->item_count = (uint32_t)PySequence_Size(items); @@ -378,7 +422,7 @@ Matcher_calculate_scores(Matcher *self, PyObject *args) { } Py_BEGIN_ALLOW_THREADS; - ok = match(self->items, self->item_lengths, self->item_count, needle, matches, final_positions, needle_char_len, self->level1, self->level2, self->level3); + ok = match(self->items, self->item_lengths, self->item_count, needle, matches, final_positions, needle_char_len, self->collator, self->level1, self->level2, self->level3); Py_END_ALLOW_THREADS; if (ok) { diff --git a/src/calibre/gui2/tweak_book/matcher.py b/src/calibre/gui2/tweak_book/matcher.py index bf7840d7be..4c4c20501d 100644 --- a/src/calibre/gui2/tweak_book/matcher.py +++ b/src/calibre/gui2/tweak_book/matcher.py @@ -6,29 +6,76 @@ from __future__ import (unicode_literals, division, absolute_import, __license__ = 'GPL v3' __copyright__ = '2014, Kovid Goyal ' +import atexit +from math import ceil from unicodedata import normalize +from threading import Thread, Lock +from Queue import Queue from itertools import izip from future_builtins import map +from calibre import detect_ncpus as cpu_count from calibre.constants import plugins -from calibre.utils.icu import primary_sort_key, find +from calibre.utils.icu import primary_sort_key, primary_find, primary_collator DEFAULT_LEVEL1 = '/' DEFAULT_LEVEL2 = '-_ 0123456789' DEFAULT_LEVEL3 = '.' +class Worker(Thread): + + daemon = True + + def __init__(self, requests, results): + Thread.__init__(self) + self.requests, self.results = requests, results + atexit.register(lambda : requests.put(None)) + + def run(self): + while True: + x = self.requests.get() + if x is None: + break + try: + self.results.put((True, self.process_query(*x))) + except: + import traceback + self.results.put((False, traceback.format_exc())) +wlock = Lock() +workers = [] + +def split(tasks, pool_size): + ''' + Split a list into a list of sub lists, with the number of sub lists being + no more than the number of workers this server supports. Each sublist contains + 2-tuples of the form (i, x) where x is an element from the original list + and i is the index of the element x in the original list. + ''' + ans, count, pos = [], 0, 0 + delta = int(ceil(len(tasks)/pool_size)) + while count < len(tasks): + section = [] + for t in tasks[pos:pos+delta]: + section.append((count, t)) + count += 1 + ans.append(section) + pos += delta + return ans + + class Matcher(object): def __init__(self, items, level1=DEFAULT_LEVEL1, level2=DEFAULT_LEVEL2, level3=DEFAULT_LEVEL3): + with wlock: + if not workers: + requests, results = Queue(), Queue() + w = [Worker(requests, results) for i in range(max(1, cpu_count()))] + [x.start() for x in w] + workers.extend(w) items = map(lambda x: normalize('NFC', unicode(x)), filter(None, items)) - items = tuple(map(lambda x: x.encode('utf-8'), items)) - sort_keys = tuple(map(primary_sort_key, items)) - - speedup, err = plugins['matcher'] - if speedup is None: - raise RuntimeError('Failed to load the matcher plugin with error: %s' % err) - self.m = speedup.Matcher(items, sort_keys, level1.encode('utf-8'), level2.encode('utf-8'), level3.encode('utf-8')) + self.items = items = tuple(items) + self.sort_keys = tuple(map(primary_sort_key, items)) def __call__(self, query): query = normalize('NFC', unicode(query)).encode('utf-8') @@ -65,7 +112,7 @@ def process_item(ctx, haystack, needle): if (len(haystack) - hidx < len(needle) - i): score = 0 break - pos = find(n, haystack[hidx:])[0] + hidx + pos = primary_find(n, haystack[hidx:])[0] + hidx if pos == -1: score = 0 break @@ -106,7 +153,7 @@ class CScorer(object): speedup, err = plugins['matcher'] if speedup is None: raise RuntimeError('Failed to load the matcher plugin with error: %s' % err) - self.m = speedup.Matcher(items, unicode(level1), unicode(level2), unicode(level3)) + self.m = speedup.Matcher(items, primary_collator().capsule, unicode(level1), unicode(level2), unicode(level3)) def __call__(self, query): query = normalize('NFC', unicode(query)) @@ -120,7 +167,7 @@ def test(): c = CScorer(items) for q in (s, c): print (q) - for item, (score, positions) in izip(items, q('mno')): + for item, (score, positions) in izip(items, q('MNO')): print (item, score, positions) def test_mem(): From 35c837b83943982cffc3042f42b4dcba773d95e9 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 8 Mar 2014 11:24:15 +0530 Subject: [PATCH 007/122] Move matcher module into the utils package --- setup/extensions.py | 4 ++-- src/calibre/{gui2/tweak_book => utils}/matcher.c | 0 src/calibre/{gui2/tweak_book => utils}/matcher.py | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename src/calibre/{gui2/tweak_book => utils}/matcher.c (100%) rename src/calibre/{gui2/tweak_book => utils}/matcher.py (100%) diff --git a/setup/extensions.py b/setup/extensions.py index c3a7fe4550..8050fce363 100644 --- a/setup/extensions.py +++ b/setup/extensions.py @@ -179,12 +179,12 @@ extensions = [ ), Extension('matcher', - ['calibre/gui2/tweak_book/matcher.c'], + ['calibre/utils/matcher.c'], headers=['calibre/utils/icu_calibre_utils.h'], libraries=icu_libs, lib_dirs=icu_lib_dirs, cflags=icu_cflags, - inc_dirs=icu_inc_dirs + ['calibre/utils'] + inc_dirs=icu_inc_dirs ), Extension('podofo', diff --git a/src/calibre/gui2/tweak_book/matcher.c b/src/calibre/utils/matcher.c similarity index 100% rename from src/calibre/gui2/tweak_book/matcher.c rename to src/calibre/utils/matcher.c diff --git a/src/calibre/gui2/tweak_book/matcher.py b/src/calibre/utils/matcher.py similarity index 100% rename from src/calibre/gui2/tweak_book/matcher.py rename to src/calibre/utils/matcher.py From 09be666ea0602a4451267e6c78204f57c4a37403 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 8 Mar 2014 18:11:08 +0530 Subject: [PATCH 008/122] When reading metadata from filenames, do not apply the fallback regexp to read metadata if the user specified regexp puts the entire filename into the title. The fallback is only used if the user specified expression does not match the filename at all. --- src/calibre/ebooks/metadata/meta.py | 26 +++++++++----------------- 1 file changed, 9 insertions(+), 17 deletions(-) diff --git a/src/calibre/ebooks/metadata/meta.py b/src/calibre/ebooks/metadata/meta.py index ecb681056c..9c3a79cc70 100644 --- a/src/calibre/ebooks/metadata/meta.py +++ b/src/calibre/ebooks/metadata/meta.py @@ -100,27 +100,17 @@ def _get_metadata(stream, stream_type, use_libprs_metadata, if use_libprs_metadata and getattr(opf, 'application_id', None) is not None: return opf - mi = MetaInformation(None, None) name = os.path.basename(getattr(stream, 'name', '')) - base = metadata_from_filename(name, pat=pattern) - if force_read_metadata or prefs['read_file_metadata']: - mi = get_file_type_metadata(stream, stream_type) - if base.title == os.path.splitext(name)[0] and \ - base.is_null('authors') and base.is_null('isbn'): - # Assume that there was no metadata in the file and the user set pattern - # to match meta info from the file name did not match. - # The regex is meant to match the standard format filenames are written - # in the library title - author.extension - base.smart_update(metadata_from_filename(name, re.compile( - r'^(?P.+)[ _]-[ _](?P<author>[^-]+)$'))) - if base.title: - base.title = base.title.replace('_', ' ') - if base.authors: - base.authors = [a.replace('_', ' ').strip() for a in base.authors] + # The fallback pattern matches the default filename format produced by calibre + base = metadata_from_filename(name, pat=pattern, fallback_pat=re.compile( + r'^(?P<title>.+) - (?P<author>[^-]+)$')) if not base.authors: base.authors = [_('Unknown')] if not base.title: base.title = _('Unknown') + mi = MetaInformation(None, None) + if force_read_metadata or prefs['read_file_metadata']: + mi = get_file_type_metadata(stream, stream_type) base.smart_update(mi) if opf is not None: base.smart_update(opf) @@ -133,7 +123,7 @@ def set_metadata(stream, mi, stream_type='lrf'): set_file_type_metadata(stream, mi, stream_type) -def metadata_from_filename(name, pat=None): +def metadata_from_filename(name, pat=None, fallback_pat=None): if isbytestring(name): name = name.decode(filesystem_encoding, 'replace') name = name.rpartition('.')[0] @@ -142,6 +132,8 @@ def metadata_from_filename(name, pat=None): pat = re.compile(prefs.get('filename_pattern')) name = name.replace('_', ' ') match = pat.search(name) + if match is None and fallback_pat is not None: + match = fallback_pat.search(name) if match is not None: try: mi.title = match.group('title') From bd88666bb001207ac24f067056d56dfaa907495a Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Sat, 8 Mar 2014 20:55:20 +0530 Subject: [PATCH 009/122] Command line interface to filesystem matcher Also fix a couple of bugs in the matcher algorithms --- src/calibre/utils/matcher.c | 15 ++-- src/calibre/utils/matcher.py | 159 +++++++++++++++++++++++++++-------- 2 files changed, 129 insertions(+), 45 deletions(-) diff --git a/src/calibre/utils/matcher.c b/src/calibre/utils/matcher.c index 6ea1062499..bf5dead056 100644 --- a/src/calibre/utils/matcher.c +++ b/src/calibre/utils/matcher.c @@ -158,10 +158,9 @@ static void convert_positions(int32_t *positions, int32_t *final_positions, UCha // The positions array stores character positions as byte offsets in string, convert them into character offsets int32_t i, *end; - if (score == 0.0) { - for (i = 0; i < char_len; i++) final_positions[i] = -1; - return; - } + if (score == 0.0) { for (i = 0; i < char_len; i++) final_positions[i] = -1; return; } + + if (char_len == byte_len) { memcpy(final_positions, positions, sizeof(*positions) * char_len); return; } end = final_positions + char_len; for (i = 0; i < byte_len && final_positions < end; i++) { @@ -293,16 +292,14 @@ static bool match(UChar **items, int32_t *item_lengths, uint32_t item_count, UCh if (stack.items == NULL || memo == NULL) {PyErr_NoMemory(); goto end;} for (i = 0; i < (int32_t)item_count; i++) { - for (r = 0; r < needle_len; r++) { - positions[r] = -1; - } + for (r = 0; r < needle_len; r++) positions[r] = -1; stack_clear(&stack); clear_memory(memo, needle_len, matches[i].haystack_len); free_searches(searches, needle_len); if (!create_searches(searches, matches[i].haystack, matches[i].haystack_len, needle, needle_len, collator)) goto end; matches[i].memo = memo; match_results[i].score = process_item(&matches[i], &stack, positions, searches); - convert_positions(positions, final_positions + i, matches[i].haystack, needle_char_len, needle_len, match_results[i].score); + convert_positions(positions, final_positions + i * needle_char_len, matches[i].haystack, needle_char_len, needle_len, match_results[i].score); } ok = TRUE; @@ -430,7 +427,7 @@ Matcher_calculate_scores(Matcher *self, PyObject *args) { score = PyFloat_FromDouble(matches[i].score); if (score == NULL) { PyErr_NoMemory(); goto end; } PyTuple_SET_ITEM(items, (Py_ssize_t)i, score); - p = final_positions + i; + p = final_positions + (i * needle_char_len); for (j = 0; j < needle_char_len; j++) { score = PyInt_FromLong((long)p[j]); if (score == NULL) { PyErr_NoMemory(); goto end; } diff --git a/src/calibre/utils/matcher.py b/src/calibre/utils/matcher.py index 4c4c20501d..a07aa75875 100644 --- a/src/calibre/utils/matcher.py +++ b/src/calibre/utils/matcher.py @@ -6,17 +6,20 @@ from __future__ import (unicode_literals, division, absolute_import, __license__ = 'GPL v3' __copyright__ = '2014, Kovid Goyal <kovid at kovidgoyal.net>' -import atexit +import atexit, os, sys from math import ceil from unicodedata import normalize from threading import Thread, Lock from Queue import Queue +from operator import itemgetter +from collections import OrderedDict +from itertools import islice from itertools import izip from future_builtins import map -from calibre import detect_ncpus as cpu_count -from calibre.constants import plugins +from calibre import detect_ncpus as cpu_count, as_unicode +from calibre.constants import plugins, filesystem_encoding from calibre.utils.icu import primary_sort_key, primary_find, primary_collator DEFAULT_LEVEL1 = '/' @@ -38,35 +41,35 @@ class Worker(Thread): if x is None: break try: - self.results.put((True, self.process_query(*x))) - except: - import traceback - self.results.put((False, traceback.format_exc())) + i, scorer, query = x + self.results.put((True, (i, scorer(query)))) + except Exception as e: + self.results.put((False, as_unicode(e))) + # import traceback + # traceback.print_exc() wlock = Lock() workers = [] def split(tasks, pool_size): ''' Split a list into a list of sub lists, with the number of sub lists being - no more than the number of workers this server supports. Each sublist contains + no more than pool_size. Each sublist contains 2-tuples of the form (i, x) where x is an element from the original list and i is the index of the element x in the original list. ''' - ans, count, pos = [], 0, 0 + ans, count = [], 0 delta = int(ceil(len(tasks)/pool_size)) - while count < len(tasks): - section = [] - for t in tasks[pos:pos+delta]: - section.append((count, t)) - count += 1 + while tasks: + section = [(count+i, task) for i, task in enumerate(tasks[:delta])] + tasks = tasks[delta:] + count += len(section) ans.append(section) - pos += delta return ans class Matcher(object): - def __init__(self, items, level1=DEFAULT_LEVEL1, level2=DEFAULT_LEVEL2, level3=DEFAULT_LEVEL3): + def __init__(self, items, level1=DEFAULT_LEVEL1, level2=DEFAULT_LEVEL2, level3=DEFAULT_LEVEL3, scorer=None): with wlock: if not workers: requests, results = Queue(), Queue() @@ -75,12 +78,57 @@ class Matcher(object): workers.extend(w) items = map(lambda x: normalize('NFC', unicode(x)), filter(None, items)) self.items = items = tuple(items) - self.sort_keys = tuple(map(primary_sort_key, items)) + tasks = split(items, len(workers)) + self.task_maps = [{j:i for j, (i, _) in enumerate(task)} for task in tasks] + scorer = scorer or CScorer + self.scorers = [scorer(tuple(map(itemgetter(1), task_items))) for task_items in tasks] + self.sort_keys = None def __call__(self, query): - query = normalize('NFC', unicode(query)).encode('utf-8') - return map(lambda x:x.decode('utf-8'), self.m.get_matches(query)) + query = normalize('NFC', unicode(query)) + with wlock: + for i, scorer in enumerate(self.scorers): + workers[0].requests.put((i, scorer, query)) + if self.sort_keys is None: + self.sort_keys = {i:primary_sort_key(x) for i, x in enumerate(self.items)} + num = len(self.task_maps) + scores, positions = {}, {} + error = None + while num > 0: + ok, x = workers[0].results.get() + num -= 1 + if ok: + task_num, vals = x + task_map = self.task_maps[task_num] + for i, (score, pos) in enumerate(vals): + item = task_map[i] + scores[item] = score + positions[item] = pos + else: + error = x + if error is not None: + raise Exception('Failed to score items: %s' % error) + items = sorted(((-scores[i], item, positions[i]) for i, item in enumerate(self.items)), + key=itemgetter(0)) + return OrderedDict(x[1:] for x in items) + +def get_items_from_dir(basedir): + if isinstance(basedir, bytes): + basedir = basedir.decode(filesystem_encoding) + relsep = os.sep != '/' + for dirpath, dirnames, filenames in os.walk(basedir): + for f in filenames: + x = os.path.join(dirpath, f) + x = os.path.relpath(x, basedir) + if relsep: + x = x.replace(os.sep, '/') + yield x + +class FilesystemMatcher(Matcher): + + def __init__(self, basedir, *args, **kwargs): + Matcher.__init__(self, get_items_from_dir(basedir), *args, **kwargs) def calc_score_for_char(ctx, prev, current, distance): factor = 1.0 @@ -112,10 +160,11 @@ def process_item(ctx, haystack, needle): if (len(haystack) - hidx < len(needle) - i): score = 0 break - pos = primary_find(n, haystack[hidx:])[0] + hidx + pos = primary_find(n, haystack[hidx:])[0] if pos == -1: score = 0 break + pos += hidx distance = pos - last_idx score_for_char = ctx.max_score_per_char if distance <= 1 else calc_score_for_char(ctx, haystack[pos-1], haystack[pos], distance) @@ -137,7 +186,7 @@ class PyScorer(object): def __init__(self, items, level1=DEFAULT_LEVEL1, level2=DEFAULT_LEVEL2, level3=DEFAULT_LEVEL3): self.level1, self.level2, self.level3 = level1, level2, level3 self.max_score_per_char = 0 - self.items = map(lambda x: normalize('NFC', unicode(x)), filter(None, items)) + self.items = items def __call__(self, needle): for item in self.items: @@ -148,7 +197,6 @@ class PyScorer(object): class CScorer(object): def __init__(self, items, level1=DEFAULT_LEVEL1, level2=DEFAULT_LEVEL2, level3=DEFAULT_LEVEL3): - items = tuple(map(lambda x: normalize('NFC', unicode(x)), filter(None, items))) speedup, err = plugins['matcher'] if speedup is None: @@ -156,23 +204,32 @@ class CScorer(object): self.m = speedup.Matcher(items, primary_collator().capsule, unicode(level1), unicode(level2), unicode(level3)) def __call__(self, query): - query = normalize('NFC', unicode(query)) scores, positions = self.m.calculate_scores(query) for score, pos in izip(scores, positions): yield score, pos -def test(): - items = ['m1mn34o/mno'] - s = PyScorer(items) - c = CScorer(items) - for q in (s, c): +def test2(): + items = ['.driveinfo.calibre', 'Suspense.xls', 'p/parsed/content.opf', 'ns.html'] + for q in (PyScorer, CScorer): print (q) - for item, (score, positions) in izip(items, q('MNO')): - print (item, score, positions) + m = Matcher(items, scorer=q) + for item, positions in m('ns').iteritems(): + print ('\tns', item, positions) + +def test(): + items = ['m1mn34o/mno', 'xxx/XXX', 'mxnxox'] + for q in (PyScorer, CScorer): + print (q) + m = Matcher(items, scorer=q) + for item, positions in m('MNO').iteritems(): + print ('\tMNO', item, positions) + for item, positions in m('xxx').iteritems(): + print ('\txxx', item, positions) def test_mem(): from calibre.utils.mem import gc_histogram, diff_hists - m = Matcher([]) + m = Matcher(['a']) + m('a') del m def doit(c): m = Matcher([c+'im/one.gif', c+'im/two.gif', c+'text/one.html',]) @@ -182,12 +239,42 @@ def test_mem(): h1 = gc_histogram() for i in xrange(100): doit(str(i)) + gc.collect() h2 = gc_histogram() diff_hists(h1, h2) +def main(basedir=None, query=None): + from calibre import prints + from calibre.utils.terminal import ColoredStream + if basedir is None: + try: + basedir = raw_input('Enter directory to scan [%s]: ' % os.getcwdu()).decode(sys.stdin.encoding).strip() or os.getcwdu() + except (EOFError, KeyboardInterrupt): + return + m = FilesystemMatcher(basedir) + emph = ColoredStream(sys.stdout, fg='red', bold=True) + while True: + if query is None: + try: + query = raw_input('Enter query: ').decode(sys.stdin.encoding) + except (EOFError, KeyboardInterrupt): + break + if not query: + break + for path, positions in islice(m(query).iteritems(), 0, 10): + positions = list(positions) + p = 0 + while positions: + pos = positions.pop(0) + if pos == -1: + break + prints(path[p:pos], end='') + with emph: + prints(path[pos], end='') + p = pos + 1 + prints(path[p:]) + query = None + if __name__ == '__main__': - test() - # m = Matcher(['image/one.png', 'image/two.gif', 'text/one.html']) - # for q in ('one', 'ONE', 'ton', 'imo'): - # print (q, '->', tuple(m(q))) - # test_mem() + # main(basedir='/t', query='ns') + main() From b8e414f18bedb21ff6e5badb19e9d3851d824551 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Sat, 8 Mar 2014 21:12:38 +0530 Subject: [PATCH 010/122] Revert a part of the previous commit that was left in by mistake and also add a test for handling of positions when the haystack contains non-BMP chars --- src/calibre/utils/matcher.c | 2 -- src/calibre/utils/matcher.py | 10 +++++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/calibre/utils/matcher.c b/src/calibre/utils/matcher.c index bf5dead056..209a7e390d 100644 --- a/src/calibre/utils/matcher.c +++ b/src/calibre/utils/matcher.c @@ -160,8 +160,6 @@ static void convert_positions(int32_t *positions, int32_t *final_positions, UCha if (score == 0.0) { for (i = 0; i < char_len; i++) final_positions[i] = -1; return; } - if (char_len == byte_len) { memcpy(final_positions, positions, sizeof(*positions) * char_len); return; } - end = final_positions + char_len; for (i = 0; i < byte_len && final_positions < end; i++) { if (positions[i] == -1) continue; diff --git a/src/calibre/utils/matcher.py b/src/calibre/utils/matcher.py index a07aa75875..de7ed95f4b 100644 --- a/src/calibre/utils/matcher.py +++ b/src/calibre/utils/matcher.py @@ -217,14 +217,17 @@ def test2(): print ('\tns', item, positions) def test(): - items = ['m1mn34o/mno', 'xxx/XXX', 'mxnxox'] + items = ['mx\U0001f431nxox'] for q in (PyScorer, CScorer): print (q) m = Matcher(items, scorer=q) for item, positions in m('MNO').iteritems(): print ('\tMNO', item, positions) - for item, positions in m('xxx').iteritems(): - print ('\txxx', item, positions) + if -1 not in positions: + for p in positions: + print (item[p], end=' ') + print () + def test_mem(): from calibre.utils.mem import gc_histogram, diff_hists @@ -277,4 +280,5 @@ def main(basedir=None, query=None): if __name__ == '__main__': # main(basedir='/t', query='ns') + # test() main() From f078cd71683ee5589519160700969b6eff8068d8 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Fri, 7 Mar 2014 21:46:01 +0530 Subject: [PATCH 011/122] Performance improvements and code cleanup for the ICU module --- src/calibre/gui2/complete2.py | 4 +- src/calibre/test_build.py | 7 +- src/calibre/utils/icu.c | 494 ++++++++++--------------- src/calibre/utils/icu.py | 666 ++++++++++------------------------ src/calibre/utils/icu_test.py | 148 ++++++++ 5 files changed, 523 insertions(+), 796 deletions(-) create mode 100644 src/calibre/utils/icu_test.py diff --git a/src/calibre/gui2/complete2.py b/src/calibre/gui2/complete2.py index 8aa28069f8..623215a6e6 100644 --- a/src/calibre/gui2/complete2.py +++ b/src/calibre/gui2/complete2.py @@ -14,13 +14,13 @@ from PyQt4.Qt import (QLineEdit, QAbstractListModel, Qt, pyqtSignal, QObject, QApplication, QListView, QPoint, QModelIndex, QFont, QFontInfo) from calibre.constants import isosx, get_osx_version -from calibre.utils.icu import sort_key, primary_startswith, primary_icu_find +from calibre.utils.icu import sort_key, primary_startswith, primary_find from calibre.gui2 import NONE from calibre.gui2.widgets import EnComboBox, LineEditECM from calibre.utils.config import tweaks def containsq(x, prefix): - return primary_icu_find(prefix, x)[0] != -1 + return primary_find(prefix, x)[0] != -1 class CompleteModel(QAbstractListModel): # {{{ diff --git a/src/calibre/test_build.py b/src/calibre/test_build.py index 618626883e..fd9a36df61 100644 --- a/src/calibre/test_build.py +++ b/src/calibre/test_build.py @@ -113,10 +113,9 @@ def test_ssl(): print ('SSL OK!') def test_icu(): - from calibre.utils.icu import _icu_not_ok, test_roundtrip - if _icu_not_ok: - raise RuntimeError('ICU module not loaded/valid') - test_roundtrip() + print ('Testing ICU') + from calibre.utils.icu_test import test_build + test_build() print ('ICU OK!') def test_wpd(): diff --git a/src/calibre/utils/icu.c b/src/calibre/utils/icu.c index 93d66a20a2..d556115c45 100644 --- a/src/calibre/utils/icu.c +++ b/src/calibre/utils/icu.c @@ -1,5 +1,9 @@ #include "icu_calibre_utils.h" +#define UPPER_CASE 0 +#define LOWER_CASE 1 +#define TITLE_CASE 2 + static PyObject* uchar_to_unicode(const UChar *src, int32_t len) { wchar_t *buf = NULL; PyObject *ans = NULL; @@ -66,20 +70,16 @@ icu_Collator_display_name(icu_Collator *self, void *closure) { const char *loc = NULL; UErrorCode status = U_ZERO_ERROR; UChar dname[400]; - char buf[100]; + int32_t sz = 0; loc = ucol_getLocaleByType(self->collator, ULOC_ACTUAL_LOCALE, &status); - if (loc == NULL || U_FAILURE(status)) { + if (loc == NULL) { PyErr_SetString(PyExc_Exception, "Failed to get actual locale"); return NULL; } - ucol_getDisplayName(loc, "en", dname, 100, &status); - if (U_FAILURE(status)) return PyErr_NoMemory(); + sz = ucol_getDisplayName(loc, "en", dname, sizeof(dname), &status); + if (U_FAILURE(status)) {PyErr_SetString(PyExc_ValueError, u_errorName(status)); return NULL; } - u_strToUTF8(buf, 100, NULL, dname, -1, &status); - if (U_FAILURE(status)) { - PyErr_SetString(PyExc_Exception, "Failed to convert dname to UTF-8"); return NULL; - } - return Py_BuildValue("s", buf); + return icu_to_python(dname, sz); } // }}} @@ -140,47 +140,29 @@ icu_Collator_capsule(icu_Collator *self, void *closure) { // Collator.sort_key {{{ static PyObject * icu_Collator_sort_key(icu_Collator *self, PyObject *args, PyObject *kwargs) { - char *input; - int32_t sz; - UChar *buf; - uint8_t *buf2; - PyObject *ans; - int32_t key_size; - UErrorCode status = U_ZERO_ERROR; + int32_t sz = 0, key_size = 0, bsz = 0; + UChar *buf = NULL; + uint8_t *buf2 = NULL; + PyObject *ans = NULL, *input = NULL; - if (!PyArg_ParseTuple(args, "es", "UTF-8", &input)) return NULL; + if (!PyArg_ParseTuple(args, "O", &input)) return NULL; + buf = python_to_icu(input, &sz, 1); + if (buf == NULL) return NULL; - sz = (int32_t)strlen(input); + bsz = 7 * sz + 1; + buf2 = (uint8_t*)calloc(bsz, sizeof(uint8_t)); + if (buf2 == NULL) { PyErr_NoMemory(); goto end; } + key_size = ucol_getSortKey(self->collator, buf, sz, buf2, bsz); + if (key_size > bsz) { + buf2 = realloc(buf2, (key_size + 1) * sizeof(uint8_t)); + if (buf2 == NULL) { PyErr_NoMemory(); goto end; } + key_size = ucol_getSortKey(self->collator, buf, sz, buf2, key_size + 1); + } + ans = PyBytes_FromStringAndSize((char*)buf2, key_size); - buf = (UChar*)calloc(sz*4 + 1, sizeof(UChar)); - - if (buf == NULL) return PyErr_NoMemory(); - - u_strFromUTF8(buf, sz*4 + 1, &key_size, input, sz, &status); - PyMem_Free(input); - - if (U_SUCCESS(status)) { - buf2 = (uint8_t*)calloc(7*sz+1, sizeof(uint8_t)); - if (buf2 == NULL) return PyErr_NoMemory(); - - key_size = ucol_getSortKey(self->collator, buf, -1, buf2, 7*sz+1); - - if (key_size == 0) { - ans = PyBytes_FromString(""); - } else { - if (key_size >= 7*sz+1) { - free(buf2); - buf2 = (uint8_t*)calloc(key_size+1, sizeof(uint8_t)); - if (buf2 == NULL) return PyErr_NoMemory(); - ucol_getSortKey(self->collator, buf, -1, buf2, key_size+1); - } - ans = PyBytes_FromString((char *)buf2); - } - free(buf2); - } else ans = PyBytes_FromString(""); - - free(buf); - if (ans == NULL) return PyErr_NoMemory(); +end: + if (buf != NULL) free(buf); + if (buf2 != NULL) free(buf2); return ans; } // }}} @@ -188,86 +170,64 @@ icu_Collator_sort_key(icu_Collator *self, PyObject *args, PyObject *kwargs) { // Collator.strcmp {{{ static PyObject * icu_Collator_strcmp(icu_Collator *self, PyObject *args, PyObject *kwargs) { - char *a_, *b_; - int32_t asz, bsz; - UChar *a, *b; - UErrorCode status = U_ZERO_ERROR; + PyObject *a_ = NULL, *b_ = NULL; + int32_t asz = 0, bsz = 0; + UChar *a = NULL, *b = NULL; UCollationResult res = UCOL_EQUAL; - if (!PyArg_ParseTuple(args, "eses", "UTF-8", &a_, "UTF-8", &b_)) return NULL; - - asz = (int32_t)strlen(a_); bsz = (int32_t)strlen(b_); + if (!PyArg_ParseTuple(args, "OO", &a_, &b_)) return NULL; - a = (UChar*)calloc(asz*4 + 1, sizeof(UChar)); - b = (UChar*)calloc(bsz*4 + 1, sizeof(UChar)); + a = python_to_icu(a_, &asz, 1); + if (a == NULL) goto end; + b = python_to_icu(b_, &bsz, 1); + if (b == NULL) goto end; + res = ucol_strcoll(self->collator, a, asz, b, bsz); +end: + if (a != NULL) free(a); if (b != NULL) free(b); - - if (a == NULL || b == NULL) return PyErr_NoMemory(); - - u_strFromUTF8(a, asz*4 + 1, NULL, a_, asz, &status); - u_strFromUTF8(b, bsz*4 + 1, NULL, b_, bsz, &status); - PyMem_Free(a_); PyMem_Free(b_); - - if (U_SUCCESS(status)) - res = ucol_strcoll(self->collator, a, -1, b, -1); - - free(a); free(b); - - return Py_BuildValue("i", res); + return (PyErr_Occurred()) ? NULL : Py_BuildValue("i", res); } // }}} // Collator.find {{{ static PyObject * icu_Collator_find(icu_Collator *self, PyObject *args, PyObject *kwargs) { - PyObject *a_, *b_; - int32_t asz, bsz; - UChar *a, *b; - wchar_t *aw, *bw; + PyObject *a_ = NULL, *b_ = NULL; + UChar *a = NULL, *b = NULL; + int32_t asz = 0, bsz = 0, pos = -1, length = -1; UErrorCode status = U_ZERO_ERROR; UStringSearch *search = NULL; - int32_t pos = -1, length = -1; - if (!PyArg_ParseTuple(args, "UU", &a_, &b_)) return NULL; - asz = (int32_t)PyUnicode_GetSize(a_); bsz = (int32_t)PyUnicode_GetSize(b_); - - a = (UChar*)calloc(asz*4 + 2, sizeof(UChar)); - b = (UChar*)calloc(bsz*4 + 2, sizeof(UChar)); - aw = (wchar_t*)calloc(asz*4 + 2, sizeof(wchar_t)); - bw = (wchar_t*)calloc(bsz*4 + 2, sizeof(wchar_t)); + if (!PyArg_ParseTuple(args, "OO", &a_, &b_)) return NULL; - if (a == NULL || b == NULL || aw == NULL || bw == NULL) return PyErr_NoMemory(); - - PyUnicode_AsWideChar((PyUnicodeObject*)a_, aw, asz*4+1); - PyUnicode_AsWideChar((PyUnicodeObject*)b_, bw, bsz*4+1); - u_strFromWCS(a, asz*4 + 1, NULL, aw, -1, &status); - u_strFromWCS(b, bsz*4 + 1, NULL, bw, -1, &status); + a = python_to_icu(a_, &asz, 1); + if (a == NULL) goto end; + b = python_to_icu(b_, &bsz, 1); + if (b == NULL) goto end; + search = usearch_openFromCollator(a, asz, b, bsz, self->collator, NULL, &status); if (U_SUCCESS(status)) { - search = usearch_openFromCollator(a, -1, b, -1, self->collator, NULL, &status); - if (U_SUCCESS(status)) { - pos = usearch_first(search, &status); - if (pos != USEARCH_DONE) - length = usearch_getMatchedLength(search); - else - pos = -1; - } - if (search != NULL) usearch_close(search); + pos = usearch_first(search, &status); + if (pos != USEARCH_DONE) + length = usearch_getMatchedLength(search); + else + pos = -1; } +end: + if (search != NULL) usearch_close(search); + if (a != NULL) free(a); + if (b != NULL) free(b); - free(a); free(b); free(aw); free(bw); - - return Py_BuildValue("ii", pos, length); + return (PyErr_Occurred()) ? NULL : Py_BuildValue("ii", pos, length); } // }}} // Collator.contractions {{{ static PyObject * icu_Collator_contractions(icu_Collator *self, PyObject *args, PyObject *kwargs) { UErrorCode status = U_ZERO_ERROR; - UChar *str; + UChar *str = NULL; UChar32 start=0, end=0; - int32_t count = 0, len = 0, dlen = 0, i; + int32_t count = 0, len = 0, i; PyObject *ans = Py_None, *pbuf; - wchar_t *buf; if (self->contractions == NULL) { self->contractions = uset_open(1, 0); @@ -275,107 +235,112 @@ icu_Collator_contractions(icu_Collator *self, PyObject *args, PyObject *kwargs) self->contractions = ucol_getTailoredSet(self->collator, &status); } status = U_ZERO_ERROR; + count = uset_getItemCount(self->contractions); str = (UChar*)calloc(100, sizeof(UChar)); - buf = (wchar_t*)calloc(4*100+2, sizeof(wchar_t)); - if (str == NULL || buf == NULL) return PyErr_NoMemory(); - - count = uset_getItemCount(self->contractions); + if (str == NULL) { PyErr_NoMemory(); goto end; } ans = PyTuple_New(count); - if (ans != NULL) { - for (i = 0; i < count; i++) { - len = uset_getItem(self->contractions, i, &start, &end, str, 1000, &status); - if (len >= 2) { - // We have a string - status = U_ZERO_ERROR; - u_strToWCS(buf, 4*100 + 1, &dlen, str, len, &status); - pbuf = PyUnicode_FromWideChar(buf, dlen); - if (pbuf == NULL) return PyErr_NoMemory(); - PyTuple_SetItem(ans, i, pbuf); - } else { - // Ranges dont make sense for contractions, ignore them - PyTuple_SetItem(ans, i, Py_None); - } + if (ans == NULL) { goto end; } + + for (i = 0; i < count; i++) { + len = uset_getItem(self->contractions, i, &start, &end, str, 1000, &status); + if (len >= 2) { + // We have a string + status = U_ZERO_ERROR; + pbuf = icu_to_python(str, len); + if (pbuf == NULL) { Py_DECREF(ans); ans = NULL; goto end; } + PyTuple_SetItem(ans, i, pbuf); + } else { + // Ranges dont make sense for contractions, ignore them + PyTuple_SetItem(ans, i, Py_None); Py_INCREF(Py_None); } } - free(str); free(buf); +end: + if (str != NULL) free(str); - return Py_BuildValue("O", ans); + return ans; } // }}} // Collator.startswith {{{ static PyObject * icu_Collator_startswith(icu_Collator *self, PyObject *args, PyObject *kwargs) { - PyObject *a_, *b_; - int32_t asz, bsz; - int32_t actual_a, actual_b; - UChar *a, *b; - wchar_t *aw, *bw; - UErrorCode status = U_ZERO_ERROR; - int ans = 0; + PyObject *a_ = NULL, *b_ = NULL; + int32_t asz = 0, bsz = 0; + UChar *a = NULL, *b = NULL; + uint8_t ans = 0; - if (!PyArg_ParseTuple(args, "UU", &a_, &b_)) return NULL; - asz = (int32_t)PyUnicode_GetSize(a_); bsz = (int32_t)PyUnicode_GetSize(b_); - if (asz < bsz) Py_RETURN_FALSE; - if (bsz == 0) Py_RETURN_TRUE; + if (!PyArg_ParseTuple(args, "OO", &a_, &b_)) return NULL; + + a = python_to_icu(a_, &asz, 1); + if (a == NULL) goto end; + b = python_to_icu(b_, &bsz, 1); + if (b == NULL) goto end; + + if (asz < bsz) goto end; + if (bsz == 0) { ans = 1; goto end; } - a = (UChar*)calloc(asz*4 + 2, sizeof(UChar)); - b = (UChar*)calloc(bsz*4 + 2, sizeof(UChar)); - aw = (wchar_t*)calloc(asz*4 + 2, sizeof(wchar_t)); - bw = (wchar_t*)calloc(bsz*4 + 2, sizeof(wchar_t)); + ans = ucol_equal(self->collator, a, bsz, b, bsz); - if (a == NULL || b == NULL || aw == NULL || bw == NULL) return PyErr_NoMemory(); +end: + if (a != NULL) free(a); + if (b != NULL) free(b); - actual_a = (int32_t)PyUnicode_AsWideChar((PyUnicodeObject*)a_, aw, asz*4+1); - actual_b = (int32_t)PyUnicode_AsWideChar((PyUnicodeObject*)b_, bw, bsz*4+1); - if (actual_a > -1 && actual_b > -1) { - u_strFromWCS(a, asz*4 + 1, &actual_a, aw, -1, &status); - u_strFromWCS(b, bsz*4 + 1, &actual_b, bw, -1, &status); - - if (U_SUCCESS(status) && ucol_equal(self->collator, a, actual_b, b, actual_b)) - ans = 1; - } - - free(a); free(b); free(aw); free(bw); - if (ans) Py_RETURN_TRUE; + if (PyErr_Occurred()) return NULL; + if (ans) { Py_RETURN_TRUE; } Py_RETURN_FALSE; } // }}} -// Collator.startswith {{{ +// Collator.collation_order {{{ static PyObject * icu_Collator_collation_order(icu_Collator *self, PyObject *args, PyObject *kwargs) { - PyObject *a_; - int32_t asz; - int32_t actual_a; - UChar *a; - wchar_t *aw; + PyObject *a_ = NULL; + int32_t asz = 0; + UChar *a = NULL; UErrorCode status = U_ZERO_ERROR; UCollationElements *iter = NULL; int order = 0, len = -1; - if (!PyArg_ParseTuple(args, "U", &a_)) return NULL; - asz = (int32_t)PyUnicode_GetSize(a_); - - a = (UChar*)calloc(asz*4 + 2, sizeof(UChar)); - aw = (wchar_t*)calloc(asz*4 + 2, sizeof(wchar_t)); + if (!PyArg_ParseTuple(args, "O", &a_)) return NULL; - if (a == NULL || aw == NULL ) return PyErr_NoMemory(); + a = python_to_icu(a_, &asz, 1); + if (a == NULL) goto end; - actual_a = (int32_t)PyUnicode_AsWideChar((PyUnicodeObject*)a_, aw, asz*4+1); - if (actual_a > -1) { - u_strFromWCS(a, asz*4 + 1, &actual_a, aw, -1, &status); - iter = ucol_openElements(self->collator, a, actual_a, &status); - if (iter != NULL && U_SUCCESS(status)) { - order = ucol_next(iter, &status); - len = ucol_getOffset(iter); - ucol_closeElements(iter); iter = NULL; - } - } - - free(a); free(aw); + iter = ucol_openElements(self->collator, a, asz, &status); + if (U_FAILURE(status)) { PyErr_SetString(PyExc_ValueError, u_errorName(status)); goto end; } + order = ucol_next(iter, &status); + len = ucol_getOffset(iter); +end: + if (iter != NULL) ucol_closeElements(iter); iter = NULL; + if (a != NULL) free(a); + if (PyErr_Occurred()) return NULL; return Py_BuildValue("ii", order, len); } // }}} +// Collator.upper_first {{{ +static PyObject * +icu_Collator_get_upper_first(icu_Collator *self, void *closure) { + UErrorCode status = U_ZERO_ERROR; + UColAttributeValue val; + + val = ucol_getAttribute(self->collator, UCOL_CASE_FIRST, &status); + if (U_FAILURE(status)) { PyErr_SetString(PyExc_ValueError, u_errorName(status)); return NULL; } + + if (val == UCOL_OFF) { Py_RETURN_NONE; } + if (val) { + Py_RETURN_TRUE; + } + Py_RETURN_FALSE; +} + +static int +icu_Collator_set_upper_first(icu_Collator *self, PyObject *val, void *closure) { + UErrorCode status = U_ZERO_ERROR; + ucol_setAttribute(self->collator, UCOL_CASE_FIRST, (val == Py_None) ? UCOL_OFF : ((PyObject_IsTrue(val)) ? UCOL_UPPER_FIRST : UCOL_LOWER_FIRST), &status); + if (U_FAILURE(status)) { PyErr_SetString(PyExc_ValueError, u_errorName(status)); return -1; } + return 0; +} +// }}} + static PyObject* icu_Collator_clone(icu_Collator *self, PyObject *args, PyObject *kwargs); @@ -432,6 +397,11 @@ static PyGetSetDef icu_Collator_getsetters[] = { (char *)"The strength of this collator.", NULL}, + {(char *)"upper_first", + (getter)icu_Collator_get_upper_first, (setter)icu_Collator_set_upper_first, + (char *)"Whether this collator should always put upper case letters before lower case. Values are: None - means use the tertiary strength of the letters. True - Always sort upper case before lower case. False - Always sort lower case before upper case.", + NULL}, + {(char *)"numeric", (getter)icu_Collator_get_numeric, (setter)icu_Collator_set_numeric, (char *)"If True the collator sorts contiguous digits as numbers rather than strings, so 2 will sort before 10.", @@ -513,139 +483,45 @@ icu_Collator_clone(icu_Collator *self, PyObject *args, PyObject *kwargs) // }}} -// upper {{{ -static PyObject * -icu_upper(PyObject *self, PyObject *args) { - char *input, *ans, *buf3 = NULL; - const char *loc; - int32_t sz; - UChar *buf, *buf2; - PyObject *ret; +// change_case {{{ + +static PyObject* icu_change_case(PyObject *self, PyObject *args) { + char *locale = NULL; + PyObject *input = NULL, *result = NULL; + int which = UPPER_CASE; UErrorCode status = U_ZERO_ERROR; - + UChar *input_buf = NULL, *output_buf = NULL; + int32_t sz = 0; - if (!PyArg_ParseTuple(args, "ses", &loc, "UTF-8", &input)) return NULL; - - sz = (int32_t)strlen(input); - - buf = (UChar*)calloc(sz*4 + 1, sizeof(UChar)); - buf2 = (UChar*)calloc(sz*8 + 1, sizeof(UChar)); - - - if (buf == NULL || buf2 == NULL) return PyErr_NoMemory(); - - u_strFromUTF8(buf, sz*4, NULL, input, sz, &status); - u_strToUpper(buf2, sz*8, buf, -1, loc, &status); - - ans = input; - sz = u_strlen(buf2); - free(buf); - - if (U_SUCCESS(status) && sz > 0) { - buf3 = (char*)calloc(sz*5+1, sizeof(char)); - if (buf3 == NULL) return PyErr_NoMemory(); - u_strToUTF8(buf3, sz*5, NULL, buf2, -1, &status); - if (U_SUCCESS(status)) ans = buf3; + if (!PyArg_ParseTuple(args, "Oiz", &input, &which, &locale)) return NULL; + if (locale == NULL) { + PyErr_SetString(PyExc_NotImplementedError, "You must specify a locale"); // We deliberately use NotImplementedError so that this error can be unambiguously identified + return NULL; } - ret = PyUnicode_DecodeUTF8(ans, strlen(ans), "replace"); - if (ret == NULL) return PyErr_NoMemory(); + input_buf = python_to_icu(input, &sz, 1); + if (input_buf == NULL) goto end; + output_buf = (UChar*) calloc(3 * sz, sizeof(UChar)); + if (output_buf == NULL) { PyErr_NoMemory(); goto end; } - free(buf2); - if (buf3 != NULL) free(buf3); - PyMem_Free(input); - - return ret; -} // }}} - -// lower {{{ -static PyObject * -icu_lower(PyObject *self, PyObject *args) { - char *input, *ans, *buf3 = NULL; - const char *loc; - int32_t sz; - UChar *buf, *buf2; - PyObject *ret; - UErrorCode status = U_ZERO_ERROR; - - - if (!PyArg_ParseTuple(args, "ses", &loc, "UTF-8", &input)) return NULL; - - sz = (int32_t)strlen(input); - - buf = (UChar*)calloc(sz*4 + 1, sizeof(UChar)); - buf2 = (UChar*)calloc(sz*8 + 1, sizeof(UChar)); - - - if (buf == NULL || buf2 == NULL) return PyErr_NoMemory(); - - u_strFromUTF8(buf, sz*4, NULL, input, sz, &status); - u_strToLower(buf2, sz*8, buf, -1, loc, &status); - - ans = input; - sz = u_strlen(buf2); - free(buf); - - if (U_SUCCESS(status) && sz > 0) { - buf3 = (char*)calloc(sz*5+1, sizeof(char)); - if (buf3 == NULL) return PyErr_NoMemory(); - u_strToUTF8(buf3, sz*5, NULL, buf2, -1, &status); - if (U_SUCCESS(status)) ans = buf3; + switch (which) { + case TITLE_CASE: + sz = u_strToTitle(output_buf, 3 * sz, input_buf, sz, NULL, locale, &status); + break; + case UPPER_CASE: + sz = u_strToUpper(output_buf, 3 * sz, input_buf, sz, locale, &status); + break; + default: + sz = u_strToLower(output_buf, 3 * sz, input_buf, sz, locale, &status); } + if (U_FAILURE(status)) { PyErr_SetString(PyExc_ValueError, u_errorName(status)); goto end; } + result = icu_to_python(output_buf, sz); - ret = PyUnicode_DecodeUTF8(ans, strlen(ans), "replace"); - if (ret == NULL) return PyErr_NoMemory(); +end: + if (input_buf != NULL) free(input_buf); + if (output_buf != NULL) free(output_buf); + return result; - free(buf2); - if (buf3 != NULL) free(buf3); - PyMem_Free(input); - - return ret; -} // }}} - -// title {{{ -static PyObject * -icu_title(PyObject *self, PyObject *args) { - char *input, *ans, *buf3 = NULL; - const char *loc; - int32_t sz; - UChar *buf, *buf2; - PyObject *ret; - UErrorCode status = U_ZERO_ERROR; - - - if (!PyArg_ParseTuple(args, "ses", &loc, "UTF-8", &input)) return NULL; - - sz = (int32_t)strlen(input); - - buf = (UChar*)calloc(sz*4 + 1, sizeof(UChar)); - buf2 = (UChar*)calloc(sz*8 + 1, sizeof(UChar)); - - - if (buf == NULL || buf2 == NULL) return PyErr_NoMemory(); - - u_strFromUTF8(buf, sz*4, NULL, input, sz, &status); - u_strToTitle(buf2, sz*8, buf, -1, NULL, loc, &status); - - ans = input; - sz = u_strlen(buf2); - free(buf); - - if (U_SUCCESS(status) && sz > 0) { - buf3 = (char*)calloc(sz*5+1, sizeof(char)); - if (buf3 == NULL) return PyErr_NoMemory(); - u_strToUTF8(buf3, sz*5, NULL, buf2, -1, &status); - if (U_SUCCESS(status)) ans = buf3; - } - - ret = PyUnicode_DecodeUTF8(ans, strlen(ans), "replace"); - if (ret == NULL) return PyErr_NoMemory(); - - free(buf2); - if (buf3 != NULL) free(buf3); - PyMem_Free(input); - - return ret; } // }}} // set_default_encoding {{{ @@ -662,7 +538,7 @@ icu_set_default_encoding(PyObject *self, PyObject *args) { } // }}} -// set_default_encoding {{{ +// set_filesystem_encoding {{{ static PyObject * icu_set_filesystem_encoding(PyObject *self, PyObject *args) { char *encoding; @@ -674,7 +550,7 @@ icu_set_filesystem_encoding(PyObject *self, PyObject *args) { } // }}} -// set_default_encoding {{{ +// get_available_transliterators {{{ static PyObject * icu_get_available_transliterators(PyObject *self, PyObject *args) { PyObject *ans, *l; @@ -835,16 +711,8 @@ icu_roundtrip(PyObject *self, PyObject *args) { // Module initialization {{{ static PyMethodDef icu_methods[] = { - {"upper", icu_upper, METH_VARARGS, - "upper(locale, unicode object) -> upper cased unicode object using locale rules." - }, - - {"lower", icu_lower, METH_VARARGS, - "lower(locale, unicode object) -> lower cased unicode object using locale rules." - }, - - {"title", icu_title, METH_VARARGS, - "title(locale, unicode object) -> Title cased unicode object using locale rules." + {"change_case", icu_change_case, METH_VARARGS, + "change_case(unicode object, which, locale) -> change case to one of UPPER_CASE, LOWER_CASE, TITLE_CASE" }, {"set_default_encoding", icu_set_default_encoding, METH_VARARGS, @@ -946,5 +814,9 @@ initicu(void) ADDUCONST(UNORM_NFKC); ADDUCONST(UNORM_FCD); + ADDUCONST(UPPER_CASE); + ADDUCONST(LOWER_CASE); + ADDUCONST(TITLE_CASE); + } // }}} diff --git a/src/calibre/utils/icu.py b/src/calibre/utils/icu.py index 39256f6fd6..df4c369365 100644 --- a/src/calibre/utils/icu.py +++ b/src/calibre/utils/icu.py @@ -1,5 +1,7 @@ #!/usr/bin/env python -# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +# vim:fileencoding=utf-8 +from __future__ import (unicode_literals, division, absolute_import, + print_function) __license__ = 'GPL v3' __copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>' @@ -7,232 +9,20 @@ __docformat__ = 'restructuredtext en' # Setup code {{{ import sys -from functools import partial from calibre.constants import plugins from calibre.utils.config_base import tweaks -_icu = _collator = _primary_collator = _sort_collator = _numeric_collator = None -_locale = None +_locale = _collator = _primary_collator = _sort_collator = _numeric_collator = _case_sensitive_collator = None _none = u'' _none2 = b'' - -def get_locale(): - global _locale - if _locale is None: - from calibre.utils.localization import get_lang - if tweaks['locale_for_sorting']: - _locale = tweaks['locale_for_sorting'] - else: - _locale = get_lang() - return _locale - -def load_icu(): - global _icu - if _icu is None: - _icu = plugins['icu'][0] - if _icu is None: - print 'Loading ICU failed with: ', plugins['icu'][1] - else: - if not getattr(_icu, 'ok', False): - print 'icu not ok' - _icu = None - return _icu - -def load_collator(): - 'The default collator for most locales takes both case and accented letters into account' - global _collator - if _collator is None: - icu = load_icu() - if icu is not None: - _collator = icu.Collator(get_locale()) - return _collator - -def primary_collator(): - 'Ignores case differences and accented characters' - global _primary_collator - if _primary_collator is None: - _primary_collator = _collator.clone() - _primary_collator.strength = _icu.UCOL_PRIMARY - return _primary_collator - -def sort_collator(): - 'Ignores case differences and recognizes numbers in strings' - global _sort_collator - if _sort_collator is None: - _sort_collator = _collator.clone() - _sort_collator.strength = _icu.UCOL_SECONDARY - if tweaks['numeric_collation']: - try: - _sort_collator.numeric = True - except AttributeError: - pass - return _sort_collator - -def py_sort_key(obj): - if not obj: - return _none - return obj.lower() - -def icu_sort_key(collator, obj): - if not obj: - return _none2 - try: - try: - return _sort_collator.sort_key(obj) - except AttributeError: - return sort_collator().sort_key(obj) - except TypeError: - if isinstance(obj, unicode): - obj = obj.replace(u'\0', u'') - else: - obj = obj.replace(b'\0', b'') - return _sort_collator.sort_key(obj) - -def numeric_collator(): - global _numeric_collator - _numeric_collator = _collator.clone() - _numeric_collator.strength = _icu.UCOL_SECONDARY - _numeric_collator.numeric = True - return _numeric_collator - -def numeric_sort_key(obj): - 'Uses natural sorting for numbers inside strings so something2 will sort before something10' - if not obj: - return _none2 - try: - try: - return _numeric_collator.sort_key(obj) - except AttributeError: - return numeric_collator().sort_key(obj) - except TypeError: - if isinstance(obj, unicode): - obj = obj.replace(u'\0', u'') - else: - obj = obj.replace(b'\0', b'') - return _numeric_collator.sort_key(obj) - -def icu_change_case(upper, locale, obj): - func = _icu.upper if upper else _icu.lower - try: - return func(locale, obj) - except TypeError: - if isinstance(obj, unicode): - obj = obj.replace(u'\0', u'') - else: - obj = obj.replace(b'\0', b'') - return func(locale, obj) - -def py_find(pattern, source): - pos = source.find(pattern) - if pos > -1: - return pos, len(pattern) - return -1, -1 - -def character_name(string): - try: - try: - return _icu.character_name(unicode(string)) or None - except AttributeError: - import unicodedata - return unicodedata.name(unicode(string)[0], None) - except (TypeError, ValueError, KeyError): - pass - -def character_name_from_code(code): - try: - try: - return _icu.character_name_from_code(code) or '' - except AttributeError: - import unicodedata - return unicodedata.name(py_safe_chr(code), '') - except (TypeError, ValueError, KeyError): - return '' - -if sys.maxunicode >= 0x10ffff: - try: - py_safe_chr = unichr - except NameError: - py_safe_chr = chr -else: - def py_safe_chr(i): - # Narrow builds of python cannot represent code point > 0xffff as a - # single character, so we need our own implementation of unichr - # that returns them as a surrogate pair - return (b"\U%s" % (hex(i)[2:].zfill(8))).decode('unicode-escape') - -def safe_chr(code): - try: - return _icu.chr(code) - except AttributeError: - return py_safe_chr(code) - -def normalize(text, mode='NFC'): - # This is very slightly slower than using unicodedata.normalize, so stick with - # that unless you have very good reasons not too. Also, it's speed - # decreases on wide python builds, where conversion to/from ICU's string - # representation is slower. - try: - return _icu.normalize(_nmodes[mode], unicode(text)) - except (AttributeError, KeyError): - import unicodedata - return unicodedata.normalize(mode, unicode(text)) - -def icu_find(collator, pattern, source): - try: - return collator.find(pattern, source) - except TypeError: - return collator.find(unicode(pattern), unicode(source)) - -def icu_startswith(collator, a, b): - try: - return collator.startswith(a, b) - except TypeError: - return collator.startswith(unicode(a), unicode(b)) - -def py_case_sensitive_sort_key(obj): - if not obj: - return _none - return obj - -def icu_case_sensitive_sort_key(collator, obj): - if not obj: - return _none2 - return collator.sort_key(obj) - -def icu_strcmp(collator, a, b): - return collator.strcmp(lower(a), lower(b)) - -def py_strcmp(a, b): - return cmp(a.lower(), b.lower()) - -def icu_case_sensitive_strcmp(collator, a, b): - return collator.strcmp(a, b) - -def icu_capitalize(s): - s = lower(s) - return s.replace(s[0], upper(s[0]), 1) if s else s - _cmap = {} -def icu_contractions(collator): - global _cmap - ans = _cmap.get(collator, None) - if ans is None: - ans = collator.contractions() - ans = frozenset(filter(None, ans)) if ans else {} - _cmap[collator] = ans - return ans -def icu_collation_order(collator, a): - try: - return collator.collation_order(a) - except TypeError: - return collator.collation_order(unicode(a)) - -load_icu() -load_collator() -_icu_not_ok = _icu is None or _collator is None +_icu, err = plugins['icu'] +if _icu is None: + raise RuntimeError('Failed to load icu with error: %s' % err) +del err icu_unicode_version = getattr(_icu, 'unicode_version', None) _nmodes = {m:getattr(_icu, 'UNORM_'+m, None) for m in ('NFC', 'NFD', 'NFKC', 'NFKD', 'NONE', 'DEFAULT', 'FCD')} @@ -252,290 +42,208 @@ try: except: pass +def collator(): + global _collator, _locale + if _collator is None: + if _locale is None: + from calibre.utils.localization import get_lang + if tweaks['locale_for_sorting']: + _locale = tweaks['locale_for_sorting'] + else: + _locale = get_lang() + try: + _collator = _icu.Collator(_locale) + except Exception as e: + print ('Failed to load collator for locale: %r with error %r, using English' % (_locale, e)) + _collator = _icu.Collator('en') + return _collator + +def change_locale(locale=None): + global _locale, _collator, _primary_collator, _sort_collator, _numeric_collator, _case_sensitive_collator + _collator = _primary_collator = _sort_collator = _numeric_collator = _case_sensitive_collator = None + _locale = locale + +def primary_collator(): + 'Ignores case differences and accented characters' + global _primary_collator + if _primary_collator is None: + _primary_collator = collator().clone() + _primary_collator.strength = _icu.UCOL_PRIMARY + return _primary_collator + +def sort_collator(): + 'Ignores case differences and recognizes numbers in strings (if the tweak is set)' + global _sort_collator + if _sort_collator is None: + _sort_collator = collator().clone() + _sort_collator.strength = _icu.UCOL_SECONDARY + _sort_collator.numeric = tweaks['numeric_collation'] + return _sort_collator + +def numeric_collator(): + 'Uses natural sorting for numbers inside strings so something2 will sort before something10' + global _numeric_collator + if _numeric_collator is None: + _numeric_collator = collator().clone() + _numeric_collator.strength = _icu.UCOL_SECONDARY + _numeric_collator.numeric = True + return _numeric_collator + +def case_sensitive_collator(): + 'Always sorts upper case letter before lower case' + global _case_sensitive_collator + if _case_sensitive_collator is None: + _case_sensitive_collator = collator().clone() + _case_sensitive_collator.numeric = sort_collator().numeric + _case_sensitive_collator.upper_first = True + return _case_sensitive_collator + +# Templates that will be used to generate various concrete +# function implementations based on different collators, to allow lazy loading +# of collators, with maximum runtime performance + +_sort_key_template = ''' +def {name}(obj): + try: + try: + return {collator}.{func}(obj) + except AttributeError: + return {collator_func}().{func}(obj) + except TypeError: + if isinstance(obj, bytes): + try: + obj = obj.decode(sys.getdefaultencoding()) + except ValueError: + return obj + return {collator}.{func}(obj) + return b'' +''' + +_strcmp_template = ''' +def {name}(a, b): + try: + try: + return {collator}.{func}(a, b) + except AttributeError: + return {collator_func}().{func}(a, b) + except TypeError: + if isinstance(a, bytes): + try: + a = a.decode(sys.getdefaultencoding()) + except ValueError: + return cmp(a, b) + elif a is None: + a = u'' + if isinstance(b, bytes): + try: + b = b.decode(sys.getdefaultencoding()) + except ValueError: + return cmp(a, b) + elif b is None: + b = u'' + return {collator}.{func}(a, b) +''' + +_change_case_template = ''' +def {name}(x): + try: + try: + return _icu.change_case(x, _icu.{which}, _locale) + except NotImplementedError: + collator() # sets _locale + return _icu.change_case(x, _icu.{which}, _locale) + except TypeError: + if isinstance(x, bytes): + try: + x = x.decode(sys.getdefaultencoding()) + except ValueError: + return x + return _icu.change_case(x, _icu.{which}, _locale) + raise +''' + +def _make_func(template, name, **kwargs): + l = globals() + kwargs['name'] = name + kwargs['func'] = kwargs.get('func', 'sort_key') + exec template.format(**kwargs) in l + return l[name] + # }}} ################# The string functions ######################################## +sort_key = _make_func(_sort_key_template, 'sort_key', collator='_sort_collator', collator_func='sort_collator') -sort_key = py_sort_key if _icu_not_ok else partial(icu_sort_key, _collator) +numeric_sort_key = _make_func(_sort_key_template, 'numeric_sort_key', collator='_numeric_collator', collator_func='numeric_collator') -strcmp = py_strcmp if _icu_not_ok else partial(icu_strcmp, _collator) +primary_sort_key = _make_func(_sort_key_template, 'primary_sort_key', collator='_primary_collator', collator_func='primary_collator') -case_sensitive_sort_key = py_case_sensitive_sort_key if _icu_not_ok else \ - partial(icu_case_sensitive_sort_key, _collator) +case_sensitive_sort_key = _make_func(_sort_key_template, 'case_sensitive_sort_key', + collator='_case_sensitive_collator', collator_func='case_sensitive_collator') -case_sensitive_strcmp = cmp if _icu_not_ok else icu_case_sensitive_strcmp +collation_order = _make_func(_sort_key_template, 'collation_order', collator='_sort_collator', collator_func='sort_collator', func='collation_order') -upper = (lambda s: s.upper()) if _icu_not_ok else \ - partial(icu_change_case, True, get_locale()) +strcmp = _make_func(_strcmp_template, 'strcmp', collator='_sort_collator', collator_func='sort_collator', func='strcmp') -lower = (lambda s: s.lower()) if _icu_not_ok else \ - partial(icu_change_case, False, get_locale()) +case_sensitive_strcmp = _make_func( + _strcmp_template, 'case_sensitive_strcmp', collator='_case_sensitive_collator', collator_func='case_sensitive_collator', func='strcmp') -title_case = (lambda s: s.title()) if _icu_not_ok else \ - partial(_icu.title, get_locale()) +primary_strcmp = _make_func(_strcmp_template, 'primary_strcmp', collator='_primary_collator', collator_func='primary_collator', func='strcmp') -capitalize = (lambda s: s.capitalize()) if _icu_not_ok else \ - (lambda s: icu_capitalize(s)) +upper = _make_func(_change_case_template, 'upper', which='UPPER_CASE') -find = (py_find if _icu_not_ok else partial(icu_find, _collator)) +lower = _make_func(_change_case_template, 'lower', which='LOWER_CASE') -contractions = ((lambda : {}) if _icu_not_ok else (partial(icu_contractions, - _collator))) +title_case = _make_func(_change_case_template, 'title_case', which='TITLE_CASE') -def primary_strcmp(a, b): - 'strcmp that ignores case and accents on letters' - if _icu_not_ok: - from calibre.utils.filenames import ascii_text - return py_strcmp(ascii_text(a), ascii_text(b)) +capitalize = lambda x: upper(x[0]) + lower(x[1:]) + +find = _make_func(_strcmp_template, 'find', collator='_collator', collator_func='collator', func='find') + +primary_find = _make_func(_strcmp_template, 'primary_find', collator='_primary_collator', collator_func='primary_collator', func='find') + +startswith = _make_func(_strcmp_template, 'startswith', collator='_collator', collator_func='collator', func='startswith') + +primary_startswith = _make_func(_strcmp_template, 'primary_startswith', collator='_primary_collator', collator_func='primary_collator', func='startswith') + +safe_chr = _icu.chr + +def character_name(string): try: - return _primary_collator.strcmp(a, b) - except AttributeError: - return primary_collator().strcmp(a, b) + return _icu.character_name(unicode(string)) or None + except (TypeError, ValueError, KeyError): + pass -def primary_find(pat, src): - 'find that ignores case and accents on letters' - if _icu_not_ok: - from calibre.utils.filenames import ascii_text - return py_find(ascii_text(pat), ascii_text(src)) - return primary_icu_find(pat, src) - -def primary_icu_find(pat, src): +def character_name_from_code(code): try: - return icu_find(_primary_collator, pat, src) - except AttributeError: - return icu_find(primary_collator(), pat, src) + return _icu.character_name_from_code(code) or '' + except (TypeError, ValueError, KeyError): + return '' -def primary_sort_key(val): - 'A sort key that ignores case and diacritics' - if _icu_not_ok: - from calibre.utils.filenames import ascii_text - return ascii_text(val).lower() - try: - return _primary_collator.sort_key(val) - except AttributeError: - return primary_collator().sort_key(val) +def normalize(text, mode='NFC'): + # This is very slightly slower than using unicodedata.normalize, so stick with + # that unless you have very good reasons not too. Also, it's speed + # decreases on wide python builds, where conversion to/from ICU's string + # representation is slower. + return _icu.normalize(_nmodes[mode], unicode(text)) -def primary_startswith(a, b): - if _icu_not_ok: - from calibre.utils.filenames import ascii_text - return ascii_text(a).lower().startswith(ascii_text(b).lower()) - try: - return icu_startswith(_primary_collator, a, b) - except AttributeError: - return icu_startswith(primary_collator(), a, b) +def contractions(col=None): + global _cmap + col = col or _collator + if col is None: + col = collator() + ans = _cmap.get(collator, None) + if ans is None: + ans = col.contractions() + ans = frozenset(filter(None, ans)) + _cmap[col] = ans + return ans -def collation_order(a): - if _icu_not_ok: - return (ord(a[0]), 1) if a else (0, 0) - try: - return icu_collation_order(_sort_collator, a) - except AttributeError: - return icu_collation_order(sort_collator(), a) ################################################################################ -def test(): # {{{ - from calibre import prints - # Data {{{ - german = ''' - Sonntag -Montag -Dienstag -Januar -Februar -März -Fuße -Fluße -Flusse -flusse -fluße -flüße -flüsse -''' - german_good = ''' - Dienstag -Februar -flusse -Flusse -fluße -Fluße -flüsse -flüße -Fuße -Januar -März -Montag -Sonntag''' - french = ''' -dimanche -lundi -mardi -janvier -février -mars -déjà -Meme -deja -même -dejà -bpef -bœg -Boef -Mémé -bœf -boef -bnef -pêche -pèché -pêché -pêche -pêché''' - french_good = ''' - bnef - boef - Boef - bœf - bœg - bpef - deja - dejà - déjà - dimanche - février - janvier - lundi - mardi - mars - Meme - Mémé - même - pèché - pêche - pêche - pêché - pêché''' - # }}} - - def create(l): - l = l.decode('utf-8').splitlines() - return [x.strip() for x in l if x.strip()] - - def test_strcmp(entries): - for x in entries: - for y in entries: - if strcmp(x, y) != cmp(sort_key(x), sort_key(y)): - print 'strcmp failed for %r, %r'%(x, y) - - german = create(german) - c = _icu.Collator('de') - c.numeric = True - gs = list(sorted(german, key=c.sort_key)) - if gs != create(german_good): - print 'German sorting failed' - return - print - french = create(french) - c = _icu.Collator('fr') - c.numeric = True - fs = list(sorted(french, key=c.sort_key)) - if fs != create(french_good): - print 'French sorting failed (note that French fails with icu < 4.6)' - return - test_strcmp(german + french) - - print '\nTesting case transforms in current locale' - from calibre.utils.titlecase import titlecase - for x in ('a', 'Alice\'s code', 'macdonald\'s machine', '02 the wars'): - print 'Upper: ', x, '->', 'py:', x.upper().encode('utf-8'), 'icu:', upper(x).encode('utf-8') - print 'Lower: ', x, '->', 'py:', x.lower().encode('utf-8'), 'icu:', lower(x).encode('utf-8') - print 'Title: ', x, '->', 'py:', x.title().encode('utf-8'), 'icu:', title_case(x).encode('utf-8'), 'titlecase:', titlecase(x).encode('utf-8') - print 'Capitalize:', x, '->', 'py:', x.capitalize().encode('utf-8'), 'icu:', capitalize(x).encode('utf-8') - print - - print '\nTesting primary collation' - for k, v in {u'pèché': u'peche', u'flüße':u'Flusse', - u'Štepánek':u'ŠtepaneK'}.iteritems(): - if primary_strcmp(k, v) != 0: - prints('primary_strcmp() failed with %s != %s'%(k, v)) - return - if primary_find(v, u' '+k)[0] != 1: - prints('primary_find() failed with %s not in %s'%(v, k)) - return - - n = character_name(safe_chr(0x1f431)) - if n != u'CAT FACE': - raise ValueError('Failed to get correct character name for 0x1f431: %r != %r' % n, u'CAT FACE') - - global _primary_collator - orig = _primary_collator - _primary_collator = _icu.Collator('es') - if primary_strcmp(u'peña', u'pena') == 0: - print 'Primary collation in Spanish locale failed' - return - _primary_collator = orig - - print '\nTesting contractions' - c = _icu.Collator('cs') - if icu_contractions(c) != frozenset([u'Z\u030c', u'z\u030c', u'Ch', - u'C\u030c', u'ch', u'cH', u'c\u030c', u's\u030c', u'r\u030c', u'CH', - u'S\u030c', u'R\u030c']): - print 'Contractions for the Czech language failed' - return - - print '\nTesting startswith' - p = primary_startswith - if (not p('asd', 'asd') or not p('asd', 'A') or - not p('x', '')): - print 'startswith() failed' - return - - print '\nTesting collation_order()' - for group in [ - ('Šaa', 'Smith', 'Solženicyn', 'Štepánek'), - ('calibre', 'Charon', 'Collins'), - ('01', '1'), - ('1', '11', '13'), - ]: - last = None - for x in group: - val = icu_collation_order(sort_collator(), x) - if val[1] != 1: - prints('collation_order() returned incorrect length for', x) - if last is None: - last = val - else: - if val != last: - prints('collation_order() returned incorrect value for', x) - last = val - -# }}} - -def test_roundtrip(): - for r in (u'xxx\0\u2219\U0001f431xxx', u'\0', u'', u'simple'): - rp = _icu.roundtrip(r) - if rp != r: - raise ValueError(u'Roundtripping failed: %r != %r' % (r, rp)) - -def test_normalize_performance(): - import os - if not os.path.exists('t.txt'): - return - raw = open('t.txt', 'rb').read().decode('utf-8') - print (len(raw)) - import time, unicodedata - st = time.time() - count = 100 - for i in xrange(count): - normalize(raw) - print ('ICU time:', time.time() - st) - st = time.time() - for i in xrange(count): - unicodedata.normalize('NFC', unicode(raw)) - print ('py time:', time.time() - st) - if __name__ == '__main__': - test_roundtrip() - test_normalize_performance() - test() + from calibre.utils.icu_test import run + run(verbosity=4) diff --git a/src/calibre/utils/icu_test.py b/src/calibre/utils/icu_test.py new file mode 100644 index 0000000000..e96397e86a --- /dev/null +++ b/src/calibre/utils/icu_test.py @@ -0,0 +1,148 @@ +#!/usr/bin/env python +# vim:fileencoding=utf-8 +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +__license__ = 'GPL v3' +__copyright__ = '2014, Kovid Goyal <kovid at kovidgoyal.net>' + +import unittest, sys +from contextlib import contextmanager + +import calibre.utils.icu as icu + + +@contextmanager +def make_collation_func(name, locale, numeric=True, template='_sort_key_template', func='strcmp'): + c = icu._icu.Collator(locale) + cname = '%s_test_collator%s' % (name, template) + setattr(icu, cname, c) + c.numeric = numeric + yield icu._make_func(getattr(icu, template), name, collator=cname, collator_func='not_used_xxx', func=func) + delattr(icu, cname) + +class TestICU(unittest.TestCase): + + ae = unittest.TestCase.assertEqual + + def setUp(self): + icu.change_locale('en') + + def test_sorting(self): + ' Test the various sorting APIs ' + german = '''Sonntag Montag Dienstag Januar Februar März Fuße Fluße Flusse flusse fluße flüße flüsse'''.split() + german_good = '''Dienstag Februar flusse Flusse fluße Fluße flüsse flüße Fuße Januar März Montag Sonntag'''.split() + french = '''dimanche lundi mardi janvier février mars déjà Meme deja même dejà bpef bœg Boef Mémé bœf boef bnef pêche pèché pêché pêche pêché'''.split() + french_good = '''bnef boef Boef bœf bœg bpef deja dejà déjà dimanche février janvier lundi mardi mars Meme Mémé même pèché pêche pêche pêché pêché'''.split() # noqa + + # Test corner cases + sort_key = icu.sort_key + s = '\U0001f431' + self.ae(sort_key(s), sort_key(s.encode(sys.getdefaultencoding())), 'UTF-8 encoded object not correctly decoded to generate sort key') + self.ae(s.encode('utf-16'), s.encode('utf-16'), 'Undecodable bytestring not returned as itself') + self.ae(b'', sort_key(None)) + self.ae(0, icu.strcmp(None, b'')) + self.ae(0, icu.strcmp(s, s.encode(sys.getdefaultencoding()))) + + # Test locales + with make_collation_func('dsk', 'de', func='sort_key') as dsk: + self.ae(german_good, sorted(german, key=dsk)) + with make_collation_func('dcmp', 'de', template='_strcmp_template') as dcmp: + for x in german: + for y in german: + self.ae(cmp(dsk(x), dsk(y)), dcmp(x, y)) + + with make_collation_func('fsk', 'fr', func='sort_key') as fsk: + self.ae(french_good, sorted(french, key=fsk)) + with make_collation_func('fcmp', 'fr', template='_strcmp_template') as fcmp: + for x in french: + for y in french: + self.ae(cmp(fsk(x), fsk(y)), fcmp(x, y)) + + with make_collation_func('ssk', 'es', func='sort_key') as ssk: + self.assertNotEqual(ssk('peña'), ssk('pena')) + with make_collation_func('scmp', 'es', template='_strcmp_template') as scmp: + self.assertNotEqual(0, scmp('pena', 'peña')) + + for k, v in {u'pèché': u'peche', u'flüße':u'Flusse', u'Štepánek':u'ŠtepaneK'}.iteritems(): + self.ae(0, icu.primary_strcmp(k, v)) + + # Test different types of collation + self.ae(icu.primary_sort_key('Aä'), icu.primary_sort_key('aa')) + self.assertLess(icu.numeric_sort_key('something 2'), icu.numeric_sort_key('something 11')) + self.assertLess(icu.case_sensitive_sort_key('A'), icu.case_sensitive_sort_key('a')) + self.ae(0, icu.strcmp('a', 'A')) + self.ae(cmp('a', 'A'), icu.case_sensitive_strcmp('a', 'A')) + self.ae(0, icu.primary_strcmp('ä', 'A')) + + def test_change_case(self): + ' Test the various ways of changing the case ' + from calibre.utils.titlecase import titlecase + # Test corner cases + self.ae('A', icu.upper(b'a')) + + for x in ('a', 'Alice\'s code', 'macdonald\'s machIne', '02 the wars'): + self.ae(icu.upper(x), x.upper()) + self.ae(icu.lower(x), x.lower()) + # ICU's title case algorithm is different from ours, when there are + # capitals inside words + self.ae(icu.title_case(x), titlecase(x).replace('machIne', 'Machine')) + self.ae(icu.capitalize(x), x[0].upper() + x[1:].lower()) + + def test_find(self): + ' Test searching for substrings ' + self.ae((1, 1), icu.find(b'a', b'1ab')) + self.ae((1, 2), icu.find('\U0001f431', 'x\U0001f431x')) + self.ae((0, 4), icu.primary_find('pena', 'peña')) + for k, v in {u'pèché': u'peche', u'flüße':u'Flusse', u'Štepánek':u'ŠtepaneK'}.iteritems(): + self.ae((1, len(k)), icu.primary_find(v, ' ' + k), 'Failed to find %s in %s' % (v, k)) + self.assertTrue(icu.startswith(b'abc', b'ab')) + self.assertTrue(icu.startswith('abc', 'abc')) + self.assertFalse(icu.startswith('xyz', 'a')) + self.assertTrue(icu.startswith('xxx', '')) + self.assertTrue(icu.primary_startswith('pena', 'peña')) + + def test_collation_order(self): + 'Testing collation ordering' + for group in [ + ('Šaa', 'Smith', 'Solženicyn', 'Štepánek'), + ('01', '1'), + ('1', '11', '13'), + ]: + last = None + for x in group: + order, length = icu.numeric_collator().collation_order(x) + if last is not None: + self.ae(last, order) + last = order + + def test_roundtrip(self): + for r in (u'xxx\0\u2219\U0001f431xxx', u'\0', u'', u'simple'): + self.ae(r, icu._icu.roundtrip(r)) + + def test_character_name(self): + self.ae(icu.character_name('\U0001f431'), 'CAT FACE') + + def test_contractions(self): + c = icu._icu.Collator('cs') + self.ae(icu.contractions(c), frozenset({u'Z\u030c', u'z\u030c', u'Ch', + u'C\u030c', u'ch', u'cH', u'c\u030c', u's\u030c', u'r\u030c', u'CH', + u'S\u030c', u'R\u030c'})) + +class TestRunner(unittest.main): + + def createTests(self): + tl = unittest.TestLoader() + self.test = tl.loadTestsFromTestCase(TestICU) + +def run(verbosity=4): + TestRunner(verbosity=verbosity, exit=False) + +def test_build(): + result = TestRunner(verbosity=0, buffer=True, catchbreak=True, failfast=True, argv=sys.argv[:1], exit=False).result + if not result.wasSuccessful(): + raise SystemExit(1) + +if __name__ == '__main__': + run(verbosity=4) + From 27327e811b5d191272c1c955a0b172887dc2503f Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Sat, 8 Mar 2014 21:19:05 +0530 Subject: [PATCH 012/122] Clearer error message when compiling on python >= 3.3 --- src/calibre/utils/icu_calibre_utils.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/calibre/utils/icu_calibre_utils.h b/src/calibre/utils/icu_calibre_utils.h index 5cab803258..a965d0c072 100644 --- a/src/calibre/utils/icu_calibre_utils.h +++ b/src/calibre/utils/icu_calibre_utils.h @@ -21,7 +21,10 @@ #include <unicode/utrans.h> #include <unicode/unorm.h> -#if PY_VERSION_HEX < 0x03030000 +#if PY_VERSION_HEX >= 0x03030000 +#error Not implemented for python >= 3.3 +#endif + // Roundtripping will need to be implemented differently for python 3.3+ where strings are stored with variable widths #ifndef NO_PYTHON_TO_ICU @@ -67,5 +70,4 @@ static PyObject* icu_to_python(UChar *src, int32_t sz) { } #endif -#endif From 4eaee89487bf3b1e85d65fdd8e10f7da3cce5602 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Sat, 8 Mar 2014 21:41:05 +0530 Subject: [PATCH 013/122] Fix ICU find returning incorrect position and length parameters when non-BMP characters are present on wide python builds --- src/calibre/utils/icu.c | 15 ++++++++++++--- src/calibre/utils/icu_test.py | 3 ++- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/src/calibre/utils/icu.c b/src/calibre/utils/icu.c index d556115c45..34649afa3f 100644 --- a/src/calibre/utils/icu.c +++ b/src/calibre/utils/icu.c @@ -191,6 +191,9 @@ end: // Collator.find {{{ static PyObject * icu_Collator_find(icu_Collator *self, PyObject *args, PyObject *kwargs) { +#if PY_VERSION_HEX >= 0x03030000 +#error Not implemented for python >= 3.3 +#endif PyObject *a_ = NULL, *b_ = NULL; UChar *a = NULL, *b = NULL; int32_t asz = 0, bsz = 0, pos = -1, length = -1; @@ -207,10 +210,16 @@ icu_Collator_find(icu_Collator *self, PyObject *args, PyObject *kwargs) { search = usearch_openFromCollator(a, asz, b, bsz, self->collator, NULL, &status); if (U_SUCCESS(status)) { pos = usearch_first(search, &status); - if (pos != USEARCH_DONE) + if (pos != USEARCH_DONE) { length = usearch_getMatchedLength(search); - else - pos = -1; +#ifdef Py_UNICODE_WIDE + // We have to return number of unicode characters since the string + // could contain surrogate pairs which are represented as a single + // character in python wide builds + length = u_countChar32(b + pos, length); + pos = u_countChar32(b, pos); +#endif + } else pos = -1; } end: if (search != NULL) usearch_close(search); diff --git a/src/calibre/utils/icu_test.py b/src/calibre/utils/icu_test.py index e96397e86a..d6d5f557f4 100644 --- a/src/calibre/utils/icu_test.py +++ b/src/calibre/utils/icu_test.py @@ -92,7 +92,8 @@ class TestICU(unittest.TestCase): def test_find(self): ' Test searching for substrings ' self.ae((1, 1), icu.find(b'a', b'1ab')) - self.ae((1, 2), icu.find('\U0001f431', 'x\U0001f431x')) + self.ae((1, 1 if sys.maxunicode >= 0x10ffff else 2), icu.find('\U0001f431', 'x\U0001f431x')) + self.ae((1 if sys.maxunicode >= 0x10ffff else 2, 1), icu.find('y', '\U0001f431y')) self.ae((0, 4), icu.primary_find('pena', 'peña')) for k, v in {u'pèché': u'peche', u'flüße':u'Flusse', u'Štepánek':u'ŠtepaneK'}.iteritems(): self.ae((1, len(k)), icu.primary_find(v, ' ' + k), 'Failed to find %s in %s' % (v, k)) From b76cc3e9ab45e5206bb57f3266156dc3e3541728 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Sat, 8 Mar 2014 22:08:31 +0530 Subject: [PATCH 014/122] Speed up searching a little by using a dedicated function for testing if a string contains a substring using primary collation (replaces using primary_find() --- src/calibre/db/search.py | 4 +-- src/calibre/gui2/complete2.py | 4 +-- src/calibre/gui2/dialogs/tag_editor.py | 4 +-- src/calibre/utils/icu.c | 37 ++++++++++++++++++++++++++ src/calibre/utils/icu.py | 4 +++ src/calibre/utils/icu_test.py | 6 +++++ 6 files changed, 53 insertions(+), 6 deletions(-) diff --git a/src/calibre/db/search.py b/src/calibre/db/search.py index d01db38552..448b0f896d 100644 --- a/src/calibre/db/search.py +++ b/src/calibre/db/search.py @@ -16,7 +16,7 @@ from calibre.constants import preferred_encoding from calibre.db.utils import force_to_bool from calibre.utils.config_base import prefs from calibre.utils.date import parse_date, UNDEFINED_DATE, now, dt_as_local -from calibre.utils.icu import primary_find, sort_key +from calibre.utils.icu import primary_contains, sort_key from calibre.utils.localization import lang_map, canonicalize_lang from calibre.utils.search_query_parser import SearchQueryParser, ParseException @@ -73,7 +73,7 @@ def _match(query, value, matchkind, use_primary_find_in_search=True): return True elif matchkind == CONTAINS_MATCH: if use_primary_find_in_search: - if primary_find(query, t)[0] != -1: + if primary_contains(query, t): return True elif query in t: return True diff --git a/src/calibre/gui2/complete2.py b/src/calibre/gui2/complete2.py index 623215a6e6..7c98e6477c 100644 --- a/src/calibre/gui2/complete2.py +++ b/src/calibre/gui2/complete2.py @@ -14,13 +14,13 @@ from PyQt4.Qt import (QLineEdit, QAbstractListModel, Qt, pyqtSignal, QObject, QApplication, QListView, QPoint, QModelIndex, QFont, QFontInfo) from calibre.constants import isosx, get_osx_version -from calibre.utils.icu import sort_key, primary_startswith, primary_find +from calibre.utils.icu import sort_key, primary_startswith, primary_contains from calibre.gui2 import NONE from calibre.gui2.widgets import EnComboBox, LineEditECM from calibre.utils.config import tweaks def containsq(x, prefix): - return primary_find(prefix, x)[0] != -1 + return primary_contains(prefix, x) class CompleteModel(QAbstractListModel): # {{{ diff --git a/src/calibre/gui2/dialogs/tag_editor.py b/src/calibre/gui2/dialogs/tag_editor.py index 15959543e5..ef16dc4704 100644 --- a/src/calibre/gui2/dialogs/tag_editor.py +++ b/src/calibre/gui2/dialogs/tag_editor.py @@ -9,7 +9,7 @@ from PyQt4.QtGui import QDialog from calibre.gui2.dialogs.tag_editor_ui import Ui_TagEditor from calibre.gui2 import question_dialog, error_dialog, gprefs from calibre.constants import islinux -from calibre.utils.icu import sort_key, primary_find +from calibre.utils.icu import sort_key, primary_contains class TagEditor(QDialog, Ui_TagEditor): @@ -178,7 +178,7 @@ class TagEditor(QDialog, Ui_TagEditor): q = icu_lower(unicode(filter_value)) for i in xrange(collection.count()): # on every available tag item = collection.item(i) - item.setHidden(bool(q and primary_find(q, unicode(item.text()))[0] == -1)) + item.setHidden(bool(q and not primary_contains(q, unicode(item.text())))) def accept(self): self.save_state() diff --git a/src/calibre/utils/icu.c b/src/calibre/utils/icu.c index 34649afa3f..22c9bbb811 100644 --- a/src/calibre/utils/icu.c +++ b/src/calibre/utils/icu.c @@ -229,6 +229,39 @@ end: return (PyErr_Occurred()) ? NULL : Py_BuildValue("ii", pos, length); } // }}} +// Collator.contains {{{ +static PyObject * +icu_Collator_contains(icu_Collator *self, PyObject *args, PyObject *kwargs) { + PyObject *a_ = NULL, *b_ = NULL; + UChar *a = NULL, *b = NULL; + int32_t asz = 0, bsz = 0, pos = -1; + uint8_t found = 0; + UErrorCode status = U_ZERO_ERROR; + UStringSearch *search = NULL; + + if (!PyArg_ParseTuple(args, "OO", &a_, &b_)) return NULL; + + a = python_to_icu(a_, &asz, 1); + if (a == NULL) goto end; + if (asz == 0) { found = TRUE; goto end; } + b = python_to_icu(b_, &bsz, 1); + if (b == NULL) goto end; + + search = usearch_openFromCollator(a, asz, b, bsz, self->collator, NULL, &status); + if (U_SUCCESS(status)) { + pos = usearch_first(search, &status); + if (pos != USEARCH_DONE) found = TRUE; + } +end: + if (search != NULL) usearch_close(search); + if (a != NULL) free(a); + if (b != NULL) free(b); + + if (PyErr_Occurred()) return NULL; + if (found) Py_RETURN_TRUE; + Py_RETURN_FALSE; +} // }}} + // Collator.contractions {{{ static PyObject * icu_Collator_contractions(icu_Collator *self, PyObject *args, PyObject *kwargs) { @@ -366,6 +399,10 @@ static PyMethodDef icu_Collator_methods[] = { "find(pattern, source) -> returns the position and length of the first occurrence of pattern in source. Returns (-1, -1) if not found." }, + {"contains", (PyCFunction)icu_Collator_contains, METH_VARARGS, + "contains(pattern, source) -> return True iff the pattern was found in the source." + }, + {"contractions", (PyCFunction)icu_Collator_contractions, METH_VARARGS, "contractions() -> returns the contractions defined for this collator." }, diff --git a/src/calibre/utils/icu.py b/src/calibre/utils/icu.py index df4c369365..f6973c72a6 100644 --- a/src/calibre/utils/icu.py +++ b/src/calibre/utils/icu.py @@ -203,6 +203,10 @@ find = _make_func(_strcmp_template, 'find', collator='_collator', collator_func= primary_find = _make_func(_strcmp_template, 'primary_find', collator='_primary_collator', collator_func='primary_collator', func='find') +contains = _make_func(_strcmp_template, 'contains', collator='_collator', collator_func='collator', func='contains') + +primary_contains = _make_func(_strcmp_template, 'primary_contains', collator='_primary_collator', collator_func='primary_collator', func='contains') + startswith = _make_func(_strcmp_template, 'startswith', collator='_collator', collator_func='collator', func='startswith') primary_startswith = _make_func(_strcmp_template, 'primary_startswith', collator='_primary_collator', collator_func='primary_collator', func='startswith') diff --git a/src/calibre/utils/icu_test.py b/src/calibre/utils/icu_test.py index d6d5f557f4..3b2775e454 100644 --- a/src/calibre/utils/icu_test.py +++ b/src/calibre/utils/icu_test.py @@ -102,6 +102,12 @@ class TestICU(unittest.TestCase): self.assertFalse(icu.startswith('xyz', 'a')) self.assertTrue(icu.startswith('xxx', '')) self.assertTrue(icu.primary_startswith('pena', 'peña')) + self.assertTrue(icu.contains('\U0001f431', '\U0001f431')) + self.assertTrue(icu.contains('something', 'some other something else')) + self.assertTrue(icu.contains('', 'a')) + self.assertTrue(icu.contains('', '')) + self.assertFalse(icu.contains('xxx', 'xx')) + self.assertTrue(icu.primary_contains('pena', 'peña')) def test_collation_order(self): 'Testing collation ordering' From 1f2aa8a55bb7e626b3b580fef02dd7fb9f18b6ea Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Sat, 8 Mar 2014 22:18:29 +0530 Subject: [PATCH 015/122] Allow merging of icu branch into trunk by falling back to the old icu module if the old binary plugin is detected. --- src/calibre/utils/icu.py | 10 + src/calibre/utils/icu_old.py | 541 +++++++++++++++++++++++++++++++++++ 2 files changed, 551 insertions(+) create mode 100644 src/calibre/utils/icu_old.py diff --git a/src/calibre/utils/icu.py b/src/calibre/utils/icu.py index f6973c72a6..6f335a5434 100644 --- a/src/calibre/utils/icu.py +++ b/src/calibre/utils/icu.py @@ -247,6 +247,16 @@ def contractions(col=None): ################################################################################ +if not hasattr(_icu, 'change_case'): + print ('You are running from source with an outdated calibre binary install. You' + ' should update the main calibre binary to at least version 1.28.') + # Dont creak calibre for people running from source until the + # next binary is available witht he update icu module + from calibre.utils.icu_old import * # noqa + + def primary_contains(pat, src): + return primary_find(pat, src)[0] != -1 + if __name__ == '__main__': from calibre.utils.icu_test import run run(verbosity=4) diff --git a/src/calibre/utils/icu_old.py b/src/calibre/utils/icu_old.py new file mode 100644 index 0000000000..39256f6fd6 --- /dev/null +++ b/src/calibre/utils/icu_old.py @@ -0,0 +1,541 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai + +__license__ = 'GPL v3' +__copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>' +__docformat__ = 'restructuredtext en' + +# Setup code {{{ +import sys +from functools import partial + +from calibre.constants import plugins +from calibre.utils.config_base import tweaks + +_icu = _collator = _primary_collator = _sort_collator = _numeric_collator = None +_locale = None + +_none = u'' +_none2 = b'' + +def get_locale(): + global _locale + if _locale is None: + from calibre.utils.localization import get_lang + if tweaks['locale_for_sorting']: + _locale = tweaks['locale_for_sorting'] + else: + _locale = get_lang() + return _locale + +def load_icu(): + global _icu + if _icu is None: + _icu = plugins['icu'][0] + if _icu is None: + print 'Loading ICU failed with: ', plugins['icu'][1] + else: + if not getattr(_icu, 'ok', False): + print 'icu not ok' + _icu = None + return _icu + +def load_collator(): + 'The default collator for most locales takes both case and accented letters into account' + global _collator + if _collator is None: + icu = load_icu() + if icu is not None: + _collator = icu.Collator(get_locale()) + return _collator + +def primary_collator(): + 'Ignores case differences and accented characters' + global _primary_collator + if _primary_collator is None: + _primary_collator = _collator.clone() + _primary_collator.strength = _icu.UCOL_PRIMARY + return _primary_collator + +def sort_collator(): + 'Ignores case differences and recognizes numbers in strings' + global _sort_collator + if _sort_collator is None: + _sort_collator = _collator.clone() + _sort_collator.strength = _icu.UCOL_SECONDARY + if tweaks['numeric_collation']: + try: + _sort_collator.numeric = True + except AttributeError: + pass + return _sort_collator + +def py_sort_key(obj): + if not obj: + return _none + return obj.lower() + +def icu_sort_key(collator, obj): + if not obj: + return _none2 + try: + try: + return _sort_collator.sort_key(obj) + except AttributeError: + return sort_collator().sort_key(obj) + except TypeError: + if isinstance(obj, unicode): + obj = obj.replace(u'\0', u'') + else: + obj = obj.replace(b'\0', b'') + return _sort_collator.sort_key(obj) + +def numeric_collator(): + global _numeric_collator + _numeric_collator = _collator.clone() + _numeric_collator.strength = _icu.UCOL_SECONDARY + _numeric_collator.numeric = True + return _numeric_collator + +def numeric_sort_key(obj): + 'Uses natural sorting for numbers inside strings so something2 will sort before something10' + if not obj: + return _none2 + try: + try: + return _numeric_collator.sort_key(obj) + except AttributeError: + return numeric_collator().sort_key(obj) + except TypeError: + if isinstance(obj, unicode): + obj = obj.replace(u'\0', u'') + else: + obj = obj.replace(b'\0', b'') + return _numeric_collator.sort_key(obj) + +def icu_change_case(upper, locale, obj): + func = _icu.upper if upper else _icu.lower + try: + return func(locale, obj) + except TypeError: + if isinstance(obj, unicode): + obj = obj.replace(u'\0', u'') + else: + obj = obj.replace(b'\0', b'') + return func(locale, obj) + +def py_find(pattern, source): + pos = source.find(pattern) + if pos > -1: + return pos, len(pattern) + return -1, -1 + +def character_name(string): + try: + try: + return _icu.character_name(unicode(string)) or None + except AttributeError: + import unicodedata + return unicodedata.name(unicode(string)[0], None) + except (TypeError, ValueError, KeyError): + pass + +def character_name_from_code(code): + try: + try: + return _icu.character_name_from_code(code) or '' + except AttributeError: + import unicodedata + return unicodedata.name(py_safe_chr(code), '') + except (TypeError, ValueError, KeyError): + return '' + +if sys.maxunicode >= 0x10ffff: + try: + py_safe_chr = unichr + except NameError: + py_safe_chr = chr +else: + def py_safe_chr(i): + # Narrow builds of python cannot represent code point > 0xffff as a + # single character, so we need our own implementation of unichr + # that returns them as a surrogate pair + return (b"\U%s" % (hex(i)[2:].zfill(8))).decode('unicode-escape') + +def safe_chr(code): + try: + return _icu.chr(code) + except AttributeError: + return py_safe_chr(code) + +def normalize(text, mode='NFC'): + # This is very slightly slower than using unicodedata.normalize, so stick with + # that unless you have very good reasons not too. Also, it's speed + # decreases on wide python builds, where conversion to/from ICU's string + # representation is slower. + try: + return _icu.normalize(_nmodes[mode], unicode(text)) + except (AttributeError, KeyError): + import unicodedata + return unicodedata.normalize(mode, unicode(text)) + +def icu_find(collator, pattern, source): + try: + return collator.find(pattern, source) + except TypeError: + return collator.find(unicode(pattern), unicode(source)) + +def icu_startswith(collator, a, b): + try: + return collator.startswith(a, b) + except TypeError: + return collator.startswith(unicode(a), unicode(b)) + +def py_case_sensitive_sort_key(obj): + if not obj: + return _none + return obj + +def icu_case_sensitive_sort_key(collator, obj): + if not obj: + return _none2 + return collator.sort_key(obj) + +def icu_strcmp(collator, a, b): + return collator.strcmp(lower(a), lower(b)) + +def py_strcmp(a, b): + return cmp(a.lower(), b.lower()) + +def icu_case_sensitive_strcmp(collator, a, b): + return collator.strcmp(a, b) + +def icu_capitalize(s): + s = lower(s) + return s.replace(s[0], upper(s[0]), 1) if s else s + +_cmap = {} +def icu_contractions(collator): + global _cmap + ans = _cmap.get(collator, None) + if ans is None: + ans = collator.contractions() + ans = frozenset(filter(None, ans)) if ans else {} + _cmap[collator] = ans + return ans + +def icu_collation_order(collator, a): + try: + return collator.collation_order(a) + except TypeError: + return collator.collation_order(unicode(a)) + +load_icu() +load_collator() +_icu_not_ok = _icu is None or _collator is None +icu_unicode_version = getattr(_icu, 'unicode_version', None) +_nmodes = {m:getattr(_icu, 'UNORM_'+m, None) for m in ('NFC', 'NFD', 'NFKC', 'NFKD', 'NONE', 'DEFAULT', 'FCD')} + +try: + senc = sys.getdefaultencoding() + if not senc or senc.lower() == 'ascii': + _icu.set_default_encoding('utf-8') + del senc +except: + pass + +try: + fenc = sys.getfilesystemencoding() + if not fenc or fenc.lower() == 'ascii': + _icu.set_filesystem_encoding('utf-8') + del fenc +except: + pass + + +# }}} + +################# The string functions ######################################## + +sort_key = py_sort_key if _icu_not_ok else partial(icu_sort_key, _collator) + +strcmp = py_strcmp if _icu_not_ok else partial(icu_strcmp, _collator) + +case_sensitive_sort_key = py_case_sensitive_sort_key if _icu_not_ok else \ + partial(icu_case_sensitive_sort_key, _collator) + +case_sensitive_strcmp = cmp if _icu_not_ok else icu_case_sensitive_strcmp + +upper = (lambda s: s.upper()) if _icu_not_ok else \ + partial(icu_change_case, True, get_locale()) + +lower = (lambda s: s.lower()) if _icu_not_ok else \ + partial(icu_change_case, False, get_locale()) + +title_case = (lambda s: s.title()) if _icu_not_ok else \ + partial(_icu.title, get_locale()) + +capitalize = (lambda s: s.capitalize()) if _icu_not_ok else \ + (lambda s: icu_capitalize(s)) + +find = (py_find if _icu_not_ok else partial(icu_find, _collator)) + +contractions = ((lambda : {}) if _icu_not_ok else (partial(icu_contractions, + _collator))) + +def primary_strcmp(a, b): + 'strcmp that ignores case and accents on letters' + if _icu_not_ok: + from calibre.utils.filenames import ascii_text + return py_strcmp(ascii_text(a), ascii_text(b)) + try: + return _primary_collator.strcmp(a, b) + except AttributeError: + return primary_collator().strcmp(a, b) + +def primary_find(pat, src): + 'find that ignores case and accents on letters' + if _icu_not_ok: + from calibre.utils.filenames import ascii_text + return py_find(ascii_text(pat), ascii_text(src)) + return primary_icu_find(pat, src) + +def primary_icu_find(pat, src): + try: + return icu_find(_primary_collator, pat, src) + except AttributeError: + return icu_find(primary_collator(), pat, src) + +def primary_sort_key(val): + 'A sort key that ignores case and diacritics' + if _icu_not_ok: + from calibre.utils.filenames import ascii_text + return ascii_text(val).lower() + try: + return _primary_collator.sort_key(val) + except AttributeError: + return primary_collator().sort_key(val) + +def primary_startswith(a, b): + if _icu_not_ok: + from calibre.utils.filenames import ascii_text + return ascii_text(a).lower().startswith(ascii_text(b).lower()) + try: + return icu_startswith(_primary_collator, a, b) + except AttributeError: + return icu_startswith(primary_collator(), a, b) + +def collation_order(a): + if _icu_not_ok: + return (ord(a[0]), 1) if a else (0, 0) + try: + return icu_collation_order(_sort_collator, a) + except AttributeError: + return icu_collation_order(sort_collator(), a) + +################################################################################ + +def test(): # {{{ + from calibre import prints + # Data {{{ + german = ''' + Sonntag +Montag +Dienstag +Januar +Februar +März +Fuße +Fluße +Flusse +flusse +fluße +flüße +flüsse +''' + german_good = ''' + Dienstag +Februar +flusse +Flusse +fluße +Fluße +flüsse +flüße +Fuße +Januar +März +Montag +Sonntag''' + french = ''' +dimanche +lundi +mardi +janvier +février +mars +déjà +Meme +deja +même +dejà +bpef +bœg +Boef +Mémé +bœf +boef +bnef +pêche +pèché +pêché +pêche +pêché''' + french_good = ''' + bnef + boef + Boef + bœf + bœg + bpef + deja + dejà + déjà + dimanche + février + janvier + lundi + mardi + mars + Meme + Mémé + même + pèché + pêche + pêche + pêché + pêché''' + # }}} + + def create(l): + l = l.decode('utf-8').splitlines() + return [x.strip() for x in l if x.strip()] + + def test_strcmp(entries): + for x in entries: + for y in entries: + if strcmp(x, y) != cmp(sort_key(x), sort_key(y)): + print 'strcmp failed for %r, %r'%(x, y) + + german = create(german) + c = _icu.Collator('de') + c.numeric = True + gs = list(sorted(german, key=c.sort_key)) + if gs != create(german_good): + print 'German sorting failed' + return + print + french = create(french) + c = _icu.Collator('fr') + c.numeric = True + fs = list(sorted(french, key=c.sort_key)) + if fs != create(french_good): + print 'French sorting failed (note that French fails with icu < 4.6)' + return + test_strcmp(german + french) + + print '\nTesting case transforms in current locale' + from calibre.utils.titlecase import titlecase + for x in ('a', 'Alice\'s code', 'macdonald\'s machine', '02 the wars'): + print 'Upper: ', x, '->', 'py:', x.upper().encode('utf-8'), 'icu:', upper(x).encode('utf-8') + print 'Lower: ', x, '->', 'py:', x.lower().encode('utf-8'), 'icu:', lower(x).encode('utf-8') + print 'Title: ', x, '->', 'py:', x.title().encode('utf-8'), 'icu:', title_case(x).encode('utf-8'), 'titlecase:', titlecase(x).encode('utf-8') + print 'Capitalize:', x, '->', 'py:', x.capitalize().encode('utf-8'), 'icu:', capitalize(x).encode('utf-8') + print + + print '\nTesting primary collation' + for k, v in {u'pèché': u'peche', u'flüße':u'Flusse', + u'Štepánek':u'ŠtepaneK'}.iteritems(): + if primary_strcmp(k, v) != 0: + prints('primary_strcmp() failed with %s != %s'%(k, v)) + return + if primary_find(v, u' '+k)[0] != 1: + prints('primary_find() failed with %s not in %s'%(v, k)) + return + + n = character_name(safe_chr(0x1f431)) + if n != u'CAT FACE': + raise ValueError('Failed to get correct character name for 0x1f431: %r != %r' % n, u'CAT FACE') + + global _primary_collator + orig = _primary_collator + _primary_collator = _icu.Collator('es') + if primary_strcmp(u'peña', u'pena') == 0: + print 'Primary collation in Spanish locale failed' + return + _primary_collator = orig + + print '\nTesting contractions' + c = _icu.Collator('cs') + if icu_contractions(c) != frozenset([u'Z\u030c', u'z\u030c', u'Ch', + u'C\u030c', u'ch', u'cH', u'c\u030c', u's\u030c', u'r\u030c', u'CH', + u'S\u030c', u'R\u030c']): + print 'Contractions for the Czech language failed' + return + + print '\nTesting startswith' + p = primary_startswith + if (not p('asd', 'asd') or not p('asd', 'A') or + not p('x', '')): + print 'startswith() failed' + return + + print '\nTesting collation_order()' + for group in [ + ('Šaa', 'Smith', 'Solženicyn', 'Štepánek'), + ('calibre', 'Charon', 'Collins'), + ('01', '1'), + ('1', '11', '13'), + ]: + last = None + for x in group: + val = icu_collation_order(sort_collator(), x) + if val[1] != 1: + prints('collation_order() returned incorrect length for', x) + if last is None: + last = val + else: + if val != last: + prints('collation_order() returned incorrect value for', x) + last = val + +# }}} + +def test_roundtrip(): + for r in (u'xxx\0\u2219\U0001f431xxx', u'\0', u'', u'simple'): + rp = _icu.roundtrip(r) + if rp != r: + raise ValueError(u'Roundtripping failed: %r != %r' % (r, rp)) + +def test_normalize_performance(): + import os + if not os.path.exists('t.txt'): + return + raw = open('t.txt', 'rb').read().decode('utf-8') + print (len(raw)) + import time, unicodedata + st = time.time() + count = 100 + for i in xrange(count): + normalize(raw) + print ('ICU time:', time.time() - st) + st = time.time() + for i in xrange(count): + unicodedata.normalize('NFC', unicode(raw)) + print ('py time:', time.time() - st) + +if __name__ == '__main__': + test_roundtrip() + test_normalize_performance() + test() + From 158c33481eefe2a948602fa926ca68782b6b9f0b Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Sun, 9 Mar 2014 08:03:24 +0530 Subject: [PATCH 016/122] ... --- src/calibre/utils/matcher.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/calibre/utils/matcher.py b/src/calibre/utils/matcher.py index de7ed95f4b..aa501fe4ff 100644 --- a/src/calibre/utils/matcher.py +++ b/src/calibre/utils/matcher.py @@ -111,19 +111,20 @@ class Matcher(object): raise Exception('Failed to score items: %s' % error) items = sorted(((-scores[i], item, positions[i]) for i, item in enumerate(self.items)), key=itemgetter(0)) - return OrderedDict(x[1:] for x in items) + return OrderedDict(x[1:] for x in filter(itemgetter(0), items)) -def get_items_from_dir(basedir): +def get_items_from_dir(basedir, acceptq=lambda x: True): if isinstance(basedir, bytes): basedir = basedir.decode(filesystem_encoding) relsep = os.sep != '/' for dirpath, dirnames, filenames in os.walk(basedir): for f in filenames: x = os.path.join(dirpath, f) - x = os.path.relpath(x, basedir) - if relsep: - x = x.replace(os.sep, '/') - yield x + if acceptq(x): + x = os.path.relpath(x, basedir) + if relsep: + x = x.replace(os.sep, '/') + yield x class FilesystemMatcher(Matcher): From ea6b203b37415939ca6e84e273ec5755656d5d6f Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Sun, 9 Mar 2014 09:23:58 +0530 Subject: [PATCH 017/122] E-book viewer: Add an option to control the maximum text height in full screen. Note that it only owrks if the viewer is in paged mode (which is the default mode). --- resources/compiled_coffeescript.zip | Bin 80305 -> 81090 bytes .../ebooks/oeb/display/full_screen.coffee | 3 +- src/calibre/ebooks/oeb/display/paged.coffee | 28 +++++++--- src/calibre/gui2/viewer/config.py | 9 ++++ src/calibre/gui2/viewer/config.ui | 50 +++++++++++++----- src/calibre/gui2/viewer/documentview.py | 3 +- 6 files changed, 71 insertions(+), 22 deletions(-) diff --git a/resources/compiled_coffeescript.zip b/resources/compiled_coffeescript.zip index e50a016181f370e2f58b763db26cd71b69d4c043..fa675be5040264752ec6a0bfc79bcaaec8c7e59c 100644 GIT binary patch delta 1080 zcmdn^ndQ(=mJMRrLgt>CF3Fm^A3Wn^U;tr7hUtYSjLMt6v$>fCa}z7#OHwOJ;xke+ z(=$qR6eic@iBJBY&pWv)FJki4Jb4L-go3RCn$pRRS0pEIEzsL6nEzJ;Y)SQ_yBYy; z3=AMF4YovddX6b0$7b6ZwM@bw?aBE$P>X=(=^9RUm?b)S&Qb2oXJ_5#n!I^wvb2Ii zNk(R|UTRueYH~?tS!#T4Vo`c#UVKS@!Q`%YGO9(XC8b4qI20x2mz3n^D%ei`zgBv( z=M}-p>dX8lCoC16JaL&Q6-~~#q`LX^vL+^`)QrvED~nk+ORdXhnton|kyQkTyQSo? z`wGRu9<z8sVKMpIS{J#@GzAT?7PLUKQ!r3aQ=8m(RMJSn4iXBeDl`?UxfGxvH3JyP zzKJCnddaDoIT{+1`;SW0E9hd^rmtY6X${wk-MPR>P|&tjNX;Om5b7^1YA5fUB`TzA zt6-#{g=9|c<ck}{Cr{mA26E!$+I0$()i#E4Ap&!9%|?49?&jSaWjK-f*LR&_MB-0g zxJMm{yZOYPQ-;{1X1%lM<c;r?kriKgSL@1)l4YjHnJ^|W8J2=_jp+2NCXA*`h20R& zJqt#z>Ds1@JPIHksOg^p353jms0Zc-Bc{9GAe{f^j2zSVnlfrJxwB4xY0Bu%yoQf) z`VMYJh3Ua&K+zic>GfuedQ3;mrmr_+tkf_tF-}ZOHZx4MFi*8eGByKJ$%#ppW@d@T zDJiMOM#dIthL))Y=F<br8I?7REK<`9%}mV=laf-*l8g+@(ozkK4K0mLOf8HPlT8hh alTDM-EX_<ns>P>Un=o>1-($}Bl@S21v3=|S delta 475 zcmX@~lV#&)mJMRrLiq_kF75BHADqU?zyQLE4AU=~Gm37u&6Z@ItguFWvVQ^Z<nFwP z$=lxwPG&9C+nkvHS3~G-n6C@Ba_{>7Xa)ulmWJq>UTeX~vAK6fEz{(88^ty|%)Z0L zR+L&&T9h|gdRYqC2=?WEFro8H#U^Jh7oBYJT6MGZiYBJb3s)7gOn$jBX0y=xY^KQ` z>jfq^Z#J8J;G^_pxlQtnnv*YXE}I;<#eVawEi#;&59~d~2oz_ZJbS;w=A-*h8BSib zOl0!fk9r`%&2K)Ix$>g;ZF;^rV*=B|5|H=2m=wFg-kkp4nvrX|jRhl*0;ZQ3kO0dJ zh<cz0jhN1Sp1#q7(VfYRY5D|9Mv>`!mW;+sv$-LhJA911(^D)NwV1MHKq7ifn~kR* zuw<;%NHVZUHA=HIvM@6?HnTKOGcr#$OG`~mNj6TkG)OfzvPiQqHZ@4IoStaKsH~Bi zY-*a6YLt?kY+{sVmY8H_V3=x_mS$#Tnqp#Zlw@jRnU-i`X=n^mEk51HoRMq$Su4h` Fi~!wPs6zk% diff --git a/src/calibre/ebooks/oeb/display/full_screen.coffee b/src/calibre/ebooks/oeb/display/full_screen.coffee index f4dece210a..7c5adc3f3c 100644 --- a/src/calibre/ebooks/oeb/display/full_screen.coffee +++ b/src/calibre/ebooks/oeb/display/full_screen.coffee @@ -25,9 +25,10 @@ class FullScreen this.initial_left_margin = bs.marginLeft this.initial_right_margin = bs.marginRight - on: (max_text_width, in_paged_mode) -> + on: (max_text_width, max_text_height, in_paged_mode) -> if in_paged_mode window.paged_display.max_col_width = max_text_width + window.paged_display.max_col_height = max_text_height else s = document.body.style s.maxWidth = max_text_width + 'px' diff --git a/src/calibre/ebooks/oeb/display/paged.coffee b/src/calibre/ebooks/oeb/display/paged.coffee index 37fb46c9ba..56c7135e90 100644 --- a/src/calibre/ebooks/oeb/display/paged.coffee +++ b/src/calibre/ebooks/oeb/display/paged.coffee @@ -26,6 +26,7 @@ class PagedDisplay this.current_margin_side = 0 this.is_full_screen_layout = false this.max_col_width = -1 + this.max_col_height = - 1 this.current_page_height = null this.document_margins = null this.use_document_margins = false @@ -71,10 +72,14 @@ class PagedDisplay this.margin_top = this.document_margins.top or margin_top this.margin_bottom = this.document_margins.bottom or margin_bottom this.margin_side = this.document_margins.left or this.document_margins.right or margin_side + this.effective_margin_top = this.margin_top + this.effective_margin_bottom = this.margin_bottom else this.margin_top = margin_top this.margin_side = margin_side this.margin_bottom = margin_bottom + this.effective_margin_top = this.margin_top + this.effective_margin_bottom = this.margin_bottom handle_rtl_body: (body_style) -> if body_style.direction == "rtl" @@ -118,7 +123,6 @@ class PagedDisplay this.col_width = col_width this.page_width = col_width + 2*sm this.screen_width = this.page_width * this.cols_per_screen - this.current_page_height = window.innerHeight - this.margin_top - this.margin_bottom fgcolor = body_style.getPropertyValue('color') @@ -142,12 +146,20 @@ class PagedDisplay if c?.nodeType == 1 c.style.setProperty('-webkit-margin-before', '0') + this.effective_margin_top = this.margin_top + this.effective_margin_bottom = this.margin_bottom + this.current_page_height = window.innerHeight - this.margin_top - this.margin_bottom + if this.max_col_height > 0 and this.current_page_height > this.max_col_height + eh = Math.ceil((this.current_page_height - this.max_col_height) / 2) + this.effective_margin_top += eh + this.effective_margin_bottom += eh + this.current_page_height -= 2 * eh bs.setProperty('overflow', 'visible') - bs.setProperty('height', (window.innerHeight - this.margin_top - this.margin_bottom) + 'px') + bs.setProperty('height', this.current_page_height + 'px') bs.setProperty('width', (window.innerWidth - 2*sm)+'px') - bs.setProperty('margin-top', this.margin_top + 'px') - bs.setProperty('margin-bottom', this.margin_bottom+'px') + bs.setProperty('margin-top', this.effective_margin_top + 'px') + bs.setProperty('margin-bottom', this.effective_margin_bottom+'px') bs.setProperty('margin-left', sm+'px') bs.setProperty('margin-right', sm+'px') for edge in ['left', 'right', 'top', 'bottom'] @@ -193,12 +205,12 @@ class PagedDisplay create_header_footer: (uuid) -> if this.header_template != null this.header = document.createElement('div') - this.header.setAttribute('style', "overflow:hidden; display:block; position:absolute; left:#{ this.side_margin }px; top: 0px; height: #{ this.margin_top }px; width: #{ this.col_width }px; margin: 0; padding: 0") + this.header.setAttribute('style', "overflow:hidden; display:block; position:absolute; left:#{ this.side_margin }px; top: 0px; height: #{ this.effective_margin_top }px; width: #{ this.col_width }px; margin: 0; padding: 0") this.header.setAttribute('id', 'pdf_page_header_'+uuid) document.body.appendChild(this.header) if this.footer_template != null this.footer = document.createElement('div') - this.footer.setAttribute('style', "overflow:hidden; display:block; position:absolute; left:#{ this.side_margin }px; top: #{ window.innerHeight - this.margin_bottom }px; height: #{ this.margin_bottom }px; width: #{ this.col_width }px; margin: 0; padding: 0") + this.footer.setAttribute('style', "overflow:hidden; display:block; position:absolute; left:#{ this.side_margin }px; top: #{ window.innerHeight - this.effective_margin_bottom }px; height: #{ this.effective_margin_bottom }px; width: #{ this.col_width }px; margin: 0; padding: 0") this.footer.setAttribute('id', 'pdf_page_footer_'+uuid) document.body.appendChild(this.footer) if this.header != null or this.footer != null @@ -501,8 +513,8 @@ class PagedDisplay continue deltax = Math.floor(this.page_width/25) deltay = Math.floor(window.innerHeight/25) - cury = this.margin_top - until cury >= (window.innerHeight - this.margin_bottom) + cury = this.effective_margin_top + until cury >= (window.innerHeight - this.effective_margin_bottom) curx = left + this.current_margin_side until curx >= (right - this.current_margin_side) cfi = window.cfi.at_point(curx-window.pageXOffset, cury-window.pageYOffset) diff --git a/src/calibre/gui2/viewer/config.py b/src/calibre/gui2/viewer/config.py index abf46b113e..f8544cb0da 100644 --- a/src/calibre/gui2/viewer/config.py +++ b/src/calibre/gui2/viewer/config.py @@ -35,6 +35,10 @@ def config(defaults=None): help=_("Set the maximum width that the book's text and pictures will take" " when in fullscreen mode. This allows you to read the book text" " without it becoming too wide.")) + c.add_opt('max_fs_height', default=-1, + help=_("Set the maximum height that the book's text and pictures will take" + " when in fullscreen mode. This allows you to read the book text" + " without it becoming too tall. Note that this setting only takes effect in paged mode (which is the default mode).")) c.add_opt('fit_images', default=True, help=_('Resize images larger than the viewer window to fit inside it')) c.add_opt('hyphenate', default=False, help=_('Hyphenate text')) @@ -211,6 +215,7 @@ class ConfigDialog(QDialog, Ui_Dialog): {'serif':0, 'sans':1, 'mono':2}[opts.standard_font]) self.css.setPlainText(opts.user_css) self.max_fs_width.setValue(opts.max_fs_width) + self.max_fs_height.setValue(opts.max_fs_height) pats, names = self.hyphenate_pats, self.hyphenate_names try: idx = pats.index(opts.hyphenate_default_lang) @@ -287,6 +292,10 @@ class ConfigDialog(QDialog, Ui_Dialog): c.set('remember_window_size', self.opt_remember_window_size.isChecked()) c.set('fit_images', self.opt_fit_images.isChecked()) c.set('max_fs_width', int(self.max_fs_width.value())) + max_fs_height = self.max_fs_height.value() + if max_fs_height <= self.max_fs_height.minimum(): + max_fs_height = -1 + c.set('max_fs_height', max_fs_height) c.set('hyphenate', self.hyphenate.isChecked()) c.set('remember_current_page', self.opt_remember_current_page.isChecked()) c.set('wheel_flips_pages', self.opt_wheel_flips_pages.isChecked()) diff --git a/src/calibre/gui2/viewer/config.ui b/src/calibre/gui2/viewer/config.ui index dd7019a157..9900ba45d3 100644 --- a/src/calibre/gui2/viewer/config.ui +++ b/src/calibre/gui2/viewer/config.ui @@ -60,7 +60,7 @@ QToolBox::tab:hover { }</string> </property> <property name="currentIndex"> - <number>0</number> + <number>2</number> </property> <widget class="QWidget" name="page"> <property name="geometry"> @@ -404,41 +404,67 @@ QToolBox::tab:hover { </property> </widget> </item> - <item row="1" column="0"> - <widget class="QCheckBox" name="opt_fullscreen_clock"> - <property name="text"> - <string>Show &clock in full screen mode</string> - </property> - </widget> - </item> - <item row="5" column="0" colspan="2"> + <item row="6" column="0" colspan="2"> <widget class="QCheckBox" name="opt_fullscreen_pos"> <property name="text"> <string>Show reading &position in full screen mode</string> </property> </widget> </item> - <item row="4" column="0" colspan="2"> + <item row="5" column="0" colspan="2"> <widget class="QCheckBox" name="opt_fullscreen_scrollbar"> <property name="text"> <string>Show &scrollbar in full screen mode</string> </property> </widget> </item> - <item row="3" column="0" colspan="2"> + <item row="4" column="0" colspan="2"> <widget class="QCheckBox" name="opt_start_in_fullscreen"> <property name="text"> <string>&Start viewer in full screen mode</string> </property> </widget> </item> - <item row="2" column="0" colspan="2"> + <item row="3" column="0" colspan="2"> <widget class="QCheckBox" name="opt_show_fullscreen_help"> <property name="text"> <string>Show &help message when starting full screen mode</string> </property> </widget> </item> + <item row="1" column="0"> + <widget class="QLabel" name="label_24"> + <property name="text"> + <string>Maximum text height in fullscreen (paged mode):</string> + </property> + </widget> + </item> + <item row="2" column="0" colspan="2"> + <widget class="QCheckBox" name="opt_fullscreen_clock"> + <property name="text"> + <string>Show &clock in full screen mode</string> + </property> + </widget> + </item> + <item row="1" column="1"> + <widget class="QSpinBox" name="max_fs_height"> + <property name="specialValueText"> + <string>Disabled</string> + </property> + <property name="suffix"> + <string> px</string> + </property> + <property name="minimum"> + <number>100</number> + </property> + <property name="maximum"> + <number>10000</number> + </property> + <property name="singleStep"> + <number>25</number> + </property> + </widget> + </item> </layout> </widget> <widget class="QWidget" name="page_6"> diff --git a/src/calibre/gui2/viewer/documentview.py b/src/calibre/gui2/viewer/documentview.py index 0691a9deb8..109ea85436 100644 --- a/src/calibre/gui2/viewer/documentview.py +++ b/src/calibre/gui2/viewer/documentview.py @@ -160,6 +160,7 @@ class Document(QWebPage): # {{{ screen_width = QApplication.desktop().screenGeometry().width() # Leave some space for the scrollbar and some border self.max_fs_width = min(opts.max_fs_width, screen_width-50) + self.max_fs_height = opts.max_fs_height self.fullscreen_clock = opts.fullscreen_clock self.fullscreen_scrollbar = opts.fullscreen_scrollbar self.fullscreen_pos = opts.fullscreen_pos @@ -310,7 +311,7 @@ class Document(QWebPage): # {{{ def switch_to_fullscreen_mode(self): self.in_fullscreen_mode = True - self.javascript('full_screen.on(%d, %s)'%(self.max_fs_width, + self.javascript('full_screen.on(%d, %d, %s)'%(self.max_fs_width, self.max_fs_height, 'true' if self.in_paged_mode else 'false')) def switch_to_window_mode(self): From 777a43308b45a340db5207d85f5c958aecacda99 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Sun, 9 Mar 2014 09:26:54 +0530 Subject: [PATCH 018/122] E-book viewer: Fix pressing the Esc key to leave full screen mode not changing the state of the full screen button --- src/calibre/gui2/viewer/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/gui2/viewer/main.py b/src/calibre/gui2/viewer/main.py index 8e1b6163b2..494683d88b 100644 --- a/src/calibre/gui2/viewer/main.py +++ b/src/calibre/gui2/viewer/main.py @@ -1119,7 +1119,7 @@ class EbookViewer(MainWindow, Ui_EbookViewer): event.accept() return if self.isFullScreen(): - self.toggle_fullscreen() + self.action_full_screen.trigger() event.accept() return try: From 5df98073089343de95b5ab24e18dfda19b992474 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Sun, 9 Mar 2014 09:31:26 +0530 Subject: [PATCH 019/122] Fix leaving full screen mode not removing max height restriction --- resources/compiled_coffeescript.zip | Bin 81090 -> 81140 bytes .../ebooks/oeb/display/full_screen.coffee | 1 + 2 files changed, 1 insertion(+) diff --git a/resources/compiled_coffeescript.zip b/resources/compiled_coffeescript.zip index fa675be5040264752ec6a0bfc79bcaaec8c7e59c..b8b7c199d5110342dcf9090e17a2e97f15a35b43 100644 GIT binary patch delta 225 zcmX@~ljX}#mJMRrLYKTVT~w~z{3F4|zyQLE4AU=~Gm37u&2D6!ydqC^^6yz(lg(HK zCZ8z~n=CY2c=GnQf}2?jV~_En>YQF+&KS>_JpF<>qc_vyZm=QK-&-?sO}DXN)M9Fz z0THRSU^HUl|2}=A1*1DtCF}GFmW(3P`79ZYneOpHICuCMd8emXGHNldo8E89sL%Lq z`hH8sDvcxyi<A^IqeN57)HI7Uv$WJSle8oQa}!I8WV1vIV*|@HQ;Vb&Ba`ikR*Z>E E09SZQ`v3p{ delta 220 zcmezJljYD)mJMRrlPAnz;Z4@u{oolV0|N*vPF^@eY_n~4BkN?_Z|sxpS=lH5EfAZ0 zyFh3<r#PeFX7j?>V|?bGnJx%T(-)aD#xog~POdLin67NWsKr<~-Oqy2iz#FVP)Kz8 zdJ9G)rn}!j?8y$_Ii^_xRk%;rwq*2RUc<*Y{Wl+@!gR1;&GdtojQWg6rhl|#tkN(r zF-}ZOHZx4MFi*8eGByKJ$%#ppW@d@TDJiMOM#dIthL))Y=F<;aGm3*O;n=>(iqVM) E0BMg(r2qf` diff --git a/src/calibre/ebooks/oeb/display/full_screen.coffee b/src/calibre/ebooks/oeb/display/full_screen.coffee index 7c5adc3f3c..2e1ee204c7 100644 --- a/src/calibre/ebooks/oeb/display/full_screen.coffee +++ b/src/calibre/ebooks/oeb/display/full_screen.coffee @@ -40,6 +40,7 @@ class FullScreen window.removeEventListener('click', this.handle_click, false) if in_paged_mode window.paged_display.max_col_width = -1 + window.paged_display.max_col_height = -1 else s = document.body.style s.maxWidth = 'none' From b4b1c021f712bb6467968dc375f1a7404f7928f6 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Sun, 9 Mar 2014 10:16:21 +0530 Subject: [PATCH 020/122] ... --- recipes/american_thinker.recipe | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/recipes/american_thinker.recipe b/recipes/american_thinker.recipe index 6ecca8549c..663753264b 100644 --- a/recipes/american_thinker.recipe +++ b/recipes/american_thinker.recipe @@ -3,7 +3,15 @@ __copyright__ = '2010, Walt Anthony <workshop.northpole at gmail.com>' ''' www.americanthinker.com ''' +import html5lib from calibre.web.feeds.news import BasicNewsRecipe +from calibre.utils.cleantext import clean_xml_chars +from lxml import etree + +def CSSSelect(expr): + from cssselect import HTMLTranslator + from lxml.etree import XPath + return XPath(HTMLTranslator().css_to_xpath(expr)) class AmericanThinker(BasicNewsRecipe): title = u'American Thinker' @@ -18,7 +26,7 @@ class AmericanThinker(BasicNewsRecipe): ignore_duplicate_articles = {'title', 'url'} remove_javascript = True - no_stylesheets = True + remove_tags_before = dict(name='h1') conversion_options = { 'comment' : description @@ -27,7 +35,14 @@ class AmericanThinker(BasicNewsRecipe): , 'language' : language , 'linearize_tables' : True } - auto_claenup = True + + def preprocess_raw_html(self, raw, url): + root = html5lib.parse( + clean_xml_chars(raw), treebuilder='lxml', + namespaceHTMLElements=False) + for x in CSSSelect('.article_body.bottom')(root): + x.getparent().remove(x) + return etree.tostring(root, encoding=unicode) feeds = [(u'http://feeds.feedburner.com/americanthinker'), (u'http://feeds.feedburner.com/AmericanThinkerBlog') From b4e2b9e93feee477d6d1bd3ef66f5ce08ae4a768 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Sun, 9 Mar 2014 14:42:31 +0530 Subject: [PATCH 021/122] UI for the subsequence matcher --- src/calibre/gui2/tweak_book/widgets.py | 217 ++++++++++++++++++++++++- src/calibre/utils/matcher.py | 12 +- 2 files changed, 223 insertions(+), 6 deletions(-) diff --git a/src/calibre/gui2/tweak_book/widgets.py b/src/calibre/gui2/tweak_book/widgets.py index c0344b178f..f9bde08f87 100644 --- a/src/calibre/gui2/tweak_book/widgets.py +++ b/src/calibre/gui2/tweak_book/widgets.py @@ -6,10 +6,15 @@ from __future__ import (unicode_literals, division, absolute_import, __license__ = 'GPL v3' __copyright__ = '2014, Kovid Goyal <kovid at kovidgoyal.net>' +import os +from itertools import izip + from PyQt4.Qt import ( QDialog, QDialogButtonBox, QGridLayout, QLabel, QLineEdit, QVBoxLayout, - QFormLayout, QHBoxLayout, QToolButton, QIcon, QApplication, Qt) + QFormLayout, QHBoxLayout, QToolButton, QIcon, QApplication, Qt, QWidget, + QPoint, QSizePolicy, QPainter, QStaticText, pyqtSignal) +from calibre import prepare_string_for_xml from calibre.gui2 import error_dialog, choose_files, choose_save_file from calibre.gui2.tweak_book import tprefs @@ -222,8 +227,212 @@ class ImportForeign(Dialog): # {{{ return src, dest # }}} +# Quick Open {{{ + +class Results(QWidget): + + EMPH = "color:magenta; font-weight:bold" + MARGIN = 4 + + item_selected = pyqtSignal() + + def __init__(self, parent=None): + QWidget.__init__(self, parent=parent) + + self.setSizePolicy(QSizePolicy.Expanding, QSizePolicy.Expanding) + self.results = () + self.current_result = -1 + self.max_result = -1 + self.mouse_hover_result = -1 + self.setMouseTracking(True) + self.setFocusPolicy(Qt.NoFocus) + + def item_from_y(self, y): + if not self.results: + return + delta = self.results[0][0].size().height() + self.MARGIN + maxy = self.height() + pos = 0 + for i, r in enumerate(self.results): + bottom = pos + delta + if pos <= y < bottom: + return i + break + pos = bottom + if pos > min(y, maxy): + break + return -1 + + def mouseMoveEvent(self, ev): + y = ev.pos().y() + prev = self.mouse_hover_result + self.mouse_hover_result = self.item_from_y(y) + if prev != self.mouse_hover_result: + self.update() + + def mousePressEvent(self, ev): + if ev.button() == 1: + i = self.item_from_y(ev.pos().y()) + if i != -1: + ev.accept() + self.current_result = i + self.update() + self.item_selected.emit() + return + return QWidget.mousePressEvent(self, ev) + + def change_current(self, delta=1): + if not self.results: + return + nc = self.current_result + delta + if 0 <= nc <= self.max_result: + self.current_result = nc + self.update() + + def __call__(self, results): + if results: + self.current_result = 0 + prefixes = [QStaticText('<b>%s</b>' % os.path.basename(x)) for x in results] + self.maxwidth = max([x.size().width() for x in prefixes]) + divider = QStaticText('\xa0→ \xa0') + divider.setTextFormat(Qt.PlainText) + self.results = tuple((prefix, divider, self.make_text(text, positions), text) + for prefix, (text, positions) in izip(prefixes, results.iteritems())) + else: + self.results = () + self.current_result = -1 + self.max_result = min(10, len(self.results) - 1) + self.mouse_hover_result = -1 + self.update() + + def make_text(self, text, positions): + positions = sorted(set(positions) - {-1}, reverse=True) + text = prepare_string_for_xml(text) + for p in positions: + text = '%s<span style="%s">%s</span>%s' % (text[:p], self.EMPH, text[p], text[p+1:]) + text = QStaticText(text) + text.setTextFormat(Qt.RichText) + return text + + def paintEvent(self, ev): + offset = QPoint(0, 0) + p = QPainter(self) + p.setClipRect(ev.rect()) + bottom = self.rect().bottom() + + if self.results: + for i, (prefix, divider, full, text) in enumerate(self.results): + size = prefix.size() + if offset.y() + size.height() > bottom: + break + self.max_result = i + offset.setX(0) + if i in (self.current_result, self.mouse_hover_result): + p.save() + if i != self.current_result: + p.setPen(Qt.DotLine) + p.drawLine(offset, QPoint(self.width(), offset.y())) + p.restore() + offset.setY(offset.y() + self.MARGIN // 2) + p.drawStaticText(offset, prefix) + offset.setX(self.maxwidth + 5) + p.drawStaticText(offset, divider) + offset.setX(offset.x() + divider.size().width()) + p.drawStaticText(offset, full) + offset.setY(offset.y() + size.height() + self.MARGIN // 2) + if i in (self.current_result, self.mouse_hover_result): + offset.setX(0) + p.save() + if i != self.current_result: + p.setPen(Qt.DotLine) + p.drawLine(offset, QPoint(self.width(), offset.y())) + p.restore() + offset.setY(offset.y()) + else: + p.drawText(self.rect(), Qt.AlignCenter, _('No results found')) + + p.end() + + @property + def selected_result(self): + try: + return self.results[self.current_result][-1] + except IndexError: + pass + +class QuickOpen(Dialog): + + def __init__(self, items, parent=None): + from calibre.utils.matcher import Matcher + self.matcher = Matcher(items) + self.matches = () + self.selected_result = None + Dialog.__init__(self, _('Choose file to edit'), 'quick-open', parent=parent) + + def sizeHint(self): + ans = Dialog.sizeHint(self) + ans.setWidth(800) + ans.setHeight(max(600, ans.height())) + return ans + + def setup_ui(self): + self.l = l = QVBoxLayout(self) + self.setLayout(l) + + self.text = t = QLineEdit(self) + t.textEdited.connect(self.update_matches) + l.addWidget(t, alignment=Qt.AlignTop) + + example = '<pre>{0}i{1}mages/{0}c{1}hapter1/{0}s{1}cene{0}3{1}.jpg</pre>'.format( + '<span style="%s">' % Results.EMPH, '</span>') + chars = '<pre style="%s">ics3</pre>' % Results.EMPH + + self.help_label = hl = QLabel(_( + '''<p>Quickly choose a file by typing in just a few characters from the file name into the field above. + For example, if want to choose the file: + {example} + Simply type in the characters: + {chars} + and press Enter.''').format(example=example, chars=chars)) + hl.setMargin(50), hl.setAlignment(Qt.AlignTop | Qt.AlignHCenter) + l.addWidget(hl) + self.results = Results(self) + self.results.setVisible(False) + self.results.item_selected.connect(self.accept) + l.addWidget(self.results) + + l.addWidget(self.bb, alignment=Qt.AlignBottom) + + def update_matches(self, text): + text = unicode(text).strip() + self.help_label.setVisible(False) + self.results.setVisible(True) + matches = self.matcher(text) + self.results(matches) + self.matches = tuple(matches) + + def keyPressEvent(self, ev): + if ev.key() in (Qt.Key_Up, Qt.Key_Down): + ev.accept() + self.results.change_current(delta=-1 if ev.key() == Qt.Key_Up else 1) + return + return Dialog.keyPressEvent(self, ev) + + def accept(self): + self.selected_result = self.results.selected_result + return Dialog.accept(self) + + @classmethod + def test(cls): + import os + from calibre.utils.matcher import get_items_from_dir + items = get_items_from_dir(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), lambda x:not x.endswith('.pyc')) + d = cls(items) + d.exec_() + print (d.selected_result) + +# }}} + if __name__ == '__main__': app = QApplication([]) - d = ImportForeign() - d.exec_() - print (d.data) + QuickOpen.test() diff --git a/src/calibre/utils/matcher.py b/src/calibre/utils/matcher.py index aa501fe4ff..7ee20a9501 100644 --- a/src/calibre/utils/matcher.py +++ b/src/calibre/utils/matcher.py @@ -26,6 +26,9 @@ DEFAULT_LEVEL1 = '/' DEFAULT_LEVEL2 = '-_ 0123456789' DEFAULT_LEVEL3 = '.' +class PluginFailed(RuntimeError): + pass + class Worker(Thread): daemon = True @@ -66,6 +69,11 @@ def split(tasks, pool_size): ans.append(section) return ans +def default_scorer(*args, **kwargs): + try: + return CScorer(*args, **kwargs) + except PluginFailed: + return PyScorer(*args, **kwargs) class Matcher(object): @@ -80,7 +88,7 @@ class Matcher(object): self.items = items = tuple(items) tasks = split(items, len(workers)) self.task_maps = [{j:i for j, (i, _) in enumerate(task)} for task in tasks] - scorer = scorer or CScorer + scorer = scorer or default_scorer self.scorers = [scorer(tuple(map(itemgetter(1), task_items))) for task_items in tasks] self.sort_keys = None @@ -201,7 +209,7 @@ class CScorer(object): speedup, err = plugins['matcher'] if speedup is None: - raise RuntimeError('Failed to load the matcher plugin with error: %s' % err) + raise PluginFailed('Failed to load the matcher plugin with error: %s' % err) self.m = speedup.Matcher(items, primary_collator().capsule, unicode(level1), unicode(level2), unicode(level3)) def __call__(self, query): From bdbc6ccfaa69a0edbf2317d5d9de879c595361e6 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Sun, 9 Mar 2014 16:44:50 +0530 Subject: [PATCH 022/122] Handle positions when matching on non BMP chars on narrow python builds correctly --- src/calibre/gui2/tweak_book/widgets.py | 5 +++-- src/calibre/utils/matcher.c | 8 ++++++++ src/calibre/utils/matcher.py | 26 +++++++++++++------------- 3 files changed, 24 insertions(+), 15 deletions(-) diff --git a/src/calibre/gui2/tweak_book/widgets.py b/src/calibre/gui2/tweak_book/widgets.py index f9bde08f87..cb63cff199 100644 --- a/src/calibre/gui2/tweak_book/widgets.py +++ b/src/calibre/gui2/tweak_book/widgets.py @@ -17,6 +17,7 @@ from PyQt4.Qt import ( from calibre import prepare_string_for_xml from calibre.gui2 import error_dialog, choose_files, choose_save_file from calibre.gui2.tweak_book import tprefs +from calibre.utils.matcher import get_char, Matcher class Dialog(QDialog): @@ -309,7 +310,8 @@ class Results(QWidget): positions = sorted(set(positions) - {-1}, reverse=True) text = prepare_string_for_xml(text) for p in positions: - text = '%s<span style="%s">%s</span>%s' % (text[:p], self.EMPH, text[p], text[p+1:]) + ch = get_char(text, p) + text = '%s<span style="%s">%s</span>%s' % (text[:p], self.EMPH, ch, text[p+len(ch):]) text = QStaticText(text) text.setTextFormat(Qt.RichText) return text @@ -363,7 +365,6 @@ class Results(QWidget): class QuickOpen(Dialog): def __init__(self, items, parent=None): - from calibre.utils.matcher import Matcher self.matcher = Matcher(items) self.matches = () self.selected_result = None diff --git a/src/calibre/utils/matcher.c b/src/calibre/utils/matcher.c index 209a7e390d..c2c2210dad 100644 --- a/src/calibre/utils/matcher.c +++ b/src/calibre/utils/matcher.c @@ -155,6 +155,10 @@ static double calc_score_for_char(MatchInfo *m, UChar32 last, UChar32 current, i } static void convert_positions(int32_t *positions, int32_t *final_positions, UChar *string, int32_t char_len, int32_t byte_len, double score) { +#if PY_VERSION_HEX >= 0x03030000 +#error Not implemented for python >= 3.3 +#endif + // The positions array stores character positions as byte offsets in string, convert them into character offsets int32_t i, *end; @@ -163,7 +167,11 @@ static void convert_positions(int32_t *positions, int32_t *final_positions, UCha end = final_positions + char_len; for (i = 0; i < byte_len && final_positions < end; i++) { if (positions[i] == -1) continue; +#ifdef Py_UNICODE_WIDE *final_positions = u_countChar32(string, positions[i]); +#else + *final_positions = positions[i]; +#endif final_positions += 1; } } diff --git a/src/calibre/utils/matcher.py b/src/calibre/utils/matcher.py index 7ee20a9501..bf74d3b42d 100644 --- a/src/calibre/utils/matcher.py +++ b/src/calibre/utils/matcher.py @@ -139,6 +139,7 @@ class FilesystemMatcher(Matcher): def __init__(self, basedir, *args, **kwargs): Matcher.__init__(self, get_items_from_dir(basedir), *args, **kwargs) +# Python implementation of the scoring algorithm {{{ def calc_score_for_char(ctx, prev, current, distance): factor = 1.0 ans = ctx.max_score_per_char @@ -202,11 +203,11 @@ class PyScorer(object): self.max_score_per_char = (1.0 / len(item) + 1.0 / len(needle)) / 2.0 self.memory = {} yield process_item(self, item, needle) +# }}} class CScorer(object): def __init__(self, items, level1=DEFAULT_LEVEL1, level2=DEFAULT_LEVEL2, level3=DEFAULT_LEVEL3): - speedup, err = plugins['matcher'] if speedup is None: raise PluginFailed('Failed to load the matcher plugin with error: %s' % err) @@ -217,14 +218,6 @@ class CScorer(object): for score, pos in izip(scores, positions): yield score, pos -def test2(): - items = ['.driveinfo.calibre', 'Suspense.xls', 'p/parsed/content.opf', 'ns.html'] - for q in (PyScorer, CScorer): - print (q) - m = Matcher(items, scorer=q) - for item, positions in m('ns').iteritems(): - print ('\tns', item, positions) - def test(): items = ['mx\U0001f431nxox'] for q in (PyScorer, CScorer): @@ -237,7 +230,6 @@ def test(): print (item[p], end=' ') print () - def test_mem(): from calibre.utils.mem import gc_histogram, diff_hists m = Matcher(['a']) @@ -255,6 +247,13 @@ def test_mem(): h2 = gc_histogram() diff_hists(h1, h2) +if sys.maxunicode >= 0x10ffff: + get_char = lambda string, pos: string[pos] +else: + def get_char(string, pos): + chs = 2 if ('\ud800' <= string[pos] <= '\udbff') else 1 # UTF-16 surrogate pair in python narrow builds + return string[pos:pos+chs] + def main(basedir=None, query=None): from calibre import prints from calibre.utils.terminal import ColoredStream @@ -279,11 +278,12 @@ def main(basedir=None, query=None): while positions: pos = positions.pop(0) if pos == -1: - break + continue prints(path[p:pos], end='') + ch = get_char(path, pos) with emph: - prints(path[pos], end='') - p = pos + 1 + prints(ch, end='') + p = pos + len(ch) prints(path[p:]) query = None From b88f26adff6eaee68488bf6d78fe26e3f5141f88 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Sun, 9 Mar 2014 17:00:14 +0530 Subject: [PATCH 023/122] Refactor matcher test suite --- src/calibre/utils/matcher.py | 62 +++++++++++++++++++++--------------- 1 file changed, 36 insertions(+), 26 deletions(-) diff --git a/src/calibre/utils/matcher.py b/src/calibre/utils/matcher.py index bf74d3b42d..d4c7c8c162 100644 --- a/src/calibre/utils/matcher.py +++ b/src/calibre/utils/matcher.py @@ -219,33 +219,43 @@ class CScorer(object): yield score, pos def test(): - items = ['mx\U0001f431nxox'] - for q in (PyScorer, CScorer): - print (q) - m = Matcher(items, scorer=q) - for item, positions in m('MNO').iteritems(): - print ('\tMNO', item, positions) - if -1 not in positions: - for p in positions: - print (item[p], end=' ') - print () + import unittest -def test_mem(): - from calibre.utils.mem import gc_histogram, diff_hists - m = Matcher(['a']) - m('a') - del m - def doit(c): - m = Matcher([c+'im/one.gif', c+'im/two.gif', c+'text/one.html',]) - m('one') - import gc - gc.collect() - h1 = gc_histogram() - for i in xrange(100): - doit(str(i)) - gc.collect() - h2 = gc_histogram() - diff_hists(h1, h2) + class Test(unittest.TestCase): + + def test_mem_leaks(self): + import gc + from calibre.utils.mem import get_memory as memory + m = Matcher(['a'], scorer=CScorer) + m('a') + def doit(c): + m = Matcher([c+'im/one.gif', c+'im/two.gif', c+'text/one.html',], scorer=CScorer) + m('one') + start = memory() + for i in xrange(10): + doit(str(i)) + gc.collect() + used10 = memory() - start + start = memory() + for i in xrange(100): + doit(str(i)) + gc.collect() + used100 = memory() - start + self.assertLessEqual(used100, 2 * used10) + + def test_non_bmp(self): + raw = '_\U0001f431-' + m = Matcher([raw], scorer=CScorer) + positions = next(m(raw).itervalues()) + self.assertEqual(positions, (0, 1, (2 if sys.maxunicode >= 0x10ffff else 3))) + + class TestRunner(unittest.main): + + def createTests(self): + tl = unittest.TestLoader() + self.test = tl.loadTestsFromTestCase(Test) + + TestRunner(verbosity=4) if sys.maxunicode >= 0x10ffff: get_char = lambda string, pos: string[pos] From 62eb796b51790a32ea5b2d138cfc98eff3166324 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Sun, 9 Mar 2014 18:16:05 +0530 Subject: [PATCH 024/122] Edit book: Add a tool to easily open a file inside the book for editing by just typing a few characters from the file name. To use it press Ctrl+T in the editor or go to Edit->Quick open a file to edit' --- src/calibre/gui2/tweak_book/boss.py | 8 ++++++++ src/calibre/gui2/tweak_book/ui.py | 4 ++++ 2 files changed, 12 insertions(+) diff --git a/src/calibre/gui2/tweak_book/boss.py b/src/calibre/gui2/tweak_book/boss.py index f81d26cb1e..c20520445c 100644 --- a/src/calibre/gui2/tweak_book/boss.py +++ b/src/calibre/gui2/tweak_book/boss.py @@ -38,6 +38,7 @@ from calibre.gui2.tweak_book.editor import editor_from_syntax, syntax_from_mime from calibre.gui2.tweak_book.editor.insert_resource import get_resource_data, NewBook from calibre.gui2.tweak_book.preferences import Preferences from calibre.gui2.tweak_book.widgets import RationalizeFolders, MultiSplit, ImportForeign +from calibre.gui2.tweak_book.widgets import QuickOpen _diff_dialogs = [] @@ -1129,6 +1130,13 @@ class Boss(QObject): _('Editing files of type %s is not supported' % mime), show=True) return self.edit_file(name, syntax) + def quick_open(self): + c = current_container() + files = [name for name, mime in c.mime_map.iteritems() if c.exists(name) and syntax_from_mime(name, mime) is not None] + d = QuickOpen(files, parent=self.gui) + if d.exec_() == d.Accepted and d.selected_result is not None: + self.edit_file_requested(d.selected_result, None, c.mime_map[d.selected_result]) + # Editor basic controls {{{ def do_editor_undo(self): ed = self.gui.central.current_editor diff --git a/src/calibre/gui2/tweak_book/ui.py b/src/calibre/gui2/tweak_book/ui.py index 6915063535..935e8ba5d5 100644 --- a/src/calibre/gui2/tweak_book/ui.py +++ b/src/calibre/gui2/tweak_book/ui.py @@ -302,6 +302,8 @@ class Main(MainWindow): self.action_new_book = reg('book.png', _('Create &new, empty book'), self.boss.new_book, 'new-book', (), _('Create a new, empty book')) self.action_import_book = reg('book.png', _('&Import an HTML or DOCX file as a new book'), self.boss.import_book, 'import-book', (), _('Import an HTML or DOCX file as a new book')) + self.action_quick_edit = reg('modified.png', _('&Quick open a file to edit'), self.boss.quick_open, 'quick-open', ('Ctrl+T'), _( + 'Quickly open a file from the book to edit it')) # Editor actions group = _('Editor actions') @@ -430,6 +432,7 @@ class Main(MainWindow): f = b.addMenu(_('&File')) f.addAction(self.action_new_file) f.addAction(self.action_import_files) + f.addSeparator() f.addAction(self.action_open_book) f.addAction(self.action_new_book) f.addAction(self.action_import_book) @@ -455,6 +458,7 @@ class Main(MainWindow): e.addAction(self.action_editor_paste) e.addAction(self.action_insert_char) e.addSeparator() + e.addAction(self.action_quick_edit) e.addAction(self.action_preferences) e = b.addMenu(_('&Tools')) From 706c0baca245329f79bb0047b86e01340efafcfe Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Sun, 9 Mar 2014 18:28:46 +0530 Subject: [PATCH 025/122] Fix matcher dialog wrapping filenames --- src/calibre/gui2/tweak_book/widgets.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/calibre/gui2/tweak_book/widgets.py b/src/calibre/gui2/tweak_book/widgets.py index cb63cff199..f0021da046 100644 --- a/src/calibre/gui2/tweak_book/widgets.py +++ b/src/calibre/gui2/tweak_book/widgets.py @@ -12,7 +12,7 @@ from itertools import izip from PyQt4.Qt import ( QDialog, QDialogButtonBox, QGridLayout, QLabel, QLineEdit, QVBoxLayout, QFormLayout, QHBoxLayout, QToolButton, QIcon, QApplication, Qt, QWidget, - QPoint, QSizePolicy, QPainter, QStaticText, pyqtSignal) + QPoint, QSizePolicy, QPainter, QStaticText, pyqtSignal, QTextOption) from calibre import prepare_string_for_xml from calibre.gui2 import error_dialog, choose_files, choose_save_file @@ -247,6 +247,8 @@ class Results(QWidget): self.mouse_hover_result = -1 self.setMouseTracking(True) self.setFocusPolicy(Qt.NoFocus) + self.text_option = to = QTextOption() + to.setWrapMode(to.NoWrap) def item_from_y(self, y): if not self.results: @@ -294,6 +296,7 @@ class Results(QWidget): if results: self.current_result = 0 prefixes = [QStaticText('<b>%s</b>' % os.path.basename(x)) for x in results] + [(p.setTextFormat(Qt.RichText), p.setTextOption(self.text_option)) for p in prefixes] self.maxwidth = max([x.size().width() for x in prefixes]) divider = QStaticText('\xa0→ \xa0') divider.setTextFormat(Qt.PlainText) @@ -313,6 +316,7 @@ class Results(QWidget): ch = get_char(text, p) text = '%s<span style="%s">%s</span>%s' % (text[:p], self.EMPH, ch, text[p+len(ch):]) text = QStaticText(text) + text.setTextOption(self.text_option) text.setTextFormat(Qt.RichText) return text @@ -349,7 +353,6 @@ class Results(QWidget): p.setPen(Qt.DotLine) p.drawLine(offset, QPoint(self.width(), offset.y())) p.restore() - offset.setY(offset.y()) else: p.drawText(self.rect(), Qt.AlignCenter, _('No results found')) @@ -427,7 +430,7 @@ class QuickOpen(Dialog): def test(cls): import os from calibre.utils.matcher import get_items_from_dir - items = get_items_from_dir(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), lambda x:not x.endswith('.pyc')) + items = get_items_from_dir(os.getcwdu(), lambda x:not x.endswith('.pyc')) d = cls(items) d.exec_() print (d.selected_result) From a708b93df5b0aff5f2e398b4e2bdba65f96f883e Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Sun, 9 Mar 2014 18:45:38 +0530 Subject: [PATCH 026/122] ... --- session.vim | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/session.vim b/session.vim index a786e71451..67502c2f73 100644 --- a/session.vim +++ b/session.vim @@ -17,9 +17,9 @@ let g:syntastic_cpp_include_dirs = [ \] let g:syntastic_c_include_dirs = g:syntastic_cpp_include_dirs -set wildignore+=resources/viewer/mathjax/** -set wildignore+=build/** -set wildignore+=dist/** +set wildignore+=resources/viewer/mathjax/* +set wildignore+=build/* +set wildignore+=dist/* fun! CalibreLog() " Setup buffers to edit the calibre changelog and version info prior to From 456b297bf061fe307f530f0f4ed56970c5a67da5 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Sun, 9 Mar 2014 19:14:35 +0530 Subject: [PATCH 027/122] ... --- src/calibre/gui2/tweak_book/widgets.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/calibre/gui2/tweak_book/widgets.py b/src/calibre/gui2/tweak_book/widgets.py index f0021da046..b770ca825d 100644 --- a/src/calibre/gui2/tweak_book/widgets.py +++ b/src/calibre/gui2/tweak_book/widgets.py @@ -249,6 +249,8 @@ class Results(QWidget): self.setFocusPolicy(Qt.NoFocus) self.text_option = to = QTextOption() to.setWrapMode(to.NoWrap) + self.divider = QStaticText('\xa0→ \xa0') + self.divider.setTextFormat(Qt.PlainText) def item_from_y(self, y): if not self.results: @@ -298,9 +300,7 @@ class Results(QWidget): prefixes = [QStaticText('<b>%s</b>' % os.path.basename(x)) for x in results] [(p.setTextFormat(Qt.RichText), p.setTextOption(self.text_option)) for p in prefixes] self.maxwidth = max([x.size().width() for x in prefixes]) - divider = QStaticText('\xa0→ \xa0') - divider.setTextFormat(Qt.PlainText) - self.results = tuple((prefix, divider, self.make_text(text, positions), text) + self.results = tuple((prefix, self.make_text(text, positions), text) for prefix, (text, positions) in izip(prefixes, results.iteritems())) else: self.results = () @@ -327,7 +327,7 @@ class Results(QWidget): bottom = self.rect().bottom() if self.results: - for i, (prefix, divider, full, text) in enumerate(self.results): + for i, (prefix, full, text) in enumerate(self.results): size = prefix.size() if offset.y() + size.height() > bottom: break @@ -342,8 +342,8 @@ class Results(QWidget): offset.setY(offset.y() + self.MARGIN // 2) p.drawStaticText(offset, prefix) offset.setX(self.maxwidth + 5) - p.drawStaticText(offset, divider) - offset.setX(offset.x() + divider.size().width()) + p.drawStaticText(offset, self.divider) + offset.setX(offset.x() + self.divider.size().width()) p.drawStaticText(offset, full) offset.setY(offset.y() + size.height() + self.MARGIN // 2) if i in (self.current_result, self.mouse_hover_result): From 1ee24745ee54cb8fe92669b3be49e7d228c17971 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Mon, 10 Mar 2014 08:16:01 +0530 Subject: [PATCH 028/122] ... --- setup/linux-installer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/linux-installer.py b/setup/linux-installer.py index e70e806b54..ddfe093c8c 100644 --- a/setup/linux-installer.py +++ b/setup/linux-installer.py @@ -449,7 +449,7 @@ def match_hostname(cert, hostname): % (hostname, ', '.join(map(repr, dnsnames)))) elif len(dnsnames) == 1: # python 2.6 does not read subjectAltName, so we do the best we can - if sys.version_info[:2] == (2, 6): + if sys.version_info[:2] <= (2, 6): if dnsnames[0] == 'calibre-ebook.com': return raise CertificateError("hostname %r " From 4836d17783abbb8647576a748a165887052c5cf9 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Mon, 10 Mar 2014 08:25:40 +0530 Subject: [PATCH 029/122] Fix #1290097 ["epub" without toc crashes](https://bugs.launchpad.net/calibre/+bug/1290097) --- src/calibre/ebooks/oeb/polish/toc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/ebooks/oeb/polish/toc.py b/src/calibre/ebooks/oeb/polish/toc.py index 98d5e534fc..ac87b7ae66 100644 --- a/src/calibre/ebooks/oeb/polish/toc.py +++ b/src/calibre/ebooks/oeb/polish/toc.py @@ -182,7 +182,7 @@ def find_existing_toc(container): def get_toc(container, verify_destinations=True): toc = find_existing_toc(container) - if toc is None: + if toc is None or not container.has_name(toc): ans = TOC() ans.lang = ans.uid = None return ans From 87fedb8a6d9faf1ba510b40ed1ffc97ea3d404cb Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Mon, 10 Mar 2014 11:53:50 +0530 Subject: [PATCH 030/122] I love linux distros --- setup/linux-installer.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/setup/linux-installer.py b/setup/linux-installer.py index ddfe093c8c..a51878328c 100644 --- a/setup/linux-installer.py +++ b/setup/linux-installer.py @@ -448,10 +448,16 @@ def match_hostname(cert, hostname): "doesn't match either of %s" % (hostname, ', '.join(map(repr, dnsnames)))) elif len(dnsnames) == 1: - # python 2.6 does not read subjectAltName, so we do the best we can - if sys.version_info[:2] <= (2, 6): - if dnsnames[0] == 'calibre-ebook.com': - return + # python 2.7.2 does not read subject alt names thanks to this + # bug: http://bugs.python.org/issue13034 + # And the utter lunacy that is the linux landscape could have + # any old version of python whatsoever with or without a hot fix for + # this bug. Not to mention that python 2.6 may or may not + # read alt names depending on its patchlevel. So we just bail on full + # verification if the python version is less than 2.7.3. + # Linux distros are one enormous, honking disaster. + if sys.version_info[:3] < (2, 7, 3) and dnsnames[0] == 'calibre-ebook.com': + return raise CertificateError("hostname %r " "doesn't match %r" % (hostname, dnsnames[0])) From 38f00298a6aeb0928bbd5dc65b2660661778ac21 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Mon, 10 Mar 2014 12:41:42 +0530 Subject: [PATCH 031/122] Possible fix for crash on OS X Mavericks when adding duplicates --- src/calibre/gui2/add.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/gui2/add.py b/src/calibre/gui2/add.py index 554b8d4a36..d849846e02 100644 --- a/src/calibre/gui2/add.py +++ b/src/calibre/gui2/add.py @@ -424,7 +424,7 @@ class Adder(QObject): # {{{ return self.duplicates_processed() self.pd.hide() from calibre.gui2.dialogs.duplicates import DuplicatesQuestion - d = DuplicatesQuestion(self.db, duplicates, self._parent) + self.__d_q = d = DuplicatesQuestion(self.db, duplicates, self._parent) duplicates = tuple(d.duplicates) if duplicates: pd = QProgressDialog(_('Adding duplicates...'), '', 0, len(duplicates), From a479db415cb4e20062fda50fd2a39e98fbc3fe6d Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Mon, 10 Mar 2014 13:44:06 +0530 Subject: [PATCH 032/122] ... --- setup/installer/windows/notes.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/setup/installer/windows/notes.rst b/setup/installer/windows/notes.rst index 7d4f315dce..88ff06abb2 100644 --- a/setup/installer/windows/notes.rst +++ b/setup/installer/windows/notes.rst @@ -60,6 +60,9 @@ to login as the normal user account with ssh. To do this, follow these steps: http://pcsupport.about.com/od/windows7/ht/auto-logon-windows-7.htm or http://pcsupport.about.com/od/windowsxp/ht/auto-logon-xp.htm to allow the machine to bootup without having to enter the password + + * The following steps must all be run in an administrator cygwin shell + * First clean out any existing cygwin ssh setup with:: net stop sshd cygrunsrv -R sshd @@ -70,7 +73,7 @@ to login as the normal user account with ssh. To do this, follow these steps: mkpasswd -cl > /etc/passwd mkgroup --local > /etc/group * Assign the necessary rights to the normal user account (administrator - command prompt needed):: + cygwin command prompt needed - editrights is available in \cygwin\bin):: editrights.exe -a SeAssignPrimaryTokenPrivilege -u kovid editrights.exe -a SeCreateTokenPrivilege -u kovid editrights.exe -a SeTcbPrivilege -u kovid From 0415fb19dd7ef2293f5a428e36284a1a93a8682c Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Mon, 10 Mar 2014 14:46:22 +0530 Subject: [PATCH 033/122] Remove ununsed DJVU input GUI widget --- src/calibre/gui2/convert/djvu_input.py | 24 ---------------------- src/calibre/gui2/convert/djvu_input.ui | 28 -------------------------- 2 files changed, 52 deletions(-) delete mode 100644 src/calibre/gui2/convert/djvu_input.py delete mode 100644 src/calibre/gui2/convert/djvu_input.ui diff --git a/src/calibre/gui2/convert/djvu_input.py b/src/calibre/gui2/convert/djvu_input.py deleted file mode 100644 index 6e52487d81..0000000000 --- a/src/calibre/gui2/convert/djvu_input.py +++ /dev/null @@ -1,24 +0,0 @@ -# coding: utf-8 -from __future__ import (unicode_literals, division, absolute_import, - print_function) - -__license__ = 'GPL v3' -__copyright__ = '2011, Anthon van der Neut <A.van.der.Neut@ruamel.eu>' - - -from calibre.gui2.convert.djvu_input_ui import Ui_Form -from calibre.gui2.convert import Widget - -class PluginWidget(Widget, Ui_Form): - - TITLE = _('DJVU Input') - HELP = _('Options specific to')+' DJVU '+_('input') - COMMIT_NAME = 'djvu_input' - ICON = I('mimetypes/djvu.png') - - def __init__(self, parent, get_option, get_help, db=None, book_id=None): - Widget.__init__(self, parent, - ['use_djvutxt', ]) - self.db, self.book_id = db, book_id - self.initialize_options(get_option, get_help, db, book_id) - diff --git a/src/calibre/gui2/convert/djvu_input.ui b/src/calibre/gui2/convert/djvu_input.ui deleted file mode 100644 index ad027f645e..0000000000 --- a/src/calibre/gui2/convert/djvu_input.ui +++ /dev/null @@ -1,28 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<ui version="4.0"> - <class>Form</class> - <widget class="QWidget" name="Form"> - <property name="geometry"> - <rect> - <x>0</x> - <y>0</y> - <width>400</width> - <height>300</height> - </rect> - </property> - <property name="windowTitle"> - <string>Form</string> - </property> - <layout class="QVBoxLayout" name="verticalLayout"> - <item> - <widget class="QCheckBox" name="opt_use_djvutxt"> - <property name="text"> - <string>Use &djvutxt, if available, for faster processing</string> - </property> - </widget> - </item> - </layout> - </widget> - <resources/> - <connections/> -</ui> From bf003b8d1f281b2c2eb0a06f5fcba4403d132139 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Mon, 10 Mar 2014 15:09:30 +0530 Subject: [PATCH 034/122] Add getafix to push destinations --- setup/installer/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup/installer/__init__.py b/setup/installer/__init__.py index 960422f750..3102a1ba80 100644 --- a/setup/installer/__init__.py +++ b/setup/installer/__init__.py @@ -14,7 +14,7 @@ from setup.build_environment import HOST, PROJECT BASE_RSYNC = ['rsync', '-avz', '--delete', '--force'] EXCLUDES = [] for x in [ - 'src/calibre/plugins', 'src/calibre/manual', 'src/calibre/trac', + 'src/calibre/plugins', 'manual', '.bzr', '.git', '.build', '.svn', 'build', 'dist', 'imgsrc', '*.pyc', '*.pyo', '*.swp', '*.swo', 'format_docs']: EXCLUDES.extend(['--exclude', x]) @@ -82,6 +82,7 @@ class Push(Command): r'kovid@win7:/cygdrive/c/Users/kovid/calibre':'Windows 7', 'kovid@win7-x64:calibre-src':'win7-x64', 'kovid@tiny:calibre':None, + 'kovid@getafix:calibre-src':None, }.iteritems(): threads[vmname or host] = thread = Thread(target=push, args=(host, vmname, available)) thread.start() From 0dc16efa89841c89d5d40b0c7d60db1231764585 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Mon, 10 Mar 2014 16:48:19 +0530 Subject: [PATCH 035/122] Update Fleshbot --- recipes/fleshbot.recipe | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/recipes/fleshbot.recipe b/recipes/fleshbot.recipe index 0059d8855d..3754c80ceb 100644 --- a/recipes/fleshbot.recipe +++ b/recipes/fleshbot.recipe @@ -20,10 +20,10 @@ class Fleshbot(BasicNewsRecipe): language = 'en' masthead_url = 'http://fbassets.s3.amazonaws.com/images/uploads/2012/01/fleshbot-logo.png' extra_css = ''' - body{font-family: "Lucida Grande",Helvetica,Arial,sans-serif} - img{margin-bottom: 1em} - h1{font-family :Arial,Helvetica,sans-serif; font-size:large} - ''' + body{font-family: "Lucida Grande",Helvetica,Arial,sans-serif} + img{margin-bottom: 1em} + h1{font-family :Arial,Helvetica,sans-serif; font-size:large} + ''' conversion_options = { 'comment' : description , 'tags' : category @@ -31,13 +31,12 @@ class Fleshbot(BasicNewsRecipe): , 'language' : language } - feeds = [(u'Articles', u'http://www.fleshbot.com/feed')] + feeds = [(u'Articles', u'http://fleshbot.com/?feed=rss2')] remove_tags = [ {'class': 'feedflare'}, ] - def preprocess_html(self, soup): return self.adeify_images(soup) From 127282840a38f9e9bff57fc9f27a2b352a411023 Mon Sep 17 00:00:00 2001 From: David Forrester <davidfor@internode.on.net> Date: Mon, 10 Mar 2014 22:41:37 +1100 Subject: [PATCH 036/122] Update dbversion for older Kobo devices Should have set the supported_dbversion to 98 in the KOBO driver to match the KOBOTOUCH driver. --- src/calibre/devices/kobo/driver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/devices/kobo/driver.py b/src/calibre/devices/kobo/driver.py index 5eaf0f3563..f57435a423 100644 --- a/src/calibre/devices/kobo/driver.py +++ b/src/calibre/devices/kobo/driver.py @@ -68,7 +68,7 @@ class KOBO(USBMS): dbversion = 0 fwversion = 0 - supported_dbversion = 95 + supported_dbversion = 98 has_kepubs = False supported_platforms = ['windows', 'osx', 'linux'] From 20ce36517c4f9040450772c6c462d1ceee380d0d Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Tue, 11 Mar 2014 09:44:21 +0530 Subject: [PATCH 037/122] Fix cursor becoming invisible when completion popup is opened --- src/calibre/gui2/complete2.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/calibre/gui2/complete2.py b/src/calibre/gui2/complete2.py index 7c98e6477c..d19e9aaff4 100644 --- a/src/calibre/gui2/complete2.py +++ b/src/calibre/gui2/complete2.py @@ -257,6 +257,7 @@ class LineEdit(QLineEdit, LineEditECM): QLineEdit.__init__(self, parent) self.sep = ',' + self.eat_focus_out = False self.space_before_sep = False self.add_separator = True self.original_cursor_pos = None @@ -302,9 +303,15 @@ class LineEdit(QLineEdit, LineEditECM): if not self.mcompleter.model().current_items: self.mcompleter.hide() return + self.eat_focus_out = True self.mcompleter.popup(select_first=select_first) self.mcompleter.scroll_to(orig) + def focusOutEvent(self, ev): + if not self.eat_focus_out: + QLineEdit.focusOutEvent(self, ev) + self.eat_focus_out = False + def relayout(self): self.mcompleter.popup() From 737fde279d73813401e40b58d6e873358f59d321 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Wed, 12 Mar 2014 09:04:54 +0530 Subject: [PATCH 038/122] Fix #1291085 [Add 'tooltip'(title) to Author in book details](https://bugs.launchpad.net/calibre/+bug/1291085) --- src/calibre/ebooks/metadata/book/render.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/ebooks/metadata/book/render.py b/src/calibre/ebooks/metadata/book/render.py index ab9c8ae454..37a94f9e4a 100644 --- a/src/calibre/ebooks/metadata/book/render.py +++ b/src/calibre/ebooks/metadata/book/render.py @@ -148,7 +148,7 @@ def mi_to_html(mi, field_list=None, default_author_link=None, use_roman_numbers= default_author_link, vals, '', vals) aut = p(aut) if link: - authors.append(u'<a calibre-data="authors" href="%s">%s</a>'%(a(link), aut)) + authors.append(u'<a calibre-data="authors" title="%s" href="%s">%s</a>'%(a(link), a(link), aut)) else: authors.append(aut) ans.append((field, row % (name, u' & '.join(authors)))) From 0e3362fd4bc60f40f20c4daadec8cf330dc89352 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Wed, 12 Mar 2014 09:10:40 +0530 Subject: [PATCH 039/122] Library backup: Avoid infinite retries if converting metadata to backup OPF for a book fails. Simply fail to backup the metadata for that book. Fixes #1291142 [Memory Error/ Failed to convert to opf for id:](https://bugs.launchpad.net/calibre/+bug/1291142) --- src/calibre/db/backup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/calibre/db/backup.py b/src/calibre/db/backup.py index f0f2b07a54..9dfda156d4 100644 --- a/src/calibre/db/backup.py +++ b/src/calibre/db/backup.py @@ -93,6 +93,7 @@ class MetadataBackup(Thread): except: prints('Failed to convert to opf for id:', book_id) traceback.print_exc() + self.db.clear_dirtied(book_id, sequence) return self.wait(self.scheduling_interval) From 13b47f4a29935c77b5773f4a1d78e1683a8389ab Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Wed, 12 Mar 2014 09:59:42 +0530 Subject: [PATCH 040/122] Edit book: Allow disabling the completion popups for the search and replace fields. Right click on the search/replace field to enable/disable the completion popup --- src/calibre/gui2/complete2.py | 10 ++++++++++ src/calibre/gui2/tweak_book/__init__.py | 1 + src/calibre/gui2/tweak_book/search.py | 21 +++++++++++++++++++-- 3 files changed, 30 insertions(+), 2 deletions(-) diff --git a/src/calibre/gui2/complete2.py b/src/calibre/gui2/complete2.py index d19e9aaff4..fe794902ea 100644 --- a/src/calibre/gui2/complete2.py +++ b/src/calibre/gui2/complete2.py @@ -76,6 +76,7 @@ class Completer(QListView): # {{{ def __init__(self, completer_widget, max_visible_items=7): QListView.__init__(self) + self.disable_popup = False self.completer_widget = weakref.ref(completer_widget) self.setWindowFlags(Qt.Popup) self.max_visible_items = max_visible_items @@ -132,6 +133,8 @@ class Completer(QListView): # {{{ self.setCurrentIndex(index) def popup(self, select_first=True): + if self.disable_popup: + return p = self m = p.model() widget = self.completer_widget() @@ -293,6 +296,13 @@ class LineEdit(QLineEdit, LineEditECM): self.mcompleter.model().set_items(items) return property(fget=fget, fset=fset) + @dynamic_property + def disable_popup(self): + def fget(self): + return self.mcompleter.disable_popup + def fset(self, val): + self.mcompleter.disable_popup = bool(val) + return property(fget=fget, fset=fset) # }}} def complete(self, show_all=False, select_first=True): diff --git a/src/calibre/gui2/tweak_book/__init__.py b/src/calibre/gui2/tweak_book/__init__.py index 1f2f5fa1bc..2085568d6c 100644 --- a/src/calibre/gui2/tweak_book/__init__.py +++ b/src/calibre/gui2/tweak_book/__init__.py @@ -40,6 +40,7 @@ d['remove_existing_links_when_linking_sheets'] = True d['charmap_favorites'] = list(map(ord, '\xa0\u2002\u2003\u2009\xad' '‘’“”‹›«»‚„' '—–§¶†‡©®™' '→⇒•·°±−×÷¼½½¾' '…µ¢£€¿¡¨´¸ˆ˜' 'ÀÁÂÃÄÅÆÇÈÉÊË' 'ÌÍÎÏÐÑÒÓÔÕÖØ' 'ŒŠÙÚÛÜÝŸÞßàá' 'âãäåæçèéêëìí' 'îïðñòóôõöøœš' 'ùúûüýÿþªºαΩ∞')) # noqa d['folders_for_types'] = {'style':'styles', 'image':'images', 'font':'fonts', 'audio':'audio', 'video':'video'} d['pretty_print_on_open'] = False +d['disable_completion_popup_for_search'] = False del d diff --git a/src/calibre/gui2/tweak_book/search.py b/src/calibre/gui2/tweak_book/search.py index ce2446d7b3..6988aa5912 100644 --- a/src/calibre/gui2/tweak_book/search.py +++ b/src/calibre/gui2/tweak_book/search.py @@ -25,6 +25,23 @@ class PushButton(QPushButton): QPushButton.__init__(self, text, parent) self.clicked.connect(lambda : parent.search_triggered.emit(action)) +class HistoryLineEdit(HistoryLineEdit2): + + def __init__(self, parent): + HistoryLineEdit2.__init__(self, parent) + self.disable_popup = tprefs['disable_completion_popup_for_search'] + + def contextMenuEvent(self, event): + menu = self.createStandardContextMenu() + menu.addSeparator() + menu.addAction((_('Show completion based on search history') if self.disable_popup else _( + 'Hide completion based on search history')), self.toggle_popups) + menu.exec_(event.globalPos()) + + def toggle_popups(self): + self.disable_popup = not bool(self.disable_popup) + tprefs['disable_completion_popup_for_search'] = self.disable_popup + class SearchWidget(QWidget): DEFAULT_STATE = { @@ -46,7 +63,7 @@ class SearchWidget(QWidget): self.fl = fl = QLabel(_('&Find:')) fl.setAlignment(Qt.AlignRight | Qt.AlignCenter) - self.find_text = ft = HistoryLineEdit2(self) + self.find_text = ft = HistoryLineEdit(self) ft.initialize('tweak_book_find_edit') ft.returnPressed.connect(lambda : self.search_triggered.emit('find')) fl.setBuddy(ft) @@ -55,7 +72,7 @@ class SearchWidget(QWidget): self.rl = rl = QLabel(_('&Replace:')) rl.setAlignment(Qt.AlignRight | Qt.AlignCenter) - self.replace_text = rt = HistoryLineEdit2(self) + self.replace_text = rt = HistoryLineEdit(self) rt.initialize('tweak_book_replace_edit') rl.setBuddy(rt) l.addWidget(rl, 1, 0) From 240f16484068812b1b6e86853d327d1a5112b8c2 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Wed, 12 Mar 2014 12:28:38 +0530 Subject: [PATCH 041/122] Better fix for missing cursor when completion popup opens --- src/calibre/gui2/complete2.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/src/calibre/gui2/complete2.py b/src/calibre/gui2/complete2.py index fe794902ea..5260e265a5 100644 --- a/src/calibre/gui2/complete2.py +++ b/src/calibre/gui2/complete2.py @@ -260,7 +260,6 @@ class LineEdit(QLineEdit, LineEditECM): QLineEdit.__init__(self, parent) self.sep = ',' - self.eat_focus_out = False self.space_before_sep = False self.add_separator = True self.original_cursor_pos = None @@ -313,17 +312,13 @@ class LineEdit(QLineEdit, LineEditECM): if not self.mcompleter.model().current_items: self.mcompleter.hide() return - self.eat_focus_out = True self.mcompleter.popup(select_first=select_first) + self.setFocus(Qt.OtherFocusReason) self.mcompleter.scroll_to(orig) - def focusOutEvent(self, ev): - if not self.eat_focus_out: - QLineEdit.focusOutEvent(self, ev) - self.eat_focus_out = False - def relayout(self): self.mcompleter.popup() + self.setFocus(Qt.OtherFocusReason) def text_edited(self, *args): if self.no_popup: From a991c5e36d0a7569809887e36b37a2d3c4f6c4df Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Wed, 12 Mar 2014 13:53:38 +0530 Subject: [PATCH 042/122] Edit Book: Add a tool to easily insert hyperlinks (click the insert hyperlink button on the toolbar) --- src/calibre/gui2/tweak_book/boss.py | 9 +- .../gui2/tweak_book/editor/smart/html.py | 26 ++- src/calibre/gui2/tweak_book/editor/text.py | 4 + src/calibre/gui2/tweak_book/editor/widget.py | 8 + src/calibre/gui2/tweak_book/widgets.py | 216 +++++++++++++++++- src/calibre/utils/matcher.py | 4 +- 6 files changed, 253 insertions(+), 14 deletions(-) diff --git a/src/calibre/gui2/tweak_book/boss.py b/src/calibre/gui2/tweak_book/boss.py index c20520445c..de605790fb 100644 --- a/src/calibre/gui2/tweak_book/boss.py +++ b/src/calibre/gui2/tweak_book/boss.py @@ -37,8 +37,8 @@ from calibre.gui2.tweak_book.toc import TOCEditor from calibre.gui2.tweak_book.editor import editor_from_syntax, syntax_from_mime from calibre.gui2.tweak_book.editor.insert_resource import get_resource_data, NewBook from calibre.gui2.tweak_book.preferences import Preferences -from calibre.gui2.tweak_book.widgets import RationalizeFolders, MultiSplit, ImportForeign -from calibre.gui2.tweak_book.widgets import QuickOpen +from calibre.gui2.tweak_book.widgets import ( + RationalizeFolders, MultiSplit, ImportForeign, QuickOpen, InsertLink) _diff_dialogs = [] @@ -641,6 +641,11 @@ class Boss(QObject): chosen_name = chosen_image_is_external[0] href = current_container().name_to_href(chosen_name, edname) ed.insert_image(href) + elif action[0] == 'insert_hyperlink': + self.commit_all_editors_to_container() + d = InsertLink(current_container(), edname, parent=self.gui) + if d.exec_() == d.Accepted: + ed.insert_hyperlink(d.href) else: ed.action_triggered(action) diff --git a/src/calibre/gui2/tweak_book/editor/smart/html.py b/src/calibre/gui2/tweak_book/editor/smart/html.py index 40d905bcbd..bd4a5b568f 100644 --- a/src/calibre/gui2/tweak_book/editor/smart/html.py +++ b/src/calibre/gui2/tweak_book/editor/smart/html.py @@ -12,6 +12,7 @@ from . import NullSmarts from PyQt4.Qt import QTextEdit +from calibre import prepare_string_for_xml from calibre.gui2 import error_dialog get_offset = itemgetter(0) @@ -128,6 +129,20 @@ def rename_tag(cursor, opening_tag, closing_tag, new_name, insert=False): cursor.insertText(text) cursor.endEditBlock() +def ensure_not_within_tag_definition(cursor): + ''' Ensure the cursor is not inside a tag definition <>. Returns True iff the cursor was moved. ''' + block, offset = cursor.block(), cursor.positionInBlock() + b, boundary = next_tag_boundary(block, offset, forward=False) + if b is None: + return False + if boundary.is_start: + # We are inside a tag + block, boundary = next_tag_boundary(block, offset) + if block is not None: + cursor.setPosition(block.position() + boundary.offset + 1) + return True + return False + class HTMLSmarts(NullSmarts): def get_extra_selections(self, editor): @@ -180,4 +195,13 @@ class HTMLSmarts(NullSmarts): return error_dialog(editor, _('No found'), _( 'No suitable block level tag was found to rename'), show=True) - + def insert_hyperlink(self, editor, target): + c = editor.textCursor() + if c.hasSelection(): + c.insertText('') # delete any existing selected text + ensure_not_within_tag_definition(c) + c.insertText('<a href="%s">' % prepare_string_for_xml(target, True)) + p = c.position() + c.insertText('</a>') + c.setPosition(p) # ensure cursor is positioned inside the newly created tag + editor.setTextCursor(c) diff --git a/src/calibre/gui2/tweak_book/editor/text.py b/src/calibre/gui2/tweak_book/editor/text.py index fb4f39183a..cccb274c97 100644 --- a/src/calibre/gui2/tweak_book/editor/text.py +++ b/src/calibre/gui2/tweak_book/editor/text.py @@ -602,6 +602,10 @@ class TextEdit(PlainTextEdit): c.setPosition(left + len(text), c.KeepAnchor) self.setTextCursor(c) + def insert_hyperlink(self, target): + if hasattr(self.smarts, 'insert_hyperlink'): + self.smarts.insert_hyperlink(self, target) + def keyPressEvent(self, ev): if ev.key() == Qt.Key_X and ev.modifiers() == Qt.AltModifier: if self.replace_possible_unicode_sequence(): diff --git a/src/calibre/gui2/tweak_book/editor/widget.py b/src/calibre/gui2/tweak_book/editor/widget.py index a15514b28a..d4c7de956a 100644 --- a/src/calibre/gui2/tweak_book/editor/widget.py +++ b/src/calibre/gui2/tweak_book/editor/widget.py @@ -56,6 +56,9 @@ def register_text_editor_actions(reg, palette): ac = reg('view-image', _('&Insert image'), ('insert_resource', 'image'), 'insert-image', (), _('Insert an image into the text')) ac.setToolTip(_('<h3>Insert image</h3>Insert an image into the text')) + ac = reg('insert-link', _('Insert &hyperlink'), ('insert_hyperlink',), 'insert-hyperlink', (), _('Insert hyperlink')) + ac.setToolTip(_('<h3>Insert hyperlink</h3>Insert a hyperlink into the text')) + for i, name in enumerate(('h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'p')): text = ('&' + name) if name == 'p' else (name[0] + '&' + name[1]) desc = _('Convert the paragraph to <%s>') % name @@ -141,6 +144,9 @@ class Editor(QMainWindow): def insert_image(self, href): self.editor.insert_image(href) + def insert_hyperlink(self, href): + self.editor.insert_hyperlink(href) + def undo(self): self.editor.undo() @@ -195,6 +201,8 @@ class Editor(QMainWindow): b.addAction(actions['pretty-current']) if self.syntax in {'html', 'css'}: b.addAction(actions['insert-image']) + if self.syntax == 'html': + b.addAction(actions['insert-hyperlink']) if self.syntax == 'html': self.format_bar = b = self.addToolBar(_('Format text')) for x in ('bold', 'italic', 'underline', 'strikethrough', 'subscript', 'superscript', 'color', 'background-color'): diff --git a/src/calibre/gui2/tweak_book/widgets.py b/src/calibre/gui2/tweak_book/widgets.py index b770ca825d..43793e9bda 100644 --- a/src/calibre/gui2/tweak_book/widgets.py +++ b/src/calibre/gui2/tweak_book/widgets.py @@ -12,13 +12,18 @@ from itertools import izip from PyQt4.Qt import ( QDialog, QDialogButtonBox, QGridLayout, QLabel, QLineEdit, QVBoxLayout, QFormLayout, QHBoxLayout, QToolButton, QIcon, QApplication, Qt, QWidget, - QPoint, QSizePolicy, QPainter, QStaticText, pyqtSignal, QTextOption) + QPoint, QSizePolicy, QPainter, QStaticText, pyqtSignal, QTextOption, + QAbstractListModel, QModelIndex, QVariant, QStyledItemDelegate, QStyle, + QListView, QTextDocument, QSize) from calibre import prepare_string_for_xml -from calibre.gui2 import error_dialog, choose_files, choose_save_file +from calibre.gui2 import error_dialog, choose_files, choose_save_file, NONE from calibre.gui2.tweak_book import tprefs +from calibre.utils.icu import primary_sort_key from calibre.utils.matcher import get_char, Matcher +ROOT = QModelIndex() + class Dialog(QDialog): def __init__(self, title, name, parent=None): @@ -230,6 +235,15 @@ class ImportForeign(Dialog): # {{{ # Quick Open {{{ +def make_highlighted_text(emph, text, positions): + positions = sorted(set(positions) - {-1}, reverse=True) + text = prepare_string_for_xml(text) + for p in positions: + ch = get_char(text, p) + text = '%s<span style="%s">%s</span>%s' % (text[:p], emph, ch, text[p+len(ch):]) + return text + + class Results(QWidget): EMPH = "color:magenta; font-weight:bold" @@ -310,12 +324,7 @@ class Results(QWidget): self.update() def make_text(self, text, positions): - positions = sorted(set(positions) - {-1}, reverse=True) - text = prepare_string_for_xml(text) - for p in positions: - ch = get_char(text, p) - text = '%s<span style="%s">%s</span>%s' % (text[:p], self.EMPH, ch, text[p+len(ch):]) - text = QStaticText(text) + text = QStaticText(make_highlighted_text(self.EMPH, text, positions)) text.setTextOption(self.text_option) text.setTextFormat(Qt.RichText) return text @@ -411,7 +420,7 @@ class QuickOpen(Dialog): text = unicode(text).strip() self.help_label.setVisible(False) self.results.setVisible(True) - matches = self.matcher(text) + matches = self.matcher(text, limit=100) self.results(matches) self.matches = tuple(matches) @@ -437,6 +446,193 @@ class QuickOpen(Dialog): # }}} +# Filterable names list {{{ + +class NamesDelegate(QStyledItemDelegate): + + def sizeHint(self, option, index): + ans = QStyledItemDelegate.sizeHint(self, option, index) + ans.setHeight(ans.height() + 10) + return ans + + def paint(self, painter, option, index): + QStyledItemDelegate.paint(self, painter, option, index) + text, positions = index.data(Qt.UserRole).toPyObject() + self.initStyleOption(option, index) + painter.save() + painter.setFont(option.font) + p = option.palette + c = p.HighlightedText if option.state & QStyle.State_Selected else p.Text + group = (p.Active if option.state & QStyle.State_Active else p.Inactive) + c = p.color(group, c) + painter.setClipRect(option.rect) + if positions is None or -1 in positions: + painter.setPen(c) + painter.drawText(option.rect, Qt.AlignLeft | Qt.AlignVCenter | Qt.TextSingleLine, text) + else: + to = QTextOption() + to.setWrapMode(to.NoWrap) + to.setAlignment(Qt.AlignLeft | Qt.AlignVCenter) + positions = sorted(set(positions) - {-1}, reverse=True) + text = '<body>%s</body>' % make_highlighted_text(Results.EMPH, text, positions) + doc = QTextDocument() + c = 'rgb(%d, %d, %d)'%c.getRgb()[:3] + doc.setDefaultStyleSheet(' body { color: %s }'%c) + doc.setHtml(text) + doc.setDefaultFont(option.font) + doc.setDocumentMargin(0.0) + doc.setDefaultTextOption(to) + height = doc.size().height() + painter.translate(option.rect.left(), option.rect.top() + (max(0, option.rect.height() - height) // 2)) + doc.drawContents(painter) + painter.restore() + +class NamesModel(QAbstractListModel): + + filtered = pyqtSignal(object) + + def __init__(self, names, parent=None): + self.items = [] + QAbstractListModel.__init__(self, parent) + self.set_names(names) + + def set_names(self, names): + self.names = names + self.matcher = Matcher(names) + self.filter('') + + def rowCount(self, parent=ROOT): + return len(self.items) + + def data(self, index, role): + if role == Qt.UserRole: + return QVariant(self.items[index.row()]) + if role == Qt.DisplayRole: + return QVariant('\xa0' * 20) + return NONE + + def filter(self, query): + query = unicode(query or '') + if not query: + self.items = tuple((text, None) for text in self.names) + else: + self.items = tuple(self.matcher(query).iteritems()) + self.reset() + self.filtered.emit(not bool(query)) + +def create_filterable_names_list(names, filter_text=None, parent=None): + nl = QListView(parent) + nl.m = m = NamesModel(names, parent=nl) + m.filtered.connect(lambda all_items: nl.scrollTo(m.index(0))) + nl.setModel(m) + nl.d = NamesDelegate(nl) + nl.setItemDelegate(nl.d) + f = QLineEdit(parent) + f.setPlaceholderText(filter_text or '') + f.textEdited.connect(m.filter) + return nl, f + +# }}} + +# Insert Link {{{ +class InsertLink(Dialog): + + def __init__(self, container, source_name, parent=None): + self.container = container + self.source_name = source_name + Dialog.__init__(self, _('Insert Hyperlink'), 'insert-hyperlink', parent=parent) + self.anchor_cache = {} + + def sizeHint(self): + return QSize(800, 600) + + def setup_ui(self): + self.l = l = QVBoxLayout(self) + self.setLayout(l) + + self.h = h = QHBoxLayout() + l.addLayout(h) + + names = [n for n, linear in self.container.spine_names] + fn, f = create_filterable_names_list(names, filter_text=_('Filter files'), parent=self) + self.file_names, self.file_names_filter = fn, f + fn.selectionModel().selectionChanged.connect(self.selected_file_changed) + self.fnl = fnl = QVBoxLayout() + self.la1 = la = QLabel(_('Choose a &file to link to:')) + la.setBuddy(fn) + fnl.addWidget(la), fnl.addWidget(fn), fnl.addWidget(f) + h.addLayout(fnl), h.setStretch(0, 2) + + fn, f = create_filterable_names_list([], filter_text=_('Filter locations'), parent=self) + self.anchor_names, self.anchor_names_filter = fn, f + fn.selectionModel().selectionChanged.connect(self.update_target) + fn.doubleClicked.connect(self.accept, type=Qt.QueuedConnection) + self.anl = fnl = QVBoxLayout() + self.la2 = la = QLabel(_('Choose a &location (anchor) in the file:')) + la.setBuddy(fn) + fnl.addWidget(la), fnl.addWidget(fn), fnl.addWidget(f) + h.addLayout(fnl), h.setStretch(1, 1) + + self.tl = tl = QHBoxLayout() + self.la3 = la = QLabel(_('&Target:')) + tl.addWidget(la) + self.target = t = QLineEdit(self) + la.setBuddy(t) + tl.addWidget(t) + l.addLayout(tl) + + l.addWidget(self.bb) + + def selected_file_changed(self, *args): + rows = list(self.file_names.selectionModel().selectedRows()) + if not rows: + self.anchor_names.model().set_names([]) + else: + name, positions = self.file_names.model().data(rows[0], Qt.UserRole).toPyObject() + self.populate_anchors(name) + + def populate_anchors(self, name): + if name not in self.anchor_cache: + from calibre.ebooks.oeb.base import XHTML_NS + root = self.container.parsed(name) + self.anchor_cache[name] = sorted( + (set(root.xpath('//*/@id')) | set(root.xpath('//h:a/@name', namespaces={'h':XHTML_NS}))) - {''}, key=primary_sort_key) + self.anchor_names.model().set_names(self.anchor_cache[name]) + self.update_target() + + def update_target(self): + rows = list(self.file_names.selectionModel().selectedRows()) + if not rows: + return + name = self.file_names.model().data(rows[0], Qt.UserRole).toPyObject()[0] + if name == self.source_name: + href = '' + else: + href = self.container.name_to_href(name, self.source_name) + frag = '' + rows = list(self.anchor_names.selectionModel().selectedRows()) + if rows: + anchor = self.anchor_names.model().data(rows[0], Qt.UserRole).toPyObject()[0] + if anchor: + frag = '#' + anchor + href += frag + self.target.setText(href or '#') + + @property + def href(self): + return unicode(self.target.text()).strip() + + @classmethod + def test(cls): + import sys + from calibre.ebooks.oeb.polish.container import get_container + c = get_container(sys.argv[-1], tweak_mode=True) + d = cls(c, next(c.spine_names)[0]) + if d.exec_() == d.Accepted: + print (d.href) + +# }}} + if __name__ == '__main__': app = QApplication([]) - QuickOpen.test() + InsertLink.test() diff --git a/src/calibre/utils/matcher.py b/src/calibre/utils/matcher.py index d4c7c8c162..895d29082a 100644 --- a/src/calibre/utils/matcher.py +++ b/src/calibre/utils/matcher.py @@ -92,7 +92,7 @@ class Matcher(object): self.scorers = [scorer(tuple(map(itemgetter(1), task_items))) for task_items in tasks] self.sort_keys = None - def __call__(self, query): + def __call__(self, query, limit=None): query = normalize('NFC', unicode(query)) with wlock: for i, scorer in enumerate(self.scorers): @@ -119,6 +119,8 @@ class Matcher(object): raise Exception('Failed to score items: %s' % error) items = sorted(((-scores[i], item, positions[i]) for i, item in enumerate(self.items)), key=itemgetter(0)) + if limit is not None: + del items[limit:] return OrderedDict(x[1:] for x in filter(itemgetter(0), items)) def get_items_from_dir(basedir, acceptq=lambda x: True): From 77071e14a5c91c1389fe29b9b87471cb241d3f5d Mon Sep 17 00:00:00 2001 From: Charles Haley <cbhaley@i.wont.say.com> Date: Wed, 12 Mar 2014 12:23:24 +0100 Subject: [PATCH 043/122] Several fixes to syncing in the wireless device driver. 1) Handle case where metadata comes from scanning books instead of from calibre. 2) Don't attempt to update a field when one of the two sync columns is None. 3) Refactor code to eliminate interactions between sync types. --- .../devices/smart_device_app/driver.py | 189 ++++++++++-------- 1 file changed, 111 insertions(+), 78 deletions(-) diff --git a/src/calibre/devices/smart_device_app/driver.py b/src/calibre/devices/smart_device_app/driver.py index 23748af774..f9b5d0d742 100644 --- a/src/calibre/devices/smart_device_app/driver.py +++ b/src/calibre/devices/smart_device_app/driver.py @@ -1233,7 +1233,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): if book: bl.add_book(book, replace_metadata=True) book.set('_is_read_', r.get('_is_read_', None)) - book.set('_is_read_changed_', r.get('_is_read_changed_', None)) + book.set('_sync_type_', r.get('_sync_type_', None)) book.set('_last_read_date_', r.get('_last_read_date_', None)) else: books_to_send.append(r['priKey']) @@ -1256,7 +1256,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): del result['_series_sort_'] book = self.json_codec.raw_to_book(result, SDBook, self.PREFIX) book.set('_is_read_', result.get('_is_read_', None)) - book.set('_is_read_changed_', result.get('_is_read_changed_', None)) + book.set('_sync_type_', result.get('_sync_type_', None)) book.set('_last_read_date_', result.get('_last_read_date_', None)) bl.add_book(book, replace_metadata=True) if '_new_book_' in result: @@ -1512,93 +1512,126 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): if self.have_bad_sync_columns: return None - is_changed = book.get('_is_read_changed_', None); + sync_type = book.get('_sync_type_', None); is_read = book.get('_is_read_', None) - # This returns UNDEFINED_DATE if the value is None + # parse_date returns UNDEFINED_DATE if the value is None is_read_date = parse_date(book.get('_last_read_date_', None)); if is_date_undefined(is_read_date): is_read_date = None - value_to_return = None - - if is_changed == 2: - # This is a special case where the user just set the sync column. In - # this case the device value wins if it is not None by falling - # through to the normal sync situation below, otherwise the calibre - # value wins. The orig_* values are set to None to force the normal - # sync code to actually sync because the values are different - orig_is_read_date = None - orig_is_read = None - - if is_read is None: - calibre_val = db.new_api.field_for(self.is_read_sync_col, - id_, default_value=None) - if calibre_val is not None: - # This forces the metadata for the book to be sent to the - # device even if the mod dates haven't changed. - book.set('_force_send_metadata_', True) - self._debug('special update is_read', book.get('title', 'huh?'), - 'to', calibre_val) - value_to_return = set() - - if is_read_date is None: - calibre_val = db.new_api.field_for(self.is_read_date_sync_col, - id_, default_value=None) - if not is_date_undefined(calibre_val): - book.set('_force_send_metadata_', True) - self._debug('special update is_read_date', book.get('title', 'huh?'), - 'to', calibre_val) - value_to_return = set() - # Fall through to the normal sync. At this point either the is_read* - # values are different from the orig_is_read* which will cause a - # sync below, or they are both None which will cause the code below - # to do nothing. If either of the calibre data fields were set, the - # method will return set(), which will force updated metadata to be - # given back to the device, effectively forcing the sync of the - # calibre values back to the device. - else: - orig_is_read = book.get(self.is_read_sync_col, None) - orig_is_read_date = book.get(self.is_read_date_sync_col, None) - + force_return_changed_books = False changed_books = set() - try: - if is_read != orig_is_read: - # The value in the device's is_read checkbox is not the same as the - # last one that came to the device from calibre during the last - # connect, meaning that the user changed it. Write the one from the - # device to calibre's db. - self._debug('standard update book is_read', book.get('title', 'huh?'), - 'to', is_read) - if self.is_read_sync_col: - changed_books = db.new_api.set_field(self.is_read_sync_col, - {id_: is_read}) - except: - self._debug('exception syncing is_read col', self.is_read_sync_col) - traceback.print_exc() - try: - if is_read_date != orig_is_read_date: - self._debug('standard update book is_read_date', book.get('title', 'huh?'), - 'to', is_read_date, 'was', orig_is_read_date) - if self.is_read_date_sync_col: - changed_books |= db.new_api.set_field(self.is_read_date_sync_col, - {id_: is_read_date}) - except: - self._debug('Exception while syncing is_read_date', self.is_read_date_sync_col) - traceback.print_exc() + if sync_type == 3: + # The book metadata was built by the device from metadata in the + # book file itself. It must not be synced, because the metadata is + # almost surely wrong. However, the fact that we got here means that + # book matching has succeeded. Arrange that calibre's metadata is + # sent back to the device. This isn't strictly necessary as sending + # back the info will be arranged in other ways. + self._debug('Book with device-generated metadata', book.get('title', 'huh?')) + book.set('_force_send_metadata_', True) + force_return_changed_books = True + elif sync_type == 2: + # This is a special case where the user just set a sync column. In + # this case the device value wins if it is not None, otherwise the + # calibre value wins. - if changed_books: - # One of the two values was synced, giving a list of changed books. - # Return that. + # Check is_read + if self.is_read_sync_col: + try: + calibre_val = db.new_api.field_for(self.is_read_sync_col, + id_, default_value=None) + if is_read is not None: + # The CC value wins. Check if it is different from calibre's + # value to avoid updating the db to the same value + if is_read != calibre_val: + self._debug('special update calibre to is_read', + book.get('title', 'huh?'), 'to', is_read, calibre_val) + changed_books = db.new_api.set_field(self.is_read_sync_col, + {id_: is_read}) + elif calibre_val is not None: + # Calibre value wins. Force the metadata for the + # book to be sent to the device even if the mod + # dates haven't changed. + self._debug('special update is_read to calibre value', + book.get('title', 'huh?'), 'to', calibre_val) + book.set('_force_send_metadata_', True) + force_return_changed_books = True + except: + self._debug('exception special syncing is_read', self.is_read_sync_col) + traceback.print_exc() + + # Check is_read_date. + if self.is_read_date_sync_col: + try: + # The db method returns None for undefined dates. + calibre_val = db.new_api.field_for(self.is_read_date_sync_col, + id_, default_value=None) + if is_read_date is not None: + if is_read_date != calibre_val: + self._debug('special update calibre to is_read_date', + book.get('title', 'huh?'), 'to', is_read_date, calibre_val) + changed_books |= db.new_api.set_field(self.is_read_date_sync_col, + {id_: is_read_date}) + elif calibre_val is not None: + self._debug('special update is_read_date to calibre value', + book.get('title', 'huh?'), 'to', calibre_val) + book.set('_force_send_metadata_', True) + force_return_changed_books = True + except: + self._debug('exception special syncing is_read_date', + self.is_read_sync_col) + traceback.print_exc() + else: + # This is the standard sync case. If the CC value has changed, it + # wins, otherwise the calibre value is synced to CC in the normal + # fashion (mod date) + if self.is_read_sync_col: + try: + orig_is_read = book.get(self.is_read_sync_col, None) + if is_read != orig_is_read: + # The value in the device's is_read checkbox is not the + # same as the last one that came to the device from + # calibre during the last connect, meaning that the user + # changed it. Write the one from the device to calibre's + # db. + self._debug('standard update is_read', book.get('title', 'huh?'), + 'to', is_read, 'was', orig_is_read) + changed_books = db.new_api.set_field(self.is_read_sync_col, + {id_: is_read}) + except: + self._debug('exception standard syncing is_read', self.is_read_sync_col) + traceback.print_exc() + + if self.is_read_date_sync_col: + try: + orig_is_read_date = book.get(self.is_read_date_sync_col, None) + if is_date_undefined(orig_is_read_date): + orig_is_read_date = None + + if is_read_date != orig_is_read_date: + self._debug('standard update is_read_date', book.get('title', 'huh?'), + 'to', is_read_date, 'was', orig_is_read_date) + changed_books |= db.new_api.set_field(self.is_read_date_sync_col, + {id_: is_read_date}) + except: + self._debug('Exception standard syncing is_read_date', + self.is_read_date_sync_col) + traceback.print_exc() + + if changed_books or force_return_changed_books: + # One of the two values was synced, giving a (perhaps empty) list of + # changed books. Return that. return changed_books - # The user might have changed the value in calibre. If so, that value - # will be sent to the device in the normal way. Note that because any - # updated value has already been synced and so will also be sent, the - # device should put the calibre value into its checkbox (or whatever it - # uses) - return value_to_return + # Nothing was synced. The user might have changed the value in calibre. + # If so, that value will be sent to the device in the normal way. Note + # that because any updated value has already been synced and so will + # also be sent, the device should put the calibre value into its + # checkbox (or whatever it uses) + return None @synchronous('sync_lock') def startup(self): From 9486fe05adb3000b1f7b9154d81ff9c7e091aca0 Mon Sep 17 00:00:00 2001 From: Gregory Riker <griker@hotmail.com> Date: Wed, 12 Mar 2014 05:12:58 -0700 Subject: [PATCH 044/122] Add optimization to listdir to return filenames only --- .../devices/idevice/libimobiledevice.py | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/src/calibre/devices/idevice/libimobiledevice.py b/src/calibre/devices/idevice/libimobiledevice.py index 4da72939dd..3b405d7977 100644 --- a/src/calibre/devices/idevice/libimobiledevice.py +++ b/src/calibre/devices/idevice/libimobiledevice.py @@ -403,13 +403,13 @@ class libiMobileDevice(): self._log_location() return self._lockdown_get_value(requested_items) - def listdir(self, path): + def listdir(self, path, get_stats=True): ''' Return a list containing the names of the entries in the iOS directory given by path. ''' self._log_location("'%s'" % path) - return self._afc_read_directory(path) + return self._afc_read_directory(path, get_stats=get_stats) def load_library(self): if islinux: @@ -1053,16 +1053,17 @@ class libiMobileDevice(): return error - def _afc_read_directory(self, directory=''): + def _afc_read_directory(self, directory='', get_stats=True): ''' Gets a directory listing of the directory requested Args: - client: (AFC_CLIENT_T) The client to get a directory listing from - dir: (const char *) The directory to list (a fully-qualified path) - list: (char ***) A char list of files in that directory, terminated by - an empty string. NULL if there was an error. - + client: (AFC_CLIENT_T) The client to get a directory listing from + dir: (const char *) The directory to list (a fully-qualified path) + list: (char ***) A char list of files in that directory, terminated by + an empty string. NULL if there was an error. + get_stats: If True, return full file stats for each file in dir (slower) + If False, return filename only (faster) Result: error: AFC_E_SUCCESS on success or an AFC_E_* error value file_stats: @@ -1094,7 +1095,10 @@ class libiMobileDevice(): path = '/' + this_item else: path = '/'.join([directory, this_item]) - file_stats[os.path.basename(path)] = self._afc_get_file_info(path) + if get_stats: + file_stats[os.path.basename(path)] = self._afc_get_file_info(path) + else: + file_stats[os.path.basename(path)] = {} self.current_dir = directory return file_stats From a32a1e6f794565f6c4cc046ae08165f5db2e229c Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Thu, 13 Mar 2014 08:30:00 +0530 Subject: [PATCH 045/122] Add an action to clear search history as well as disabling the popup --- src/calibre/gui2/tweak_book/search.py | 12 +++++++----- src/calibre/gui2/widgets2.py | 5 +++++ 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/src/calibre/gui2/tweak_book/search.py b/src/calibre/gui2/tweak_book/search.py index 6988aa5912..ca29189046 100644 --- a/src/calibre/gui2/tweak_book/search.py +++ b/src/calibre/gui2/tweak_book/search.py @@ -27,15 +27,17 @@ class PushButton(QPushButton): class HistoryLineEdit(HistoryLineEdit2): - def __init__(self, parent): + def __init__(self, parent, clear_msg): HistoryLineEdit2.__init__(self, parent) self.disable_popup = tprefs['disable_completion_popup_for_search'] + self.clear_msg = clear_msg def contextMenuEvent(self, event): menu = self.createStandardContextMenu() menu.addSeparator() - menu.addAction((_('Show completion based on search history') if self.disable_popup else _( - 'Hide completion based on search history')), self.toggle_popups) + menu.addAction(self.clear_msg, self.clear_history) + menu.addAction((_('Enable completion based on search history') if self.disable_popup else _( + 'Disable completion based on search history')), self.toggle_popups) menu.exec_(event.globalPos()) def toggle_popups(self): @@ -63,7 +65,7 @@ class SearchWidget(QWidget): self.fl = fl = QLabel(_('&Find:')) fl.setAlignment(Qt.AlignRight | Qt.AlignCenter) - self.find_text = ft = HistoryLineEdit(self) + self.find_text = ft = HistoryLineEdit(self, _('Clear search history')) ft.initialize('tweak_book_find_edit') ft.returnPressed.connect(lambda : self.search_triggered.emit('find')) fl.setBuddy(ft) @@ -72,7 +74,7 @@ class SearchWidget(QWidget): self.rl = rl = QLabel(_('&Replace:')) rl.setAlignment(Qt.AlignRight | Qt.AlignCenter) - self.replace_text = rt = HistoryLineEdit(self) + self.replace_text = rt = HistoryLineEdit(self, _('Clear replace history')) rt.initialize('tweak_book_replace_edit') rl.setBuddy(rt) l.addWidget(rl, 1, 0) diff --git a/src/calibre/gui2/widgets2.py b/src/calibre/gui2/widgets2.py index c53ae2e93f..3a52d72078 100644 --- a/src/calibre/gui2/widgets2.py +++ b/src/calibre/gui2/widgets2.py @@ -34,3 +34,8 @@ class HistoryLineEdit2(LineEdit): history.set(self.store_name, self.history) self.update_items_cache(self.history) + def clear_history(self): + self.history = [] + history.set(self.store_name, self.history) + self.update_items_cache(self.history) + From d78b9c3f0db41864b8c8784e2840db1e38a56c4d Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Thu, 13 Mar 2014 08:32:30 +0530 Subject: [PATCH 046/122] Move the filter text boxes to the top so that they are closer to the first result --- src/calibre/gui2/tweak_book/widgets.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/calibre/gui2/tweak_book/widgets.py b/src/calibre/gui2/tweak_book/widgets.py index 43793e9bda..445eaf493a 100644 --- a/src/calibre/gui2/tweak_book/widgets.py +++ b/src/calibre/gui2/tweak_book/widgets.py @@ -560,7 +560,7 @@ class InsertLink(Dialog): self.fnl = fnl = QVBoxLayout() self.la1 = la = QLabel(_('Choose a &file to link to:')) la.setBuddy(fn) - fnl.addWidget(la), fnl.addWidget(fn), fnl.addWidget(f) + fnl.addWidget(la), fnl.addWidget(f), fnl.addWidget(fn) h.addLayout(fnl), h.setStretch(0, 2) fn, f = create_filterable_names_list([], filter_text=_('Filter locations'), parent=self) @@ -570,7 +570,7 @@ class InsertLink(Dialog): self.anl = fnl = QVBoxLayout() self.la2 = la = QLabel(_('Choose a &location (anchor) in the file:')) la.setBuddy(fn) - fnl.addWidget(la), fnl.addWidget(fn), fnl.addWidget(f) + fnl.addWidget(la), fnl.addWidget(f), fnl.addWidget(fn) h.addLayout(fnl), h.setStretch(1, 1) self.tl = tl = QHBoxLayout() From aea66db0189619472115ec183ed472d2ffe199ef Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Thu, 13 Mar 2014 08:52:58 +0530 Subject: [PATCH 047/122] Show the search expression for the virtual library in a tooltip when hovering over the tab for the virtual library. Fixes #1291691 [[Enhancement] VL search expression tooltip on hover](https://bugs.launchpad.net/calibre/+bug/1291691) --- src/calibre/gui2/init.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/calibre/gui2/init.py b/src/calibre/gui2/init.py index 7c952ebaf3..2a626f784a 100644 --- a/src/calibre/gui2/init.py +++ b/src/calibre/gui2/init.py @@ -315,7 +315,8 @@ class VLTabs(QTabBar): # {{{ def rebuild(self): self.currentChanged.disconnect(self.tab_changed) db = self.current_db - virt_libs = frozenset(db.prefs.get('virtual_libraries', {})) + vl_map = db.prefs.get('virtual_libraries', {}) + virt_libs = frozenset(vl_map) hidden = frozenset(db.prefs['virt_libs_hidden']) if hidden - virt_libs: db.prefs['virt_libs_hidden'] = list(hidden.intersection(virt_libs)) @@ -328,6 +329,9 @@ class VLTabs(QTabBar): # {{{ order = {x:i for i, x in enumerate(order)} for i, vl in enumerate(sorted(virt_libs, key=lambda x:(order.get(x, 0), sort_key(x)))): self.addTab(vl.replace('&', '&&') or _('All books')) + sexp = vl_map.get(vl, None) + if sexp is not None: + self.setTabToolTip(i, _('Search expression for this virtual library:') + '\n\n' + sexp) self.setTabData(i, vl) if vl == current_lib: current_idx = i From f0d676502fbddca0aedb1d6ac79d0e0b25a182a7 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Thu, 13 Mar 2014 14:13:29 +0530 Subject: [PATCH 048/122] Edit book: Fix file permissions for the edited book being changed on Linux and OS X Since the editor saves by using a temp file and then renaming the temp file to overwrite the original, we have to explicitly copy over permissions and owner metadata to the temp file to ensure they remain unchanged. --- src/calibre/gui2/tweak_book/save.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/calibre/gui2/tweak_book/save.py b/src/calibre/gui2/tweak_book/save.py index 0627c27241..36688dc5d3 100644 --- a/src/calibre/gui2/tweak_book/save.py +++ b/src/calibre/gui2/tweak_book/save.py @@ -22,6 +22,13 @@ from calibre.utils.ipc import RC def save_container(container, path): temp = PersistentTemporaryFile( prefix=('_' if iswindows else '.'), suffix=os.path.splitext(path)[1], dir=os.path.dirname(path)) + if hasattr(os, 'fchmod'): + # Ensure file permissions and owner information is preserved + fno = temp.fileno() + st = os.stat(path) + os.fchmod(fno, st.st_mode) + os.fchown(fno, st.st_uid, st.st_gid) + temp.close() temp = temp.name try: From 5024757c94cc7a5cbf5a86f7732c4da690ade8f2 Mon Sep 17 00:00:00 2001 From: Charles Haley <cbhaley@i.wont.say.com> Date: Thu, 13 Mar 2014 12:52:27 +0100 Subject: [PATCH 049/122] Wireless device: Do not attempt to sync data when sending books for the first time. --- .../devices/smart_device_app/driver.py | 29 +++++++++++++------ 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/src/calibre/devices/smart_device_app/driver.py b/src/calibre/devices/smart_device_app/driver.py index f9b5d0d742..9d89e264cf 100644 --- a/src/calibre/devices/smart_device_app/driver.py +++ b/src/calibre/devices/smart_device_app/driver.py @@ -1513,12 +1513,23 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): return None sync_type = book.get('_sync_type_', None); - is_read = book.get('_is_read_', None) + # We need to check if our attributes are in the book. If they are not + # then this is metadata coming from calibre to the device for the first + # time, in which case we must not sync it. + if hasattr(book, '_is_read_'): + is_read = book.get('_is_read_', None) + has_is_read = True + else: + has_is_read = False - # parse_date returns UNDEFINED_DATE if the value is None - is_read_date = parse_date(book.get('_last_read_date_', None)); - if is_date_undefined(is_read_date): - is_read_date = None + if hasattr(book, '_last_read_date_'): + # parse_date returns UNDEFINED_DATE if the value is None + is_read_date = parse_date(book.get('_last_read_date_', None)); + if is_date_undefined(is_read_date): + is_read_date = None + has_is_read_date = True + else: + has_is_read_date = False force_return_changed_books = False changed_books = set() @@ -1539,7 +1550,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): # calibre value wins. # Check is_read - if self.is_read_sync_col: + if has_is_read and self.is_read_sync_col: try: calibre_val = db.new_api.field_for(self.is_read_sync_col, id_, default_value=None) @@ -1564,7 +1575,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): traceback.print_exc() # Check is_read_date. - if self.is_read_date_sync_col: + if has_is_read_date and self.is_read_date_sync_col: try: # The db method returns None for undefined dates. calibre_val = db.new_api.field_for(self.is_read_date_sync_col, @@ -1588,7 +1599,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): # This is the standard sync case. If the CC value has changed, it # wins, otherwise the calibre value is synced to CC in the normal # fashion (mod date) - if self.is_read_sync_col: + if has_is_read and self.is_read_sync_col: try: orig_is_read = book.get(self.is_read_sync_col, None) if is_read != orig_is_read: @@ -1605,7 +1616,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): self._debug('exception standard syncing is_read', self.is_read_sync_col) traceback.print_exc() - if self.is_read_date_sync_col: + if has_is_read_date and self.is_read_date_sync_col: try: orig_is_read_date = book.get(self.is_read_date_sync_col, None) if is_date_undefined(orig_is_read_date): From 7bf63b3290a5638560e351bd108756836a141cdb Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Fri, 14 Mar 2014 09:51:05 +0530 Subject: [PATCH 050/122] version 1.28 --- Changelog.yaml | 40 ++++++++++++++++++++++++++++++++++++++++ src/calibre/constants.py | 2 +- 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/Changelog.yaml b/Changelog.yaml index 0b5965fa2d..1251195ab6 100644 --- a/Changelog.yaml +++ b/Changelog.yaml @@ -20,6 +20,46 @@ # new recipes: # - title: +- version: 1.28.0 + date: 2014-03-14 + + new features: + - title: "Edit Book: Add a tool to easily insert hyperlinks (click the insert hyperlink button on the toolbar)" + + - title: "Edit book: Add a tool to easily open a file inside the book for editing by just typing a few characters from the file name. To use it press Ctrl+T in the editor or go to Edit->Quick open a file to edit'" + + - title: "Edit book: Allow disabling the completion popups for the search and replace fields. Right click on the search/replace field to enable/disable the completion popup" + + - title: "E-book viewer: Add an option to control the maximum text height in full screen. Note that it only works if the viewer is in paged mode (which is the default mode)." + + - title: "Show the search expression for the virtual library in a tooltip when hovering over the tab for the virtual library." + tickets: [1291691] + + - title: "Book details panel: Show author URL in a tooltip when hovering over author names" + + - title: "Kobo driver: Update to handle updated Kobo firmware" + + bug fixes: + - title: "Library backup: Avoid infinite retries if converting metadata to backup OPF for a book fails. Simply fail to backup the metadata for that book." + tickets: [1291142] + + - title: "Edit book: Fix file permissions for the edited book being changed on Linux and OS X" + + - title: "Fix text entry cursor becoming invisible when completion popup is opened" + + - title: "Possible fix for crash on OS X Mavericks when adding duplicates" + + - title: "E-book viewer: Fix pressing the Esc key to leave full screen mode not changing the state of the full screen button" + + - title: "When reading metadata from filenames, do not apply the fallback regexp to read metadata if the user specified regexp puts the entire filename into the title. The fallback is only used if the user specified expression does not match the filename at all." + + - title: "Linux binary install script: Fix error on linux systems where the system python has an encoding of None set on stdout. Assume encoding is utf-8 in this case." + + - title: "Content server: Fix (maybe) an error on some windows computers with a non-standard default encoding" + + improved recipes: + - Fleshbot + - version: 1.27.0 date: 2014-03-07 diff --git a/src/calibre/constants.py b/src/calibre/constants.py index 5d78357f6c..61983d0358 100644 --- a/src/calibre/constants.py +++ b/src/calibre/constants.py @@ -4,7 +4,7 @@ __license__ = 'GPL v3' __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net' __docformat__ = 'restructuredtext en' __appname__ = u'calibre' -numeric_version = (1, 27, 0) +numeric_version = (1, 28, 0) __version__ = u'.'.join(map(unicode, numeric_version)) __author__ = u"Kovid Goyal <kovid@kovidgoyal.net>" From 5864635e4a4dd5f74315fc6c325609c1ed5972c1 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Sat, 15 Mar 2014 08:42:04 +0530 Subject: [PATCH 051/122] When installing plugins, if an error occurs while installing the plugin, the plugin was not automatically un-installed due to a typo. --- src/calibre/gui2/dialogs/plugin_updater.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/calibre/gui2/dialogs/plugin_updater.py b/src/calibre/gui2/dialogs/plugin_updater.py index 920949894e..1a3b7d4030 100644 --- a/src/calibre/gui2/dialogs/plugin_updater.py +++ b/src/calibre/gui2/dialogs/plugin_updater.py @@ -748,8 +748,8 @@ class PluginUpdaterDialog(SizePersistedDialog): det_msg=traceback.format_exc(), show=True) if DEBUG: prints('Due to error now uninstalling plugin: %s'%display_plugin.name) - remove_plugin(display_plugin.name) - display_plugin.plugin = None + remove_plugin(display_plugin.name) + display_plugin.plugin = None display_plugin.uninstall_plugins = [] if self.proxy_model.filter_criteria in [FILTER_NOT_INSTALLED, FILTER_UPDATE_AVAILABLE]: From 6d60befc70eef3e3e3bf699e8d4e22d5c67cad0c Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Sat, 15 Mar 2014 18:59:33 +0530 Subject: [PATCH 052/122] Edit book: Fix check book failling in the presence of empty <style/> tags. Fixes #1292841 [Edit book / check crashes on empty inline style tag](https://bugs.launchpad.net/calibre/+bug/1292841) --- src/calibre/ebooks/oeb/polish/check/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/ebooks/oeb/polish/check/main.py b/src/calibre/ebooks/oeb/polish/check/main.py index 3f5c2fafad..466e96afd3 100644 --- a/src/calibre/ebooks/oeb/polish/check/main.py +++ b/src/calibre/ebooks/oeb/polish/check/main.py @@ -50,7 +50,7 @@ def run_checks(container): for name, mt, raw in html_items: root = container.parsed(name) for style in root.xpath('//*[local-name()="style"]'): - if style.get('type', 'text/css') == 'text/css': + if style.get('type', 'text/css') == 'text/css' and style.text: errors.extend(check_css_parsing(name, style.text, line_offset=style.sourceline - 1)) for elem in root.xpath('//*[@style]'): raw = elem.get('style') From b5dba545cd5a41a36510231fa755b79ed9978ac6 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Sat, 15 Mar 2014 19:17:36 +0530 Subject: [PATCH 053/122] Edit book: Fix syntax highlighting in HTML files breaks if the closing of a comment or processing instruction is a tthe start of a new line. --- src/calibre/gui2/tweak_book/editor/syntax/html.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/calibre/gui2/tweak_book/editor/syntax/html.py b/src/calibre/gui2/tweak_book/editor/syntax/html.py index 25bcb3189b..5890481323 100644 --- a/src/calibre/gui2/tweak_book/editor/syntax/html.py +++ b/src/calibre/gui2/tweak_book/editor/syntax/html.py @@ -286,7 +286,7 @@ def closing_tag(state, text, i, formats): def in_comment(state, text, i, formats): ' Comment, processing instruction or doctype ' end = {state.IN_COMMENT:'-->', state.IN_PI:'?>'}.get(state.parse, '>') - pos = text.find(end, i+1) + pos = text.find(end, i) fmt = formats['comment' if state.parse == state.IN_COMMENT else 'preproc'] if pos == -1: num = len(text) - i @@ -371,6 +371,8 @@ if __name__ == '__main__': launch_editor('''\ <!DOCTYPE html> <html xml:lang="en" lang="en"> +<!-- +--> <head> <meta charset="utf-8" /> <title>A title with a tag <span> in it, the tag is treated as normal text From 82e10006266fed2ee810604e0c453c1f40f34c34 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 16 Mar 2014 10:58:27 +0530 Subject: [PATCH 054/122] Edit Book: When inserting hyperlinks, allow specifying the text for the hyperlink in the insert hyperlink dialog --- src/calibre/gui2/tweak_book/boss.py | 4 +- .../gui2/tweak_book/editor/smart/__init__.py | 3 ++ .../gui2/tweak_book/editor/smart/html.py | 37 ++++++++++++++++--- src/calibre/gui2/tweak_book/editor/text.py | 9 +++-- src/calibre/gui2/tweak_book/editor/widget.py | 7 +++- src/calibre/gui2/tweak_book/widgets.py | 23 ++++++++---- 6 files changed, 64 insertions(+), 19 deletions(-) diff --git a/src/calibre/gui2/tweak_book/boss.py b/src/calibre/gui2/tweak_book/boss.py index de605790fb..c5fbfeccf5 100644 --- a/src/calibre/gui2/tweak_book/boss.py +++ b/src/calibre/gui2/tweak_book/boss.py @@ -643,9 +643,9 @@ class Boss(QObject): ed.insert_image(href) elif action[0] == 'insert_hyperlink': self.commit_all_editors_to_container() - d = InsertLink(current_container(), edname, parent=self.gui) + d = InsertLink(current_container(), edname, initial_text=ed.get_smart_selection(), parent=self.gui) if d.exec_() == d.Accepted: - ed.insert_hyperlink(d.href) + ed.insert_hyperlink(d.href, d.text) else: ed.action_triggered(action) diff --git a/src/calibre/gui2/tweak_book/editor/smart/__init__.py b/src/calibre/gui2/tweak_book/editor/smart/__init__.py index 7cb00ca997..b13c22032a 100644 --- a/src/calibre/gui2/tweak_book/editor/smart/__init__.py +++ b/src/calibre/gui2/tweak_book/editor/smart/__init__.py @@ -14,3 +14,6 @@ class NullSmarts(object): def get_extra_selections(self, editor): return () + def get_smart_selection(self, editor, update=True): + return editor.selected_text + diff --git a/src/calibre/gui2/tweak_book/editor/smart/html.py b/src/calibre/gui2/tweak_book/editor/smart/html.py index bd4a5b568f..a0b2a1ba77 100644 --- a/src/calibre/gui2/tweak_book/editor/smart/html.py +++ b/src/calibre/gui2/tweak_book/editor/smart/html.py @@ -129,7 +129,7 @@ def rename_tag(cursor, opening_tag, closing_tag, new_name, insert=False): cursor.insertText(text) cursor.endEditBlock() -def ensure_not_within_tag_definition(cursor): +def ensure_not_within_tag_definition(cursor, forward=True): ''' Ensure the cursor is not inside a tag definition <>. Returns True iff the cursor was moved. ''' block, offset = cursor.block(), cursor.positionInBlock() b, boundary = next_tag_boundary(block, offset, forward=False) @@ -137,10 +137,15 @@ def ensure_not_within_tag_definition(cursor): return False if boundary.is_start: # We are inside a tag - block, boundary = next_tag_boundary(block, offset) - if block is not None: - cursor.setPosition(block.position() + boundary.offset + 1) + if forward: + block, boundary = next_tag_boundary(block, offset) + if block is not None: + cursor.setPosition(block.position() + boundary.offset + 1) + return True + else: + cursor.setPosition(b.position() + boundary.offset) return True + return False class HTMLSmarts(NullSmarts): @@ -195,7 +200,27 @@ class HTMLSmarts(NullSmarts): return error_dialog(editor, _('No found'), _( 'No suitable block level tag was found to rename'), show=True) - def insert_hyperlink(self, editor, target): + def get_smart_selection(self, editor, update=True): + cursor = editor.textCursor() + if not cursor.hasSelection(): + return '' + left = min(cursor.anchor(), cursor.position()) + right = max(cursor.anchor(), cursor.position()) + + cursor.setPosition(left) + ensure_not_within_tag_definition(cursor) + left = cursor.position() + + cursor.setPosition(right) + ensure_not_within_tag_definition(cursor, forward=False) + right = cursor.position() + + cursor.setPosition(left), cursor.setPosition(right, cursor.KeepAnchor) + if update: + editor.setTextCursor(cursor) + return editor.selected_text_from_cursor(cursor) + + def insert_hyperlink(self, editor, target, text): c = editor.textCursor() if c.hasSelection(): c.insertText('') # delete any existing selected text @@ -204,4 +229,6 @@ class HTMLSmarts(NullSmarts): p = c.position() c.insertText('') c.setPosition(p) # ensure cursor is positioned inside the newly created tag + if text: + c.insertText(text) editor.setTextCursor(c) diff --git a/src/calibre/gui2/tweak_book/editor/text.py b/src/calibre/gui2/tweak_book/editor/text.py index cccb274c97..d310906ea4 100644 --- a/src/calibre/gui2/tweak_book/editor/text.py +++ b/src/calibre/gui2/tweak_book/editor/text.py @@ -101,9 +101,12 @@ class PlainTextEdit(QPlainTextEdit): self.copy() self.textCursor().removeSelectedText() + def selected_text_from_cursor(self, cursor): + return unicodedata.normalize('NFC', unicode(cursor.selectedText()).replace(PARAGRAPH_SEPARATOR, '\n').rstrip('\0')) + @property def selected_text(self): - return unicodedata.normalize('NFC', unicode(self.textCursor().selectedText()).replace(PARAGRAPH_SEPARATOR, '\n').rstrip('\0')) + return self.selected_text_from_cursor(self.textCursor()) def selection_changed(self): # Workaround Qt replacing nbsp with normal spaces on copy @@ -602,9 +605,9 @@ class TextEdit(PlainTextEdit): c.setPosition(left + len(text), c.KeepAnchor) self.setTextCursor(c) - def insert_hyperlink(self, target): + def insert_hyperlink(self, target, text): if hasattr(self.smarts, 'insert_hyperlink'): - self.smarts.insert_hyperlink(self, target) + self.smarts.insert_hyperlink(self, target, text) def keyPressEvent(self, ev): if ev.key() == Qt.Key_X and ev.modifiers() == Qt.AltModifier: diff --git a/src/calibre/gui2/tweak_book/editor/widget.py b/src/calibre/gui2/tweak_book/editor/widget.py index d4c7de956a..81a54d17a9 100644 --- a/src/calibre/gui2/tweak_book/editor/widget.py +++ b/src/calibre/gui2/tweak_book/editor/widget.py @@ -144,8 +144,8 @@ class Editor(QMainWindow): def insert_image(self, href): self.editor.insert_image(href) - def insert_hyperlink(self, href): - self.editor.insert_hyperlink(href) + def insert_hyperlink(self, href, text): + self.editor.insert_hyperlink(href, text) def undo(self): self.editor.undo() @@ -157,6 +157,9 @@ class Editor(QMainWindow): def selected_text(self): return self.editor.selected_text + def get_smart_selection(self, update=True): + return self.editor.smarts.get_smart_selection(self.editor, update=update) + # Search and replace {{{ def mark_selected_text(self): self.editor.mark_selected_text() diff --git a/src/calibre/gui2/tweak_book/widgets.py b/src/calibre/gui2/tweak_book/widgets.py index 445eaf493a..0ca22d1832 100644 --- a/src/calibre/gui2/tweak_book/widgets.py +++ b/src/calibre/gui2/tweak_book/widgets.py @@ -537,9 +537,10 @@ def create_filterable_names_list(names, filter_text=None, parent=None): # Insert Link {{{ class InsertLink(Dialog): - def __init__(self, container, source_name, parent=None): + def __init__(self, container, source_name, initial_text=None, parent=None): self.container = container self.source_name = source_name + self.initial_text = initial_text Dialog.__init__(self, _('Insert Hyperlink'), 'insert-hyperlink', parent=parent) self.anchor_cache = {} @@ -573,14 +574,18 @@ class InsertLink(Dialog): fnl.addWidget(la), fnl.addWidget(f), fnl.addWidget(fn) h.addLayout(fnl), h.setStretch(1, 1) - self.tl = tl = QHBoxLayout() - self.la3 = la = QLabel(_('&Target:')) - tl.addWidget(la) + self.tl = tl = QFormLayout() self.target = t = QLineEdit(self) - la.setBuddy(t) - tl.addWidget(t) + t.setPlaceholderText(_('The destination (href) for the link')) + tl.addRow(_('&Target:'), t) l.addLayout(tl) + self.text_edit = t = QLineEdit(self) + la.setBuddy(t) + tl.addRow(_('Te&xt:'), t) + t.setText(self.initial_text or '') + t.setPlaceholderText(_('The (optional) text for the link')) + l.addWidget(self.bb) def selected_file_changed(self, *args): @@ -622,6 +627,10 @@ class InsertLink(Dialog): def href(self): return unicode(self.target.text()).strip() + @property + def text(self): + return unicode(self.text_edit.text()).strip() + @classmethod def test(cls): import sys @@ -629,7 +638,7 @@ class InsertLink(Dialog): c = get_container(sys.argv[-1], tweak_mode=True) d = cls(c, next(c.spine_names)[0]) if d.exec_() == d.Accepted: - print (d.href) + print (d.href, d.text) # }}} From 7ee3a1703d23f73e74097586070a5340579c7f1a Mon Sep 17 00:00:00 2001 From: Charles Haley Date: Sun, 16 Mar 2014 15:56:40 +0100 Subject: [PATCH 055/122] If the user closes calibre while connected to a wireless device, cleanly disconnect and flush the metadata cache. --- src/calibre/devices/smart_device_app/driver.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/calibre/devices/smart_device_app/driver.py b/src/calibre/devices/smart_device_app/driver.py index 9d89e264cf..81f41db06b 100644 --- a/src/calibre/devices/smart_device_app/driver.py +++ b/src/calibre/devices/smart_device_app/driver.py @@ -1779,6 +1779,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): @synchronous('sync_lock') def shutdown(self): + self._close_device_socket() if getattr(self, 'listen_socket', None) is not None: self.connection_listener.stop() try: From e6e4a61ecccfde9de0225b7b901a5012f79fbb06 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 17 Mar 2014 09:16:05 +0530 Subject: [PATCH 056/122] Update Wired Daily Edition --- recipes/wired_daily.recipe | 30 ++++++++---------------------- 1 file changed, 8 insertions(+), 22 deletions(-) diff --git a/recipes/wired_daily.recipe b/recipes/wired_daily.recipe index df59c7c826..7b1f233a7d 100644 --- a/recipes/wired_daily.recipe +++ b/recipes/wired_daily.recipe @@ -2,10 +2,8 @@ __license__ = 'GPL v3' __docformat__ = 'restructuredtext en' -import re from calibre.web.feeds.news import BasicNewsRecipe -from calibre.ebooks.chardet import xml_to_unicode class Wired_Daily(BasicNewsRecipe): @@ -14,22 +12,13 @@ class Wired_Daily(BasicNewsRecipe): description = 'Technology news' timefmt = ' [%Y%b%d %H%M]' language = 'en' - + use_embedded_content = False no_stylesheets = True - preprocess_regexps = [(re.compile(r'', re.DOTALL), lambda m: - '')] - - remove_tags_before = dict(name='div', id='content') - remove_tags = [dict(id=['header', 'commenting_module', 'post_nav', - 'social_tools', 'sidebar', 'footer', 'social_wishlist', 'pgwidget', - 'outerWrapper', 'inf_widget']), - {'class':['entryActions', 'advertisement', 'entryTags']}, - dict(name=['noscript', 'script']), - dict(name='h4', attrs={'class':re.compile(r'rat\d+')}), - {'class':lambda x: x and x.startswith('contentjump')}, - dict(name='li', attrs={'class':['entryCategories', 'entryEdit']})] + keep_only_tags = [ # dict(name= 'div', id ='liveblog-hdr'), + dict(name='div', attrs={'class': 'post'})] + remove_tags = [dict(name='div', attrs={'class': 'social-top'})] feeds = [ ('Top News', 'http://feeds.wired.com/wired/index'), @@ -49,11 +38,8 @@ class Wired_Daily(BasicNewsRecipe): ('Science', 'http://www.wired.com/wiredscience/feed/'), ] - def populate_article_metadata(self, article, soup, first): - if article.text_summary: - article.text_summary = xml_to_unicode(article.text_summary, - resolve_entities=True)[0] - - def print_version(self, url): - return url + '/all/1' + def preprocess_html(self, soup): + for img in soup.findAll('img', attrs={'data-lazy-src':True}): + img['src'] = img['data-lazy-src'] + return soup From a0d19d2f40b3cbeaff17ec53d983dd2bc787bee3 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 17 Mar 2014 09:36:04 +0530 Subject: [PATCH 057/122] Edit book: Fix a regression in the previous release that broke saving a copy of the current book on linux and OS X --- src/calibre/gui2/tweak_book/save.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/calibre/gui2/tweak_book/save.py b/src/calibre/gui2/tweak_book/save.py index 36688dc5d3..cd78860796 100644 --- a/src/calibre/gui2/tweak_book/save.py +++ b/src/calibre/gui2/tweak_book/save.py @@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import, __license__ = 'GPL v3' __copyright__ = '2013, Kovid Goyal ' -import shutil, os +import shutil, os, errno from threading import Thread from Queue import LifoQueue, Empty @@ -25,7 +25,14 @@ def save_container(container, path): if hasattr(os, 'fchmod'): # Ensure file permissions and owner information is preserved fno = temp.fileno() - st = os.stat(path) + try: + st = os.stat(path) + except EnvironmentError as err: + if err.errno != errno.ENOENT: + raise + # path may not exist if we are saving a copy, in which case we use + # the metadata from the original book + st = os.stat(container.path_to_ebook) os.fchmod(fno, st.st_mode) os.fchown(fno, st.st_uid, st.st_gid) From 5c2dec8022a1e11bae1c4de89509139da795675c Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 17 Mar 2014 10:02:23 +0530 Subject: [PATCH 058/122] Edit Book: Preview panel: Add a copy selected text action to the context menu --- src/calibre/gui2/tweak_book/preview.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/calibre/gui2/tweak_book/preview.py b/src/calibre/gui2/tweak_book/preview.py index 7c12c35292..fc986bbc87 100644 --- a/src/calibre/gui2/tweak_book/preview.py +++ b/src/calibre/gui2/tweak_book/preview.py @@ -416,6 +416,9 @@ class WebView(QWebView): def contextMenuEvent(self, ev): menu = QMenu(self) + ca = self.pageAction(QWebPage.Copy) + if ca.isEnabled(): + menu.addAction(ca) menu.addAction(actions['reload-preview']) menu.addAction(QIcon(I('debug.png')), _('Inspect element'), self.inspect) menu.exec_(ev.globalPos()) From 77e79d75cb9a055bf2dd2c6842147c2adad8547a Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 17 Mar 2014 10:20:16 +0530 Subject: [PATCH 059/122] Edit Book: Fix saving of empty files not working --- src/calibre/gui2/tweak_book/editor/text.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/calibre/gui2/tweak_book/editor/text.py b/src/calibre/gui2/tweak_book/editor/text.py index d310906ea4..79e57c1804 100644 --- a/src/calibre/gui2/tweak_book/editor/text.py +++ b/src/calibre/gui2/tweak_book/editor/text.py @@ -81,8 +81,11 @@ class PlainTextEdit(QPlainTextEdit): if hasattr(ans, 'rstrip'): ans = ans.rstrip('\0') else: # QString - while ans[-1] == '\0': - ans.chop(1) + try: + while ans[-1] == '\0': + ans.chop(1) + except IndexError: + pass # ans is an empty string return ans @pyqtSlot() From 2e40d47b5ff558f704bbdebbab6f59d3f6ba474b Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 17 Mar 2014 11:40:41 +0530 Subject: [PATCH 060/122] Fix side margin not defined in paged mode --- resources/compiled_coffeescript.zip | Bin 81140 -> 81197 bytes src/calibre/ebooks/oeb/display/paged.coffee | 2 ++ 2 files changed, 2 insertions(+) diff --git a/resources/compiled_coffeescript.zip b/resources/compiled_coffeescript.zip index b8b7c199d5110342dcf9090e17a2e97f15a35b43..e404bffba289e3d1009a0a53189efb82e702db00 100644 GIT binary patch delta 199 zcmezJlV$BMmJJhT@UDy~bXj1r`($Sv14HNLxibWqxQjDWQsZ+Ii_$amHqV)rBf^8= zE7)%SzOK0wRoC=(3&toWkMEP~OGPI;eCL>E$*9G+X}Y#0qX)A%KjZY@e2fay!Gh1H zAGBoDXV*7pWcbHaJ^i5-qsVk_E5_C8X{p9WhRH_esU{W{7Ri>WX{JW0CPpSnCaH-A gNfu^Crb(t|CZ-0{4_GrwOs_L%P^9s}T^#zz`5OIpMqH=It}SF>XFND_Uf;q49!f<4U>{m%#w@@%+gW~j14V~O-wC}6O&C1l9Nr7(k#tPlynr7CO4J} LZ`ZSC{Lcsg07^Od diff --git a/src/calibre/ebooks/oeb/display/paged.coffee b/src/calibre/ebooks/oeb/display/paged.coffee index 56c7135e90..946edd619b 100644 --- a/src/calibre/ebooks/oeb/display/paged.coffee +++ b/src/calibre/ebooks/oeb/display/paged.coffee @@ -22,6 +22,7 @@ class PagedDisplay this.set_geometry() this.page_width = 0 this.screen_width = 0 + this.side_margin = 0 this.in_paged_mode = false this.current_margin_side = 0 this.is_full_screen_layout = false @@ -122,6 +123,7 @@ class PagedDisplay col_width = Math.max(100, ((ww - adjust)/n) - 2*sm) this.col_width = col_width this.page_width = col_width + 2*sm + this.side_margin = sm this.screen_width = this.page_width * this.cols_per_screen fgcolor = body_style.getPropertyValue('color') From 43e325b79e8cd9b250cc049892d4739fb864fdea Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 17 Mar 2014 11:44:15 +0530 Subject: [PATCH 061/122] E-book viewer: Fix right margin for last page in a chapter sometimes disappearing when changing font size. Fixes #1292822 [Right margin of text when resize in reader](https://bugs.launchpad.net/calibre/+bug/1292822) --- src/calibre/gui2/viewer/documentview.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/src/calibre/gui2/viewer/documentview.py b/src/calibre/gui2/viewer/documentview.py index 109ea85436..130af04424 100644 --- a/src/calibre/gui2/viewer/documentview.py +++ b/src/calibre/gui2/viewer/documentview.py @@ -281,11 +281,16 @@ class Document(QWebPage): # {{{ )) force_fullscreen_layout = bool(getattr(last_loaded_path, 'is_single_page', False)) - f = 'true' if force_fullscreen_layout else 'false' - side_margin = self.javascript('window.paged_display.layout(%s)'%f, typ=int) + self.update_contents_size_for_paged_mode(force_fullscreen_layout) + + def update_contents_size_for_paged_mode(self, force_fullscreen_layout=None): # Setup the contents size to ensure that there is a right most margin. # Without this WebKit renders the final column with no margin, as the # columns extend beyond the boundaries (and margin) of body + if force_fullscreen_layout is None: + force_fullscreen_layout = self.javascript('window.paged_display.is_full_screen_layout', typ=bool) + f = 'true' if force_fullscreen_layout else 'false' + side_margin = self.javascript('window.paged_display.layout(%s)'%f, typ=int) mf = self.mainFrame() sz = mf.contentsSize() scroll_width = self.javascript('document.body.scrollWidth', int) @@ -354,6 +359,8 @@ class Document(QWebPage): # {{{ return ans[0] if ans[1] else 0.0 if typ == 'string': return unicode(ans.toString()) + if typ in {bool, 'bool'}: + return ans.toBool() return ans def javaScriptConsoleMessage(self, msg, lineno, msgid): @@ -1104,8 +1111,12 @@ class DocumentView(QWebView): # {{{ def fget(self): return self.zoomFactor() def fset(self, val): + oval = self.zoomFactor() self.setZoomFactor(val) - self.magnification_changed.emit(val) + if val != oval: + if self.document.in_paged_mode: + self.document.update_contents_size_for_paged_mode() + self.magnification_changed.emit(val) return property(fget=fget, fset=fset) def magnify_fonts(self, amount=None): From 934ab58b88070f63c75540166d694a717fd37d66 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 17 Mar 2014 15:49:57 +0530 Subject: [PATCH 062/122] Update TIME Fixes #1292474 [Private bug](https://bugs.launchpad.net/calibre/+bug/1292474) --- recipes/time_magazine.recipe | 119 +++++++++++------------------------ 1 file changed, 38 insertions(+), 81 deletions(-) diff --git a/recipes/time_magazine.recipe b/recipes/time_magazine.recipe index b44cb9823b..775819b5e6 100644 --- a/recipes/time_magazine.recipe +++ b/recipes/time_magazine.recipe @@ -13,17 +13,17 @@ from calibre.web.feeds.jsnews import JavascriptRecipe from lxml import html def wait_for_load(browser): - # This element is present in the black login bar at the top - browser.wait_for_element('#site-header p.constrain', timeout=180) + # This element is present next to the main TIME logo in the left hand side nav bar + browser.wait_for_element('.signedin-wrap a[href]', timeout=180) # Keep the login method as standalone, so it can be easily tested def do_login(browser, username, password): from calibre.web.jsbrowser.browser import Timeout - browser.visit('http://www.time.com/time/magazine') - form = browser.select_form('#magazine-signup') + browser.visit('http://time.com/magazine') + form = browser.select_form('#sign-in-form') form['username'] = username form['password'] = password - browser.submit('#paid-wall-submit') + browser.submit('#Sign_In') try: wait_for_load(browser) except Timeout: @@ -40,100 +40,57 @@ class Time(JavascriptRecipe): no_stylesheets = True remove_javascript = True - keep_only_tags = ['article.post'] - remove_tags = ['meta', '.entry-sharing', '.entry-footer', '.wp-paginate', - '.post-rail', '.entry-comments', '.entry-tools', - '#paid-wall-cm-ad'] - - recursions = 1 - links_from_selectors = ['.wp-paginate a.page[href]'] - - extra_css = '.entry-date { padding-left: 2ex }' + keep_only_tags = ['.article-viewport .full-article'] + remove_tags = ['.read-more-list', '.read-more-inline', '.article-footer', '.subscribe', '.tooltip', '#first-visit'] def do_login(self, browser, username, password): do_login(browser, username, password) - def get_publication_data(self, browser): - selector = 'section.sec-mag-showcase ul.ul-mag-showcase img[src]' + def get_time_cover(self, browser): + selector = '#rail-articles img.magazine-thumb' cover = browser.css_select(selector) # URL for large cover - cover_url = unicode(cover.evaluateJavaScript('this.src').toString()).replace('_400.', '_600.') - raw = browser.html - ans = {'cover': browser.get_resource(cover_url)} + cover_url = unicode(cover.evaluateJavaScript('this.src').toString()).partition('?')[0] + '?w=814' + return browser.get_resource(cover_url) + + def get_publication_data(self, browser): # We are already at the magazine page thanks to the do_login() method + ans = {} + raw = browser.html root = html.fromstring(raw) - dates = ''.join(root.xpath('//time[@class="updated"]/text()')) + dates = ''.join(root.xpath('//*[@class="rail-article-magazine-issue"]/date/text()')) if dates: self.timefmt = ' [%s]'%dates - feeds = [] - parent = root.xpath('//div[@class="content-main-aside"]')[0] - for sec in parent.xpath( - 'descendant::section[contains(@class, "sec-mag-section")]'): - h3 = sec.xpath('./h3') - if h3: - section = html.tostring(h3[0], encoding=unicode, - method='text').strip().capitalize() - self.log('Found section', section) - articles = list(self.find_articles(sec)) - if articles: - feeds.append((section, articles)) + parent = root.xpath('//section[@id="rail-articles"]')[0] + articles = [] + for h3 in parent.xpath( + 'descendant::h3[contains(@class, "rail-article-title")]'): + title = html.tostring(h3[0], encoding=unicode, method='text').strip() + a = h3.xpath('descendant::a[@href]')[0] + url = a.get('href') + h2 = h3.xpath('following-sibling::h2[@class="rail-article-excerpt"]') + desc = '' + if h2: + desc = html.tostring(h2[0], encoding=unicode, method='text').strip() + self.log('\nFound article:', title) + self.log('\t' + desc) + articles.append({'title':title, 'url':url, 'date':'', 'description':desc}) - ans['index'] = feeds + ans['index'] = [('Articles', articles)] + ans['cover'] = self.get_time_cover(browser) return ans - def find_articles(self, sec): - for article in sec.xpath('./article'): - h2 = article.xpath('./*[@class="entry-title"]') - if not h2: - continue - a = h2[0].xpath('./a[@href]') - if not a: - continue - title = html.tostring(a[0], encoding=unicode, - method='text').strip() - if not title: - continue - url = a[0].get('href') - if url.startswith('/'): - url = 'http://www.time.com'+url - desc = '' - p = article.xpath('./*[@class="entry-content"]') - if p: - desc = html.tostring(p[0], encoding=unicode, - method='text') - self.log('\t', title, ':\n\t\t', url) - yield { - 'title' : title, - 'url' : url, - 'date' : '', - 'description' : desc - } - - def load_complete(self, browser, url, recursion_level): - # This is needed as without it, subscriber content is blank. time.com - # appears to be using some crazy iframe+js callback for loading content - wait_for_load(browser) + def load_complete(self, browser, url, rl): + browser.wait_for_element('footer.article-footer') return True def postprocess_html(self, article, root, url, recursion_level): - # Remove the header and page n of m messages from pages after the first - # page - if recursion_level > 0: - for h in root.xpath('//header[@class="entry-header"]|//span[@class="page"]'): - h.getparent().remove(h) - # Unfloat the article images and also remove them from pages after the - # first page as they are repeated on every page. - for fig in root.xpath('//figure'): - parent = fig.getparent() - if recursion_level > 0: - parent.remove(fig) - else: - idx = parent.index(fig) - for img in reversed(fig.xpath('descendant::img')): - parent.insert(idx, img) - parent.remove(fig) + # get rid of the first visit div which for some reason remove_tags is + # not removing + for div in root.xpath('//*[@id="first-visit"]'): + div.getparent().remove(div) return root if __name__ == '__main__': From ca1fdf47b8b5de49d442c4e392625260af7317d0 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 17 Mar 2014 19:36:50 +0530 Subject: [PATCH 063/122] ... --- resources/default_tweaks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/default_tweaks.py b/resources/default_tweaks.py index 207137914c..58901ac6f2 100644 --- a/resources/default_tweaks.py +++ b/resources/default_tweaks.py @@ -147,7 +147,7 @@ sort_columns_at_startup = None # d the day as number without a leading zero (1 to 31) # dd the day as number with a leading zero (01 to 31) # ddd the abbreviated localized day name (e.g. 'Mon' to 'Sun'). -# dddd the long localized day name (e.g. 'Monday' to 'Qt::Sunday'). +# dddd the long localized day name (e.g. 'Monday' to 'Sunday'). # M the month as number without a leading zero (1-12) # MM the month as number with a leading zero (01-12) # MMM the abbreviated localized month name (e.g. 'Jan' to 'Dec'). From 0301890d595f6b37cf0b3a435711adfe8d65249e Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 17 Mar 2014 22:06:22 +0530 Subject: [PATCH 064/122] Edit book: When generating inline Table of Contents, mark it as such in the guide section of the OPF. Fixes #1287018 [ebook-edit should add coresponding guide reference element after adding inline toc.xhtml](https://bugs.launchpad.net/calibre/+bug/1287018) --- src/calibre/ebooks/oeb/polish/opf.py | 45 ++++++++++++++++++++++++++++ src/calibre/ebooks/oeb/polish/toc.py | 12 +++++++- 2 files changed, 56 insertions(+), 1 deletion(-) create mode 100644 src/calibre/ebooks/oeb/polish/opf.py diff --git a/src/calibre/ebooks/oeb/polish/opf.py b/src/calibre/ebooks/oeb/polish/opf.py new file mode 100644 index 0000000000..416fefd400 --- /dev/null +++ b/src/calibre/ebooks/oeb/polish/opf.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python +# vim:fileencoding=utf-8 +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +__license__ = 'GPL v3' +__copyright__ = '2014, Kovid Goyal ' + +from lxml import etree + +from calibre.ebooks.oeb.polish.container import OPF_NAMESPACES +from calibre.utils.localization import canonicalize_lang + +def get_book_language(container): + for lang in container.opf_xpath('//dc:language'): + raw = lang.text + if raw: + code = canonicalize_lang(raw.split(',')[0].strip()) + if code: + return code + +def set_guide_item(container, item_type, title, name, frag=None): + guides = container.opf_xpath('//opf:guide') + if not guides: + g = container.opf.makeelement('{%s}guide' % OPF_NAMESPACES['opf'], nsmap={'opf':OPF_NAMESPACES['opf']}) + container.insert_into_xml(container.opf, g) + guides = [g] + ref_tag = '{%s}reference' % OPF_NAMESPACES['opf'] + href = container.name_to_href(name, container.opf_name) + if frag: + href += '#' + frag + + for guide in guides: + matches = [] + for child in guide.iterchildren(etree.Element): + if child.tag == ref_tag and child.get('type', '').lower() == item_type.lower(): + matches.append(child) + if not matches: + r = guide.makeelement(ref_tag, type=item_type, nsmap={'opf':OPF_NAMESPACES['opf']}) + container.insert_into_xml(guide, r) + matches.append(r) + for m in matches: + m.set('title', title), m.set('href', href), m.set('type', item_type) + container.dirty(container.opf_name) + diff --git a/src/calibre/ebooks/oeb/polish/toc.py b/src/calibre/ebooks/oeb/polish/toc.py index ac87b7ae66..fe9766ff34 100644 --- a/src/calibre/ebooks/oeb/polish/toc.py +++ b/src/calibre/ebooks/oeb/polish/toc.py @@ -20,7 +20,9 @@ from calibre import __version__ from calibre.ebooks.oeb.base import XPath, uuid_id, xml2text, NCX, NCX_NS, XML, XHTML, XHTML_NS, serialize from calibre.ebooks.oeb.polish.errors import MalformedMarkup from calibre.ebooks.oeb.polish.utils import guess_type +from calibre.ebooks.oeb.polish.opf import set_guide_item, get_book_language from calibre.ebooks.oeb.polish.pretty import pretty_html_tree +from calibre.translations.dynamic import translate from calibre.utils.localization import get_lang, canonicalize_lang, lang_as_iso639_1 ns = etree.FunctionNamespace('calibre_xpath_extensions') @@ -481,7 +483,12 @@ def find_inline_toc(container): return name def create_inline_toc(container, title=None): - title = title or _('Table of Contents') + lang = get_book_language(container) + default_title = 'Table of Contents' + if lang: + lang = lang_as_iso639_1(lang) or lang + default_title = translate(lang, default_title) + title = title or default_title toc = get_toc(container) if len(toc) == 0: return None @@ -529,6 +536,8 @@ def create_inline_toc(container, title=None): name = toc_name for child in toc: process_node(html[1][1], child) + if lang: + html.set('lang', lang) pretty_html_tree(container, html) raw = serialize(html, 'text/html') if name is None: @@ -540,5 +549,6 @@ def create_inline_toc(container, title=None): else: with container.open(name, 'wb') as f: f.write(raw) + set_guide_item(container, 'toc', title, name, frag='calibre_generated_inline_toc') return name From 16c3101490838ccf336e5d5f89634973e12bfaa5 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 18 Mar 2014 09:37:42 +0530 Subject: [PATCH 065/122] Remember no more than 100 search expressions in the history --- src/calibre/gui2/tweak_book/search.py | 2 ++ src/calibre/gui2/widgets2.py | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/src/calibre/gui2/tweak_book/search.py b/src/calibre/gui2/tweak_book/search.py index ca29189046..31386fc4a1 100644 --- a/src/calibre/gui2/tweak_book/search.py +++ b/src/calibre/gui2/tweak_book/search.py @@ -27,6 +27,8 @@ class PushButton(QPushButton): class HistoryLineEdit(HistoryLineEdit2): + max_history_items = 100 + def __init__(self, parent, clear_msg): HistoryLineEdit2.__init__(self, parent) self.disable_popup = tprefs['disable_completion_popup_for_search'] diff --git a/src/calibre/gui2/widgets2.py b/src/calibre/gui2/widgets2.py index 3a52d72078..fe8a1f1b3d 100644 --- a/src/calibre/gui2/widgets2.py +++ b/src/calibre/gui2/widgets2.py @@ -11,6 +11,8 @@ from calibre.gui2.widgets import history class HistoryLineEdit2(LineEdit): + max_history_items = None + @property def store_name(self): return 'lineedit_history_'+self._name @@ -31,6 +33,8 @@ class HistoryLineEdit2(LineEdit): except ValueError: pass self.history.insert(0, ct) + if self.max_history_items is not None: + del self.history[self.max_history_items:] history.set(self.store_name, self.history) self.update_items_cache(self.history) From c53e46512e46fd95d5edd67e94d05e32e3e61df0 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 18 Mar 2014 10:01:07 +0530 Subject: [PATCH 066/122] Sort items in completion popups for history by MRU --- src/calibre/gui2/complete2.py | 13 +++++++------ src/calibre/gui2/widgets2.py | 3 +++ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/calibre/gui2/complete2.py b/src/calibre/gui2/complete2.py index 5260e265a5..34e7d576b7 100644 --- a/src/calibre/gui2/complete2.py +++ b/src/calibre/gui2/complete2.py @@ -24,15 +24,16 @@ def containsq(x, prefix): class CompleteModel(QAbstractListModel): # {{{ - def __init__(self, parent=None): + def __init__(self, parent=None, sort_func=sort_key): QAbstractListModel.__init__(self, parent) + self.sort_func = sort_func self.all_items = self.current_items = () self.current_prefix = '' def set_items(self, items): items = [unicode(x.strip()) for x in items] items = [x for x in items if x] - items = tuple(sorted(items, key=sort_key)) + items = tuple(sorted(items, key=self.sort_func)) self.all_items = self.current_items = items self.current_prefix = '' self.reset() @@ -74,7 +75,7 @@ class Completer(QListView): # {{{ item_selected = pyqtSignal(object) relayout_needed = pyqtSignal() - def __init__(self, completer_widget, max_visible_items=7): + def __init__(self, completer_widget, max_visible_items=7, sort_func=sort_key): QListView.__init__(self) self.disable_popup = False self.completer_widget = weakref.ref(completer_widget) @@ -85,7 +86,7 @@ class Completer(QListView): # {{{ self.setSelectionBehavior(self.SelectRows) self.setSelectionMode(self.SingleSelection) self.setAlternatingRowColors(True) - self.setModel(CompleteModel(self)) + self.setModel(CompleteModel(self, sort_func=sort_func)) self.setMouseTracking(True) self.entered.connect(self.item_entered) self.activated.connect(self.item_chosen) @@ -256,7 +257,7 @@ class LineEdit(QLineEdit, LineEditECM): to complete non multiple fields as well. ''' - def __init__(self, parent=None, completer_widget=None): + def __init__(self, parent=None, completer_widget=None, sort_func=sort_key): QLineEdit.__init__(self, parent) self.sep = ',' @@ -266,7 +267,7 @@ class LineEdit(QLineEdit, LineEditECM): completer_widget = (self if completer_widget is None else completer_widget) - self.mcompleter = Completer(completer_widget) + self.mcompleter = Completer(completer_widget, sort_func=sort_func) self.mcompleter.item_selected.connect(self.completion_selected, type=Qt.QueuedConnection) self.mcompleter.relayout_needed.connect(self.relayout) diff --git a/src/calibre/gui2/widgets2.py b/src/calibre/gui2/widgets2.py index fe8a1f1b3d..8d801a91bd 100644 --- a/src/calibre/gui2/widgets2.py +++ b/src/calibre/gui2/widgets2.py @@ -13,6 +13,9 @@ class HistoryLineEdit2(LineEdit): max_history_items = None + def __init__(self, parent=None, completer_widget=None, sort_func=lambda x:None): + LineEdit.__init__(self, parent=parent, completer_widget=completer_widget, sort_func=sort_func) + @property def store_name(self): return 'lineedit_history_'+self._name From f973436000aa298c7e514802bc99af1b112d818d Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 18 Mar 2014 13:18:07 +0530 Subject: [PATCH 067/122] Edit Book: New tool to specify semantics in EPUB books (semantics are items in the guide such as preface, title-page, dedication, etc.). Fixes #1287025 [[ebook-edit] Implement of defining more reference types of guide section](https://bugs.launchpad.net/calibre/+bug/1287025) --- src/calibre/ebooks/oeb/polish/opf.py | 20 ++- src/calibre/gui2/tweak_book/boss.py | 14 +- src/calibre/gui2/tweak_book/ui.py | 3 + src/calibre/gui2/tweak_book/widgets.py | 208 ++++++++++++++++++++++++- 4 files changed, 233 insertions(+), 12 deletions(-) diff --git a/src/calibre/ebooks/oeb/polish/opf.py b/src/calibre/ebooks/oeb/polish/opf.py index 416fefd400..45af043cf0 100644 --- a/src/calibre/ebooks/oeb/polish/opf.py +++ b/src/calibre/ebooks/oeb/polish/opf.py @@ -20,26 +20,32 @@ def get_book_language(container): return code def set_guide_item(container, item_type, title, name, frag=None): + ref_tag = '{%s}reference' % OPF_NAMESPACES['opf'] + href = None + if name: + href = container.name_to_href(name, container.opf_name) + if frag: + href += '#' + frag + guides = container.opf_xpath('//opf:guide') - if not guides: + if not guides and href: g = container.opf.makeelement('{%s}guide' % OPF_NAMESPACES['opf'], nsmap={'opf':OPF_NAMESPACES['opf']}) container.insert_into_xml(container.opf, g) guides = [g] - ref_tag = '{%s}reference' % OPF_NAMESPACES['opf'] - href = container.name_to_href(name, container.opf_name) - if frag: - href += '#' + frag for guide in guides: matches = [] for child in guide.iterchildren(etree.Element): if child.tag == ref_tag and child.get('type', '').lower() == item_type.lower(): matches.append(child) - if not matches: + if not matches and href: r = guide.makeelement(ref_tag, type=item_type, nsmap={'opf':OPF_NAMESPACES['opf']}) container.insert_into_xml(guide, r) matches.append(r) for m in matches: - m.set('title', title), m.set('href', href), m.set('type', item_type) + if href: + m.set('title', title), m.set('href', href), m.set('type', item_type) + else: + container.remove_from_xml(m) container.dirty(container.opf_name) diff --git a/src/calibre/gui2/tweak_book/boss.py b/src/calibre/gui2/tweak_book/boss.py index c5fbfeccf5..c7ad6d53f7 100644 --- a/src/calibre/gui2/tweak_book/boss.py +++ b/src/calibre/gui2/tweak_book/boss.py @@ -38,7 +38,7 @@ from calibre.gui2.tweak_book.editor import editor_from_syntax, syntax_from_mime from calibre.gui2.tweak_book.editor.insert_resource import get_resource_data, NewBook from calibre.gui2.tweak_book.preferences import Preferences from calibre.gui2.tweak_book.widgets import ( - RationalizeFolders, MultiSplit, ImportForeign, QuickOpen, InsertLink) + RationalizeFolders, MultiSplit, ImportForeign, QuickOpen, InsertLink, InsertSemantics) _diff_dialogs = [] @@ -649,6 +649,18 @@ class Boss(QObject): else: ed.action_triggered(action) + def set_semantics(self): + self.commit_all_editors_to_container() + c = current_container() + if c.book_type == 'azw3': + return error_dialog(self.gui, _('Not supported'), _( + 'Semantics are not supported for the AZW3 format.'), show=True) + d = InsertSemantics(c, parent=self.gui) + if d.exec_() == d.Accepted and d.changed_type_map: + self.add_savepoint(_('Before: Set Semantics')) + d.apply_changes(current_container()) + self.apply_container_update_to_gui() + def show_find(self): self.gui.central.show_find() ed = self.gui.central.current_editor diff --git a/src/calibre/gui2/tweak_book/ui.py b/src/calibre/gui2/tweak_book/ui.py index 935e8ba5d5..f478738b1a 100644 --- a/src/calibre/gui2/tweak_book/ui.py +++ b/src/calibre/gui2/tweak_book/ui.py @@ -343,6 +343,8 @@ class Main(MainWindow): _('Insert special character')) self.action_rationalize_folders = reg('mimetypes/dir.png', _('&Arrange into folders'), self.boss.rationalize_folders, 'rationalize-folders', (), _('Arrange into folders')) + self.action_set_semantics = reg('tags.png', _('Set &Semantics'), self.boss.set_semantics, 'set-semantics', (), + _('Set Semantics')) # Polish actions group = _('Polish Book') @@ -472,6 +474,7 @@ class Main(MainWindow): e.addAction(self.action_fix_html_all) e.addAction(self.action_pretty_all) e.addAction(self.action_rationalize_folders) + e.addAction(self.action_set_semantics) e.addAction(self.action_check_book) e = b.addMenu(_('&View')) diff --git a/src/calibre/gui2/tweak_book/widgets.py b/src/calibre/gui2/tweak_book/widgets.py index 0ca22d1832..9a0b9df508 100644 --- a/src/calibre/gui2/tweak_book/widgets.py +++ b/src/calibre/gui2/tweak_book/widgets.py @@ -8,18 +8,19 @@ __copyright__ = '2014, Kovid Goyal ' import os from itertools import izip +from collections import OrderedDict from PyQt4.Qt import ( QDialog, QDialogButtonBox, QGridLayout, QLabel, QLineEdit, QVBoxLayout, QFormLayout, QHBoxLayout, QToolButton, QIcon, QApplication, Qt, QWidget, QPoint, QSizePolicy, QPainter, QStaticText, pyqtSignal, QTextOption, QAbstractListModel, QModelIndex, QVariant, QStyledItemDelegate, QStyle, - QListView, QTextDocument, QSize) + QListView, QTextDocument, QSize, QComboBox, QFrame) from calibre import prepare_string_for_xml -from calibre.gui2 import error_dialog, choose_files, choose_save_file, NONE +from calibre.gui2 import error_dialog, choose_files, choose_save_file, NONE, info_dialog from calibre.gui2.tweak_book import tprefs -from calibre.utils.icu import primary_sort_key +from calibre.utils.icu import primary_sort_key, sort_key from calibre.utils.matcher import get_char, Matcher ROOT = QModelIndex() @@ -520,6 +521,11 @@ class NamesModel(QAbstractListModel): self.reset() self.filtered.emit(not bool(query)) + def find_name(self, name): + for i, (text, positions) in enumerate(self.items): + if text == name: + return i + def create_filterable_names_list(names, filter_text=None, parent=None): nl = QListView(parent) nl.m = m = NamesModel(names, parent=nl) @@ -642,6 +648,200 @@ class InsertLink(Dialog): # }}} +# Insert Semantics {{{ + +class InsertSemantics(Dialog): + + def __init__(self, container, parent=None): + self.container = container + self.anchor_cache = {} + self.original_type_map = {item.get('type', ''):(container.href_to_name(item.get('href'), container.opf_name), item.get('href', '').partition('#')[-1]) + for item in container.opf_xpath('//opf:guide/opf:reference[@href and @type]')} + self.final_type_map = self.original_type_map.copy() + self.create_known_type_map() + Dialog.__init__(self, _('Set Semantics'), 'insert-semantics', parent=parent) + + def sizeHint(self): + return QSize(800, 600) + + def create_known_type_map(self): + _ = lambda x: x + self.known_type_map = { + 'title-page': _('Title Page'), + 'toc': _('Table of Contents'), + 'index': _('Index'), + 'glossary': _('Glossary'), + 'acknowledgements': _('Acknowledgements'), + 'bibliography': _('Bibliography'), + 'colophon': _('Colophon'), + 'copyright-page': _('Copyright page'), + 'dedication': _('Dedication'), + 'epigraph': _('Epigraph'), + 'foreword': _('Foreword'), + 'loi': _('List of Illustrations'), + 'lot': _('List of Tables'), + 'notes:': _('Notes'), + 'preface': _('Preface'), + 'text': _('Text'), + } + _ = __builtins__['_'] + type_map_help = { + 'title-page': _('Page with title, author, publisher, etc.'), + 'index': _('Back-of-book style index'), + 'text': _('First "real" page of content'), + } + t = _ + all_types = [(k, (('%s (%s)' % (t(v), type_map_help[k])) if k in type_map_help else t(v))) for k, v in self.known_type_map.iteritems()] + all_types.sort(key=lambda x: sort_key(x[1])) + self.all_types = OrderedDict(all_types) + + def setup_ui(self): + self.l = l = QVBoxLayout(self) + self.setLayout(l) + + self.tl = tl = QFormLayout() + self.semantic_type = QComboBox(self) + for key, val in self.all_types.iteritems(): + self.semantic_type.addItem(val, key) + tl.addRow(_('Type of &semantics:'), self.semantic_type) + self.target = t = QLineEdit(self) + t.setPlaceholderText(_('The destination (href) for the link')) + tl.addRow(_('&Target:'), t) + l.addLayout(tl) + + self.hline = hl = QFrame(self) + hl.setFrameStyle(hl.HLine) + l.addWidget(hl) + + self.h = h = QHBoxLayout() + l.addLayout(h) + + names = [n for n, linear in self.container.spine_names] + fn, f = create_filterable_names_list(names, filter_text=_('Filter files'), parent=self) + self.file_names, self.file_names_filter = fn, f + fn.selectionModel().selectionChanged.connect(self.selected_file_changed) + self.fnl = fnl = QVBoxLayout() + self.la1 = la = QLabel(_('Choose a &file:')) + la.setBuddy(fn) + fnl.addWidget(la), fnl.addWidget(f), fnl.addWidget(fn) + h.addLayout(fnl), h.setStretch(0, 2) + + fn, f = create_filterable_names_list([], filter_text=_('Filter locations'), parent=self) + self.anchor_names, self.anchor_names_filter = fn, f + fn.selectionModel().selectionChanged.connect(self.update_target) + fn.doubleClicked.connect(self.accept, type=Qt.QueuedConnection) + self.anl = fnl = QVBoxLayout() + self.la2 = la = QLabel(_('Choose a &location (anchor) in the file:')) + la.setBuddy(fn) + fnl.addWidget(la), fnl.addWidget(f), fnl.addWidget(fn) + h.addLayout(fnl), h.setStretch(1, 1) + + self.bb.addButton(self.bb.Help) + self.bb.helpRequested.connect(self.help_requested) + l.addWidget(self.bb) + self.semantic_type_changed() + self.semantic_type.currentIndexChanged.connect(self.semantic_type_changed) + self.target.textChanged.connect(self.target_text_changed) + + def help_requested(self): + d = info_dialog(self, _('About semantics'), _( + 'Semantics refer to additional information about specific locations in the book.' + ' For example, you can specify that a particular location is the dedication or the preface' + ' or the table of contents and so on.\n\nFirst choose the type of semantic information, then' + ' choose a file and optionally a location within the file to point to.\n\nThe' + ' semantic information will be written in the section of the opf file.')) + d.resize(d.sizeHint()) + d.exec_() + + def semantic_type_changed(self): + item_type = unicode(self.semantic_type.itemData(self.semantic_type.currentIndex()).toString()) + name, frag = self.final_type_map.get(item_type, (None, None)) + self.show_type(name, frag) + + def show_type(self, name, frag): + self.file_names_filter.clear(), self.anchor_names_filter.clear() + self.file_names.clearSelection(), self.anchor_names.clearSelection() + if name is not None: + row = self.file_names.model().find_name(name) + if row is not None: + sm = self.file_names.selectionModel() + sm.select(self.file_names.model().index(row), sm.ClearAndSelect) + if frag: + row = self.anchor_names.model().find_name(frag) + if row is not None: + sm = self.anchor_names.selectionModel() + sm.select(self.anchor_names.model().index(row), sm.ClearAndSelect) + self.target.blockSignals(True) + if name is not None: + self.target.setText(name + (('#' + frag) if frag else '')) + else: + self.target.setText('') + self.target.blockSignals(False) + + def target_text_changed(self): + name, frag = unicode(self.target.text()).partition('#')[::2] + item_type = unicode(self.semantic_type.itemData(self.semantic_type.currentIndex()).toString()) + self.final_type_map[item_type] = (name, frag or None) + + def selected_file_changed(self, *args): + rows = list(self.file_names.selectionModel().selectedRows()) + if not rows: + self.anchor_names.model().set_names([]) + else: + name, positions = self.file_names.model().data(rows[0], Qt.UserRole).toPyObject() + self.populate_anchors(name) + + def populate_anchors(self, name): + if name not in self.anchor_cache: + from calibre.ebooks.oeb.base import XHTML_NS + root = self.container.parsed(name) + self.anchor_cache[name] = sorted( + (set(root.xpath('//*/@id')) | set(root.xpath('//h:a/@name', namespaces={'h':XHTML_NS}))) - {''}, key=primary_sort_key) + self.anchor_names.model().set_names(self.anchor_cache[name]) + self.update_target() + + def update_target(self): + rows = list(self.file_names.selectionModel().selectedRows()) + if not rows: + return + name = self.file_names.model().data(rows[0], Qt.UserRole).toPyObject()[0] + href = name + frag = '' + rows = list(self.anchor_names.selectionModel().selectedRows()) + if rows: + anchor = self.anchor_names.model().data(rows[0], Qt.UserRole).toPyObject()[0] + if anchor: + frag = '#' + anchor + href += frag + self.target.setText(href or '#') + + @property + def changed_type_map(self): + return {k:v for k, v in self.final_type_map.iteritems() if v != self.original_type_map.get(k, None)} + + def apply_changes(self, container): + from calibre.ebooks.oeb.polish.opf import set_guide_item, get_book_language + from calibre.translations.dynamic import translate + lang = get_book_language(container) + for item_type, (name, frag) in self.changed_type_map.iteritems(): + title = self.known_type_map[item_type] + if lang: + title = translate(lang, title) + set_guide_item(container, item_type, title, name, frag=frag) + + @classmethod + def test(cls): + import sys + from calibre.ebooks.oeb.polish.container import get_container + c = get_container(sys.argv[-1], tweak_mode=True) + d = cls(c) + if d.exec_() == d.Accepted: + import pprint + pprint.pprint(d.changed_type_map) + d.apply_changes(d.container) + +# }}} + if __name__ == '__main__': app = QApplication([]) - InsertLink.test() + InsertSemantics.test() From f36389d5289a20d877c963bd92b791986f7f4242 Mon Sep 17 00:00:00 2001 From: Charles Haley Date: Tue, 18 Mar 2014 09:45:04 +0100 Subject: [PATCH 068/122] Wireless device: remove books from the metadata cache that are no longer on the device. --- .../devices/smart_device_app/driver.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/calibre/devices/smart_device_app/driver.py b/src/calibre/devices/smart_device_app/driver.py index 81f41db06b..fdabf4d854 100644 --- a/src/calibre/devices/smart_device_app/driver.py +++ b/src/calibre/devices/smart_device_app/driver.py @@ -1223,6 +1223,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): books_on_device.append(result) books_to_send = [] + lpaths_on_device = set() for r in books_on_device: if r.get('lpath', None): book = self._metadata_in_cache(r['uuid'], r['lpath'], @@ -1231,6 +1232,8 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): book = self._metadata_in_cache(r['uuid'], r['extension'], r['last_modified']) if book: + if self.client_cache_uses_lpaths: + lpaths_on_device.add(r.get('lpath')) bl.add_book(book, replace_metadata=True) book.set('_is_read_', r.get('_is_read_', None)) book.set('_sync_type_', r.get('_sync_type_', None)) @@ -1238,6 +1241,22 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): else: books_to_send.append(r['priKey']) + count_of_cache_items_deleted = 0 + if self.client_cache_uses_lpaths: + for lpath in self.known_metadata.keys(): + if lpath not in lpaths_on_device: + try: + uuid = self.known_metadata[lpath].get('uuid', None) + if uuid is not None: + key = self._make_metadata_cache_key(uuid, lpath) + self.device_book_cache.pop(key, None) + self.known_metadata.pop(lpath, None) + count_of_cache_items_deleted += 1 + except: + self._debug('Exception while deleting book from caches', lpath) + traceback.print_exc() + self._debug('removed', count_of_cache_items_deleted, 'books from caches') + count = len(books_to_send) self._debug('caching. Need count from device', count) From 8e2066b9df47021632cc04e31548c75a4c8c8c9f Mon Sep 17 00:00:00 2001 From: Charles Haley Date: Tue, 18 Mar 2014 09:47:04 +0100 Subject: [PATCH 069/122] Update the CC production release version number --- src/calibre/devices/smart_device_app/driver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/devices/smart_device_app/driver.py b/src/calibre/devices/smart_device_app/driver.py index fdabf4d854..cfece66235 100644 --- a/src/calibre/devices/smart_device_app/driver.py +++ b/src/calibre/devices/smart_device_app/driver.py @@ -226,7 +226,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): PURGE_CACHE_ENTRIES_DAYS = 30 - CURRENT_CC_VERSION = 64 + CURRENT_CC_VERSION = 73 ZEROCONF_CLIENT_STRING = b'calibre wireless device client' From 130965ba41d793f5c02f06b37f7d33ee1a150969 Mon Sep 17 00:00:00 2001 From: Piotr Parafiniuk Date: Tue, 18 Mar 2014 18:53:25 +0100 Subject: [PATCH 070/122] applefobia recipe --- recipes/applefobia.recipe | 16 ++++++++++++++++ recipes/icons/applefobia.png | Bin 0 -> 4353 bytes 2 files changed, 16 insertions(+) create mode 100644 recipes/applefobia.recipe create mode 100644 recipes/icons/applefobia.png diff --git a/recipes/applefobia.recipe b/recipes/applefobia.recipe new file mode 100644 index 0000000000..78e4b79b21 --- /dev/null +++ b/recipes/applefobia.recipe @@ -0,0 +1,16 @@ +class BasicUserRecipe1395137685(AutomaticNewsRecipe): + title = u'Applefobia' + oldest_article = 7 + max_articles_per_feed = 100 + auto_cleanup = True + language = 'pl' # dwuliterowe oznaczenie jzyka + remove_empty_feeds = True # usu puste wiadomoci + remove_javascript = True # usu skrypty JavaScript + conversion_options = { 'tags' : u'newsy, Apple, humor', + 'smarten_punctuation' : True, + 'authors' : 'Ogrodnik January', + 'publisher' : 'Blogspot.pl' + } # opcje konwersji. Tu mona ustawi warto pl w metadanych ksiki. + reverse_article_order = True # odwraca domyln kolejno i czytamy od najstarszej do najnowszej a nie odwrotnie. + + feeds = [(u'Aktualne', u'http://applefobia.blogspot.com/feeds/posts/default')] diff --git a/recipes/icons/applefobia.png b/recipes/icons/applefobia.png new file mode 100644 index 0000000000000000000000000000000000000000..6cb38f329841bd70db987dce574197ddd86dc0a8 GIT binary patch literal 4353 zcmeH~SyU5Q8pjI(37agk2?`-B!A25x3=-C`1&C}m5yVMILJ}a0NiYFhjG)peAOa%F zE;_id7*G&I5dqOwOtT2estAIJXd|l_nBaJNUgpJzd6;vob8g+LTmSmicmH+IcfOnA zkMUHL)0P7OK+)R^6)5hN*BfYq_?&TuVhaG$dhx+nP9UBOX0jNO)EE+&6UQWhNjz#K z0Py-NLc%iNn<$9L>NQe^`npeE@3q;LJ1%U-K@W7C=!gZ3~kOb zoXtkf#W}sRI{0>K`2L-EyXlx=R^0Ii*e3O_ilt`IyN}0NE! z$V?(1Ees`3qZQ?7IsocZMXrLZ){R2c<1EFeM?pJ1K4n?3HQJ>WGJdhMlpOq>mys01 zXQawTro3!^hw!2=}j2nksI4-3tQtRJoKIckFNcJatzoq4&yG!`sEp z$;uI4`kpDUs5y&Pt)QnVS;fbwv)gmkg3yQ->WO!$om;dKny|oqoufV`f=ijE56T*@ zJ@4};_}SReyE0Zg>TsbkSI05iN_xWkW)4jX$_iy7Xpfj~$$I$KGamP_1LgOsJ?!2l zbZFChjR`Y<$}9DPH@&uYsmY%lQ%DsUnClu#f+&UgFTI}qMWnMLHydovZq|NhSk_5Y zP-wuEC&6ZPpNcZ=N@wbLZ;EoQvKrQs5LQ zjc@Bo_im=|FXQOos-=yh`%8G!yo%bZ#T7Ei4cFfXb@UwG=XPE6>d9}cRpO_9`BqsivO;k}%Efyq8*d_!b~gR zjUKSBCAIE%u1#CwDZ^$VecUK@mrr>d@I@6Ue{)Z*01XAX+j&$b$K0wKN2EJRk0Iq} zJ|7YenmT5PRuKy`lT(tARiLv{J&W?0FzdU?u5Gs9R8;xwDL)zrujZo2iEEP6#A>65otkczz_kgQAV+w@sIf8n!dyj#`CUu5GxqI5L5`lUFCsnE3yi zDyd8kVLeqMQv`~9X9dX>-)4|^Ov;T-%O}zkMaS2SZm93-eHrUS8&K``_to8sSq@31 z?4FL7m-XbMuSB2GKuo9>Xy&}hn_1XkX{Oo70qa4_QMU1o$M0tEii0_aP3WvkAhr_vCE{tmhW3b3@3j_iIH@AdaTAGO!X6!gR z2hTI3vkle}|7M^_YywMc9hE@`uQTxxj93m50uet4e}@wZf3umfte9_PA^}c{A<;;5 z4jXO(vw(lo6AAVn3>FP9exFLilSyzUoeY7mM+^TRu6XwFzo)+*!~g#H`H>&;_@`Yz z?fM}Hen|Xtb^WyKhaC7J@z2%u|7Dlle+noPU0gMB#pMyO#&;H%RmL9hk?D#3LF6Cc!0*H1hX4c{euyh+pFYN7%;A3@U|jH8cK$gnjJoooa6s<<$mzJXQ4`!9ka za;C0DHgA+?lb*OIfMiacr=?&8}(f z<|c+-2VT^Mt(m_SWIL?&$DQB$YcGv*!TbRsro7Tj@=W)9R?Rm482S*-oV&Z}DVjKT zqPYVwv~P{5O23&fqyaAHurGBQdElM1F%CsXzSayKQ?nef74JK|AJw0_7F(sM`)IR(rBuQ+=70q{^0_SWf>X|w%6ehdmuw%;86pSULly?tP{%`@ zHsm_2tv0Fm{ZmPuVOW_yxK`~j62iP}5yRE%9qB2|)m~i?1#K#7ykob#Fq>B9+1GD2 j;A1MkIjBV=b5%myxuC!1O55+^9|m~4V^Ec@xWs<}pZo~< literal 0 HcmV?d00001 From 6e81aa844261e2aeb5f8b59d4ef7df2302bd4f4f Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 19 Mar 2014 09:40:29 +0530 Subject: [PATCH 071/122] pep8 --- recipes/guardian.recipe | 66 ++++++++++++++++++++--------------------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/recipes/guardian.recipe b/recipes/guardian.recipe index 8bff4f9be8..444130a296 100644 --- a/recipes/guardian.recipe +++ b/recipes/guardian.recipe @@ -48,14 +48,14 @@ class Guardian(BasicNewsRecipe): # article history link dict(name='a', attrs={'class':["rollover history-link"]}), # "a version of this article ..." speil - dict(name='div' , attrs = { 'class' : ['section']}), + dict(name='div' , attrs={'class' : ['section']}), # "about this article" js dialog dict(name='div', attrs={'class':["share-top",]}), # author picture dict(name='img', attrs={'class':["contributor-pic-small"]}), # embedded videos/captions dict(name='span',attrs={'class' : ['inline embed embed-media']}), - #dict(name='img'), + # dict(name='img'), ] use_embedded_content = False @@ -72,12 +72,12 @@ class Guardian(BasicNewsRecipe): ''' def get_article_url(self, article): - url = article.get('guid', None) - if '/video/' in url or '/flyer/' in url or '/quiz/' in url or \ - '/gallery/' in url or 'ivebeenthere' in url or \ - 'pickthescore' in url or 'audioslideshow' in url : - url = None - return url + url = article.get('guid', None) + if '/video/' in url or '/flyer/' in url or '/quiz/' in url or \ + '/gallery/' in url or 'ivebeenthere' in url or \ + 'pickthescore' in url or 'audioslideshow' in url : + url = None + return url def populate_article_metadata(self, article, soup, first): if first and hasattr(self, 'add_toc_thumbnail'): @@ -87,39 +87,39 @@ class Guardian(BasicNewsRecipe): def preprocess_html(self, soup): - # multiple html sections in soup, useful stuff in the first - html = soup.find('html') - soup2 = BeautifulSoup() - soup2.insert(0,html) - - soup = soup2 - - for item in soup.findAll(style=True): - del item['style'] + # multiple html sections in soup, useful stuff in the first + html = soup.find('html') + soup2 = BeautifulSoup() + soup2.insert(0,html) - for item in soup.findAll(face=True): - del item['face'] - for tag in soup.findAll(name=['ul','li']): - tag.name = 'div' - - # removes number next to rating stars - items_to_remove = [] - rating_container = soup.find('div', attrs = {'class': ['rating-container']}) - if rating_container: + soup = soup2 + + for item in soup.findAll(style=True): + del item['style'] + + for item in soup.findAll(face=True): + del item['face'] + for tag in soup.findAll(name=['ul','li']): + tag.name = 'div' + + # removes number next to rating stars + items_to_remove = [] + rating_container = soup.find('div', attrs={'class': ['rating-container']}) + if rating_container: for item in rating_container: if isinstance(item, Tag) and str(item.name) == 'span': items_to_remove.append(item) - - for item in items_to_remove: + + for item in items_to_remove: item.extract() - - return soup + + return soup def find_sections(self): # soup = self.index_to_soup("http://www.guardian.co.uk/theobserver") soup = self.index_to_soup(self.base_url) # find cover pic - img = soup.find( 'img',attrs ={'alt':self.cover_pic}) + img = soup.find('img',attrs={'alt':self.cover_pic}) if img is not None: self.cover_url = img['src'] # end find cover pic @@ -149,7 +149,8 @@ class Guardian(BasicNewsRecipe): continue tt = li.find('div', attrs={'class':'trailtext'}) if tt is not None: - for da in tt.findAll('a'): da.extract() + for da in tt.findAll('a'): + da.extract() desc = self.tag_to_string(tt).strip() yield { 'title': title, 'url':url, 'description':desc, @@ -161,4 +162,3 @@ class Guardian(BasicNewsRecipe): for title, href in self.find_sections(): feeds.append((title, list(self.find_articles(href)))) return feeds - From db2aabb34b6c89f54eda14a0ef60ca535a080850 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 19 Mar 2014 09:42:24 +0530 Subject: [PATCH 072/122] ... --- recipes/guardian.recipe | 2 ++ 1 file changed, 2 insertions(+) diff --git a/recipes/guardian.recipe b/recipes/guardian.recipe index 444130a296..533c1f0a27 100644 --- a/recipes/guardian.recipe +++ b/recipes/guardian.recipe @@ -30,6 +30,8 @@ class Guardian(BasicNewsRecipe): max_articles_per_feed = 100 remove_javascript = True encoding = 'utf-8' + compress_news_images = True + compress_news_images_auto_size = 8 # List of section titles to ignore # For example: ['Sport'] From 15802fe3cbc612db5414c2465a8b624cb59e3e8f Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 19 Mar 2014 09:51:26 +0530 Subject: [PATCH 073/122] cleanup applefobia --- recipes/applefobia.recipe | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/recipes/applefobia.recipe b/recipes/applefobia.recipe index 78e4b79b21..9dcc5c2954 100644 --- a/recipes/applefobia.recipe +++ b/recipes/applefobia.recipe @@ -1,16 +1,22 @@ -class BasicUserRecipe1395137685(AutomaticNewsRecipe): +# vim:fileencoding=UTF-8 +from __future__ import unicode_literals +from calibre.web.feeds.news import BasicNewsRecipe + +class BasicUserRecipe1395137685(BasicNewsRecipe): title = u'Applefobia' + __author__ = 'koliberek' oldest_article = 7 max_articles_per_feed = 100 auto_cleanup = True - language = 'pl' # dwuliterowe oznaczenie jzyka - remove_empty_feeds = True # usu puste wiadomoci - remove_javascript = True # usu skrypty JavaScript - conversion_options = { 'tags' : u'newsy, Apple, humor', + language = 'pl' + remove_empty_feeds = True + remove_javascript = True + conversion_options = { + 'tags' : u'newsy, Apple, humor', 'smarten_punctuation' : True, 'authors' : 'Ogrodnik January', 'publisher' : 'Blogspot.pl' - } # opcje konwersji. Tu mona ustawi warto pl w metadanych ksiki. - reverse_article_order = True # odwraca domyln kolejno i czytamy od najstarszej do najnowszej a nie odwrotnie. + } + reverse_article_order = True feeds = [(u'Aktualne', u'http://applefobia.blogspot.com/feeds/posts/default')] From 7256c9bf4e9be72c5cdef74463932c856055218d Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 19 Mar 2014 21:05:41 +0530 Subject: [PATCH 074/122] Fix a regression in the previous release that broke downloading metadata for authors witha double initial such as R. A. Salvatore. Fixes #1294529 [Metadata download fails on author with 2 intials](https://bugs.launchpad.net/calibre/+bug/1294529) --- src/calibre/utils/icu.py | 6 +++++- src/calibre/utils/icu_test.py | 2 ++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/calibre/utils/icu.py b/src/calibre/utils/icu.py index 6f335a5434..1c61d3e739 100644 --- a/src/calibre/utils/icu.py +++ b/src/calibre/utils/icu.py @@ -197,7 +197,11 @@ lower = _make_func(_change_case_template, 'lower', which='LOWER_CASE') title_case = _make_func(_change_case_template, 'title_case', which='TITLE_CASE') -capitalize = lambda x: upper(x[0]) + lower(x[1:]) +def capitalize(x): + try: + return upper(x[0]) + lower(x[1:]) + except (IndexError, TypeError, AttributeError): + return x find = _make_func(_strcmp_template, 'find', collator='_collator', collator_func='collator', func='find') diff --git a/src/calibre/utils/icu_test.py b/src/calibre/utils/icu_test.py index 3b2775e454..2c24348169 100644 --- a/src/calibre/utils/icu_test.py +++ b/src/calibre/utils/icu_test.py @@ -80,6 +80,8 @@ class TestICU(unittest.TestCase): from calibre.utils.titlecase import titlecase # Test corner cases self.ae('A', icu.upper(b'a')) + for x in ('', None, False, 1): + self.ae(x, icu.capitalize(x)) for x in ('a', 'Alice\'s code', 'macdonald\'s machIne', '02 the wars'): self.ae(icu.upper(x), x.upper()) From 828406fdc29a08170723d73fd5110d99534e8944 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 19 Mar 2014 21:07:40 +0530 Subject: [PATCH 075/122] Get rid of the backwards compat code for people running from source that have not updated their binary calibre builds as 1.28 has been out for a while --- src/calibre/utils/icu.py | 10 - src/calibre/utils/icu_old.py | 541 ----------------------------------- 2 files changed, 551 deletions(-) delete mode 100644 src/calibre/utils/icu_old.py diff --git a/src/calibre/utils/icu.py b/src/calibre/utils/icu.py index 1c61d3e739..551a1f4c04 100644 --- a/src/calibre/utils/icu.py +++ b/src/calibre/utils/icu.py @@ -251,16 +251,6 @@ def contractions(col=None): ################################################################################ -if not hasattr(_icu, 'change_case'): - print ('You are running from source with an outdated calibre binary install. You' - ' should update the main calibre binary to at least version 1.28.') - # Dont creak calibre for people running from source until the - # next binary is available witht he update icu module - from calibre.utils.icu_old import * # noqa - - def primary_contains(pat, src): - return primary_find(pat, src)[0] != -1 - if __name__ == '__main__': from calibre.utils.icu_test import run run(verbosity=4) diff --git a/src/calibre/utils/icu_old.py b/src/calibre/utils/icu_old.py deleted file mode 100644 index 39256f6fd6..0000000000 --- a/src/calibre/utils/icu_old.py +++ /dev/null @@ -1,541 +0,0 @@ -#!/usr/bin/env python -# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai - -__license__ = 'GPL v3' -__copyright__ = '2010, Kovid Goyal ' -__docformat__ = 'restructuredtext en' - -# Setup code {{{ -import sys -from functools import partial - -from calibre.constants import plugins -from calibre.utils.config_base import tweaks - -_icu = _collator = _primary_collator = _sort_collator = _numeric_collator = None -_locale = None - -_none = u'' -_none2 = b'' - -def get_locale(): - global _locale - if _locale is None: - from calibre.utils.localization import get_lang - if tweaks['locale_for_sorting']: - _locale = tweaks['locale_for_sorting'] - else: - _locale = get_lang() - return _locale - -def load_icu(): - global _icu - if _icu is None: - _icu = plugins['icu'][0] - if _icu is None: - print 'Loading ICU failed with: ', plugins['icu'][1] - else: - if not getattr(_icu, 'ok', False): - print 'icu not ok' - _icu = None - return _icu - -def load_collator(): - 'The default collator for most locales takes both case and accented letters into account' - global _collator - if _collator is None: - icu = load_icu() - if icu is not None: - _collator = icu.Collator(get_locale()) - return _collator - -def primary_collator(): - 'Ignores case differences and accented characters' - global _primary_collator - if _primary_collator is None: - _primary_collator = _collator.clone() - _primary_collator.strength = _icu.UCOL_PRIMARY - return _primary_collator - -def sort_collator(): - 'Ignores case differences and recognizes numbers in strings' - global _sort_collator - if _sort_collator is None: - _sort_collator = _collator.clone() - _sort_collator.strength = _icu.UCOL_SECONDARY - if tweaks['numeric_collation']: - try: - _sort_collator.numeric = True - except AttributeError: - pass - return _sort_collator - -def py_sort_key(obj): - if not obj: - return _none - return obj.lower() - -def icu_sort_key(collator, obj): - if not obj: - return _none2 - try: - try: - return _sort_collator.sort_key(obj) - except AttributeError: - return sort_collator().sort_key(obj) - except TypeError: - if isinstance(obj, unicode): - obj = obj.replace(u'\0', u'') - else: - obj = obj.replace(b'\0', b'') - return _sort_collator.sort_key(obj) - -def numeric_collator(): - global _numeric_collator - _numeric_collator = _collator.clone() - _numeric_collator.strength = _icu.UCOL_SECONDARY - _numeric_collator.numeric = True - return _numeric_collator - -def numeric_sort_key(obj): - 'Uses natural sorting for numbers inside strings so something2 will sort before something10' - if not obj: - return _none2 - try: - try: - return _numeric_collator.sort_key(obj) - except AttributeError: - return numeric_collator().sort_key(obj) - except TypeError: - if isinstance(obj, unicode): - obj = obj.replace(u'\0', u'') - else: - obj = obj.replace(b'\0', b'') - return _numeric_collator.sort_key(obj) - -def icu_change_case(upper, locale, obj): - func = _icu.upper if upper else _icu.lower - try: - return func(locale, obj) - except TypeError: - if isinstance(obj, unicode): - obj = obj.replace(u'\0', u'') - else: - obj = obj.replace(b'\0', b'') - return func(locale, obj) - -def py_find(pattern, source): - pos = source.find(pattern) - if pos > -1: - return pos, len(pattern) - return -1, -1 - -def character_name(string): - try: - try: - return _icu.character_name(unicode(string)) or None - except AttributeError: - import unicodedata - return unicodedata.name(unicode(string)[0], None) - except (TypeError, ValueError, KeyError): - pass - -def character_name_from_code(code): - try: - try: - return _icu.character_name_from_code(code) or '' - except AttributeError: - import unicodedata - return unicodedata.name(py_safe_chr(code), '') - except (TypeError, ValueError, KeyError): - return '' - -if sys.maxunicode >= 0x10ffff: - try: - py_safe_chr = unichr - except NameError: - py_safe_chr = chr -else: - def py_safe_chr(i): - # Narrow builds of python cannot represent code point > 0xffff as a - # single character, so we need our own implementation of unichr - # that returns them as a surrogate pair - return (b"\U%s" % (hex(i)[2:].zfill(8))).decode('unicode-escape') - -def safe_chr(code): - try: - return _icu.chr(code) - except AttributeError: - return py_safe_chr(code) - -def normalize(text, mode='NFC'): - # This is very slightly slower than using unicodedata.normalize, so stick with - # that unless you have very good reasons not too. Also, it's speed - # decreases on wide python builds, where conversion to/from ICU's string - # representation is slower. - try: - return _icu.normalize(_nmodes[mode], unicode(text)) - except (AttributeError, KeyError): - import unicodedata - return unicodedata.normalize(mode, unicode(text)) - -def icu_find(collator, pattern, source): - try: - return collator.find(pattern, source) - except TypeError: - return collator.find(unicode(pattern), unicode(source)) - -def icu_startswith(collator, a, b): - try: - return collator.startswith(a, b) - except TypeError: - return collator.startswith(unicode(a), unicode(b)) - -def py_case_sensitive_sort_key(obj): - if not obj: - return _none - return obj - -def icu_case_sensitive_sort_key(collator, obj): - if not obj: - return _none2 - return collator.sort_key(obj) - -def icu_strcmp(collator, a, b): - return collator.strcmp(lower(a), lower(b)) - -def py_strcmp(a, b): - return cmp(a.lower(), b.lower()) - -def icu_case_sensitive_strcmp(collator, a, b): - return collator.strcmp(a, b) - -def icu_capitalize(s): - s = lower(s) - return s.replace(s[0], upper(s[0]), 1) if s else s - -_cmap = {} -def icu_contractions(collator): - global _cmap - ans = _cmap.get(collator, None) - if ans is None: - ans = collator.contractions() - ans = frozenset(filter(None, ans)) if ans else {} - _cmap[collator] = ans - return ans - -def icu_collation_order(collator, a): - try: - return collator.collation_order(a) - except TypeError: - return collator.collation_order(unicode(a)) - -load_icu() -load_collator() -_icu_not_ok = _icu is None or _collator is None -icu_unicode_version = getattr(_icu, 'unicode_version', None) -_nmodes = {m:getattr(_icu, 'UNORM_'+m, None) for m in ('NFC', 'NFD', 'NFKC', 'NFKD', 'NONE', 'DEFAULT', 'FCD')} - -try: - senc = sys.getdefaultencoding() - if not senc or senc.lower() == 'ascii': - _icu.set_default_encoding('utf-8') - del senc -except: - pass - -try: - fenc = sys.getfilesystemencoding() - if not fenc or fenc.lower() == 'ascii': - _icu.set_filesystem_encoding('utf-8') - del fenc -except: - pass - - -# }}} - -################# The string functions ######################################## - -sort_key = py_sort_key if _icu_not_ok else partial(icu_sort_key, _collator) - -strcmp = py_strcmp if _icu_not_ok else partial(icu_strcmp, _collator) - -case_sensitive_sort_key = py_case_sensitive_sort_key if _icu_not_ok else \ - partial(icu_case_sensitive_sort_key, _collator) - -case_sensitive_strcmp = cmp if _icu_not_ok else icu_case_sensitive_strcmp - -upper = (lambda s: s.upper()) if _icu_not_ok else \ - partial(icu_change_case, True, get_locale()) - -lower = (lambda s: s.lower()) if _icu_not_ok else \ - partial(icu_change_case, False, get_locale()) - -title_case = (lambda s: s.title()) if _icu_not_ok else \ - partial(_icu.title, get_locale()) - -capitalize = (lambda s: s.capitalize()) if _icu_not_ok else \ - (lambda s: icu_capitalize(s)) - -find = (py_find if _icu_not_ok else partial(icu_find, _collator)) - -contractions = ((lambda : {}) if _icu_not_ok else (partial(icu_contractions, - _collator))) - -def primary_strcmp(a, b): - 'strcmp that ignores case and accents on letters' - if _icu_not_ok: - from calibre.utils.filenames import ascii_text - return py_strcmp(ascii_text(a), ascii_text(b)) - try: - return _primary_collator.strcmp(a, b) - except AttributeError: - return primary_collator().strcmp(a, b) - -def primary_find(pat, src): - 'find that ignores case and accents on letters' - if _icu_not_ok: - from calibre.utils.filenames import ascii_text - return py_find(ascii_text(pat), ascii_text(src)) - return primary_icu_find(pat, src) - -def primary_icu_find(pat, src): - try: - return icu_find(_primary_collator, pat, src) - except AttributeError: - return icu_find(primary_collator(), pat, src) - -def primary_sort_key(val): - 'A sort key that ignores case and diacritics' - if _icu_not_ok: - from calibre.utils.filenames import ascii_text - return ascii_text(val).lower() - try: - return _primary_collator.sort_key(val) - except AttributeError: - return primary_collator().sort_key(val) - -def primary_startswith(a, b): - if _icu_not_ok: - from calibre.utils.filenames import ascii_text - return ascii_text(a).lower().startswith(ascii_text(b).lower()) - try: - return icu_startswith(_primary_collator, a, b) - except AttributeError: - return icu_startswith(primary_collator(), a, b) - -def collation_order(a): - if _icu_not_ok: - return (ord(a[0]), 1) if a else (0, 0) - try: - return icu_collation_order(_sort_collator, a) - except AttributeError: - return icu_collation_order(sort_collator(), a) - -################################################################################ - -def test(): # {{{ - from calibre import prints - # Data {{{ - german = ''' - Sonntag -Montag -Dienstag -Januar -Februar -März -Fuße -Fluße -Flusse -flusse -fluße -flüße -flüsse -''' - german_good = ''' - Dienstag -Februar -flusse -Flusse -fluße -Fluße -flüsse -flüße -Fuße -Januar -März -Montag -Sonntag''' - french = ''' -dimanche -lundi -mardi -janvier -février -mars -déjà -Meme -deja -même -dejà -bpef -bœg -Boef -Mémé -bœf -boef -bnef -pêche -pèché -pêché -pêche -pêché''' - french_good = ''' - bnef - boef - Boef - bœf - bœg - bpef - deja - dejà - déjà - dimanche - février - janvier - lundi - mardi - mars - Meme - Mémé - même - pèché - pêche - pêche - pêché - pêché''' - # }}} - - def create(l): - l = l.decode('utf-8').splitlines() - return [x.strip() for x in l if x.strip()] - - def test_strcmp(entries): - for x in entries: - for y in entries: - if strcmp(x, y) != cmp(sort_key(x), sort_key(y)): - print 'strcmp failed for %r, %r'%(x, y) - - german = create(german) - c = _icu.Collator('de') - c.numeric = True - gs = list(sorted(german, key=c.sort_key)) - if gs != create(german_good): - print 'German sorting failed' - return - print - french = create(french) - c = _icu.Collator('fr') - c.numeric = True - fs = list(sorted(french, key=c.sort_key)) - if fs != create(french_good): - print 'French sorting failed (note that French fails with icu < 4.6)' - return - test_strcmp(german + french) - - print '\nTesting case transforms in current locale' - from calibre.utils.titlecase import titlecase - for x in ('a', 'Alice\'s code', 'macdonald\'s machine', '02 the wars'): - print 'Upper: ', x, '->', 'py:', x.upper().encode('utf-8'), 'icu:', upper(x).encode('utf-8') - print 'Lower: ', x, '->', 'py:', x.lower().encode('utf-8'), 'icu:', lower(x).encode('utf-8') - print 'Title: ', x, '->', 'py:', x.title().encode('utf-8'), 'icu:', title_case(x).encode('utf-8'), 'titlecase:', titlecase(x).encode('utf-8') - print 'Capitalize:', x, '->', 'py:', x.capitalize().encode('utf-8'), 'icu:', capitalize(x).encode('utf-8') - print - - print '\nTesting primary collation' - for k, v in {u'pèché': u'peche', u'flüße':u'Flusse', - u'Štepánek':u'ŠtepaneK'}.iteritems(): - if primary_strcmp(k, v) != 0: - prints('primary_strcmp() failed with %s != %s'%(k, v)) - return - if primary_find(v, u' '+k)[0] != 1: - prints('primary_find() failed with %s not in %s'%(v, k)) - return - - n = character_name(safe_chr(0x1f431)) - if n != u'CAT FACE': - raise ValueError('Failed to get correct character name for 0x1f431: %r != %r' % n, u'CAT FACE') - - global _primary_collator - orig = _primary_collator - _primary_collator = _icu.Collator('es') - if primary_strcmp(u'peña', u'pena') == 0: - print 'Primary collation in Spanish locale failed' - return - _primary_collator = orig - - print '\nTesting contractions' - c = _icu.Collator('cs') - if icu_contractions(c) != frozenset([u'Z\u030c', u'z\u030c', u'Ch', - u'C\u030c', u'ch', u'cH', u'c\u030c', u's\u030c', u'r\u030c', u'CH', - u'S\u030c', u'R\u030c']): - print 'Contractions for the Czech language failed' - return - - print '\nTesting startswith' - p = primary_startswith - if (not p('asd', 'asd') or not p('asd', 'A') or - not p('x', '')): - print 'startswith() failed' - return - - print '\nTesting collation_order()' - for group in [ - ('Šaa', 'Smith', 'Solženicyn', 'Štepánek'), - ('calibre', 'Charon', 'Collins'), - ('01', '1'), - ('1', '11', '13'), - ]: - last = None - for x in group: - val = icu_collation_order(sort_collator(), x) - if val[1] != 1: - prints('collation_order() returned incorrect length for', x) - if last is None: - last = val - else: - if val != last: - prints('collation_order() returned incorrect value for', x) - last = val - -# }}} - -def test_roundtrip(): - for r in (u'xxx\0\u2219\U0001f431xxx', u'\0', u'', u'simple'): - rp = _icu.roundtrip(r) - if rp != r: - raise ValueError(u'Roundtripping failed: %r != %r' % (r, rp)) - -def test_normalize_performance(): - import os - if not os.path.exists('t.txt'): - return - raw = open('t.txt', 'rb').read().decode('utf-8') - print (len(raw)) - import time, unicodedata - st = time.time() - count = 100 - for i in xrange(count): - normalize(raw) - print ('ICU time:', time.time() - st) - st = time.time() - for i in xrange(count): - unicodedata.normalize('NFC', unicode(raw)) - print ('py time:', time.time() - st) - -if __name__ == '__main__': - test_roundtrip() - test_normalize_performance() - test() - From 48c0a77c53dc70b03dec05e6609314a30b4570f3 Mon Sep 17 00:00:00 2001 From: Gregory Riker Date: Wed, 19 Mar 2014 11:05:22 -0700 Subject: [PATCH 076/122] Updated diagnostic logging. Optimized logging when debugging disabled. Added 'silent' switch to exists(), _afc_get_file_info(). Changed buffer for device name from fixed size to variable size. --- .../devices/idevice/libimobiledevice.py | 144 +++++++++--------- 1 file changed, 73 insertions(+), 71 deletions(-) diff --git a/src/calibre/devices/idevice/libimobiledevice.py b/src/calibre/devices/idevice/libimobiledevice.py index 3b405d7977..b8bbac4881 100644 --- a/src/calibre/devices/idevice/libimobiledevice.py +++ b/src/calibre/devices/idevice/libimobiledevice.py @@ -184,7 +184,9 @@ class libiMobileDevice(): def __init__(self, **kwargs): self.verbose = kwargs.get('verbose', False) - + if not self.verbose: + self._log = self.__null + self._log_location = self.__null self._log_location() self.afc = None self.app_version = 0 @@ -230,7 +232,7 @@ class libiMobileDevice(): src: file on local filesystem dst: file to be created on iOS filesystem ''' - self._log_location("src=%s, dst=%s" % (repr(src), repr(dst))) + self._log_location("src:{0} dst:{1}".format(repr(src), repr(dst))) mode = 'rb' with open(src, mode) as f: content = bytearray(f.read()) @@ -239,7 +241,7 @@ class libiMobileDevice(): handle = self._afc_file_open(str(dst), mode=mode) if handle is not None: success = self._afc_file_write(handle, content, mode=mode) - self._log(" success: %s" % success) + self._log(" success: {0}".format(success)) self._afc_file_close(handle) else: self._log(" could not create copy") @@ -251,7 +253,10 @@ class libiMobileDevice(): src: path to file on iDevice dst: file object on local filesystem ''' - self._log_location("src='%s', dst='%s'" % (src, dst.name)) + self._log_location() + self._log("src: {0}".format(repr(src))) + self._log("dst: {0}".format(dst.name)) + BUFFER_SIZE = 10 * 1024 * 1024 data = None mode = 'rb' @@ -287,7 +292,7 @@ class libiMobileDevice(): else: self._log(" could not open file") - raise libiMobileDeviceIOException("could not open file %s for reading" % repr(src)) + raise libiMobileDeviceIOException("could not open file {0} for reading".format(repr(src))) def disconnect_idevice(self): ''' @@ -310,14 +315,14 @@ class libiMobileDevice(): self._idevice_free() self.device_mounted = False - def exists(self, path): + def exists(self, path, silent=False): ''' Determine if path exists Returns file_info or {} ''' - self._log_location("'%s'" % path) - return self._afc_get_file_info(path) + self._log_location("{0}".format(repr(path))) + return self._afc_get_file_info(path, silent=silent) def get_device_info(self): ''' @@ -408,7 +413,7 @@ class libiMobileDevice(): Return a list containing the names of the entries in the iOS directory given by path. ''' - self._log_location("'%s'" % path) + self._log_location("{0}".format(repr(path))) return self._afc_read_directory(path, get_stats=get_stats) def load_library(self): @@ -438,8 +443,8 @@ class libiMobileDevice(): self.plist_lib = cdll.LoadLibrary('libplist.dll') self._log_location(env) - self._log(" libimobiledevice loaded from '%s'" % self.lib._name) - self._log(" libplist loaded from '%s'" % self.plist_lib._name) + self._log(" libimobiledevice loaded from '{0}'".format(self.lib._name)) + self._log(" libplist loaded from '{0}'".format(self.plist_lib._name)) if False: self._idevice_set_debug_level(DEBUG) @@ -449,7 +454,7 @@ class libiMobileDevice(): Mimic mkdir(), creating a directory at path. Does not create intermediate folders ''' - self._log_location("'%s'" % path) + self._log_location("{0}".format(repr(path))) return self._afc_make_directory(path) def mount_ios_app(self, app_name=None, app_id=None): @@ -481,7 +486,7 @@ class libiMobileDevice(): self._instproxy_client_free() if not app_name in self.installed_apps: - self._log(" '%s' not installed on this iDevice" % app_name) + self._log(" {0} not installed on this iDevice".format(repr(app_name))) self.disconnect_idevice() else: # Mount the app's Container @@ -517,9 +522,9 @@ class libiMobileDevice(): self.disconnect_idevice() if self.device_mounted: - self._log_location("'%s' mounted" % (app_name if app_name else app_id)) + self._log_location("'{0}' mounted".format(app_name if app_name else app_id)) else: - self._log_location("unable to mount '%s'" % (app_name if app_name else app_id)) + self._log_location("unable to mount '{0}'".format(app_name if app_name else app_id)) return self.device_mounted def mount_ios_media_folder(self): @@ -559,7 +564,7 @@ class libiMobileDevice(): Use for small files. For larger files copied to local file, use copy_from_idevice() ''' - self._log_location("'%s', mode='%s'" % (path, mode)) + self._log_location("{0} mode='{1}'".format(repr(path), mode)) data = None handle = self._afc_file_open(path, mode) @@ -569,7 +574,7 @@ class libiMobileDevice(): self._afc_file_close(handle) else: self._log(" could not open file") - raise libiMobileDeviceIOException("could not open file %s for reading" % repr(path)) + raise libiMobileDeviceIOException("could not open file {0} for reading".format(repr(path))) return data @@ -581,13 +586,13 @@ class libiMobileDevice(): from_name: (const char *) The fully-qualified path to rename from to_name: (const char *) The fully-qualified path to rename to ''' - self._log_location("from: '%s' to: '%s'" % (from_name, to_name)) + self._log_location("from: {0} to: {1}".format(repr(from_name), repr(to_name))) error = self.lib.afc_rename_path(byref(self.afc), str(from_name), str(to_name)) if error: - self._log(" ERROR: %s" % self._afc_error(error)) + self._log(" ERROR: {0}".format(self._afc_error(error))) def remove(self, path): ''' @@ -596,12 +601,12 @@ class libiMobileDevice(): client (afc_client_t) The client to use path (const char *) The fully-qualified path to delete ''' - self._log_location("'%s'" % path) + self._log_location("{0}".format(repr(path))) error = self.lib.afc_remove_path(byref(self.afc), str(path)) if error: - self._log_error(" ERROR: %s path:%s" % (self._afc_error(error), repr(path))) + self._log_error(" ERROR: {0} path:{1}".format(self._afc_error(error), repr(path))) def stat(self, path): ''' @@ -615,19 +620,19 @@ class libiMobileDevice(): 'st_birthtime': xxx.yyy} ''' - self._log_location("'%s'" % path) + self._log_location("{0}".format(repr(path))) return self._afc_get_file_info(path) def write(self, content, destination, mode='w'): ''' Convenience method to write to path on iDevice ''' - self._log_location(destination) + self._log_location("{0}".format(repr(destination))) handle = self._afc_file_open(destination, mode=mode) if handle is not None: success = self._afc_file_write(handle, content, mode=mode) - self._log(" success: %s" % success) + self._log(" success: {0}".format(success)) self._afc_file_close(handle) else: self._log(" could not open file for writing") @@ -650,7 +655,7 @@ class libiMobileDevice(): error = self.lib.afc_client_free(byref(self.afc)) & 0xFFFF if error: - self._log_error(" ERROR: %s" % self._afc_error(error)) + self._log_error(" ERROR: {0}".format(self._afc_error(error))) def _afc_client_new(self): ''' @@ -805,12 +810,12 @@ class libiMobileDevice(): File closed ''' - self._log_location(handle.value) + self._log_location("handle:{0}".format(handle.value)) error = self.lib.afc_file_close(byref(self.afc), handle) & 0xFFFF if error: - self._log_error(" ERROR: %s handle:%s" % (self._afc_error(error), handle)) + self._log_error(" ERROR: {0} handle:{1}".format(self._afc_error(error), handle)) def _afc_file_open(self, filename, mode='r'): ''' @@ -834,7 +839,7 @@ class libiMobileDevice(): error: (afc_error_t) AFC_E_SUCCESS (0) on success or AFC_E_* error value ''' - self._log_location("%s, mode='%s'" % (repr(filename), mode)) + self._log_location("{0} mode='{1}'".format(repr(filename), mode)) handle = c_ulonglong(0) @@ -850,7 +855,7 @@ class libiMobileDevice(): byref(handle)) & 0xFFFF if error: - self._log_error(" ERROR: %s filename:%s" % (self._afc_error(error), repr(filename))) + self._log_error(" ERROR: {0} filename:{1}".format(self._afc_error(error), repr(filename))) return None else: return handle @@ -874,7 +879,7 @@ class libiMobileDevice(): error (afc_error_t) AFC_E_SUCCESS (0) on success or AFC_E_* error value ''' - self._log_location("%s, size=%d, mode='%s'" % (handle.value, size, mode)) + self._log_location("handle:{0} size:{1:,} mode='{2}'".format(handle.value, size, mode)) bytes_read = c_uint(0) @@ -887,13 +892,13 @@ class libiMobileDevice(): size, byref(bytes_read)) & 0xFFFF if error: - self._log_error(" ERROR: %s handle:%s" % (self._afc_error(error), handle)) + self._log_error(" ERROR: {0} handle:{1}".format(self._afc_error(error), handle)) return data else: data = create_string_buffer(size) error = self.lib.afc_file_read(byref(self.afc), handle, byref(data), size, byref(bytes_read)) if error: - self._log_error(" ERROR: %s handle:%s" % (self._afc_error(error), handle)) + self._log_error(" ERROR: {0} handle:{1}".format(self._afc_error(error), handle)) return data.value def _afc_file_write(self, handle, content, mode='w'): @@ -915,7 +920,7 @@ class libiMobileDevice(): error: (afc_error_t) AFC_E_SUCCESS (0) on success or AFC_E_* error value ''' - self._log_location("handle=%d, mode='%s'" % (handle.value, mode)) + self._log_location("handle:{0} mode='{1}'".format(handle.value, mode)) bytes_written = c_uint(0) @@ -933,7 +938,7 @@ class libiMobileDevice(): len(content), byref(bytes_written)) & 0xFFFF if error: - self._log_error(" ERROR: %s handle:%s" % (self._afc_error(error), handle)) + self._log_error(" ERROR: {0} handle:{1}".format(self._afc_error(error), handle)) return False return True @@ -976,12 +981,12 @@ class libiMobileDevice(): for key in device_info.keys(): self._log("{0:>16}: {1}".format(key, device_info[key])) else: - self._log(" ERROR: %s" % self._afc_error(error)) + self._log(" ERROR: {0}".format(self._afc_error(error))) else: self._log(" ERROR: AFC not initialized, can't get device info") return device_info - def _afc_get_file_info(self, path): + def _afc_get_file_info(self, path, silent=False): ''' Gets information about a specific file @@ -1003,7 +1008,7 @@ class libiMobileDevice(): 'st_birthtime': xxx.yyy} ''' - self._log_location("'%s'" % path) + self._log_location("{0}".format(repr(path))) infolist_p = c_char * 1024 infolist = POINTER(POINTER(infolist_p))() @@ -1012,7 +1017,8 @@ class libiMobileDevice(): byref(infolist)) & 0xFFFF file_stats = {} if error: - self._log_error(" ERROR: %s path:%s" % (self._afc_error(error), repr(path))) + if not silent or self.verbose: + self._log_error(" ERROR: {0} path:{1}".format(self._afc_error(error), repr(path))) else: num_items = 0 item_list = [] @@ -1023,14 +1029,14 @@ class libiMobileDevice(): if item_list[i].contents.value in ['st_mtime', 'st_birthtime']: integer = item_list[i+1].contents.value[:10] decimal = item_list[i+1].contents.value[10:] - value = float("%s.%s" % (integer, decimal)) + value = float("{0}.{1}".format(integer, decimal)) else: value = item_list[i+1].contents.value file_stats[item_list[i].contents.value] = value if False and self.verbose: for key in file_stats.keys(): - self._log(" %s: %s" % (key, file_stats[key])) + self._log(" {0}: {1}".format(key, file_stats[key])) return file_stats def _afc_make_directory(self, path): @@ -1044,12 +1050,12 @@ class libiMobileDevice(): Result: error: AFC_E_SUCCESS on success or an AFC_E_* error value ''' - self._log_location("%s" % repr(path)) + self._log_location("{0}".format(repr(path))) error = self.lib.afc_make_directory(byref(self.afc), str(path)) & 0xFFFF if error: - self._log_error(" ERROR: %s path:%s" % (self._afc_error(error), repr(path))) + self._log_error(" ERROR: {0} path: {1}".format(self._afc_error(error), repr(path))) return error @@ -1070,7 +1076,7 @@ class libiMobileDevice(): {'': {} ...} ''' - self._log_location("'%s'" % directory) + self._log_location("{0}".format(repr(directory))) file_stats = {} dirs_p = c_char_p @@ -1079,7 +1085,7 @@ class libiMobileDevice(): str(directory), byref(dirs)) & 0xFFFF if error: - self._log_error(" ERROR: %s directory:%s" % (self._afc_error(error), repr(directory))) + self._log_error(" ERROR: {0} directory: {1}".format(self._afc_error(error), repr(directory))) else: num_dirs = 0 dir_list = [] @@ -1130,7 +1136,7 @@ class libiMobileDevice(): error = self.lib.house_arrest_client_free(byref(self.house_arrest)) & 0xFFFF if error: error = error - 0x10000 - self._log_error(" ERROR: %s" % self._house_arrest_error(error)) + self._log_error(" ERROR: {0}".format(self._house_arrest_error(error))) def _house_arrest_client_new(self): ''' @@ -1222,9 +1228,9 @@ class libiMobileDevice(): # To determine success, we need to inspect the returned plist if 'Status' in result: - self._log(" STATUS: %s" % result['Status']) + self._log(" STATUS: {0}".format(result['Status'])) elif 'Error' in result: - self._log(" ERROR: %s" % result['Error']) + self._log(" ERROR: {0}".format(result['Error'])) raise libiMobileDeviceException(result['Error']) def _house_arrest_send_command(self, command=None, appid=None): @@ -1248,12 +1254,12 @@ class libiMobileDevice(): to call house_arrest_get_result(). ''' - self._log_location("command='%s' appid='%s'" % (command, appid)) + self._log_location("command={0} appid={1}".format(repr(command), repr(appid))) commands = ['VendContainer', 'VendDocuments'] if command not in commands: - self._log(" ERROR: available commands: %s" % ', '.join(commands)) + self._log(" ERROR: available commands: {0}".format(', '.join(commands))) return _command = create_string_buffer(command) @@ -1306,7 +1312,7 @@ class libiMobileDevice(): if error: error = error - 0x10000 - self._log_error(" ERROR: %s" % self._idevice_error(error)) + self._log_error(" ERROR: {0}".format(self._idevice_error(error))) def _idevice_get_device_list(self): ''' @@ -1330,7 +1336,7 @@ class libiMobileDevice(): self._log(" no connected devices") else: device_list = None - self._log_error(" ERROR: %s" % self._idevice_error(error)) + self._log_error(" ERROR: {0}".format(self._idevice_error(error))) else: index = 0 while devices[index]: @@ -1338,7 +1344,7 @@ class libiMobileDevice(): if devices[index].contents.value not in device_list: device_list.append(devices[index].contents.value) index += 1 - self._log(" %s" % repr(device_list)) + self._log(" {0}".format(repr(device_list))) #self.lib.idevice_device_list_free() return device_list @@ -1372,8 +1378,8 @@ class libiMobileDevice(): if idevice_t.contents.conn_type == 1: self._log(" conn_type: CONNECTION_USBMUXD") else: - self._log(" conn_type: Unknown (%d)" % idevice_t.contents.conn_type) - self._log(" udid: %s" % idevice_t.contents.udid) + self._log(" conn_type: Unknown ({0})".format(idevice_t.contents.conn_type)) + self._log(" udid: {0}".format(idevice_t.contents.udid)) return idevice_t.contents def _idevice_set_debug_level(self, debug): @@ -1410,7 +1416,7 @@ class libiMobileDevice(): else: # Get the number of apps #app_count = self.lib.plist_array_get_size(apps) - #self._log(" app_count: %d" % app_count) + #self._log(" app_count: {0}".format(app_count)) # Convert the app plist to xml xml = POINTER(c_void_p)() @@ -1428,7 +1434,7 @@ class libiMobileDevice(): else: self._log(" unable to find app name in bundle:") for key in sorted(app.keys()): - self._log(" %s %s" % (repr(key), repr(app[key]))) + self._log(" {0} {1}".format(repr(key), repr(app[key]))) continue if not applist: @@ -1487,7 +1493,7 @@ class libiMobileDevice(): ''' Specify the type of apps we want to browse ''' - self._log_location("'%s', '%s'" % (app_type, domain)) + self._log_location("{0}, {1}".format(repr(app_type), repr(domain))) self.lib.instproxy_client_options_add(self.client_options, app_type, domain, None) @@ -1579,11 +1585,11 @@ class libiMobileDevice(): self._log_location() lockdownd_client_t = POINTER(LOCKDOWND_CLIENT_T)() - SERVICE_NAME = create_string_buffer('calibre') + #SERVICE_NAME = create_string_buffer('calibre') + SERVICE_NAME = c_void_p() error = self.lib.lockdownd_client_new_with_handshake(byref(self.device), byref(lockdownd_client_t), SERVICE_NAME) & 0xFFFF - if error: error = error - 0x10000 error_description = self.LIB_ERROR_TEMPLATE.format( @@ -1653,8 +1659,7 @@ class libiMobileDevice(): ''' self._log_location() - device_name_b = c_char * 32 - device_name_p = POINTER(device_name_b)() + device_name_p = c_char_p() device_name = None error = self.lib.lockdownd_get_device_name(byref(self.control), byref(device_name_p)) & 0xFFFF @@ -1666,8 +1671,8 @@ class libiMobileDevice(): desc=self._lockdown_error(error)) raise libiMobileDeviceException(error_description) else: - device_name = device_name_p.contents.value - self._log(" device_name: %s" % device_name) + device_name = device_name_p.value + self._log(" device_name: {0}".format(device_name)) return device_name def _lockdown_get_value(self, requested_items=[]): @@ -1815,7 +1820,7 @@ class libiMobileDevice(): if self.control: error = self.lib.lockdownd_goodbye(byref(self.control)) & 0xFFFF error = error - 0x10000 - self._log(" ERROR: %s" % self.error_lockdown(error)) + self._log(" ERROR: {0}".format(self.error_lockdown(error))) else: self._log(" connection already closed") @@ -1855,11 +1860,8 @@ class libiMobileDevice(): ''' Print msg to console ''' - if not self.verbose: - return - if msg: - debug_print(" %s" % msg) + debug_print(" {0}".format(msg)) else: debug_print() @@ -1880,9 +1882,6 @@ class libiMobileDevice(): def _log_location(self, *args): ''' ''' - if not self.verbose: - return - arg1 = arg2 = '' if len(args) > 0: @@ -1892,3 +1891,6 @@ class libiMobileDevice(): debug_print(self.LOCATION_TEMPLATE.format(cls=self.__class__.__name__, func=sys._getframe(1).f_code.co_name, arg1=arg1, arg2=arg2)) + + def __null(self, *args, **kwargs): + pass From 74f4bafae2da2e499680de6cc710137e3ed91c27 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 20 Mar 2014 11:02:19 +0530 Subject: [PATCH 077/122] Start work on saved searches dialog --- src/calibre/gui2/tweak_book/search.py | 241 ++++++++++++++++++++++---- 1 file changed, 206 insertions(+), 35 deletions(-) diff --git a/src/calibre/gui2/tweak_book/search.py b/src/calibre/gui2/tweak_book/search.py index 31386fc4a1..2d3d630716 100644 --- a/src/calibre/gui2/tweak_book/search.py +++ b/src/calibre/gui2/tweak_book/search.py @@ -6,14 +6,18 @@ from __future__ import (unicode_literals, division, absolute_import, __license__ = 'GPL v3' __copyright__ = '2013, Kovid Goyal ' +from functools import partial + from PyQt4.Qt import ( QWidget, QToolBar, Qt, QHBoxLayout, QSize, QIcon, QGridLayout, QLabel, - QPushButton, pyqtSignal, QComboBox, QCheckBox, QSizePolicy) + QPushButton, pyqtSignal, QComboBox, QCheckBox, QSizePolicy, QVBoxLayout, + QLineEdit, QToolButton, QListView, QFrame, QApplication) import regex from calibre.gui2.widgets2 import HistoryLineEdit2 from calibre.gui2.tweak_book import tprefs +from calibre.gui2.tweak_book.widgets import Dialog REGEX_FLAGS = regex.VERSION1 | regex.WORD | regex.FULLCASE | regex.MULTILINE | regex.UNICODE @@ -46,6 +50,60 @@ class HistoryLineEdit(HistoryLineEdit2): self.disable_popup = not bool(self.disable_popup) tprefs['disable_completion_popup_for_search'] = self.disable_popup +class WhereBox(QComboBox): + + def __init__(self, parent): + QComboBox.__init__(self) + self.addItems([_('Current file'), _('All text files'), _('All style files'), _('Selected files'), _('Marked text')]) + self.setToolTip('' + _( + ''' + Where to search/replace: +
+
Current file
+
Search only inside the currently opened file
+
All text files
+
Search in all text (HTML) files
+
All style files
+
Search in all style (CSS) files
+
Selected files
+
Search in the files currently selected in the Files Browser
+
Marked text
+
Search only within the marked text in the currently opened file. You can mark text using the Search menu.
+
''')) + + @dynamic_property + def where(self): + wm = {0:'current', 1:'text', 2:'styles', 3:'selected', 4:'selected-text'} + def fget(self): + return wm[self.currentIndex()] + def fset(self, val): + self.setCurrentIndex({v:k for k, v in wm.iteritems()}[val]) + return property(fget=fget, fset=fset) + +class DirectionBox(QComboBox): + + def __init__(self, parent): + QComboBox.__init__(self, parent) + self.addItems([_('Down'), _('Up')]) + self.setToolTip('' + _( + ''' + Direction to search: +
+
Down
+
Search for the next match from your current position
+
Up
+
Search for the previous match from your current position
+
''')) + + @dynamic_property + def direction(self): + def fget(self): + return 'down' if self.currentIndex() == 0 else 'up' + def fset(self, val): + self.setCurrentIndex(1 if val == 'up' else 0) + return property(fget=fget, fset=fset) + + class SearchWidget(QWidget): DEFAULT_STATE = { @@ -111,38 +169,12 @@ class SearchWidget(QWidget): ml.setBuddy(mb) ol.addWidget(mb) - self.where_box = wb = QComboBox(self) - wb.setSizePolicy(QSizePolicy.Maximum, QSizePolicy.Maximum) - wb.addItems([_('Current file'), _('All text files'), _('All style files'), _('Selected files'), _('Marked text')]) - wb.setToolTip('' + _( - ''' - Where to search/replace: -
-
Current file
-
Search only inside the currently opened file
-
All text files
-
Search in all text (HTML) files
-
All style files
-
Search in all style (CSS) files
-
Selected files
-
Search in the files currently selected in the Files Browser
-
Marked text
-
Search only within the marked text in the currently opened file. You can mark text using the Search menu.
-
''')) + self.where_box = wb = WhereBox(self) + wb.setSizePolicy(QSizePolicy.Maximum, QSizePolicy.Fixed) ol.addWidget(wb) - self.direction_box = db = QComboBox(self) + self.direction_box = db = DirectionBox(self) db.setSizePolicy(QSizePolicy.Maximum, QSizePolicy.Maximum) - db.addItems([_('Down'), _('Up')]) - db.setToolTip('' + _( - ''' - Direction to search: -
-
Down
-
Search for the next match from your current position
-
Up
-
Search for the previous match from your current position
-
''')) ol.addWidget(db) self.cs = cs = QCheckBox(_('&Case sensitive')) @@ -190,11 +222,10 @@ class SearchWidget(QWidget): @dynamic_property def where(self): - wm = {0:'current', 1:'text', 2:'styles', 3:'selected', 4:'selected-text'} def fget(self): - return wm[self.where_box.currentIndex()] + return self.where_box.where def fset(self, val): - self.where_box.setCurrentIndex({v:k for k, v in wm.iteritems()}[val]) + self.where_box.where = val return property(fget=fget, fset=fset) @dynamic_property @@ -208,9 +239,9 @@ class SearchWidget(QWidget): @dynamic_property def direction(self): def fget(self): - return 'down' if self.direction_box.currentIndex() == 0 else 'up' + return self.direction_box.direction def fset(self, val): - self.direction_box.setCurrentIndex(1 if val == 'up' else 0) + self.direction_box.direction = val return property(fget=fget, fset=fset) @dynamic_property @@ -320,3 +351,143 @@ class SearchPanel(QWidget): else: return QWidget.keyPressEvent(self, ev) +class SavedSearches(Dialog): + + def __init__(self, parent=None): + Dialog.__init__(self, _('Saved Searches'), 'saved-searches', parent=parent) + + def sizeHint(self): + return QSize(800, 650) + + def setup_ui(self): + self.l = l = QVBoxLayout(self) + self.setLayout(l) + + self.h = h = QHBoxLayout() + self.filter_text = ft = QLineEdit(self) + ft.textChanged.connect(self.do_filter) + ft.setPlaceholderText(_('Filter displayed searches')) + h.addWidget(ft) + self.cft = cft = QToolButton(self) + cft.setToolTip(_('Clear filter')), cft.setIcon(QIcon(I('clear_left.png'))) + cft.clicked.connect(ft.clear) + h.addWidget(cft) + l.addLayout(h) + + self.h2 = h = QHBoxLayout() + self.searches = searches = QListView(self) + h.addWidget(searches, stretch=10) + self.v = v = QVBoxLayout() + h.addLayout(v) + l.addLayout(h) + + def pb(text, tooltip=None): + b = QPushButton(text, self) + b.setToolTip(tooltip or '') + b.setSizePolicy(QSizePolicy.Preferred, QSizePolicy.Fixed) + return b + + mulmsg = '\n\n' + _('The entries are tried in order until the first one matches.') + + for text, action, tooltip in [ + (_('&Find'), 'find', _('Run the search using the selected entries.') + mulmsg), + (_('&Replace'), 'replace', _('Run replace using the selected entries.') + mulmsg), + (_('Replace a&nd Find'), 'replace-find', _('Run replace and then find using the selected entries.') + mulmsg), + (_('Replace &all'), 'replace-all', _('Run Replace All for all selected entries in the order selected')), + (_('&Count all'), 'count-all', _('Run Count All for all selected entries')), + ]: + b = pb(text, tooltip) + v.addWidget(b) + b.clicked.connect(partial(self.run_search, action)) + + self.d1 = d = QFrame(self) + d.setFrameStyle(QFrame.HLine) + v.addWidget(d) + + self.h3 = h = QHBoxLayout() + self.upb = b = QToolButton(self) + b.setIcon(QIcon(I('arrow-up.png'))), b.setToolTip(_('Move selected entries up')) + b.clicked.connect(partial(self.move_entry, -1)) + self.dnb = b = QToolButton(self) + b.setIcon(QIcon(I('arrow-down.png'))), b.setToolTip(_('Move selected entries down')) + b.clicked.connect(partial(self.move_entry, 1)) + h.addWidget(self.upb), h.addWidget(self.dnb) + v.addLayout(h) + + self.eb = b = pb(_('&Edit search'), _('Edit the currently selected search')) + b.clicked.connect(self.edit_search) + v.addWidget(b) + + self.eb = b = pb(_('&Remove search'), _('Remove the currently selected searches')) + b.clicked.connect(self.remove_search) + v.addWidget(b) + + self.eb = b = pb(_('&Add search'), _('Add a new saved search')) + b.clicked.connect(self.add_search) + v.addWidget(b) + + self.d2 = d = QFrame(self) + d.setFrameStyle(QFrame.HLine) + v.addWidget(d) + + self.where_box = wb = WhereBox(self) + v.addWidget(wb) + self.direction_box = db = DirectionBox(self) + v.addWidget(db) + + self.wr = wr = QCheckBox(_('&Wrap')) + wr.setToolTip('

'+_('When searching reaches the end, wrap around to the beginning and continue the search')) + v.addWidget(wr) + + l.addWidget(self.bb) + self.bb.clear() + self.bb.addButton(self.bb.Close) + + self.searches.setFocus(Qt.OtherFocusReason) + + @dynamic_property + def where(self): + def fget(self): + return self.where_box.where + def fset(self, val): + self.where_box.where = val + return property(fget=fget, fset=fset) + + @dynamic_property + def direction(self): + def fget(self): + return self.direction_box.direction + def fset(self, val): + self.direction_box.direction = val + return property(fget=fget, fset=fset) + + @dynamic_property + def wrap(self): + def fget(self): + return self.wr.isChecked() + def fset(self, val): + self.wr.setChecked(bool(val)) + return property(fget=fget, fset=fset) + + def do_filter(self, text): + pass + + def run_search(self, action): + pass + + def move_entry(self, delta): + pass + + def edit_search(self): + pass + + def remove_search(self): + pass + + def add_search(self): + pass + +if __name__ == '__main__': + app = QApplication([]) + d = SavedSearches() + d.exec_() From c59bf51c707710259ab1c67294af6a9481443903 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 20 Mar 2014 16:00:58 +0530 Subject: [PATCH 078/122] Ordering, editing, adding, removing for saved searches --- src/calibre/gui2/tweak_book/__init__.py | 1 + src/calibre/gui2/tweak_book/search.py | 220 +++++++++++++++++++++++- 2 files changed, 212 insertions(+), 9 deletions(-) diff --git a/src/calibre/gui2/tweak_book/__init__.py b/src/calibre/gui2/tweak_book/__init__.py index 2085568d6c..bfdc16b610 100644 --- a/src/calibre/gui2/tweak_book/__init__.py +++ b/src/calibre/gui2/tweak_book/__init__.py @@ -41,6 +41,7 @@ d['charmap_favorites'] = list(map(ord, '\xa0\u2002\u2003\u2009\xad' '‘’“ d['folders_for_types'] = {'style':'styles', 'image':'images', 'font':'fonts', 'audio':'audio', 'video':'video'} d['pretty_print_on_open'] = False d['disable_completion_popup_for_search'] = False +d['saved_searches'] = [] del d diff --git a/src/calibre/gui2/tweak_book/search.py b/src/calibre/gui2/tweak_book/search.py index 2d3d630716..6f8b1dfba1 100644 --- a/src/calibre/gui2/tweak_book/search.py +++ b/src/calibre/gui2/tweak_book/search.py @@ -11,14 +11,18 @@ from functools import partial from PyQt4.Qt import ( QWidget, QToolBar, Qt, QHBoxLayout, QSize, QIcon, QGridLayout, QLabel, QPushButton, pyqtSignal, QComboBox, QCheckBox, QSizePolicy, QVBoxLayout, - QLineEdit, QToolButton, QListView, QFrame, QApplication) + QLineEdit, QToolButton, QListView, QFrame, QApplication, QStyledItemDelegate, + QAbstractListModel, QVariant, QFormLayout, QModelIndex) import regex +from calibre.gui2 import NONE, error_dialog from calibre.gui2.widgets2 import HistoryLineEdit2 from calibre.gui2.tweak_book import tprefs from calibre.gui2.tweak_book.widgets import Dialog +from calibre.utils.icu import primary_contains + REGEX_FLAGS = regex.VERSION1 | regex.WORD | regex.FULLCASE | regex.MULTILINE | regex.UNICODE # The search panel {{{ @@ -288,7 +292,7 @@ class SearchWidget(QWidget): regex_cache = {} -class SearchPanel(QWidget): +class SearchPanel(QWidget): # {{{ search_triggered = pyqtSignal(object) @@ -350,6 +354,143 @@ class SearchPanel(QWidget): ev.accept() else: return QWidget.keyPressEvent(self, ev) +# }}} + +class SearchesModel(QAbstractListModel): + + def __init__(self, parent): + QAbstractListModel.__init__(self, parent) + self.searches = tprefs['saved_searches'] + self.filtered_searches = list(xrange(len(self.searches))) + + def rowCount(self, parent=QModelIndex()): + return len(self.filtered_searches) + + def data(self, index, role): + if role == Qt.DisplayRole: + search = self.searches[self.filtered_searches[index.row()]] + return QVariant(search['name']) + if role == Qt.ToolTipRole: + search = self.searches[self.filtered_searches[index.row()]] + tt = '\n'.join((search['find'], search['replace'])) + return QVariant(tt) + if role == Qt.UserRole: + search = self.searches[self.filtered_searches[index.row()]] + return QVariant((self.filtered_searches[index.row()], search)) + return NONE + + def do_filter(self, text): + text = unicode(text) + self.filtered_searches = [] + for i, search in enumerate(self.searches): + if primary_contains(text, search['name']): + self.filtered_searches.append(i) + self.reset() + + def move_entry(self, row, delta): + a, b = row, row + delta + if 0 <= b < len(self.filtered_searches): + ai, bi = self.filtered_searches[a], self.filtered_searches[b] + self.searches[ai], self.searches[bi] = self.searches[bi], self.searches[ai] + self.dataChanged.emit(self.index(a), self.index(a)) + self.dataChanged.emit(self.index(b), self.index(b)) + tprefs['saved_searches'] = self.searches + + def add_search(self): + self.searches = tprefs['saved_searches'] + self.filtered_searches.append(len(self.searches) - 1) + self.reset() + + def remove_searches(self, rows): + rows = sorted(set(rows), reverse=True) + indices = [self.filtered_searches[row] for row in rows] + for row in rows: + self.beginRemoveRows(QModelIndex(), row, row) + del self.filtered_searches[row] + self.endRemoveRows() + for idx in sorted(indices, reverse=True): + del self.searches[idx] + tprefs['saved_searches'] = self.searches + +class EditSearch(Dialog): # {{{ + + def __init__(self, search=None, search_index=-1, parent=None): + self.search = search or {} + self.original_name = self.search.get('name', None) + self.search_index = search_index + Dialog.__init__(self, _('Edit search'), 'edit-saved-search', parent=parent) + + def sizeHint(self): + ans = Dialog.sizeHint(self) + ans.setWidth(600) + return ans + + def setup_ui(self): + self.l = l = QFormLayout(self) + self.setLayout(l) + + self.search_name = n = QLineEdit(self.search.get('name', ''), self) + n.setPlaceholderText(_('The name with which to save this search')) + l.addRow(_('&Name:'), n) + + self.find = f = QLineEdit(self.search.get('find', ''), self) + f.setPlaceholderText(_('The regular expression to search for')) + l.addRow(_('&Find:'), f) + + self.replace = r = QLineEdit(self.search.get('replace', ''), self) + r.setPlaceholderText(_('The replace expression')) + l.addRow(_('&Replace:'), r) + + self.case_sensitive = c = QCheckBox(_('Case sensitive')) + c.setChecked(self.search.get('case_sensitive', SearchWidget.DEFAULT_STATE['case_sensitive'])) + l.addRow(c) + + self.dot_all = d = QCheckBox(_('Dot matches all')) + d.setChecked(self.search.get('dot_all', SearchWidget.DEFAULT_STATE['dot_all'])) + l.addRow(d) + + l.addRow(self.bb) + + def accept(self): + searches = tprefs['saved_searches'] + all_names = {x['name'] for x in searches} - {self.original_name} + n = unicode(self.search_name.text()).strip() + search = self.search + if not n: + return error_dialog(self, _('Must specify name'), _( + 'You must specify a search name'), show=True) + if n in all_names: + return error_dialog(self, _('Name exists'), _( + 'Another search with the name %s already exists') % n, show=True) + search['name'] = n + + f = unicode(self.find.text()) + if not f: + return error_dialog(self, _('Must specify find'), _( + 'You must specify a find expression'), show=True) + search['find'] = f + + r = unicode(self.replace.text()) + search['replace'] = r + + search['dot_all'] = bool(self.dot_all.isChecked()) + search['case_sensitive'] = bool(self.case_sensitive.isChecked()) + + if self.search_index == -1: + searches.append(search) + else: + searches[self.search_index] = search + tprefs.set('saved_searches', searches) + + Dialog.accept(self) +# }}} + +class SearchDelegate(QStyledItemDelegate): + + def sizeHint(self, *args): + ans = QStyledItemDelegate.sizeHint(self, *args) + ans.setHeight(ans.height() + 4) + return ans class SavedSearches(Dialog): @@ -357,7 +498,7 @@ class SavedSearches(Dialog): Dialog.__init__(self, _('Saved Searches'), 'saved-searches', parent=parent) def sizeHint(self): - return QSize(800, 650) + return QSize(800, 675) def setup_ui(self): self.l = l = QVBoxLayout(self) @@ -376,6 +517,15 @@ class SavedSearches(Dialog): self.h2 = h = QHBoxLayout() self.searches = searches = QListView(self) + searches.doubleClicked.connect(self.edit_search) + self.model = SearchesModel(self.searches) + self.model.dataChanged.connect(self.show_details) + searches.setModel(self.model) + searches.selectionModel().currentChanged.connect(self.show_details) + searches.setSelectionMode(searches.ExtendedSelection) + self.delegate = SearchDelegate(searches) + searches.setItemDelegate(self.delegate) + searches.setAlternatingRowColors(True) h.addWidget(searches, stretch=10) self.v = v = QVBoxLayout() h.addLayout(v) @@ -431,14 +581,21 @@ class SavedSearches(Dialog): v.addWidget(d) self.where_box = wb = WhereBox(self) + self.where = SearchWidget.DEFAULT_STATE['where'] v.addWidget(wb) self.direction_box = db = DirectionBox(self) + self.direction = SearchWidget.DEFAULT_STATE['direction'] v.addWidget(db) self.wr = wr = QCheckBox(_('&Wrap')) wr.setToolTip('

'+_('When searching reaches the end, wrap around to the beginning and continue the search')) + self.wr.setChecked(SearchWidget.DEFAULT_STATE['wrap']) v.addWidget(wr) + self.description = d = QLabel(' \n \n ') + d.setTextFormat(Qt.PlainText) + l.addWidget(d) + l.addWidget(self.bb) self.bb.clear() self.bb.addButton(self.bb.Close) @@ -470,22 +627,67 @@ class SavedSearches(Dialog): return property(fget=fget, fset=fset) def do_filter(self, text): - pass + self.model.do_filter(text) + self.searches.scrollTo(self.model.index(0)) def run_search(self, action): - pass + searches, seen = [], set() + for index in self.searches.selectionModel().selectedIndexes(): + if index.row() in seen: + continue + seen.add(index.row()) + search = SearchWidget.DEFAULT_STATE.copy() + search_index, s = index.data(Qt.UserRole).toPyObject() + search.update(s) + search['wrap'] = self.wrap + search['direction'] = self.direction + search['where'] = self.where + searches.append(search) + if not searches: + return def move_entry(self, delta): - pass + rows = {index.row() for index in self.searches.selectionModel().selectedIndexes()} - {-1} + if rows: + with tprefs: + for row in sorted(rows, reverse=delta > 0): + self.model.move_entry(row, delta) + nrow = row + delta + index = self.model.index(nrow) + if index.isValid(): + sm = self.searches.selectionModel() + sm.setCurrentIndex(index, sm.ClearAndSelect) def edit_search(self): - pass + index = self.searches.currentIndex() + if index.isValid(): + search_index, search = index.data(Qt.UserRole).toPyObject() + d = EditSearch(search=search, search_index=search_index, parent=self) + if d.exec_() == d.Accepted: + self.model.dataChanged.emit(index, index) def remove_search(self): - pass + rows = {index.row() for index in self.searches.selectionModel().selectedIndexes()} - {-1} + self.model.remove_searches(rows) + self.show_details() def add_search(self): - pass + d = EditSearch(parent=self) + if d.exec_() == d.Accepted: + self.model.add_search() + index = self.model.index(self.model.rowCount() - 1) + self.searches.scrollTo(index) + sm = self.searches.selectionModel() + sm.setCurrentIndex(index, sm.ClearAndSelect) + self.show_details() + + def show_details(self): + self.description.setText(' \n \n ') + i = self.searches.currentIndex() + if i.isValid(): + search_index, search = i.data(Qt.UserRole).toPyObject() + self.description.setText(_('{2}\nFind: {0}\nReplace: {1}').format( + search.get('find', ''), search.get('replace', ''), search.get('name', ''))) if __name__ == '__main__': app = QApplication([]) From c951290eb47bb7e15e2664d3f7a5764700619b34 Mon Sep 17 00:00:00 2001 From: Gregory Riker Date: Thu, 20 Mar 2014 04:58:48 -0700 Subject: [PATCH 079/122] Fix for lp:1294983. Added test for bools_are_tristate when processing rules for bool fields. --- src/calibre/library/catalogs/epub_mobi_builder.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/calibre/library/catalogs/epub_mobi_builder.py b/src/calibre/library/catalogs/epub_mobi_builder.py index 503a9081ab..736b7db6da 100644 --- a/src/calibre/library/catalogs/epub_mobi_builder.py +++ b/src/calibre/library/catalogs/epub_mobi_builder.py @@ -584,10 +584,11 @@ class CatalogBuilder(object): if field_contents == '': field_contents = None - if (self.db.metadata_for_field(rule['field'])['datatype'] == 'bool' and + # Handle condition where bools_are_tristate is False, + # field is a bool and contents is None, which is displayed as No + if (not self.db.prefs.get('bools_are_tristate') and + self.db.metadata_for_field(rule['field'])['datatype'] == 'bool' and field_contents is None): - # Handle condition where field is a bool and contents is None, - # which is displayed as No field_contents = _('False') if field_contents is not None: @@ -1021,8 +1022,11 @@ class CatalogBuilder(object): data = self.plugin.search_sort_db(self.db, self.opts) data = self.process_exclusions(data) - if self.prefix_rules and self.DEBUG: - self.opts.log.info(" Added prefixes:") + if self.DEBUG: + if self.prefix_rules: + self.opts.log.info(" Added prefixes (bools_are_tristate: {0}):".format(self.db.prefs.get('bools_are_tristate'))) + else: + self.opts.log.info(" No added prefixes") # Populate this_title{} from data[{},{}] titles = [] From 91a993643eadc1d288b79c60bad21e871f2ef5a1 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 20 Mar 2014 17:58:39 +0530 Subject: [PATCH 080/122] Refactor the search code in preparation for multi-searches --- src/calibre/gui2/tweak_book/boss.py | 178 ++---------------- src/calibre/gui2/tweak_book/char_select.py | 3 +- src/calibre/gui2/tweak_book/check.py | 3 +- src/calibre/gui2/tweak_book/search.py | 207 +++++++++++++++++++-- src/calibre/gui2/tweak_book/widgets.py | 10 +- 5 files changed, 210 insertions(+), 191 deletions(-) diff --git a/src/calibre/gui2/tweak_book/boss.py b/src/calibre/gui2/tweak_book/boss.py index c7ad6d53f7..9c2168a43b 100644 --- a/src/calibre/gui2/tweak_book/boss.py +++ b/src/calibre/gui2/tweak_book/boss.py @@ -7,14 +7,14 @@ __license__ = 'GPL v3' __copyright__ = '2013, Kovid Goyal ' import tempfile, shutil, sys, os -from collections import OrderedDict from functools import partial, wraps from PyQt4.Qt import ( - QObject, QApplication, QDialog, QGridLayout, QLabel, QSize, Qt, QCursor, - QDialogButtonBox, QIcon, QTimer, QPixmap, QTextBrowser, QVBoxLayout, QInputDialog) + QObject, QApplication, QDialog, QGridLayout, QLabel, QSize, Qt, + QDialogButtonBox, QIcon, QTimer, QPixmap, QTextBrowser, QVBoxLayout, + QInputDialog) -from calibre import prints, prepare_string_for_xml, isbytestring +from calibre import prints, isbytestring from calibre.ptempfile import PersistentTemporaryDirectory, TemporaryDirectory from calibre.ebooks.oeb.base import urlnormalize from calibre.ebooks.oeb.polish.main import SUPPORTED, tweak_polish @@ -27,7 +27,6 @@ from calibre.ebooks.oeb.polish.toc import remove_names_from_toc, find_existing_t from calibre.ebooks.oeb.polish.utils import link_stylesheets, setup_cssutils_serialization as scs from calibre.gui2 import error_dialog, choose_files, question_dialog, info_dialog, choose_save_file from calibre.gui2.dialogs.confirm_delete import confirm -from calibre.gui2.dialogs.message_box import MessageBox from calibre.gui2.tweak_book import set_current_container, current_container, tprefs, actions, editors from calibre.gui2.tweak_book.undo import GlobalUndoHistory from calibre.gui2.tweak_book.file_list import NewFileDialog @@ -37,8 +36,10 @@ from calibre.gui2.tweak_book.toc import TOCEditor from calibre.gui2.tweak_book.editor import editor_from_syntax, syntax_from_mime from calibre.gui2.tweak_book.editor.insert_resource import get_resource_data, NewBook from calibre.gui2.tweak_book.preferences import Preferences +from calibre.gui2.tweak_book.search import validate_search_request, run_search from calibre.gui2.tweak_book.widgets import ( - RationalizeFolders, MultiSplit, ImportForeign, QuickOpen, InsertLink, InsertSemantics) + RationalizeFolders, MultiSplit, ImportForeign, QuickOpen, InsertLink, + InsertSemantics, BusyCursor) _diff_dialogs = [] @@ -58,14 +59,6 @@ def get_container(*args, **kwargs): def setup_cssutils_serialization(): scs(tprefs['editor_tab_stop_width']) -class BusyCursor(object): - - def __enter__(self): - QApplication.setOverrideCursor(QCursor(Qt.WaitCursor)) - - def __exit__(self, *args): - QApplication.restoreOverrideCursor() - def in_thread_job(func): @wraps(func) def ans(*args, **kwargs): @@ -675,7 +668,7 @@ class Boss(QObject): # Ensure the search panel is visible sp.setVisible(True) ed = self.gui.central.current_editor - name = editor = None + name = None for n, x in editors.iteritems(): if x is ed: name = n @@ -684,158 +677,11 @@ class Boss(QObject): if overrides: state.update(overrides) searchable_names = self.gui.file_list.searchable_names - where = state['where'] - err = None - if name is None and where in {'current', 'selected-text'}: - err = _('No file is being edited.') - elif where == 'selected' and not searchable_names['selected']: - err = _('No files are selected in the Files Browser') - elif where == 'selected-text' and not ed.has_marked_text: - err = _('No text is marked. First select some text, and then use' - ' The "Mark selected text" action in the Search menu to mark it.') - if not err and not state['find']: - err = _('No search query specified') - if err: - return error_dialog(self.gui, _('Cannot search'), err, show=True) - del err + if not validate_search_request(name, searchable_names, getattr(ed, 'has_marked_text', False), state, self.gui): + return - files = OrderedDict() - do_all = state['wrap'] or action in {'replace-all', 'count'} - marked = False - if where == 'current': - editor = ed - elif where in {'styles', 'text', 'selected'}: - files = searchable_names[where] - if name in files: - # Start searching in the current editor - editor = ed - # Re-order the list of other files so that we search in the same - # order every time. Depending on direction, search the files - # that come after the current file, or before the current file, - # first. - lfiles = list(files) - idx = lfiles.index(name) - before, after = lfiles[:idx], lfiles[idx+1:] - if state['direction'] == 'up': - lfiles = list(reversed(before)) - if do_all: - lfiles += list(reversed(after)) + [name] - else: - lfiles = after - if do_all: - lfiles += before + [name] - files = OrderedDict((m, files[m]) for m in lfiles) - else: - editor = ed - marked = True - - def no_match(): - QApplication.restoreOverrideCursor() - msg = '

' + _('No matches were found for %s') % ('

' + prepare_string_for_xml(state['find']) + '
') - if not state['wrap']: - msg += '

' + _('You have turned off search wrapping, so all text might not have been searched.' - ' Try the search again, with wrapping enabled. Wrapping is enabled via the' - ' "Wrap" checkbox at the bottom of the search panel.') - return error_dialog( - self.gui, _('Not found'), msg, show=True) - - pat = sp.get_regex(state) - - def do_find(): - if editor is not None: - if editor.find(pat, marked=marked, save_match='gui'): - return - if not files: - if not state['wrap']: - return no_match() - return editor.find(pat, wrap=True, marked=marked, save_match='gui') or no_match() - for fname, syntax in files.iteritems(): - if fname in editors: - if not editors[fname].find(pat, complete=True, save_match='gui'): - continue - return self.show_editor(fname) - raw = current_container().raw_data(fname) - if pat.search(raw) is not None: - self.edit_file(fname, syntax) - if editors[fname].find(pat, complete=True, save_match='gui'): - return - return no_match() - - def no_replace(prefix=''): - QApplication.restoreOverrideCursor() - if prefix: - prefix += ' ' - error_dialog( - self.gui, _('Cannot replace'), prefix + _( - 'You must first click Find, before trying to replace'), show=True) - return False - - def do_replace(): - if editor is None: - return no_replace() - if not editor.replace(pat, state['replace'], saved_match='gui'): - return no_replace(_( - 'Currently selected text does not match the search query.')) - return True - - def count_message(action, count, show_diff=False): - msg = _('%(action)s %(num)s occurrences of %(query)s' % dict(num=count, query=state['find'], action=action)) - if show_diff and count > 0: - d = MessageBox(MessageBox.INFO, _('Searching done'), prepare_string_for_xml(msg), parent=self.gui, show_copy_button=False) - d.diffb = b = d.bb.addButton(_('See what &changed'), d.bb.ActionRole) - b.setIcon(QIcon(I('diff.png'))), d.set_details(None), b.clicked.connect(d.accept) - b.clicked.connect(partial(self.show_current_diff, allow_revert=True)) - d.exec_() - else: - info_dialog(self.gui, _('Searching done'), prepare_string_for_xml(msg), show=True) - - def do_all(replace=True): - count = 0 - if not files and editor is None: - return 0 - lfiles = files or {name:editor.syntax} - - for n, syntax in lfiles.iteritems(): - if n in editors: - raw = editors[n].get_raw_data() - else: - raw = current_container().raw_data(n) - if replace: - raw, num = pat.subn(state['replace'], raw) - else: - num = len(pat.findall(raw)) - count += num - if replace and num > 0: - if n in editors: - editors[n].replace_data(raw) - else: - with current_container().open(n, 'wb') as f: - f.write(raw.encode('utf-8')) - QApplication.restoreOverrideCursor() - count_message(_('Replaced') if replace else _('Found'), count, show_diff=replace) - return count - - with BusyCursor(): - if action == 'find': - return do_find() - if action == 'replace': - return do_replace() - if action == 'replace-find' and do_replace(): - return do_find() - if action == 'replace-all': - if marked: - return count_message(_('Replaced'), editor.all_in_marked(pat, state['replace'])) - self.add_savepoint(_('Before: Replace all')) - count = do_all() - if count == 0: - self.rewind_savepoint() - else: - self.set_modified() - return - if action == 'count': - if marked: - return count_message(_('Found'), editor.all_in_marked(pat)) - return do_all(replace=False) + run_search(state, action, ed, name, searchable_names, + self.gui, self.show_editor, self.edit_file, self.show_current_diff, self.add_savepoint, self.rewind_savepoint, self.set_modified) def create_checkpoint(self): text, ok = QInputDialog.getText(self.gui, _('Choose name'), _( diff --git a/src/calibre/gui2/tweak_book/char_select.py b/src/calibre/gui2/tweak_book/char_select.py index 9530087210..46d1f4b254 100644 --- a/src/calibre/gui2/tweak_book/char_select.py +++ b/src/calibre/gui2/tweak_book/char_select.py @@ -21,7 +21,7 @@ from calibre.constants import plugins, cache_dir from calibre.gui2 import NONE from calibre.gui2.widgets2 import HistoryLineEdit2 from calibre.gui2.tweak_book import tprefs -from calibre.gui2.tweak_book.widgets import Dialog +from calibre.gui2.tweak_book.widgets import Dialog, BusyCursor from calibre.utils.icu import safe_chr as chr, icu_unicode_version, character_name_from_code ROOT = QModelIndex() @@ -765,7 +765,6 @@ class CharSelect(Dialog): self.char_view.setFocus(Qt.OtherFocusReason) def do_search(self): - from calibre.gui2.tweak_book.boss import BusyCursor text = unicode(self.search.text()).strip() if not text: return self.clear_search() diff --git a/src/calibre/gui2/tweak_book/check.py b/src/calibre/gui2/tweak_book/check.py index 2c248b4c9c..fba761dbee 100644 --- a/src/calibre/gui2/tweak_book/check.py +++ b/src/calibre/gui2/tweak_book/check.py @@ -15,6 +15,7 @@ from PyQt4.Qt import ( from calibre.ebooks.oeb.polish.check.base import WARN, INFO, DEBUG, ERROR, CRITICAL from calibre.ebooks.oeb.polish.check.main import run_checks, fix_errors from calibre.gui2.tweak_book import tprefs +from calibre.gui2.tweak_book.widgets import BusyCursor def icon_for_level(level): if level > WARN: @@ -160,7 +161,6 @@ class Check(QSplitter): template % (err.HELP, ifix, fix_tt, fix_msg, run_tt, run_msg)) def run_checks(self, container): - from calibre.gui2.tweak_book.boss import BusyCursor with BusyCursor(): self.show_busy() QApplication.processEvents() @@ -179,7 +179,6 @@ class Check(QSplitter): self.clear_help(_('No problems found')) def fix_errors(self, container, errors): - from calibre.gui2.tweak_book.boss import BusyCursor with BusyCursor(): self.show_busy(_('Running fixers, please wait...')) QApplication.processEvents() diff --git a/src/calibre/gui2/tweak_book/search.py b/src/calibre/gui2/tweak_book/search.py index 6f8b1dfba1..80c365ed82 100644 --- a/src/calibre/gui2/tweak_book/search.py +++ b/src/calibre/gui2/tweak_book/search.py @@ -7,6 +7,7 @@ __license__ = 'GPL v3' __copyright__ = '2013, Kovid Goyal ' from functools import partial +from collections import OrderedDict from PyQt4.Qt import ( QWidget, QToolBar, Qt, QHBoxLayout, QSize, QIcon, QGridLayout, QLabel, @@ -16,10 +17,12 @@ from PyQt4.Qt import ( import regex -from calibre.gui2 import NONE, error_dialog +from calibre import prepare_string_for_xml +from calibre.gui2 import NONE, error_dialog, info_dialog +from calibre.gui2.dialogs.message_box import MessageBox from calibre.gui2.widgets2 import HistoryLineEdit2 -from calibre.gui2.tweak_book import tprefs -from calibre.gui2.tweak_book.widgets import Dialog +from calibre.gui2.tweak_book import tprefs, editors, current_container +from calibre.gui2.tweak_book.widgets import Dialog, BusyCursor from calibre.utils.icu import primary_contains @@ -332,22 +335,6 @@ class SearchPanel(QWidget): # {{{ def set_where(self, val): self.widget.where = val - def get_regex(self, state): - raw = state['find'] - if state['mode'] != 'regex': - raw = regex.escape(raw, special_only=True) - flags = REGEX_FLAGS - if not state['case_sensitive']: - flags |= regex.IGNORECASE - if state['mode'] == 'regex' and state['dot_all']: - flags |= regex.DOTALL - if state['direction'] == 'up': - flags |= regex.REVERSE - ans = regex_cache.get((flags, raw), None) - if ans is None: - ans = regex_cache[(flags, raw)] = regex.compile(raw, flags=flags) - return ans - def keyPressEvent(self, ev): if ev.key() == Qt.Key_Escape: self.hide_panel() @@ -544,7 +531,7 @@ class SavedSearches(Dialog): (_('&Replace'), 'replace', _('Run replace using the selected entries.') + mulmsg), (_('Replace a&nd Find'), 'replace-find', _('Run replace and then find using the selected entries.') + mulmsg), (_('Replace &all'), 'replace-all', _('Run Replace All for all selected entries in the order selected')), - (_('&Count all'), 'count-all', _('Run Count All for all selected entries')), + (_('&Count all'), 'count', _('Run Count All for all selected entries')), ]: b = pb(text, tooltip) v.addWidget(b) @@ -642,6 +629,7 @@ class SavedSearches(Dialog): search['wrap'] = self.wrap search['direction'] = self.direction search['where'] = self.where + search['mode'] = 'regex' searches.append(search) if not searches: return @@ -689,6 +677,185 @@ class SavedSearches(Dialog): self.description.setText(_('{2}\nFind: {0}\nReplace: {1}').format( search.get('find', ''), search.get('replace', ''), search.get('name', ''))) +def validate_search_request(name, searchable_names, has_marked_text, state, gui_parent): + err = None + where = state['where'] + if name is None and where in {'current', 'selected-text'}: + err = _('No file is being edited.') + elif where == 'selected' and not searchable_names['selected']: + err = _('No files are selected in the Files Browser') + elif where == 'selected-text' and not has_marked_text: + err = _('No text is marked. First select some text, and then use' + ' The "Mark selected text" action in the Search menu to mark it.') + if not err and not state['find']: + err = _('No search query specified') + if err: + error_dialog(gui_parent, _('Cannot search'), err, show=True) + return False + return True + +def get_search_regex(state): + raw = state['find'] + if state['mode'] != 'regex': + raw = regex.escape(raw, special_only=True) + flags = REGEX_FLAGS + if not state['case_sensitive']: + flags |= regex.IGNORECASE + if state['mode'] == 'regex' and state['dot_all']: + flags |= regex.DOTALL + if state['direction'] == 'up': + flags |= regex.REVERSE + ans = regex_cache.get((flags, raw), None) + if ans is None: + ans = regex_cache[(flags, raw)] = regex.compile(raw, flags=flags) + return ans + +def initialize_search_request(state, action, current_editor, current_editor_name, searchable_names): + editor = None + where = state['where'] + files = OrderedDict() + do_all = state['wrap'] or action in {'replace-all', 'count'} + marked = False + if where == 'current': + editor = current_editor + elif where in {'styles', 'text', 'selected'}: + files = searchable_names[where] + if current_editor_name in files: + # Start searching in the current editor + editor = current_editor + # Re-order the list of other files so that we search in the same + # order every time. Depending on direction, search the files + # that come after the current file, or before the current file, + # first. + lfiles = list(files) + idx = lfiles.index(current_editor_name) + before, after = lfiles[:idx], lfiles[idx+1:] + if state['direction'] == 'up': + lfiles = list(reversed(before)) + if do_all: + lfiles += list(reversed(after)) + [current_editor_name] + else: + lfiles = after + if do_all: + lfiles += before + [current_editor_name] + files = OrderedDict((m, files[m]) for m in lfiles) + else: + editor = current_editor + marked = True + + return editor, where, files, do_all, marked, get_search_regex(state) + +def run_search( + state, action, current_editor, current_editor_name, searchable_names, + gui_parent, show_editor, edit_file, show_current_diff, add_savepoint, rewind_savepoint, set_modified): + + editor, where, files, do_all, marked, pat = initialize_search_request(state, action, current_editor, current_editor_name, searchable_names) + def no_match(): + QApplication.restoreOverrideCursor() + msg = '

' + _('No matches were found for %s') % ('

' + prepare_string_for_xml(state['find']) + '
') + if not state['wrap']: + msg += '

' + _('You have turned off search wrapping, so all text might not have been searched.' + ' Try the search again, with wrapping enabled. Wrapping is enabled via the' + ' "Wrap" checkbox at the bottom of the search panel.') + return error_dialog( + gui_parent, _('Not found'), msg, show=True) + + def do_find(): + if editor is not None: + if editor.find(pat, marked=marked, save_match='gui'): + return + if not files: + if not state['wrap']: + return no_match() + return editor.find(pat, wrap=True, marked=marked, save_match='gui') or no_match() + for fname, syntax in files.iteritems(): + if fname in editors: + if not editors[fname].find(pat, complete=True, save_match='gui'): + continue + return show_editor(fname) + raw = current_container().raw_data(fname) + if pat.search(raw) is not None: + edit_file(fname, syntax) + if editors[fname].find(pat, complete=True, save_match='gui'): + return + return no_match() + + def no_replace(prefix=''): + QApplication.restoreOverrideCursor() + if prefix: + prefix += ' ' + error_dialog( + gui_parent, _('Cannot replace'), prefix + _( + 'You must first click Find, before trying to replace'), show=True) + return False + + def do_replace(): + if editor is None: + return no_replace() + if not editor.replace(pat, state['replace'], saved_match='gui'): + return no_replace(_( + 'Currently selected text does not match the search query.')) + return True + + def count_message(action, count, show_diff=False): + msg = _('%(action)s %(num)s occurrences of %(query)s' % dict(num=count, query=state['find'], action=action)) + if show_diff and count > 0: + d = MessageBox(MessageBox.INFO, _('Searching done'), prepare_string_for_xml(msg), parent=gui_parent, show_copy_button=False) + d.diffb = b = d.bb.addButton(_('See what &changed'), d.bb.ActionRole) + b.setIcon(QIcon(I('diff.png'))), d.set_details(None), b.clicked.connect(d.accept) + b.clicked.connect(partial(show_current_diff, allow_revert=True)) + d.exec_() + else: + info_dialog(gui_parent, _('Searching done'), prepare_string_for_xml(msg), show=True) + + def do_all(replace=True): + count = 0 + if not files and editor is None: + return 0 + lfiles = files or {current_editor_name:editor.syntax} + + for n, syntax in lfiles.iteritems(): + if n in editors: + raw = editors[n].get_raw_data() + else: + raw = current_container().raw_data(n) + if replace: + raw, num = pat.subn(state['replace'], raw) + else: + num = len(pat.findall(raw)) + count += num + if replace and num > 0: + if n in editors: + editors[n].replace_data(raw) + else: + with current_container().open(n, 'wb') as f: + f.write(raw.encode('utf-8')) + QApplication.restoreOverrideCursor() + count_message(_('Replaced') if replace else _('Found'), count, show_diff=replace) + return count + + with BusyCursor(): + if action == 'find': + return do_find() + if action == 'replace': + return do_replace() + if action == 'replace-find' and do_replace(): + return do_find() + if action == 'replace-all': + if marked: + return count_message(_('Replaced'), editor.all_in_marked(pat, state['replace'])) + add_savepoint(_('Before: Replace all')) + count = do_all() + if count == 0: + rewind_savepoint() + else: + set_modified() + return + if action == 'count': + if marked: + return count_message(_('Found'), editor.all_in_marked(pat)) + return do_all(replace=False) + if __name__ == '__main__': app = QApplication([]) d = SavedSearches() diff --git a/src/calibre/gui2/tweak_book/widgets.py b/src/calibre/gui2/tweak_book/widgets.py index 9a0b9df508..e3ddc0d13b 100644 --- a/src/calibre/gui2/tweak_book/widgets.py +++ b/src/calibre/gui2/tweak_book/widgets.py @@ -15,7 +15,7 @@ from PyQt4.Qt import ( QFormLayout, QHBoxLayout, QToolButton, QIcon, QApplication, Qt, QWidget, QPoint, QSizePolicy, QPainter, QStaticText, pyqtSignal, QTextOption, QAbstractListModel, QModelIndex, QVariant, QStyledItemDelegate, QStyle, - QListView, QTextDocument, QSize, QComboBox, QFrame) + QListView, QTextDocument, QSize, QComboBox, QFrame, QCursor) from calibre import prepare_string_for_xml from calibre.gui2 import error_dialog, choose_files, choose_save_file, NONE, info_dialog @@ -25,6 +25,14 @@ from calibre.utils.matcher import get_char, Matcher ROOT = QModelIndex() +class BusyCursor(object): + + def __enter__(self): + QApplication.setOverrideCursor(QCursor(Qt.WaitCursor)) + + def __exit__(self, *args): + QApplication.restoreOverrideCursor() + class Dialog(QDialog): def __init__(self, title, name, parent=None): From cb2b41f28e1fe5be1f32397bac2584145329fdc6 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 20 Mar 2014 22:19:55 +0530 Subject: [PATCH 081/122] Implement multi-searches --- src/calibre/gui2/tweak_book/search.py | 103 ++++++++++++++++---------- 1 file changed, 64 insertions(+), 39 deletions(-) diff --git a/src/calibre/gui2/tweak_book/search.py b/src/calibre/gui2/tweak_book/search.py index 80c365ed82..aeadb40010 100644 --- a/src/calibre/gui2/tweak_book/search.py +++ b/src/calibre/gui2/tweak_book/search.py @@ -743,17 +743,28 @@ def initialize_search_request(state, action, current_editor, current_editor_name editor = current_editor marked = True - return editor, where, files, do_all, marked, get_search_regex(state) + return editor, where, files, do_all, marked def run_search( - state, action, current_editor, current_editor_name, searchable_names, + searches, action, current_editor, current_editor_name, searchable_names, gui_parent, show_editor, edit_file, show_current_diff, add_savepoint, rewind_savepoint, set_modified): - editor, where, files, do_all, marked, pat = initialize_search_request(state, action, current_editor, current_editor_name, searchable_names) + if isinstance(searches, dict): + searches = [searches] + + editor, where, files, do_all, marked = initialize_search_request(searches[0], action, current_editor, current_editor_name, searchable_names) + wrap = searches[0]['wrap'] + + errfind = searches[0]['find'] + if len(searches) > 1: + errfind = _('the selected searches') + + searches = [(get_search_regex(search), search['replace']) for search in searches] + def no_match(): QApplication.restoreOverrideCursor() - msg = '

' + _('No matches were found for %s') % ('

' + prepare_string_for_xml(state['find']) + '
') - if not state['wrap']: + msg = '

' + _('No matches were found for %s') % ('

' + prepare_string_for_xml(errfind) + '
') + if not wrap: msg += '

' + _('You have turned off search wrapping, so all text might not have been searched.' ' Try the search again, with wrapping enabled. Wrapping is enabled via the' ' "Wrap" checkbox at the bottom of the search panel.') @@ -761,23 +772,25 @@ def run_search( gui_parent, _('Not found'), msg, show=True) def do_find(): - if editor is not None: - if editor.find(pat, marked=marked, save_match='gui'): - return - if not files: - if not state['wrap']: - return no_match() - return editor.find(pat, wrap=True, marked=marked, save_match='gui') or no_match() - for fname, syntax in files.iteritems(): - if fname in editors: - if not editors[fname].find(pat, complete=True, save_match='gui'): - continue - return show_editor(fname) - raw = current_container().raw_data(fname) - if pat.search(raw) is not None: - edit_file(fname, syntax) - if editors[fname].find(pat, complete=True, save_match='gui'): + for p, __ in searches: + if editor is not None: + if editor.find(p, marked=marked, save_match='gui'): return + if wrap and not files and editor.find(p, wrap=True, marked=marked, save_match='gui'): + return + for fname, syntax in files.iteritems(): + ed = editors.get(fname, None) + if ed is not None: + if not wrap and ed is editor: + continue + if ed.find(p, complete=True, save_match='gui'): + return show_editor(fname) + else: + raw = current_container().raw_data(fname) + if p.search(raw) is not None: + edit_file(fname, syntax) + if editors[fname].find(p, complete=True, save_match='gui'): + return return no_match() def no_replace(prefix=''): @@ -792,13 +805,14 @@ def run_search( def do_replace(): if editor is None: return no_replace() - if not editor.replace(pat, state['replace'], saved_match='gui'): - return no_replace(_( - 'Currently selected text does not match the search query.')) - return True + for p, repl in searches: + if editor.replace(p, repl, saved_match='gui'): + return True + return no_replace(_( + 'Currently selected text does not match the search query.')) def count_message(action, count, show_diff=False): - msg = _('%(action)s %(num)s occurrences of %(query)s' % dict(num=count, query=state['find'], action=action)) + msg = _('%(action)s %(num)s occurrences of %(query)s' % dict(num=count, query=errfind, action=action)) if show_diff and count > 0: d = MessageBox(MessageBox.INFO, _('Searching done'), prepare_string_for_xml(msg), parent=gui_parent, show_copy_button=False) d.diffb = b = d.bb.addButton(_('See what &changed'), d.bb.ActionRole) @@ -813,23 +827,34 @@ def run_search( if not files and editor is None: return 0 lfiles = files or {current_editor_name:editor.syntax} - + updates = set() + raw_data = {} for n, syntax in lfiles.iteritems(): if n in editors: raw = editors[n].get_raw_data() else: raw = current_container().raw_data(n) - if replace: - raw, num = pat.subn(state['replace'], raw) - else: - num = len(pat.findall(raw)) - count += num - if replace and num > 0: - if n in editors: - editors[n].replace_data(raw) + raw_data[n] = raw + + for p, repl in searches: + for n, syntax in lfiles.iteritems(): + raw = raw_data[n] + if replace: + raw, num = p.subn(repl, raw) + if num > 0: + updates.add(n) + raw_data[n] = raw else: - with current_container().open(n, 'wb') as f: - f.write(raw.encode('utf-8')) + num = len(p.findall(raw)) + count += num + + for n in updates: + raw = raw_data[n] + if n in editors: + editors[n].replace_data(raw) + else: + with current_container().open(n, 'wb') as f: + f.write(raw.encode('utf-8')) QApplication.restoreOverrideCursor() count_message(_('Replaced') if replace else _('Found'), count, show_diff=replace) return count @@ -843,7 +868,7 @@ def run_search( return do_find() if action == 'replace-all': if marked: - return count_message(_('Replaced'), editor.all_in_marked(pat, state['replace'])) + return count_message(_('Replaced'), sum(editor.all_in_marked(p, repl) for p, repl in searches)) add_savepoint(_('Before: Replace all')) count = do_all() if count == 0: @@ -853,7 +878,7 @@ def run_search( return if action == 'count': if marked: - return count_message(_('Found'), editor.all_in_marked(pat)) + return count_message(_('Found'), sum(editor.all_in_marked(p) for p, __ in searches)) return do_all(replace=False) if __name__ == '__main__': From 4a34618df483726e596d57abf7c9817f7e2c43f3 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 20 Mar 2014 22:40:35 +0530 Subject: [PATCH 082/122] Edit Book: Add support for saved searches. Click Search->Saved searches to bring up a dialog where you can create and manage saved searches Needs testing --- src/calibre/gui2/tweak_book/boss.py | 23 +++++++++++++++++++++++ src/calibre/gui2/tweak_book/search.py | 25 ++++++++++++++++++++++++- src/calibre/gui2/tweak_book/ui.py | 5 +++++ 3 files changed, 52 insertions(+), 1 deletion(-) diff --git a/src/calibre/gui2/tweak_book/boss.py b/src/calibre/gui2/tweak_book/boss.py index 9c2168a43b..89ebd51e8c 100644 --- a/src/calibre/gui2/tweak_book/boss.py +++ b/src/calibre/gui2/tweak_book/boss.py @@ -106,6 +106,8 @@ class Boss(QObject): self.gui.image_browser.image_activated.connect(self.image_activated) self.gui.checkpoints.revert_requested.connect(self.revert_requested) self.gui.checkpoints.compare_requested.connect(self.compare_requested) + self.gui.saved_searches.run_saved_searches.connect(self.run_saved_searches) + self.gui.central.search_panel.save_search.connect(self.save_search) def preferences(self): p = Preferences(self.gui) @@ -683,6 +685,27 @@ class Boss(QObject): run_search(state, action, ed, name, searchable_names, self.gui, self.show_editor, self.edit_file, self.show_current_diff, self.add_savepoint, self.rewind_savepoint, self.set_modified) + def saved_searches(self): + self.gui.saved_searches.show(), self.gui.saved_searches.raise_() + + def save_search(self): + state = self.gui.central.search_panel.state + self.gui.saved_searches.show(), self.gui.saved_searches.raise_() + self.gui.saved_searches.add_predefined_search(state) + + def run_saved_searches(self, searches, action): + ed = self.gui.central.current_editor + name = None + for n, x in editors.iteritems(): + if x is ed: + name = n + break + searchable_names = self.gui.file_list.searchable_names + if not searches or not validate_search_request(name, searchable_names, getattr(ed, 'has_marked_text', False), searches[0], self.gui): + return + run_search(searches, action, ed, name, searchable_names, + self.gui, self.show_editor, self.edit_file, self.show_current_diff, self.add_savepoint, self.rewind_savepoint, self.set_modified) + def create_checkpoint(self): text, ok = QInputDialog.getText(self.gui, _('Choose name'), _( 'Choose a name for the checkpoint.\nYou can later restore the book' diff --git a/src/calibre/gui2/tweak_book/search.py b/src/calibre/gui2/tweak_book/search.py index aeadb40010..ea9759defc 100644 --- a/src/calibre/gui2/tweak_book/search.py +++ b/src/calibre/gui2/tweak_book/search.py @@ -39,6 +39,7 @@ class PushButton(QPushButton): class HistoryLineEdit(HistoryLineEdit2): max_history_items = 100 + save_search = pyqtSignal() def __init__(self, parent, clear_msg): HistoryLineEdit2.__init__(self, parent) @@ -51,6 +52,8 @@ class HistoryLineEdit(HistoryLineEdit2): menu.addAction(self.clear_msg, self.clear_history) menu.addAction((_('Enable completion based on search history') if self.disable_popup else _( 'Disable completion based on search history')), self.toggle_popups) + menu.addSeparator() + menu.addAction(_('Save current search'), self.save_search.emit) menu.exec_(event.globalPos()) def toggle_popups(self): @@ -123,6 +126,7 @@ class SearchWidget(QWidget): } search_triggered = pyqtSignal(object) + save_search = pyqtSignal() def __init__(self, parent=None): QWidget.__init__(self, parent) @@ -133,6 +137,7 @@ class SearchWidget(QWidget): self.fl = fl = QLabel(_('&Find:')) fl.setAlignment(Qt.AlignRight | Qt.AlignCenter) self.find_text = ft = HistoryLineEdit(self, _('Clear search history')) + ft.save_search.connect(self.save_search) ft.initialize('tweak_book_find_edit') ft.returnPressed.connect(lambda : self.search_triggered.emit('find')) fl.setBuddy(ft) @@ -142,6 +147,7 @@ class SearchWidget(QWidget): self.rl = rl = QLabel(_('&Replace:')) rl.setAlignment(Qt.AlignRight | Qt.AlignCenter) self.replace_text = rt = HistoryLineEdit(self, _('Clear replace history')) + rt.save_search.connect(self.save_search) rt.initialize('tweak_book_replace_edit') rl.setBuddy(rt) l.addWidget(rl, 1, 0) @@ -298,6 +304,7 @@ regex_cache = {} class SearchPanel(QWidget): # {{{ search_triggered = pyqtSignal(object) + save_search = pyqtSignal() def __init__(self, parent=None): QWidget.__init__(self, parent) @@ -316,6 +323,7 @@ class SearchPanel(QWidget): # {{{ l.addWidget(self.widget) self.restore_state, self.save_state = self.widget.restore_state, self.widget.save_state self.widget.search_triggered.connect(self.search_triggered) + self.widget.save_search.connect(self.save_search) self.pre_fill = self.widget.pre_fill def hide_panel(self): @@ -401,11 +409,16 @@ class SearchesModel(QAbstractListModel): class EditSearch(Dialog): # {{{ - def __init__(self, search=None, search_index=-1, parent=None): + def __init__(self, search=None, search_index=-1, parent=None, state=None): self.search = search or {} self.original_name = self.search.get('name', None) self.search_index = search_index Dialog.__init__(self, _('Edit search'), 'edit-saved-search', parent=parent) + if state is not None: + self.find.setText(state['find']) + self.replace.setText(state['replace']) + self.case_sensitive.setChecked(state['case_sensitive']) + self.dot_all.setChecked(state['dot_all']) def sizeHint(self): ans = Dialog.sizeHint(self) @@ -481,6 +494,8 @@ class SearchDelegate(QStyledItemDelegate): class SavedSearches(Dialog): + run_saved_searches = pyqtSignal(object, object) + def __init__(self, parent=None): Dialog.__init__(self, _('Saved Searches'), 'saved-searches', parent=parent) @@ -633,6 +648,7 @@ class SavedSearches(Dialog): searches.append(search) if not searches: return + self.run_saved_searches.emit(searches, action) def move_entry(self, delta): rows = {index.row() for index in self.searches.selectionModel().selectedIndexes()} - {-1} @@ -661,6 +677,9 @@ class SavedSearches(Dialog): def add_search(self): d = EditSearch(parent=self) + self._add_search(d) + + def _add_search(self, d): if d.exec_() == d.Accepted: self.model.add_search() index = self.model.index(self.model.rowCount() - 1) @@ -669,6 +688,10 @@ class SavedSearches(Dialog): sm.setCurrentIndex(index, sm.ClearAndSelect) self.show_details() + def add_predefined_search(self, state): + d = EditSearch(parent=self, state=state) + self._add_search(d) + def show_details(self): self.description.setText(' \n \n ') i = self.searches.currentIndex() diff --git a/src/calibre/gui2/tweak_book/ui.py b/src/calibre/gui2/tweak_book/ui.py index f478738b1a..1aa1d60adf 100644 --- a/src/calibre/gui2/tweak_book/ui.py +++ b/src/calibre/gui2/tweak_book/ui.py @@ -29,6 +29,7 @@ from calibre.gui2.tweak_book.undo import CheckpointView from calibre.gui2.tweak_book.preview import Preview from calibre.gui2.tweak_book.search import SearchPanel from calibre.gui2.tweak_book.check import Check +from calibre.gui2.tweak_book.search import SavedSearches from calibre.gui2.tweak_book.toc import TOCViewer from calibre.gui2.tweak_book.char_select import CharSelect from calibre.gui2.tweak_book.editor.widget import register_text_editor_actions @@ -221,6 +222,7 @@ class Main(MainWindow): self.setCentralWidget(self.central) self.check_book = Check(self) self.toc_view = TOCViewer(self) + self.saved_searches = SavedSearches(self) self.image_browser = InsertImage(self, for_browsing=True) self.insert_char = CharSelect(self) @@ -393,6 +395,7 @@ class Main(MainWindow): 'count', keys=('Ctrl+N'), description=_('Count number of matches')) self.action_mark = reg(None, _('&Mark selected text'), self.boss.mark_selected_text, 'mark-selected-text', ('Ctrl+Shift+M',), _('Mark selected text')) self.action_go_to_line = reg(None, _('Go to &line'), self.boss.go_to_line_number, 'go-to-line-number', ('Ctrl+.',), _('Go to line number')) + self.action_saved_searches = reg(None, _('Sa&ved searches'), self.boss.saved_searches, 'saved-searches', (), _('Show the saved searches dialog')) # Check Book actions group = _('Check Book') @@ -507,6 +510,8 @@ class Main(MainWindow): a(self.action_mark) e.addSeparator() a(self.action_go_to_line) + e.addSeparator() + a(self.action_saved_searches) e = b.addMenu(_('&Help')) a = e.addAction From 1e2febdb37fb9cc2ab7c93855a4623a1607de3d0 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 21 Mar 2014 09:20:33 +0530 Subject: [PATCH 083/122] Linux build: Workaround for systems that have broken libc implementations that change the behavior of truncate() on file descriptors with O_APPEND set. Fixes #1295366 [json config files get corrupted when using Python 2.7.6 on Linux](https://bugs.launchpad.net/calibre/+bug/1295366) --- src/calibre/utils/lock.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/calibre/utils/lock.py b/src/calibre/utils/lock.py index b2156d48c8..5090c11cf8 100644 --- a/src/calibre/utils/lock.py +++ b/src/calibre/utils/lock.py @@ -8,7 +8,7 @@ Secure access to locked files from multiple processes. from calibre.constants import iswindows, __appname__, \ win32api, win32event, winerror, fcntl -import time, atexit, os +import time, atexit, os, stat class LockError(Exception): pass @@ -105,6 +105,12 @@ class WindowsExclFile(object): def closed(self): return self._handle is None +def unix_open(path): + # We cannot use open(a+b) directly because Fedora apparently ships with a + # broken libc that causes seek(0) followed by truncate() to not work for + # files with O_APPEND set. + fd = os.open(path, os.O_RDWR | os.O_CREAT, stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH) + return os.fdopen(fd, 'r+b') class ExclusiveFile(object): @@ -113,7 +119,7 @@ class ExclusiveFile(object): self.timeout = timeout def __enter__(self): - self.file = WindowsExclFile(self.path, self.timeout) if iswindows else open(self.path, 'a+b') + self.file = WindowsExclFile(self.path, self.timeout) if iswindows else unix_open(self.path) self.file.seek(0) timeout = self.timeout if not iswindows: From eb10b2d93f7769f79d601971178e671a8234ebc5 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 21 Mar 2014 09:48:05 +0530 Subject: [PATCH 084/122] Fix incorrect replace when doing multi S&R and non-leading search expression matches --- src/calibre/gui2/tweak_book/editor/text.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/calibre/gui2/tweak_book/editor/text.py b/src/calibre/gui2/tweak_book/editor/text.py index 79e57c1804..fb949f63eb 100644 --- a/src/calibre/gui2/tweak_book/editor/text.py +++ b/src/calibre/gui2/tweak_book/editor/text.py @@ -315,7 +315,7 @@ class TextEdit(PlainTextEdit): # Center search result on screen self.centerCursor() if save_match is not None: - self.saved_matches[save_match] = m + self.saved_matches[save_match] = (pat, m) return True def all_in_marked(self, pat, template=None): @@ -372,7 +372,7 @@ class TextEdit(PlainTextEdit): # Center search result on screen self.centerCursor() if save_match is not None: - self.saved_matches[save_match] = m + self.saved_matches[save_match] = (pat, m) return True def replace(self, pat, template, saved_match='gui'): @@ -385,8 +385,8 @@ class TextEdit(PlainTextEdit): # the saved match matches the currently selected text and # use it, if so. if saved_match is not None and saved_match in self.saved_matches: - saved = self.saved_matches.pop(saved_match) - if saved.group() == raw: + saved_pat, saved = self.saved_matches.pop(saved_match) + if saved_pat == pat and saved.group() == raw: m = saved if m is None: return False From c6ce43cf9c7fd000d9cbce5548307a36aa37e67d Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 21 Mar 2014 10:06:50 +0530 Subject: [PATCH 085/122] version 1.29 --- Changelog.yaml | 46 ++++++++++++++++++++++++++++++++++++++++ src/calibre/constants.py | 2 +- 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/Changelog.yaml b/Changelog.yaml index 1251195ab6..65f307facf 100644 --- a/Changelog.yaml +++ b/Changelog.yaml @@ -20,6 +20,52 @@ # new recipes: # - title: +- version: 1.29.0 + date: 2014-03-21 + + new features: + - title: "Edit Book: Add support for saved searches. Click Search->Saved Searches to bring up a dialog where you can create and manage saved searches" + + - title: "Edit Book: New tool to specify semantics in EPUB books (semantics are items in the guide such as preface, title-page, dedication, etc.). To use it, go to Tools->Set Semantics" + tickets: [1287025] + + - title: "Edit Book: Preview panel: Add a copy selected text action to the context menu" + + - title: "Edit Book: When inserting hyperlinks, allow specifying the text for the hyperlink in the insert hyperlink dialog" + + bug fixes: + - title: "Fix a regression in the previous release that broke downloading metadata for authors with a double initial such as R. A. Salvatore." + tickets: [1294529] + + - title: "Edit book: When generating inline Table of Contents, mark it as such in the guide section of the OPF." + tickets: [1287018] + + - title: "E-book viewer: Fix right margin for last page in a chapter sometimes disappearing when changing font size." + tickets: [1292822] + + - title: "Edit Book: Fix saving of empty files not working" + + - title: "Edit book: Fix a regression in the previous release that broke saving a copy of the current book on linux and OS X" + + - title: "Edit book: Fix syntax highlighting in HTML files breaks if the closing of a comment or processing instruction is at the start of a new line." + + - title: "Edit book: Fix check book failing in the presence of empty ' + _( + '''Select how the search expression is interpreted +

+
Normal
+
The search expression is treated as normal text, calibre will look for the exact text.
+
Regex
+
The search expression is interpreted as a regular expression. See the User Manual for more help on using regular expressions.
+
''')) + + @dynamic_property + def mode(self): + def fget(self): + return 'normal' if self.currentIndex() == 0 else 'regex' + def fset(self, val): + self.setCurrentIndex({'regex':1}.get(val, 0)) + return property(fget=fget, fset=fset) + class SearchWidget(QWidget): @@ -169,17 +191,8 @@ class SearchWidget(QWidget): ml.setAlignment(Qt.AlignRight | Qt.AlignCenter) l.addWidget(ml, 2, 0) l.addLayout(ol, 2, 1, 1, 3) - self.mode_box = mb = QComboBox(self) + self.mode_box = mb = ModeBox(self) mb.setSizePolicy(QSizePolicy.Maximum, QSizePolicy.Maximum) - mb.addItems([_('Normal'), _('Regex')]) - mb.setToolTip('' + _( - '''Select how the search expression is interpreted -
-
Normal
-
The search expression is treated as normal text, calibre will look for the exact text.
-
Regex
-
The search expression is interpreted as a regular expression. See the User Manual for more help on using regular expressions.
-
''')) ml.setBuddy(mb) ol.addWidget(mb) @@ -212,9 +225,9 @@ class SearchWidget(QWidget): @dynamic_property def mode(self): def fget(self): - return 'normal' if self.mode_box.currentIndex() == 0 else 'regex' + return self.mode_box.mode def fset(self, val): - self.mode_box.setCurrentIndex({'regex':1}.get(val, 0)) + self.mode_box.mode = val self.da.setVisible(self.mode == 'regex') return property(fget=fget, fset=fset) @@ -420,6 +433,7 @@ class EditSearch(Dialog): # {{{ self.replace.setText(state['replace']) self.case_sensitive.setChecked(state['case_sensitive']) self.dot_all.setChecked(state['dot_all']) + self.mode_box.mode = state.get('mode') def sizeHint(self): ans = Dialog.sizeHint(self) @@ -450,6 +464,10 @@ class EditSearch(Dialog): # {{{ d.setChecked(self.search.get('dot_all', SearchWidget.DEFAULT_STATE['dot_all'])) l.addRow(d) + self.mode_box = m = ModeBox(self) + self.mode_box.mode = self.search.get('mode', 'regex') + l.addRow(_('&Mode:'), m) + l.addRow(self.bb) def accept(self): @@ -476,6 +494,7 @@ class EditSearch(Dialog): # {{{ search['dot_all'] = bool(self.dot_all.isChecked()) search['case_sensitive'] = bool(self.case_sensitive.isChecked()) + search['mode'] = self.mode_box.mode if self.search_index == -1: searches.append(search) From 1379bbf852d09c69b99ac78131cd8c10fbff39f2 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 22 Mar 2014 10:33:01 +0530 Subject: [PATCH 092/122] ... --- src/calibre/gui2/tweak_book/search.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/gui2/tweak_book/search.py b/src/calibre/gui2/tweak_book/search.py index e2b5c34625..2b4a608a71 100644 --- a/src/calibre/gui2/tweak_book/search.py +++ b/src/calibre/gui2/tweak_book/search.py @@ -740,7 +740,7 @@ class SavedSearches(Dialog): def err(): error_dialog(self, _('Invalid data'), _( 'The file %s does not contain valid saved searches') % path, show=True) - if not isinstance(obj, dict) or not 'version' in obj or not 'searches' in obj: + if not isinstance(obj, dict) or not 'version' in obj or not 'searches' in obj or obj['version'] not in (1,): return err() searches = [] for item in obj['searches']: From cfad8a5076a3b222245ad72caf0fef4e724738a9 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 22 Mar 2014 10:39:14 +0530 Subject: [PATCH 093/122] Allow opening saved searches dialog directly via right click --- src/calibre/gui2/tweak_book/boss.py | 6 +++++- src/calibre/gui2/tweak_book/search.py | 7 +++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/calibre/gui2/tweak_book/boss.py b/src/calibre/gui2/tweak_book/boss.py index 89ebd51e8c..d1b05207bd 100644 --- a/src/calibre/gui2/tweak_book/boss.py +++ b/src/calibre/gui2/tweak_book/boss.py @@ -108,6 +108,7 @@ class Boss(QObject): self.gui.checkpoints.compare_requested.connect(self.compare_requested) self.gui.saved_searches.run_saved_searches.connect(self.run_saved_searches) self.gui.central.search_panel.save_search.connect(self.save_search) + self.gui.central.search_panel.show_saved_searches.connect(self.show_saved_searches) def preferences(self): p = Preferences(self.gui) @@ -690,9 +691,12 @@ class Boss(QObject): def save_search(self): state = self.gui.central.search_panel.state - self.gui.saved_searches.show(), self.gui.saved_searches.raise_() + self.show_saved_searches() self.gui.saved_searches.add_predefined_search(state) + def show_saved_searches(self): + self.gui.saved_searches.show(), self.gui.saved_searches.raise_() + def run_saved_searches(self, searches, action): ed = self.gui.central.current_editor name = None diff --git a/src/calibre/gui2/tweak_book/search.py b/src/calibre/gui2/tweak_book/search.py index 2b4a608a71..c77790fd26 100644 --- a/src/calibre/gui2/tweak_book/search.py +++ b/src/calibre/gui2/tweak_book/search.py @@ -41,6 +41,7 @@ class HistoryLineEdit(HistoryLineEdit2): max_history_items = 100 save_search = pyqtSignal() + show_saved_searches = pyqtSignal() def __init__(self, parent, clear_msg): HistoryLineEdit2.__init__(self, parent) @@ -55,6 +56,7 @@ class HistoryLineEdit(HistoryLineEdit2): 'Disable completion based on search history')), self.toggle_popups) menu.addSeparator() menu.addAction(_('Save current search'), self.save_search.emit) + menu.addAction(_('Show saved searches'), self.show_saved_searches.emit) menu.exec_(event.globalPos()) def toggle_popups(self): @@ -150,6 +152,7 @@ class SearchWidget(QWidget): search_triggered = pyqtSignal(object) save_search = pyqtSignal() + show_saved_searches = pyqtSignal() def __init__(self, parent=None): QWidget.__init__(self, parent) @@ -161,6 +164,7 @@ class SearchWidget(QWidget): fl.setAlignment(Qt.AlignRight | Qt.AlignCenter) self.find_text = ft = HistoryLineEdit(self, _('Clear search history')) ft.save_search.connect(self.save_search) + ft.show_saved_searches.connect(self.show_saved_searches) ft.initialize('tweak_book_find_edit') ft.returnPressed.connect(lambda : self.search_triggered.emit('find')) fl.setBuddy(ft) @@ -171,6 +175,7 @@ class SearchWidget(QWidget): rl.setAlignment(Qt.AlignRight | Qt.AlignCenter) self.replace_text = rt = HistoryLineEdit(self, _('Clear replace history')) rt.save_search.connect(self.save_search) + rt.show_saved_searches.connect(self.show_saved_searches) rt.initialize('tweak_book_replace_edit') rl.setBuddy(rt) l.addWidget(rl, 1, 0) @@ -319,6 +324,7 @@ class SearchPanel(QWidget): # {{{ search_triggered = pyqtSignal(object) save_search = pyqtSignal() + show_saved_searches = pyqtSignal() def __init__(self, parent=None): QWidget.__init__(self, parent) @@ -338,6 +344,7 @@ class SearchPanel(QWidget): # {{{ self.restore_state, self.save_state = self.widget.restore_state, self.widget.save_state self.widget.search_triggered.connect(self.search_triggered) self.widget.save_search.connect(self.save_search) + self.widget.show_saved_searches.connect(self.show_saved_searches) self.pre_fill = self.widget.pre_fill def hide_panel(self): From eaae5b235e9e4279dbce684f7a0f09485c1bd75d Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 22 Mar 2014 17:06:02 +0530 Subject: [PATCH 094/122] Cover Browser: Add an option to show covers with their original aspect ratio instead of resizing them all to have the same width and height. Option is in Preferences->Look & Feel->Cover Browser. Fixes #1295902 [[Enhancement Request] Allow cover browser to show book covers in actual dimensions/orientation](https://bugs.launchpad.net/calibre/+bug/1295902) --- src/calibre/gui2/__init__.py | 1 + src/calibre/gui2/cover_flow.py | 8 +- src/calibre/gui2/pictureflow/pictureflow.cpp | 118 +++++++++++++------ src/calibre/gui2/pictureflow/pictureflow.h | 11 ++ src/calibre/gui2/pictureflow/pictureflow.sip | 4 + src/calibre/gui2/preferences/look_feel.py | 2 + src/calibre/gui2/preferences/look_feel.ui | 17 ++- 7 files changed, 121 insertions(+), 40 deletions(-) diff --git a/src/calibre/gui2/__init__.py b/src/calibre/gui2/__init__.py index 706e3b4ab6..015df60143 100644 --- a/src/calibre/gui2/__init__.py +++ b/src/calibre/gui2/__init__.py @@ -123,6 +123,7 @@ defs['cover_grid_texture'] = None defs['show_vl_tabs'] = False defs['show_highlight_toggle_button'] = False defs['add_comments_to_email'] = False +defs['cb_preserve_aspect_ratio'] = False del defs # }}} diff --git a/src/calibre/gui2/cover_flow.py b/src/calibre/gui2/cover_flow.py index 81cdf9b90d..1eb11adac2 100644 --- a/src/calibre/gui2/cover_flow.py +++ b/src/calibre/gui2/cover_flow.py @@ -21,6 +21,7 @@ pictureflow, pictureflowerror = plugins['pictureflow'] if pictureflow is not None: class EmptyImageList(pictureflow.FlowImages): + def __init__(self): pictureflow.FlowImages.__init__(self) @@ -108,7 +109,6 @@ if pictureflow is not None: def image(self, index): return self.model.cover(index) - class CoverFlow(pictureflow.PictureFlow): dc_signal = pyqtSignal() @@ -125,6 +125,10 @@ if pictureflow is not None: type=Qt.QueuedConnection) self.context_menu = None self.setContextMenuPolicy(Qt.DefaultContextMenu) + try: + self.setPreserveAspectRatio(gprefs['cb_preserve_aspect_ratio']) + except AttributeError: + pass # source checkout without updated binary if hasattr(self, 'setSubtitleFont'): self.setSubtitleFont(QFont(rating_font())) if not gprefs['cover_browser_reflections']: @@ -290,7 +294,6 @@ class CoverFlowMixin(object): self.library_view.setCurrentIndex(idx) self.library_view.scroll_to_row(idx.row()) - def show_cover_browser(self): d = CBDialog(self, self.cover_flow) d.addAction(self.cb_splitter.action_toggle) @@ -313,7 +316,6 @@ class CoverFlowMixin(object): self.cb_dialog = None self.cb_splitter.button.set_state_to_show() - def sync_cf_to_listview(self, current, previous): if self.cover_flow_sync_flag and self.cover_flow.isVisible() and \ self.cover_flow.currentSlide() != current.row(): diff --git a/src/calibre/gui2/pictureflow/pictureflow.cpp b/src/calibre/gui2/pictureflow/pictureflow.cpp index 173a080301..d9811f02e7 100644 --- a/src/calibre/gui2/pictureflow/pictureflow.cpp +++ b/src/calibre/gui2/pictureflow/pictureflow.cpp @@ -318,6 +318,9 @@ struct SlideInfo PFreal cy; }; +static const QString OFFSET_KEY("offset"); +static const QString WIDTH_KEY("width"); + // PicturePlowPrivate {{{ class PictureFlowPrivate @@ -367,6 +370,7 @@ public: QTime previousPosTimestamp; int pixelDistanceMoved; int pixelsToMovePerSlide; + bool preserveAspectRatio; QFont subtitleFont; void setImages(FlowImages *images); @@ -421,6 +425,7 @@ PictureFlowPrivate::PictureFlowPrivate(PictureFlow* w, int queueLength_) slideHeight = 200; fontSize = 10; doReflections = true; + preserveAspectRatio = false; centerIndex = 0; queueLength = queueLength_; @@ -598,41 +603,58 @@ void PictureFlowPrivate::resetSlides() } } -static QImage prepareSurface(QImage img, int w, int h, bool doReflections) +static QImage prepareSurface(QImage srcimg, int w, int h, bool doReflections, bool preserveAspectRatio) { - img = img.scaled(w, h, Qt::IgnoreAspectRatio, Qt::SmoothTransformation); + // slightly larger, to accommodate for the reflection + int hs = int(h * REFLECTION_FACTOR), left = 0, top = 0, a = 0, r = 0, g = 0, b = 0, ht, x, y, bpp; + QImage img = (preserveAspectRatio) ? QImage(w, h, srcimg.format()) : srcimg.scaled(w, h, Qt::IgnoreAspectRatio, Qt::SmoothTransformation); + QRgb color; - // slightly larger, to accommodate for the reflection - int hs = int(h * REFLECTION_FACTOR); + // offscreen buffer: black is sweet + QImage result(hs, w, QImage::Format_RGB16); + result.fill(0); - // offscreen buffer: black is sweet - QImage result(hs, w, QImage::Format_RGB16); - result.fill(0); - - // transpose the image, this is to speed-up the rendering - // because we process one column at a time - // (and much better and faster to work row-wise, i.e in one scanline) - for(int x = 0; x < w; x++) - for(int y = 0; y < h; y++) - result.setPixel(y, x, img.pixel(x, y)); - - if (doReflections) { - // create the reflection - int ht = hs - h; - for(int x = 0; x < w; x++) - for(int y = 0; y < ht; y++) - { - QRgb color = img.pixel(x, img.height()-y-1); - //QRgb565 color = img.scanLine(img.height()-y-1) + x*sizeof(QRgb565); //img.pixel(x, img.height()-y-1); - int a = qAlpha(color); - int r = qRed(color) * a / 256 * (ht - y) / ht * 3/5; - int g = qGreen(color) * a / 256 * (ht - y) / ht * 3/5; - int b = qBlue(color) * a / 256 * (ht - y) / ht * 3/5; - result.setPixel(h+y, x, qRgb(r, g, b)); + if (preserveAspectRatio) { + QImage temp = srcimg.scaled(w, h, Qt::KeepAspectRatio, Qt::SmoothTransformation); + img = QImage(w, h, temp.format()); + img.fill(0); + left = (w - temp.width()) / 2; + top = h - temp.height(); + bpp = img.bytesPerLine() / img.width(); + x = temp.width() * bpp; + result.setText(OFFSET_KEY, QString::number(left)); + result.setText(WIDTH_KEY, QString::number(temp.width())); + for (y = 0; y < temp.height(); y++) { + const uchar *src = temp.scanLine(y); + uchar *dest = img.scanLine(top + y) + (bpp * left); + memcpy(dest, src, x); } - } + } - return result; + // transpose the image, this is to speed-up the rendering + // because we process one column at a time + // (and much better and faster to work row-wise, i.e in one scanline) + for(x = 0; x < w; x++) + for(y = 0; y < h; y++) + result.setPixel(y, x, img.pixel(x, y)); + + if (doReflections) { + // create the reflection + ht = hs - h; + for(x = 0; x < w; x++) + for(y = 0; y < ht; y++) + { + color = img.pixel(x, img.height()-y-1); + //QRgb565 color = img.scanLine(img.height()-y-1) + x*sizeof(QRgb565); //img.pixel(x, img.height()-y-1); + a = qAlpha(color); + r = qRed(color) * a / 256 * (ht - y) / ht * 3/5; + g = qGreen(color) * a / 256 * (ht - y) / ht * 3/5; + b = qBlue(color) * a / 256 * (ht - y) / ht * 3/5; + result.setPixel(h+y, x, qRgb(r, g, b)); + } + } + + return result; } @@ -668,12 +690,12 @@ QImage* PictureFlowPrivate::surface(int slideIndex) painter.setBrush(QBrush()); painter.drawRect(2, 2, slideWidth-3, slideHeight-3); painter.end(); - blankSurface = prepareSurface(blankSurface, slideWidth, slideHeight, doReflections); + blankSurface = prepareSurface(blankSurface, slideWidth, slideHeight, doReflections, preserveAspectRatio); } return &blankSurface; } - surfaceCache.insert(slideIndex, new QImage(prepareSurface(img, slideWidth, slideHeight, doReflections))); + surfaceCache.insert(slideIndex, new QImage(prepareSurface(img, slideWidth, slideHeight, doReflections, preserveAspectRatio))); return surfaceCache[slideIndex]; } @@ -874,8 +896,7 @@ QRect PictureFlowPrivate::renderCenterSlide(const SlideInfo &slide) { // Renders a slide to offscreen buffer. Returns a rect of the rendered area. // alpha=256 means normal, alpha=0 is fully black, alpha=128 half transparent // col1 and col2 limit the column for rendering. -QRect PictureFlowPrivate::renderSlide(const SlideInfo &slide, int alpha, -int col1, int col2) +QRect PictureFlowPrivate::renderSlide(const SlideInfo &slide, int alpha, int col1, int col2) { QImage* src = surface(slide.slideIndex); if(!src) @@ -913,6 +934,13 @@ int col1, int col2) bool flag = false; rect.setLeft(xi); + int img_offset = 0, img_width = 0; + bool slide_moving_to_center = false; + if (preserveAspectRatio) { + img_offset = src->text(OFFSET_KEY).toInt(); + img_width = src->text(WIDTH_KEY).toInt(); + slide_moving_to_center = slide.slideIndex == target && target != centerIndex; + } for(int x = qMax(xi, col1); x <= col2; x++) { PFreal hity = 0; @@ -935,6 +963,17 @@ int col1, int col2) break; if(column < 0) continue; + if (preserveAspectRatio && !slide_moving_to_center) { + // We dont want a black border at the edge of narrow images when the images are in the left or right stacks + if (slide.slideIndex < centerIndex) { + column = qMin(column + img_offset, sw - 1); + } else if (slide.slideIndex == centerIndex) { + if (target > centerIndex) column = qMin(column + img_offset, sw - 1); + else if (target < centerIndex) column = qMax(column - sw + img_offset + img_width, 0); + } else { + column = qMax(column - sw + img_offset + img_width, 0); + } + } rect.setRight(x); if(!flag) @@ -1196,6 +1235,17 @@ void PictureFlow::setSlideSize(QSize size) d->setSlideSize(size); } +bool PictureFlow::preserveAspectRatio() const +{ + return d->preserveAspectRatio; +} + +void PictureFlow::setPreserveAspectRatio(bool preserve) +{ + d->preserveAspectRatio = preserve; + clearCaches(); +} + void PictureFlow::setSubtitleFont(QFont font) { d->subtitleFont = font; diff --git a/src/calibre/gui2/pictureflow/pictureflow.h b/src/calibre/gui2/pictureflow/pictureflow.h index bc427e8580..9d50b89edc 100644 --- a/src/calibre/gui2/pictureflow/pictureflow.h +++ b/src/calibre/gui2/pictureflow/pictureflow.h @@ -93,6 +93,7 @@ Q_OBJECT Q_PROPERTY(int currentSlide READ currentSlide WRITE setCurrentSlide) Q_PROPERTY(QSize slideSize READ slideSize WRITE setSlideSize) Q_PROPERTY(QFont subtitleFont READ subtitleFont WRITE setSubtitleFont) + Q_PROPERTY(bool preserveAspectRatio READ preserveAspectRatio WRITE setPreserveAspectRatio) public: /*! @@ -121,6 +122,16 @@ public: */ void setSlideSize(QSize size); + /*! + Returns whether aspect ration is preserved when scaling images + */ + bool preserveAspectRatio() const; + + /*! + Whether to preserve aspect ration when scaling images + */ + void setPreserveAspectRatio(bool preserve); + /*! Turn the reflections on/off. */ diff --git a/src/calibre/gui2/pictureflow/pictureflow.sip b/src/calibre/gui2/pictureflow/pictureflow.sip index 0fab379147..3754a538ce 100644 --- a/src/calibre/gui2/pictureflow/pictureflow.sip +++ b/src/calibre/gui2/pictureflow/pictureflow.sip @@ -41,6 +41,10 @@ public : void setSlideSize(QSize size); + bool preserveAspectRatio() const; + + void setPreserveAspectRatio(bool preserve); + QFont subtitleFont() const; void setSubtitleFont(QFont font); diff --git a/src/calibre/gui2/preferences/look_feel.py b/src/calibre/gui2/preferences/look_feel.py index e886cf8dec..6ced51f3e1 100644 --- a/src/calibre/gui2/preferences/look_feel.py +++ b/src/calibre/gui2/preferences/look_feel.py @@ -183,6 +183,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): r('use_roman_numerals_for_series_number', config) r('separate_cover_flow', config, restart_required=True) r('cb_fullscreen', gprefs) + r('cb_preserve_aspect_ratio', gprefs) choices = [(_('Off'), 'off'), (_('Small'), 'small'), (_('Medium'), 'medium'), (_('Large'), 'large')] @@ -461,6 +462,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): gui.library_view.refresh_book_details() if hasattr(gui.cover_flow, 'setShowReflections'): gui.cover_flow.setShowReflections(gprefs['cover_browser_reflections']) + gui.cover_flow.setPreserveAspectRatio(gprefs['cb_preserve_aspect_ratio']) gui.library_view.refresh_row_sizing() gui.grid_view.refresh_settings() diff --git a/src/calibre/gui2/preferences/look_feel.ui b/src/calibre/gui2/preferences/look_feel.ui index c1153d49b3..6721fb1e7c 100644 --- a/src/calibre/gui2/preferences/look_feel.ui +++ b/src/calibre/gui2/preferences/look_feel.ui @@ -897,7 +897,7 @@ a few top-level elements. - + Qt::Vertical @@ -913,14 +913,14 @@ a few top-level elements. - + When showing cover browser in separate window, show it &fullscreen - + margin-left: 1.5em @@ -940,6 +940,17 @@ a few top-level elements. + + + + Show covers in their original aspect ratio instead of resizing +them to all have the same width and height + + + Preserve &aspect ratio of covers displayed in the cover browser + + + From b7913ec9719e77ad0ddc1314e0e6eb6be8af4f04 Mon Sep 17 00:00:00 2001 From: Andres Gomez Date: Sat, 22 Mar 2014 12:46:28 +0200 Subject: [PATCH 095/122] driver: Nokia Maemo/MeeGo devices support PDF --- src/calibre/devices/nokia/driver.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/calibre/devices/nokia/driver.py b/src/calibre/devices/nokia/driver.py index f993817461..1f72e8790f 100644 --- a/src/calibre/devices/nokia/driver.py +++ b/src/calibre/devices/nokia/driver.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- __license__ = 'GPL v3' -__copyright__ = '2009, John Schember ' +__copyright__ = '2009-2014, John Schember and Andres Gomez ' __docformat__ = 'restructuredtext en' ''' @@ -15,12 +15,12 @@ class N770(USBMS): name = 'Nokia 770 Device Interface' gui_name = 'Nokia 770' description = _('Communicate with the Nokia 770 internet tablet.') - author = 'John Schember' + author = 'John Schember and Andres Gomez' supported_platforms = ['windows', 'linux', 'osx'] # Ordered list of supported formats - FORMATS = ['mobi', 'prc', 'epub', 'html', 'zip', 'fb2', 'chm', 'pdb', - 'tcr', 'txt', 'rtf'] + FORMATS = ['mobi', 'prc', 'epub', 'pdf', 'html', 'zip', 'fb2', 'chm', + 'pdb', 'tcr', 'txt', 'rtf'] VENDOR_ID = [0x421] PRODUCT_ID = [0x431] From 2b3b87af71bf0406c2e52a0b6bee16f683d597bb Mon Sep 17 00:00:00 2001 From: Andres Gomez Date: Sat, 22 Mar 2014 13:50:05 +0200 Subject: [PATCH 096/122] driver: Updated the Nokia 770 device information --- src/calibre/devices/nokia/driver.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/calibre/devices/nokia/driver.py b/src/calibre/devices/nokia/driver.py index 1f72e8790f..84bd97f28f 100644 --- a/src/calibre/devices/nokia/driver.py +++ b/src/calibre/devices/nokia/driver.py @@ -14,7 +14,7 @@ class N770(USBMS): name = 'Nokia 770 Device Interface' gui_name = 'Nokia 770' - description = _('Communicate with the Nokia 770 internet tablet.') + description = _('Communicate with the Nokia 770 Internet Tablet.') author = 'John Schember and Andres Gomez' supported_platforms = ['windows', 'linux', 'osx'] @@ -29,7 +29,7 @@ class N770(USBMS): VENDOR_NAME = 'NOKIA' WINDOWS_MAIN_MEM = '770' - MAIN_MEMORY_VOLUME_LABEL = 'N770 Main Memory' + MAIN_MEMORY_VOLUME_LABEL = 'Nokia 770 Main Memory' EBOOK_DIR_MAIN = 'My Ebooks' SUPPORTS_SUB_DIRS = True From 1eb1a3225433b43fe17593a508a62d22b7444f95 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 22 Mar 2014 17:38:34 +0530 Subject: [PATCH 097/122] Cover Browser: Fix visual "pop" when scrolling the first time --- src/calibre/gui2/pictureflow/pictureflow.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/calibre/gui2/pictureflow/pictureflow.cpp b/src/calibre/gui2/pictureflow/pictureflow.cpp index d9811f02e7..1068dd56dd 100644 --- a/src/calibre/gui2/pictureflow/pictureflow.cpp +++ b/src/calibre/gui2/pictureflow/pictureflow.cpp @@ -496,9 +496,9 @@ void PictureFlowPrivate::setCurrentSlide(int index) { animateTimer.stop(); step = 0; - centerIndex = qBound(index, 0, slideImages->count()-1); + centerIndex = qBound(0, index, qMax(0, slideImages->count()-1)); target = centerIndex; - slideFrame = ((long long)index) << 16; + slideFrame = ((long long)centerIndex) << 16; resetSlides(); triggerRender(); widget->emitcurrentChanged(centerIndex); From 34c1e4c24fb05e17b6f60c3d18c26070994fbdb7 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 22 Mar 2014 18:17:09 +0530 Subject: [PATCH 098/122] PDF Output: Fix using __SECTION__ in header and footer templates resolving to the inscorrect section if a page with no sections follows a page with multiple sections. Fixes #1295236 [[Conversion HTML-> PDF] PDF header/footer uses wrong _SECTION_ when a single page contains more than one section](https://bugs.launchpad.net/calibre/+bug/1295236) --- src/calibre/ebooks/pdf/render/from_html.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/calibre/ebooks/pdf/render/from_html.py b/src/calibre/ebooks/pdf/render/from_html.py index d6367223db..1e882d8385 100644 --- a/src/calibre/ebooks/pdf/render/from_html.py +++ b/src/calibre/ebooks/pdf/render/from_html.py @@ -10,6 +10,7 @@ __docformat__ = 'restructuredtext en' import json, os from future_builtins import map from math import floor +from collections import defaultdict from PyQt4.Qt import (QObject, QPainter, Qt, QSize, QString, QTimer, pyqtProperty, QEventLoop, QPixmap, QRect, pyqtSlot) @@ -310,7 +311,7 @@ class PDFWriter(QObject): evaljs('document.getElementById("MathJax_Message").style.display="none";') def get_sections(self, anchor_map): - sections = {} + sections = defaultdict(list) ci = os.path.abspath(os.path.normcase(self.current_item)) if self.toc is not None: for toc in self.toc.flat(): @@ -323,8 +324,7 @@ class PDFWriter(QObject): col = 0 if frag and frag in anchor_map: col = anchor_map[frag]['column'] - if col not in sections: - sections[col] = toc.text or _('Untitled') + sections[col].append(toc.text or _('Untitled')) return sections @@ -380,7 +380,11 @@ class PDFWriter(QObject): mf = self.view.page().mainFrame() while True: if col in sections: - self.current_section = sections[col] + self.current_section = sections[col][0] + elif col - 1 in sections: + # Ensure we are using the last section on the previous page as + # the section for this page, since this page has no sections + self.current_section = sections[col][-1] self.doc.init_page() if self.header or self.footer: evaljs('paged_display.update_header_footer(%d)'%self.current_page_num) From e37f3c7171acc9c624ef6232fe3755f6c7f6884c Mon Sep 17 00:00:00 2001 From: Andres Gomez Date: Sat, 22 Mar 2014 14:48:57 +0200 Subject: [PATCH 099/122] driver: Added Nokia N800 device support --- src/calibre/devices/nokia/driver.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/calibre/devices/nokia/driver.py b/src/calibre/devices/nokia/driver.py index 84bd97f28f..a5ebf59bfe 100644 --- a/src/calibre/devices/nokia/driver.py +++ b/src/calibre/devices/nokia/driver.py @@ -35,16 +35,16 @@ class N770(USBMS): SUPPORTS_SUB_DIRS = True class N810(N770): - name = 'Nokia 810 Device Interface' - gui_name = 'Nokia 810/900/9' - description = _('Communicate with the Nokia 810/900 internet tablet.') + name = 'Nokia N800/N810/N900/N9 Device Interface' + gui_name = 'Nokia N800/N810/N900/N9' + description = _('Communicate with the Nokia N800/N810/N900/N9 Maemo/MeeGo devices.') - PRODUCT_ID = [0x96, 0x1c7, 0x0518] + PRODUCT_ID = [0x4c3, 0x96, 0x1c7, 0x0518] BCD = [0x316] - WINDOWS_MAIN_MEM = ['N810', 'N900', 'NOKIA_N9'] + WINDOWS_MAIN_MEM = ['N800', 'N810', 'N900', 'NOKIA_N9'] - MAIN_MEMORY_VOLUME_LABEL = 'Nokia Tablet Main Memory' + MAIN_MEMORY_VOLUME_LABEL = 'Nokia Maemo/MeeGo device Main Memory' class E71X(USBMS): From 201909c853b4ebf0a6ff7651cbb21d888f6c8e7a Mon Sep 17 00:00:00 2001 From: Andres Gomez Date: Sat, 22 Mar 2014 14:50:22 +0200 Subject: [PATCH 100/122] driver: Added Nokia N950 device support --- src/calibre/devices/nokia/driver.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/calibre/devices/nokia/driver.py b/src/calibre/devices/nokia/driver.py index a5ebf59bfe..7749f85e26 100644 --- a/src/calibre/devices/nokia/driver.py +++ b/src/calibre/devices/nokia/driver.py @@ -35,14 +35,14 @@ class N770(USBMS): SUPPORTS_SUB_DIRS = True class N810(N770): - name = 'Nokia N800/N810/N900/N9 Device Interface' - gui_name = 'Nokia N800/N810/N900/N9' - description = _('Communicate with the Nokia N800/N810/N900/N9 Maemo/MeeGo devices.') + name = 'Nokia N800/N810/N900/N950/N9 Device Interface' + gui_name = 'Nokia N800/N810/N900/N950/N9' + description = _('Communicate with the Nokia N800/N810/N900/N950/N9 Maemo/MeeGo devices.') - PRODUCT_ID = [0x4c3, 0x96, 0x1c7, 0x0518] + PRODUCT_ID = [0x4c3, 0x96, 0x1c7, 0x3d1, 0x518] BCD = [0x316] - WINDOWS_MAIN_MEM = ['N800', 'N810', 'N900', 'NOKIA_N9'] + WINDOWS_MAIN_MEM = ['N800', 'N810', 'N900', 'NOKIA_N950', 'NOKIA_N9'] MAIN_MEMORY_VOLUME_LABEL = 'Nokia Maemo/MeeGo device Main Memory' From dc655ac435adcd97456e07c0848a6c7c5e96bb3c Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 22 Mar 2014 21:44:35 +0530 Subject: [PATCH 101/122] ... --- src/calibre/utils/icu.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/calibre/utils/icu.py b/src/calibre/utils/icu.py index 551a1f4c04..33bcc480f3 100644 --- a/src/calibre/utils/icu.py +++ b/src/calibre/utils/icu.py @@ -32,7 +32,8 @@ try: _icu.set_default_encoding('utf-8') del senc except: - pass + import traceback + traceback.print_exc() try: fenc = sys.getfilesystemencoding() @@ -40,7 +41,8 @@ try: _icu.set_filesystem_encoding('utf-8') del fenc except: - pass + import traceback + traceback.print_exc() def collator(): global _collator, _locale From 9f56b608e0a2cb3fd6184980a6d452f54b54fca2 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 22 Mar 2014 22:59:01 +0530 Subject: [PATCH 102/122] ... --- src/calibre/utils/icu.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/calibre/utils/icu.py b/src/calibre/utils/icu.py index 33bcc480f3..0fa9262de9 100644 --- a/src/calibre/utils/icu.py +++ b/src/calibre/utils/icu.py @@ -28,8 +28,8 @@ _nmodes = {m:getattr(_icu, 'UNORM_'+m, None) for m in ('NFC', 'NFD', 'NFKC', 'NF try: senc = sys.getdefaultencoding() - if not senc or senc.lower() == 'ascii': - _icu.set_default_encoding('utf-8') + if not senc or senc.lower() == b'ascii': + _icu.set_default_encoding(b'utf-8') del senc except: import traceback @@ -37,8 +37,8 @@ except: try: fenc = sys.getfilesystemencoding() - if not fenc or fenc.lower() == 'ascii': - _icu.set_filesystem_encoding('utf-8') + if not fenc or fenc.lower() == b'ascii': + _icu.set_filesystem_encoding(b'utf-8') del fenc except: import traceback From 9e9d2db2d013beb683c831d60c42c5b6a3c64856 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 23 Mar 2014 09:32:04 +0530 Subject: [PATCH 103/122] Update Der Tagesspiegel --- recipes/tagesspiegel.recipe | 105 ++++++++++++------------------------ 1 file changed, 34 insertions(+), 71 deletions(-) diff --git a/recipes/tagesspiegel.recipe b/recipes/tagesspiegel.recipe index 71191065f1..7c0ccede9c 100644 --- a/recipes/tagesspiegel.recipe +++ b/recipes/tagesspiegel.recipe @@ -1,20 +1,18 @@ -__license__ = 'GPL v3' -__copyright__ = '2010 Ingo Paschke ' - -''' -Fetch Tagesspiegel. -''' -import string, re -from calibre import strftime from calibre.web.feeds.news import BasicNewsRecipe -class TagesspiegelRSS(BasicNewsRecipe): +class TagesspiegelRss(BasicNewsRecipe): title = u'Der Tagesspiegel' - __author__ = 'Ingo Paschke' - language = 'de' - oldest_article = 7 + oldest_article = 1 max_articles_per_feed = 100 + language = 'de' publication_type = 'newspaper' + auto_cleanup = True + no_stylesheets = True + remove_stylesheets = True + remove_javascript = True + remove_empty_feeds = True + encoding = 'utf-8' + use_embedded_content = False extra_css = ''' .hcf-overline{color:#990000; font-family:Arial,Helvetica,sans-serif;font-size:xx-small;display:block} @@ -30,69 +28,34 @@ class TagesspiegelRSS(BasicNewsRecipe): .hcf-smart-box{font-family: Arial, Helvetica, sans-serif; font-size: xx-small; margin: 0px 15px 8px 0px; width: 300px;} ''' - no_stylesheets = True - no_javascript = True - remove_empty_feeds = True - encoding = 'utf-8' remove_tags = [{'class':'hcf-header'}, {'class':'hcf-atlas'}, {'class':'hcf-colon'}, {'class':'hcf-date hcf-separate'}] + feeds = [ + (u'Politik', u'http://www.tagesspiegel.de/contentexport/feed/politik'), + (u'Meinung', u'http://www.tagesspiegel.de/contentexport/feed/meinung'), + (u'Berlin', u'http://www.tagesspiegel.de/contentexport/feed/berlin'), + (u'Wirtschaft', u'http://www.tagesspiegel.de/contentexport/feed/wirtschaft'), + (u'Sport', u'http://www.tagesspiegel.de/contentexport/feed/sport'), + (u'Kultur', u'http://www.tagesspiegel.de/contentexport/feed/kultur'), + (u'Weltspiegel', u'http://www.tagesspiegel.de/contentexport/feed/weltspiegel'), + (u'Medien', u'http://www.tagesspiegel.de/contentexport/feed/medien'), + (u'Wissen', u'http://www.tagesspiegel.de/contentexport/feed/wissen') + ] + def print_version(self, url): - url = url.split('/') + # print url + u = url.find('0L0Stagesspiegel0Bde') + u = 'http://www.tagesspiegel.de' + url[u + 20:] + u = u.replace('0C', '/') + u = u.replace('0E', '-') + u = u.replace('A', '') + u = u.replace('0B', '.') + u = u.replace('.html/story01.htm', '.html') + url = u.split('/') url[-1] = 'v_print,%s?p='%url[-1] - return '/'.join(url) + u = '/'.join(url) + # print u + return u def get_masthead_url(self): return 'http://www.tagesspiegel.de/images/tsp_logo/3114/6.png' - - def parse_index(self): - soup = self.index_to_soup('http://www.tagesspiegel.de/zeitung/') - - def feed_title(div): - return ''.join(div.findAll(text=True, recursive=False)).strip() if div is not None else None - - articles = {} - links = set() - key = None - ans = [] - maincol = soup.find('div', attrs={'class':re.compile('hcf-main-col')}) - - for div in maincol.findAll(True, attrs={'class':['hcf-teaser', 'hcf-header', 'story headline', 'hcf-teaser hcf-last']}): - - if div['class'] == 'hcf-header': - try: - key = string.capwords(feed_title(div.em)) - articles[key] = [] - ans.append(key) - except: - continue - - elif div['class'] in ['hcf-teaser', 'hcf-teaser hcf-last'] and getattr(div.contents[0],'name','') == 'h2': - a = div.find('a', href=True) - if not a: - continue - url = 'http://www.tagesspiegel.de' + a['href'] - - # check for duplicates - if url in links: - continue - links.add(url) - - title = self.tag_to_string(a, use_alt=True).strip() - description = '' - pubdate = strftime('%a, %d %b') - summary = div.find('p', attrs={'class':'hcf-teaser'}) - if summary: - description = self.tag_to_string(summary, use_alt=False) - - feed = key if key is not None else 'Uncategorized' - if not articles.has_key(feed): - articles[feed] = [] - if not 'podcasts' in url: - articles[feed].append( - dict(title=title, url=url, date=pubdate, - description=re.sub('mehr$', '', description), - content='')) - - ans = [(key, articles[key]) for key in ans if articles.has_key(key)] - - return ans From bbc75357184b34b3556951c48352ae02d0b790ec Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 23 Mar 2014 11:20:25 +0530 Subject: [PATCH 104/122] ... --- src/calibre/ebooks/pdf/render/from_html.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/ebooks/pdf/render/from_html.py b/src/calibre/ebooks/pdf/render/from_html.py index 1e882d8385..d4711381df 100644 --- a/src/calibre/ebooks/pdf/render/from_html.py +++ b/src/calibre/ebooks/pdf/render/from_html.py @@ -384,7 +384,7 @@ class PDFWriter(QObject): elif col - 1 in sections: # Ensure we are using the last section on the previous page as # the section for this page, since this page has no sections - self.current_section = sections[col][-1] + self.current_section = sections[col-1][-1] self.doc.init_page() if self.header or self.footer: evaljs('paged_display.update_header_footer(%d)'%self.current_page_num) From 1fa5a5af1aee6591d8ac79a8e610ef92678b02be Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 23 Mar 2014 11:41:21 +0530 Subject: [PATCH 105/122] PDF Output: Enable using javascript inside header and footer templates --- manual/conversion.rst | 8 +++++++- resources/compiled_coffeescript.zip | Bin 81197 -> 81622 bytes src/calibre/ebooks/oeb/display/paged.coffee | 6 ++++++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/manual/conversion.rst b/manual/conversion.rst index 5576209e01..ad7440e99f 100644 --- a/manual/conversion.rst +++ b/manual/conversion.rst @@ -840,7 +840,7 @@ template:: This will display the title at the left and the author at the right, in a font size smaller than the main text. -Finally, you can also use the current section in templates, as shown below:: +You can also use the current section in templates, as shown below::

_SECTION_

@@ -850,6 +850,12 @@ Outline). If the document has no table of contents then it will be replaced by empty text. If a single PDF page has multiple sections, the first section on the page will be used. +You can even use javascript inside the header and footer templates, for +example, the following template will cause page numbers to start at 4 instead +of 1:: + +

+ .. note:: When adding headers and footers make sure you set the page top and bottom margins to large enough values, under the Page Setup section of the conversion dialog. diff --git a/resources/compiled_coffeescript.zip b/resources/compiled_coffeescript.zip index e404bffba289e3d1009a0a53189efb82e702db00..69db556d5f71ed039e15bffa957fd020e1aaf28b 100644 GIT binary patch delta 428 zcmZ4ci{;v1mJJhT@F~QUyR3UHx!Nd!fx#$Ya>94X$=hf2@aQNMmF5*E7iAWd6i>d8 zt)YkDD%dKdmF6XvWaj5-6eJd<=9OqFRC6f+L0Mvv0z}XBRicdQ%s|=cg4&Ey91uZu z&FS|=8H3p%yvdC0!v66^sl}x^CB=FLrNtQ{2Nqk!y@+xv7TR~{Cf9JQZk zvh?vy@<7SdlG36)BsCf(8JWd;Y5DmjsYR0w-|B7tbo^ieS}09FXu%lC^zp~!`cl#9 zhL(&*%+c(O)6cUq%1m#vWYl6hATWKMC8Iaf1jXsUEgAKgZdgp$wqiV@lVogeVU(O| zWNwgRU~FQZmTH)6U}0=*nv`l`W}a%2oMx0}k!+f1rlg~wG`X=vH8-hLfy$HkJwM1b7Irv zovT$h&pvsu08R1qEGx!HCXesa4XqeOr{A(-G-BGsI-Sp&(VbbGpK|9Hl$8kZW@Tdl0xlqY#l*m%Z_dc@kEvQ|a$~97^dHuY3pCPFjg1VGjm%R` pEG#UNEmPA>jZ#gFOp;7e6Ah9q%#2KvOwCM84N|usuwfKq0st@aMAHBO diff --git a/src/calibre/ebooks/oeb/display/paged.coffee b/src/calibre/ebooks/oeb/display/paged.coffee index 946edd619b..8ed5ae0da1 100644 --- a/src/calibre/ebooks/oeb/display/paged.coffee +++ b/src/calibre/ebooks/oeb/display/paged.coffee @@ -8,6 +8,10 @@ log = window.calibre_utils.log +runscripts = (parent) -> + for script in parent.getElementsByTagName('script') + eval(script.text || script.textContent || script.innerHTML || '') + class PagedDisplay # This class is a namespace to expose functions via the # window.paged_display object. The most important functions are: @@ -238,8 +242,10 @@ class PagedDisplay section = py_bridge.section() if this.header != null this.header.innerHTML = this.header_template.replace(/_PAGENUM_/g, pagenum+"").replace(/_TITLE_/g, title+"").replace(/_AUTHOR_/g, author+"").replace(/_SECTION_/g, section+"") + runscripts(this.header) if this.footer != null this.footer.innerHTML = this.footer_template.replace(/_PAGENUM_/g, pagenum+"").replace(/_TITLE_/g, title+"").replace(/_AUTHOR_/g, author+"").replace(/_SECTION_/g, section+"") + runscripts(this.footer) fit_images: () -> # Ensure no images are wider than the available width in a column. Note From c2246fdd5a63f0a4ff39ec3e71d2fdf85a938764 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 23 Mar 2014 11:58:41 +0530 Subject: [PATCH 106/122] When restoring a db on windows if renaming the old db fails because some other process has locked the file, wait a little and try again --- src/calibre/db/restore.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/calibre/db/restore.py b/src/calibre/db/restore.py index 3f513de100..c26f4dfe70 100644 --- a/src/calibre/db/restore.py +++ b/src/calibre/db/restore.py @@ -5,7 +5,7 @@ __license__ = 'GPL v3' __copyright__ = '2010, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import re, os, traceback, shutil +import re, os, traceback, shutil, time from threading import Thread from operator import itemgetter @@ -269,7 +269,14 @@ class Restore(Thread): save_path = self.olddb = os.path.splitext(dbpath)[0]+'_pre_restore.db' if os.path.exists(save_path): os.remove(save_path) - os.rename(dbpath, save_path) + try: + os.rename(dbpath, save_path) + except OSError as err: + if getattr(err, 'winerror', None) == 32: # ERROR_SHARING_VIOLATION + time.sleep(4) # Wait a little for dropbox or the antivirus or whatever to release the file + os.rename(dbpath, save_path) + else: + raise shutil.copyfile(ndbpath, dbpath) From 462b429071eeeebc0bf2611c4ddcfe77b7e101a6 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 24 Mar 2014 11:11:14 +0530 Subject: [PATCH 107/122] ... --- src/calibre/utils/config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/calibre/utils/config.py b/src/calibre/utils/config.py index 2da7863192..c671dbe826 100644 --- a/src/calibre/utils/config.py +++ b/src/calibre/utils/config.py @@ -204,7 +204,7 @@ class DynamicConfig(dict): def decouple(self, prefix): self.file_path = os.path.join(os.path.dirname(self.file_path), prefix + os.path.basename(self.file_path)) - self.refresh(clear_current=False) + self.refresh() def refresh(self, clear_current=True): d = {} @@ -287,7 +287,7 @@ class XMLConfig(dict): def decouple(self, prefix): self.file_path = os.path.join(os.path.dirname(self.file_path), prefix + os.path.basename(self.file_path)) - self.refresh(clear_current=False) + self.refresh() def refresh(self, clear_current=True): d = {} From 587e5aba65c754068f209fbf15a5578a66087972 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 25 Mar 2014 00:36:05 +0530 Subject: [PATCH 108/122] Linux binary build: Fix worker processes not working on linux systems with bash >= 4.3 --- setup/installer/linux/freeze2.py | 23 +++------- setup/installer/linux/launcher.c | 74 ++++++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+), 16 deletions(-) create mode 100644 setup/installer/linux/launcher.c diff --git a/setup/installer/linux/freeze2.py b/setup/installer/linux/freeze2.py index 13da77f8db..daae99cafb 100644 --- a/setup/installer/linux/freeze2.py +++ b/setup/installer/linux/freeze2.py @@ -279,6 +279,12 @@ class LinuxFreeze(Command): modules['console'].append('calibre.linux') basenames['console'].append('calibre_postinstall') functions['console'].append('main') + c_launcher = '/tmp/calibre-c-launcher' + lsrc = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'launcher.c') + cmd = ['gcc', '-O2', '-DMAGICK_BASE="%s"' % self.magick_base, '-o', c_launcher, lsrc, ] + self.info('Compiling launcher') + self.run_builder(cmd, verbose=False) + for typ in ('console', 'gui', ): self.info('Processing %s launchers'%typ) for mod, bname, func in zip(modules[typ], basenames[typ], @@ -288,20 +294,6 @@ class LinuxFreeze(Command): xflags += ['-DMODULE="%s"'%mod, '-DBASENAME="%s"'%bname, '-DFUNCTION="%s"'%func] - launcher = textwrap.dedent('''\ - #!/bin/sh - path=`readlink -f $0` - base=`dirname $path` - lib=$base/lib - export QT_ACCESSIBILITY=0 # qt-at-spi causes crashes and performance issues in various distros, so disable it - export LD_LIBRARY_PATH=$lib:$LD_LIBRARY_PATH - export MAGICK_HOME=$base - export MAGICK_CONFIGURE_PATH=$lib/{1}/config - export MAGICK_CODER_MODULE_PATH=$lib/{1}/modules-Q16/coders - export MAGICK_CODER_FILTER_PATH=$lib/{1}/modules-Q16/filters - exec $base/bin/{0} "$@" - ''') - dest = self.j(self.obj_dir, bname+'.o') if self.newer(dest, [src, __file__]+headers): self.info('Compiling', bname) @@ -309,8 +301,7 @@ class LinuxFreeze(Command): self.run_builder(cmd, verbose=False) exe = self.j(self.bin_dir, bname) sh = self.j(self.base, bname) - with open(sh, 'wb') as f: - f.write(launcher.format(bname, self.magick_base)) + shutil.copy2(c_launcher, sh) os.chmod(sh, stat.S_IREAD|stat.S_IEXEC|stat.S_IWRITE|stat.S_IRGRP|stat.S_IXGRP|stat.S_IROTH|stat.S_IXOTH) diff --git a/setup/installer/linux/launcher.c b/setup/installer/linux/launcher.c new file mode 100644 index 0000000000..7e501b1dea --- /dev/null +++ b/setup/installer/linux/launcher.c @@ -0,0 +1,74 @@ +/* + * launcher.c + * Copyright (C) 2014 Kovid Goyal + * + * Distributed under terms of the GPL3 license. + */ + +#include +#include +#include +#include +#include + +#define PATHLEN 1023 + +int main(int argc, char **argv) { + static char buf[PATHLEN+1] = {0}, lib[PATHLEN+1] = {0}, base[PATHLEN+1] = {0}, exe[PATHLEN+1] = {0}, *ldp = NULL; + + if (readlink("/proc/self/exe", buf, PATHLEN) == -1) { + fprintf(stderr, "Failed to read path of executable with error: %s\n", strerror(errno)); + return 1; + } + strncpy(lib, buf, PATHLEN); + strncpy(base, dirname(lib), PATHLEN); + snprintf(exe, PATHLEN, "%s/bin/%s", base, basename(buf)); + memset(lib, 0, PATHLEN); + snprintf(lib, PATHLEN, "%s/lib", base); + + /* qt-at-spi causes crashes and performance issues in various distros, so disable it */ + if (setenv("QT_ACCESSIBILITY", "0", 1) != 0) { + fprintf(stderr, "Failed to set environment variable with error: %s\n", strerror(errno)); + return 1; + } + + if (setenv("MAGICK_HOME", base, 1) != 0) { + fprintf(stderr, "Failed to set environment variable with error: %s\n", strerror(errno)); + return 1; + } + memset(buf, 0, PATHLEN); snprintf(buf, PATHLEN, "%s/%s/config", lib, MAGICK_BASE); + if (setenv("MAGICK_CONFIGURE_PATH", buf, 1) != 0) { + fprintf(stderr, "Failed to set environment variable with error: %s\n", strerror(errno)); + return 1; + } + memset(buf, 0, PATHLEN); snprintf(buf, PATHLEN, "%s/%s/modules-Q16/coders", lib, MAGICK_BASE); + if (setenv("MAGICK_CODER_MODULE_PATH", buf, 1) != 0) { + fprintf(stderr, "Failed to set environment variable with error: %s\n", strerror(errno)); + return 1; + } + memset(buf, 0, PATHLEN); snprintf(buf, PATHLEN, "%s/%s/modules-Q16/filters", lib, MAGICK_BASE); + if (setenv("MAGICK_CODER_FILTER_PATH", buf, 1) != 0) { + fprintf(stderr, "Failed to set environment variable with error: %s\n", strerror(errno)); + return 1; + } + + memset(buf, 0, PATHLEN); + ldp = getenv("LD_LIBRARY_PATH"); + if (ldp == NULL) strncpy(buf, lib, PATHLEN); + else snprintf(buf, PATHLEN, "%s:%s", lib, ldp); + if (setenv("LD_LIBRARY_PATH", buf, 1) != 0) { + fprintf(stderr, "Failed to set environment variable with error: %s\n", strerror(errno)); + return 1; + } + + argv[0] = exe; + if (execv(exe, argv) == -1) { + fprintf(stderr, "Failed to execute binary: %s with error: %s\n", exe, strerror(errno)); + return 1; + } + + return 0; +} + + + From 14ddd035b9bdffc707056e1b088c6f0804a2d591 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 25 Mar 2014 13:10:28 +0530 Subject: [PATCH 109/122] Use abstract named sockets on linux for IPC, to avoid use of temp files --- src/calibre/gui2/main.py | 4 ++-- src/calibre/utils/ipc/__init__.py | 11 ++++++---- src/calibre/utils/ipc/server.py | 35 ++++++++++++++++++++++--------- 3 files changed, 34 insertions(+), 16 deletions(-) diff --git a/src/calibre/gui2/main.py b/src/calibre/gui2/main.py index f88238d44f..6956f14204 100644 --- a/src/calibre/gui2/main.py +++ b/src/calibre/gui2/main.py @@ -365,7 +365,7 @@ def cant_start(msg=_('If you are sure it is not running')+', ', else: where += _('lower right region of the screen.') if what is None: - if iswindows: + if iswindows or islinux: what = _('try rebooting your computer.') else: what = _('try deleting the file')+': '+ gui_socket_address() @@ -436,7 +436,7 @@ def main(args=sys.argv): try: listener = Listener(address=gui_socket_address()) except socket.error: - if iswindows: + if iswindows or islinux: cant_start() if os.path.exists(gui_socket_address()): os.remove(gui_socket_address()) diff --git a/src/calibre/utils/ipc/__init__.py b/src/calibre/utils/ipc/__init__.py index 54c10b5058..9735478a40 100644 --- a/src/calibre/utils/ipc/__init__.py +++ b/src/calibre/utils/ipc/__init__.py @@ -9,7 +9,7 @@ __docformat__ = 'restructuredtext en' import os, errno from threading import Thread -from calibre.constants import iswindows, get_windows_username +from calibre.constants import iswindows, get_windows_username, islinux ADDRESS = None @@ -37,12 +37,15 @@ def gui_socket_address(): if user: ADDRESS += '-' + user[:100] + 'x' else: - from tempfile import gettempdir - tmp = gettempdir() user = os.environ.get('USER', '') if not user: user = os.path.basename(os.path.expanduser('~')) - ADDRESS = os.path.join(tmp, user+'-calibre-gui.socket') + if islinux: + ADDRESS = (u'\0%s-calibre-gui.socket' % user).encode('ascii') + else: + from tempfile import gettempdir + tmp = gettempdir() + ADDRESS = os.path.join(tmp, user+'-calibre-gui.socket') return ADDRESS class RC(Thread): diff --git a/src/calibre/utils/ipc/server.py b/src/calibre/utils/ipc/server.py index fbbe411f84..20a2b0ed64 100644 --- a/src/calibre/utils/ipc/server.py +++ b/src/calibre/utils/ipc/server.py @@ -6,7 +6,7 @@ __license__ = 'GPL v3' __copyright__ = '2009, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import sys, os, cPickle, time, tempfile +import sys, os, cPickle, time, tempfile, errno from math import ceil from threading import Thread, RLock from Queue import Queue, Empty @@ -18,7 +18,7 @@ from calibre.utils.ipc import eintr_retry_call from calibre.utils.ipc.launch import Worker from calibre.utils.ipc.worker import PARALLEL_FUNCS from calibre import detect_ncpus as cpu_count -from calibre.constants import iswindows, DEBUG +from calibre.constants import iswindows, DEBUG, islinux from calibre.ptempfile import base_dir _counter = 0 @@ -84,6 +84,22 @@ class ConnectedWorker(Thread): class CriticalError(Exception): pass +_name_counter = 0 + +def create_linux_listener(authkey, backlog=4): + # Use abstract named sockets on linux to avoid creating unnecessary temp files + global _name_counter + prefix = u'\0calibre-ipc-listener-%d-%%d' % os.getpid() + while True: + _name_counter += 1 + address = (prefix % _name_counter).encode('ascii') + try: + return address, Listener(address=address, authkey=authkey, backlog=backlog) + except EnvironmentError as err: + if err.errno == errno.EADDRINUSE: + continue + raise + class Server(Thread): def __init__(self, notify_on_job_done=lambda x: x, pool_size=None, @@ -99,11 +115,13 @@ class Server(Thread): self.pool_size = limit if pool_size is None else pool_size self.notify_on_job_done = notify_on_job_done self.auth_key = os.urandom(32) - self.address = arbitrary_address('AF_PIPE' if iswindows else 'AF_UNIX') - if iswindows and self.address[1] == ':': - self.address = self.address[2:] - self.listener = Listener(address=self.address, - authkey=self.auth_key, backlog=4) + if islinux: + self.address, self.listener = create_linux_listener(self.auth_key, backlog=4) + else: + self.address = arbitrary_address('AF_PIPE' if iswindows else 'AF_UNIX') + if iswindows and self.address[1] == ':': + self.address = self.address[2:] + self.listener = Listener(address=self.address, authkey=self.auth_key, backlog=4) self.add_jobs_queue, self.changed_jobs_queue = Queue(), Queue() self.kill_queue = Queue() self.waiting_jobs = [] @@ -162,7 +180,6 @@ class Server(Thread): w = self.launch_worker(gui=gui, redirect_output=redirect_output) w.start_job(job) - def run(self): while True: try: @@ -280,8 +297,6 @@ class Server(Thread): pos += delta return ans - - def close(self): try: self.add_jobs_queue.put(None) From 2989b9b81932ffb31d16c23750fba2907e1e6ed7 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 25 Mar 2014 13:45:17 +0530 Subject: [PATCH 110/122] Replace another use of temp files for named sockets on linux Also prevent the multiprocessing module from calling unlink() on abstract named sockets. --- src/calibre/utils/ipc/server.py | 47 +++++++++++++++----------- src/calibre/utils/ipc/simple_worker.py | 8 ++--- 2 files changed, 30 insertions(+), 25 deletions(-) diff --git a/src/calibre/utils/ipc/server.py b/src/calibre/utils/ipc/server.py index 20a2b0ed64..9350163be6 100644 --- a/src/calibre/utils/ipc/server.py +++ b/src/calibre/utils/ipc/server.py @@ -86,19 +86,32 @@ class CriticalError(Exception): _name_counter = 0 -def create_linux_listener(authkey, backlog=4): - # Use abstract named sockets on linux to avoid creating unnecessary temp files - global _name_counter - prefix = u'\0calibre-ipc-listener-%d-%%d' % os.getpid() - while True: - _name_counter += 1 - address = (prefix % _name_counter).encode('ascii') - try: - return address, Listener(address=address, authkey=authkey, backlog=backlog) - except EnvironmentError as err: - if err.errno == errno.EADDRINUSE: - continue - raise +if islinux: + def create_listener(authkey, backlog=4): + # Use abstract named sockets on linux to avoid creating unnecessary temp files + global _name_counter + prefix = u'\0calibre-ipc-listener-%d-%%d' % os.getpid() + while True: + _name_counter += 1 + address = (prefix % _name_counter).encode('ascii') + try: + l = Listener(address=address, authkey=authkey, backlog=backlog) + if hasattr(l._listener._unlink, 'cancel'): + # multiprocessing tries to call unlink even on abstract + # named sockets, prevent it from doing so. + l._listener._unlink.cancel() + return address, l + except EnvironmentError as err: + if err.errno == errno.EADDRINUSE: + continue + raise +else: + def create_listener(authkey, backlog=4): + address = arbitrary_address('AF_PIPE' if iswindows else 'AF_UNIX') + if iswindows and address[1] == ':': + address = address[2:] + listener = Listener(address=address, authkey=authkey, backlog=backlog) + return address, listener class Server(Thread): @@ -115,13 +128,7 @@ class Server(Thread): self.pool_size = limit if pool_size is None else pool_size self.notify_on_job_done = notify_on_job_done self.auth_key = os.urandom(32) - if islinux: - self.address, self.listener = create_linux_listener(self.auth_key, backlog=4) - else: - self.address = arbitrary_address('AF_PIPE' if iswindows else 'AF_UNIX') - if iswindows and self.address[1] == ':': - self.address = self.address[2:] - self.listener = Listener(address=self.address, authkey=self.auth_key, backlog=4) + self.address, self.listener = create_listener(self.auth_key, backlog=4) self.add_jobs_queue, self.changed_jobs_queue = Queue(), Queue() self.kill_queue = Queue() self.waiting_jobs = [] diff --git a/src/calibre/utils/ipc/simple_worker.py b/src/calibre/utils/ipc/simple_worker.py index 2d24fec22b..d06550cdce 100644 --- a/src/calibre/utils/ipc/simple_worker.py +++ b/src/calibre/utils/ipc/simple_worker.py @@ -9,7 +9,7 @@ __docformat__ = 'restructuredtext en' import os, cPickle, traceback, time, importlib from binascii import hexlify, unhexlify -from multiprocessing.connection import Listener, arbitrary_address, Client +from multiprocessing.connection import Client from threading import Thread from contextlib import closing @@ -117,11 +117,9 @@ def communicate(ans, worker, listener, args, timeout=300, heartbeat=None, ans['result'] = cw.res['result'] def create_worker(env, priority='normal', cwd=None, func='main'): - address = arbitrary_address('AF_PIPE' if iswindows else 'AF_UNIX') - if iswindows and address[1] == ':': - address = address[2:] + from calibre.utils.ipc.server import create_listener auth_key = os.urandom(32) - listener = Listener(address=address, authkey=auth_key) + address, listener = create_listener(auth_key) env = dict(env) env.update({ From f5e1a13ac699129c7fa6b6a9ae156df67bc35b67 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 25 Mar 2014 16:39:08 +0530 Subject: [PATCH 111/122] Updated expired CA certificate --- resources/calibre-ebook-root-CA.crt | 60 +++++++++++++++-------------- setup/linux-installer.py | 60 +++++++++++++++-------------- 2 files changed, 62 insertions(+), 58 deletions(-) diff --git a/resources/calibre-ebook-root-CA.crt b/resources/calibre-ebook-root-CA.crt index cd47d2829b..df404b1272 100644 --- a/resources/calibre-ebook-root-CA.crt +++ b/resources/calibre-ebook-root-CA.crt @@ -1,32 +1,34 @@ -----BEGIN CERTIFICATE----- -MIIFlzCCA3+gAwIBAgIJAI67A/kD1DLtMA0GCSqGSIb3DQEBBQUAMGIxCzAJBgNV +MIIFzjCCA7agAwIBAgIJAPE9riMS7RUZMA0GCSqGSIb3DQEBBQUAMGIxCzAJBgNV BAYTAklOMRQwEgYDVQQIDAtNYWhhcmFzaHRyYTEPMA0GA1UEBwwGTXVtYmFpMRAw -DgYDVQQKDAdjYWxpYnJlMRowGAYDVQQDDBFjYWxpYnJlLWVib29rLmNvbTAeFw0x -NDAyMjMwNDAzNDFaFw0xNDAzMjUwNDAzNDFaMGIxCzAJBgNVBAYTAklOMRQwEgYD -VQQIDAtNYWhhcmFzaHRyYTEPMA0GA1UEBwwGTXVtYmFpMRAwDgYDVQQKDAdjYWxp -YnJlMRowGAYDVQQDDBFjYWxpYnJlLWVib29rLmNvbTCCAiIwDQYJKoZIhvcNAQEB -BQADggIPADCCAgoCggIBALZW3gMUCsloaMcGhqjIeZLUYarC0ers47qlpgfjJnwt -DYuOZjkqNkf7rBUE2XrK2FKKNsgYTDefArC3rmmkH7D3g7LO8yfY19L/xmFEt7zO -6hOea7kVrtINdTabli2ZKr3MOYFYt2SWMf8qkxBpQgxsY11bPYhIPi++QXJvcvO6 -JW3GQOh/wm0eZT9f7V3Msm9UwSDbk3IONPEp4nmPx6ZwNa9zUAfTMH0nHV9PB0wd -AXPHtKs/q9QTYt8GWXKzaalocOl/UJB4oBmgzaaZlqnNUOZ8cZNqwttRkYOep6er -dxDUDHLRNykyX0fE8DN9zf3X3IKGw2f2U56IKnRUMnBToL0+JiGbF3bCb+rJsoZZ -FKsntj1fF3EzSa/sEcyDf/rtt4wvgmk9FNAOew/D1GVYU/mbIV4wfdSqPISxNUpi -ZHb9m8RVeNm7HpoUsWVgrbHNjb/Pw7PllVdNMXwA8pvi6JMxKqn3Cvb5JDBsxYe8 -M3e2KjzqzBjgnvbx9QqC91TubKz1ftDKdX4yBoJuUiIZJckX2niIxXsqA0QOnvBF -6yN8TrK5F1zCQ74Z3RCTmGKqZWPuJC4VtF3k2Yyuwpg+fcUbRWFmld3XDJWlm1cb -mO3YLIju4lM7WGNE6OWQxMXB3puzxD1E8hYovS4W3EiXlw2qjxTMYofl9Iqir54v -AgMBAAGjUDBOMB0GA1UdDgQWBBRFarPkQ6DkrU6tIqmV5H6Wi5XGxDAfBgNVHSME -GDAWgBRFarPkQ6DkrU6tIqmV5H6Wi5XGxDAMBgNVHRMEBTADAQH/MA0GCSqGSIb3 -DQEBBQUAA4ICAQBAlBhF+greu0vYEDzz04HQjgfamxWQ4nXete8++9et1mcRw16i -RbEz/1ZeELz9KMwMpooVPIaYAWgqe+UNWuzHt0+jrH30NBcBv407G8eR/FWOU/cx -y/YMk3nXsAARoOcsFN1YSS1dNL1osezfsRStET8/bOEqpWD0yvt8wRWwh1hOCPVD -OpWTZx7+dZcK1Zh64Rm5mPYzbhWYxGGqNuZGFCuR9yI2bHHsFI69LryUKNf1cJ/N -dfvHt4GDxfF5ie4PWNgTp52wuI3YxNpsHgz9SmSEey6uVlA13vTO1QFX8Ymbyn6K -FRhr2LHY4iBdY+Gw47WnAqdo7uXpyM3wT6jI4gn7oENvCSUyM/JMSQqE1Etw0LBr -NIlC/RxN5wjcDvVCL/uS3PL6IW7R0wxrCQwBU3f5wMOnDM/R4EWJdS96zyb7Xnh3 -PQGoj6/vllymI7tuwRhEuvFknRRihu3vilHgtGczVXTG73nFJftLzvN/OhqSSQG/ -3c2JDX+vAy5jwPT/M3nPkrs68M4P77da1/BDZ0/KgJb/JzYZyNpq1nhWo3nMn+Sx -jq7y+h6ry8Omnlw7a/7CnNgvkLfP/uTfllL4erETFntHNh6LqCvpPNOqrvAP5keB -EB8yoJraypfuiNELOw1zSRksMxe2ac4b/dhDNStBTPC0egfRSm3FA0XoOQ== +DgYDVQQKDAdjYWxpYnJlMRowGAYDVQQDDBFjYWxpYnJlLWVib29rLmNvbTAgFw0x +NDAzMjUxMDU2MThaGA8yMTE0MDMwMTEwNTYxOFowYjELMAkGA1UEBhMCSU4xFDAS +BgNVBAgMC01haGFyYXNodHJhMQ8wDQYDVQQHDAZNdW1iYWkxEDAOBgNVBAoMB2Nh +bGlicmUxGjAYBgNVBAMMEWNhbGlicmUtZWJvb2suY29tMIICIjANBgkqhkiG9w0B +AQEFAAOCAg8AMIICCgKCAgEAtlbeAxQKyWhoxwaGqMh5ktRhqsLR6uzjuqWmB+Mm +fC0Ni45mOSo2R/usFQTZesrYUoo2yBhMN58CsLeuaaQfsPeDss7zJ9jX0v/GYUS3 +vM7qE55ruRWu0g11NpuWLZkqvcw5gVi3ZJYx/yqTEGlCDGxjXVs9iEg+L75Bcm9y +87olbcZA6H/CbR5lP1/tXcyyb1TBINuTcg408SnieY/HpnA1r3NQB9MwfScdX08H +TB0Bc8e0qz+r1BNi3wZZcrNpqWhw6X9QkHigGaDNppmWqc1Q5nxxk2rC21GRg56n +p6t3ENQMctE3KTJfR8TwM33N/dfcgobDZ/ZTnogqdFQycFOgvT4mIZsXdsJv6smy +hlkUqye2PV8XcTNJr+wRzIN/+u23jC+CaT0U0A57D8PUZVhT+ZshXjB91Ko8hLE1 +SmJkdv2bxFV42bsemhSxZWCtsc2Nv8/Ds+WVV00xfADym+LokzEqqfcK9vkkMGzF +h7wzd7YqPOrMGOCe9vH1CoL3VO5srPV+0Mp1fjIGgm5SIhklyRfaeIjFeyoDRA6e +8EXrI3xOsrkXXMJDvhndEJOYYqplY+4kLhW0XeTZjK7CmD59xRtFYWaV3dcMlaWb +VxuY7dgsiO7iUztYY0To5ZDExcHem7PEPUTyFii9LhbcSJeXDaqPFMxih+X0iqKv +ni8CAwEAAaOBhDCBgTAxBgNVHREEKjAoghFjYWxpYnJlLWVib29rLmNvbYITKi5j +YWxpYnJlLWVib29rLmNvbTAdBgNVHQ4EFgQURWqz5EOg5K1OrSKpleR+louVxsQw +HwYDVR0jBBgwFoAURWqz5EOg5K1OrSKpleR+louVxsQwDAYDVR0TBAUwAwEB/zAN +BgkqhkiG9w0BAQUFAAOCAgEANxijK3JQNZnrDYv7E5Ny17EtxV6ADggs8BIFLHrp +tRISYw8HpFIrIF/MDbHgYGp/xkefGKGEHeS7rUPYwdAbKM0sfoxKXm5e8GGe9L5K +pdG+ig1Ptm+Pae2Rcdj9RHKGmpAiKIF8a15l/Yj3jDVk06kx+lnT5fOePGhZBeuj +duBZ2vP39rFfcBtTvFmoQRwfoa46fZEoWoXb3YwzBqIhBg9m80R+E79/HsRPwA4L +pOvcFTr28jNp1OadgZ92sY9EYabes23amebz/P6IOjutqssIdrPSKqM9aphlGLXE +7YDxS9nSfX165Aa8NIWO95ivdbZplisnQ3rQM4pIdk7Z8FPhHftMdhekDREMxYKX +KXepi5tLyVnhETj+ifYBwqxZ024rlnpnHUWgjxRz5atKTAsbAgcxHOYTKMZoRAod +BK7lvjZ7+C/cqUc2c9FSG/HxkrfMpJHJlzMsanTBJ1+MeUybeBtp5E7gdNALbfh/ +BJ4eWw7X7q2oKape+7+OMX7aKAIysM7d2iVRuBofLBxOqzY6mzP8+Ro8zIgwFUeh +r6pbEa8P2DXnuZ+PtcMiClYKuSLlf6xRRDMnHCxvsu1zA/Ga3vZ6g0bd487DIsGP +tXHCYXttMGNxZDNVKS6rkrY2sT5xnJwvHwWmiooUZmSUFUdpqsvV5r9v89NMQ87L +gNA= -----END CERTIFICATE----- diff --git a/setup/linux-installer.py b/setup/linux-installer.py index a51878328c..aa7ae1ff0b 100644 --- a/setup/linux-installer.py +++ b/setup/linux-installer.py @@ -500,36 +500,38 @@ else: CACERT = b'''\ -----BEGIN CERTIFICATE----- -MIIFlzCCA3+gAwIBAgIJAI67A/kD1DLtMA0GCSqGSIb3DQEBBQUAMGIxCzAJBgNV +MIIFzjCCA7agAwIBAgIJAPE9riMS7RUZMA0GCSqGSIb3DQEBBQUAMGIxCzAJBgNV BAYTAklOMRQwEgYDVQQIDAtNYWhhcmFzaHRyYTEPMA0GA1UEBwwGTXVtYmFpMRAw -DgYDVQQKDAdjYWxpYnJlMRowGAYDVQQDDBFjYWxpYnJlLWVib29rLmNvbTAeFw0x -NDAyMjMwNDAzNDFaFw0xNDAzMjUwNDAzNDFaMGIxCzAJBgNVBAYTAklOMRQwEgYD -VQQIDAtNYWhhcmFzaHRyYTEPMA0GA1UEBwwGTXVtYmFpMRAwDgYDVQQKDAdjYWxp -YnJlMRowGAYDVQQDDBFjYWxpYnJlLWVib29rLmNvbTCCAiIwDQYJKoZIhvcNAQEB -BQADggIPADCCAgoCggIBALZW3gMUCsloaMcGhqjIeZLUYarC0ers47qlpgfjJnwt -DYuOZjkqNkf7rBUE2XrK2FKKNsgYTDefArC3rmmkH7D3g7LO8yfY19L/xmFEt7zO -6hOea7kVrtINdTabli2ZKr3MOYFYt2SWMf8qkxBpQgxsY11bPYhIPi++QXJvcvO6 -JW3GQOh/wm0eZT9f7V3Msm9UwSDbk3IONPEp4nmPx6ZwNa9zUAfTMH0nHV9PB0wd -AXPHtKs/q9QTYt8GWXKzaalocOl/UJB4oBmgzaaZlqnNUOZ8cZNqwttRkYOep6er -dxDUDHLRNykyX0fE8DN9zf3X3IKGw2f2U56IKnRUMnBToL0+JiGbF3bCb+rJsoZZ -FKsntj1fF3EzSa/sEcyDf/rtt4wvgmk9FNAOew/D1GVYU/mbIV4wfdSqPISxNUpi -ZHb9m8RVeNm7HpoUsWVgrbHNjb/Pw7PllVdNMXwA8pvi6JMxKqn3Cvb5JDBsxYe8 -M3e2KjzqzBjgnvbx9QqC91TubKz1ftDKdX4yBoJuUiIZJckX2niIxXsqA0QOnvBF -6yN8TrK5F1zCQ74Z3RCTmGKqZWPuJC4VtF3k2Yyuwpg+fcUbRWFmld3XDJWlm1cb -mO3YLIju4lM7WGNE6OWQxMXB3puzxD1E8hYovS4W3EiXlw2qjxTMYofl9Iqir54v -AgMBAAGjUDBOMB0GA1UdDgQWBBRFarPkQ6DkrU6tIqmV5H6Wi5XGxDAfBgNVHSME -GDAWgBRFarPkQ6DkrU6tIqmV5H6Wi5XGxDAMBgNVHRMEBTADAQH/MA0GCSqGSIb3 -DQEBBQUAA4ICAQBAlBhF+greu0vYEDzz04HQjgfamxWQ4nXete8++9et1mcRw16i -RbEz/1ZeELz9KMwMpooVPIaYAWgqe+UNWuzHt0+jrH30NBcBv407G8eR/FWOU/cx -y/YMk3nXsAARoOcsFN1YSS1dNL1osezfsRStET8/bOEqpWD0yvt8wRWwh1hOCPVD -OpWTZx7+dZcK1Zh64Rm5mPYzbhWYxGGqNuZGFCuR9yI2bHHsFI69LryUKNf1cJ/N -dfvHt4GDxfF5ie4PWNgTp52wuI3YxNpsHgz9SmSEey6uVlA13vTO1QFX8Ymbyn6K -FRhr2LHY4iBdY+Gw47WnAqdo7uXpyM3wT6jI4gn7oENvCSUyM/JMSQqE1Etw0LBr -NIlC/RxN5wjcDvVCL/uS3PL6IW7R0wxrCQwBU3f5wMOnDM/R4EWJdS96zyb7Xnh3 -PQGoj6/vllymI7tuwRhEuvFknRRihu3vilHgtGczVXTG73nFJftLzvN/OhqSSQG/ -3c2JDX+vAy5jwPT/M3nPkrs68M4P77da1/BDZ0/KgJb/JzYZyNpq1nhWo3nMn+Sx -jq7y+h6ry8Omnlw7a/7CnNgvkLfP/uTfllL4erETFntHNh6LqCvpPNOqrvAP5keB -EB8yoJraypfuiNELOw1zSRksMxe2ac4b/dhDNStBTPC0egfRSm3FA0XoOQ== +DgYDVQQKDAdjYWxpYnJlMRowGAYDVQQDDBFjYWxpYnJlLWVib29rLmNvbTAgFw0x +NDAzMjUxMDU2MThaGA8yMTE0MDMwMTEwNTYxOFowYjELMAkGA1UEBhMCSU4xFDAS +BgNVBAgMC01haGFyYXNodHJhMQ8wDQYDVQQHDAZNdW1iYWkxEDAOBgNVBAoMB2Nh +bGlicmUxGjAYBgNVBAMMEWNhbGlicmUtZWJvb2suY29tMIICIjANBgkqhkiG9w0B +AQEFAAOCAg8AMIICCgKCAgEAtlbeAxQKyWhoxwaGqMh5ktRhqsLR6uzjuqWmB+Mm +fC0Ni45mOSo2R/usFQTZesrYUoo2yBhMN58CsLeuaaQfsPeDss7zJ9jX0v/GYUS3 +vM7qE55ruRWu0g11NpuWLZkqvcw5gVi3ZJYx/yqTEGlCDGxjXVs9iEg+L75Bcm9y +87olbcZA6H/CbR5lP1/tXcyyb1TBINuTcg408SnieY/HpnA1r3NQB9MwfScdX08H +TB0Bc8e0qz+r1BNi3wZZcrNpqWhw6X9QkHigGaDNppmWqc1Q5nxxk2rC21GRg56n +p6t3ENQMctE3KTJfR8TwM33N/dfcgobDZ/ZTnogqdFQycFOgvT4mIZsXdsJv6smy +hlkUqye2PV8XcTNJr+wRzIN/+u23jC+CaT0U0A57D8PUZVhT+ZshXjB91Ko8hLE1 +SmJkdv2bxFV42bsemhSxZWCtsc2Nv8/Ds+WVV00xfADym+LokzEqqfcK9vkkMGzF +h7wzd7YqPOrMGOCe9vH1CoL3VO5srPV+0Mp1fjIGgm5SIhklyRfaeIjFeyoDRA6e +8EXrI3xOsrkXXMJDvhndEJOYYqplY+4kLhW0XeTZjK7CmD59xRtFYWaV3dcMlaWb +VxuY7dgsiO7iUztYY0To5ZDExcHem7PEPUTyFii9LhbcSJeXDaqPFMxih+X0iqKv +ni8CAwEAAaOBhDCBgTAxBgNVHREEKjAoghFjYWxpYnJlLWVib29rLmNvbYITKi5j +YWxpYnJlLWVib29rLmNvbTAdBgNVHQ4EFgQURWqz5EOg5K1OrSKpleR+louVxsQw +HwYDVR0jBBgwFoAURWqz5EOg5K1OrSKpleR+louVxsQwDAYDVR0TBAUwAwEB/zAN +BgkqhkiG9w0BAQUFAAOCAgEANxijK3JQNZnrDYv7E5Ny17EtxV6ADggs8BIFLHrp +tRISYw8HpFIrIF/MDbHgYGp/xkefGKGEHeS7rUPYwdAbKM0sfoxKXm5e8GGe9L5K +pdG+ig1Ptm+Pae2Rcdj9RHKGmpAiKIF8a15l/Yj3jDVk06kx+lnT5fOePGhZBeuj +duBZ2vP39rFfcBtTvFmoQRwfoa46fZEoWoXb3YwzBqIhBg9m80R+E79/HsRPwA4L +pOvcFTr28jNp1OadgZ92sY9EYabes23amebz/P6IOjutqssIdrPSKqM9aphlGLXE +7YDxS9nSfX165Aa8NIWO95ivdbZplisnQ3rQM4pIdk7Z8FPhHftMdhekDREMxYKX +KXepi5tLyVnhETj+ifYBwqxZ024rlnpnHUWgjxRz5atKTAsbAgcxHOYTKMZoRAod +BK7lvjZ7+C/cqUc2c9FSG/HxkrfMpJHJlzMsanTBJ1+MeUybeBtp5E7gdNALbfh/ +BJ4eWw7X7q2oKape+7+OMX7aKAIysM7d2iVRuBofLBxOqzY6mzP8+Ro8zIgwFUeh +r6pbEa8P2DXnuZ+PtcMiClYKuSLlf6xRRDMnHCxvsu1zA/Ga3vZ6g0bd487DIsGP +tXHCYXttMGNxZDNVKS6rkrY2sT5xnJwvHwWmiooUZmSUFUdpqsvV5r9v89NMQ87L +gNA= -----END CERTIFICATE----- ''' From 88f19064dcc4ad1c909bd03d5075c3785585aecc Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 25 Mar 2014 16:44:49 +0530 Subject: [PATCH 112/122] ... --- setup/linux-installer.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup/linux-installer.py b/setup/linux-installer.py index aa7ae1ff0b..780af4426f 100644 --- a/setup/linux-installer.py +++ b/setup/linux-installer.py @@ -19,6 +19,7 @@ py3 = sys.version_info[0] > 2 enc = getattr(sys.stdout, 'encoding', 'UTF-8') or 'utf-8' calibre_version = signature = None urllib = __import__('urllib.request' if py3 else 'urllib', fromlist=1) + if py3: unicode = str raw_input = input From 7a4f106f3d46ae349fe3d2b322576ae054c22489 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 25 Mar 2014 16:47:51 +0530 Subject: [PATCH 113/122] Update Courrier International --- recipes/courrierinternational.recipe | 58 ++++++++++++++++++++++------ 1 file changed, 46 insertions(+), 12 deletions(-) diff --git a/recipes/courrierinternational.recipe b/recipes/courrierinternational.recipe index 0ad471b9ba..aaea5c8995 100644 --- a/recipes/courrierinternational.recipe +++ b/recipes/courrierinternational.recipe @@ -19,23 +19,57 @@ class CourrierInternational(BasicNewsRecipe): max_articles_per_feed = 50 no_stylesheets = True + ignore_duplicate_articles = {'title', 'url'} + html2lrf_options = ['--base-font-size', '10'] + keep_only_tags = [ + dict(name='div', attrs={'class':'dessin'}), + dict(name='div', attrs={'class':'story-content'}), + ] + remove_tags = [ + dict(name='div', attrs={'class':re.compile('story-share storylinks|pager|event-expand')}), + dict(name='li', attrs={'class':'event-partage_outils'}), + dict(name='li', attrs={'class':'story-comment-link'}), + ] + + needs_subscription = "optional" + login_url = 'http://www.courrierinternational.com/login' + + def get_browser(self): + br = BasicNewsRecipe.get_browser(self) + if self.username: + br.open(self.login_url) + br.select_form(nr=1) + br['name'] = self.username + br['pass'] = self.password + br.submit() + return br + + def preprocess_html(self, soup): + for link in soup.findAll("a",href=re.compile('^/(notule|sources|comment)')): + link["href"]='http://www.courrierinternational.com' + link["href"] + return soup + feeds = [ # Some articles requiring subscription fails on download. ('A la Une', 'http://www.courrierinternational.com/rss/rss_a_la_une.xml'), + ('France', 'http://courrierint.com/rss/rp/14/0/rss.xml'), + ('Europe', 'http://courrierint.com/rss/rp/15/0/rss.xml'), + ('Amerique', 'http://courrierint.com/rss/rp/16/0/rss.xml'), + ('Asie', 'http://courrierint.com/rss/rp/17/0/rss.xml'), + ('Afrique', 'http://courrierint.com/rss/rp/18/0/rss.xml'), + ('Moyen-Orient', 'http://courrierint.com/rss/rp/19/0/rss.xml'), + ('Economie', 'http://courrierint.com/rss/rp/20/0/rss.xml'), + ('Multimedia', 'http://courrierint.com/rss/rp/23/0/rss.xml'), + ('Sciences', 'http://courrierint.com/rss/rp/22/0/rss.xml'), + ('Culture', 'http://courrierint.com/rss/rp/24/0/rss.xml'), + ('Insolites', 'http://courrierint.com/rss/rp/26/0/rss.xml'), + ('Cartoons', 'http://cs.courrierint.com/rss/all/rss.xml'), + ('Environnement', 'http://vt.courrierint.com/rss/all/rss.xml'), + ('Cinema', 'http://ca.courrierint.com/rss/all/rss.xml'), + ('Sport', 'http://st.courrierint.com/rss/all/rss.xml'), ] - preprocess_regexps = [ (re.compile(i[0], re.IGNORECASE|re.DOTALL), i[1]) for i in - [ - #Handle Depeches - (r'.*]*>([0-9][0-9]/.*

).*', lambda match : '
'+match.group(1)+'
'), - #Handle Articles - (r'.*]*>(Courrier international.*?) .*', lambda match : '
'+match.group(1)+''), - ] - ] - - def print_version(self, url): - return re.sub('/[a-zA-Z]+\.asp','/imprimer.asp' ,url) - + return url + '?page=all' From acfd853f5af37a2b54b1455a90189593f2eb0042 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 25 Mar 2014 21:23:25 +0530 Subject: [PATCH 114/122] Prevent busy errors when reloading the db --- src/calibre/db/cache.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index 5c1ffdc8f2..54bfe9c012 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -196,11 +196,12 @@ class Cache(object): def reload_from_db(self, clear_caches=True): if clear_caches: self._clear_caches() - self.backend.prefs.load_from_db() - self._search_api.saved_searches.load_from_db() - for field in self.fields.itervalues(): - if hasattr(field, 'table'): - field.table.read(self.backend) # Reread data from metadata.db + with self.backend.conn: # Prevent other processes, such as calibredb from interrupting the reload by locking the db + self.backend.prefs.load_from_db() + self._search_api.saved_searches.load_from_db() + for field in self.fields.itervalues(): + if hasattr(field, 'table'): + field.table.read(self.backend) # Reread data from metadata.db @property def field_metadata(self): From a64ae5911378992b1b1aef65b67e93682716b1c8 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 25 Mar 2014 21:31:58 +0530 Subject: [PATCH 115/122] Increase busy timeout to 10 seconds --- src/calibre/db/backend.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/db/backend.py b/src/calibre/db/backend.py index c860c3dec6..7d4d4fb04d 100644 --- a/src/calibre/db/backend.py +++ b/src/calibre/db/backend.py @@ -233,7 +233,7 @@ def AumSortedConcatenate(): class Connection(apsw.Connection): # {{{ - BUSY_TIMEOUT = 2000 # milliseconds + BUSY_TIMEOUT = 10000 # milliseconds def __init__(self, path): apsw.Connection.__init__(self, path) From 4836eb97ba9974014a58d0eeccf71102304ef1e4 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 25 Mar 2014 21:34:41 +0530 Subject: [PATCH 116/122] ... --- recipes/courrierinternational.recipe | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/recipes/courrierinternational.recipe b/recipes/courrierinternational.recipe index aaea5c8995..bd2b71a09b 100644 --- a/recipes/courrierinternational.recipe +++ b/recipes/courrierinternational.recipe @@ -1,7 +1,8 @@ #!/usr/bin/env python __license__ = 'GPL v3' -__copyright__ = '2009, Mathieu Godlewski ' +__copyright__ = '''2009, Mathieu Godlewski +2014, Rémi Vanicat ''' ''' Courrier International ''' @@ -47,7 +48,7 @@ class CourrierInternational(BasicNewsRecipe): return br def preprocess_html(self, soup): - for link in soup.findAll("a",href=re.compile('^/(notule|sources|comment)')): + for link in soup.findAll("a",href=re.compile('^/')): link["href"]='http://www.courrierinternational.com' + link["href"] return soup From 5c03567d8eab0a763a77e21fcfd8d38a1de0c92a Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 25 Mar 2014 21:51:43 +0530 Subject: [PATCH 117/122] Update The Atlantic --- recipes/atlantic.recipe | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/recipes/atlantic.recipe b/recipes/atlantic.recipe index 55e02b2ad1..1fca16827c 100644 --- a/recipes/atlantic.recipe +++ b/recipes/atlantic.recipe @@ -18,13 +18,14 @@ class TheAtlantic(BasicNewsRecipe): INDEX = 'http://www.theatlantic.com/magazine/toc/0/' language = 'en' - remove_tags_before = dict(name='div', id='articleHead') - remove_tags_after = dict(id='copyright') - remove_tags = [dict(id=['header', 'printAds', 'pageControls'])] + keep_only_tags = [{'attrs':{'class':['article', 'articleHead', 'articleText']}}] + remove_tags = [dict(attrs={'class':'footer'})] no_stylesheets = True - preprocess_regexps = [(re.compile(r'', re.DOTALL), lambda m: '')] - + preprocess_regexps = [ + (re.compile(r'', re.DOTALL), lambda m: ''), + (re.compile(r'.* Date: Wed, 26 Mar 2014 09:55:05 +0530 Subject: [PATCH 118/122] Fix #1297532 [Edit Book: Wrong indent in beautify](https://bugs.launchpad.net/calibre/+bug/1297532) --- src/calibre/ebooks/oeb/polish/pretty.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/ebooks/oeb/polish/pretty.py b/src/calibre/ebooks/oeb/polish/pretty.py index 8f89549d9e..2b08f253d0 100644 --- a/src/calibre/ebooks/oeb/polish/pretty.py +++ b/src/calibre/ebooks/oeb/polish/pretty.py @@ -93,7 +93,7 @@ BLOCK_TAGS = frozenset(map(XHTML, ( 'div', 'dl', 'dt', 'fieldset', 'figcaption', 'figure', 'footer', 'form', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'header', 'hgroup', 'hr', 'li', 'noscript', 'ol', 'output', 'p', 'pre', 'script', 'section', 'style', 'table', 'tbody', 'td', - 'tfoot', 'thead', 'tr', 'ul', 'video'))) | {SVG_TAG} + 'tfoot', 'thead', 'tr', 'ul', 'video', 'img'))) | {SVG_TAG} def isblock(x): From f0ee52cf5d9160f103806e8ab95aeebda5e09240 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 26 Mar 2014 10:58:29 +0530 Subject: [PATCH 119/122] DOCX Input: Fix text from some paragraphs not being converted if the paragraph contains an inline forced page break and no formatted text or line breaks. Fixes #1296817 [Private bug](https://bugs.launchpad.net/calibre/+bug/1296817) [Private bug](https://bugs.launchpad.net/calibre/+bug/1296817) --- src/calibre/ebooks/docx/cleanup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/ebooks/docx/cleanup.py b/src/calibre/ebooks/docx/cleanup.py index 941893ab4f..77533991cd 100644 --- a/src/calibre/ebooks/docx/cleanup.py +++ b/src/calibre/ebooks/docx/cleanup.py @@ -171,7 +171,7 @@ def cleanup_markup(log, root, styles, dest_dir, detect_cover): if prefix: prefix += '; ' p.set('style', prefix + 'page-break-after:always') - p.text = NBSP + p.text = NBSP if not p.text else p.text if detect_cover: # Check if the first image in the document is possibly a cover From db0e425ba6756c9299c9bdb47c65cd3adf01e8e6 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 26 Mar 2014 11:08:45 +0530 Subject: [PATCH 120/122] Fix #1296816 [[F. Request]: Showing "mode" in Saved Searches](https://bugs.launchpad.net/calibre/+bug/1296816) --- src/calibre/gui2/tweak_book/search.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/calibre/gui2/tweak_book/search.py b/src/calibre/gui2/tweak_book/search.py index c77790fd26..df038ef161 100644 --- a/src/calibre/gui2/tweak_book/search.py +++ b/src/calibre/gui2/tweak_book/search.py @@ -456,7 +456,7 @@ class EditSearch(Dialog): # {{{ l.addRow(_('&Name:'), n) self.find = f = QLineEdit(self.search.get('find', ''), self) - f.setPlaceholderText(_('The regular expression to search for')) + f.setPlaceholderText(_('The expression to search for')) l.addRow(_('&Find:'), f) self.replace = r = QLineEdit(self.search.get('replace', ''), self) @@ -734,8 +734,12 @@ class SavedSearches(Dialog): search_index, search = i.data(Qt.UserRole).toPyObject() cs = '✓' if search.get('case_sensitive', SearchWidget.DEFAULT_STATE['case_sensitive']) else '✗' da = '✓' if search.get('dot_all', SearchWidget.DEFAULT_STATE['dot_all']) else '✗' - self.description.setText(_('{2} (Case sensitive: {3} Dot All: {4})\nFind: {0}\nReplace: {1}').format( - search.get('find', ''), search.get('replace', ''), search.get('name', ''), cs, da)) + if search.get('mode', SearchWidget.DEFAULT_STATE['mode']) == 'regex': + ts = _('(Case sensitive: {0} Dot All: {1})').format(cs, da) + else: + ts = _('(Case sensitive: {0} [Normal search])').format(cs) + self.description.setText(_('{2} {3}\nFind: {0}\nReplace: {1}').format( + search.get('find', ''), search.get('replace', ''), search.get('name', ''), ts)) def import_searches(self): path = choose_files(self, 'import_saved_searches', _('Choose file'), filters=[ From a258f4a2015a10cee1e044f47bf895cefbc334f2 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 26 Mar 2014 11:15:05 +0530 Subject: [PATCH 121/122] Increase the max allowed cover size a little, to match current publishing guidelines for covers --- resources/default_tweaks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/default_tweaks.py b/resources/default_tweaks.py index 58901ac6f2..ae595a7cf8 100644 --- a/resources/default_tweaks.py +++ b/resources/default_tweaks.py @@ -444,7 +444,7 @@ public_smtp_relay_delay = 301 # All covers in the calibre library will be resized, preserving aspect ratio, # to fit within this size. This is to prevent slowdowns caused by extremely # large covers -maximum_cover_size = (1450, 2000) +maximum_cover_size = (1650, 2200) #: Where to send downloaded news # When automatically sending downloaded news to a connected device, calibre From 99db7985bd6d221e9d875ab428a412532117e628 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 26 Mar 2014 15:12:57 +0530 Subject: [PATCH 122/122] AZW3 Input: Handle files with garbage bytes in their table of contents. Fixes #1297713 [private](https://bugs.launchpad.net/calibre/+bug/1297713) --- src/calibre/ebooks/metadata/toc.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/calibre/ebooks/metadata/toc.py b/src/calibre/ebooks/metadata/toc.py index f2f49e2c63..9c19f6b59e 100644 --- a/src/calibre/ebooks/metadata/toc.py +++ b/src/calibre/ebooks/metadata/toc.py @@ -13,6 +13,7 @@ from lxml.builder import ElementMaker from calibre.constants import __appname__, __version__ from calibre.ebooks.BeautifulSoup import BeautifulSoup from calibre.ebooks.chardet import xml_to_unicode +from calibre.utils.cleantext import clean_xml_chars NCX_NS = "http://www.daisy.org/z3986/2005/ncx/" CALIBRE_NS = "http://calibre.kovidgoyal.net/2009/metadata" @@ -136,7 +137,7 @@ class TOC(list): try: if not os.path.exists(toc): bn = os.path.basename(toc) - bn = bn.replace('_top.htm', '_toc.htm') # Bug in BAEN OPF files + bn = bn.replace('_top.htm', '_toc.htm') # Bug in BAEN OPF files toc = os.path.join(os.path.dirname(toc), bn) self.read_html_toc(toc) @@ -258,6 +259,7 @@ class TOC(list): text = '' c[1] += 1 item_id = 'num_%d'%c[1] + text = clean_xml_chars(text) elem = E.navPoint( E.navLabel(E.text(re.sub(r'\s+', ' ', text))), E.content(src=unicode(np.href)+(('#' + unicode(np.fragment))