diff --git a/resources/compiled_coffeescript.zip b/resources/compiled_coffeescript.zip index d7fad735ba..c026c894de 100644 Binary files a/resources/compiled_coffeescript.zip and b/resources/compiled_coffeescript.zip differ diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py index af5bfc74fb..d5e911ef1b 100644 --- a/src/calibre/customize/builtins.py +++ b/src/calibre/customize/builtins.py @@ -1203,7 +1203,7 @@ class StoreAmazonFRKindleStore(StoreBase): description = u'Tous les ebooks Kindle' actual_plugin = 'calibre.gui2.store.stores.amazon_fr_plugin:AmazonFRKindleStore' - headquarters = 'DE' + headquarters = 'FR' formats = ['KINDLE'] affiliate = True diff --git a/src/calibre/devices/udisks.py b/src/calibre/devices/udisks.py index 18771dbeb2..7a536f59eb 100644 --- a/src/calibre/devices/udisks.py +++ b/src/calibre/devices/udisks.py @@ -159,16 +159,29 @@ def get_udisks(ver=None): return u return UDisks2() if ver == 2 else UDisks() +def get_udisks1(): + u = None + try: + u = UDisks() + except NoUDisks1: + try: + u = UDisks2() + except NoUDisks2: + pass + if u is None: + raise EnvironmentError('UDisks not available on your system') + return u + def mount(node_path): - u = UDisks() + u = get_udisks1() u.mount(node_path) def eject(node_path): - u = UDisks() + u = get_udisks1() u.eject(node_path) def umount(node_path): - u = UDisks() + u = get_udisks1() u.unmount(node_path) def test_udisks(ver=None): diff --git a/src/calibre/ebooks/oeb/display/paged.coffee b/src/calibre/ebooks/oeb/display/paged.coffee index 695ca3af7f..3dd930d92f 100644 --- a/src/calibre/ebooks/oeb/display/paged.coffee +++ b/src/calibre/ebooks/oeb/display/paged.coffee @@ -6,48 +6,9 @@ Released under the GPLv3 License ### -log = (args...) -> # {{{ - if args - msg = args.join(' ') - if window?.console?.log - window.console.log(msg) - else if process?.stdout?.write - process.stdout.write(msg + '\n') -# }}} - -window_scroll_pos = (win=window) -> # {{{ - if typeof(win.pageXOffset) == 'number' - x = win.pageXOffset - y = win.pageYOffset - else # IE < 9 - if document.body and ( document.body.scrollLeft or document.body.scrollTop ) - x = document.body.scrollLeft - y = document.body.scrollTop - else if document.documentElement and ( document.documentElement.scrollLeft or document.documentElement.scrollTop) - y = document.documentElement.scrollTop - x = document.documentElement.scrollLeft - return [x, y] -# }}} - -viewport_to_document = (x, y, doc=window?.document) -> # {{{ - until doc == window.document - # We are in a frame - frame = doc.defaultView.frameElement - rect = frame.getBoundingClientRect() - x += rect.left - y += rect.top - doc = frame.ownerDocument - win = doc.defaultView - [wx, wy] = window_scroll_pos(win) - x += wx - y += wy - return [x, y] -# }}} - -absleft = (elem) -> # {{{ - r = elem.getBoundingClientRect() - return viewport_to_document(r.left, 0, elem.ownerDocument)[0] -# }}} +log = window.calibre_utils.log +viewport_to_document = window.calibre_utils.viewport_to_document +absleft = window.calibre_utils.absleft class PagedDisplay # This class is a namespace to expose functions via the @@ -75,6 +36,7 @@ class PagedDisplay this.cols_per_screen = cols_per_screen layout: () -> + # start_time = new Date().getTime() body_style = window.getComputedStyle(document.body) # When laying body out in columns, webkit bleeds the top margin of the # first block element out above the columns, leading to an extra top @@ -160,8 +122,12 @@ class PagedDisplay this.in_paged_mode = true this.current_margin_side = sm + # log('Time to layout:', new Date().getTime() - start_time) return sm + fit_images: () -> + null + scroll_to_pos: (frac) -> # Scroll to the position represented by frac (number between 0 and 1) xpos = Math.floor(document.body.scrollWidth * frac) diff --git a/src/calibre/ebooks/oeb/display/utils.coffee b/src/calibre/ebooks/oeb/display/utils.coffee new file mode 100644 index 0000000000..cc98a06d04 --- /dev/null +++ b/src/calibre/ebooks/oeb/display/utils.coffee @@ -0,0 +1,96 @@ +#!/usr/bin/env coffee +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai + +### + Copyright 2012, Kovid Goyal + Released under the GPLv3 License +### + +class CalibreUtils + # This class is a namespace to expose functions via the + # window.calibre_utils object. + + constructor: () -> + if not this instanceof arguments.callee + throw new Error('CalibreUtils constructor called as function') + this.dom_attr = 'calibre_f3fa75ca98eb4413a4ee413f20f60226' + this.dom_data = [] + + # Data API {{{ + + retrieve: (node, key, def=null) -> + # Retrieve data previously stored on node (a DOM node) with key (a + # string). If no such data is found then return the value of def. + idx = parseInt(node.getAttribute(this.dom_attr)) + if isNaN(idx) + return def + data = this.dom_data[idx] + if not data.hasOwnProperty(key) + return def + return data[key] + + store: (node, key, val) -> + # Store arbitrary javscript object val on DOM node node with key (a + # string). This can be later retrieved by the retrieve method. + idx = parseInt(node.getAttribute(this.dom_attr)) + if isNaN(idx) + idx = this.dom_data.length + node.setAttribute(this.dom_attr, idx+'') + this.dom_data.push({}) + this.dom_data[idx][key] = val + # }}} + + log: (args...) -> # {{{ + # Output args to the window.console object. args are automatically + # coerced to strings + if args + msg = args.join(' ') + if window?.console?.log + window.console.log(msg) + else if process?.stdout?.write + process.stdout.write(msg + '\n') + # }}} + + window_scroll_pos: (win=window) -> # {{{ + # The current scroll position of the browser window + if typeof(win.pageXOffset) == 'number' + x = win.pageXOffset + y = win.pageYOffset + else # IE < 9 + if document.body and ( document.body.scrollLeft or document.body.scrollTop ) + x = document.body.scrollLeft + y = document.body.scrollTop + else if document.documentElement and ( document.documentElement.scrollLeft or document.documentElement.scrollTop) + y = document.documentElement.scrollTop + x = document.documentElement.scrollLeft + return [x, y] + # }}} + + viewport_to_document: (x, y, doc=window?.document) -> # {{{ + # Convert x, y from the viewport (window) co-ordinate system to the + # document (body) co-ordinate system + until doc == window.document + # We are in a frame + frame = doc.defaultView.frameElement + rect = frame.getBoundingClientRect() + x += rect.left + y += rect.top + doc = frame.ownerDocument + win = doc.defaultView + [wx, wy] = this.window_scroll_pos(win) + x += wx + y += wy + return [x, y] + # }}} + + absleft: (elem) -> # {{{ + # The left edge of elem in document co-ords. Works in all + # circumstances, including column layout. Note that this will cause + # a relayout if the render tree is dirty. + r = elem.getBoundingClientRect() + return this.viewport_to_document(r.left, 0, elem.ownerDocument)[0] + # }}} + +if window? + window.calibre_utils = new CalibreUtils() + diff --git a/src/calibre/gui2/actions/copy_to_library.py b/src/calibre/gui2/actions/copy_to_library.py index f901d5ce30..a6699b0fcb 100644 --- a/src/calibre/gui2/actions/copy_to_library.py +++ b/src/calibre/gui2/actions/copy_to_library.py @@ -216,7 +216,8 @@ class CopyToLibraryAction(InterfaceAction): if ci.isValid(): row = ci.row() - v.model().delete_books_by_id(self.worker.processed) + v.model().delete_books_by_id(self.worker.processed, + permanent=True) self.gui.iactions['Remove Books'].library_ids_deleted( self.worker.processed, row) diff --git a/src/calibre/gui2/book_details.py b/src/calibre/gui2/book_details.py index 00bd4135b1..3e57442591 100644 --- a/src/calibre/gui2/book_details.py +++ b/src/calibre/gui2/book_details.py @@ -104,8 +104,11 @@ def render_data(mi, use_roman_numbers=True, all_fields=False): field = 'title_sort' if all_fields: display = True - if (not display or not metadata or mi.is_null(field) or - field == 'comments'): + if metadata['datatype'] == 'bool': + isnull = mi.get(field) is None + else: + isnull = mi.is_null(field) + if (not display or not metadata or isnull or field == 'comments'): continue name = metadata['name'] if not name: diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py index e2706c0a54..f0296a32c7 100644 --- a/src/calibre/gui2/library/models.py +++ b/src/calibre/gui2/library/models.py @@ -220,16 +220,15 @@ class BooksModel(QAbstractTableModel): # {{{ self.count_changed() self.reset() - def delete_books(self, indices): + def delete_books(self, indices, permanent=False): ids = map(self.id, indices) - for id in ids: - self.db.delete_book(id, notify=False) - self.books_deleted() + self.delete_books_by_id(ids, permanent=permanent) return ids - def delete_books_by_id(self, ids): + def delete_books_by_id(self, ids, permanent=False): for id in ids: - self.db.delete_book(id) + self.db.delete_book(id, permanent=permanent, do_clean=False) + self.db.clean() self.books_deleted() def books_added(self, num): diff --git a/src/calibre/gui2/preferences/tweaks.py b/src/calibre/gui2/preferences/tweaks.py index ad0561dd8a..ebea8574a7 100644 --- a/src/calibre/gui2/preferences/tweaks.py +++ b/src/calibre/gui2/preferences/tweaks.py @@ -243,7 +243,8 @@ class Tweaks(QAbstractListModel, SearchQueryParser): # {{{ query = lower(query) for r in candidates: dat = self.data(self.index(r), Qt.UserRole) - if query in lower(dat.name):# or query in lower(dat.doc): + var_names = u' '.join(dat.default_values) + if query in lower(dat.name) or query in lower(var_names): ans.add(r) return ans diff --git a/src/calibre/gui2/store/stores/foyles_uk_plugin.py b/src/calibre/gui2/store/stores/foyles_uk_plugin.py index 906d47b7ea..b684dd533d 100644 --- a/src/calibre/gui2/store/stores/foyles_uk_plugin.py +++ b/src/calibre/gui2/store/stores/foyles_uk_plugin.py @@ -40,32 +40,27 @@ class FoylesUKStore(BasicStoreConfig, StorePlugin): d.exec_() def search(self, query, max_results=10, timeout=60): - url = 'http://www.foyles.co.uk/Public/Shop/Search.aspx?fFacetId=1015&searchBy=1&quick=true&term=' + urllib2.quote(query) + url = 'http://ebooks.foyles.co.uk/search_for-' + urllib2.quote(query) br = browser() counter = max_results with closing(br.open(url, timeout=timeout)) as f: doc = html.fromstring(f.read()) - for data in doc.xpath('//table[contains(@id, "MainContent")]/tr/td/div[contains(@class, "Item")]'): + for data in doc.xpath('//div[@class="doc-item"]'): if counter <= 0: break - id = ''.join(data.xpath('.//a[@class="Title"]/@href')).strip() - if not id: + id_ = ''.join(data.xpath('.//p[@class="doc-cover"]/a/@href')).strip() + if not id_: continue - # filter out the audio books - if not data.xpath('boolean(.//div[@class="Relative"]/ul/li[contains(text(), "ePub")])'): - continue - - cover_url = ''.join(data.xpath('.//a[@class="Jacket"]/img/@src')) - title = ''.join(data.xpath('.//a[@class="Title"]/text()')) - author = ', '.join(data.xpath('.//span[@class="Author"]/text()')) - price = ''.join(data.xpath('./ul/li[@class="Strong"]/text()')) - mo = re.search('£[\d\.]+', price) - if mo is None: - continue - price = mo.group(0) + cover_url = ''.join(data.xpath('.//p[@class="doc-cover"]/a/img/@src')) + title = ''.join(data.xpath('.//span[@class="title"]/a/text()')) + author = ', '.join(data.xpath('.//span[@class="author"]/span[@class="author"]/text()')) + price = ''.join(data.xpath('.//span[@class="price"]/text()')) + format_ = ''.join(data.xpath('.//p[@class="doc-meta-format"]/span[last()]/text()')) + format_, ign, drm = format_.partition(' ') + drm = SearchResult.DRM_LOCKED if 'DRM' in drm else SearchResult.DRM_UNLOCKED counter -= 1 @@ -74,8 +69,8 @@ class FoylesUKStore(BasicStoreConfig, StorePlugin): s.title = title.strip() s.author = author.strip() s.price = price - s.detail_item = id - s.drm = SearchResult.DRM_LOCKED - s.formats = 'ePub' + s.detail_item = id_ + s.drm = drm + s.formats = format_ yield s diff --git a/src/calibre/gui2/viewer/documentview.py b/src/calibre/gui2/viewer/documentview.py index 10d10b7155..8beea63df4 100644 --- a/src/calibre/gui2/viewer/documentview.py +++ b/src/calibre/gui2/viewer/documentview.py @@ -136,7 +136,7 @@ class Document(QWebPage): # {{{ self.max_fs_width = min(opts.max_fs_width, screen_width-50) def fit_images(self): - if self.do_fit_images: + if self.do_fit_images and not self.in_paged_mode: self.javascript('setup_image_scaling_handlers()') def add_window_objects(self): @@ -219,6 +219,7 @@ class Document(QWebPage): # {{{ if scroll_width > self.window_width: sz.setWidth(scroll_width+side_margin) self.setPreferredContentsSize(sz) + self.javascript('window.paged_display.fit_images()') @property def column_boundaries(self): diff --git a/src/calibre/gui2/viewer/javascript.py b/src/calibre/gui2/viewer/javascript.py index 1594a1d5db..2aedaf9e13 100644 --- a/src/calibre/gui2/viewer/javascript.py +++ b/src/calibre/gui2/viewer/javascript.py @@ -31,10 +31,11 @@ class JavaScriptLoader(object): 'cfi':'ebooks.oeb.display.cfi', 'indexing':'ebooks.oeb.display.indexing', 'paged':'ebooks.oeb.display.paged', + 'utils':'ebooks.oeb.display.utils', } ORDER = ('jquery', 'jquery_scrollTo', 'bookmarks', 'referencing', 'images', - 'hyphenation', 'hyphenator', 'cfi', 'indexing', 'paged') + 'hyphenation', 'hyphenator', 'utils', 'cfi', 'indexing', 'paged') def __init__(self, dynamic_coffeescript=False): diff --git a/src/calibre/library/catalogs/epub_mobi_builder.py b/src/calibre/library/catalogs/epub_mobi_builder.py index 1383379db2..6267651aff 100644 --- a/src/calibre/library/catalogs/epub_mobi_builder.py +++ b/src/calibre/library/catalogs/epub_mobi_builder.py @@ -14,6 +14,7 @@ from calibre.ebooks.chardet import substitute_entites from calibre.ptempfile import PersistentTemporaryDirectory from calibre.utils.config import config_dir from calibre.utils.date import format_date, is_date_undefined, now as nowf +from calibre.utils.filenames import ascii_text from calibre.utils.icu import capitalize from calibre.utils.magick.draw import thumbnail from calibre.utils.zipfile import ZipFile @@ -689,7 +690,7 @@ Author '{0}': this_title['series'] = None this_title['series_index'] = 0.0 - this_title['title_sort'] = self.generateSortTitle(this_title['title']) + this_title['title_sort'] = self.generateSortTitle(ascii_text(this_title['title'])) if 'authors' in record: # from calibre.ebooks.metadata import authors_to_string # return authors_to_string(self.authors) @@ -704,6 +705,7 @@ Author '{0}': this_title['author_sort'] = record['author_sort'] else: this_title['author_sort'] = self.author_to_author_sort(this_title['author']) + this_title['author_sort'] = ascii_text(this_title['author_sort']) if record['publisher']: this_title['publisher'] = re.sub('&', '&', record['publisher']) @@ -3768,7 +3770,7 @@ Author '{0}': else: word = '%10.0f' % (float(word)) translated.append(word) - return ' '.join(translated) + return ascii_text(' '.join(translated)) def generateThumbnail(self, title, image_dir, thumb_file): ''' diff --git a/src/calibre/library/cli.py b/src/calibre/library/cli.py index 6c24a1e455..d5defe96c7 100644 --- a/src/calibre/library/cli.py +++ b/src/calibre/library/cli.py @@ -999,6 +999,55 @@ def command_saved_searches(args, dbpath): return 0 +def backup_metadata_option_parser(): + parser = get_parser(_('''\ +%prog backup_metadata [options] + +Backup the metadata stored in the database into individual OPF files in each +books directory. This normally happens automatically, but you can run this +command to force re-generation of the OPF files, with the --all option. + +Note that there is normally no need to do this, as the OPF files are backed up +automatically, every time metadata is changed. +''')) + parser.add_option('--all', default=False, action='store_true', + help=_('Normally, this command only operates on books that have' + ' out of date OPF files. This option makes it operate on all' + ' books.')) + return parser + +class BackupProgress(object): + + def __init__(self): + self.total = 0 + self.count = 0 + + def __call__(self, book_id, mi, ok): + if mi is True: + self.total = book_id + else: + self.count += 1 + prints(u'%.1f%% %s - %s'%((self.count*100)/float(self.total), + book_id, mi.title)) + +def command_backup_metadata(args, dbpath): + parser = backup_metadata_option_parser() + opts, args = parser.parse_args(args) + if len(args) != 0: + parser.print_help() + return 1 + + if opts.library_path is not None: + dbpath = opts.library_path + if isbytestring(dbpath): + dbpath = dbpath.decode(preferred_encoding) + db = LibraryDatabase2(dbpath) + book_ids = None + if opts.all: + book_ids = db.all_ids() + db.dump_metadata(book_ids=book_ids, callback=BackupProgress()) + + def check_library_option_parser(): from calibre.library.check_library import CHECKS parser = get_parser(_('''\ @@ -1275,7 +1324,7 @@ COMMANDS = ('list', 'add', 'remove', 'add_format', 'remove_format', 'show_metadata', 'set_metadata', 'export', 'catalog', 'saved_searches', 'add_custom_column', 'custom_columns', 'remove_custom_column', 'set_custom', 'restore_database', - 'check_library', 'list_categories') + 'check_library', 'list_categories', 'backup_metadata') def option_parser(): diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 32cb6737a8..e60350b307 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -808,18 +808,30 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): pass def dump_metadata(self, book_ids=None, remove_from_dirtied=True, - commit=True): + commit=True, callback=None): ''' - Write metadata for each record to an individual OPF file + Write metadata for each record to an individual OPF file. If callback + is not None, it is called once at the start with the number of book_ids + being processed. And once for every book_id, with arguments (book_id, + mi, ok). ''' if book_ids is None: book_ids = [x[0] for x in self.conn.get( 'SELECT book FROM metadata_dirtied', all=True)] + + if callback is not None: + book_ids = tuple(book_ids) + callback(len(book_ids), True, False) + for book_id in book_ids: if not self.data.has_id(book_id): + if callback is not None: + callback(book_id, None, False) continue path, mi, sequence = self.get_metadata_for_dump(book_id) if path is None: + if callback is not None: + callback(book_id, mi, False) continue try: raw = metadata_to_opf(mi) @@ -829,6 +841,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): self.clear_dirtied(book_id, sequence) except: pass + if callback is not None: + callback(book_id, mi, True) if commit: self.conn.commit() @@ -1411,7 +1425,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): opath = self.format_abspath(book_id, nfmt, index_is_id=True) return fmt if opath is None else nfmt - def delete_book(self, id, notify=True, commit=True, permanent=False): + def delete_book(self, id, notify=True, commit=True, permanent=False, + do_clean=True): ''' Removes book from the result cache and the underlying database. If you set commit to False, you must call clean() manually afterwards @@ -1428,7 +1443,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): self.conn.execute('DELETE FROM books WHERE id=?', (id,)) if commit: self.conn.commit() - self.clean() + if do_clean: + self.clean() self.data.books_deleted([id]) if notify: self.notify('delete', [id]) diff --git a/src/calibre/library/server/mobile.py b/src/calibre/library/server/mobile.py index eef131e89f..aa98412d27 100644 --- a/src/calibre/library/server/mobile.py +++ b/src/calibre/library/server/mobile.py @@ -97,6 +97,7 @@ def build_index(books, num, search, sort, order, start, total, url_base, CKEYS, search_box = build_search_box(num, search, sort, order, prefix) navigation = build_navigation(start, num, total, prefix+url_base) + navigation2 = build_navigation(start, num, total, prefix+url_base) bookt = TABLE(id='listing') body = BODY( @@ -104,7 +105,9 @@ def build_index(books, num, search, sort, order, start, total, url_base, CKEYS, search_box, navigation, HR(CLASS('spacer')), - bookt + bookt, + HR(CLASS('spacer')), + navigation2 ) # Book list {{{ @@ -155,7 +158,6 @@ def build_index(books, num, search, sort, order, start, total, url_base, CKEYS, bookt.append(TR(thumbnail, data)) # }}} - body.append(HR()) body.append(DIV( A(_('Switch to the full interface (non-mobile interface)'), href=prefix+"/browse", diff --git a/src/calibre/linux.py b/src/calibre/linux.py index f1a53603e7..90cacd0180 100644 --- a/src/calibre/linux.py +++ b/src/calibre/linux.py @@ -354,8 +354,8 @@ class PostInstall: check_call('xdg-icon-resource install --noupdate --context mimetypes --size 128 calibre-lrf.png text-lrs', shell=True) self.icon_resources.append(('mimetypes', 'application-lrs', '128')) - render_img('lt.png', 'calibre-gui.png') - check_call('xdg-icon-resource install --noupdate --size 128 calibre-gui.png calibre-gui', shell=True) + render_img('lt.png', 'calibre-gui.png', width=256, height=256) + check_call('xdg-icon-resource install --noupdate --size 256 calibre-gui.png calibre-gui', shell=True) self.icon_resources.append(('apps', 'calibre-gui', '128')) render_img('viewer.png', 'calibre-viewer.png') check_call('xdg-icon-resource install --size 128 calibre-viewer.png calibre-viewer', shell=True) diff --git a/src/calibre/utils/icu.c b/src/calibre/utils/icu.c index 7bf2341fde..f4d820bff4 100644 --- a/src/calibre/utils/icu.c +++ b/src/calibre/utils/icu.c @@ -5,6 +5,7 @@ #include #include #include +#include // Collator object definition {{{ @@ -12,6 +13,7 @@ typedef struct { PyObject_HEAD // Type-specific fields go here. UCollator *collator; + USet *contractions; } icu_Collator; @@ -19,6 +21,7 @@ static void icu_Collator_dealloc(icu_Collator* self) { if (self->collator != NULL) ucol_close(self->collator); + if (self->contractions != NULL) uset_close(self->contractions); self->collator = NULL; self->ob_type->tp_free((PyObject*)self); } @@ -29,18 +32,19 @@ icu_Collator_new(PyTypeObject *type, PyObject *args, PyObject *kwds) icu_Collator *self; const char *loc; UErrorCode status = U_ZERO_ERROR; + UCollator *collator; if (!PyArg_ParseTuple(args, "s", &loc)) return NULL; + collator = ucol_open(loc, &status); + if (collator == NULL || U_FAILURE(status)) { + PyErr_SetString(PyExc_Exception, "Failed to create collator."); + return NULL; + } self = (icu_Collator *)type->tp_alloc(type, 0); if (self != NULL) { - self->collator = ucol_open(loc, &status); - if (self->collator == NULL || U_FAILURE(status)) { - PyErr_SetString(PyExc_Exception, "Failed to create collator."); - self->collator = NULL; - Py_DECREF(self); - return NULL; - } + self->collator = collator; + self->contractions = NULL; } return (PyObject *)self; @@ -63,13 +67,30 @@ icu_Collator_display_name(icu_Collator *self, void *closure) { u_strToUTF8(buf, 100, NULL, dname, -1, &status); if (U_FAILURE(status)) { - PyErr_SetString(PyExc_Exception, "Failed ot convert dname to UTF-8"); return NULL; + PyErr_SetString(PyExc_Exception, "Failed to convert dname to UTF-8"); return NULL; } return Py_BuildValue("s", buf); } // }}} +// Collator.strength {{{ +static PyObject * +icu_Collator_get_strength(icu_Collator *self, void *closure) { + return Py_BuildValue("i", ucol_getStrength(self->collator)); +} + +static int +icu_Collator_set_strength(icu_Collator *self, PyObject *val, void *closure) { + if (!PyInt_Check(val)) { + PyErr_SetString(PyExc_TypeError, "Strength must be an integer."); + return -1; + } + ucol_setStrength(self->collator, (int)PyInt_AS_LONG(val)); + return 0; +} +// }}} + // Collator.actual_locale {{{ static PyObject * icu_Collator_actual_locale(icu_Collator *self, void *closure) { @@ -164,7 +185,126 @@ icu_Collator_strcmp(icu_Collator *self, PyObject *args, PyObject *kwargs) { return Py_BuildValue("i", res); } // }}} +// Collator.find {{{ +static PyObject * +icu_Collator_find(icu_Collator *self, PyObject *args, PyObject *kwargs) { + PyObject *a_, *b_; + size_t asz, bsz; + UChar *a, *b; + wchar_t *aw, *bw; + UErrorCode status = U_ZERO_ERROR; + UStringSearch *search = NULL; + int32_t pos = -1, length = -1; + + if (!PyArg_ParseTuple(args, "UU", &a_, &b_)) return NULL; + asz = PyUnicode_GetSize(a_); bsz = PyUnicode_GetSize(b_); + + a = (UChar*)calloc(asz*4 + 2, sizeof(UChar)); + b = (UChar*)calloc(bsz*4 + 2, sizeof(UChar)); + aw = (wchar_t*)calloc(asz*4 + 2, sizeof(wchar_t)); + bw = (wchar_t*)calloc(bsz*4 + 2, sizeof(wchar_t)); + if (a == NULL || b == NULL || aw == NULL || bw == NULL) return PyErr_NoMemory(); + + PyUnicode_AsWideChar((PyUnicodeObject*)a_, aw, asz*4+1); + PyUnicode_AsWideChar((PyUnicodeObject*)b_, bw, bsz*4+1); + u_strFromWCS(a, asz*4 + 1, NULL, aw, -1, &status); + u_strFromWCS(b, bsz*4 + 1, NULL, bw, -1, &status); + + if (U_SUCCESS(status)) { + search = usearch_openFromCollator(a, -1, b, -1, self->collator, NULL, &status); + if (U_SUCCESS(status)) { + pos = usearch_first(search, &status); + if (pos != USEARCH_DONE) + length = (pos == USEARCH_DONE) ? -1 : usearch_getMatchedLength(search); + else + pos = -1; + } + if (search != NULL) usearch_close(search); + } + + free(a); free(b); free(aw); free(bw); + + return Py_BuildValue("ii", pos, length); +} // }}} + +// Collator.contractions {{{ +static PyObject * +icu_Collator_contractions(icu_Collator *self, PyObject *args, PyObject *kwargs) { + UErrorCode status = U_ZERO_ERROR; + UChar *str; + UChar32 start=0, end=0; + int32_t count = 0, len = 0, dlen = 0, i; + PyObject *ans = Py_None, *pbuf; + wchar_t *buf; + + if (self->contractions == NULL) { + self->contractions = uset_open(1, 0); + if (self->contractions == NULL) return PyErr_NoMemory(); + ucol_getContractionsAndExpansions(self->collator, self->contractions, NULL, 0, &status); + } + status = U_ZERO_ERROR; + + str = (UChar*)calloc(100, sizeof(UChar)); + buf = (wchar_t*)calloc(4*100+2, sizeof(wchar_t)); + if (str == NULL || buf == NULL) return PyErr_NoMemory(); + + count = uset_getItemCount(self->contractions); + ans = PyTuple_New(count); + if (ans != NULL) { + for (i = 0; i < count; i++) { + len = uset_getItem(self->contractions, i, &start, &end, str, 1000, &status); + if (len >= 2) { + // We have a string + status = U_ZERO_ERROR; + u_strToWCS(buf, 4*100 + 1, &dlen, str, len, &status); + pbuf = PyUnicode_FromWideChar(buf, dlen); + if (pbuf == NULL) return PyErr_NoMemory(); + PyTuple_SetItem(ans, i, pbuf); + } else { + // Ranges dont make sense for contractions, ignore them + PyTuple_SetItem(ans, i, Py_None); + } + } + } + free(str); free(buf); + + return Py_BuildValue("O", ans); +} // }}} + +// Collator.span_contractions {{{ +static PyObject * +icu_Collator_span_contractions(icu_Collator *self, PyObject *args, PyObject *kwargs) { + int span_type; + UErrorCode status = U_ZERO_ERROR; + PyObject *str; + size_t slen = 0; + wchar_t *buf; + UChar *s; + + if (!PyArg_ParseTuple(args, "Ui", &str, &span_type)) return NULL; + + if (self->contractions == NULL) { + self->contractions = uset_open(1, 0); + if (self->contractions == NULL) return PyErr_NoMemory(); + ucol_getContractionsAndExpansions(self->collator, self->contractions, NULL, 0, &status); + } + status = U_ZERO_ERROR; + + slen = PyUnicode_GetSize(str); + buf = (wchar_t*)calloc(slen*4 + 2, sizeof(wchar_t)); + s = (UChar*)calloc(slen*4 + 2, sizeof(UChar)); + if (buf == NULL || s == NULL) return PyErr_NoMemory(); + slen = PyUnicode_AsWideChar((PyUnicodeObject*)str, buf, slen); + u_strFromWCS(s, slen*4+1, NULL, buf, slen, &status); + + free(buf); free(s); + return Py_BuildValue("i", uset_span(self->contractions, s, slen, span_type)); +} // }}} + + +static PyObject* +icu_Collator_clone(icu_Collator *self, PyObject *args, PyObject *kwargs); static PyMethodDef icu_Collator_methods[] = { {"sort_key", (PyCFunction)icu_Collator_sort_key, METH_VARARGS, @@ -175,6 +315,22 @@ static PyMethodDef icu_Collator_methods[] = { "strcmp(unicode object, unicode object) -> strcmp(a, b) <=> cmp(sorty_key(a), sort_key(b)), but faster." }, + {"find", (PyCFunction)icu_Collator_find, METH_VARARGS, + "find(pattern, source) -> returns the position and length of the first occurrence of pattern in source. Returns (-1, -1) if not found." + }, + + {"contractions", (PyCFunction)icu_Collator_contractions, METH_VARARGS, + "contractions() -> returns the contractions defined for this collator." + }, + + {"span_contractions", (PyCFunction)icu_Collator_span_contractions, METH_VARARGS, + "span_contractions(src, span_condition) -> returns the length of the initial substring according to span_condition in the set of contractions for this collator. Returns 0 if src does not fit the span_condition. The span_condition can be one of USET_SPAN_NOT_CONTAINED, USET_SPAN_CONTAINED, USET_SPAN_SIMPLE." + }, + + {"clone", (PyCFunction)icu_Collator_clone, METH_VARARGS, + "clone() -> returns a clone of this collator." + }, + {NULL} /* Sentinel */ }; @@ -189,6 +345,12 @@ static PyGetSetDef icu_Collator_getsetters[] = { (char *)"Display name of this collator in English. The name reflects the actual data source used.", NULL}, + {(char *)"strength", + (getter)icu_Collator_get_strength, (setter)icu_Collator_set_strength, + (char *)"The strength of this collator.", + NULL}, + + {NULL} /* Sentinel */ }; @@ -236,6 +398,31 @@ static PyTypeObject icu_CollatorType = { // {{{ // }} +// Collator.clone {{{ +static PyObject* +icu_Collator_clone(icu_Collator *self, PyObject *args, PyObject *kwargs) +{ + UCollator *collator; + UErrorCode status = U_ZERO_ERROR; + int32_t bufsize = -1; + icu_Collator *clone; + + collator = ucol_safeClone(self->collator, NULL, &bufsize, &status); + + if (collator == NULL || U_FAILURE(status)) { + PyErr_SetString(PyExc_Exception, "Failed to create collator."); + return NULL; + } + + clone = PyObject_New(icu_Collator, &icu_CollatorType); + if (clone == NULL) return PyErr_NoMemory(); + + clone->collator = collator; + clone->contractions = NULL; + + return (PyObject*) clone; + +} // }}} // }}} @@ -411,6 +598,7 @@ static PyMethodDef icu_methods[] = { {NULL} /* Sentinel */ }; +#define ADDUCONST(x) PyModule_AddIntConstant(m, #x, x) PyMODINIT_FUNC initicu(void) @@ -432,5 +620,22 @@ initicu(void) // uint8_t must be the same size as char PyModule_AddIntConstant(m, "ok", (U_SUCCESS(status) && sizeof(uint8_t) == sizeof(char)) ? 1 : 0); + ADDUCONST(USET_SPAN_NOT_CONTAINED); + ADDUCONST(USET_SPAN_CONTAINED); + ADDUCONST(USET_SPAN_SIMPLE); + ADDUCONST(UCOL_DEFAULT); + ADDUCONST(UCOL_PRIMARY); + ADDUCONST(UCOL_SECONDARY); + ADDUCONST(UCOL_TERTIARY); + ADDUCONST(UCOL_DEFAULT_STRENGTH); + ADDUCONST(UCOL_QUATERNARY); + ADDUCONST(UCOL_IDENTICAL); + ADDUCONST(UCOL_OFF); + ADDUCONST(UCOL_ON); + ADDUCONST(UCOL_SHIFTED); + ADDUCONST(UCOL_NON_IGNORABLE); + ADDUCONST(UCOL_LOWER_FIRST); + ADDUCONST(UCOL_UPPER_FIRST); + } // }}} diff --git a/src/calibre/utils/icu.py b/src/calibre/utils/icu.py index 4daec9d553..d9ae3c602c 100644 --- a/src/calibre/utils/icu.py +++ b/src/calibre/utils/icu.py @@ -12,7 +12,7 @@ from functools import partial from calibre.constants import plugins from calibre.utils.config_base import tweaks -_icu = _collator = None +_icu = _collator = _primary_collator = None _locale = None _none = u'' @@ -48,6 +48,12 @@ def load_collator(): _collator = icu.Collator(get_locale()) return _collator +def primary_collator(): + global _primary_collator + if _primary_collator is None: + _primary_collator = _collator.clone() + _primary_collator.strength = _icu.UCOL_PRIMARY + return _primary_collator def py_sort_key(obj): if not obj: @@ -59,6 +65,18 @@ def icu_sort_key(collator, obj): return _none2 return collator.sort_key(lower(obj)) +def py_find(pattern, source): + pos = source.find(pattern) + if pos > -1: + return pos, len(pattern) + return -1, -1 + +def icu_find(collator, pattern, source): + try: + return collator.find(pattern, source) + except TypeError: + return collator.find(unicode(pattern), unicode(source)) + def py_case_sensitive_sort_key(obj): if not obj: return _none @@ -72,7 +90,7 @@ def icu_case_sensitive_sort_key(collator, obj): def icu_strcmp(collator, a, b): return collator.strcmp(lower(a), lower(b)) -def py_strcmp(a, b): +def py_strcmp(a, b, strength=None): return cmp(a.lower(), b.lower()) def icu_case_sensitive_strcmp(collator, a, b): @@ -82,6 +100,30 @@ def icu_capitalize(s): s = lower(s) return s.replace(s[0], upper(s[0]), 1) if s else s +_cmap = {} +def icu_contractions(collator): + global _cmap + ans = _cmap.get(collator, None) + if ans is None: + ans = collator.contractions() + ans = frozenset(filter(None, ans)) if ans else {} + _cmap[collator] = ans + return ans + +def py_span_contractions(*args, **kwargs): + return 0 + +def icu_span_contractions(src, span_type=None, collator=None): + global _collator + if collator is None: + collator = _collator + if span_type is None: + span_type = _icu.USET_SPAN_SIMPLE + try: + return collator.span_contractions(src, span_type) + except TypeError: + return collator.span_contractions(unicode(src), span_type) + load_icu() load_collator() _icu_not_ok = _icu is None or _collator is None @@ -117,6 +159,28 @@ title_case = (lambda s: s.title()) if _icu_not_ok else \ capitalize = (lambda s: s.capitalize()) if _icu_not_ok else \ (lambda s: icu_capitalize(s)) +find = (py_find if _icu_not_ok else partial(icu_find, _collator)) + +contractions = ((lambda : {}) if _icu_not_ok else (partial(icu_contractions, + _collator))) + +span_contractions = (py_span_contractions if _icu_not_ok else + icu_span_contractions) + +def primary_strcmp(a, b): + 'strcmp that ignores case and accents on letters' + if _icu_not_ok: + from calibre.utils.filenames import ascii_text + return py_strcmp(ascii_text(a), ascii_text(b)) + return primary_collator().strcmp(a, b) + +def primary_find(pat, src): + 'find that ignores case and accents on letters' + if _icu_not_ok: + from calibre.utils.filenames import ascii_text + return py_find(ascii_text(pat), ascii_text(src)) + return icu_find(primary_collator(), pat, src) + ################################################################################ def test(): # {{{ @@ -240,6 +304,18 @@ pêché''' print 'Capitalize:', x, '->', 'py:', x.capitalize().encode('utf-8'), 'icu:', capitalize(x).encode('utf-8') print + print '\nTesting primary collation' + for k, v in {u'pèché': u'peche', u'flüße':u'flusse'}.iteritems(): + if primary_strcmp(k, v) != 0: + print 'primary_strcmp() failed with %s != %s'%(k, v) + if primary_find(v, u' '+k)[0] != 1: + print 'primary_find() failed with %s not in %s'%(v, k) + + global _primary_collator + _primary_collator = _icu.Collator('es') + if primary_strcmp(u'peña', u'pena') == 0: + print 'Primary collation in Spanish locale failed' + # }}} if __name__ == '__main__':