Merge from trunk

src/calibre/customize/builtins.py

@@ -1203,7 +1203,7 @@ class StoreAmazonFRKindleStore(StoreBase):
     description = u'Tous les ebooks Kindle'
     actual_plugin = 'calibre.gui2.store.stores.amazon_fr_plugin:AmazonFRKindleStore'
 
-    headquarters = 'DE'
+    headquarters = 'FR'
     formats = ['KINDLE']
     affiliate = True
 

src/calibre/devices/udisks.py

@@ -159,16 +159,29 @@ def get_udisks(ver=None):
         return u
     return UDisks2() if ver == 2 else UDisks()
 
-def mount(node_path):
+def get_udisks1():
+    u = None
+    try:
         u = UDisks()
+    except NoUDisks1:
+        try:
+            u = UDisks2()
+        except NoUDisks2:
+            pass
+    if u is None:
+        raise EnvironmentError('UDisks not available on your system')
+    return u
+
+def mount(node_path):
+    u = get_udisks1()
     u.mount(node_path)
 
 def eject(node_path):
-    u = UDisks()
+    u = get_udisks1()
     u.eject(node_path)
 
 def umount(node_path):
-    u = UDisks()
+    u = get_udisks1()
     u.unmount(node_path)
 
 def test_udisks(ver=None):

src/calibre/ebooks/oeb/display/paged.coffee

@@ -6,48 +6,9 @@
  Released under the GPLv3 License
 ###
 
-log = (args...) -> # {{{
+log = window.calibre_utils.log
-    if args
+viewport_to_document = window.calibre_utils.viewport_to_document
-        msg = args.join(' ')
+absleft = window.calibre_utils.absleft
-        if window?.console?.log
-            window.console.log(msg)
-        else if process?.stdout?.write
-            process.stdout.write(msg + '\n')
-# }}}
-
-window_scroll_pos = (win=window) -> # {{{
-    if typeof(win.pageXOffset) == 'number'
-        x = win.pageXOffset
-        y = win.pageYOffset
-    else # IE < 9
-        if document.body and ( document.body.scrollLeft or document.body.scrollTop )
-            x = document.body.scrollLeft
-            y = document.body.scrollTop
-        else if document.documentElement and ( document.documentElement.scrollLeft or document.documentElement.scrollTop)
-            y = document.documentElement.scrollTop
-            x = document.documentElement.scrollLeft
-    return [x, y]
-# }}}
-
-viewport_to_document = (x, y, doc=window?.document) -> # {{{
-    until doc == window.document
-        # We are in a frame
-        frame = doc.defaultView.frameElement
-        rect = frame.getBoundingClientRect()
-        x += rect.left
-        y += rect.top
-        doc = frame.ownerDocument
-    win = doc.defaultView
-    [wx, wy] = window_scroll_pos(win)
-    x += wx
-    y += wy
-    return [x, y]
-# }}}
-
-absleft = (elem) -> # {{{
-    r = elem.getBoundingClientRect()
-    return viewport_to_document(r.left, 0, elem.ownerDocument)[0]
-# }}}
 
 class PagedDisplay
     # This class is a namespace to expose functions via the
@@ -75,6 +36,7 @@ class PagedDisplay
         this.cols_per_screen = cols_per_screen
 
     layout: () ->
+        # start_time = new Date().getTime()
         body_style = window.getComputedStyle(document.body)
         # When laying body out in columns, webkit bleeds the top margin of the
         # first block element out above the columns, leading to an extra top
@@ -160,8 +122,12 @@ class PagedDisplay
 
         this.in_paged_mode = true
         this.current_margin_side = sm
+        # log('Time to layout:', new Date().getTime() - start_time)
         return sm
 
+    fit_images: () ->
+        null
+
     scroll_to_pos: (frac) ->
         # Scroll to the position represented by frac (number between 0 and 1)
         xpos = Math.floor(document.body.scrollWidth * frac)

src/calibre/ebooks/oeb/display/utils.coffee

@@ -0,0 +1,96 @@
+#!/usr/bin/env coffee
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+
+###
+ Copyright 2012, Kovid Goyal <kovid@kovidgoyal.net>
+ Released under the GPLv3 License
+###
+
+class CalibreUtils
+    # This class is a namespace to expose functions via the
+    # window.calibre_utils object.
+
+    constructor: () ->
+        if not this instanceof arguments.callee
+            throw new Error('CalibreUtils constructor called as function')
+        this.dom_attr = 'calibre_f3fa75ca98eb4413a4ee413f20f60226'
+        this.dom_data = []
+
+    # Data API {{{
+
+    retrieve: (node, key, def=null) ->
+        # Retrieve data previously stored on node (a DOM node) with key (a
+        # string). If no such data is found then return the value of def.
+        idx = parseInt(node.getAttribute(this.dom_attr))
+        if isNaN(idx)
+            return def
+        data = this.dom_data[idx]
+        if not data.hasOwnProperty(key)
+            return def
+        return data[key]
+
+    store: (node, key, val) ->
+        # Store arbitrary javscript object val on DOM node node with key (a
+        # string). This can be later retrieved by the retrieve method.
+        idx = parseInt(node.getAttribute(this.dom_attr))
+        if isNaN(idx)
+            idx = this.dom_data.length
+            node.setAttribute(this.dom_attr, idx+'')
+            this.dom_data.push({})
+        this.dom_data[idx][key] = val
+    # }}}
+
+    log: (args...) -> # {{{
+        # Output args to the window.console object. args are automatically
+        # coerced to strings
+        if args
+            msg = args.join(' ')
+            if window?.console?.log
+                window.console.log(msg)
+            else if process?.stdout?.write
+                process.stdout.write(msg + '\n')
+    # }}}
+
+    window_scroll_pos: (win=window) -> # {{{
+        # The current scroll position of the browser window
+        if typeof(win.pageXOffset) == 'number'
+            x = win.pageXOffset
+            y = win.pageYOffset
+        else # IE < 9
+            if document.body and ( document.body.scrollLeft or document.body.scrollTop )
+                x = document.body.scrollLeft
+                y = document.body.scrollTop
+            else if document.documentElement and ( document.documentElement.scrollLeft or document.documentElement.scrollTop)
+                y = document.documentElement.scrollTop
+                x = document.documentElement.scrollLeft
+        return [x, y]
+    # }}}
+
+    viewport_to_document: (x, y, doc=window?.document) -> # {{{
+        # Convert x, y from the viewport (window) co-ordinate system to the
+        # document (body) co-ordinate system
+        until doc == window.document
+            # We are in a frame
+            frame = doc.defaultView.frameElement
+            rect = frame.getBoundingClientRect()
+            x += rect.left
+            y += rect.top
+            doc = frame.ownerDocument
+        win = doc.defaultView
+        [wx, wy] = this.window_scroll_pos(win)
+        x += wx
+        y += wy
+        return [x, y]
+    # }}}
+
+    absleft: (elem) -> # {{{
+        # The left edge of elem in document co-ords. Works in all
+        # circumstances, including column layout. Note that this will cause
+        # a relayout if the render tree is dirty.
+        r = elem.getBoundingClientRect()
+        return this.viewport_to_document(r.left, 0, elem.ownerDocument)[0]
+    # }}}
+
+if window?
+    window.calibre_utils = new CalibreUtils()
+

src/calibre/gui2/actions/copy_to_library.py

@@ -216,7 +216,8 @@ class CopyToLibraryAction(InterfaceAction):
                 if ci.isValid():
                     row = ci.row()
 
-                v.model().delete_books_by_id(self.worker.processed)
+                v.model().delete_books_by_id(self.worker.processed,
+                        permanent=True)
                 self.gui.iactions['Remove Books'].library_ids_deleted(
                         self.worker.processed, row)
 

src/calibre/gui2/book_details.py

@@ -104,8 +104,11 @@ def render_data(mi, use_roman_numbers=True, all_fields=False):
             field = 'title_sort'
         if all_fields:
             display = True
-        if (not display or not metadata or mi.is_null(field) or
+        if metadata['datatype'] == 'bool':
-                field == 'comments'):
+            isnull = mi.get(field) is None
+        else:
+            isnull = mi.is_null(field)
+        if (not display or not metadata or isnull or field == 'comments'):
             continue
         name = metadata['name']
         if not name:

src/calibre/gui2/library/models.py

@@ -220,16 +220,15 @@ class BooksModel(QAbstractTableModel): # {{{
         self.count_changed()
         self.reset()
 
-    def delete_books(self, indices):
+    def delete_books(self, indices, permanent=False):
         ids = map(self.id, indices)
-        for id in ids:
+        self.delete_books_by_id(ids, permanent=permanent)
-            self.db.delete_book(id, notify=False)
-        self.books_deleted()
         return ids
 
-    def delete_books_by_id(self, ids):
+    def delete_books_by_id(self, ids, permanent=False):
         for id in ids:
-            self.db.delete_book(id)
+            self.db.delete_book(id, permanent=permanent, do_clean=False)
+        self.db.clean()
         self.books_deleted()
 
     def books_added(self, num):

src/calibre/gui2/preferences/tweaks.py

@@ -243,7 +243,8 @@ class Tweaks(QAbstractListModel, SearchQueryParser): # {{{
         query = lower(query)
         for r in candidates:
             dat = self.data(self.index(r), Qt.UserRole)
-            if query in lower(dat.name):# or query in lower(dat.doc):
+            var_names = u' '.join(dat.default_values)
+            if query in lower(dat.name) or query in lower(var_names):
                 ans.add(r)
         return ans
 

src/calibre/gui2/store/stores/foyles_uk_plugin.py

@@ -40,32 +40,27 @@ class FoylesUKStore(BasicStoreConfig, StorePlugin):
             d.exec_()
 
     def search(self, query, max_results=10, timeout=60):
-        url = 'http://www.foyles.co.uk/Public/Shop/Search.aspx?fFacetId=1015&searchBy=1&quick=true&term=' + urllib2.quote(query)
+        url = 'http://ebooks.foyles.co.uk/search_for-' + urllib2.quote(query)
 
         br = browser()
 
         counter = max_results
         with closing(br.open(url, timeout=timeout)) as f:
             doc = html.fromstring(f.read())
-            for data in doc.xpath('//table[contains(@id, "MainContent")]/tr/td/div[contains(@class, "Item")]'):
+            for data in doc.xpath('//div[@class="doc-item"]'):
                 if counter <= 0:
                     break
-                id = ''.join(data.xpath('.//a[@class="Title"]/@href')).strip()
+                id_ = ''.join(data.xpath('.//p[@class="doc-cover"]/a/@href')).strip()
-                if not id:
+                if not id_:
                     continue
 
-                # filter out the audio books
+                cover_url = ''.join(data.xpath('.//p[@class="doc-cover"]/a/img/@src'))
-                if not data.xpath('boolean(.//div[@class="Relative"]/ul/li[contains(text(), "ePub")])'):
+                title = ''.join(data.xpath('.//span[@class="title"]/a/text()'))
-                    continue
+                author = ', '.join(data.xpath('.//span[@class="author"]/span[@class="author"]/text()'))
-
+                price = ''.join(data.xpath('.//span[@class="price"]/text()'))
-                cover_url = ''.join(data.xpath('.//a[@class="Jacket"]/img/@src'))
+                format_ = ''.join(data.xpath('.//p[@class="doc-meta-format"]/span[last()]/text()'))
-                title = ''.join(data.xpath('.//a[@class="Title"]/text()'))
+                format_, ign, drm = format_.partition(' ')
-                author = ', '.join(data.xpath('.//span[@class="Author"]/text()'))
+                drm = SearchResult.DRM_LOCKED if 'DRM' in drm else SearchResult.DRM_UNLOCKED
-                price = ''.join(data.xpath('./ul/li[@class="Strong"]/text()'))
-                mo = re.search('£[\d\.]+', price)
-                if mo is None:
-                    continue
-                price = mo.group(0)
 
                 counter -= 1
 
@@ -74,8 +69,8 @@ class FoylesUKStore(BasicStoreConfig, StorePlugin):
                 s.title = title.strip()
                 s.author = author.strip()
                 s.price = price
-                s.detail_item = id
+                s.detail_item = id_
-                s.drm = SearchResult.DRM_LOCKED
+                s.drm = drm
-                s.formats = 'ePub'
+                s.formats = format_
 
                 yield s

src/calibre/gui2/viewer/documentview.py

@@ -136,7 +136,7 @@ class Document(QWebPage): # {{{
         self.max_fs_width = min(opts.max_fs_width, screen_width-50)
 
     def fit_images(self):
-        if self.do_fit_images:
+        if self.do_fit_images and not self.in_paged_mode:
             self.javascript('setup_image_scaling_handlers()')
 
     def add_window_objects(self):
@@ -219,6 +219,7 @@ class Document(QWebPage): # {{{
         if scroll_width > self.window_width:
             sz.setWidth(scroll_width+side_margin)
             self.setPreferredContentsSize(sz)
+        self.javascript('window.paged_display.fit_images()')
 
     @property
     def column_boundaries(self):

src/calibre/gui2/viewer/javascript.py

@@ -31,10 +31,11 @@ class JavaScriptLoader(object):
             'cfi':'ebooks.oeb.display.cfi',
             'indexing':'ebooks.oeb.display.indexing',
             'paged':'ebooks.oeb.display.paged',
+            'utils':'ebooks.oeb.display.utils',
         }
 
     ORDER = ('jquery', 'jquery_scrollTo', 'bookmarks', 'referencing', 'images',
-            'hyphenation', 'hyphenator', 'cfi', 'indexing', 'paged')
+            'hyphenation', 'hyphenator', 'utils', 'cfi', 'indexing', 'paged')
 
 
     def __init__(self, dynamic_coffeescript=False):

src/calibre/library/catalogs/epub_mobi_builder.py

@@ -14,6 +14,7 @@ from calibre.ebooks.chardet import substitute_entites
 from calibre.ptempfile import PersistentTemporaryDirectory
 from calibre.utils.config import config_dir
 from calibre.utils.date import format_date, is_date_undefined, now as nowf
+from calibre.utils.filenames import ascii_text
 from calibre.utils.icu import capitalize
 from calibre.utils.magick.draw import thumbnail
 from calibre.utils.zipfile import ZipFile
@@ -689,7 +690,7 @@ Author '{0}':
                 this_title['series'] = None
                 this_title['series_index'] = 0.0
 
-            this_title['title_sort'] = self.generateSortTitle(this_title['title'])
+            this_title['title_sort'] = self.generateSortTitle(ascii_text(this_title['title']))
             if 'authors' in record:
                 # from calibre.ebooks.metadata import authors_to_string
                 # return authors_to_string(self.authors)
@@ -704,6 +705,7 @@ Author '{0}':
                 this_title['author_sort'] = record['author_sort']
             else:
                 this_title['author_sort'] = self.author_to_author_sort(this_title['author'])
+            this_title['author_sort'] = ascii_text(this_title['author_sort'])
 
             if record['publisher']:
                 this_title['publisher'] = re.sub('&', '&', record['publisher'])
@@ -3768,7 +3770,7 @@ Author '{0}':
                     else:
                         word = '%10.0f' % (float(word))
                 translated.append(word)
-        return ' '.join(translated)
+        return ascii_text(' '.join(translated))
 
     def generateThumbnail(self, title, image_dir, thumb_file):
         '''

src/calibre/library/cli.py

@@ -999,6 +999,55 @@ def command_saved_searches(args, dbpath):
 
     return 0
 
+def backup_metadata_option_parser():
+    parser = get_parser(_('''\
+%prog backup_metadata [options]
+
+Backup the metadata stored in the database into individual OPF files in each
+books directory. This normally happens automatically, but you can run this
+command to force re-generation of the OPF files, with the --all option.
+
+Note that there is normally no need to do this, as the OPF files are backed up
+automatically, every time metadata is changed.
+'''))
+    parser.add_option('--all', default=False, action='store_true',
+        help=_('Normally, this command only operates on books that have'
+            ' out of date OPF files. This option makes it operate on all'
+            ' books.'))
+    return parser
+
+class BackupProgress(object):
+
+    def __init__(self):
+        self.total = 0
+        self.count = 0
+
+    def __call__(self, book_id, mi, ok):
+        if mi is True:
+            self.total = book_id
+        else:
+            self.count += 1
+            prints(u'%.1f%% %s - %s'%((self.count*100)/float(self.total),
+                book_id, mi.title))
+
+def command_backup_metadata(args, dbpath):
+    parser = backup_metadata_option_parser()
+    opts, args = parser.parse_args(args)
+    if len(args) != 0:
+        parser.print_help()
+        return 1
+
+    if opts.library_path is not None:
+        dbpath = opts.library_path
+    if isbytestring(dbpath):
+        dbpath = dbpath.decode(preferred_encoding)
+    db = LibraryDatabase2(dbpath)
+    book_ids = None
+    if opts.all:
+        book_ids = db.all_ids()
+    db.dump_metadata(book_ids=book_ids, callback=BackupProgress())
+
+
 def check_library_option_parser():
     from calibre.library.check_library import CHECKS
     parser = get_parser(_('''\
@@ -1275,7 +1324,7 @@ COMMANDS = ('list', 'add', 'remove', 'add_format', 'remove_format',
             'show_metadata', 'set_metadata', 'export', 'catalog',
             'saved_searches', 'add_custom_column', 'custom_columns',
             'remove_custom_column', 'set_custom', 'restore_database',
-            'check_library', 'list_categories')
+            'check_library', 'list_categories', 'backup_metadata')
 
 
 def option_parser():

src/calibre/library/database2.py

@@ -808,18 +808,30 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                 pass
 
     def dump_metadata(self, book_ids=None, remove_from_dirtied=True,
-            commit=True):
+            commit=True, callback=None):
         '''
-        Write metadata for each record to an individual OPF file
+        Write metadata for each record to an individual OPF file. If callback
+        is not None, it is called once at the start with the number of book_ids
+        being processed. And once for every book_id, with arguments (book_id,
+        mi, ok).
         '''
         if book_ids is None:
             book_ids = [x[0] for x in self.conn.get(
                 'SELECT book FROM metadata_dirtied', all=True)]
+
+        if callback is not None:
+            book_ids = tuple(book_ids)
+            callback(len(book_ids), True, False)
+
         for book_id in book_ids:
             if not self.data.has_id(book_id):
+                if callback is not None:
+                    callback(book_id, None, False)
                 continue
             path, mi, sequence = self.get_metadata_for_dump(book_id)
             if path is None:
+                if callback is not None:
+                    callback(book_id, mi, False)
                 continue
             try:
                 raw = metadata_to_opf(mi)
@@ -829,6 +841,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                     self.clear_dirtied(book_id, sequence)
             except:
                 pass
+            if callback is not None:
+                callback(book_id, mi, True)
         if commit:
             self.conn.commit()
 
@@ -1411,7 +1425,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         opath = self.format_abspath(book_id, nfmt, index_is_id=True)
         return fmt if opath is None else nfmt
 
-    def delete_book(self, id, notify=True, commit=True, permanent=False):
+    def delete_book(self, id, notify=True, commit=True, permanent=False,
+            do_clean=True):
         '''
         Removes book from the result cache and the underlying database.
         If you set commit to False, you must call clean() manually afterwards
@@ -1428,6 +1443,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         self.conn.execute('DELETE FROM books WHERE id=?', (id,))
         if commit:
             self.conn.commit()
+            if do_clean:
                 self.clean()
         self.data.books_deleted([id])
         if notify:

src/calibre/library/server/mobile.py

@@ -97,6 +97,7 @@ def build_index(books, num, search, sort, order, start, total, url_base, CKEYS,
 
     search_box = build_search_box(num, search, sort, order, prefix)
     navigation = build_navigation(start, num, total, prefix+url_base)
+    navigation2 = build_navigation(start, num, total, prefix+url_base)
     bookt = TABLE(id='listing')
 
     body = BODY(
@@ -104,7 +105,9 @@ def build_index(books, num, search, sort, order, start, total, url_base, CKEYS,
         search_box,
         navigation,
         HR(CLASS('spacer')),
-        bookt
+        bookt,
+        HR(CLASS('spacer')),
+        navigation2
     )
 
     # Book list {{{
@@ -155,7 +158,6 @@ def build_index(books, num, search, sort, order, start, total, url_base, CKEYS,
         bookt.append(TR(thumbnail, data))
     # }}}
 
-    body.append(HR())
     body.append(DIV(
         A(_('Switch to the full interface (non-mobile interface)'),
             href=prefix+"/browse",

src/calibre/linux.py

@@ -354,8 +354,8 @@ class PostInstall:
                 check_call('xdg-icon-resource install --noupdate --context mimetypes --size 128 calibre-lrf.png text-lrs', shell=True)
                 self.icon_resources.append(('mimetypes', 'application-lrs',
                 '128'))
-                render_img('lt.png', 'calibre-gui.png')
+                render_img('lt.png', 'calibre-gui.png', width=256, height=256)
-                check_call('xdg-icon-resource install --noupdate --size 128 calibre-gui.png calibre-gui', shell=True)
+                check_call('xdg-icon-resource install --noupdate --size 256 calibre-gui.png calibre-gui', shell=True)
                 self.icon_resources.append(('apps', 'calibre-gui', '128'))
                 render_img('viewer.png', 'calibre-viewer.png')
                 check_call('xdg-icon-resource install --size 128 calibre-viewer.png calibre-viewer', shell=True)

src/calibre/utils/icu.c

@@ -5,6 +5,7 @@
 #include <unicode/uclean.h>
 #include <unicode/ucol.h>
 #include <unicode/ustring.h>
+#include <unicode/usearch.h>
 
 
 // Collator object definition {{{
@@ -12,6 +13,7 @@ typedef struct {
     PyObject_HEAD
     // Type-specific fields go here.
     UCollator *collator;
+    USet *contractions;
 
 } icu_Collator;
 
@@ -19,6 +21,7 @@ static void
 icu_Collator_dealloc(icu_Collator* self)
 {
     if (self->collator != NULL) ucol_close(self->collator);
+    if (self->contractions != NULL) uset_close(self->contractions);
     self->collator = NULL;
     self->ob_type->tp_free((PyObject*)self);
 }
@@ -29,18 +32,19 @@ icu_Collator_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
     icu_Collator *self;
     const char *loc;
     UErrorCode status = U_ZERO_ERROR;
+    UCollator *collator;
 
     if (!PyArg_ParseTuple(args, "s", &loc)) return NULL;
+    collator = ucol_open(loc, &status);
+    if (collator == NULL || U_FAILURE(status)) { 
+        PyErr_SetString(PyExc_Exception, "Failed to create collator.");
+        return NULL;
+    }
 
     self = (icu_Collator *)type->tp_alloc(type, 0);
     if (self != NULL) {
-        self->collator = ucol_open(loc, &status);
+        self->collator = collator;
-        if (self->collator == NULL || U_FAILURE(status)) { 
+        self->contractions = NULL;
-            PyErr_SetString(PyExc_Exception, "Failed to create collator.");
-            self->collator = NULL;
-            Py_DECREF(self);
-            return NULL;
-        }
     }
 
     return (PyObject *)self;
@@ -63,13 +67,30 @@ icu_Collator_display_name(icu_Collator *self, void *closure) {
 
     u_strToUTF8(buf, 100, NULL, dname, -1, &status);
     if (U_FAILURE(status)) {
-        PyErr_SetString(PyExc_Exception, "Failed ot convert dname to UTF-8"); return NULL;
+        PyErr_SetString(PyExc_Exception, "Failed to convert dname to UTF-8"); return NULL;
     }
     return Py_BuildValue("s", buf);
 }
 
 // }}}
 
+// Collator.strength {{{
+static PyObject *
+icu_Collator_get_strength(icu_Collator *self, void *closure) {
+    return Py_BuildValue("i", ucol_getStrength(self->collator));
+}
+
+static int
+icu_Collator_set_strength(icu_Collator *self, PyObject *val, void *closure) {
+    if (!PyInt_Check(val)) {
+        PyErr_SetString(PyExc_TypeError, "Strength must be an integer.");
+        return -1;
+    }
+    ucol_setStrength(self->collator, (int)PyInt_AS_LONG(val));
+    return 0;
+}
+// }}}
+
 // Collator.actual_locale {{{
 static PyObject *
 icu_Collator_actual_locale(icu_Collator *self, void *closure) {
@@ -164,7 +185,126 @@ icu_Collator_strcmp(icu_Collator *self, PyObject *args, PyObject *kwargs) {
     return Py_BuildValue("i", res);
 } // }}}
 
+// Collator.find {{{
+static PyObject *
+icu_Collator_find(icu_Collator *self, PyObject *args, PyObject *kwargs) {
+    PyObject *a_, *b_;
+    size_t asz, bsz;
+    UChar *a, *b;
+    wchar_t *aw, *bw;
+    UErrorCode status = U_ZERO_ERROR;
+    UStringSearch *search = NULL;
+    int32_t pos = -1, length = -1;
   
+    if (!PyArg_ParseTuple(args, "UU", &a_, &b_)) return NULL;
+    asz = PyUnicode_GetSize(a_); bsz = PyUnicode_GetSize(b_);
+    
+    a = (UChar*)calloc(asz*4 + 2, sizeof(UChar));
+    b = (UChar*)calloc(bsz*4 + 2, sizeof(UChar));
+    aw = (wchar_t*)calloc(asz*4 + 2, sizeof(wchar_t));
+    bw = (wchar_t*)calloc(bsz*4 + 2, sizeof(wchar_t));
+
+    if (a == NULL || b == NULL || aw == NULL || bw == NULL) return PyErr_NoMemory();
+
+    PyUnicode_AsWideChar((PyUnicodeObject*)a_, aw, asz*4+1);
+    PyUnicode_AsWideChar((PyUnicodeObject*)b_, bw, bsz*4+1);
+    u_strFromWCS(a, asz*4 + 1, NULL, aw, -1, &status);
+    u_strFromWCS(b, bsz*4 + 1, NULL, bw, -1, &status);
+
+    if (U_SUCCESS(status)) {
+        search = usearch_openFromCollator(a, -1, b, -1, self->collator, NULL, &status);
+        if (U_SUCCESS(status)) {
+            pos = usearch_first(search, &status);
+            if (pos != USEARCH_DONE) 
+                length = (pos == USEARCH_DONE) ? -1 : usearch_getMatchedLength(search);
+            else
+                pos = -1;
+        }
+        if (search != NULL) usearch_close(search);
+    }
+
+    free(a); free(b); free(aw); free(bw);
+
+    return Py_BuildValue("ii", pos, length);
+} // }}}
+
+// Collator.contractions {{{
+static PyObject *
+icu_Collator_contractions(icu_Collator *self, PyObject *args, PyObject *kwargs) {
+    UErrorCode status = U_ZERO_ERROR;
+    UChar *str;
+    UChar32 start=0, end=0;
+    int32_t count = 0, len = 0, dlen = 0, i;
+    PyObject *ans = Py_None, *pbuf;
+    wchar_t *buf;
+
+    if (self->contractions == NULL) {
+        self->contractions = uset_open(1, 0);
+        if (self->contractions == NULL) return PyErr_NoMemory();
+        ucol_getContractionsAndExpansions(self->collator, self->contractions, NULL, 0, &status);
+    }
+    status = U_ZERO_ERROR; 
+
+    str = (UChar*)calloc(100, sizeof(UChar));
+    buf = (wchar_t*)calloc(4*100+2, sizeof(wchar_t));
+    if (str == NULL || buf == NULL) return PyErr_NoMemory();
+
+    count = uset_getItemCount(self->contractions);
+    ans = PyTuple_New(count);
+    if (ans != NULL) {
+        for (i = 0; i < count; i++) {
+            len = uset_getItem(self->contractions, i, &start, &end, str, 1000, &status);
+            if (len >= 2) {
+                // We have a string
+                status = U_ZERO_ERROR;
+                u_strToWCS(buf, 4*100 + 1, &dlen, str, len, &status);
+                pbuf = PyUnicode_FromWideChar(buf, dlen);
+                if (pbuf == NULL) return PyErr_NoMemory();
+                PyTuple_SetItem(ans, i, pbuf);
+            } else {
+                // Ranges dont make sense for contractions, ignore them
+                PyTuple_SetItem(ans, i, Py_None);
+            }
+        }
+    }
+    free(str); free(buf);
+  
+    return Py_BuildValue("O", ans);
+} // }}}
+
+// Collator.span_contractions {{{
+static PyObject *
+icu_Collator_span_contractions(icu_Collator *self, PyObject *args, PyObject *kwargs) {
+    int span_type;
+    UErrorCode status = U_ZERO_ERROR;
+    PyObject *str;
+    size_t slen = 0;
+    wchar_t *buf;
+    UChar *s;
+
+    if (!PyArg_ParseTuple(args, "Ui", &str, &span_type)) return NULL;
+
+    if (self->contractions == NULL) {
+        self->contractions = uset_open(1, 0);
+        if (self->contractions == NULL) return PyErr_NoMemory();
+        ucol_getContractionsAndExpansions(self->collator, self->contractions, NULL, 0, &status);
+    }
+    status = U_ZERO_ERROR; 
+
+    slen = PyUnicode_GetSize(str);
+    buf = (wchar_t*)calloc(slen*4 + 2, sizeof(wchar_t));
+    s = (UChar*)calloc(slen*4 + 2, sizeof(UChar));
+    if (buf == NULL || s == NULL) return PyErr_NoMemory();
+    slen = PyUnicode_AsWideChar((PyUnicodeObject*)str, buf, slen);
+    u_strFromWCS(s, slen*4+1, NULL, buf, slen, &status);
+
+    free(buf); free(s);
+    return Py_BuildValue("i", uset_span(self->contractions, s, slen, span_type));
+} // }}}
+
+
+static PyObject*
+icu_Collator_clone(icu_Collator *self, PyObject *args, PyObject *kwargs);
 
 static PyMethodDef icu_Collator_methods[] = {
     {"sort_key", (PyCFunction)icu_Collator_sort_key, METH_VARARGS,
@@ -175,6 +315,22 @@ static PyMethodDef icu_Collator_methods[] = {
      "strcmp(unicode object, unicode object) -> strcmp(a, b) <=> cmp(sorty_key(a), sort_key(b)), but faster."
     },
 
+    {"find", (PyCFunction)icu_Collator_find, METH_VARARGS,
+        "find(pattern, source) -> returns the position and length of the first occurrence of pattern in source. Returns (-1, -1) if not found."
+    },
+
+    {"contractions", (PyCFunction)icu_Collator_contractions, METH_VARARGS,
+        "contractions() -> returns the contractions defined for this collator."
+    },
+
+    {"span_contractions", (PyCFunction)icu_Collator_span_contractions, METH_VARARGS,
+        "span_contractions(src, span_condition) -> returns the length of the initial substring according to span_condition in the set of contractions for this collator. Returns 0 if src does not fit the span_condition. The span_condition can be one of USET_SPAN_NOT_CONTAINED, USET_SPAN_CONTAINED, USET_SPAN_SIMPLE."
+    },
+
+    {"clone", (PyCFunction)icu_Collator_clone, METH_VARARGS,
+        "clone() -> returns a clone of this collator."
+    },
+
     {NULL}  /* Sentinel */
 };
 
@@ -189,6 +345,12 @@ static PyGetSetDef  icu_Collator_getsetters[] = {
      (char *)"Display name of this collator in English. The name reflects the actual data source used.",
      NULL},
 
+    {(char *)"strength",
+     (getter)icu_Collator_get_strength, (setter)icu_Collator_set_strength,
+     (char *)"The strength of this collator.",
+     NULL},
+
+
     {NULL}  /* Sentinel */
 };
 
@@ -236,6 +398,31 @@ static PyTypeObject icu_CollatorType = { // {{{
 
 // }}
 
+// Collator.clone {{{
+static PyObject*
+icu_Collator_clone(icu_Collator *self, PyObject *args, PyObject *kwargs)
+{
+    UCollator *collator;
+    UErrorCode status = U_ZERO_ERROR;
+    int32_t bufsize = -1;
+    icu_Collator *clone;
+
+    collator = ucol_safeClone(self->collator, NULL, &bufsize, &status);
+
+    if (collator == NULL || U_FAILURE(status)) {
+        PyErr_SetString(PyExc_Exception, "Failed to create collator.");
+        return NULL;
+    }
+
+    clone = PyObject_New(icu_Collator, &icu_CollatorType);
+    if (clone == NULL) return PyErr_NoMemory();
+
+    clone->collator = collator;
+    clone->contractions = NULL;
+
+    return (PyObject*) clone;
+
+} // }}}
 
 // }}}
 
@@ -411,6 +598,7 @@ static PyMethodDef icu_methods[] = {
     {NULL}  /* Sentinel */
 };
 
+#define ADDUCONST(x) PyModule_AddIntConstant(m, #x, x)
 
 PyMODINIT_FUNC
 initicu(void) 
@@ -432,5 +620,22 @@ initicu(void)
     // uint8_t must be the same size as char
     PyModule_AddIntConstant(m, "ok", (U_SUCCESS(status) && sizeof(uint8_t) == sizeof(char)) ? 1 : 0);
 
+    ADDUCONST(USET_SPAN_NOT_CONTAINED);
+    ADDUCONST(USET_SPAN_CONTAINED);
+    ADDUCONST(USET_SPAN_SIMPLE);
+    ADDUCONST(UCOL_DEFAULT);
+    ADDUCONST(UCOL_PRIMARY);
+    ADDUCONST(UCOL_SECONDARY);
+    ADDUCONST(UCOL_TERTIARY);
+    ADDUCONST(UCOL_DEFAULT_STRENGTH);
+    ADDUCONST(UCOL_QUATERNARY);
+    ADDUCONST(UCOL_IDENTICAL);
+    ADDUCONST(UCOL_OFF);
+    ADDUCONST(UCOL_ON);
+    ADDUCONST(UCOL_SHIFTED);
+    ADDUCONST(UCOL_NON_IGNORABLE);
+    ADDUCONST(UCOL_LOWER_FIRST);
+    ADDUCONST(UCOL_UPPER_FIRST);
+
 }
 // }}}

src/calibre/utils/icu.py

@@ -12,7 +12,7 @@ from functools import partial
 from calibre.constants import plugins
 from calibre.utils.config_base import tweaks
 
-_icu = _collator = None
+_icu = _collator = _primary_collator = None
 _locale = None
 
 _none = u''
@@ -48,6 +48,12 @@ def load_collator():
             _collator = icu.Collator(get_locale())
     return _collator
 
+def primary_collator():
+    global _primary_collator
+    if _primary_collator is None:
+        _primary_collator = _collator.clone()
+        _primary_collator.strength = _icu.UCOL_PRIMARY
+    return _primary_collator
 
 def py_sort_key(obj):
     if not obj:
@@ -59,6 +65,18 @@ def icu_sort_key(collator, obj):
         return _none2
     return collator.sort_key(lower(obj))
 
+def py_find(pattern, source):
+    pos = source.find(pattern)
+    if pos > -1:
+        return pos, len(pattern)
+    return -1, -1
+
+def icu_find(collator, pattern, source):
+    try:
+        return collator.find(pattern, source)
+    except TypeError:
+        return collator.find(unicode(pattern), unicode(source))
+
 def py_case_sensitive_sort_key(obj):
     if not obj:
         return _none
@@ -72,7 +90,7 @@ def icu_case_sensitive_sort_key(collator, obj):
 def icu_strcmp(collator, a, b):
     return collator.strcmp(lower(a), lower(b))
 
-def py_strcmp(a, b):
+def py_strcmp(a, b, strength=None):
     return cmp(a.lower(), b.lower())
 
 def icu_case_sensitive_strcmp(collator, a, b):
@@ -82,6 +100,30 @@ def icu_capitalize(s):
     s = lower(s)
     return s.replace(s[0], upper(s[0]), 1) if s else s
 
+_cmap = {}
+def icu_contractions(collator):
+    global _cmap
+    ans = _cmap.get(collator, None)
+    if ans is None:
+        ans = collator.contractions()
+        ans = frozenset(filter(None, ans)) if ans else {}
+        _cmap[collator] = ans
+    return ans
+
+def py_span_contractions(*args, **kwargs):
+    return 0
+
+def icu_span_contractions(src, span_type=None, collator=None):
+    global _collator
+    if collator is None:
+        collator = _collator
+    if span_type is None:
+        span_type = _icu.USET_SPAN_SIMPLE
+    try:
+        return collator.span_contractions(src, span_type)
+    except TypeError:
+        return collator.span_contractions(unicode(src), span_type)
+
 load_icu()
 load_collator()
 _icu_not_ok = _icu is None or _collator is None
@@ -117,6 +159,28 @@ title_case = (lambda s: s.title()) if _icu_not_ok else \
 capitalize = (lambda s: s.capitalize()) if _icu_not_ok else \
     (lambda s: icu_capitalize(s))
 
+find = (py_find if _icu_not_ok else partial(icu_find, _collator))
+
+contractions = ((lambda : {}) if _icu_not_ok else (partial(icu_contractions,
+    _collator)))
+
+span_contractions = (py_span_contractions if _icu_not_ok else
+        icu_span_contractions)
+
+def primary_strcmp(a, b):
+    'strcmp that ignores case and accents on letters'
+    if _icu_not_ok:
+        from calibre.utils.filenames import ascii_text
+        return py_strcmp(ascii_text(a), ascii_text(b))
+    return primary_collator().strcmp(a, b)
+
+def primary_find(pat, src):
+    'find that ignores case and accents on letters'
+    if _icu_not_ok:
+        from calibre.utils.filenames import ascii_text
+        return py_find(ascii_text(pat), ascii_text(src))
+    return icu_find(primary_collator(), pat, src)
+
 ################################################################################
 
 def test(): # {{{
@@ -240,6 +304,18 @@ pêché'''
         print 'Capitalize:', x, '->', 'py:', x.capitalize().encode('utf-8'), 'icu:', capitalize(x).encode('utf-8')
         print
 
+    print '\nTesting primary collation'
+    for k, v in {u'pèché': u'peche', u'flüße':u'flusse'}.iteritems():
+        if primary_strcmp(k, v) != 0:
+            print 'primary_strcmp() failed with %s != %s'%(k, v)
+        if primary_find(v, u' '+k)[0] != 1:
+            print 'primary_find() failed with %s not in %s'%(v, k)
+
+    global _primary_collator
+    _primary_collator = _icu.Collator('es')
+    if primary_strcmp(u'peña', u'pena') == 0:
+        print 'Primary collation in Spanish locale failed'
+
 # }}}
 
 if __name__ == '__main__':