Use ICU for sorting

setup/build_environment.py

@@ -91,11 +91,15 @@ podofo_inc = '/usr/include/podofo'
 podofo_lib = '/usr/lib'
 chmlib_inc_dirs = chmlib_lib_dirs = []
 sqlite_inc_dirs = []
+icu_inc_dirs = []
+icu_lib_dirs = []
 
 if iswindows:
     prefix  = r'C:\cygwin\home\kovid\sw'
     sw_inc_dir  = os.path.join(prefix, 'include')
     sw_lib_dir  = os.path.join(prefix, 'lib')
+    icu_inc_dirs = [sw_inc_dir]
+    icu_lib_dirs = [sw_lib_dir]
     sqlite_inc_dirs = [sw_inc_dir]
     fc_inc = os.path.join(sw_inc_dir, 'fontconfig')
     fc_lib = sw_lib_dir

setup/extensions.py

@@ -18,7 +18,8 @@ from setup.build_environment import fc_inc, fc_lib, chmlib_inc_dirs, \
         QMAKE, msvc, MT, win_inc, win_lib, png_inc_dirs, win_ddk, \
         magick_inc_dirs, magick_lib_dirs, png_lib_dirs, png_libs, \
         magick_error, magick_libs, ft_lib_dirs, ft_libs, jpg_libs, \
-        jpg_lib_dirs, chmlib_lib_dirs, sqlite_inc_dirs
+        jpg_lib_dirs, chmlib_lib_dirs, sqlite_inc_dirs, icu_inc_dirs, \
+        icu_lib_dirs
 MT
 isunix = islinux or isosx or isfreebsd
 
@@ -56,8 +57,19 @@ pdfreflow_libs = []
 if iswindows:
     pdfreflow_libs = ['advapi32', 'User32', 'Gdi32', 'zlib']
 
+icu_libs = ['icudata', 'icui18n', 'icuuc', 'icuio']
+if iswindows:
+    icu_libs = ['icudt', 'icuin', 'icuuc', 'icuio']
+
 extensions = [
 
+    Extension('icu',
+        ['calibre/utils/icu.c'],
+        libraries=icu_libs,
+        lib_dirs=icu_lib_dirs,
+        inc_dirs=icu_inc_dirs,
+        ),
+
     Extension('sqlite_custom',
         ['calibre/library/sqlite_custom.c'],
         inc_dirs=sqlite_inc_dirs

setup/installer/windows/freeze.py

@@ -199,7 +199,7 @@ class Win32Freeze(Command, WixMixIn):
         for pat in ('*.dll',):
             for f in glob.glob(os.path.join(bindir, pat)):
                 ok = True
-                for ex in ('expatw',):
+                for ex in ('expatw', 'testplug'):
                     if ex in f.lower():
                         ok = False
                 if not ok: continue

setup/installer/windows/notes.rst

@@ -77,6 +77,15 @@ Test it on the target system with
 
 calibre-debug -c "import _imaging, _imagingmath, _imagingft, _imagingcms"
 
+ICU
+-------
+
+Download the win32 msvc9 binary from http://www.icu-project.org/download/4.4.html
+
+Note that 4.4 is the last version of ICU that can be compiled (is precompiled) with msvc9
+
+Put the dlls into sw/bin and the unicode dir into sw/include and the contents of lib int sw/lib
+
 Libunrar
 ----------
 

src/calibre/constants.py

@@ -67,7 +67,8 @@ if plugins is None:
                 'pdfreflow',
                 'progress_indicator',
                 'chmlib',
-                'chm_extra'
+                'chm_extra',
+                'icu',
             ] + \
                     (['winutil'] if iswindows else []) + \
                     (['usbobserver'] if isosx else []):

src/calibre/library/caches.py

@@ -796,11 +796,13 @@ class SortKey(object):
 class SortKeyGenerator(object):
 
     def __init__(self, fields, field_metadata, data):
+        from calibre.utils.icu import sort_key
         self.field_metadata = field_metadata
         self.orders = [-1 if x[1] else 1 for x in fields]
         self.entries = [(x[0], field_metadata[x[0]]) for x in fields]
         self.library_order = tweaks['title_series_sorting'] == 'library_order'
         self.data = data
+        self.string_sort_key = sort_key
 
     def __call__(self, record):
         values = tuple(self.itervals(self.data[record]))
@@ -821,17 +823,14 @@ class SortKeyGenerator(object):
                 if val is None:
                     val = ('', 1)
                 else:
-                    val = val.lower()
                     if self.library_order:
                         val = title_sort(val)
                     sidx_fm = self.field_metadata[name + '_index']
                     sidx = record[sidx_fm['rec_index']]
-                    val = (val, sidx)
+                    val = (self.string_sort_key(val), sidx)
 
             elif dt in ('text', 'comments', 'composite', 'enumeration'):
-                if val is None:
+                val = self.string_sort_key(val)
-                    val = ''
-                val = val.lower()
 
             elif dt == 'bool':
                 val = {True: 1, False: 2, None: 3}.get(val, 3)

src/calibre/utils/icu.c

@@ -0,0 +1,220 @@
+#define UNICODE
+#define PY_SSIZE_T_CLEAN
+#include <Python.h>
+#include <unicode/utypes.h>
+#include <unicode/uclean.h>
+#include <unicode/ucol.h>
+#include <unicode/ustring.h>
+
+
+// Collator object definition {{{
+typedef struct {
+    PyObject_HEAD
+    // Type-specific fields go here.
+    UCollator *collator;
+
+} icu_Collator;
+
+static void
+icu_Collator_dealloc(icu_Collator* self)
+{
+    if (self->collator != NULL) ucol_close(self->collator);
+    self->collator = NULL;
+    self->ob_type->tp_free((PyObject*)self);
+}
+
+static PyObject *
+icu_Collator_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
+{
+    icu_Collator *self;
+    const char *loc;
+    UErrorCode status = U_ZERO_ERROR;
+
+    if (!PyArg_ParseTuple(args, "s", &loc)) return NULL;
+
+    self = (icu_Collator *)type->tp_alloc(type, 0);
+    if (self != NULL) {
+        self->collator = ucol_open(loc, &status);
+        if (self->collator == NULL || U_FAILURE(status)) { 
+            PyErr_SetString(PyExc_Exception, "Failed to create collator.");
+            self->collator = NULL;
+            Py_DECREF(self);
+            return NULL;
+        }
+    }
+
+    return (PyObject *)self;
+}
+
+// Collator.display_name {{{
+static PyObject *
+icu_Collator_display_name(icu_Collator *self, void *closure) {
+    const char *loc = NULL;
+    UErrorCode status = U_ZERO_ERROR;
+    UChar dname[400];
+    char buf[100];
+
+    loc = ucol_getLocaleByType(self->collator, ULOC_ACTUAL_LOCALE, &status);
+    if (loc == NULL || U_FAILURE(status)) {
+        PyErr_SetString(PyExc_Exception, "Failed to get actual locale"); return NULL;
+    }
+    ucol_getDisplayName(loc, "en", dname, 100, &status);
+    if (U_FAILURE(status)) return PyErr_NoMemory();
+
+    u_strToUTF8(buf, 100, NULL, dname, -1, &status);
+    if (U_FAILURE(status)) {
+        PyErr_SetString(PyExc_Exception, "Failed ot convert dname to UTF-8"); return NULL;
+    }
+    return Py_BuildValue("s", buf);
+}
+
+// }}}
+
+// Collator.actual_locale {{{
+static PyObject *
+icu_Collator_actual_locale(icu_Collator *self, void *closure) {
+    const char *loc = NULL;
+    UErrorCode status = U_ZERO_ERROR;
+
+    loc = ucol_getLocaleByType(self->collator, ULOC_ACTUAL_LOCALE, &status);
+    if (loc == NULL || U_FAILURE(status)) {
+        PyErr_SetString(PyExc_Exception, "Failed to get actual locale"); return NULL;
+    }
+    return Py_BuildValue("s", loc);
+}
+
+// }}}
+
+// Collator.sort_key {{{
+static PyObject *
+icu_Collator_sort_key(icu_Collator *self, PyObject *args, PyObject *kwargs) {
+    PyObject *o;
+    Py_ssize_t sz;
+    wchar_t *buf;
+    UChar *buf2;
+    uint8_t *buf3;
+    PyObject *ans;
+    UErrorCode status = U_ZERO_ERROR;
+   
+    if (!PyArg_ParseTuple(args, "U", &o)) return NULL;
+
+    sz = PyUnicode_GetSize(o);
+
+    buf = (wchar_t*)calloc(sz*2 + 1, sizeof(wchar_t));
+    buf2 = (UChar*)calloc(sz*2 + 1, sizeof(UChar));
+    buf3 = (uint8_t*)calloc(sz*4 + 1, sizeof(uint8_t));
+
+    if (buf == NULL || buf2 == NULL || buf3 == NULL) return PyErr_NoMemory();
+
+    PyUnicode_AsWideChar((PyUnicodeObject *)o, buf, sz);
+
+    u_strFromWCS(buf2, 2*sz+1, NULL, buf, -1, &status);
+    if (U_SUCCESS(status))
+        ucol_getSortKey(self->collator, buf2, -1, buf3, sz*4+1);
+
+    ans = PyBytes_FromString((char *)buf3);
+    free(buf3); free(buf); free(buf2);
+    if (ans == NULL) return PyErr_NoMemory();
+
+    return ans;
+}
+
+static PyMethodDef icu_Collator_methods[] = {
+    {"sort_key", (PyCFunction)icu_Collator_sort_key, METH_VARARGS,
+     "sort_key(unicode object) -> Return a sort key for the given object as a bytestring. The idea is that these bytestring will sort using the builtin cmp function, just like the original unicode strings would sort in the current locale with ICU."
+    },
+
+    {NULL}  /* Sentinel */
+};
+
+static PyGetSetDef  icu_Collator_getsetters[] = {
+    {(char *)"actual_locale", 
+     (getter)icu_Collator_actual_locale, NULL,
+     (char *)"Actual locale used by this collator.",
+     NULL},
+
+    {(char *)"display_name", 
+     (getter)icu_Collator_display_name, NULL,
+     (char *)"Display name of this collator in English. The name reflects the actual data source used.",
+     NULL},
+
+    {NULL}  /* Sentinel */
+};
+
+static PyTypeObject icu_CollatorType = { // {{{
+    PyObject_HEAD_INIT(NULL)
+    0,                         /*ob_size*/
+    "icu.Collator",            /*tp_name*/
+    sizeof(icu_Collator),      /*tp_basicsize*/
+    0,                         /*tp_itemsize*/
+    (destructor)icu_Collator_dealloc, /*tp_dealloc*/
+    0,                         /*tp_print*/
+    0,                         /*tp_getattr*/
+    0,                         /*tp_setattr*/
+    0,                         /*tp_compare*/
+    0,                         /*tp_repr*/
+    0,                         /*tp_as_number*/
+    0,                         /*tp_as_sequence*/
+    0,                         /*tp_as_mapping*/
+    0,                         /*tp_hash */
+    0,                         /*tp_call*/
+    0,                         /*tp_str*/
+    0,                         /*tp_getattro*/
+    0,                         /*tp_setattro*/
+    0,                         /*tp_as_buffer*/
+    Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE,        /*tp_flags*/
+    "Collator",                  /* tp_doc */
+    0,		               /* tp_traverse */
+    0,		               /* tp_clear */
+    0,		               /* tp_richcompare */
+    0,		               /* tp_weaklistoffset */
+    0,		               /* tp_iter */
+    0,		               /* tp_iternext */
+    icu_Collator_methods,             /* tp_methods */
+    0,             /* tp_members */
+    icu_Collator_getsetters,                         /* tp_getset */
+    0,                         /* tp_base */
+    0,                         /* tp_dict */
+    0,                         /* tp_descr_get */
+    0,                         /* tp_descr_set */
+    0,                         /* tp_dictoffset */
+    0,      /* tp_init */
+    0,                         /* tp_alloc */
+    icu_Collator_new,                 /* tp_new */
+}; // }}}
+
+// }}
+
+
+// }}}
+
+
+// Module initialization {{{
+
+static PyMethodDef icu_methods[] = {
+    {NULL}  /* Sentinel */
+};
+
+
+PyMODINIT_FUNC
+initicu(void) 
+{
+    PyObject* m;
+    UErrorCode status = U_ZERO_ERROR;
+
+    u_init(&status);
+
+
+    if (PyType_Ready(&icu_CollatorType) < 0)
+        return;
+
+    m = Py_InitModule3("icu", icu_methods,
+                       "Wrapper for the ICU internationalization library");
+
+    Py_INCREF(&icu_CollatorType);
+    PyModule_AddObject(m, "Collator", (PyObject *)&icu_CollatorType);
+    // uint8_t must be the same size as char
+    PyModule_AddIntConstant(m, "ok", (U_SUCCESS(status) && sizeof(uint8_t) == sizeof(char)) ? 1 : 0);
+
+}
+// }}}

src/calibre/utils/icu.py

@@ -0,0 +1,53 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+
+__license__   = 'GPL v3'
+__copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+from functools import partial
+
+from calibre.constants import plugins
+
+_icu = _collator = None
+
+_none = u''
+_none2 = b''
+
+def load_icu():
+    global _icu
+    if _icu is None:
+        _icu = plugins['icu'][0]
+        if _icu is None:
+            print plugins['icu'][1]
+        else:
+            if not _icu.ok:
+                print 'icu not ok'
+                _icu = None
+    return _icu
+
+def load_collator():
+    global _collator
+    from calibre.utils.localization import get_lang
+    if _collator is None:
+        icu = load_icu()
+        if icu is not None:
+            _collator = icu.Collator(get_lang())
+    return _collator
+
+
+def py_sort_key(obj):
+    if not obj:
+        return _none
+    return obj.lower()
+
+def icu_sort_key(collator, obj):
+    if not obj:
+        return _none2
+    return collator.sort_key(obj.lower())
+
+load_icu()
+load_collator()
+sort_key = py_sort_key if _icu is None or _collator is None else \
+        partial(icu_sort_key, _collator)
+