Change the filesystem encoding used by python to utf-8 if it is ascii

src/calibre/constants.py

@@ -66,10 +66,8 @@ else:
             filesystem_encoding = 'utf-8'
             # On linux, unicode arguments to os file functions are coerced to an ascii
             # bytestring if sys.getfilesystemencoding() == 'ascii', which is
-            # just plain dumb. So issue a warning.
+            # just plain dumb. This is fixed by the icu.py module which, when
-            print ('WARNING: You do not have the LANG environment variable set correctly. '
+            # imported changes ascii to utf-8
-                    'This will cause problems with non-ascii filenames. '
-                    'Set it to something like en_US.UTF-8.\n')
     except:
         filesystem_encoding = 'utf-8'
 

src/calibre/utils/icu.c

@@ -661,6 +661,17 @@ icu_set_default_encoding(PyObject *self, PyObject *args) {
 }
 // }}}
 
+// set_default_encoding {{{
+static PyObject *
+icu_set_filesystem_encoding(PyObject *self, PyObject *args) {
+    char *encoding;
+    if (!PyArg_ParseTuple(args, "s:setfilesystemencoding", &encoding))
+        return NULL;
+    Py_FileSystemDefaultEncoding = strdup(encoding);
+    Py_RETURN_NONE;
+
+}
+// }}}
 // set_default_encoding {{{
 static PyObject *
 icu_get_available_transliterators(PyObject *self, PyObject *args) {
@@ -707,6 +718,10 @@ static PyMethodDef icu_methods[] = {
         "set_default_encoding(encoding) -> Set the default encoding for the python unicode implementation."
     },
 
+    {"set_filesystem_encoding", icu_set_filesystem_encoding, METH_VARARGS,
+        "set_filesystem_encoding(encoding) -> Set the filesystem encoding for python."
+    },
+
     {"get_available_transliterators", icu_get_available_transliterators, METH_VARARGS,
         "get_available_transliterators() -> Return list of available transliterators. This list is rather limited on OS X."
     },

src/calibre/utils/icu.py

@@ -163,11 +163,22 @@ load_collator()
 _icu_not_ok = _icu is None or _collator is None
 
 try:
-    if sys.getdefaultencoding().lower() == 'ascii':
+    senc = sys.getdefaultencoding()
+    if not senc or senc.lower() == 'ascii':
         _icu.set_default_encoding('utf-8')
+    del senc
 except:
     pass
 
+try:
+    fenc = sys.getfilesystemencoding()
+    if not fenc or fenc.lower() == 'ascii':
+        _icu.set_filesystem_encoding('utf-8')
+    del fenc
+except:
+    pass
+
+
 # }}}
 
 ################# The string functions ########################################
@@ -247,7 +258,7 @@ def collation_order(a):
 
 ################################################################################
 
-def test(): # {{{
+def test():  # {{{
     from calibre import prints
     # Data {{{
     german = '''