Convert all unicode characters to ASCII when creating file paths on the device

installer/linux/freeze.py

@@ -49,6 +49,7 @@ def freeze():
                        '/usr/lib/libMagickCore.so',
                        '/usr/lib/libgcrypt.so.11',
                        '/usr/lib/libgpg-error.so.0',
+                       '/usr/lib/libphonon.so.4',
                        ]
 
     binary_includes += [os.path.join(QTDIR, 'lib%s.so.4'%x) for x in QTDLLS]

src/calibre/devices/usbms/device.py

@@ -15,7 +15,7 @@ from calibre.devices.interface import DevicePlugin
 from calibre.devices.errors import DeviceError
 from calibre.devices.usbms.deviceconfig import DeviceConfig
 from calibre import iswindows, islinux, isosx, __appname__
-from calibre import sanitize_file_name as sanitize
+from calibre.utils.filenames import ascii_filename as sanitize
 
 class Device(DeviceConfig, DevicePlugin):
     '''

src/calibre/utils/filenames.py

@@ -1,97 +1,11 @@
-# -*- coding: utf-8 -*-
 '''
-Make strings safe for use as ASCII filenames, while trying to preserve as much 
+Make strings safe for use as ASCII filenames, while trying to preserve as much
 meaning as possible.
 '''
 
-import re, string
+from calibre.ebooks.unidecode.unidecoder import Unidecoder
-
+from calibre import sanitize_file_name
-MAP = {
+udc = Unidecoder()
-        u"‘" : "'",
-        u"’" : "'",
-        u"«" : '"',
-        u"»" : '"',
-        u"…" : "...",
-        u"№" : "#",
-        u"Щ" : "Shh",
-        u"Ё" : "Jo",
-        u"Ж" : "Zh",
-        u"Ц" : "C",
-        u"Ч" : "Ch",
-        u"Ш" : "Sh",
-        u"Ы" : "Y",
-        u"Ю" : "Ju",
-        u"Я" : "Ja",
-        u"Б" : "B",
-        u"Г" : "G",
-        u"Д" : "D",
-        u"И" : "I",
-        u"Й" : "J",
-        u"К" : "K",
-        u"Л" : "L",
-        u"П" : "P",
-        u"Ф" : "F",
-        u"Э" : "E",
-        u"Ъ" : "`",
-        u"Ь" : "'",
-        u"щ" : "shh",
-        u"ё" : "jo",
-        u"ж" : "zh",
-        u"ц" : "c",
-        u"ч" : "ch",
-        u"ш" : "sh",
-        u"ы" : "y",
-        u"ю" : "ju",
-        u"я" : "ja",
-        u"б" : "b",
-        u"в" : "v",
-        u"г" : "g",
-        u"д" : "d",
-        u"з" : "z",
-        u"и" : "i",
-        u"й" : "j",
-        u"к" : "k",
-        u"л" : "l",
-        u"м" : "m",
-        u"н" : "n",
-        u"о" : "o",
-        u"п" : "p",
-        u"т" : "t",
-        u"ф" : "f",
-        u"э" : "e",
-        u"ъ" : "`",
-        u"ь" : "'",
-        u"А" : "A",
-        u"В" : "V",
-        u"Е" : "Je",
-        u"З" : "Z",
-        u"М" : "M",
-        u"Н" : "N",
-        u"О" : "O",
-        u"Р" : "R",
-        u"С" : "S",
-        u"Т" : "T",
-        u"У" : "U",
-        u"Х" : "Kh",
-        u"Є" : "Je",
-        u"Ї" : "Ji",
-        u"а" : "a",
-        u"е" : "je",
-        u"р" : "r",
-        u"с" : "s",
-        u"у" : "u",
-        u"х" : "kh",
-        u"є" : "je",
-}  #: Translation table
-
-for c in string.whitespace:
-    MAP[c] = ' '
-PAT = re.compile('['+u''.join(MAP.keys())+']')
 
 def ascii_filename(orig):
-    orig =  PAT.sub(lambda m:MAP[m.group()], orig)
+    return sanitize_file_name(udc.decode(orig).replace('?', '_'))
-    buf = []
-    for i in range(len(orig)):
-        val = ord(orig[i])
-        buf.append('_' if val < 33 or val > 126 else orig[i])
-    return (''.join(buf)).encode('ascii')