Use msgpack instead of pickle for ISO 639 and 3166 databases

setup/translations.py

@@ -713,7 +713,7 @@ class ISO639(Command):  # {{{
 
     description = 'Compile language code maps for performance'
     DEST = os.path.join(os.path.dirname(POT.SRC), 'resources', 'localization',
-            'iso639.pickle')
+            'iso639.calibre_msgpack')
 
     def run(self, opts):
         src = self.j(self.d(self.SRC), 'setup', 'iso_639_3.xml')
@@ -763,7 +763,9 @@ class ISO639(Command):  # {{{
         x = {'by_2':by_2, 'by_3b':by_3b, 'by_3t':by_3t, 'codes2':codes2,
                 'codes3b':codes3b, 'codes3t':codes3t, '2to3':m2to3,
                 '3to2':m3to2, '3bto3t':m3bto3t, 'name_map':nm}
-        cPickle.dump(x, open(dest, 'wb'), -1)
+        from calibre.utils.serialize import msgpack_dumps
+        with open(dest, 'wb') as f:
+            f.write(msgpack_dumps(x))
 
     def clean(self):
         if os.path.exists(self.DEST):
@@ -776,7 +778,7 @@ class ISO3166(ISO639):  # {{{
 
     description = 'Compile country code maps for performance'
     DEST = os.path.join(os.path.dirname(POT.SRC), 'resources', 'localization',
-            'iso3166.pickle')
+            'iso3166.calibre_msgpack')
 
     def run(self, opts):
         src = self.j(self.d(self.SRC), 'setup', 'iso3166.xml')
@@ -803,5 +805,7 @@ class ISO3166(ISO639):  # {{{
             if three:
                 three_map[three] = two
         x = {'names':name_map, 'codes':frozenset(codes), 'three_map':three_map}
-        cPickle.dump(x, open(dest, 'wb'), -1)
+        from calibre.utils.serialize import msgpack_dumps
+        with open(dest, 'wb') as f:
+            f.write(msgpack_dumps(x))
 # }}}

src/calibre/gui2/tweak_book/spell.py

@@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import,
 __license__ = 'GPL v3'
 __copyright__ = '2014, Kovid Goyal <kovid at kovidgoyal.net>'
 
-import cPickle, os, sys
+import os, sys
 from collections import defaultdict, OrderedDict
 from itertools import chain
 from threading import Thread
@@ -48,7 +48,8 @@ _country_map = None
 def country_map():
     global _country_map
     if _country_map is None:
-        _country_map = cPickle.loads(P('localization/iso3166.pickle', data=True, allow_user_override=False))
+        from calibre.utils.serialize import msgpack_loads
+        _country_map = msgpack_loads(P('localization/iso3166.calibre_msgpack', data=True, allow_user_override=False))
     return _country_map
 
 

src/calibre/spell/__init__.py

@@ -6,7 +6,6 @@ from __future__ import (unicode_literals, division, absolute_import,
 __license__ = 'GPL v3'
 __copyright__ = '2014, Kovid Goyal <kovid at kovidgoyal.net>'
 
-import cPickle
 from collections import namedtuple
 
 from calibre.utils.localization import canonicalize_lang
@@ -19,7 +18,8 @@ ccodes, ccodemap, country_names = None, None, None
 def get_codes():
     global ccodes, ccodemap, country_names
     if ccodes is None:
-        data = cPickle.loads(P('localization/iso3166.pickle', allow_user_override=False, data=True))
+        from calibre.utils.serialize import msgpack_loads
+        data = msgpack_loads(P('localization/iso3166.calibre_msgpack', allow_user_override=False, data=True))
         ccodes, ccodemap, country_names = data['codes'], data['three_map'], data['names']
     return ccodes, ccodemap
 
@@ -39,5 +39,3 @@ def parse_lang_code(raw):
         else:
             cc = ccodemap.get(q, None)
     return DictionaryLocale(lc, cc)
-
-

src/calibre/utils/localization.py

@@ -347,9 +347,9 @@ for k in _extra_lang_codes:
 def _load_iso639():
     global _iso639
     if _iso639 is None:
-        ip = P('localization/iso639.pickle', allow_user_override=False)
-        with open(ip, 'rb') as f:
-            _iso639 = cPickle.load(f)
+        ip = P('localization/iso639.calibre_msgpack', allow_user_override=False, data=True)
+        from calibre.utils.serialize import msgpack_loads
+        _iso639 = msgpack_loads(ip)
     return _iso639