Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-09 03:04:10 -04:00
Code Issues Packages Projects Releases Wiki Activity

Use msgpack instead of pickle for ISO 639 and 3166 databases

Browse Source
This commit is contained in:
Kovid Goyal 2018-09-10 17:14:03 +05:30
parent cedf06b55d
commit 1d54c2f874
No known key found for this signature in database
GPG Key ID: 06BC317B515ACE7C
4 changed files with 16 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
setup/translations.py
View File

@ -713,7 +713,7 @@ class ISO639(Command): # {{{
description = 'Compile language code maps for performance' description = 'Compile language code maps for performance'
DEST = os.path.join(os.path.dirname(POT.SRC), 'resources', 'localization', DEST = os.path.join(os.path.dirname(POT.SRC), 'resources', 'localization',
'iso639.pickle') 'iso639.calibre_msgpack')
def run(self, opts): def run(self, opts):
src = self.j(self.d(self.SRC), 'setup', 'iso_639_3.xml') src = self.j(self.d(self.SRC), 'setup', 'iso_639_3.xml')
@ -763,7 +763,9 @@ class ISO639(Command): # {{{
x = {'by_2':by_2, 'by_3b':by_3b, 'by_3t':by_3t, 'codes2':codes2, x = {'by_2':by_2, 'by_3b':by_3b, 'by_3t':by_3t, 'codes2':codes2,
'codes3b':codes3b, 'codes3t':codes3t, '2to3':m2to3, 'codes3b':codes3b, 'codes3t':codes3t, '2to3':m2to3,
'3to2':m3to2, '3bto3t':m3bto3t, 'name_map':nm} '3to2':m3to2, '3bto3t':m3bto3t, 'name_map':nm}
cPickle.dump(x, open(dest, 'wb'), -1) from calibre.utils.serialize import msgpack_dumps
with open(dest, 'wb') as f:
f.write(msgpack_dumps(x))
def clean(self): def clean(self):
if os.path.exists(self.DEST): if os.path.exists(self.DEST):
@ -776,7 +778,7 @@ class ISO3166(ISO639): # {{{
description = 'Compile country code maps for performance' description = 'Compile country code maps for performance'
DEST = os.path.join(os.path.dirname(POT.SRC), 'resources', 'localization', DEST = os.path.join(os.path.dirname(POT.SRC), 'resources', 'localization',
'iso3166.pickle') 'iso3166.calibre_msgpack')
def run(self, opts): def run(self, opts):
src = self.j(self.d(self.SRC), 'setup', 'iso3166.xml') src = self.j(self.d(self.SRC), 'setup', 'iso3166.xml')
@ -803,5 +805,7 @@ class ISO3166(ISO639): # {{{
if three: if three:
three_map[three] = two three_map[three] = two
x = {'names':name_map, 'codes':frozenset(codes), 'three_map':three_map} x = {'names':name_map, 'codes':frozenset(codes), 'three_map':three_map}
cPickle.dump(x, open(dest, 'wb'), -1) from calibre.utils.serialize import msgpack_dumps
with open(dest, 'wb') as f:
f.write(msgpack_dumps(x))
# }}} # }}}

5
src/calibre/gui2/tweak_book/spell.py
View File

@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import,
__license__ = 'GPL v3' __license__ = 'GPL v3'
__copyright__ = '2014, Kovid Goyal <kovid at kovidgoyal.net>' __copyright__ = '2014, Kovid Goyal <kovid at kovidgoyal.net>'
import cPickle, os, sys import os, sys
from collections import defaultdict, OrderedDict from collections import defaultdict, OrderedDict
from itertools import chain from itertools import chain
from threading import Thread from threading import Thread
@ -48,7 +48,8 @@ _country_map = None
def country_map(): def country_map():
global _country_map global _country_map
if _country_map is None: if _country_map is None:
_country_map = cPickle.loads(P('localization/iso3166.pickle', data=True, allow_user_override=False)) from calibre.utils.serialize import msgpack_loads
_country_map = msgpack_loads(P('localization/iso3166.calibre_msgpack', data=True, allow_user_override=False))
return _country_map return _country_map

6
src/calibre/spell/__init__.py
View File

@ -6,7 +6,6 @@ from __future__ import (unicode_literals, division, absolute_import,
__license__ = 'GPL v3' __license__ = 'GPL v3'
__copyright__ = '2014, Kovid Goyal <kovid at kovidgoyal.net>' __copyright__ = '2014, Kovid Goyal <kovid at kovidgoyal.net>'
import cPickle
from collections import namedtuple from collections import namedtuple
from calibre.utils.localization import canonicalize_lang from calibre.utils.localization import canonicalize_lang
@ -19,7 +18,8 @@ ccodes, ccodemap, country_names = None, None, None
def get_codes(): def get_codes():
global ccodes, ccodemap, country_names global ccodes, ccodemap, country_names
if ccodes is None: if ccodes is None:
data = cPickle.loads(P('localization/iso3166.pickle', allow_user_override=False, data=True)) from calibre.utils.serialize import msgpack_loads
data = msgpack_loads(P('localization/iso3166.calibre_msgpack', allow_user_override=False, data=True))
ccodes, ccodemap, country_names = data['codes'], data['three_map'], data['names'] ccodes, ccodemap, country_names = data['codes'], data['three_map'], data['names']
return ccodes, ccodemap return ccodes, ccodemap
@ -39,5 +39,3 @@ def parse_lang_code(raw):
else: else:
cc = ccodemap.get(q, None) cc = ccodemap.get(q, None)
return DictionaryLocale(lc, cc) return DictionaryLocale(lc, cc)

6
src/calibre/utils/localization.py
View File

@ -347,9 +347,9 @@ for k in _extra_lang_codes:
def _load_iso639(): def _load_iso639():
global _iso639 global _iso639
if _iso639 is None: if _iso639 is None:
ip = P('localization/iso639.pickle', allow_user_override=False) ip = P('localization/iso639.calibre_msgpack', allow_user_override=False, data=True)
with open(ip, 'rb') as f: from calibre.utils.serialize import msgpack_loads
_iso639 = cPickle.load(f) _iso639 = msgpack_loads(ip)
return _iso639 return _iso639