Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-06-23 15:30:45 -04:00
Code Issues Packages Projects Releases Wiki Activity

Replace more use of pickle for databases

Browse Source
This commit is contained in:
Kovid Goyal 2018-09-10 18:26:46 +05:30
parent 0911053a8a
commit 8b3ea8fb83
No known key found for this signature in database
GPG Key ID: 06BC317B515ACE7C
2 changed files with 22 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
setup/resources.py
View File

@ -7,7 +7,7 @@ __license__ = 'GPL v3'
__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>' __copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
__docformat__ = 'restructuredtext en' __docformat__ = 'restructuredtext en'
import os, cPickle, re, shutil, marshal, zipfile, glob, time, sys, hashlib, json, errno import os, re, shutil, zipfile, glob, time, sys, hashlib, json, errno, cPickle
from zlib import compress from zlib import compress
from itertools import chain from itertools import chain
is_ci = os.environ.get('CI', '').lower() == 'true' is_ci = os.environ.get('CI', '').lower() == 'true'
@ -137,7 +137,7 @@ class Kakasi(Command): # {{{
self.records = {} self.records = {}
src = self.j(self.KAKASI_PATH, 'kakasidict.utf8') src = self.j(self.KAKASI_PATH, 'kakasidict.utf8')
dest = self.j(self.RESOURCES, 'localization', dest = self.j(self.RESOURCES, 'localization',
'pykakasi','kanwadict2.pickle') 'pykakasi','kanwadict2.calibre_msgpack')
base = os.path.dirname(dest) base = os.path.dirname(dest)
if not os.path.exists(base): if not os.path.exists(base):
os.makedirs(base) os.makedirs(base)
@ -151,7 +151,7 @@ class Kakasi(Command): # {{{
src = self.j(self.KAKASI_PATH, 'itaijidict.utf8') src = self.j(self.KAKASI_PATH, 'itaijidict.utf8')
dest = self.j(self.RESOURCES, 'localization', dest = self.j(self.RESOURCES, 'localization',
'pykakasi','itaijidict2.pickle') 'pykakasi','itaijidict2.calibre_msgpack')
if self.newer(dest, src): if self.newer(dest, src):
self.info('\tGenerating Itaijidict') self.info('\tGenerating Itaijidict')
@ -159,7 +159,7 @@ class Kakasi(Command): # {{{
src = self.j(self.KAKASI_PATH, 'kanadict.utf8') src = self.j(self.KAKASI_PATH, 'kanadict.utf8')
dest = self.j(self.RESOURCES, 'localization', dest = self.j(self.RESOURCES, 'localization',
'pykakasi','kanadict2.pickle') 'pykakasi','kanadict2.calibre_msgpack')
if self.newer(dest, src): if self.newer(dest, src):
self.info('\tGenerating kanadict') self.info('\tGenerating kanadict')
@ -175,7 +175,9 @@ class Kakasi(Command): # {{{
continue continue
pair = re.sub(r'\\u([0-9a-fA-F]{4})', lambda x:unichr(int(x.group(1),16)), line) pair = re.sub(r'\\u([0-9a-fA-F]{4})', lambda x:unichr(int(x.group(1),16)), line)
dic[pair[0]] = pair[1] dic[pair[0]] = pair[1]
cPickle.dump(dic, open(dst, 'wb'), protocol=-1) # pickle from calibre.utils.serialize import msgpack_dumps
with open(dst, 'wb') as f:
f.write(msgpack_dumps(dic))
def mkkanadict(self, src, dst): def mkkanadict(self, src, dst):
dic = {} dic = {}
@ -187,7 +189,9 @@ class Kakasi(Command): # {{{
continue continue
(alpha, kana) = line.split(' ') (alpha, kana) = line.split(' ')
dic[kana] = alpha dic[kana] = alpha
cPickle.dump(dic, open(dst, 'wb'), protocol=-1) # pickle from calibre.utils.serialize import msgpack_dumps
with open(dst, 'wb') as f:
f.write(msgpack_dumps(dic))
def parsekdict(self, line): def parsekdict(self, line):
line = line.decode("utf-8").strip() line = line.decode("utf-8").strip()
@ -215,11 +219,12 @@ class Kakasi(Command): # {{{
self.records[key][kanji]=[(yomi, tail)] self.records[key][kanji]=[(yomi, tail)]
def kanwaout(self, out): def kanwaout(self, out):
from calibre.utils.serialize import msgpack_dumps
with open(out, 'wb') as f: with open(out, 'wb') as f:
dic = {} dic = {}
for k, v in self.records.iteritems(): for k, v in self.records.iteritems():
dic[k] = compress(marshal.dumps(v)) dic[k] = compress(msgpack_dumps(v))
cPickle.dump(dic, f, -1) f.write(msgpack_dumps(dic))
def clean(self): def clean(self):
kakasi = self.j(self.RESOURCES, 'localization', 'pykakasi') kakasi = self.j(self.RESOURCES, 'localization', 'pykakasi')

18
src/calibre/ebooks/unihandecode/pykakasi/jisyo.py
View File

@ -2,7 +2,6 @@
# jisyo.py # jisyo.py
# #
# Copyright 2011 Hiroshi Miura <miurahr@linux.com> # Copyright 2011 Hiroshi Miura <miurahr@linux.com>
import cPickle, marshal
from zlib import decompress from zlib import decompress
@ -21,15 +20,16 @@ class jisyo (object):
return self return self
def __init__(self): def __init__(self):
from calibre.utils.serialize import msgpack_loads
if self.kanwadict is None: if self.kanwadict is None:
self.kanwadict = cPickle.loads( self.kanwadict = msgpack_loads(
P('localization/pykakasi/kanwadict2.pickle', data=True)) P('localization/pykakasi/kanwadict2.calibre_msgpack', data=True))
if self.itaijidict is None: if self.itaijidict is None:
self.itaijidict = cPickle.loads( self.itaijidict = msgpack_loads(
P('localization/pykakasi/itaijidict2.pickle', data=True)) P('localization/pykakasi/itaijidict2.calibre_msgpack', data=True))
if self.kanadict is None: if self.kanadict is None:
self.kanadict = cPickle.loads( self.kanadict = msgpack_loads(
P('localization/pykakasi/kanadict2.pickle', data=True)) P('localization/pykakasi/kanadict2.calibre_msgpack', data=True))
def load_jisyo(self, char): def load_jisyo(self, char):
try: # python2 try: # python2
@ -40,9 +40,9 @@ class jisyo (object):
try: # already exist? try: # already exist?
table = self.jisyo_table[key] table = self.jisyo_table[key]
except: except:
from calibre.utils.serialize import msgpack_loads
try: try:
table = self.jisyo_table[key] = marshal.loads(decompress(self.kanwadict[key])) table = self.jisyo_table[key] = msgpack_loads(decompress(self.kanwadict[key]))
except: except:
return None return None
return table return table