From 8b3ea8fb83baf7cdc704ffbb84f45f9e2b2c660c Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Mon, 10 Sep 2018 18:26:46 +0530
Subject: [PATCH] Replace more use of pickle for databases

---
 setup/resources.py                            | 21 ++++++++++++-------
 .../ebooks/unihandecode/pykakasi/jisyo.py     | 18 ++++++++--------
 2 files changed, 22 insertions(+), 17 deletions(-)

diff --git a/setup/resources.py b/setup/resources.py
index 783e6e7f75..4600e67c1e 100644
--- a/setup/resources.py
+++ b/setup/resources.py
@@ -7,7 +7,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import os, cPickle, re, shutil, marshal, zipfile, glob, time, sys, hashlib, json, errno
+import os, re, shutil, zipfile, glob, time, sys, hashlib, json, errno, cPickle
 from zlib import compress
 from itertools import chain
 is_ci = os.environ.get('CI', '').lower() == 'true'
@@ -137,7 +137,7 @@ class Kakasi(Command):  # {{{
         self.records = {}
         src = self.j(self.KAKASI_PATH, 'kakasidict.utf8')
         dest = self.j(self.RESOURCES, 'localization',
-                'pykakasi','kanwadict2.pickle')
+                'pykakasi','kanwadict2.calibre_msgpack')
         base = os.path.dirname(dest)
         if not os.path.exists(base):
             os.makedirs(base)
@@ -151,7 +151,7 @@ class Kakasi(Command):  # {{{
 
         src = self.j(self.KAKASI_PATH, 'itaijidict.utf8')
         dest = self.j(self.RESOURCES, 'localization',
-                'pykakasi','itaijidict2.pickle')
+                'pykakasi','itaijidict2.calibre_msgpack')
 
         if self.newer(dest, src):
             self.info('\tGenerating Itaijidict')
@@ -159,7 +159,7 @@ class Kakasi(Command):  # {{{
 
         src = self.j(self.KAKASI_PATH, 'kanadict.utf8')
         dest = self.j(self.RESOURCES, 'localization',
-                'pykakasi','kanadict2.pickle')
+                'pykakasi','kanadict2.calibre_msgpack')
 
         if self.newer(dest, src):
             self.info('\tGenerating kanadict')
@@ -175,7 +175,9 @@ class Kakasi(Command):  # {{{
                 continue
             pair = re.sub(r'\\u([0-9a-fA-F]{4})', lambda x:unichr(int(x.group(1),16)), line)
             dic[pair[0]] = pair[1]
-        cPickle.dump(dic, open(dst, 'wb'), protocol=-1)  # pickle
+        from calibre.utils.serialize import msgpack_dumps
+        with open(dst, 'wb') as f:
+            f.write(msgpack_dumps(dic))
 
     def mkkanadict(self, src, dst):
         dic = {}
@@ -187,7 +189,9 @@ class Kakasi(Command):  # {{{
                 continue
             (alpha, kana) = line.split(' ')
             dic[kana] = alpha
-        cPickle.dump(dic, open(dst, 'wb'), protocol=-1)  # pickle
+        from calibre.utils.serialize import msgpack_dumps
+        with open(dst, 'wb') as f:
+            f.write(msgpack_dumps(dic))
 
     def parsekdict(self, line):
         line = line.decode("utf-8").strip()
@@ -215,11 +219,12 @@ class Kakasi(Command):  # {{{
             self.records[key][kanji]=[(yomi, tail)]
 
     def kanwaout(self, out):
+        from calibre.utils.serialize import msgpack_dumps
         with open(out, 'wb') as f:
             dic = {}
             for k, v in self.records.iteritems():
-                dic[k] = compress(marshal.dumps(v))
-            cPickle.dump(dic, f, -1)
+                dic[k] = compress(msgpack_dumps(v))
+            f.write(msgpack_dumps(dic))
 
     def clean(self):
         kakasi = self.j(self.RESOURCES, 'localization', 'pykakasi')
diff --git a/src/calibre/ebooks/unihandecode/pykakasi/jisyo.py b/src/calibre/ebooks/unihandecode/pykakasi/jisyo.py
index 1f7cea332b..1496eea111 100644
--- a/src/calibre/ebooks/unihandecode/pykakasi/jisyo.py
+++ b/src/calibre/ebooks/unihandecode/pykakasi/jisyo.py
@@ -2,7 +2,6 @@
 #  jisyo.py
 #
 # Copyright 2011 Hiroshi Miura <miurahr@linux.com>
-import cPickle, marshal
 from zlib import decompress
 
 
@@ -21,15 +20,16 @@ class jisyo (object):
         return self
 
     def __init__(self):
+        from calibre.utils.serialize import msgpack_loads
         if self.kanwadict is None:
-            self.kanwadict = cPickle.loads(
-                P('localization/pykakasi/kanwadict2.pickle', data=True))
+            self.kanwadict = msgpack_loads(
+                P('localization/pykakasi/kanwadict2.calibre_msgpack', data=True))
         if self.itaijidict is None:
-            self.itaijidict = cPickle.loads(
-                P('localization/pykakasi/itaijidict2.pickle', data=True))
+            self.itaijidict = msgpack_loads(
+                P('localization/pykakasi/itaijidict2.calibre_msgpack', data=True))
         if self.kanadict is None:
-            self.kanadict = cPickle.loads(
-                P('localization/pykakasi/kanadict2.pickle', data=True))
+            self.kanadict = msgpack_loads(
+                P('localization/pykakasi/kanadict2.calibre_msgpack', data=True))
 
     def load_jisyo(self, char):
         try:  # python2
@@ -40,9 +40,9 @@ class jisyo (object):
         try:  # already exist?
             table = self.jisyo_table[key]
         except:
+            from calibre.utils.serialize import msgpack_loads
             try:
-                table = self.jisyo_table[key]  = marshal.loads(decompress(self.kanwadict[key]))
+                table = self.jisyo_table[key]  = msgpack_loads(decompress(self.kanwadict[key]))
             except:
                 return None
         return table
-