From b9767b2b92a73c87486e3e1378d73d99976b4be8 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 10 Sep 2018 18:35:19 +0530 Subject: [PATCH] Move more databases to msgpack instead of pickle --- .gitignore | 4 ++-- setup/resources.py | 16 +++++++++------- src/calibre/linux.py | 5 +++-- src/calibre/utils/complete.py | 7 ++++--- 4 files changed, 18 insertions(+), 14 deletions(-) diff --git a/.gitignore b/.gitignore index 8879d21afe..a7c280204e 100644 --- a/.gitignore +++ b/.gitignore @@ -14,8 +14,8 @@ build dist docs resources/localization -resources/scripts.pickle -resources/ebook-convert-complete.pickle +resources/scripts.calibre_msgpack +resources/ebook-convert-complete.calibre_msgpack resources/builtin_recipes.xml resources/builtin_recipes.zip resources/template-functions.json diff --git a/setup/resources.py b/setup/resources.py index 4600e67c1e..55febce79a 100644 --- a/setup/resources.py +++ b/setup/resources.py @@ -7,7 +7,7 @@ __license__ = 'GPL v3' __copyright__ = '2009, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import os, re, shutil, zipfile, glob, time, sys, hashlib, json, errno, cPickle +import os, re, shutil, zipfile, glob, time, sys, hashlib, json, errno from zlib import compress from itertools import chain is_ci = os.environ.get('CI', '').lower() == 'true' @@ -297,11 +297,12 @@ class Resources(Command): # {{{ continue scripts[name] = x - dest = self.j(self.RESOURCES, 'scripts.pickle') + dest = self.j(self.RESOURCES, 'scripts.calibre_msgpack') if self.newer(dest, self.j(self.SRC, 'calibre', 'linux.py')): - self.info('\tCreating scripts.pickle') - f = open(dest, 'wb') - cPickle.dump(scripts, f, -1) + self.info('\tCreating ' + os.path.basename(dest)) + from calibre.utils.serialize import msgpack_dumps + with open(dest, 'wb') as f: + f.write(msgpack_dumps(scripts)) from calibre.web.feeds.recipes.collection import \ serialize_builtin_recipes, iterate_over_builtin_recipe_files @@ -326,7 +327,7 @@ class Resources(Command): # {{{ with open(n, 'rb') as f: zf.writestr(os.path.basename(n), f.read()) - dest = self.j(self.RESOURCES, 'ebook-convert-complete.pickle') + dest = self.j(self.RESOURCES, 'ebook-convert-complete.calibre_msgpack') files = [] for x in os.walk(self.j(self.SRC, 'calibre')): for f in x[-1]: @@ -356,7 +357,8 @@ class Resources(Command): # {{{ complete[(inf, ouf)] = [x+' 'for x in get_opts_from_parser(p)] - cPickle.dump(complete, open(dest, 'wb'), -1) + with open(dest, 'wb') as f: + f.write(msgpack_dumps(complete)) self.info('\tCreating template-functions.json') dest = self.j(self.RESOURCES, 'template-functions.json') diff --git a/src/calibre/linux.py b/src/calibre/linux.py index deed629ddb..8edeb4cb2d 100644 --- a/src/calibre/linux.py +++ b/src/calibre/linux.py @@ -4,7 +4,7 @@ __copyright__ = '2008, Kovid Goyal ' ''' Post installation script for linux ''' -import sys, os, cPickle, textwrap, stat, errno +import sys, os, textwrap, stat, errno from subprocess import check_call, check_output from functools import partial @@ -672,7 +672,8 @@ class PostInstall: self.opts.staging_etc = '/etc' if self.opts.staging_root == '/usr' else \ os.path.join(self.opts.staging_root, 'etc') - scripts = cPickle.loads(P('scripts.pickle', data=True)) + from calibre.utils.serialize import msgpack_loads + scripts = msgpack_loads(P('scripts.calibre_msgpack', data=True)) self.manifest = manifest or [] if getattr(sys, 'frozen_path', False): if os.access(self.opts.staging_bindir, os.W_OK): diff --git a/src/calibre/utils/complete.py b/src/calibre/utils/complete.py index 1903f0b12d..e2ae94e808 100644 --- a/src/calibre/utils/complete.py +++ b/src/calibre/utils/complete.py @@ -12,7 +12,7 @@ BASH completion for calibre commands that are too complex for simple completion. ''' -import sys, os, shlex, glob, re, cPickle +import sys, os, shlex, glob, re def prints(*args, **kwargs): @@ -113,8 +113,9 @@ class EbookConvert(object): self.words = words self.prefix = prefix self.previous = words[-2 if prefix else -1] - self.cache = cPickle.load(open(os.path.join(sys.resources_location, - 'ebook-convert-complete.pickle'), 'rb')) + from calibre.utils.serialize import msgpack_loads + self.cache = msgpack_loads(open(os.path.join(sys.resources_location, + 'ebook-convert-complete.calibre_msgpack'), 'rb').read()) self.complete(wc) def complete(self, wc):