diff --git a/setup/__init__.py b/setup/__init__.py index a6e86df9db..2ae25238b1 100644 --- a/setup/__init__.py +++ b/setup/__init__.py @@ -46,6 +46,15 @@ def newer(targets, sources): return newest_source > oldest_target +def dump_json(obj, path, indent=4): + import json + with open(path, 'wb') as f: + data = json.dumps(obj, indent=indent) + if not isinstance(data, bytes): + data = data.encode('utf-8') + f.write(data) + + def download_securely(url): # We use curl here as on some OSes (OS X) when bootstrapping calibre, # python will be unable to validate certificates until after cacerts is diff --git a/setup/check.py b/setup/check.py index f0d3c9fd17..fe7cd845bb 100644 --- a/setup/check.py +++ b/setup/check.py @@ -7,7 +7,7 @@ __copyright__ = '2009, Kovid Goyal ' __docformat__ = 'restructuredtext en' import sys, os, json, subprocess, errno, hashlib -from setup import Command, build_cache_dir, edit_file +from setup import Command, build_cache_dir, edit_file, dump_json class Message: @@ -72,8 +72,7 @@ class Check(Command): return self.j(build_cache_dir(), self.CACHE) def save_cache(self, cache): - with open(self.cache_file, 'wb') as f: - json.dump(cache, f) + dump_json(cache, self.cache_file) def file_has_errors(self, f): ext = os.path.splitext(f)[1] diff --git a/setup/plugins_mirror.py b/setup/plugins_mirror.py index 087770e419..eeb14039a4 100644 --- a/setup/plugins_mirror.py +++ b/setup/plugins_mirror.py @@ -568,8 +568,11 @@ def update_stats(): if m is not None: plugin = m.group(1).decode('utf-8') stats[plugin] = stats.get(plugin, 0) + 1 + data = json.dumps(stats, indent=2) + if not isinstance(data, bytes): + data = data.encode('utf-8') with open('stats.json', 'wb') as f: - json.dump(stats, f, indent=2) + f.write(data) return stats diff --git a/setup/resources.py b/setup/resources.py index edaf56d01c..15d2af3945 100644 --- a/setup/resources.py +++ b/setup/resources.py @@ -11,7 +11,7 @@ from zlib import compress from itertools import chain is_ci = os.environ.get('CI', '').lower() == 'true' -from setup import Command, basenames, __appname__, download_securely +from setup import Command, basenames, __appname__, download_securely, dump_json from polyglot.builtins import codepoint_to_chr, itervalues, iteritems @@ -374,8 +374,7 @@ class Resources(Command): # {{{ continue lines = ''.join(lines) function_dict[obj.name] = lines - import json - json.dump(function_dict, open(dest, 'wb'), indent=4) + dump_json(function_dict, dest) self.info('\tCreating editor-functions.json') dest = self.j(self.RESOURCES, 'editor-functions.json') @@ -391,13 +390,13 @@ class Resources(Command): # {{{ if imports: src = '\n'.join(imports) + '\n\n' + src function_dict[func.name] = src - json.dump(function_dict, open(dest, 'wb'), indent=4) + dump_json(function_dict, dest) self.info('\tCreating user-manual-translation-stats.json') d = {} for lc, stats in iteritems(json.load(open(self.j(self.d(self.SRC), 'manual', 'locale', 'completed.json')))): total = sum(itervalues(stats)) d[lc] = stats['translated'] / float(total) - json.dump(d, open(self.j(self.RESOURCES, 'user-manual-translation-stats.json'), 'wb'), indent=4) + dump_json(d, self.j(self.RESOURCES, 'user-manual-translation-stats.json')) def clean(self): for x in ('scripts', 'ebook-convert-complete'): diff --git a/setup/translations.py b/setup/translations.py index d1e580c66b..202264170a 100644 --- a/setup/translations.py +++ b/setup/translations.py @@ -11,7 +11,7 @@ from collections import defaultdict from locale import normalize as normalize_locale from functools import partial -from setup import Command, __appname__, __version__, require_git_master, build_cache_dir, edit_file +from setup import Command, __appname__, __version__, require_git_master, build_cache_dir, edit_file, dump_json from setup.parallel_build import parallel_check_output from polyglot.builtins import codepoint_to_chr, iteritems, range is_ci = os.environ.get('CI', '').lower() == 'true' @@ -288,8 +288,11 @@ class Translations(POT): # {{{ cname = self.cache_name(src) + '.stats.json' with open(self.j(self.cache_dir, cname), ('rb' if data is None else 'wb')) as f: if data is None: - return json.load(f) - json.dump(data, f) + return json.loads(f.read()) + data = json.dumps(data) + if not isinstance(data, bytes): + data = data.encode('utf-8') + f.write(data) for src, dest in files: base = os.path.dirname(dest) @@ -543,14 +546,12 @@ class Translations(POT): # {{{ self.compile_group(files, handle_stats=handle_stats) for locale, stats in iteritems(all_stats): - with open(self.j(srcbase, locale, 'stats.json'), 'wb') as f: - json.dump(stats, f) + dump_json(stats, self.j(srcbase, locale, 'stats.json')) total = stats['translated'] + stats['untranslated'] # Raise the 30% threshold in the future if total and (stats['translated'] / float(total)) > 0.3: complete[locale] = stats - with open(self.j(destbase, 'completed.json'), 'wb') as f: - json.dump(complete, f, indent=True, sort_keys=True) + dump_json(complete, self.j(destbase, 'completed.json')) def clean(self): if os.path.exists(self.stats): diff --git a/src/calibre/db/cli/cmd_list.py b/src/calibre/db/cli/cmd_list.py index a6635236d4..fdd7b145ed 100644 --- a/src/calibre/db/cli/cmd_list.py +++ b/src/calibre/db/cli/cmd_list.py @@ -161,12 +161,14 @@ def do_list( fields = ['id'] + fields stringify(data, metadata, for_machine) if for_machine: - json.dump( + raw = json.dumps( list(as_machine_data(book_ids, data, metadata)), - sys.stdout, indent=2, sort_keys=True ) + if not isinstance(raw, bytes): + raw = raw.encode('utf-8') + getattr(sys.stdout, 'buffer', sys.stdout).write(raw) return from calibre.utils.terminal import ColoredStream, geometry diff --git a/src/calibre/gui2/icon_theme.py b/src/calibre/gui2/icon_theme.py index e5d1c3a0a0..ad23b71af2 100644 --- a/src/calibre/gui2/icon_theme.py +++ b/src/calibre/gui2/icon_theme.py @@ -260,8 +260,11 @@ class ThemeCreateDialog(Dialog): } def save_metadata(self): + data = json.dumps(self.metadata, indent=2) + if not isinstance(data, bytes): + data = data.encode('utf-8') with open(os.path.join(self.report.path, THEME_METADATA), 'wb') as f: - json.dump(self.metadata, f, indent=2) + f.write(data) def refresh(self): self.save_metadata()