Dont store tweaks in a .py file

Use a .json file instead. As with the previosu migrations of pickle and
.py config files this causes the settings to become disjoint if an
upgrage/downgrade is done.

resources/default_tweaks.py

@@ -1,14 +1,11 @@
 #!/usr/bin/env python2
 # vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
-__license__   = 'GPL v3'
+# License: GPLv3 Copyright: 2010, Kovid Goyal <kovid at kovidgoyal.net>
-__copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
+from __future__ import unicode_literals
-__docformat__ = 'restructuredtext en'
 
-'''
+# Contains various tweaks that affect calibre behavior. Only edit this file if
-Contains various tweaks that affect calibre behavior. Only edit this file if
+# you know what you are doing. If you delete this file, it will be recreated from
-you know what you are doing. If you delete this file, it will be recreated from
+# defaults.
-defaults.
-'''
 
 #: Auto increment series index
 # The algorithm used to assign a book added to an existing series a series number.
@@ -220,7 +217,7 @@ per_language_title_sort_articles = {
         'eng'  : (r'A\s+', r'The\s+', r'An\s+'),
 
         # Esperanto
-        'epo': (r'La\s+', r"L'", 'L\xb4'),
+        'epo': (r'La\s+', r"L'", 'L´'),
 
         # Spanish
         'spa'  : (r'El\s+', r'La\s+', r'Lo\s+', r'Los\s+', r'Las\s+', r'Un\s+',
@@ -230,10 +227,10 @@ per_language_title_sort_articles = {
                   r'Des\s+', r'De\s+La\s+', r'De\s+', r"D'", u'D´', u'L’'),
 
         # Italian
-        'ita': ('Lo\\s+', 'Il\\s+', "L'", 'L\xb4', 'La\\s+', 'Gli\\s+',
+        'ita': ('Lo\\s+', 'Il\\s+', "L'", 'L´', 'La\\s+', 'Gli\\s+',
                 'I\\s+', 'Le\\s+', 'Uno\\s+', 'Un\\s+', 'Una\\s+', "Un'",
-                'Un\xb4', 'Dei\\s+', 'Degli\\s+', 'Delle\\s+', 'Del\\s+',
+                'Un´', 'Dei\\s+', 'Degli\\s+', 'Delle\\s+', 'Del\\s+',
-                'Della\\s+', 'Dello\\s+', "Dell'", 'Dell\xb4'),
+                'Della\\s+', 'Dello\\s+', "Dell'", 'Dell´'),
 
         # Portuguese
         'por'  : (r'A\s+', r'O\s+', r'Os\s+', r'As\s+', r'Um\s+', r'Uns\s+',

src/calibre/gui2/preferences/tweaks.py

@@ -13,13 +13,13 @@ from collections import OrderedDict
 from calibre.gui2.preferences import ConfigWidgetBase, test_widget, AbortCommit
 from calibre.gui2.search_box import SearchBox2
 from calibre.gui2 import error_dialog, info_dialog
-from calibre.utils.config import read_raw_tweaks, write_tweaks
+from calibre.utils.config_base import read_custom_tweaks, write_custom_tweaks, exec_tweaks, default_tweaks_raw
 from calibre.gui2.widgets import PythonHighlighter
 from calibre import isbytestring
 from calibre.utils.icu import lower
 from calibre.utils.search_query_parser import (ParseException,
         SearchQueryParser)
-from polyglot.builtins import iteritems, unicode_type, range
+from polyglot.builtins import iteritems, unicode_type, range, map
 
 from PyQt5.Qt import (
     QAbstractListModel, Qt, QStyledItemDelegate, QStyle, QStyleOptionViewItem,
@@ -30,6 +30,14 @@ from PyQt5.Qt import (
 ROOT = QModelIndex()
 
 
+def normalize(val):
+    if isinstance(val, (list, tuple)):
+        return tuple(map(normalize, val))
+    if isinstance(val, dict):
+        return {k: normalize(v) for k, v in iteritems(val)}
+    return val
+
+
 def format_doc(doc):
     current_indent = default_indent = None
     lines = ['']
@@ -98,7 +106,7 @@ class Tweak(object):  # {{{
                 ans.append('# ' + line)
         for key, val in iteritems(self.default_values):
             val = self.custom_values.get(key, val)
-            ans.append('%s = %r'%(key, val))
+            ans.append(u'%s = %r'%(key, val))
         ans = '\n'.join(ans)
         if isinstance(ans, unicode_type):
             ans = ans.encode('utf-8')
@@ -111,16 +119,21 @@ class Tweak(object):  # {{{
     @property
     def is_customized(self):
         for x, val in iteritems(self.default_values):
-            if self.custom_values.get(x, val) != val:
+            cval = self.custom_values.get(x, val)
+            if normalize(cval) != normalize(val):
                 return True
         return False
 
     @property
     def edit_text(self):
+        from pprint import pformat
         ans = ['# %s'%self.name]
         for x, val in iteritems(self.default_values):
             val = self.custom_values.get(x, val)
-            ans.append('%s = %r'%(x, val))
+            if isinstance(val, (list, tuple, dict, set, frozenset)):
+                ans.append(u'%s = %s' % (x, pformat(val)))
+            else:
+                ans.append(u'%s = %r'%(x, val))
         return '\n\n'.join(ans)
 
     def restore_to_default(self):
@@ -137,9 +150,7 @@ class Tweaks(QAbstractListModel, AdaptSQP):  # {{{
     def __init__(self, parent=None):
         QAbstractListModel.__init__(self, parent)
         SearchQueryParser.__init__(self, ['all'])
-        raw_defaults, raw_custom = read_raw_tweaks()
+        self.parse_tweaks()
-
-        self.parse_tweaks(raw_defaults, raw_custom)
 
     def rowCount(self, *args):
         return len(self.tweaks)
@@ -168,32 +179,32 @@ class Tweaks(QAbstractListModel, AdaptSQP):  # {{{
             return tweak
         return None
 
-    def parse_tweaks(self, defaults, custom):
+    def parse_tweaks(self):
-        l, g = {}, {}
         try:
-            exec(custom, g, l)
+            custom_tweaks = read_custom_tweaks()
         except:
             print('Failed to load custom tweaks file')
             import traceback
             traceback.print_exc()
-        dl, dg = {}, {}
+            custom_tweaks = {}
-        exec(defaults, dg, dl)
+        default_tweaks = exec_tweaks(default_tweaks_raw())
+        defaults = default_tweaks_raw().decode('utf-8')
         lines = defaults.splitlines()
         pos = 0
         self.tweaks = []
         while pos < len(lines):
             line = lines[pos]
             if line.startswith('#:'):
-                pos = self.read_tweak(lines, pos, dl, l)
+                pos = self.read_tweak(lines, pos, default_tweaks, custom_tweaks)
             pos += 1
 
         self.tweaks.sort()
-        default_keys = set(dl)
+        default_keys = set(default_tweaks)
-        custom_keys = set(l)
+        custom_keys = set(custom_tweaks)
 
         self.plugin_tweaks = {}
         for key in custom_keys - default_keys:
-            self.plugin_tweaks[key] = l[key]
+            self.plugin_tweaks[key] = custom_tweaks[key]
 
     def read_tweak(self, lines, pos, defaults, custom):
         name = lines[pos][2:].strip()
@@ -259,20 +270,20 @@ class Tweaks(QAbstractListModel, AdaptSQP):  # {{{
                ' edit it unless you know what you are doing.', '',
             ]
         for tweak in self.tweaks:
-            ans.extend(['', str(tweak), ''])
+            ans.extend(['', unicode_type(tweak), ''])
 
         if self.plugin_tweaks:
             ans.extend(['', '',
                 '# The following are tweaks for installed plugins', ''])
             for key, val in iteritems(self.plugin_tweaks):
-                ans.extend(['%s = %r'%(key, val), '', ''])
+                ans.extend([u'%s = %r'%(key, val), '', ''])
         return '\n'.join(ans)
 
     @property
     def plugin_tweaks_string(self):
         ans = []
         for key, val in iteritems(self.plugin_tweaks):
-            ans.extend(['%s = %r'%(key, val), '', ''])
+            ans.extend([u'%s = %r'%(key, val), '', ''])
         ans = '\n'.join(ans)
         if isbytestring(ans):
             ans = ans.decode('utf-8')
@@ -369,7 +380,6 @@ class TweaksView(QListView):
         self.setSizePolicy(QSizePolicy.Minimum, QSizePolicy.Expanding)
         self.setAlternatingRowColors(True)
         self.setSpacing(5)
-        self.setUniformItemSizes(True)
         self.setVerticalScrollMode(self.ScrollPerPixel)
 
     def currentChanged(self, cur, prev):
@@ -544,8 +554,10 @@ class ConfigWidget(ConfigWidgetBase):
 
     def commit(self):
         raw = self.tweaks.to_string()
+        if not isinstance(raw, bytes):
+            raw = raw.encode('utf-8')
         try:
-            exec(raw)
+            custom_tweaks = exec_tweaks(raw)
         except:
             import traceback
             error_dialog(self, _('Invalid tweaks'),
@@ -554,7 +566,7 @@ class ConfigWidget(ConfigWidgetBase):
                         ' you find the invalid setting.'),
                     det_msg=traceback.format_exc(), show=True)
             raise AbortCommit('abort')
-        write_tweaks(raw)
+        write_custom_tweaks(custom_tweaks)
         ConfigWidgetBase.commit(self)
         return True
 

src/calibre/utils/config.py

@@ -19,7 +19,7 @@ from calibre.constants import (
 from calibre.utils.config_base import (
     Config, ConfigInterface, ConfigProxy, Option, OptionSet, OptionValues,
     StringConfig, json_dumps, json_loads, make_config_dir, plugin_dir, prefs,
-    read_raw_tweaks, read_tweaks, tweaks, write_tweaks, from_json, to_json
+    tweaks, from_json, to_json
 )
 from calibre.utils.lock import ExclusiveFile
 
@@ -30,9 +30,8 @@ optparse._ = _
 
 if False:
     # Make pyflakes happy
-    Config, ConfigProxy, Option, OptionValues, StringConfig
+    Config, ConfigProxy, Option, OptionValues, StringConfig, OptionSet,
-    OptionSet, ConfigInterface, read_tweaks, write_tweaks
+    ConfigInterface, tweaks, plugin_dir, prefs, from_json, to_json
-    read_raw_tweaks, tweaks, plugin_dir, prefs, from_json, to_json
 
 
 def check_config_write_access():

src/calibre/utils/config_base.py

@@ -12,8 +12,8 @@ from collections import defaultdict
 from copy import deepcopy
 
 from calibre.utils.lock import ExclusiveFile
-from calibre.constants import config_dir, CONFIG_DIR_MODE, ispy3
+from calibre.constants import config_dir, CONFIG_DIR_MODE, ispy3, preferred_encoding
-from polyglot.builtins import unicode_type
+from polyglot.builtins import unicode_type, iteritems, map
 
 plugin_dir = os.path.join(config_dir, 'plugins')
 
@@ -522,50 +522,78 @@ if prefs['installation_uuid'] is None:
 # Read tweaks
 
 
-def read_raw_tweaks():
+def tweaks_file():
+    return os.path.join(config_dir, u'tweaks.json')
+
+
+def make_unicode(obj):
+    if isinstance(obj, bytes):
+        try:
+            return obj.decode('utf-8')
+        except UnicodeDecodeError:
+            return obj.decode(preferred_encoding, errors='replace')
+    if isinstance(obj, (list, tuple)):
+        return list(map(make_unicode, obj))
+    if isinstance(obj, dict):
+        return {make_unicode(k): make_unicode(v) for k, v in iteritems(obj)}
+    return obj
+
+
+def write_custom_tweaks(tweaks_dict):
     make_config_dir()
-    default_tweaks = P('default_tweaks.py', data=True,
+    raw = json_dumps(make_unicode(tweaks_dict))
-            allow_user_override=False)
+    with open(tweaks_file(), 'wb') as f:
-    tweaks_file = os.path.join(config_dir, 'tweaks.py')
+        f.write(raw)
-    if not os.path.exists(tweaks_file):
+
-        with open(tweaks_file, 'wb') as f:
+
-            f.write(default_tweaks)
+def exec_tweaks(path):
-    with open(tweaks_file, 'rb') as f:
+    if isinstance(path, bytes):
-        return default_tweaks, f.read()
+        raw = path
+        fname = '<string>'
+    else:
+        with open(path, 'rb') as f:
+            raw = f.read()
+            fname = f.name
+    code = compile(raw, fname, 'exec')
+    l = {}
+    g = {'__file__': fname}
+    exec(code, g, l)
+    return l
+
+
+def read_custom_tweaks():
+    make_config_dir()
+    tf = tweaks_file()
+    if os.path.exists(tf):
+        with open(tf, 'rb') as f:
+            raw = f.read()
+        return json_loads(raw)
+    old_tweaks_file = tf.rpartition(u'.')[0] + u'.py'
+    ans = {}
+    if os.path.exists(old_tweaks_file):
+        ans = exec_tweaks(old_tweaks_file)
+        ans = make_unicode(ans)
+        write_custom_tweaks(ans)
+    return ans
+
+
+def default_tweaks_raw():
+    return P('default_tweaks.py', data=True, allow_user_override=False)
 
 
 def read_tweaks():
-    default_tweaks, tweaks = read_raw_tweaks()
+    default_tweaks = exec_tweaks(default_tweaks_raw())
-    l, g = {}, {}
+    default_tweaks.update(read_custom_tweaks())
-    try:
+    return default_tweaks
-        exec(tweaks, g, l)
-    except:
-        import traceback
-        print('Failed to load custom tweaks file')
-        traceback.print_exc()
-    dl, dg = {}, {}
-    exec(default_tweaks, dg, dl)
-    dl.update(l)
-    return dl
-
-
-def write_tweaks(raw):
-    make_config_dir()
-    tweaks_file = os.path.join(config_dir, 'tweaks.py')
-    with open(tweaks_file, 'wb') as f:
-        f.write(raw)
 
 
 tweaks = read_tweaks()
 
 
 def reset_tweaks_to_default():
-    default_tweaks = P('default_tweaks.py', data=True,
+    default_tweaks = exec_tweaks(default_tweaks_raw())
-            allow_user_override=False)
-    dl, dg = {}, {}
-    exec(default_tweaks, dg, dl)
     tweaks.clear()
-    tweaks.update(dl)
+    tweaks.update(default_tweaks)
 
 
 class Tweak(object):