diff --git a/setup/commands.py b/setup/commands.py index e1f21010b8..9439f79739 100644 --- a/setup/commands.py +++ b/setup/commands.py @@ -13,7 +13,7 @@ __all__ = [ 'git_version', 'develop', 'install', 'kakasi', 'coffee', 'rapydscript', 'cacerts', 'recent_uas', 'resources', - 'check', 'test', + 'check', 'to3', 'test', 'sdist', 'bootstrap', 'manual', 'tag_release', 'upload_to_server', @@ -55,6 +55,8 @@ gui = GUI() from setup.check import Check check = Check() +from setup.port import To3 +to3 = To3() from setup.test import Test test = Test() diff --git a/setup/port.py b/setup/port.py new file mode 100644 index 0000000000..8b96d2c712 --- /dev/null +++ b/setup/port.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python +# vim:fileencoding=utf-8 +# License: GPL v3 Copyright: 2019, Kovid Goyal + +from __future__ import absolute_import, division, print_function, unicode_literals + +import errno +import hashlib +import json +import os +import re +import sys +from contextlib import contextmanager + +from setup import Command, build_cache_dir, dump_json + + +@contextmanager +def modified_file(path, modify): + with open(path, 'r+b') as f: + raw = f.read() + nraw = modify(raw) + modified = nraw != raw + if modified: + f.seek(0), f.truncate(), f.write(nraw), f.flush() + f.seek(0) + try: + yield + finally: + if modified: + f.seek(0), f.truncate(), f.write(raw) + + +def no2to3(raw): + return re.sub(br'^.+?\s+# no2to3$', b'', raw, flags=re.M) + + +def run_2to3(path, show_diffs=False): + from lib2to3.main import main + with modified_file(path, no2to3): + cmd = [ + '-f', 'all', + '-f', 'buffer', + '-f', 'idioms', + '-f', 'set_literal', + '-x', 'future', + path, + ] + if not show_diffs: + cmd.append('--no-diffs') + + ret = main('lib2to3.fixes', cmd + [path]) + return ret + + +class To3(Command): + + description = 'Run 2to3 and fix anything it reports' + CACHE = 'check2to3.json' + + @property + def cache_file(self): + return self.j(build_cache_dir(), self.CACHE) + + def is_cache_valid(self, f, cache): + return cache.get(f) == self.file_hash(f) + + def save_cache(self, cache): + dump_json(cache, self.cache_file) + + def get_files(self): + from calibre import walk + for path in walk(os.path.join(self.SRC, 'calibre')): + if path.endswith('.py'): + yield path + + def file_hash(self, f): + try: + return self.fhash_cache[f] + except KeyError: + self.fhash_cache[f] = ans = hashlib.sha1(open(f, 'rb').read()).hexdigest() + return ans + + def file_has_errors(self, f): + from polyglot.io import PolyglotStringIO + oo, oe = sys.stdout, sys.stderr + sys.stdout = sys.stderr = buf = PolyglotStringIO() + try: + run_2to3(f) + finally: + sys.stdout, sys.stderr = oo, oe + output = buf.getvalue() + return re.search(r'^RefactoringTool: No changes to ' + f, output, flags=re.M) is None + + def run(self, opts): + self.fhash_cache = {} + cache = {} + try: + cache = json.load(open(self.cache_file, 'rb')) + except EnvironmentError as err: + if err.errno != errno.ENOENT: + raise + dirty_files = tuple(f for f in self.get_files() if not self.is_cache_valid(f, cache)) + try: + for i, f in enumerate(dirty_files): + self.info('\tChecking', f) + if self.file_has_errors(f): + run_2to3(f, show_diffs=True) + self.info('%d files left to check' % (len(dirty_files) - i - 1)) + raise SystemExit(1) + cache[f] = self.file_hash(f) + finally: + self.save_cache(cache) + + def clean(self): + try: + os.remove(self.cache_file) + except EnvironmentError as err: + if err.errno != errno.ENOENT: + raise diff --git a/src/calibre/constants.py b/src/calibre/constants.py index 8716aace1c..85c57ec2c7 100644 --- a/src/calibre/constants.py +++ b/src/calibre/constants.py @@ -67,8 +67,8 @@ def get_osx_version(): ver = platform.mac_ver()[0].split('.') if len(ver) == 2: ver.append(0) - _osx_ver = OSX(*(map(int, ver))) - except: + _osx_ver = OSX(*map(int, ver)) # no2to3 + except Exception: _osx_ver = OSX(0, 0, 0) return _osx_ver diff --git a/src/calibre/linux.py b/src/calibre/linux.py index 4efe1a8b9d..e4f92ad1dc 100644 --- a/src/calibre/linux.py +++ b/src/calibre/linux.py @@ -223,10 +223,10 @@ class ZshCompleter(object): # {{{ lo = [x+'=' for x in lo] so = [x+'+' for x in so] ostrings = lo + so - ostrings = u'{%s}'%','.join(ostrings) if len(ostrings) > 1 else ostrings[0] - exclude = u'' + ostrings = '{%s}'%','.join(ostrings) if len(ostrings) > 1 else ostrings[0] + exclude = '' if opt.dest is None: - exclude = u"'(- *)'" + exclude = "'(- *)'" h = opt.help or '' h = h.replace('"', "'").replace('[', '(').replace( ']', ')').replace('\n', ' ').replace(':', '\\:').replace('`', "'") @@ -254,8 +254,8 @@ class ZshCompleter(object): # {{{ arg += "'_files -g \"%s\"'"%(' '.join('*.%s'%x for x in tuple(pics) + tuple(x.upper() for x in pics))) - help_txt = u'"[%s]"'%h - yield u'%s%s%s%s '%(exclude, ostrings, help_txt, arg) + help_txt = '"[%s]"'%h + yield '%s%s%s%s '%(exclude, ostrings, help_txt, arg) def opts_and_exts(self, name, op, exts, cover_opts=('--cover',), opf_opts=('--opf',), file_map={}): @@ -295,7 +295,7 @@ class ZshCompleter(object): # {{{ w('\n "--list-recipes:List builtin recipe names"') for recipe in sorted(set(get_builtin_recipe_titles())): recipe = recipe.replace(':', '\\:').replace('"', '\\"') - w(u'\n "%s.recipe"'%(recipe)) + w('\n "%s.recipe"'%(recipe)) w('\n ); _describe -t recipes "ebook-convert builtin recipes" extras') w('\n _files -g "%s"'%' '.join(('*.%s'%x for x in iexts))) w('\n}\n') @@ -384,16 +384,16 @@ class ZshCompleter(object): # {{{ lo = [x+'=' for x in lo] so = [x+'+' for x in so] ostrings = lo + so - ostrings = u'{%s}'%','.join(ostrings) if len(ostrings) > 1 else '"%s"'%ostrings[0] + ostrings = '{%s}'%','.join(ostrings) if len(ostrings) > 1 else '"%s"'%ostrings[0] h = opt.help or '' h = h.replace('"', "'").replace('[', '(').replace( ']', ')').replace('\n', ' ').replace(':', '\\:').replace('`', "'") h = h.replace('%default', unicode_type(opt.default)) - help_txt = u'"[%s]"'%h + help_txt = '"[%s]"'%h opt_lines.append(ostrings + help_txt + ' \\') opt_lines = ('\n' + (' ' * 8)).join(opt_lines) - polyglot_write(f)((u''' + polyglot_write(f)((''' _ebook_edit() { local curcontext="$curcontext" state line ebookfile expl typeset -A opt_args @@ -705,7 +705,7 @@ class PostInstall: if getattr(sys, 'frozen_path', False): if os.access(self.opts.staging_bindir, os.W_OK): self.info('Creating symlinks...') - for exe in scripts.keys(): + for exe in scripts: dest = os.path.join(self.opts.staging_bindir, exe) if os.path.lexists(dest): os.unlink(dest) @@ -835,7 +835,7 @@ class PostInstall: for size in sizes: install_single_icon(iconsrc, basename, size, context, is_last_icon and size is sizes[-1]) - icons = list(filter(None, [x.strip() for x in '''\ + icons = [x.strip() for x in '''\ mimetypes/lrf.png application-lrf mimetypes mimetypes/lrf.png text-lrs mimetypes mimetypes/mobi.png application-x-mobipocket-ebook mimetypes @@ -845,7 +845,7 @@ class PostInstall: lt.png calibre-gui apps viewer.png calibre-viewer apps tweak.png calibre-ebook-edit apps - '''.splitlines()])) + '''.splitlines() if x.strip()] for line in icons: iconsrc, basename, context = line.split() install_icons(iconsrc, basename, context, is_last_icon=line is icons[-1]) diff --git a/src/calibre/test_build.py b/src/calibre/test_build.py index 509c6e2bd0..816299aafe 100644 --- a/src/calibre/test_build.py +++ b/src/calibre/test_build.py @@ -127,7 +127,7 @@ class BuildTest(unittest.TestCase): s = msgpack_dumps(obj) self.assertEqual(obj, msgpack_loads(s)) self.assertEqual(type(msgpack_loads(msgpack_dumps(b'b'))), bytes) - self.assertEqual(type(msgpack_loads(msgpack_dumps(u'b'))), unicode_type) + self.assertEqual(type(msgpack_loads(msgpack_dumps('b'))), unicode_type) large = b'x' * (100 * 1024 * 1024) msgpack_loads(msgpack_dumps(large)) @@ -153,14 +153,14 @@ class BuildTest(unittest.TestCase): au(d['decimal_point'], 'localeconv') for k, v in iteritems(d): au(v, k) - for k in os.environ.keys(): + for k in os.environ: au(winutil.getenv(unicode_type(k)), 'getenv-' + k) os.environ['XXXTEST'] = 'YYY' - self.assertEqual(winutil.getenv(u'XXXTEST'), u'YYY') + self.assertEqual(winutil.getenv('XXXTEST'), 'YYY') del os.environ['XXXTEST'] - self.assertIsNone(winutil.getenv(u'XXXTEST')) + self.assertIsNone(winutil.getenv('XXXTEST')) t = time.localtime() - fmt = u'%Y%a%b%e%H%M' + fmt = '%Y%a%b%e%H%M' for fmt in (fmt, fmt.encode('ascii')): x = strftime(fmt, t) au(x, 'strftime') @@ -189,7 +189,7 @@ class BuildTest(unittest.TestCase): # it should just work because the hard-coded paths of the Qt # installation should work. If they do not, then it is a distro # problem. - fmts = set(map(lambda x: x.data().decode('utf-8'), QImageReader.supportedImageFormats())) + fmts = set(map(lambda x: x.data().decode('utf-8'), QImageReader.supportedImageFormats())) # no2to3 testf = {'jpg', 'png', 'svg', 'ico', 'gif'} self.assertEqual(testf.intersection(fmts), testf, "Qt doesn't seem to be able to load some of its image plugins. Available plugins: %s" % fmts) data = P('images/blank.png', allow_user_override=False, data=True) @@ -309,7 +309,7 @@ class BuildTest(unittest.TestCase): if isosx: cafile = ssl.get_default_verify_paths().cafile if not cafile or not cafile.endswith('/mozilla-ca-certs.pem') or not os.access(cafile, os.R_OK): - self.assert_('Mozilla CA certs not loaded') + raise AssertionError('Mozilla CA certs not loaded') def find_tests():