From 05d920905307feb8efac67f0dd0ff20aac8fb9a5 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 25 Jun 2016 11:26:23 +0530 Subject: [PATCH] Also check RapydScript files when running python setup.py check Use sha1 hashes for the check cache instead of mtimes. More reliable and should work with the travis cache as well, if you decide to run check on travis --- .gitignore | 2 +- .travis.yml | 1 + setup/__init__.py | 11 ++++- setup/check.py | 116 +++++++++++++++++++++++++++++----------------- 4 files changed, 86 insertions(+), 44 deletions(-) diff --git a/.gitignore b/.gitignore index 62017e5d57..5f1ae0e02e 100644 --- a/.gitignore +++ b/.gitignore @@ -4,7 +4,7 @@ *.pyj-cached .bzr .bzrignore -.check-cache.pickle +.build-cache src/calibre/plugins resources/images.qrc manual/generated diff --git a/.travis.yml b/.travis.yml index 0cfaeded71..ff276e0858 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,6 +5,7 @@ nodejs: cache: directories: - node_modules + - .build-cache env: - SW=$HOME/sw PATH=$SW/bin:$PATH CFLAGS=-I$SW/include LDFLAGS=-L$SW/lib LD_LIBRARY_PATH=$SW/qt/lib:$SW/lib PKG_CONFIG_PATH=$SW/lib/pkgconfig QMAKE=$SW/qt/bin/qmake QT_PLUGIN_PATH=$SW/qt/plugins diff --git a/setup/__init__.py b/setup/__init__.py index 8662a824d0..6af45bdb50 100644 --- a/setup/__init__.py +++ b/setup/__init__.py @@ -6,7 +6,7 @@ __license__ = 'GPL v3' __copyright__ = '2009, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import sys, re, os, platform, subprocess, time +import sys, re, os, platform, subprocess, time, errno is64bit = platform.architecture()[0] == '64bit' iswindows = re.search('win(32|64)', sys.platform) @@ -25,6 +25,15 @@ sys.running_from_setup = True __version__ = __appname__ = modules = functions = basenames = scripts = None +def build_cache_dir(): + ans = os.path.join(os.path.dirname(SRC), '.build-cache') + try: + os.mkdir(ans) + except EnvironmentError as err: + if err.errno != errno.EEXIST: + raise + return ans + def require_git_master(): if subprocess.check_output(['git', 'symbolic-ref', '--short', 'HEAD']).strip() != 'master': print >>sys.stderr, 'You must be in the master git branch' diff --git a/setup/check.py b/setup/check.py index 02b5c688b1..99f0200da5 100644 --- a/setup/check.py +++ b/setup/check.py @@ -6,8 +6,8 @@ __license__ = 'GPL v3' __copyright__ = '2009, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import sys, os, cPickle, subprocess -from setup import Command +import sys, os, json, subprocess, errno, hashlib +from setup import Command, build_cache_dir import __builtin__ def set_builtins(builtins): @@ -28,63 +28,95 @@ class Check(Command): description = 'Check for errors in the calibre source code' - CACHE = '.check-cache.pickle' + CACHE = 'check.json' - def get_files(self, cache): + def get_files(self): for x in os.walk(self.j(self.SRC, 'calibre')): for f in x[-1]: y = self.j(x[0], f) if x[0].endswith('calibre/ebooks/markdown'): continue - mtime = os.stat(y).st_mtime - if cache.get(y, 0) == mtime: - continue if (f.endswith('.py') and f not in ( 'feedparser.py', 'markdown.py') and 'prs500/driver.py' not in y) and not f.endswith('_ui.py'): - yield y, mtime + yield y if f.endswith('.coffee'): - yield y, mtime + yield y for x in os.walk(self.j(self.d(self.SRC), 'recipes')): for f in x[-1]: f = self.j(x[0], f) - mtime = os.stat(f).st_mtime - if f.endswith('.recipe') and cache.get(f, 0) != mtime: - yield f, mtime + if f.endswith('.recipe'): + yield f - def run(self, opts): - cache = {} - if os.path.exists(self.CACHE): - cache = cPickle.load(open(self.CACHE, 'rb')) - for f, mtime in self.get_files(cache): - self.info('\tChecking', f) - errors = False - ext = os.path.splitext(f)[1] - if ext in {'.py', '.recipe'}: - p = subprocess.Popen(['flake8-python2', '--ignore=E,W', f]) - if p.wait() != 0: - errors = True - else: - from calibre.utils.serve_coffee import check_coffeescript - try: - check_coffeescript(f) - except: - errors = True - if errors: - cPickle.dump(cache, open(self.CACHE, 'wb'), -1) - subprocess.call(['gvim', '-S', - self.j(self.SRC, '../session.vim'), '-f', f]) - raise SystemExit(1) - cache[f] = mtime - cPickle.dump(cache, open(self.CACHE, 'wb'), -1) - wn_path = os.path.expanduser('~/work/srv/main/static') - if os.path.exists(wn_path): - sys.path.insert(0, wn_path) - self.info('\tChecking Changelog...') + for x in os.walk(self.j(self.SRC, 'pyj')): + for f in x[-1]: + f = self.j(x[0], f) + if f.endswith('.pyj'): + yield f + if self.has_changelog_check: + yield self.j(self.d(self.SRC), 'Changelog.yaml') + + def read_file(self, f): + with open(f, 'rb') as f: + return f.read() + + def file_hash(self, f): + try: + return self.fhash_cache[f] + except KeyError: + self.fhash_cache[f] = ans = hashlib.sha1(self.read_file(f)).hexdigest() + return ans + + def is_cache_valid(self, f, cache): + return cache.get(f) == self.file_hash(f) + + def save_cache(self, cache): + with open(self.j(build_cache_dir(), self.CACHE), 'wb') as f: + json.dump(cache, f) + + def file_has_errors(self, f): + ext = os.path.splitext(f)[1] + if ext in {'.py', '.recipe'}: + p = subprocess.Popen(['flake8-python2', '--ignore=E,W', f]) + return p.wait() != 0 + elif ext == '.pyj': + p = subprocess.Popen(['rapydscript', 'lint', f]) + return p.wait() != 0 + elif ext == '.yaml': + sys.path.insert(0, self.wn_path) import whats_new whats_new.render_changelog(self.j(self.d(self.SRC), 'Changelog.yaml')) - sys.path.remove(wn_path) + sys.path.remove(self.wn_path) + else: + from calibre.utils.serve_coffee import check_coffeescript + try: + check_coffeescript(f) + except: + return True + + def run(self, opts): + self.fhash_cache = {} + cache = {} + self.wn_path = os.path.expanduser('~/work/srv/main/static') + self.has_changelog_check = os.path.exists(self.wn_path) + try: + cache = json.load(open(self.j(build_cache_dir(), self.CACHE), 'rb')) + except EnvironmentError as err: + if err.errno != errno.ENOENT: + raise + try: + for f in self.get_files(): + if self.is_cache_valid(f, cache): + continue + self.info('\tChecking', f) + if self.file_has_errors(f): + subprocess.call(['gvim', '-S', + self.j(self.SRC, '../session.vim'), '-f', f]) + raise SystemExit(1) + cache[f] = self.file_hash(f) + finally: + self.save_cache(cache) def report_errors(self, errors): for err in errors: