Also check RapydScript files when running python setup.py check

Use sha1 hashes for the check cache instead of mtimes. More reliable
and should work with the travis cache as well, if you decide to run
check on travis

.gitignore

@@ -4,7 +4,7 @@
 *.pyj-cached
 .bzr
 .bzrignore
-.check-cache.pickle
+.build-cache
 src/calibre/plugins
 resources/images.qrc
 manual/generated

.travis.yml

@@ -5,6 +5,7 @@ nodejs:
 cache:
   directories:
     - node_modules
+    - .build-cache
 
 env:
     - SW=$HOME/sw PATH=$SW/bin:$PATH CFLAGS=-I$SW/include LDFLAGS=-L$SW/lib LD_LIBRARY_PATH=$SW/qt/lib:$SW/lib PKG_CONFIG_PATH=$SW/lib/pkgconfig QMAKE=$SW/qt/bin/qmake QT_PLUGIN_PATH=$SW/qt/plugins

setup/__init__.py

@@ -6,7 +6,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import sys, re, os, platform, subprocess, time
+import sys, re, os, platform, subprocess, time, errno
 
 is64bit = platform.architecture()[0] == '64bit'
 iswindows = re.search('win(32|64)', sys.platform)
@@ -25,6 +25,15 @@ sys.running_from_setup = True
 
 __version__ = __appname__ = modules = functions = basenames = scripts = None
 
+def build_cache_dir():
+    ans = os.path.join(os.path.dirname(SRC), '.build-cache')
+    try:
+        os.mkdir(ans)
+    except EnvironmentError as err:
+        if err.errno != errno.EEXIST:
+            raise
+    return ans
+
 def require_git_master():
     if subprocess.check_output(['git', 'symbolic-ref', '--short', 'HEAD']).strip() != 'master':
         print >>sys.stderr, 'You must be in the master git branch'

setup/check.py

@@ -6,8 +6,8 @@ __license__   = 'GPL v3'
 __copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import sys, os, cPickle, subprocess
+import sys, os, json, subprocess, errno, hashlib
-from setup import Command
+from setup import Command, build_cache_dir
 import __builtin__
 
 def set_builtins(builtins):
@@ -28,63 +28,95 @@ class Check(Command):
 
     description = 'Check for errors in the calibre source code'
 
-    CACHE = '.check-cache.pickle'
+    CACHE = 'check.json'
 
-    def get_files(self, cache):
+    def get_files(self):
         for x in os.walk(self.j(self.SRC, 'calibre')):
             for f in x[-1]:
                 y = self.j(x[0], f)
                 if x[0].endswith('calibre/ebooks/markdown'):
                     continue
-                mtime = os.stat(y).st_mtime
-                if cache.get(y, 0) == mtime:
-                    continue
                 if (f.endswith('.py') and f not in (
                         'feedparser.py', 'markdown.py') and
                         'prs500/driver.py' not in y) and not f.endswith('_ui.py'):
-                    yield y, mtime
+                    yield y
                 if f.endswith('.coffee'):
-                    yield y, mtime
+                    yield y
 
         for x in os.walk(self.j(self.d(self.SRC), 'recipes')):
             for f in x[-1]:
                 f = self.j(x[0], f)
-                mtime = os.stat(f).st_mtime
+                if f.endswith('.recipe'):
-                if f.endswith('.recipe') and cache.get(f, 0) != mtime:
+                    yield f
-                    yield f, mtime
 
-    def run(self, opts):
+        for x in os.walk(self.j(self.SRC, 'pyj')):
-        cache = {}
+            for f in x[-1]:
-        if os.path.exists(self.CACHE):
+                f = self.j(x[0], f)
-            cache = cPickle.load(open(self.CACHE, 'rb'))
+                if f.endswith('.pyj'):
-        for f, mtime in self.get_files(cache):
+                    yield f
-            self.info('\tChecking', f)
+        if self.has_changelog_check:
-            errors = False
+            yield self.j(self.d(self.SRC), 'Changelog.yaml')
-            ext = os.path.splitext(f)[1]
+
-            if ext in {'.py', '.recipe'}:
+    def read_file(self, f):
-                p = subprocess.Popen(['flake8-python2', '--ignore=E,W', f])
+        with open(f, 'rb') as f:
-                if p.wait() != 0:
+            return f.read()
-                    errors = True
+
-            else:
+    def file_hash(self, f):
-                from calibre.utils.serve_coffee import check_coffeescript
+        try:
-                try:
+            return self.fhash_cache[f]
-                    check_coffeescript(f)
+        except KeyError:
-                except:
+            self.fhash_cache[f] = ans = hashlib.sha1(self.read_file(f)).hexdigest()
-                    errors = True
+            return ans
-            if errors:
+
-                cPickle.dump(cache, open(self.CACHE, 'wb'), -1)
+    def is_cache_valid(self, f, cache):
-                subprocess.call(['gvim', '-S',
+        return cache.get(f) == self.file_hash(f)
-                                 self.j(self.SRC, '../session.vim'), '-f', f])
+
-                raise SystemExit(1)
+    def save_cache(self, cache):
-            cache[f] = mtime
+        with open(self.j(build_cache_dir(), self.CACHE), 'wb') as f:
-        cPickle.dump(cache, open(self.CACHE, 'wb'), -1)
+            json.dump(cache, f)
-        wn_path = os.path.expanduser('~/work/srv/main/static')
+
-        if os.path.exists(wn_path):
+    def file_has_errors(self, f):
-            sys.path.insert(0, wn_path)
+        ext = os.path.splitext(f)[1]
-            self.info('\tChecking Changelog...')
+        if ext in {'.py', '.recipe'}:
+            p = subprocess.Popen(['flake8-python2', '--ignore=E,W', f])
+            return p.wait() != 0
+        elif ext == '.pyj':
+            p = subprocess.Popen(['rapydscript', 'lint', f])
+            return p.wait() != 0
+        elif ext == '.yaml':
+            sys.path.insert(0, self.wn_path)
             import whats_new
             whats_new.render_changelog(self.j(self.d(self.SRC), 'Changelog.yaml'))
-            sys.path.remove(wn_path)
+            sys.path.remove(self.wn_path)
+        else:
+            from calibre.utils.serve_coffee import check_coffeescript
+            try:
+                check_coffeescript(f)
+            except:
+                return True
+
+    def run(self, opts):
+        self.fhash_cache = {}
+        cache = {}
+        self.wn_path = os.path.expanduser('~/work/srv/main/static')
+        self.has_changelog_check = os.path.exists(self.wn_path)
+        try:
+            cache = json.load(open(self.j(build_cache_dir(), self.CACHE), 'rb'))
+        except EnvironmentError as err:
+            if err.errno != errno.ENOENT:
+                raise
+        try:
+            for f in self.get_files():
+                if self.is_cache_valid(f, cache):
+                    continue
+                self.info('\tChecking', f)
+                if self.file_has_errors(f):
+                    subprocess.call(['gvim', '-S',
+                                    self.j(self.SRC, '../session.vim'), '-f', f])
+                    raise SystemExit(1)
+                cache[f] = self.file_hash(f)
+        finally:
+            self.save_cache(cache)
 
     def report_errors(self, errors):
         for err in errors: