Update RapydScript

.gitignore

@@ -1,6 +1,7 @@
 *_ui.py
 *.pyc
 *.pyo
+*.pyj-cached
 .bzr
 .bzrignore
 .check-cache.pickle

resources/rapydscript/lib/math.pyj

@@ -0,0 +1,192 @@
+###########################################################
+# RapydScript Standard Library
+# Author: Alexander Tsepkov
+# Copyright 2013 Pyjeon Software LLC
+# License: Apache License    2.0
+# This library is covered under Apache license, so that
+# you can distribute it with your RapydScript applications.
+###########################################################
+
+
+# basic implementation of Python's 'math' library
+
+# NOTE: this is only meant to aid those porting lots of Python code into RapydScript,
+# if you're writing a new RapydScript application, in most cases you probably want to
+# use JavaScript's Math module directly instead
+
+
+pi = Math.PI
+e = Math.E
+
+########################################
+# Number-theoretic and representation functions
+########################################
+def ceil(x):
+    return Math.ceil(x)
+def copysign(x, y):
+    x = Math.abs(x)
+    if y < 0:
+        return -x
+    else:
+        return x
+def fabs(x):
+    return Math.abs(x)
+def factorial(x):
+    if Math.abs(int(x)) != x:
+        raise ValueError("factorial() only accepts integral values")
+    factorial.cache = []
+    r = def(n):
+        if n == 0 or n == 1:
+            return 1
+        if not factorial.cache[n]:
+            factorial.cache[n] = r(n-1) * n
+        return factorial.cache[n]
+    return r(x)
+def floor(x):
+    return Math.floor(x)
+def fmod(x, y):
+    # javascript's % operator isn't consistent with C fmod implementation, this function is
+    while y <= x:
+        x -= y
+    return x
+def fsum(iterable):
+    # like Python's fsum, this method is much more resilient to rounding errors than regular sum
+    partials = []   # sorted, non-overlapping partial sums
+    for x in iterable:
+        i = 0
+        for y in partials:
+            if Math.abs(x) < Math.abs(y):
+                x, y = y, x
+            hi = x + y
+            lo = y - (hi - x)
+            if lo:
+                partials[i] = lo
+                i += 1
+            x = hi
+        #partials[i:] = [x]
+        partials.splice(i, partials.length-i, x)
+    return sum(partials)
+def isinf(x):
+    return not isFinite(x)
+def isnan(x):
+    return isNaN(x)
+def modf(x):
+    m = fmod(x, 1)
+    return m, x-m
+def trunc(x):
+    return x | 0
+
+########################################
+# Power and logarithmic functions
+########################################
+def exp(x):
+    return Math.exp(x)
+def expm1(x):
+    # NOTE: Math.expm1() is currently only implemented in Firefox, this provides alternative implementation
+    # https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Math/expm1
+    #return Math.expm1(x)
+    if Math.abs(x) < 1e-5:
+        return x + 0.5*x*x
+    else:
+        return Math.exp(x) - 1
+def log(x, base=e):
+    return Math.log(x)/Math.log(base)
+def log1p(x):
+    # NOTE: Math.log1p() is currently only implemented in Firefox, this provides alternative implementation
+    # https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Math/log1p
+    # this version has been taken from http://phpjs.org/functions/log1p/
+    # admittedly it's not as accurate as MDN version, as you can see from math.log1p(1) result
+    ret = 0
+    n = 50
+    if x <= -1:
+        return Number.NEGATIVE_INFINITY
+    if x < 0 or x > 1:
+        return Math.log(1 + x)
+    for i in range(1, n):
+        if i % 2 == 0:
+            ret -= Math.pow(x, i) / i
+        else:
+            ret += Math.pow(x, i) / i
+    return ret
+def log10(x):
+    # NOTE: Math.log10() is currently only implemented in Firefox, this provides alternative implementation
+    # https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Math/log10
+    # I didn't find a more accurate algorithm so I'm using the basic implementation
+    return Math.log(x)/Math.LN10
+def pow(x, y):
+    if x < 0 and int(y) != y:
+        raise ValueError('math domain error')
+    if isnan(y) and x == 1:
+        return 1
+    return Math.pow(x, y)
+def sqrt(x):
+    return Math.sqrt(x)
+
+########################################
+# Trigonometric functions
+########################################
+def acos(x):        return Math.acos(x)
+def asin(x):        return Math.asin(x)
+def atan(x):        return Math.atan(x)
+def atan2(y, x):    return Math.atan2(y, x)
+def cos(x):         return Math.cos(x)
+def sin(x):         return Math.sin(x)
+def hypot(x, y):    return Math.sqrt(x*x + y*y)
+def tan(x):         return Math.tan(x)
+
+########################################
+# Angular conversion
+########################################
+def degrees(x):     return x*180/pi
+def radians(x):     return x*pi/180
+
+########################################
+# Hyperbolic functions
+########################################
+def acosh(x):
+    # NOTE: will be replaced with official, when it becomes mainstream
+    # https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Math/acosh
+    return Math.log(x + Math.sqrt(x*x - 1))
+def asinh(x):
+    # NOTE: will be replaced with official, when it becomes mainstream
+    # https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Math/asinh
+    return Math.log(x + Math.sqrt(x*x + 1))
+def atanh(x):
+    # NOTE: will be replaced with official, when it becomes mainstream
+    # https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Math/atanh
+    return 0.5 * Math.log((1 + x) / (1 - x))
+def cosh(x):
+    # NOTE: will be replaced with official, when it becomes mainstream
+    # https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Math/cosh
+    return (Math.exp(x) + Math.exp(-x)) / 2
+def sinh(x):
+    # NOTE: will be replaced with official, when it becomes mainstream
+    # https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Math/sinh
+    return (Math.exp(x) - Math.exp(-x)) / 2
+def tanh(x):
+    # NOTE: will be replaced with official, when it becomes mainstream
+    # https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Math/tanh
+    return (Math.exp(x) - Math.exp(-x)) / (Math.exp(x) + Math.exp(-x))
+
+
+
+#import stdlib
+#print(math.ceil(4.2))
+#print(math.floor(4.2))
+#print(math.fabs(-6))
+#print(math.copysign(-5, 7))
+#print(math.factorial(4))
+#print(math.fmod(-1e100, 1e100))
+#
+#d = [0.9999999, 1, 2, 3]
+#print(sum(d), math.fsum(d))
+#print(math.isinf(5), math.isinf(Infinity))
+#print(math.modf(5.5))
+#print(math.trunc(2.6), math.trunc(-2.6))
+#print(math.exp(1e-5), math.expm1(1e-5))
+#print(math.log(10), math.log(10, 1000))
+#print(math.log1p(1e-15), math.log1p(1))
+#print(math.log10(1000), math.log(1000, 10))
+#print(math.pow(1, 0), math.pow(1, NaN), math.pow(0, 0), math.pow(NaN, 0), math.pow(4,3), math.pow(100, -2))
+#print(math.hypot(3,4))
+#print(math.acosh(2), math.asinh(1), math.atanh(0.5), math.cosh(1), math.cosh(-1), math.sinh(1), math.tanh(1))

resources/rapydscript/lib/operator.pyj

@@ -0,0 +1,11 @@
+add = __add__ = def(x, y): return x + y
+sub = __sub__ = def(x, y): return x - y
+mul = __mul__ = def(x, y): return x * y
+div = __div__ = def(x, y): return x / y
+
+lt = __lt__ = def(x, y): return x < y
+le = __le__ = def(x, y): return x <= y
+eq = __eq__ = def(x, y): return x == y
+ne = __ne__ = def(x, y): return x != y
+ge = __ge__ = def(x, y): return x >= y
+gt = __gt__ = def(x, y): return x > y

resources/rapydscript/lib/random.pyj

@@ -0,0 +1,93 @@
+###########################################################
+# RapydScript Standard Library
+# Author: Alexander Tsepkov
+# Copyright 2013 Pyjeon Software LLC
+# License: Apache License    2.0
+# This library is covered under Apache license, so that
+# you can distribute it with your RapydScript applications.
+###########################################################
+
+
+# basic implementation of Python's 'random' library
+
+# JavaScript's Math.random() does not allow seeding its random generator, to bypass that, this module implements its own
+# version that can be seeded. I decided on RC4 algorithm for this.
+
+# please don't mess with this from the outside
+
+_$rapyd$_seed_state = {
+    'key': [],
+    'key_i': 0,
+    'key_j': 0
+}
+
+_$rapyd$_get_random_byte = def():
+    _$rapyd$_seed_state.key_i = (_$rapyd$_seed_state.key_i + 1) % 256
+    _$rapyd$_seed_state.key_j = (_$rapyd$_seed_state.key_j + _$rapyd$_seed_state.key[_$rapyd$_seed_state.key_i]) % 256
+    _$rapyd$_seed_state.key[_$rapyd$_seed_state.key_i], _$rapyd$_seed_state.key[_$rapyd$_seed_state.key_j] = \
+            _$rapyd$_seed_state.key[_$rapyd$_seed_state.key_j], _$rapyd$_seed_state.key[_$rapyd$_seed_state.key_i]
+    return _$rapyd$_seed_state.key[(_$rapyd$_seed_state.key[_$rapyd$_seed_state.key_i] + \
+            _$rapyd$_seed_state.key[_$rapyd$_seed_state.key_j]) % 256]
+
+def seed(x=Date().getTime()):
+    if type(x) is 'number':
+        x = x.toString()
+    elif type(x) is not 'string':
+        raise TypeError("unhashable type: '" + type(x) + "'")
+    for i in range(256):
+        _$rapyd$_seed_state.key[i] = i
+    j = 0
+    for i in range(256):
+        j = (j + _$rapyd$_seed_state.key[i] + x.charCodeAt(i % x.length)) % 256
+        _$rapyd$_seed_state.key[i], _$rapyd$_seed_state.key[j] = _$rapyd$_seed_state.key[j], _$rapyd$_seed_state.key[i]
+seed()
+
+def random():
+    n = 0
+    m = 1
+    for i in range(8):
+        n += _$rapyd$_get_random_byte() * m
+        m *= 256
+    return n / 18446744073709551616
+
+# unlike the python version, this DOES build a range object, feel free to reimplement
+def randrange():
+    return choice(range.apply(this, arguments))
+
+def randint(a, b):
+    return int(random()*(b-a+1) + a)
+
+def uniform(a, b):
+    return random()*(b-a) + a
+
+def choice(seq):
+    if seq.length > 0:
+        return seq[Math.floor(random()*seq.length)]
+    else:
+        raise IndexError()
+
+# uses Fisher-Yates algorithm to shuffle an array
+def shuffle(x, random_f=random):
+    for i in range(x.length):
+        j = Math.floor(random_f() * (i+1))
+        x[i], x[j] = x[j], x[i]
+    return x
+
+# similar to shuffle, but only shuffles a subset and creates a copy
+def sample(population, k):
+    x = population.slice()
+    for i in range(population.length-1, population.length-k-1, -1):
+        j = Math.floor(random() * (i+1))
+        x[i], x[j] = x[j], x[i]
+    return x.slice(population.length-k)
+
+
+#import stdlib
+#a = range(50)
+#random.seed(5)
+#print(random.choice(a))
+#print(random.shuffle(a))
+#print(random.randrange(10))
+#print(random.randint(1,5))
+#print(random.uniform(1,5))
+#print(random.sample(range(20),5))

src/calibre/utils/rapydscript.py

@@ -6,7 +6,7 @@ from __future__ import (unicode_literals, division, absolute_import,
 __license__ = 'GPL v3'
 __copyright__ = '2015, Kovid Goyal <kovid at kovidgoyal.net>'
 
-import os, json, sys, re, atexit, errno
+import os, sys, atexit, errno, subprocess, bz2, glob, shutil, json
 from threading import local
 from functools import partial
 from threading import Thread
@@ -16,105 +16,30 @@ from duktape import Context, JSError, to_python
 from calibre.constants import cache_dir
 from calibre.utils.terminal import ANSIStream
 
-COMPILER_PATH = 'rapydscript/compiler.js'
+COMPILER_PATH = 'rapydscript/compiler.js.bz2'
 
 def abspath(x):
     return os.path.realpath(os.path.abspath(x))
 
 # Update RapydScript {{{
-def parse_baselib(src):
-    # duktape does not store function source code, so we have to do it manually
-    start = re.compile(r'''['"]([a-zA-Z0-9()]+)['"]\s*:''')
-    in_func = None
-    funcs = {}
-    for line in src.splitlines():
-        line = line.rstrip()
-        if in_func is None:
-            m = start.match(line)
-            if m is not None:
-                funcs[m.group(1)] = in_func = [line.partition(':')[-1].lstrip()]
-        else:
-            if line in (',', '}'):
-                in_func = None
-            else:
-                in_func.append(line)
-    funcs = {k:'\n'.join(v) for k, v in funcs.iteritems()}
-    return funcs
-
-def compile_baselib(ctx, baselib, beautify=True):
-    ctx.g.current_output_options = {'beautify':beautify, 'private_scope':False, 'write_name':False}
-    ctx.g.filename = 'baselib.pyj'
-    ctx.g.basedir = ''
-    ctx.g.libdir = ''
-
-    def doit(src):
-        src += '\n'
-        ctx.g.code = src
-        try:
-            return ctx.eval(COMPILER_JS)
-        except Exception as e:
-            print ('Failed to compile source:')
-            print (src)
-            raise SystemExit(str(e))
-
-    return {k:doit(v) for k, v in sorted(baselib.iteritems())}
-
 def update_rapydscript():
-    vm_js = '''
-    exports.createContext = function(x) { x.AST_Node = {}; return x; }
-    exports.runInContext = function() { return null; }
-    '''
-    fs_js = '''
-    exports.realpathSync = function(x) { return x; }
-    exports.readFileSync = function() { return ""; }
-    '''
-    path_js = '''
-    exports.join = function(x, y) { return x + '/' + y; }
-    exports.dirname = function(x) { return x; }
-    exports.resolve = function(x) { return x; }
-    '''
-
     d = os.path.dirname
     base = d(d(d(d(d(abspath(__file__))))))
     base = os.path.join(base, 'rapydscript')
-    ctx = Context(base_dirs=(base,), builtin_modules={'path':path_js, 'fs':fs_js, 'vm':vm_js})
+    raw = subprocess.check_output(['node', '--harmony', os.path.join(base, 'bin', 'export')])
-    ctx.g.require.id = 'rapydscript/bin'
+    path = P(COMPILER_PATH, allow_user_override=False)
-    try:
+    with open(path, 'wb') as f:
-        ctx.eval('RapydScript = require("../tools/compiler")', fname='bin/rapydscript')
+        f.write(bz2.compress(raw, 9))
-    except JSError as e:
+    base = os.path.join(base, 'src', 'lib')
-        raise SystemExit('%s:%s:%s' % (e.fileName, e.lineNumber, e.message))
+    dest = os.path.join(P('rapydscript', allow_user_override=False), 'lib')
-    data = b'\n\n'.join(open(os.path.join(base, 'lib', x + '.js'), 'rb').read() for x in ctx.g.RapydScript.FILENAMES)
+    if not os.path.exists(dest):
-
+        os.mkdir(dest)
-    package = json.load(open(os.path.join(base, 'package.json')))
+    for x in glob.glob(os.path.join(base, '*.pyj')):
-    baselib = parse_baselib(open(os.path.join(base, 'src', 'baselib.pyj'), 'rb').read().decode('utf-8'))
+        shutil.copy2(x, dest)
-    ctx = Context()
-    ctx.eval(data.decode('utf-8'))
-    baselib = {'beautifed': compile_baselib(ctx, baselib), 'minified': compile_baselib(ctx, baselib, False)}
-    repl = open(os.path.join(base, 'tools', 'repl.js'), 'rb').read()
-
-    with open(P(COMPILER_PATH, allow_user_override=False), 'wb') as f:
-        f.write(data)
-        f.write(b'\n\nrs_baselib_pyj = ' + json.dumps(baselib) + b';')
-        f.write(b'\n\nrs_repl_js = ' + json.dumps(repl) + b';')
-        f.write(b'\n\nrs_package_version = ' + json.dumps(package['version']) + b';\n')
 # }}}
 
 # Compiler {{{
 tls = local()
-COMPILER_JS = '''
-(function() {
-var output = OutputStream(current_output_options);
-var ast = parse(code, {
-    filename: filename,
-    readfile: Duktape.readfile,
-    basedir: basedir,
-    auto_bind: false,
-    libdir: libdir
-});
-ast.print(output);
-return output.get();
-})();
-'''
 
 def to_dict(obj):
     return dict(zip(obj.keys(), obj.values()))
@@ -122,9 +47,9 @@ def to_dict(obj):
 def compiler():
     c = getattr(tls, 'compiler', None)
     if c is None:
-        c = tls.compiler = Context(base_dirs=(P('rapydscript', allow_user_override=False),))
+        c = tls.compiler = Context()
-        c.eval(P(COMPILER_PATH, data=True, allow_user_override=False).decode('utf-8'), fname='rapydscript-compiler.js')
+        c.eval('exports = {}; sha1sum = Duktape.sha1sum;', noreturn=True)
-        c.g.current_output_options = {}
+        c.eval(bz2.decompress(P(COMPILER_PATH, data=True, allow_user_override=False)), fname=COMPILER_PATH, noreturn=True)
     return c
 
 class PYJError(Exception):
@@ -133,31 +58,20 @@ class PYJError(Exception):
         Exception.__init__(self, '')
         self.errors = errors
 
-def compile_pyj(data, filename='<stdin>', beautify=True, private_scope=True, libdir=None, omit_baselib=False, write_name=True):
+def compile_pyj(data, filename='<stdin>', beautify=True, private_scope=True, libdir=None, omit_baselib=False):
-    import duktape
     if isinstance(data, bytes):
         data = data.decode('utf-8')
     c = compiler()
-    c.g.current_output_options = {
+    c.g.current_options = {
         'beautify':beautify,
         'private_scope':private_scope,
         'omit_baselib': omit_baselib,
-        'write_name': write_name,
+        'libdir': libdir or P('rapydscript/lib', allow_user_override=False),
-        'baselib':dict(dict(c.g.rs_baselib_pyj)['beautifed' if beautify else 'minified']),
+        'basedir': os.getcwdu() if not filename or filename == '<stdin>' else os.path.dirname(filename),
+        'filename': filename,
     }
-    d = os.path.dirname
+    c.g.rs_source_code = data
-    c.g.libdir = libdir or os.path.join(d(d(d(abspath(__file__)))), 'pyj')
+    return c.eval('exports["compile"](rs_source_code, %s, current_options)' % json.dumps(filename))
-    c.g.code = data
-    c.g.filename = filename
-    c.g.basedir = os.getcwdu() if not filename or filename == '<stdin>' else d(filename)
-    errors = []
-    c.g.AST_Node.warn = lambda templ, data:errors.append(to_dict(data))
-    try:
-        return c.eval(COMPILER_JS)
-    except duktape.JSError:
-        if errors:
-            raise PYJError(errors)
-        raise
 # }}}
 
 # REPL {{{
@@ -192,30 +106,14 @@ class Repl(Thread):
         self.start()
 
     def init_ctx(self):
-        cc = '''
-        exports.AST_Node = AST_Node;
-        exports.ALL_KEYWORDS = ALL_KEYWORDS;
-        exports.tokenizer = tokenizer;
-        exports.parse = parse;
-        exports.OutputStream = OutputStream;
-        exports.IDENTIFIER_PAT = IDENTIFIER_PAT;
-        '''
         self.prompt = self.ps1
-        readline = '''
+
-        exports.createInterface = function(options) { rl.completer = options.completer; return rl; }
+        self.ctx = compiler()
-        '''
-        self.ctx = Context(builtin_modules={'readline':readline, 'compiler':cc})
         self.ctx.g.Duktape.write = self.output.write
         self.ctx.eval(r'''console = { log: function() { Duktape.write(Array.prototype.slice.call(arguments).join(' ') + '\n');}};
                       console['error'] = console['log'];''')
-        cc = P(COMPILER_PATH, data=True, allow_user_override=False)
-        self.ctx.eval(cc)
-        baselib = dict(dict(self.ctx.g.rs_baselib_pyj)['beautifed'])
-        baselib = '\n\n'.join(baselib.itervalues())
-        self.ctx.eval('module = {}')
-        self.ctx.eval(self.ctx.g.rs_repl_js, fname='repl.js')
         self.ctx.g.repl_options = {
-            'baselib': baselib, 'show_js': self.show_js,
+            'show_js': self.show_js,
             'histfile':False,
             'input':True, 'output':True, 'ps1':self.ps1, 'ps2':self.ps2,
             'terminal':self.output.isatty,
@@ -257,9 +155,10 @@ class Repl(Thread):
             send_interrupt: function() { listeners['SIGINT'](); },
             close: function() {listeners['close'](); },
         };
-        ''')
+        repl_options.readline = { createInterface: function(options) { rl.completer = options.completer; return rl; }};
+        exports.init_repl(repl_options)
+        ''', fname='<init repl>')
         rl = self.ctx.g.rl
-        self.ctx.eval('module.exports(repl_options)')
         completer = to_python(rl.completer)
         send_interrupt = to_python(rl.send_interrupt)
         send_line = to_python(rl.send_line)
@@ -333,7 +232,7 @@ class Repl(Thread):
 
 def main(args=sys.argv):
     import argparse
-    ver = compiler().g.rs_package_version
+    ver = compiler().g.exports.rs_version
     parser = argparse.ArgumentParser(prog='pyj',
         description='RapydScript compiler and REPL. If passed input on stdin, it is compiled and written to stdout. Otherwise a REPL is started.')
     parser.add_argument('--version', action='version',