From 2012b0f6b4dc4c25bd7e6e8894f7aed208e054d6 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 27 Mar 2016 14:24:43 +0530 Subject: [PATCH] Update RapydScript --- resources/rapydscript/lib/encodings.pyj | 96 +++++++++++++++++++++++++ src/pyj/read_book/db.pyj | 2 +- src/pyj/utils.pyj | 42 ----------- 3 files changed, 97 insertions(+), 43 deletions(-) create mode 100644 resources/rapydscript/lib/encodings.pyj diff --git a/resources/rapydscript/lib/encodings.pyj b/resources/rapydscript/lib/encodings.pyj new file mode 100644 index 0000000000..a59440937f --- /dev/null +++ b/resources/rapydscript/lib/encodings.pyj @@ -0,0 +1,96 @@ +# vim:fileencoding=utf-8 +# License: BSD Copyright: 2016, Kovid Goyal + +def base64encode(bytes): + # Convert an array of bytes into a base-64 encoded string + l = bytes.length + remainder = l % 3 + main_length = l - remainder + encodings = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/' + ans = v'[]' + for v'var i = 0; i < main_length; i += 3': + chunk = (bytes[i] << 16) | (bytes[i + 1] << 8) | bytes[i + 2] + ans.push(encodings[(chunk & 16515072) >> 18], encodings[(chunk & 258048) >> 12], encodings[(chunk & 4032) >> 6], encodings[chunk & 63]) + if remainder is 1: + chunk = bytes[main_length] + ans.push(encodings[(chunk & 252) >> 2], encodings[(chunk & 3) << 4], '=', '=') + elif remainder is 2: + chunk = (bytes[main_length] << 8) | bytes[main_length + 1] + ans.push(encodings[(chunk & 64512) >> 10], encodings[(chunk & 1008) >> 4], encodings[(chunk & 15) << 2], '=') + return ans.join('') + +def base64decode(string): + # convert the output of base64encode back into an array of bytes (Uint8Array) + if type(window) is not 'undefined': + chars = window.atob(string) + else: + chars = new Buffer(string, 'base64').toString('binary') # noqa: undef + ans = Uint8Array(chars.length) + for i in range(ans.length): + ans[i] = chars.charCodeAt(i) + return ans + +utf8_decoder_table = v'''[ + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 00..1f + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 20..3f + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 40..5f + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, // 60..7f + 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9, // 80..9f + 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7, // a0..bf + 8,8,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, // c0..df + 0xa,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x3,0x4,0x3,0x3, // e0..ef + 0xb,0x6,0x6,0x6,0x5,0x8,0x8,0x8,0x8,0x8,0x8,0x8,0x8,0x8,0x8,0x8, // f0..ff + 0x0,0x1,0x2,0x3,0x5,0x8,0x7,0x1,0x1,0x1,0x4,0x6,0x1,0x1,0x1,0x1, // s0..s0 + 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,0,1,0,1,1,1,1,1,1, // s1..s2 + 1,2,1,1,1,1,1,2,1,2,1,1,1,1,1,1,1,1,1,1,1,1,1,2,1,1,1,1,1,1,1,1, // s3..s4 + 1,2,1,1,1,1,1,1,1,2,1,1,1,1,1,1,1,1,1,1,1,1,1,3,1,3,1,1,1,1,1,1, // s5..s6 + 1,3,1,1,1,1,1,3,1,3,1,1,1,1,1,1,1,3,1,1,1,1,1,1,1,1,1,1,1,1,1,1, // s7..s8 +]''' + +def _from_code_point(x): + if x <= 0xFFFF: + return String.fromCharCode(x) + x -= 0x10000 + return String.fromCharCode((x >> 10) + 0xD800, (x % 0x400) + 0xDC00) + +def utf8_decode(bytes, errors, replacement): + # Convert an array of UTF-8 encoded bytes into a string + state = 0 + ans = v'[]' + + for v'var i = 0, l = bytes.length; i < l; i++': # noqa + byte = bytes[i] + typ = utf8_decoder_table[byte] + codep = (byte & 0x3f) | (codep << 6) if state is not 0 else (0xff >> typ) & (byte) + state = utf8_decoder_table[256 + state*16 + typ] + if state is 0: + ans.push(_from_code_point(codep)) + elif state is 1: + if not errors or errors is 'strict': + raise UnicodeDecodeError(str.format('The byte 0x{:02x} at position {} is not valid UTF-8', byte, i)) + elif errors is 'replace': + ans.push(replacement or '?') + return ans.join('') + +def utf8_encode_js(string): + # Encode a string as an array of UTF-8 bytes + escstr = encodeURIComponent(string) + ans = v'[]' + for v'var i = 0; i < escstr.length; i++': + ch = escstr[i] + if ch is '%': + ans.push(parseInt(escstr[i+1:i+3], 16)) + i += 2 + else: + ans.push(ch.charCodeAt(0)) + return Uint8Array(ans) + +if type(TextEncoder) is 'function': + _u8enc = TextEncoder('utf-8') + utf8_encode = _u8enc.encode.bind(_u8enc) + _u8enc = undefined +else: + utf8_encode = utf8_encode_js + +def utf8_encode_native(string): + return _u8enc.encode(string) diff --git a/src/pyj/read_book/db.pyj b/src/pyj/read_book/db.pyj index 5c70dee0e2..1369949d80 100644 --- a/src/pyj/read_book/db.pyj +++ b/src/pyj/read_book/db.pyj @@ -2,7 +2,7 @@ # License: GPL v3 Copyright: 2016, Kovid Goyal from gettext import gettext as _ -from utils import base64encode, base64decode +from encodings import base64encode, base64decode def upgrade_schema(idb, old_version, new_version): print('upgrade_schema:', old_version, new_version) diff --git a/src/pyj/utils.pyj b/src/pyj/utils.pyj index cf333b13a5..83a1d47c3d 100644 --- a/src/pyj/utils.pyj +++ b/src/pyj/utils.pyj @@ -21,48 +21,6 @@ def debounce(func, wait, immediate=False): if call_now: func.apply(context, args) -def to_utf8(string): - if type(TextEncoder) is 'function': - return TextEncoder('utf-8').encode(string) - escstr = encodeURIComponent(string) - binstr = escstr.replace(/%([0-9A-F]{2})/g, def(match, p1): - return String.fromCharCode('0x' + p1) - ) - ua = Uint8Array(binstr.length) - Array.prototype.forEach.call(binstr, def(ch, i): - ua[i] = ch.charCodeAt(0) - ) - return ua - -def base64encode(bytes): - # Convert an array of bytes into a base-64 encoded string - l = bytes.length - remainder = l % 3 - main_length = l - remainder - encodings = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/' - ans = v'[]' - for v'var i = 0; i < main_length; i += 3': - chunk = (bytes[i] << 16) | (bytes[i + 1] << 8) | bytes[i + 2] - ans.push(encodings[(chunk & 16515072) >> 18], encodings[(chunk & 258048) >> 12], encodings[(chunk & 4032) >> 6], encodings[chunk & 63]) - if remainder is 1: - chunk = bytes[main_length] - ans.push(encodings[(chunk & 252) >> 2], encodings[(chunk & 3) << 4], '=', '=') - elif remainder is 2: - chunk = (bytes[main_length] << 8) | bytes[main_length + 1] - ans.push(encodings[(chunk & 64512) >> 10], encodings[(chunk & 1008) >> 4], encodings[(chunk & 15) << 2], '=') - return ans.join('') - -def base64decode(string): - # convert the output of base64encode back into an array of bytes (Uint8Array) - if type(window) is not 'undefined': - chars = window.atob(string) - else: - chars = new Buffer(string, 'base64').toString('binary') # noqa: undef - ans = Uint8Array(chars.length) - for i in range(ans.length): - ans[i] = chars.charCodeAt(i) - return ans - def parse_url_params(url=None, allow_multiple=False): url = url or window.location.href qs = url.indexOf('?')