From 449511cd8e5c9b9d007a0db3933602a4e64f2f5b Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 20 Mar 2016 22:53:48 +0530 Subject: [PATCH] Implement storing of rendered book in IndexedDB --- src/calibre/srv/render_book.py | 2 +- src/pyj/read_book/db.pyj | 41 +++++++++++++++++++++++++--------- src/pyj/read_book/ui.pyj | 23 +++++++++++++++---- src/pyj/utils.pyj | 19 ++++++++++++++++ 4 files changed, 69 insertions(+), 16 deletions(-) diff --git a/src/calibre/srv/render_book.py b/src/calibre/srv/render_book.py index 6316e11bf9..92dfdc8135 100644 --- a/src/calibre/srv/render_book.py +++ b/src/calibre/srv/render_book.py @@ -68,7 +68,7 @@ class Container(ContainerBase): self.virtualized_names = set() self.virtualize_resources() def manifest_data(name): - return {'size':os.path.getsize(self.name_path_map[name]), 'is_virtualized': name in self.virtualized_names} + return {'size':os.path.getsize(self.name_path_map[name]), 'is_virtualized': name in self.virtualized_names, 'mimetype':self.mime_map.get(name)} data['files'] = {name:manifest_data(name) for name in set(self.name_path_map) - excluded_names} self.commit() for name in excluded_names: diff --git a/src/pyj/read_book/db.pyj b/src/pyj/read_book/db.pyj index d1b05ff2d0..1d86a5b419 100644 --- a/src/pyj/read_book/db.pyj +++ b/src/pyj/read_book/db.pyj @@ -2,6 +2,7 @@ # License: GPL v3 Copyright: 2016, Kovid Goyal from gettext import gettext as _ +from utils import base64encode def upgrade_schema(idb, old_version, new_version): print('upgrade_schema:', old_version, new_version) @@ -10,6 +11,16 @@ def upgrade_schema(idb, old_version, new_version): if not idb.objectStoreNames.contains('files'): idb.createObjectStore('files') +def file_store_name(book, name): + return book.book_hash + ' ' + name + +def get_error_details(event): + desc = event.target + if desc.error and desc.error.toString: + desc = desc.error.toString() + elif desc.errorCode: + desc = desc.errorCode + DB_NAME = 'calibre-books-db-test' # TODO: Remove test suffix class DB: @@ -41,12 +52,7 @@ class DB: msg = msg or _( 'There was an error while interacting with the' ' database used to store books for offline reading. Click "Show details" for more information.') - desc = event.target - if desc.error and desc.error.toString: - desc = desc.error.toString() - elif desc.errorCode: - desc = desc.errorCode - self.show_error(_('Cannot read book'), msg, desc) + self.show_error(_('Cannot read book'), msg, get_error_details(event)) def do_op(self, stores, data, error_msg, proceed, op='get', store=None): store = store or stores[0] @@ -71,7 +77,7 @@ class DB: proceed(result or { 'key':key, 'is_complete':False, - 'b64_encoded_files': v'[]', + 'stored_files': {}, 'book_hash':None, 'last_read': Date(), 'metadata': metadata, @@ -83,14 +89,27 @@ class DB: book.manifest = manifest book.metadata = manifest.metadata book.book_hash = manifest.book_hash.hash - book.is_complete = False - book.b64_encoded_files = v'[]' + book.stored_files = {} book.is_complete = False v'delete manifest["metadata"]' self.do_op(['books'], book, _('Failed to write to the books database'), proceed, op='put') - def store_file(self, book, fname, xhr): - pass + def store_file(self, book, name, xhr, proceed): + store_as_text = xhr.responseType is 'text' + fname = file_store_name(book, name) + needs_encoding = not store_as_text and not self.supports_blobs + book.stored_files[fname] = {'encoded':needs_encoding, 'mimetype':book.manifest.files[name].mimetype, 'store_as_text':store_as_text} + data = xhr.response + if needs_encoding: + data = base64encode(Uint8Array(data)) + req = self.idb.transaction(['files'], 'readwrite').objectStore('files').put(data, fname) + req.onsuccess = def(event): proceed() + req.onerror = def(event): + proceed(str.format(_('Failed to store book data ({0}) with error: {1}'), name, get_error_details(event))) + + def finish_book(self, book, proceed): + book.is_complete = True + self.do_op(['books'], book, _('Failed to write to the books database'), proceed, op='put') def create_db(ui, interface_data): if not window.indexedDB: diff --git a/src/pyj/read_book/ui.pyj b/src/pyj/read_book/ui.pyj index fc502328b0..4306a07d8c 100644 --- a/src/pyj/read_book/ui.pyj +++ b/src/pyj/read_book/ui.pyj @@ -167,19 +167,31 @@ class ReadUI: pbar.setAttribute('value', x + '') progress.lastChild.textContent = str.format(_('Downloaded {0}, {1} left'), human_readable(x), human_readable(total - x)) + def on_stored(err): + files_left.discard(this) + if err: + failed_files.append([this, err]) + if len(files_left): + return + if failed_files.length: + det = [str.format('

{}

{}
', fname, err_html) for fname, err_html in failed_files].join('') + self.show_error(_('Could not download book'), _( + 'Failed to download some book data, click "Show details" for more information'), det) + return + self.db.finish_book(book, self.display_book.bind(self, book)) + def on_complete(end_type, xhr, ev): self.downloads_in_progress.remove(xhr) - files_left.discard(this) progress_track[this] = files[this].size update_progress() if end_type is 'abort': + files_left.discard(this) return if end_type is 'load': - self.db.store_file(book, fname, xhr) + self.db.store_file(book, this, xhr, on_stored.bind(this)) else: failed_files.append([this, xhr.error_html]) - if len(files_left): - return + files_left.discard(this) def on_progress(loaded): progress_track[this] = loaded @@ -187,6 +199,9 @@ class ReadUI: for fname in files_left: xhr = ajax(base_path + encodeURIComponent(fname), on_complete.bind(fname), on_progress=on_progress.bind(fname), query=query, progress_totals_needed=False) + xhr.responseType = 'text' + if not book.manifest.files[name].is_virtualized: + xhr.responseType = 'blob' if self.db.supports_blobs else 'arraybuffer' xhr.send() self.downloads_in_progress.append(xhr) diff --git a/src/pyj/utils.pyj b/src/pyj/utils.pyj index 82d391b9d2..2d2da81df7 100644 --- a/src/pyj/utils.pyj +++ b/src/pyj/utils.pyj @@ -34,6 +34,24 @@ def to_utf8(string): ) return ua +def base64encode(bytes): + # Convert an array of bytes into a base-64 encoded string + l = bytes.length + remainder = l % 3 + main_length = l - remainder + encodings = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/' + ans = v'[]' + for v'var i = 0; i < main_length; i += 3': + chunk = (bytes[i] << 16) | (bytes[i + 1] << 8) | bytes[i + 2] + ans.push(encodings[(chunk & 16515072) >> 18], encodings[(chunk & 258048) >> 12], encodings[(chunk & 4032) >> 6], encodings[chunk & 63]) + if remainder is 1: + chunk = bytes[main_length] + ans.push(encodings[(chunk & 252) >> 2], encodings[(chunk & 3) << 4], '=', '=') + elif remainder is 2: + chunk = (bytes[main_length] << 8) | bytes[main_length + 1] + ans.push(encodings[(chunk & 64512) >> 10], encodings[(chunk & 1008) >> 4], encodings[(chunk & 15) << 2], '=') + return ans.join('') + def parse_url_params(url=None, allow_multiple=False): url = url or window.location.href qs = url.indexOf('?') @@ -96,3 +114,4 @@ def human_readable(size, sep=' '): if __name__ is '__main__': print(fmt_sidx(10), fmt_sidx(1.2)) print(list(map(human_readable, [1, 1024.0, 1025, 1024*1024*2.3]))) + print(base64encode(list(range(256))))