Implement storing of rendered book in IndexedDB

src/calibre/srv/render_book.py

@@ -68,7 +68,7 @@ class Container(ContainerBase):
         self.virtualized_names = set()
         self.virtualize_resources()
         def manifest_data(name):
-            return {'size':os.path.getsize(self.name_path_map[name]), 'is_virtualized': name in self.virtualized_names}
+            return {'size':os.path.getsize(self.name_path_map[name]), 'is_virtualized': name in self.virtualized_names, 'mimetype':self.mime_map.get(name)}
         data['files'] = {name:manifest_data(name) for name in set(self.name_path_map) - excluded_names}
         self.commit()
         for name in excluded_names:

src/pyj/read_book/db.pyj

@@ -2,6 +2,7 @@
 # License: GPL v3 Copyright: 2016, Kovid Goyal <kovid at kovidgoyal.net>
 
 from gettext import gettext as _
+from utils import base64encode
 
 def upgrade_schema(idb, old_version, new_version):
     print('upgrade_schema:', old_version, new_version)
@@ -10,6 +11,16 @@ def upgrade_schema(idb, old_version, new_version):
     if not idb.objectStoreNames.contains('files'):
         idb.createObjectStore('files')
 
+def file_store_name(book, name):
+    return book.book_hash + ' ' + name
+
+def get_error_details(event):
+    desc = event.target
+    if desc.error and desc.error.toString:
+        desc = desc.error.toString()
+    elif desc.errorCode:
+        desc = desc.errorCode
+
 DB_NAME = 'calibre-books-db-test'  # TODO: Remove test suffix
 
 class DB:
@@ -41,12 +52,7 @@ class DB:
         msg = msg or _(
             'There was an error while interacting with the'
             ' database used to store books for offline reading. Click "Show details" for more information.')
-        desc = event.target
+        self.show_error(_('Cannot read book'), msg, get_error_details(event))
-        if desc.error and desc.error.toString:
-            desc = desc.error.toString()
-        elif desc.errorCode:
-            desc = desc.errorCode
-        self.show_error(_('Cannot read book'), msg, desc)
 
     def do_op(self, stores, data, error_msg, proceed, op='get', store=None):
         store = store or stores[0]
@@ -71,7 +77,7 @@ class DB:
             proceed(result or {
                 'key':key,
                 'is_complete':False,
-                'b64_encoded_files': v'[]',
+                'stored_files': {},
                 'book_hash':None,
                 'last_read': Date(),
                 'metadata': metadata,
@@ -83,14 +89,27 @@ class DB:
         book.manifest = manifest
         book.metadata = manifest.metadata
         book.book_hash = manifest.book_hash.hash
-        book.is_complete = False
+        book.stored_files = {}
-        book.b64_encoded_files = v'[]'
         book.is_complete = False
         v'delete manifest["metadata"]'
         self.do_op(['books'], book, _('Failed to write to the books database'), proceed, op='put')
 
-    def store_file(self, book, fname, xhr):
+    def store_file(self, book, name, xhr, proceed):
-        pass
+        store_as_text = xhr.responseType is 'text'
+        fname = file_store_name(book, name)
+        needs_encoding = not store_as_text and not self.supports_blobs
+        book.stored_files[fname] = {'encoded':needs_encoding, 'mimetype':book.manifest.files[name].mimetype, 'store_as_text':store_as_text}
+        data = xhr.response
+        if needs_encoding:
+            data = base64encode(Uint8Array(data))
+        req = self.idb.transaction(['files'], 'readwrite').objectStore('files').put(data, fname)
+        req.onsuccess = def(event): proceed()
+        req.onerror = def(event):
+            proceed(str.format(_('Failed to store book data ({0}) with error: {1}'), name, get_error_details(event)))
+
+    def finish_book(self, book, proceed):
+        book.is_complete = True
+        self.do_op(['books'], book, _('Failed to write to the books database'), proceed, op='put')
 
 def create_db(ui, interface_data):
     if not window.indexedDB:

src/pyj/read_book/ui.pyj

@@ -167,19 +167,31 @@ class ReadUI:
             pbar.setAttribute('value', x + '')
             progress.lastChild.textContent = str.format(_('Downloaded {0}, {1} left'), human_readable(x), human_readable(total - x))
 
+        def on_stored(err):
+            files_left.discard(this)
+            if err:
+                failed_files.append([this, err])
+            if len(files_left):
+                return
+            if failed_files.length:
+                det = [str.format('<h4>{}</h4><div>{}</div><hr>', fname, err_html) for fname, err_html in failed_files].join('')
+                self.show_error(_('Could not download book'), _(
+                    'Failed to download some book data, click "Show details" for more information'), det)
+                return
+            self.db.finish_book(book, self.display_book.bind(self, book))
+
         def on_complete(end_type, xhr, ev):
             self.downloads_in_progress.remove(xhr)
-            files_left.discard(this)
             progress_track[this] = files[this].size
             update_progress()
             if end_type is 'abort':
+                files_left.discard(this)
                 return
             if end_type is 'load':
-                self.db.store_file(book, fname, xhr)
+                self.db.store_file(book, this, xhr, on_stored.bind(this))
             else:
                 failed_files.append([this, xhr.error_html])
-            if len(files_left):
+                files_left.discard(this)
-                return
 
         def on_progress(loaded):
             progress_track[this] = loaded
@@ -187,6 +199,9 @@ class ReadUI:
 
         for fname in files_left:
             xhr = ajax(base_path + encodeURIComponent(fname), on_complete.bind(fname), on_progress=on_progress.bind(fname), query=query, progress_totals_needed=False)
+            xhr.responseType = 'text'
+            if not book.manifest.files[name].is_virtualized:
+                xhr.responseType = 'blob' if self.db.supports_blobs else 'arraybuffer'
             xhr.send()
             self.downloads_in_progress.append(xhr)
 

src/pyj/utils.pyj

@@ -34,6 +34,24 @@ def to_utf8(string):
     )
     return ua
 
+def base64encode(bytes):
+    # Convert an array of bytes into a base-64 encoded string
+    l = bytes.length
+    remainder = l % 3
+    main_length = l - remainder
+    encodings = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/'
+    ans = v'[]'
+    for v'var i = 0; i < main_length; i += 3':
+        chunk = (bytes[i] << 16) | (bytes[i + 1] << 8) | bytes[i + 2]
+        ans.push(encodings[(chunk & 16515072) >> 18], encodings[(chunk & 258048) >> 12], encodings[(chunk & 4032) >> 6], encodings[chunk & 63])
+    if remainder is 1:
+        chunk = bytes[main_length]
+        ans.push(encodings[(chunk & 252) >> 2], encodings[(chunk & 3) << 4], '=', '=')
+    elif remainder is 2:
+        chunk = (bytes[main_length] << 8) | bytes[main_length + 1]
+        ans.push(encodings[(chunk & 64512) >> 10], encodings[(chunk & 1008) >> 4], encodings[(chunk & 15) << 2], '=')
+    return ans.join('')
+
 def parse_url_params(url=None, allow_multiple=False):
     url = url or window.location.href
     qs = url.indexOf('?')
@@ -96,3 +114,4 @@ def human_readable(size, sep=' '):
 if __name__ is '__main__':
     print(fmt_sidx(10), fmt_sidx(1.2))
     print(list(map(human_readable, [1, 1024.0, 1025, 1024*1024*2.3])))
+    print(base64encode(list(range(256))))