Start work on storing books in IndexedDB for offline reading

src/calibre/srv/books.py

@@ -8,10 +8,11 @@ from hashlib import sha1
 from functools import partial
 from threading import RLock
 from cPickle import dumps
-import errno, os, tempfile, shutil, time
+import errno, os, tempfile, shutil, time, json as jsonlib
 
 from calibre.constants import cache_dir, iswindows
 from calibre.customize.ui import plugin_for_input_format
+from calibre.srv.metadata import book_as_json
 from calibre.srv.render_book import RENDER_VERSION
 from calibre.srv.errors import HTTPNotFound
 from calibre.srv.routes import endpoint, json
@@ -69,8 +70,9 @@ def queue_job(ctx, copy_format_to, bhash, fmt, book_id, size, mtime):
     with os.fdopen(fd, 'wb') as f:
         copy_format_to(f)
     tdir = tempfile.mkdtemp('', '', tdir)
-    job_id = ctx.start_job('Render book %s (%s)' % (book_id, fmt), 'calibre.srv.render_book', 'render', args=(pathtoebook, tdir, (size, mtime)),
-                           job_done_callback=job_done, job_data=(bhash, pathtoebook, tdir))
+    job_id = ctx.start_job('Render book %s (%s)' % (book_id, fmt), 'calibre.srv.render_book', 'render', args=(
+        pathtoebook, tdir, {'size':size, 'mtime':mtime, 'hash':bhash}),
+        job_done_callback=job_done, job_data=(bhash, pathtoebook, tdir))
     queued_jobs[bhash] = job_id
     return job_id
 
@@ -127,7 +129,10 @@ def book_manifest(ctx, rd, book_id, fmt):
             mpath = abspath(os.path.join(books_cache_dir(), 'f', bhash, 'calibre-book-manifest.json'))
             try:
                 os.utime(mpath, None)
-                return lopen(mpath, 'rb')
+                with lopen(mpath, 'rb') as f:
+                    ans = jsonlib.load(f)
+                ans['metadata'] = book_as_json(db, book_id)
+                return ans
             except EnvironmentError as e:
                 if e.errno != errno.ENOENT:
                     raise

src/calibre/srv/render_book.py

@@ -20,7 +20,7 @@ from calibre.ebooks.oeb.polish.utils import guess_type
 from calibre.utils.short_uuid import uuid4
 from calibre.utils.logging import default_log
 
-RENDER_VERSION = 1
+RENDER_VERSION = 1  # Also change this in read_book.ui.pyj
 
 def encode_component(x):
     return x.replace(',', ',c').replace('|', ',p')

src/pyj/book_list/book_details.pyj

@@ -393,7 +393,7 @@ class BookDetailsPanel:
         self.download_format(fmt)
 
     def read_format(self, fmt):
-        pass
+        get_boss().read_book(self.current_book_id, fmt, self.interface_data.metadata[self.current_book_id])
 
     def read_book(self):
         book_id = self.current_book_id

src/pyj/book_list/boss.pyj

@@ -12,6 +12,7 @@ from utils import parse_url_params
 from book_list.globals import get_session_data, set_boss, set_current_query
 from book_list.theme import get_color
 from book_list.ui import UI
+from read_book.ui import ReadUI
 
 class Boss:
 
@@ -28,6 +29,9 @@ class Boss:
         div = E.div(id='book-list-container')
         document.body.appendChild(div)
         self.ui = UI(interface_data, div)
+        div = E.div(id='read-book-container', style="display:none")
+        document.body.appendChild(div)
+        self.read_ui = ReadUI(interface_data, div)
         window.onerror = self.onerror.bind(self)
         self.history_count = 0
         data = parse_url_params()
@@ -36,10 +40,20 @@ class Boss:
         if not data.mode or data.mode is 'book_list':
             if data.panel is not self.ui.current_panel:
                 self.ui.show_panel(data.panel, push_state=False)
+        elif data.mode is 'read_book':
+            self.current_mode = data.mode
+            self.apply_mode()
+            self.read_book(int(data.book_id), data.fmt)
         setTimeout(def():
             window.onpopstate = self.onpopstate.bind(self)
         , 0)  # We do this after event loop ticks over to avoid catching popstate events that some browsers send on page load
 
+    def apply_mode(self, mode):
+        mode = mode or self.current_mode
+        divid = 'read-book-container' if mode is 'read_book' else 'book-list-container'
+        for x in ['book-list-container', 'read-book-container']:
+            document.getElementById(x).style.display = 'block' if x is divid else 'none'
+
     @property
     def has_history(self):
         return self.history_count > 0
@@ -62,14 +76,23 @@ class Boss:
     def onpopstate(self, ev):
         data = parse_url_params()
         set_current_query(data)
-        mode = data.mode or 'book_list'
+        self.current_mode = mode = data.mode or 'book_list'
         self.history_count -= 1
+        self.apply_mode()
         if mode is 'book_list':
             search = data.search or ''
             if data.panel is not self.ui.current_panel:
                 self.ui.show_panel(data.panel, push_state=False)
             if search is not self.ui.books_view.interface_data.search_result.query:
                 self.ui.books_view.change_search(search, push_state=False, panel_to_show=data.panel)
+        elif mode is 'read_book':
+            self.read_book(int(data.book_id), data.fmt)
+
+    def read_book(self, book_id, fmt, metadata):
+        self.current_mode = 'read_book'
+        self.apply_mode()
+        self.push_state(extra_query_data={'book_id':book_id, 'fmt':fmt})
+        self.read_ui.load_book(book_id, fmt, metadata)
 
     def change_books(self, data):
         data.search_result.sort = str.split(data.search_result.sort, ',')[:2].join(',')
@@ -84,20 +107,20 @@ class Boss:
 
     def push_state(self, replace=False, extra_query_data=None):
         query = {}
+        idata = self.interface_data
         if extra_query_data:
             for k in extra_query_data:
                 query[k] = extra_query_data[k]
         if self.current_mode is 'book_list':
             if self.ui.current_panel is not self.ui.ROOT_PANEL:
                 query.panel = self.ui.current_panel
+            sq = idata.search_result.query
+            if sq:
+                query.search = sq
         else:
-            query.current_mode = self.current_mode
-        idata = self.interface_data
+            query.mode = self.current_mode
         if idata.library_id is not idata.default_library:
             query.library_id = idata.library_id
-        sq = idata.search_result.query
-        if sq:
-            query.search = sq
         set_current_query(query)
         query = encode_query(query) or '?'
         if replace:

src/pyj/read_book/__init__.pyj

@@ -0,0 +1,5 @@
+# vim:fileencoding=utf-8
+# License: GPL v3 Copyright: 2016, Kovid Goyal <kovid at kovidgoyal.net>
+
+
+

src/pyj/read_book/db.pyj

@@ -0,0 +1,91 @@
+# vim:fileencoding=utf-8
+# License: GPL v3 Copyright: 2016, Kovid Goyal <kovid at kovidgoyal.net>
+
+from gettext import gettext as _
+from modals import error_dialog
+
+def upgrade_schema(idb, old_version, new_version):
+    if old_version < 2:
+        idb.createObjectStore('books', {'keyPath':'key'})
+        idb.createObjectStore('files')
+
+class DB:
+
+    DB_NAME = 'calibre-read-books-db-test'  # TODO: Remove test suffix
+
+    def __init__(self, idb, interface_data):
+        self.interface_data = interface_data
+        self.idb = idb
+
+        idb.onerror = def(event):
+            self.display_error(None, event)
+            if console.dir:
+                console.dir(event)
+            else:
+                console.log(event)
+
+        idb.onversionchange = def(event):
+            self.db.close()
+            error_dialog(_('Database upgraded!'), _(
+                'A newer version of calibre is available, please click the reload button in your browser.'))
+
+    def display_error(self, msg, event):
+        if event.already_displayed_by_calibre:
+            return
+        event.already_displayed_by_calibre = True
+        msg = msg or _(
+            'There was an error while interacting with the'
+            ' database used to store books for offline reading. Click "Show details" for more information.')
+        desc = event.target
+        if desc.error and desc.error.toString:
+            desc = desc.error.toString()
+        elif desc.errorCode:
+            desc = desc.errorCode
+        error_dialog(_('Cannot read book'), msg, desc)
+
+    def do_op(self, stores, data, error_msg, proceed, op='get', store=None):
+        store = store or stores[0]
+        if op is 'get':
+            transaction = self.idb.transaction(transaction)
+            req = transaction.objectStore(store).get(data)
+            req.onsuccess = def(event): proceed(req.result)
+        elif op is 'put':
+            transaction = self.idb.transaction(transaction, 'readwrite')
+            req = transaction.objectStore(store).put(data)
+            req.onsuccess = proceed
+        req.onerror = def(event):
+            self.display_error(error_msg, event)
+
+    def get_book(self, book_id, fmt, metadata, proceed):
+        fmt = fmt.toUpperCase()
+        key = [self.interface_data.library_id, book_id, fmt]
+        self.do_op(['books'], key, _('Failed to read from the books database'), def(result):
+            proceed(result or {
+                'key':key,
+                'is_complete':False,
+                'stores_blobs': True,
+                'book_hash':None,
+                'last_read': Date(),
+                'metadata': metadata,
+                'manifest': None,
+            })
+        )
+
+    def save_manifest(self, book, manifest, proceed):
+        book.manifest = manifest
+        book.metadata = manifest.metadata
+        v'delete manifest["metadata"]'
+        self.do_op(['books'], book, _('Failed to write to the books database'), proceed, op='put')
+
+def create_db(ui, interface_data):
+    if not window.indexedDB:
+        ui.db = _('Your browser does not support IndexedDB. Cannot read books. Consider using a modern browser, such as Firefox, Chrome or Edge.')
+        return
+    request = window.indexedDB.open(DB.DB_NAME, 2)
+    request.onerror = def(event):
+        ui.db = _('You must allow calibre to use IndexedDB storage in your browser to read books')
+    request.onsuccess = def(event):
+        ui.db = DB(event.target.result, interface_data)
+        ui.db_initialized()
+    request.onupgradeneeded = def(event):
+        upgrade_schema(event.target.result, event.oldVersion, event.newVersion)

src/pyj/read_book/ui.pyj

@@ -0,0 +1,76 @@
+# vim:fileencoding=utf-8
+# License: GPL v3 Copyright: 2016, Kovid Goyal <kovid at kovidgoyal.net>
+
+from ajax import ajax
+from gettext import gettext as _
+from modals import error_dialog
+from read_book.db import create_db
+
+RENDER_VERSION = 1  # Also change this in render_book.py
+
+class ReadUI:
+
+    def __init__(self, interface_data, container):
+        self.interface_data = interface_data
+        self.db = None
+        self.current_metadata = None
+        self.manifest_xhr = None
+        create_db(self, interface_data)
+        self.pending_load = None
+
+    def load_book(self, book_id, fmt, metadata):
+        if self.db is None:
+            self.pending_load = [book_id, fmt, metadata]
+            return
+        self.start_load(book_id, fmt, metadata)
+
+    def db_initialized(self):
+        if self.pending_load is not None:
+            pl, self.pending_load = self.pending_load, None
+            self.start_load(*pl)
+
+    def start_load(self, book_id, fmt, metadata):
+        if type(self.db) is 'string':
+            error_dialog(_('Cannot read book'), self.db)
+            return
+        metadata = metadata or self.interface_data.metadata[book_id]
+        self.current_metadata = metadata or {'title':_('Current Book')}
+        self.db.get_book(book_id, fmt, metadata, self.got_book.bind(self))
+
+    def got_book(self, book):
+        if not book.manifest or book.manifest.version != RENDER_VERSION:
+            self.get_manifest(book)
+            return
+        self.download_book(book)
+
+    def get_manifest(self, book):
+        library_id, book_id, fmt = book.key
+        if self.manifest_xhr:
+            self.manifest_xhr.abort()
+        query = {'library_id': library_id}
+        self.manifest_xhr = ajax(('book-manifest/' + encodeURIComponent(book_id) + '/' + encodeURIComponent(fmt)),
+                                 self.got_manifest.bind(self, book), query=query)
+        self.manifest_xhr.send()
+
+    def got_manifest(self, book, end_type, xhr, ev):
+        self.manifest_xhr = None
+        if end_type is 'abort':
+            return
+        if end_type is not 'load':
+            return error_dialog(_('Failed to load book manifest'), str.format(
+                _('Could not open {title} as book manifest failed to load, click "Show Details" for more information.'), title=self.current_metadata.title),
+                                xhr.error_html)
+        try:
+            manifest = JSON.parse(xhr.responseText)
+        except Exception as err:
+            return error_dialog(_('Failed to load book manifest'), str.format(
+                _('The manifest for {title} is not valid'), title=self.current_metadata.title),
+                                err.stack or err.toString())
+        if manifest.version != RENDER_VERSION:
+            return error_dialog(_('calibre upgraded!'), _(
+                'A newer version of calibre is available, please click the reload button in your browser.'))
+        self.current_metadata = manifest.metadata
+        self.db.save_manifest(book, manifest, self.download_book.bind(self, book))
+
+    def download_book(self, book):
+        pass