Implement automatic syncing to last read position for the most recently read books

Note that position syncing only works with user accounts (anonymous
users do not have syncing)

src/calibre/srv/books.py

@@ -147,6 +147,8 @@ def book_manifest(ctx, rd, book_id, fmt):
                 with lopen(mpath, 'rb') as f:
                     ans = jsonlib.load(f)
                 ans['metadata'] = book_as_json(db, book_id)
+                user = rd.username or None
+                ans['last_read_positions'] = db.get_last_read_positions(book_id, fmt, user)
                 return ans
             except EnvironmentError as e:
                 if e.errno != errno.ENOENT:
@@ -216,6 +218,7 @@ def set_last_read_position(ctx, rd, library_id, book_id, fmt):
         raise HTTPNotFound('Invalid data')
     db.set_last_read_position(
         book_id, fmt, user=user, device=device, cfi=cfi or None, pos_frac=pos_frac)
+    rd.outheaders['Content-type'] = 'text/plain'
     return b''
 
 

src/pyj/book_list/home.pyj

@@ -5,14 +5,15 @@ from __python__ import bound_methods, hash_literals
 from elementmaker import E
 from gettext import gettext as _
 
+from ajax import ajax
 from book_list.cover_grid import BORDER_RADIUS
 from book_list.globals import get_db
 from book_list.router import open_book, update_window_title
 from book_list.top_bar import create_top_bar
 from book_list.ui import set_default_panel_handler, show_panel
 from dom import add_extra_css, build_rule, ensure_id
-from session import get_interface_data
+from session import get_device_uuid, get_interface_data
-from utils import conditional_timeout
+from utils import conditional_timeout, username_key
 from widgets import create_button
 
 CLASS_NAME = 'home-page'
@@ -42,6 +43,63 @@ def read_book(library_id, book_id, fmt):
     open_book(book_id, fmt, library_id)
 
 
+def get_last_read_position(last_read_positions, prev_last_read):
+    prev_epoch = prev_last_read.getTime()
+    dev = get_device_uuid()
+    newest_epoch = ans = None
+    for data in last_read_positions:
+        if data.device is not dev and data.epoch > prev_epoch:
+            if ans is None or data.epoch > newest_epoch:
+                newest_epoch = data.epoch
+                ans = data
+    return ans
+
+
+def sync_data_received(library_id, lrmap, load_type, xhr, ev):
+    if load_type is not 'load':
+        print('Failed to get book sync data')
+        return
+    data = JSON.parse(xhr.responseText)
+    for key in data:
+        prev_last_read = lrmap[key]
+        if not prev_last_read:
+            continue
+        last_read_positions = data[key]
+        new_last_read = get_last_read_position(last_read_positions, prev_last_read)
+        if not new_last_read:
+            continue
+        last_read = new Date(new_last_read.epoch * 1000)
+        cfi = new_last_read.cfi
+        if cfi:
+            db = get_db()
+            book_id, fmt = key.partition(':')[::2]
+            db.update_last_read_data_from_key(library_id, int(book_id), fmt, last_read, cfi)
+
+
+def sync_library_books(library_id, to_sync):
+    url = f'book-get-last-read-position/{library_id}/'
+    which = v'[]'
+    lrmap = {}
+    for key, last_read in to_sync:
+        library_id, book_id, fmt = key
+        fmt = fmt.upper()
+        which.push(f'{book_id}-{fmt}')
+        lrmap[f'{book_id}:{fmt}'] = last_read
+    url += which.join('_')
+    ajax(url, sync_data_received.bind(None, library_id, lrmap)).send()
+
+
+def start_sync(to_sync):
+    libraries = {}
+    for key, last_read in to_sync:
+        library_id = key[0]
+        if not libraries[library_id]:
+            libraries[library_id] = v'[]'
+        libraries[library_id].push(v'[key, last_read]')
+    for lid in libraries:
+        sync_library_books(lid, libraries[lid])
+
+
 def show_recent_stage2(books):
     container = document.getElementById(this)
     if not container or not books.length:
@@ -54,7 +112,12 @@ def show_recent_stage2(books):
     container.appendChild(E.div(style='display:flex'))
     images = container.lastChild
     db = get_db()
+    to_sync = v'[]'
+    username = get_interface_data().username
     for book in books:
+        if username and to_sync.length < 10:
+            lr = book.last_read[username_key(username)] or new Date(0)  # noqa: unused-local
+            to_sync.push(v'[book.key, lr]')
         img = E.img(
             alt=_('{} by {}').format(book.metadata.title, book.metadata.authors.join(' & '))
         )
@@ -66,6 +129,8 @@ def show_recent_stage2(books):
         ))
         if book.cover_name:
             db.get_file(book, book.cover_name, show_cover.bind(img_id))
+    if username:
+        start_sync(to_sync)
     container.appendChild(E.div(style='margin: 1rem 1rem',
         create_button(
             _('Browse all previously downloaded books…'),

src/pyj/read_book/db.pyj

@@ -1,12 +1,14 @@
 # vim:fileencoding=utf-8
 # License: GPL v3 Copyright: 2016, Kovid Goyal <kovid at kovidgoyal.net>
-from __python__ import hash_literals, bound_methods
+from __python__ import bound_methods, hash_literals
 
+from encodings import base64decode, base64encode
 from gettext import gettext as _
-from encodings import base64encode, base64decode
-from modals import error_dialog
 
 from book_list.router import is_reading_book
+from modals import error_dialog
+from session import get_interface_data
+from utils import username_key
 
 
 def upgrade_schema(idb, old_version, new_version, transaction):
@@ -22,8 +24,8 @@ def upgrade_schema(idb, old_version, new_version, transaction):
 
     # Create indices
     books_store = transaction.objectStore('books')
-    if not books_store.indexNames.contains('last_read_index'):
+    if not books_store.indexNames.contains('recent_index'):
-        books_store.createIndex('last_read_index', 'last_read')
+        books_store.createIndex('recent_index', 'recent_date')
 
 def file_store_name(book, name):
     return book.book_hash + ' ' + name
@@ -35,8 +37,8 @@ def get_error_details(event):
     elif desc.errorCode:
         desc = desc.errorCode
 
-DB_NAME = 'calibre-books-db-testing'  # TODO: Remove test suffix and change version back to 1
+DB_NAME = 'calibre-books-db-testingx3'  # TODO: Remove test suffix and change version back to 1
-DB_VERSION = 3
+DB_VERSION = 1
 
 class DB:
 
@@ -134,6 +136,7 @@ class DB:
         # The key has to be a JavaScript array as otherwise it cannot be stored
         # into indexed db, because the RapydScript list has properties that
         # refer to non-serializable objects like functions.
+        book_id = int(book_id)
         key = v'[library_id, book_id, fmt]'
         self.do_op(['books'], key, _('Failed to read from the books database'), def(result):
             proceed(result or {
@@ -141,12 +144,13 @@ class DB:
                 'is_complete':False,
                 'stored_files': {},
                 'book_hash':None,
-                'last_read': Date(),
                 'metadata': metadata,
                 'manifest': None,
                 'cover_width': None,
                 'cover_height': None,
                 'cover_name': None,
+                'recent_date': new Date(),
+                'last_read': {},
                 'last_read_position': {},
             })
         )
@@ -162,7 +166,19 @@ class DB:
         book.book_hash = manifest.book_hash.hash
         book.stored_files = {}
         book.is_complete = False
+        if get_interface_data().username:
+            newest_epoch = newest_pos = None
+            for pos in manifest.last_read_positions:
+                if newest_epoch is None or pos.epoch > newest_epoch:
+                    newest_epoch = pos.epoch
+                    newest_pos = pos.cfi
+            if newest_pos:
+                unkey = username_key(get_interface_data().username)
+                book.last_read[unkey] = new Date(newest_epoch * 1000)
+                book.last_read_position[unkey] = newest_pos
+
         v'delete manifest["metadata"]'
+        v'delete manifest["last_read_positions"]'
         self.do_op(['books'], book, _('Failed to write to the books database'), proceed, op='put')
 
     def store_file(self, book, name, xhr, proceed, is_cover):
@@ -229,9 +245,20 @@ class DB:
         self.do_op(['objects'], mathjax_info, _('Failed to write to the objects database'), proceed, op='put')
 
     def update_last_read_time(self, book):
-        book.last_read = Date()
+        unkey = username_key(get_interface_data().username)
+        now = new Date()
+        book.last_read[unkey] = book.recent_date = now
         self.do_op(['books'], book, _('Failed to write to the books database'), op='put')
 
+    def update_last_read_data_from_key(self, library_id, book_id, fmt, last_read, last_read_position):
+        unkey = username_key(get_interface_data().username)
+        self.get_book(library_id, book_id, fmt, None, def(book):
+            if book.metadata:  # book exists
+                book.last_read[unkey] = book.recent_date = last_read
+                book.last_read_position[unkey] = last_read_position
+                self.do_op(['books'], book, _('Failed to write to the books database'), op='put')
+        )
+
     def get_file(self, book, name, proceed):
         key = file_store_name(book, name)
         err = _(
@@ -267,7 +294,7 @@ class DB:
 
     def get_recently_read_books(self, proceed, limit):
         limit = limit or 3
-        c = self.idb.transaction(['books'], 'readonly').objectStore('books').index('last_read_index').openCursor(None, 'prev')
+        c = self.idb.transaction(['books'], 'readonly').objectStore('books').index('recent_index').openCursor(None, 'prev')
         books = v'[]'
         c.onerror = def(event):
             err = _('Failed to read recent books from local storage')

src/pyj/read_book/view.pyj

@@ -384,18 +384,20 @@ class View:
     def on_update_cfi(self, data):
         self.currently_showing.bookpos = data.cfi
         push_state(self.ui.url_data, replace=data.replace_history, mode=read_book_mode)
-        unkey = username_key(get_interface_data().username)
+        username = get_interface_data().username
+        unkey = username_key(username)
         if not self.book.last_read_position:
             self.book.last_read_position = {}
         self.book.last_read_position[unkey] = data.cfi
         self.ui.db.update_last_read_time(self.book)
         lrd = {'device':get_device_uuid(), 'cfi':data.cfi, 'pos_frac':data.progress_frac}
         key = self.book.key
-        ajax_send('book-set-last-read-position/{library_id}/{book_id}/{fmt}'.format(
+        if username:
-            library_id=key[0], book_id=key[1], fmt=key[2]), lrd, def(end_type, xhr, ev):
+            ajax_send('book-set-last-read-position/{library_id}/{book_id}/{fmt}'.format(
-                  if end_type is not 'load':
+                library_id=key[0], book_id=key[1], fmt=key[2]), lrd, def(end_type, xhr, ev):
-                    print('Failed to update last read position, AJAX call did not succeed')
+                    if end_type is not 'load':
-        )
+                        print('Failed to update last read position, AJAX call did not succeed')
+            )
 
     def on_update_toc_position(self, data):
         update_visible_toc_nodes(data.visible_anchors)