Fix discover home page shelf query, add indexes for libraryItems and mediaProgresses table

server/models/LibraryItem.js

@@ -448,7 +448,10 @@ module.exports = (sequelize) => {
      * @returns {object} { libraryItems:oldLibraryItem[], count:number }
      */
     static async getByFilterAndSort(library, user, options) {
+      let start = Date.now()
       const { libraryItems, count } = await libraryFilters.getFilteredLibraryItems(library, user, options)
+      Logger.debug(`Loaded ${libraryItems.length} of ${count} items for libary page in ${((Date.now() - start) / 1000).toFixed(2)}s`)
+
       return {
         libraryItems: libraryItems.map(li => {
           const oldLibraryItem = this.getOldLibraryItem(li).toJSONMinified()
@@ -692,7 +695,15 @@ module.exports = (sequelize) => {
     extraData: DataTypes.JSON
   }, {
     sequelize,
-    modelName: 'libraryItem'
+    modelName: 'libraryItem',
+    indexes: [
+      {
+        fields: ['createdAt']
+      },
+      {
+        fields: ['mediaId']
+      }
+    ]
   })
 
   const { library, libraryFolder, book, podcast } = sequelize.models

server/models/MediaProgress.js

@@ -90,7 +90,12 @@ module.exports = (sequelize) => {
     extraData: DataTypes.JSON
   }, {
     sequelize,
-    modelName: 'mediaProgress'
+    modelName: 'mediaProgress',
+    indexes: [
+      {
+        fields: ['updatedAt']
+      }
+    ]
   })
 
   const { book, podcastEpisode, user } = sequelize.models

server/utils/migrations/dbMigration.js

@@ -1620,11 +1620,12 @@ async function migrationPatch2BookSeries(ctx, offset = 0) {
  */
 module.exports.migrationPatch2 = async (ctx) => {
   const queryInterface = ctx.sequelize.getQueryInterface()
+  const libraryItemsIndexes = await queryInterface.showIndex('libraryItems')
   const feedTableDescription = await queryInterface.describeTable('feeds')
   const authorsTableDescription = await queryInterface.describeTable('authors')
   const bookAuthorsTableDescription = await queryInterface.describeTable('bookAuthors')
 
-  if (feedTableDescription?.coverPath && authorsTableDescription?.lastFirst && bookAuthorsTableDescription?.createdAt) {
+  if (feedTableDescription?.coverPath && authorsTableDescription?.lastFirst && bookAuthorsTableDescription?.createdAt && libraryItemsIndexes.some(lii => lii.name === 'library_items_created_at')) {
     Logger.info(`[dbMigration] Migration patch 2.3.3+ - columns already on model`)
     return false
   }
@@ -1633,13 +1634,29 @@ module.exports.migrationPatch2 = async (ctx) => {
   try {
     await queryInterface.sequelize.transaction(t => {
       const queries = [
-        queryInterface.addColumn('bookAuthors', 'createdAt', {
+        queryInterface.addIndex('libraryItems', {
-          type: DataTypes.DATE
+          fields: ['mediaId'],
-        }, { transaction: t }),
+          transaction: t
-        queryInterface.addColumn('bookSeries', 'createdAt', {
+        }),
-          type: DataTypes.DATE
+        queryInterface.addIndex('libraryItems', {
-        }, { transaction: t }),
+          fields: ['createdAt'],
+          transaction: t
+        }),
+        queryInterface.addIndex('mediaProgresses', {
+          fields: ['updatedAt'],
+          transaction: t
+        })
       ]
+      if (!bookAuthorsTableDescription?.createdAt) {
+        queries.push(...[
+          queryInterface.addColumn('bookAuthors', 'createdAt', {
+            type: DataTypes.DATE
+          }, { transaction: t }),
+          queryInterface.addColumn('bookSeries', 'createdAt', {
+            type: DataTypes.DATE
+          }, { transaction: t }),
+        ])
+      }
       if (!authorsTableDescription?.lastFirst) {
         queries.push(...[
           queryInterface.addColumn('authors', 'lastFirst', {
@@ -1691,11 +1708,13 @@ module.exports.migrationPatch2 = async (ctx) => {
       await migrationPatch2Series(ctx, 0)
     }
 
-    // Patch bookAuthors createdAt column
+    if (!bookAuthorsTableDescription?.createdAt) {
-    await migrationPatch2BookAuthors(ctx, 0)
+      // Patch bookAuthors createdAt column
+      await migrationPatch2BookAuthors(ctx, 0)
 
-    // Patch bookSeries createdAt column
+      // Patch bookSeries createdAt column
-    await migrationPatch2BookSeries(ctx, 0)
+      await migrationPatch2BookSeries(ctx, 0)
+    }
 
     Logger.info(`[dbMigration] Migration patch 2.3.3+ finished`)
     return true

server/utils/queries/libraryItemsBookFilters.js

@@ -261,7 +261,7 @@ module.exports = {
       }
     } else if (sortBy === 'sequence') {
       const nullDir = sortDesc ? 'DESC NULLS FIRST' : 'ASC NULLS LAST'
-      return [[Sequelize.literal(`CAST(\`series.bookSeries.sequence\` AS INTEGER) ${nullDir}`)]]
+      return [[Sequelize.literal(`CAST(\`series.bookSeries.sequence\` AS FLOAT) ${nullDir}`)]]
     } else if (sortBy === 'progress') {
       return [[Sequelize.literal('mediaProgresses.updatedAt'), dir]]
     }
@@ -299,7 +299,7 @@ module.exports = {
         }
       ],
       order: [
-        Sequelize.literal('CAST(`books.bookSeries.sequence` AS INTEGER) ASC NULLS LAST')
+        Sequelize.literal('CAST(`books.bookSeries.sequence` AS FLOAT) ASC NULLS LAST')
       ]
     })
     const bookSeriesToInclude = []
@@ -458,7 +458,7 @@ module.exports = {
       })
       if (sortBy !== 'sequence') {
         // Secondary sort by sequence
-        sortOrder.push([Sequelize.literal('CAST(`series.bookSeries.sequence` AS INTEGER) ASC NULLS LAST')])
+        sortOrder.push([Sequelize.literal('CAST(`series.bookSeries.sequence` AS FLOAT) ASC NULLS LAST')])
       }
     } else if (filterGroup === 'issues') {
       libraryItemWhere[Sequelize.Op.or] = [
@@ -670,7 +670,7 @@ module.exports = {
         separate: true,
         subQuery: false,
         order: [
-          [Sequelize.literal('CAST(sequence AS INTEGER) ASC NULLS LAST')]
+          [Sequelize.literal('CAST(sequence AS FLOAT) ASC NULLS LAST')]
         ],
         where: {
           '$book.mediaProgresses.isFinished$': {
@@ -769,11 +769,13 @@ module.exports = {
           where: userPermissionBookWhere.bookWhere
         },
         order: [
-          [Sequelize.literal('CAST(sequence AS INTEGER) ASC NULLS LAST')]
+          [Sequelize.literal('CAST(sequence AS FLOAT) ASC NULLS LAST')]
         ],
         limit: 1
       },
-      subQuery: false
+      subQuery: false,
+      limit,
+      order: Database.sequelize.random()
     })
 
     const booksFromSeriesToInclude = seriesNotStarted.map(se => se.bookSeries?.[0]?.bookId).filter(bid => bid)