fix(mobile): speed up RenderList creation for timeline (#4103)

Co-authored-by: Alex <alex.tran1502@gmail.com>

mobile/lib/modules/archive/providers/archive_asset_provider.dart

@@ -13,8 +13,9 @@ final archiveProvider = StreamProvider<RenderList>((ref) async* {
   final query = ref
       .watch(dbProvider)
       .assets
+      .where()
+      .ownerIdEqualToAnyChecksum(user.isarId)
       .filter()
-      .ownerIdEqualTo(user.isarId)
       .isArchivedEqualTo(true)
       .sortByFileCreatedAt();
   final settings = ref.watch(appSettingsServiceProvider);

mobile/lib/modules/favorite/providers/favorite_provider.dart

@@ -13,8 +13,9 @@ final favoriteAssetsProvider = StreamProvider<RenderList>((ref) async* {
   final query = ref
       .watch(dbProvider)
       .assets
+      .where()
+      .ownerIdEqualToAnyChecksum(user.isarId)
       .filter()
-      .ownerIdEqualTo(user.isarId)
       .isFavoriteEqualTo(true)
       .sortByFileCreatedAt();
   final settings = ref.watch(appSettingsServiceProvider);

mobile/lib/modules/home/ui/asset_grid/asset_grid_data_structure.dart

@@ -142,7 +142,7 @@ class RenderList {
   ) async {
     final List<RenderAssetGridElement> elements = [];
 
-    const pageSize = 500;
+    const pageSize = 50000;
     const sectionSize = 60; // divides evenly by 2,3,4,5,6
 
     if (groupBy == GroupAssetsBy.none) {

mobile/lib/shared/models/asset.dart

@@ -100,12 +100,6 @@ class Asset {
 
   /// stores the raw SHA1 bytes as a base64 String
   /// because Isar cannot sort lists of byte arrays
-  @Index(
-    unique: true,
-    replace: false,
-    type: IndexType.hash,
-    composite: [CompositeIndex("ownerId")],
-  )
   String checksum;
 
   @Index(unique: false, replace: false, type: IndexType.hash)
@@ -114,6 +108,11 @@ class Asset {
   @Index(unique: false, replace: false, type: IndexType.hash)
   String? localId;
 
+  @Index(
+    unique: true,
+    replace: false,
+    composite: [CompositeIndex("checksum", type: IndexType.hash)],
+  )
   int ownerId;
 
   DateTime fileCreatedAt;

mobile/lib/shared/models/asset.g.dart

@@ -100,24 +100,6 @@ const AssetSchema = CollectionSchema(
   deserializeProp: _assetDeserializeProp,
   idName: r'id',
   indexes: {
-    r'checksum_ownerId': IndexSchema(
-      id: 5611361749756160119,
-      name: r'checksum_ownerId',
-      unique: true,
-      replace: false,
-      properties: [
-        IndexPropertySchema(
-          name: r'checksum',
-          type: IndexType.hash,
-          caseSensitive: true,
-        ),
-        IndexPropertySchema(
-          name: r'ownerId',
-          type: IndexType.value,
-          caseSensitive: false,
-        )
-      ],
-    ),
     r'remoteId': IndexSchema(
       id: 6301175856541681032,
       name: r'remoteId',
@@ -143,6 +125,24 @@ const AssetSchema = CollectionSchema(
           caseSensitive: true,
         )
       ],
+    ),
+    r'ownerId_checksum': IndexSchema(
+      id: -3295822444433175883,
+      name: r'ownerId_checksum',
+      unique: true,
+      replace: false,
+      properties: [
+        IndexPropertySchema(
+          name: r'ownerId',
+          type: IndexType.value,
+          caseSensitive: false,
+        ),
+        IndexPropertySchema(
+          name: r'checksum',
+          type: IndexType.hash,
+          caseSensitive: true,
+        )
+      ],
     )
   },
   links: {},
@@ -302,89 +302,89 @@ void _assetAttach(IsarCollection<dynamic> col, Id id, Asset object) {
 }
 
 extension AssetByIndex on IsarCollection<Asset> {
-  Future<Asset?> getByChecksumOwnerId(String checksum, int ownerId) {
+  Future<Asset?> getByOwnerIdChecksum(int ownerId, String checksum) {
-    return getByIndex(r'checksum_ownerId', [checksum, ownerId]);
+    return getByIndex(r'ownerId_checksum', [ownerId, checksum]);
   }
 
-  Asset? getByChecksumOwnerIdSync(String checksum, int ownerId) {
+  Asset? getByOwnerIdChecksumSync(int ownerId, String checksum) {
-    return getByIndexSync(r'checksum_ownerId', [checksum, ownerId]);
+    return getByIndexSync(r'ownerId_checksum', [ownerId, checksum]);
   }
 
-  Future<bool> deleteByChecksumOwnerId(String checksum, int ownerId) {
+  Future<bool> deleteByOwnerIdChecksum(int ownerId, String checksum) {
-    return deleteByIndex(r'checksum_ownerId', [checksum, ownerId]);
+    return deleteByIndex(r'ownerId_checksum', [ownerId, checksum]);
   }
 
-  bool deleteByChecksumOwnerIdSync(String checksum, int ownerId) {
+  bool deleteByOwnerIdChecksumSync(int ownerId, String checksum) {
-    return deleteByIndexSync(r'checksum_ownerId', [checksum, ownerId]);
+    return deleteByIndexSync(r'ownerId_checksum', [ownerId, checksum]);
   }
 
-  Future<List<Asset?>> getAllByChecksumOwnerId(
+  Future<List<Asset?>> getAllByOwnerIdChecksum(
-      List<String> checksumValues, List<int> ownerIdValues) {
+      List<int> ownerIdValues, List<String> checksumValues) {
-    final len = checksumValues.length;
+    final len = ownerIdValues.length;
-    assert(ownerIdValues.length == len,
+    assert(checksumValues.length == len,
         'All index values must have the same length');
     final values = <List<dynamic>>[];
     for (var i = 0; i < len; i++) {
-      values.add([checksumValues[i], ownerIdValues[i]]);
+      values.add([ownerIdValues[i], checksumValues[i]]);
     }
 
-    return getAllByIndex(r'checksum_ownerId', values);
+    return getAllByIndex(r'ownerId_checksum', values);
   }
 
-  List<Asset?> getAllByChecksumOwnerIdSync(
+  List<Asset?> getAllByOwnerIdChecksumSync(
-      List<String> checksumValues, List<int> ownerIdValues) {
+      List<int> ownerIdValues, List<String> checksumValues) {
-    final len = checksumValues.length;
+    final len = ownerIdValues.length;
-    assert(ownerIdValues.length == len,
+    assert(checksumValues.length == len,
         'All index values must have the same length');
     final values = <List<dynamic>>[];
     for (var i = 0; i < len; i++) {
-      values.add([checksumValues[i], ownerIdValues[i]]);
+      values.add([ownerIdValues[i], checksumValues[i]]);
     }
 
-    return getAllByIndexSync(r'checksum_ownerId', values);
+    return getAllByIndexSync(r'ownerId_checksum', values);
   }
 
-  Future<int> deleteAllByChecksumOwnerId(
+  Future<int> deleteAllByOwnerIdChecksum(
-      List<String> checksumValues, List<int> ownerIdValues) {
+      List<int> ownerIdValues, List<String> checksumValues) {
-    final len = checksumValues.length;
+    final len = ownerIdValues.length;
-    assert(ownerIdValues.length == len,
+    assert(checksumValues.length == len,
         'All index values must have the same length');
     final values = <List<dynamic>>[];
     for (var i = 0; i < len; i++) {
-      values.add([checksumValues[i], ownerIdValues[i]]);
+      values.add([ownerIdValues[i], checksumValues[i]]);
     }
 
-    return deleteAllByIndex(r'checksum_ownerId', values);
+    return deleteAllByIndex(r'ownerId_checksum', values);
   }
 
-  int deleteAllByChecksumOwnerIdSync(
+  int deleteAllByOwnerIdChecksumSync(
-      List<String> checksumValues, List<int> ownerIdValues) {
+      List<int> ownerIdValues, List<String> checksumValues) {
-    final len = checksumValues.length;
+    final len = ownerIdValues.length;
-    assert(ownerIdValues.length == len,
+    assert(checksumValues.length == len,
         'All index values must have the same length');
     final values = <List<dynamic>>[];
     for (var i = 0; i < len; i++) {
-      values.add([checksumValues[i], ownerIdValues[i]]);
+      values.add([ownerIdValues[i], checksumValues[i]]);
     }
 
-    return deleteAllByIndexSync(r'checksum_ownerId', values);
+    return deleteAllByIndexSync(r'ownerId_checksum', values);
   }
 
-  Future<Id> putByChecksumOwnerId(Asset object) {
+  Future<Id> putByOwnerIdChecksum(Asset object) {
-    return putByIndex(r'checksum_ownerId', object);
+    return putByIndex(r'ownerId_checksum', object);
   }
 
-  Id putByChecksumOwnerIdSync(Asset object, {bool saveLinks = true}) {
+  Id putByOwnerIdChecksumSync(Asset object, {bool saveLinks = true}) {
-    return putByIndexSync(r'checksum_ownerId', object, saveLinks: saveLinks);
+    return putByIndexSync(r'ownerId_checksum', object, saveLinks: saveLinks);
   }
 
-  Future<List<Id>> putAllByChecksumOwnerId(List<Asset> objects) {
+  Future<List<Id>> putAllByOwnerIdChecksum(List<Asset> objects) {
-    return putAllByIndex(r'checksum_ownerId', objects);
+    return putAllByIndex(r'ownerId_checksum', objects);
   }
 
-  List<Id> putAllByChecksumOwnerIdSync(List<Asset> objects,
+  List<Id> putAllByOwnerIdChecksumSync(List<Asset> objects,
       {bool saveLinks = true}) {
-    return putAllByIndexSync(r'checksum_ownerId', objects,
+    return putAllByIndexSync(r'ownerId_checksum', objects,
         saveLinks: saveLinks);
   }
 }
@@ -463,145 +463,6 @@ extension AssetQueryWhere on QueryBuilder<Asset, Asset, QWhereClause> {
     });
   }
 
-  QueryBuilder<Asset, Asset, QAfterWhereClause> checksumEqualToAnyOwnerId(
-      String checksum) {
-    return QueryBuilder.apply(this, (query) {
-      return query.addWhereClause(IndexWhereClause.equalTo(
-        indexName: r'checksum_ownerId',
-        value: [checksum],
-      ));
-    });
-  }
-
-  QueryBuilder<Asset, Asset, QAfterWhereClause> checksumNotEqualToAnyOwnerId(
-      String checksum) {
-    return QueryBuilder.apply(this, (query) {
-      if (query.whereSort == Sort.asc) {
-        return query
-            .addWhereClause(IndexWhereClause.between(
-              indexName: r'checksum_ownerId',
-              lower: [],
-              upper: [checksum],
-              includeUpper: false,
-            ))
-            .addWhereClause(IndexWhereClause.between(
-              indexName: r'checksum_ownerId',
-              lower: [checksum],
-              includeLower: false,
-              upper: [],
-            ));
-      } else {
-        return query
-            .addWhereClause(IndexWhereClause.between(
-              indexName: r'checksum_ownerId',
-              lower: [checksum],
-              includeLower: false,
-              upper: [],
-            ))
-            .addWhereClause(IndexWhereClause.between(
-              indexName: r'checksum_ownerId',
-              lower: [],
-              upper: [checksum],
-              includeUpper: false,
-            ));
-      }
-    });
-  }
-
-  QueryBuilder<Asset, Asset, QAfterWhereClause> checksumOwnerIdEqualTo(
-      String checksum, int ownerId) {
-    return QueryBuilder.apply(this, (query) {
-      return query.addWhereClause(IndexWhereClause.equalTo(
-        indexName: r'checksum_ownerId',
-        value: [checksum, ownerId],
-      ));
-    });
-  }
-
-  QueryBuilder<Asset, Asset, QAfterWhereClause>
-      checksumEqualToOwnerIdNotEqualTo(String checksum, int ownerId) {
-    return QueryBuilder.apply(this, (query) {
-      if (query.whereSort == Sort.asc) {
-        return query
-            .addWhereClause(IndexWhereClause.between(
-              indexName: r'checksum_ownerId',
-              lower: [checksum],
-              upper: [checksum, ownerId],
-              includeUpper: false,
-            ))
-            .addWhereClause(IndexWhereClause.between(
-              indexName: r'checksum_ownerId',
-              lower: [checksum, ownerId],
-              includeLower: false,
-              upper: [checksum],
-            ));
-      } else {
-        return query
-            .addWhereClause(IndexWhereClause.between(
-              indexName: r'checksum_ownerId',
-              lower: [checksum, ownerId],
-              includeLower: false,
-              upper: [checksum],
-            ))
-            .addWhereClause(IndexWhereClause.between(
-              indexName: r'checksum_ownerId',
-              lower: [checksum],
-              upper: [checksum, ownerId],
-              includeUpper: false,
-            ));
-      }
-    });
-  }
-
-  QueryBuilder<Asset, Asset, QAfterWhereClause>
-      checksumEqualToOwnerIdGreaterThan(
-    String checksum,
-    int ownerId, {
-    bool include = false,
-  }) {
-    return QueryBuilder.apply(this, (query) {
-      return query.addWhereClause(IndexWhereClause.between(
-        indexName: r'checksum_ownerId',
-        lower: [checksum, ownerId],
-        includeLower: include,
-        upper: [checksum],
-      ));
-    });
-  }
-
-  QueryBuilder<Asset, Asset, QAfterWhereClause> checksumEqualToOwnerIdLessThan(
-    String checksum,
-    int ownerId, {
-    bool include = false,
-  }) {
-    return QueryBuilder.apply(this, (query) {
-      return query.addWhereClause(IndexWhereClause.between(
-        indexName: r'checksum_ownerId',
-        lower: [checksum],
-        upper: [checksum, ownerId],
-        includeUpper: include,
-      ));
-    });
-  }
-
-  QueryBuilder<Asset, Asset, QAfterWhereClause> checksumEqualToOwnerIdBetween(
-    String checksum,
-    int lowerOwnerId,
-    int upperOwnerId, {
-    bool includeLower = true,
-    bool includeUpper = true,
-  }) {
-    return QueryBuilder.apply(this, (query) {
-      return query.addWhereClause(IndexWhereClause.between(
-        indexName: r'checksum_ownerId',
-        lower: [checksum, lowerOwnerId],
-        includeLower: includeLower,
-        upper: [checksum, upperOwnerId],
-        includeUpper: includeUpper,
-      ));
-    });
-  }
-
   QueryBuilder<Asset, Asset, QAfterWhereClause> remoteIdIsNull() {
     return QueryBuilder.apply(this, (query) {
       return query.addWhereClause(IndexWhereClause.equalTo(
@@ -731,6 +592,141 @@ extension AssetQueryWhere on QueryBuilder<Asset, Asset, QWhereClause> {
       }
     });
   }
+
+  QueryBuilder<Asset, Asset, QAfterWhereClause> ownerIdEqualToAnyChecksum(
+      int ownerId) {
+    return QueryBuilder.apply(this, (query) {
+      return query.addWhereClause(IndexWhereClause.equalTo(
+        indexName: r'ownerId_checksum',
+        value: [ownerId],
+      ));
+    });
+  }
+
+  QueryBuilder<Asset, Asset, QAfterWhereClause> ownerIdNotEqualToAnyChecksum(
+      int ownerId) {
+    return QueryBuilder.apply(this, (query) {
+      if (query.whereSort == Sort.asc) {
+        return query
+            .addWhereClause(IndexWhereClause.between(
+              indexName: r'ownerId_checksum',
+              lower: [],
+              upper: [ownerId],
+              includeUpper: false,
+            ))
+            .addWhereClause(IndexWhereClause.between(
+              indexName: r'ownerId_checksum',
+              lower: [ownerId],
+              includeLower: false,
+              upper: [],
+            ));
+      } else {
+        return query
+            .addWhereClause(IndexWhereClause.between(
+              indexName: r'ownerId_checksum',
+              lower: [ownerId],
+              includeLower: false,
+              upper: [],
+            ))
+            .addWhereClause(IndexWhereClause.between(
+              indexName: r'ownerId_checksum',
+              lower: [],
+              upper: [ownerId],
+              includeUpper: false,
+            ));
+      }
+    });
+  }
+
+  QueryBuilder<Asset, Asset, QAfterWhereClause> ownerIdGreaterThanAnyChecksum(
+    int ownerId, {
+    bool include = false,
+  }) {
+    return QueryBuilder.apply(this, (query) {
+      return query.addWhereClause(IndexWhereClause.between(
+        indexName: r'ownerId_checksum',
+        lower: [ownerId],
+        includeLower: include,
+        upper: [],
+      ));
+    });
+  }
+
+  QueryBuilder<Asset, Asset, QAfterWhereClause> ownerIdLessThanAnyChecksum(
+    int ownerId, {
+    bool include = false,
+  }) {
+    return QueryBuilder.apply(this, (query) {
+      return query.addWhereClause(IndexWhereClause.between(
+        indexName: r'ownerId_checksum',
+        lower: [],
+        upper: [ownerId],
+        includeUpper: include,
+      ));
+    });
+  }
+
+  QueryBuilder<Asset, Asset, QAfterWhereClause> ownerIdBetweenAnyChecksum(
+    int lowerOwnerId,
+    int upperOwnerId, {
+    bool includeLower = true,
+    bool includeUpper = true,
+  }) {
+    return QueryBuilder.apply(this, (query) {
+      return query.addWhereClause(IndexWhereClause.between(
+        indexName: r'ownerId_checksum',
+        lower: [lowerOwnerId],
+        includeLower: includeLower,
+        upper: [upperOwnerId],
+        includeUpper: includeUpper,
+      ));
+    });
+  }
+
+  QueryBuilder<Asset, Asset, QAfterWhereClause> ownerIdChecksumEqualTo(
+      int ownerId, String checksum) {
+    return QueryBuilder.apply(this, (query) {
+      return query.addWhereClause(IndexWhereClause.equalTo(
+        indexName: r'ownerId_checksum',
+        value: [ownerId, checksum],
+      ));
+    });
+  }
+
+  QueryBuilder<Asset, Asset, QAfterWhereClause>
+      ownerIdEqualToChecksumNotEqualTo(int ownerId, String checksum) {
+    return QueryBuilder.apply(this, (query) {
+      if (query.whereSort == Sort.asc) {
+        return query
+            .addWhereClause(IndexWhereClause.between(
+              indexName: r'ownerId_checksum',
+              lower: [ownerId],
+              upper: [ownerId, checksum],
+              includeUpper: false,
+            ))
+            .addWhereClause(IndexWhereClause.between(
+              indexName: r'ownerId_checksum',
+              lower: [ownerId, checksum],
+              includeLower: false,
+              upper: [ownerId],
+            ));
+      } else {
+        return query
+            .addWhereClause(IndexWhereClause.between(
+              indexName: r'ownerId_checksum',
+              lower: [ownerId, checksum],
+              includeLower: false,
+              upper: [ownerId],
+            ))
+            .addWhereClause(IndexWhereClause.between(
+              indexName: r'ownerId_checksum',
+              lower: [ownerId],
+              upper: [ownerId, checksum],
+              includeUpper: false,
+            ));
+      }
+    });
+  }
 }
 
 extension AssetQueryFilter on QueryBuilder<Asset, Asset, QFilterCondition> {

mobile/lib/shared/providers/asset.provider.dart

@@ -186,8 +186,9 @@ final assetsProvider =
   final query = ref
       .watch(dbProvider)
       .assets
+      .where()
+      .ownerIdEqualToAnyChecksum(userId)
       .filter()
-      .ownerIdEqualTo(userId)
       .isArchivedEqualTo(false)
       .sortByFileCreatedAtDesc();
   final settings = ref.watch(appSettingsServiceProvider);

mobile/lib/shared/services/sync.service.dart

@@ -123,7 +123,7 @@ class SyncService {
   /// Syncs a new asset to the db. Returns `true` if successful
   Future<bool> _syncNewAssetToDb(Asset a) async {
     final Asset? inDb =
-        await _db.assets.getByChecksumOwnerId(a.checksum, a.ownerId);
+        await _db.assets.getByOwnerIdChecksum(a.ownerId, a.checksum);
     if (inDb != null) {
       // unify local/remote assets by replacing the
       // local-only asset in the DB with a local&remote asset
@@ -195,8 +195,8 @@ class SyncService {
       return false;
     }
     final List<Asset> inDb = await _db.assets
-        .filter()
+        .where()
-        .ownerIdEqualTo(user.isarId)
+        .ownerIdEqualToAnyChecksum(user.isarId)
         .sortByChecksum()
         .findAll();
     assert(inDb.isSorted(Asset.compareByChecksum), "inDb not sorted!");
@@ -638,9 +638,9 @@ class SyncService {
   ) async {
     if (assets.isEmpty) return ([].cast<Asset>(), [].cast<Asset>());
 
-    final List<Asset?> inDb = await _db.assets.getAllByChecksumOwnerId(
+    final List<Asset?> inDb = await _db.assets.getAllByOwnerIdChecksum(
-      assets.map((a) => a.checksum).toList(growable: false),
       assets.map((a) => a.ownerId).toInt64List(),
+      assets.map((a) => a.checksum).toList(growable: false),
     );
     assert(inDb.length == assets.length);
     final List<Asset> existing = [], toUpsert = [];
@@ -683,9 +683,9 @@ class SyncService {
       );
       // give details on the errors
       assets.sort(Asset.compareByOwnerChecksum);
-      final inDb = await _db.assets.getAllByChecksumOwnerId(
+      final inDb = await _db.assets.getAllByOwnerIdChecksum(
-        assets.map((e) => e.checksum).toList(growable: false),
         assets.map((e) => e.ownerId).toInt64List(),
+        assets.map((e) => e.checksum).toList(growable: false),
       );
       for (int i = 0; i < assets.length; i++) {
         final Asset a = assets[i];