From e2622980905bd119a8ae2dcbc352ebefcbd8d212 Mon Sep 17 00:00:00 2001 From: Michael Manganiello Date: Sat, 6 Jan 2024 20:36:12 -0500 Subject: [PATCH] fix(server): Split database queries based on PostgreSQL bound params limit (#6034) * fix(server): Split database queries based on PostgreSQL bound params limit PostgreSQL uses a 16-bit integer to indicate the number of bound parameters. This means that the maximum number of parameters for any query is 65535. Any query that tries to bind more than that (e.g. searching by a list of IDs) requires splitting the query into multiple chunks. This change includes refactoring every Repository that runs queries using a list of ids, and either flattening or merging results. Fixes #5788, #5997. Also, potentially a fix for #4648 (at least based on [this comment](https://github.com/immich-app/immich/issues/4648#issuecomment-1826134027)). References: * https://github.com/typeorm/typeorm/issues/7565 * [PostgreSQL message format - Bind](https://www.postgresql.org/docs/15/protocol-message-formats.html#PROTOCOL-MESSAGE-FORMATS-BIND) * misc: Create Chunked decorator to simplify implementation * feat: Add ChunkedArray/ChunkedSet decorators --- server/src/domain/album/album.service.spec.ts | 2 +- server/src/domain/album/album.service.ts | 2 +- server/src/domain/domain.util.ts | 27 + .../domain/repositories/album.repository.ts | 2 +- server/src/infra/infra.util.ts | 6 + server/src/infra/infra.utils.ts | 42 ++ .../infra/repositories/access.repository.ts | 481 ++++++++++-------- .../infra/repositories/album.repository.ts | 31 +- .../infra/repositories/asset.repository.ts | 7 +- .../infra/repositories/person.repository.ts | 11 +- .../repositories/system-config.repository.ts | 2 + 11 files changed, 392 insertions(+), 221 deletions(-) diff --git a/server/src/domain/album/album.service.spec.ts b/server/src/domain/album/album.service.spec.ts index fa421342c8d71..10b6dde5e5ee0 100644 --- a/server/src/domain/album/album.service.spec.ts +++ b/server/src/domain/album/album.service.spec.ts @@ -679,7 +679,7 @@ describe(AlbumService.name, () => { ]); expect(albumMock.update).toHaveBeenCalledWith({ id: 'album-123', updatedAt: expect.any(Date) }); - expect(albumMock.removeAssets).toHaveBeenCalledWith({ assetIds: ['asset-id'], albumId: 'album-123' }); + expect(albumMock.removeAssets).toHaveBeenCalledWith('album-123', ['asset-id']); }); it('should skip assets not in the album', async () => { diff --git a/server/src/domain/album/album.service.ts b/server/src/domain/album/album.service.ts index 941fcf4c5cf14..898e3f52638ff 100644 --- a/server/src/domain/album/album.service.ts +++ b/server/src/domain/album/album.service.ts @@ -231,7 +231,7 @@ export class AlbumService { const removedIds = results.filter(({ success }) => success).map(({ id }) => id); if (removedIds.length > 0) { - await this.albumRepository.removeAssets({ albumId: id, assetIds: removedIds }); + await this.albumRepository.removeAssets(id, removedIds); await this.albumRepository.update({ id, updatedAt: new Date() }); if (album.albumThumbnailAssetId && removedIds.includes(album.albumThumbnailAssetId)) { await this.albumRepository.updateThumbnails(); diff --git a/server/src/domain/domain.util.ts b/server/src/domain/domain.util.ts index e6bc41e7b2eaf..79a1913b8ee9c 100644 --- a/server/src/domain/domain.util.ts +++ b/server/src/domain/domain.util.ts @@ -13,6 +13,7 @@ import { ValidationOptions, } from 'class-validator'; import { CronJob } from 'cron'; +import _ from 'lodash'; import { basename, extname } from 'node:path'; import sanitize from 'sanitize-filename'; @@ -175,6 +176,32 @@ export function Optional({ nullable, ...validationOptions }: OptionalOptions = { return ValidateIf((obj: any, v: any) => v !== undefined, validationOptions); } +/** + * Chunks an array or set into smaller arrays of the specified size. + * + * @param collection The collection to chunk. + * @param size The size of each chunk. + */ +export function chunks(collection: Array | Set, size: number): T[][] { + if (collection instanceof Set) { + const result = []; + let chunk = []; + for (const elem of collection) { + chunk.push(elem); + if (chunk.length === size) { + result.push(chunk); + chunk = []; + } + } + if (chunk.length > 0) { + result.push(chunk); + } + return result; + } else { + return _.chunk(collection, size); + } +} + // NOTE: The following Set utils have been added here, to easily determine where they are used. // They should be replaced with native Set operations, when they are added to the language. // Proposal reference: https://github.com/tc39/proposal-set-methods diff --git a/server/src/domain/repositories/album.repository.ts b/server/src/domain/repositories/album.repository.ts index 10b789b4b3faa..eb4d4bf3d459d 100644 --- a/server/src/domain/repositories/album.repository.ts +++ b/server/src/domain/repositories/album.repository.ts @@ -31,7 +31,7 @@ export interface IAlbumRepository { getAssetIds(albumId: string, assetIds?: string[]): Promise>; hasAsset(asset: AlbumAsset): Promise; removeAsset(assetId: string): Promise; - removeAssets(assets: AlbumAssets): Promise; + removeAssets(albumId: string, assetIds: string[]): Promise; getMetadataForIds(ids: string[]): Promise; getInvalidThumbnail(): Promise; getOwned(ownerId: string): Promise; diff --git a/server/src/infra/infra.util.ts b/server/src/infra/infra.util.ts index 35ed7c4b4ac67..8dcf6bf1ac4dc 100644 --- a/server/src/infra/infra.util.ts +++ b/server/src/infra/infra.util.ts @@ -19,3 +19,9 @@ export const DummyValue = { DATE: new Date(), TIME_BUCKET: '2024-01-01T00:00:00.000Z', }; + +// PostgreSQL uses a 16-bit integer to indicate the number of bound parameters. This means that the +// maximum number of parameters is 65535. Any query that tries to bind more than that (e.g. searching +// by a list of IDs) requires splitting the query into multiple chunks. +// We are rounding down this limit, as queries commonly include other filters and parameters. +export const DATABASE_PARAMETER_CHUNK_SIZE = 65500; diff --git a/server/src/infra/infra.utils.ts b/server/src/infra/infra.utils.ts index 6956b2fbd73f1..608f844088ea9 100644 --- a/server/src/infra/infra.utils.ts +++ b/server/src/infra/infra.utils.ts @@ -1,5 +1,8 @@ import { Paginated, PaginationOptions } from '@app/domain'; +import _ from 'lodash'; import { Between, FindOneOptions, LessThanOrEqual, MoreThanOrEqual, ObjectLiteral, Repository } from 'typeorm'; +import { chunks, setUnion } from '../domain/domain.util'; +import { DATABASE_PARAMETER_CHUNK_SIZE } from './infra.util'; /** * Allows optional values unlike the regular Between and uses MoreThanOrEqual @@ -40,3 +43,42 @@ export const isValidInteger = (value: number, options: { min?: number; max?: num const { min = Number.MIN_SAFE_INTEGER, max = Number.MAX_SAFE_INTEGER } = options; return Number.isInteger(value) && value >= min && value <= max; }; + +/** + * Wraps a method that takes a collection of parameters and sequentially calls it with chunks of the collection, + * to overcome the maximum number of parameters allowed by the database driver. + * + * @param options.paramIndex The index of the function parameter to chunk. Defaults to 0. + * @param options.flatten Whether to flatten the results. Defaults to false. + */ +export function Chunked(options: { paramIndex?: number; mergeFn?: (results: any) => any } = {}): MethodDecorator { + return (target: any, propertyKey: string | symbol, descriptor: PropertyDescriptor) => { + const originalMethod = descriptor.value; + const paramIndex = options.paramIndex ?? 0; + descriptor.value = async function (...args: any[]) { + const arg = args[paramIndex]; + + // Early return if argument length is less than or equal to the chunk size. + if ( + (arg instanceof Array && arg.length <= DATABASE_PARAMETER_CHUNK_SIZE) || + (arg instanceof Set && arg.size <= DATABASE_PARAMETER_CHUNK_SIZE) + ) { + return await originalMethod.apply(this, args); + } + + return Promise.all( + chunks(arg, DATABASE_PARAMETER_CHUNK_SIZE).map(async (chunk) => { + await originalMethod.apply(this, [...args.slice(0, paramIndex), chunk, ...args.slice(paramIndex + 1)]); + }), + ).then((results) => (options.mergeFn ? options.mergeFn(results) : results)); + }; + }; +} + +export function ChunkedArray(options?: { paramIndex?: number }): MethodDecorator { + return Chunked({ ...options, mergeFn: _.flatten }); +} + +export function ChunkedSet(options?: { paramIndex?: number }): MethodDecorator { + return Chunked({ ...options, mergeFn: setUnion }); +} diff --git a/server/src/infra/repositories/access.repository.ts b/server/src/infra/repositories/access.repository.ts index 6c5b24aaca44d..359dca3943294 100644 --- a/server/src/infra/repositories/access.repository.ts +++ b/server/src/infra/repositories/access.repository.ts @@ -1,6 +1,7 @@ import { IAccessRepository } from '@app/domain'; import { InjectRepository } from '@nestjs/typeorm'; import { Brackets, In, Repository } from 'typeorm'; +import { chunks, setUnion } from '../../domain/domain.util'; import { ActivityEntity, AlbumEntity, @@ -12,6 +13,7 @@ import { SharedLinkEntity, UserTokenEntity, } from '../entities'; +import { DATABASE_PARAMETER_CHUNK_SIZE } from '../infra.util'; export class AccessRepository implements IAccessRepository { constructor( @@ -32,15 +34,19 @@ export class AccessRepository implements IAccessRepository { return new Set(); } - return this.activityRepository - .find({ - select: { id: true }, - where: { - id: In([...activityIds]), - userId, - }, - }) - .then((activities) => new Set(activities.map((activity) => activity.id))); + return Promise.all( + chunks(activityIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) => + this.activityRepository + .find({ + select: { id: true }, + where: { + id: In(idChunk), + userId, + }, + }) + .then((activities) => new Set(activities.map((activity) => activity.id))), + ), + ).then((results) => setUnion(...results)); }, checkAlbumOwnerAccess: async (userId: string, activityIds: Set): Promise> => { @@ -48,17 +54,21 @@ export class AccessRepository implements IAccessRepository { return new Set(); } - return this.activityRepository - .find({ - select: { id: true }, - where: { - id: In([...activityIds]), - album: { - ownerId: userId, - }, - }, - }) - .then((activities) => new Set(activities.map((activity) => activity.id))); + return Promise.all( + chunks(activityIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) => + this.activityRepository + .find({ + select: { id: true }, + where: { + id: In(idChunk), + album: { + ownerId: userId, + }, + }, + }) + .then((activities) => new Set(activities.map((activity) => activity.id))), + ), + ).then((results) => setUnion(...results)); }, checkCreateAccess: async (userId: string, albumIds: Set): Promise> => { @@ -66,19 +76,23 @@ export class AccessRepository implements IAccessRepository { return new Set(); } - return this.albumRepository - .createQueryBuilder('album') - .select('album.id') - .leftJoin('album.sharedUsers', 'sharedUsers') - .where('album.id IN (:...albumIds)', { albumIds: [...albumIds] }) - .andWhere('album.isActivityEnabled = true') - .andWhere( - new Brackets((qb) => { - qb.where('album.ownerId = :userId', { userId }).orWhere('sharedUsers.id = :userId', { userId }); - }), - ) - .getMany() - .then((albums) => new Set(albums.map((album) => album.id))); + return Promise.all( + chunks(albumIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) => + this.albumRepository + .createQueryBuilder('album') + .select('album.id') + .leftJoin('album.sharedUsers', 'sharedUsers') + .where('album.id IN (:...albumIds)', { albumIds: idChunk }) + .andWhere('album.isActivityEnabled = true') + .andWhere( + new Brackets((qb) => { + qb.where('album.ownerId = :userId', { userId }).orWhere('sharedUsers.id = :userId', { userId }); + }), + ) + .getMany() + .then((albums) => new Set(albums.map((album) => album.id))), + ), + ).then((results) => setUnion(...results)); }, }; @@ -88,15 +102,19 @@ export class AccessRepository implements IAccessRepository { return new Set(); } - return this.libraryRepository - .find({ - select: { id: true }, - where: { - id: In([...libraryIds]), - ownerId: userId, - }, - }) - .then((libraries) => new Set(libraries.map((library) => library.id))); + return Promise.all( + chunks(libraryIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) => + this.libraryRepository + .find({ + select: { id: true }, + where: { + id: In(idChunk), + ownerId: userId, + }, + }) + .then((libraries) => new Set(libraries.map((library) => library.id))), + ), + ).then((results) => setUnion(...results)); }, checkPartnerAccess: async (userId: string, partnerIds: Set): Promise> => { @@ -104,13 +122,17 @@ export class AccessRepository implements IAccessRepository { return new Set(); } - return this.partnerRepository - .createQueryBuilder('partner') - .select('partner.sharedById') - .where('partner.sharedById IN (:...partnerIds)', { partnerIds: [...partnerIds] }) - .andWhere('partner.sharedWithId = :userId', { userId }) - .getMany() - .then((partners) => new Set(partners.map((partner) => partner.sharedById))); + return Promise.all( + chunks(partnerIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) => + this.partnerRepository + .createQueryBuilder('partner') + .select('partner.sharedById') + .where('partner.sharedById IN (:...partnerIds)', { partnerIds: idChunk }) + .andWhere('partner.sharedWithId = :userId', { userId }) + .getMany() + .then((partners) => new Set(partners.map((partner) => partner.sharedById))), + ), + ).then((results) => setUnion(...results)); }, }; @@ -120,13 +142,17 @@ export class AccessRepository implements IAccessRepository { return new Set(); } - return this.partnerRepository - .createQueryBuilder('partner') - .select('partner.sharedById') - .where('partner.sharedById IN (:...partnerIds)', { partnerIds: [...partnerIds] }) - .andWhere('partner.sharedWithId = :userId', { userId }) - .getMany() - .then((partners) => new Set(partners.map((partner) => partner.sharedById))); + return Promise.all( + chunks(partnerIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) => + this.partnerRepository + .createQueryBuilder('partner') + .select('partner.sharedById') + .where('partner.sharedById IN (:...partnerIds)', { partnerIds: idChunk }) + .andWhere('partner.sharedWithId = :userId', { userId }) + .getMany() + .then((partners) => new Set(partners.map((partner) => partner.sharedById))), + ), + ).then((results) => setUnion(...results)); }, }; @@ -136,33 +162,37 @@ export class AccessRepository implements IAccessRepository { return new Set(); } - return this.albumRepository - .createQueryBuilder('album') - .innerJoin('album.assets', 'asset') - .leftJoin('album.sharedUsers', 'sharedUsers') - .select('asset.id', 'assetId') - .addSelect('asset.livePhotoVideoId', 'livePhotoVideoId') - .where('array["asset"."id", "asset"."livePhotoVideoId"] && array[:...assetIds]::uuid[]', { - assetIds: [...assetIds], - }) - .andWhere( - new Brackets((qb) => { - qb.where('album.ownerId = :userId', { userId }).orWhere('sharedUsers.id = :userId', { userId }); - }), - ) - .getRawMany() - .then((rows) => { - const allowedIds = new Set(); - for (const row of rows) { - if (row.assetId && assetIds.has(row.assetId)) { - allowedIds.add(row.assetId); - } - if (row.livePhotoVideoId && assetIds.has(row.livePhotoVideoId)) { - allowedIds.add(row.livePhotoVideoId); - } - } - return allowedIds; - }); + return Promise.all( + chunks(assetIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) => + this.albumRepository + .createQueryBuilder('album') + .innerJoin('album.assets', 'asset') + .leftJoin('album.sharedUsers', 'sharedUsers') + .select('asset.id', 'assetId') + .addSelect('asset.livePhotoVideoId', 'livePhotoVideoId') + .where('array["asset"."id", "asset"."livePhotoVideoId"] && array[:...assetIds]::uuid[]', { + assetIds: idChunk, + }) + .andWhere( + new Brackets((qb) => { + qb.where('album.ownerId = :userId', { userId }).orWhere('sharedUsers.id = :userId', { userId }); + }), + ) + .getRawMany() + .then((rows) => { + const allowedIds = new Set(); + for (const row of rows) { + if (row.assetId && assetIds.has(row.assetId)) { + allowedIds.add(row.assetId); + } + if (row.livePhotoVideoId && assetIds.has(row.livePhotoVideoId)) { + allowedIds.add(row.livePhotoVideoId); + } + } + return allowedIds; + }), + ), + ).then((results) => setUnion(...results)); }, checkOwnerAccess: async (userId: string, assetIds: Set): Promise> => { @@ -170,16 +200,20 @@ export class AccessRepository implements IAccessRepository { return new Set(); } - return this.assetRepository - .find({ - select: { id: true }, - where: { - id: In([...assetIds]), - ownerId: userId, - }, - withDeleted: true, - }) - .then((assets) => new Set(assets.map((asset) => asset.id))); + return Promise.all( + chunks(assetIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) => + this.assetRepository + .find({ + select: { id: true }, + where: { + id: In(idChunk), + ownerId: userId, + }, + withDeleted: true, + }) + .then((assets) => new Set(assets.map((asset) => asset.id))), + ), + ).then((results) => setUnion(...results)); }, checkPartnerAccess: async (userId: string, assetIds: Set): Promise> => { @@ -187,15 +221,19 @@ export class AccessRepository implements IAccessRepository { return new Set(); } - return this.partnerRepository - .createQueryBuilder('partner') - .innerJoin('partner.sharedBy', 'sharedBy') - .innerJoin('sharedBy.assets', 'asset') - .select('asset.id', 'assetId') - .where('partner.sharedWithId = :userId', { userId }) - .andWhere('asset.id IN (:...assetIds)', { assetIds: [...assetIds] }) - .getRawMany() - .then((rows) => new Set(rows.map((row) => row.assetId))); + return Promise.all( + chunks(assetIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) => + this.partnerRepository + .createQueryBuilder('partner') + .innerJoin('partner.sharedBy', 'sharedBy') + .innerJoin('sharedBy.assets', 'asset') + .select('asset.id', 'assetId') + .where('partner.sharedWithId = :userId', { userId }) + .andWhere('asset.id IN (:...assetIds)', { assetIds: idChunk }) + .getRawMany() + .then((rows) => new Set(rows.map((row) => row.assetId))), + ), + ).then((results) => setUnion(...results)); }, checkSharedLinkAccess: async (sharedLinkId: string, assetIds: Set): Promise> => { @@ -203,41 +241,45 @@ export class AccessRepository implements IAccessRepository { return new Set(); } - return this.sharedLinkRepository - .createQueryBuilder('sharedLink') - .leftJoin('sharedLink.album', 'album') - .leftJoin('sharedLink.assets', 'assets') - .leftJoin('album.assets', 'albumAssets') - .select('assets.id', 'assetId') - .addSelect('albumAssets.id', 'albumAssetId') - .addSelect('assets.livePhotoVideoId', 'assetLivePhotoVideoId') - .addSelect('albumAssets.livePhotoVideoId', 'albumAssetLivePhotoVideoId') - .where('sharedLink.id = :sharedLinkId', { sharedLinkId }) - .andWhere( - 'array["assets"."id", "assets"."livePhotoVideoId", "albumAssets"."id", "albumAssets"."livePhotoVideoId"] && array[:...assetIds]::uuid[]', - { - assetIds: [...assetIds], - }, - ) - .getRawMany() - .then((rows) => { - const allowedIds = new Set(); - for (const row of rows) { - if (row.assetId && assetIds.has(row.assetId)) { - allowedIds.add(row.assetId); - } - if (row.assetLivePhotoVideoId && assetIds.has(row.assetLivePhotoVideoId)) { - allowedIds.add(row.assetLivePhotoVideoId); - } - if (row.albumAssetId && assetIds.has(row.albumAssetId)) { - allowedIds.add(row.albumAssetId); - } - if (row.albumAssetLivePhotoVideoId && assetIds.has(row.albumAssetLivePhotoVideoId)) { - allowedIds.add(row.albumAssetLivePhotoVideoId); - } - } - return allowedIds; - }); + return Promise.all( + chunks(assetIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) => + this.sharedLinkRepository + .createQueryBuilder('sharedLink') + .leftJoin('sharedLink.album', 'album') + .leftJoin('sharedLink.assets', 'assets') + .leftJoin('album.assets', 'albumAssets') + .select('assets.id', 'assetId') + .addSelect('albumAssets.id', 'albumAssetId') + .addSelect('assets.livePhotoVideoId', 'assetLivePhotoVideoId') + .addSelect('albumAssets.livePhotoVideoId', 'albumAssetLivePhotoVideoId') + .where('sharedLink.id = :sharedLinkId', { sharedLinkId }) + .andWhere( + 'array["assets"."id", "assets"."livePhotoVideoId", "albumAssets"."id", "albumAssets"."livePhotoVideoId"] && array[:...assetIds]::uuid[]', + { + assetIds: idChunk, + }, + ) + .getRawMany() + .then((rows) => { + const allowedIds = new Set(); + for (const row of rows) { + if (row.assetId && assetIds.has(row.assetId)) { + allowedIds.add(row.assetId); + } + if (row.assetLivePhotoVideoId && assetIds.has(row.assetLivePhotoVideoId)) { + allowedIds.add(row.assetLivePhotoVideoId); + } + if (row.albumAssetId && assetIds.has(row.albumAssetId)) { + allowedIds.add(row.albumAssetId); + } + if (row.albumAssetLivePhotoVideoId && assetIds.has(row.albumAssetLivePhotoVideoId)) { + allowedIds.add(row.albumAssetLivePhotoVideoId); + } + } + return allowedIds; + }), + ), + ).then((results) => setUnion(...results)); }, }; @@ -247,15 +289,19 @@ export class AccessRepository implements IAccessRepository { return new Set(); } - return this.tokenRepository - .find({ - select: { id: true }, - where: { - userId, - id: In([...deviceIds]), - }, - }) - .then((tokens) => new Set(tokens.map((token) => token.id))); + return Promise.all( + chunks(deviceIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) => + this.tokenRepository + .find({ + select: { id: true }, + where: { + userId, + id: In(idChunk), + }, + }) + .then((tokens) => new Set(tokens.map((token) => token.id))), + ), + ).then((results) => setUnion(...results)); }, }; @@ -265,15 +311,19 @@ export class AccessRepository implements IAccessRepository { return new Set(); } - return this.albumRepository - .find({ - select: { id: true }, - where: { - id: In([...albumIds]), - ownerId: userId, - }, - }) - .then((albums) => new Set(albums.map((album) => album.id))); + return Promise.all( + chunks(albumIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) => + this.albumRepository + .find({ + select: { id: true }, + where: { + id: In(idChunk), + ownerId: userId, + }, + }) + .then((albums) => new Set(albums.map((album) => album.id))), + ), + ).then((results) => setUnion(...results)); }, checkSharedAlbumAccess: async (userId: string, albumIds: Set): Promise> => { @@ -281,17 +331,21 @@ export class AccessRepository implements IAccessRepository { return new Set(); } - return this.albumRepository - .find({ - select: { id: true }, - where: { - id: In([...albumIds]), - sharedUsers: { - id: userId, - }, - }, - }) - .then((albums) => new Set(albums.map((album) => album.id))); + return Promise.all( + chunks(albumIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) => + this.albumRepository + .find({ + select: { id: true }, + where: { + id: In(idChunk), + sharedUsers: { + id: userId, + }, + }, + }) + .then((albums) => new Set(albums.map((album) => album.id))), + ), + ).then((results) => setUnion(...results)); }, checkSharedLinkAccess: async (sharedLinkId: string, albumIds: Set): Promise> => { @@ -299,18 +353,22 @@ export class AccessRepository implements IAccessRepository { return new Set(); } - return this.sharedLinkRepository - .find({ - select: { albumId: true }, - where: { - id: sharedLinkId, - albumId: In([...albumIds]), - }, - }) - .then( - (sharedLinks) => - new Set(sharedLinks.flatMap((sharedLink) => (!!sharedLink.albumId ? [sharedLink.albumId] : []))), - ); + return Promise.all( + chunks(albumIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) => + this.sharedLinkRepository + .find({ + select: { albumId: true }, + where: { + id: sharedLinkId, + albumId: In(idChunk), + }, + }) + .then( + (sharedLinks) => + new Set(sharedLinks.flatMap((sharedLink) => (!!sharedLink.albumId ? [sharedLink.albumId] : []))), + ), + ), + ).then((results) => setUnion(...results)); }, }; @@ -320,32 +378,41 @@ export class AccessRepository implements IAccessRepository { return new Set(); } - return this.personRepository - .find({ - select: { id: true }, - where: { - id: In([...personIds]), - ownerId: userId, - }, - }) - .then((persons) => new Set(persons.map((person) => person.id))); + return Promise.all( + chunks(personIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) => + this.personRepository + .find({ + select: { id: true }, + where: { + id: In(idChunk), + ownerId: userId, + }, + }) + .then((persons) => new Set(persons.map((person) => person.id))), + ), + ).then((results) => setUnion(...results)); }, checkFaceOwnerAccess: async (userId: string, assetFaceIds: Set): Promise> => { if (assetFaceIds.size === 0) { return new Set(); } - return this.assetFaceRepository - .find({ - select: { id: true }, - where: { - id: In([...assetFaceIds]), - asset: { - ownerId: userId, - }, - }, - }) - .then((faces) => new Set(faces.map((face) => face.id))); + + return Promise.all( + chunks(assetFaceIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) => + this.assetFaceRepository + .find({ + select: { id: true }, + where: { + id: In(idChunk), + asset: { + ownerId: userId, + }, + }, + }) + .then((faces) => new Set(faces.map((face) => face.id))), + ), + ).then((results) => setUnion(...results)); }, }; @@ -355,13 +422,17 @@ export class AccessRepository implements IAccessRepository { return new Set(); } - return this.partnerRepository - .createQueryBuilder('partner') - .select('partner.sharedById') - .where('partner.sharedById IN (:...partnerIds)', { partnerIds: [...partnerIds] }) - .andWhere('partner.sharedWithId = :userId', { userId }) - .getMany() - .then((partners) => new Set(partners.map((partner) => partner.sharedById))); + return Promise.all( + chunks(partnerIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) => + this.partnerRepository + .createQueryBuilder('partner') + .select('partner.sharedById') + .where('partner.sharedById IN (:...partnerIds)', { partnerIds: idChunk }) + .andWhere('partner.sharedWithId = :userId', { userId }) + .getMany() + .then((partners) => new Set(partners.map((partner) => partner.sharedById))), + ), + ).then((results) => setUnion(...results)); }, }; } diff --git a/server/src/infra/repositories/album.repository.ts b/server/src/infra/repositories/album.repository.ts index ed92bbfe154b6..aa66ba2dc8019 100644 --- a/server/src/infra/repositories/album.repository.ts +++ b/server/src/infra/repositories/album.repository.ts @@ -1,10 +1,13 @@ import { AlbumAsset, AlbumAssetCount, AlbumAssets, AlbumInfoOptions, IAlbumRepository } from '@app/domain'; import { Injectable } from '@nestjs/common'; import { InjectDataSource, InjectRepository } from '@nestjs/typeorm'; +import _ from 'lodash'; import { DataSource, FindOptionsOrder, FindOptionsRelations, In, IsNull, Not, Repository } from 'typeorm'; +import { setUnion } from '../../domain/domain.util'; import { dataSource } from '../database.config'; import { AlbumEntity, AssetEntity } from '../entities'; -import { DummyValue, GenerateSql } from '../infra.util'; +import { DATABASE_PARAMETER_CHUNK_SIZE, DummyValue, GenerateSql } from '../infra.util'; +import { Chunked, ChunkedArray } from '../infra.utils'; @Injectable() export class AlbumRepository implements IAlbumRepository { @@ -39,6 +42,7 @@ export class AlbumRepository implements IAlbumRepository { } @GenerateSql({ params: [[DummyValue.UUID]] }) + @ChunkedArray() getByIds(ids: string[]): Promise { return this.repository.find({ where: { @@ -64,6 +68,7 @@ export class AlbumRepository implements IAlbumRepository { } @GenerateSql({ params: [[DummyValue.UUID]] }) + @ChunkedArray() async getMetadataForIds(ids: string[]): Promise { // Guard against running invalid query when ids list is empty. if (!ids.length) { @@ -188,15 +193,16 @@ export class AlbumRepository implements IAlbumRepository { .execute(); } - @GenerateSql({ params: [{ albumId: DummyValue.UUID, assetIds: [DummyValue.UUID] }] }) - async removeAssets(asset: AlbumAssets): Promise { + @GenerateSql({ params: [DummyValue.UUID, [DummyValue.UUID]] }) + @Chunked({ paramIndex: 1 }) + async removeAssets(albumId: string, assetIds: string[]): Promise { await this.dataSource .createQueryBuilder() .delete() .from('albums_assets_assets') .where({ - albumsId: asset.albumId, - assetsId: In(asset.assetIds), + albumsId: albumId, + assetsId: In(assetIds), }) .execute(); } @@ -216,12 +222,19 @@ export class AlbumRepository implements IAlbumRepository { .from('albums_assets_assets', 'albums_assets') .where('"albums_assets"."albumsId" = :albumId', { albumId }); - if (assetIds?.length) { - query.andWhere('"albums_assets"."assetsId" IN (:...assetIds)', { assetIds }); + if (!assetIds?.length) { + const result = await query.getRawMany(); + return new Set(result.map((row) => row['assetId'])); } - const result = await query.getRawMany(); - return new Set(result.map((row) => row['assetId'])); + return Promise.all( + _.chunk(assetIds, DATABASE_PARAMETER_CHUNK_SIZE).map((idChunk) => + query + .andWhere('"albums_assets"."assetsId" IN (:...assetIds)', { assetIds: idChunk }) + .getRawMany() + .then((result) => new Set(result.map((row) => row['assetId']))), + ), + ).then((results) => setUnion(...results)); } @GenerateSql({ params: [{ albumId: DummyValue.UUID, assetId: DummyValue.UUID }] }) diff --git a/server/src/infra/repositories/asset.repository.ts b/server/src/infra/repositories/asset.repository.ts index 31bf554861831..f250dbeda87e8 100644 --- a/server/src/infra/repositories/asset.repository.ts +++ b/server/src/infra/repositories/asset.repository.ts @@ -27,7 +27,7 @@ import { DateTime } from 'luxon'; import { And, FindOptionsRelations, FindOptionsWhere, In, IsNull, LessThan, Not, Repository } from 'typeorm'; import { AssetEntity, AssetJobStatusEntity, AssetType, ExifEntity, SmartInfoEntity } from '../entities'; import { DummyValue, GenerateSql } from '../infra.util'; -import { OptionalBetween, paginate } from '../infra.utils'; +import { Chunked, ChunkedArray, OptionalBetween, paginate } from '../infra.utils'; const DEFAULT_SEARCH_SIZE = 250; @@ -248,6 +248,7 @@ export class AssetRepository implements IAssetRepository { } @GenerateSql({ params: [[DummyValue.UUID]] }) + @ChunkedArray() getByIds(ids: string[], relations?: FindOptionsRelations): Promise { if (!relations) { relations = { @@ -301,6 +302,7 @@ export class AssetRepository implements IAssetRepository { } @GenerateSql({ params: [[DummyValue.UUID]] }) + @ChunkedArray() getByLibraryId(libraryIds: string[]): Promise { return this.repository.find({ where: { library: { id: In(libraryIds) } }, @@ -380,14 +382,17 @@ export class AssetRepository implements IAssetRepository { } @GenerateSql({ params: [[DummyValue.UUID], { deviceId: DummyValue.STRING }] }) + @Chunked() async updateAll(ids: string[], options: Partial): Promise { await this.repository.update({ id: In(ids) }, options); } + @Chunked() async softDeleteAll(ids: string[]): Promise { await this.repository.softDelete({ id: In(ids), isExternal: false }); } + @Chunked() async restoreAll(ids: string[]): Promise { await this.repository.restore({ id: In(ids) }); } diff --git a/server/src/infra/repositories/person.repository.ts b/server/src/infra/repositories/person.repository.ts index 64fe71d1feb5e..b49859483a5bc 100644 --- a/server/src/infra/repositories/person.repository.ts +++ b/server/src/infra/repositories/person.repository.ts @@ -10,7 +10,7 @@ import { InjectRepository } from '@nestjs/typeorm'; import { In, Repository } from 'typeorm'; import { AssetEntity, AssetFaceEntity, PersonEntity } from '../entities'; import { DummyValue, GenerateSql } from '../infra.util'; -import { asVector } from '../infra.utils'; +import { Chunked, ChunkedArray, asVector } from '../infra.utils'; export class PersonRepository implements IPersonRepository { constructor( @@ -32,12 +32,16 @@ export class PersonRepository implements IPersonRepository { .getRawMany(); const assetIds = results.map(({ assetId }) => assetId); - - await this.assetFaceRepository.delete({ personId: oldPersonId, assetId: In(assetIds) }); + await this.deletePersonFromAssets(oldPersonId, assetIds); return assetIds; } + @Chunked({ paramIndex: 1 }) + async deletePersonFromAssets(personId: string, assetIds: string[]): Promise { + await this.assetFaceRepository.delete({ personId: personId, assetId: In(assetIds) }); + } + @GenerateSql({ params: [{ oldPersonId: DummyValue.UUID, newPersonId: DummyValue.UUID }] }) async reassignFaces({ oldPersonId, newPersonId }: UpdateFacesData): Promise { const result = await this.assetFaceRepository @@ -234,6 +238,7 @@ export class PersonRepository implements IPersonRepository { } @GenerateSql({ params: [[{ assetId: DummyValue.UUID, personId: DummyValue.UUID }]] }) + @ChunkedArray() async getFacesByIds(ids: AssetFaceId[]): Promise { return this.assetFaceRepository.find({ where: ids, relations: { asset: true }, withDeleted: true }); } diff --git a/server/src/infra/repositories/system-config.repository.ts b/server/src/infra/repositories/system-config.repository.ts index 57eb6c1fd4b01..4ab35b4d635da 100644 --- a/server/src/infra/repositories/system-config.repository.ts +++ b/server/src/infra/repositories/system-config.repository.ts @@ -5,6 +5,7 @@ import { readFile } from 'fs/promises'; import { In, Repository } from 'typeorm'; import { SystemConfigEntity } from '../entities'; import { DummyValue, GenerateSql } from '../infra.util'; +import { Chunked } from '../infra.utils'; export class SystemConfigRepository implements ISystemConfigRepository { constructor( @@ -29,6 +30,7 @@ export class SystemConfigRepository implements ISystemConfigRepository { } @GenerateSql({ params: [DummyValue.STRING] }) + @Chunked() async deleteKeys(keys: string[]): Promise { await this.repository.delete({ key: In(keys) }); }