Cutlery/immich
1
0
Fork 1
mirror of https://github.com/immich-app/immich.git synced 2025-07-09 03:04:16 -04:00
Code Issues Packages Projects Releases Wiki Activity

refactor: sync repository (#19581)

Browse Source
This commit is contained in:
Jason Rasmussen 2025-06-27 13:47:06 -04:00 committed by GitHub
parent 6feca56da8
commit df76735f4a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
10 changed files with 968 additions and 879 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
server/src/bin/sync-sql.ts
View File

@ -15,6 +15,7 @@ import { repositories } from 'src/repositories';
import { AccessRepository } from 'src/repositories/access.repository'; import { AccessRepository } from 'src/repositories/access.repository';
import { ConfigRepository } from 'src/repositories/config.repository'; import { ConfigRepository } from 'src/repositories/config.repository';
import { LoggingRepository } from 'src/repositories/logging.repository'; import { LoggingRepository } from 'src/repositories/logging.repository';
import { SyncRepository } from 'src/repositories/sync.repository';
import { AuthService } from 'src/services/auth.service'; import { AuthService } from 'src/services/auth.service';
import { getKyselyConfig } from 'src/utils/database'; import { getKyselyConfig } from 'src/utils/database';
@ -111,7 +112,7 @@ class SqlGenerator {
data.push(...(await this.runTargets(instance, `${Repository.name}`))); data.push(...(await this.runTargets(instance, `${Repository.name}`)));
// nested repositories // nested repositories
if (Repository.name === AccessRepository.name) { if (Repository.name === AccessRepository.name || Repository.name === SyncRepository.name) {
for (const key of Object.keys(instance)) { for (const key of Object.keys(instance)) {
const subInstance = (instance as any)[key]; const subInstance = (instance as any)[key];
data.push(...(await this.runTargets(subInstance, `${Repository.name}.${key}`))); data.push(...(await this.runTargets(subInstance, `${Repository.name}.${key}`)));

15
server/src/queries/sync.checkpoint.repository.sql Normal file
View File

@ -0,0 +1,15 @@
-- NOTE: This file is auto generated by ./sql-generator
-- SyncCheckpointRepository.getAll
select
"type",
"ack"
from
"session_sync_checkpoints"
where
"sessionId" = $1
-- SyncCheckpointRepository.deleteAll
delete from "session_sync_checkpoints"
where
"sessionId" = $1

1028
server/src/queries/sync.repository.sql
View File

File diff suppressed because it is too large Load Diff

2
server/src/repositories/index.ts
View File

@ -34,6 +34,7 @@ import { SessionRepository } from 'src/repositories/session.repository';
import { SharedLinkRepository } from 'src/repositories/shared-link.repository'; import { SharedLinkRepository } from 'src/repositories/shared-link.repository';
import { StackRepository } from 'src/repositories/stack.repository'; import { StackRepository } from 'src/repositories/stack.repository';
import { StorageRepository } from 'src/repositories/storage.repository'; import { StorageRepository } from 'src/repositories/storage.repository';
import { SyncCheckpointRepository } from 'src/repositories/sync-checkpoint.repository';
import { SyncRepository } from 'src/repositories/sync.repository'; import { SyncRepository } from 'src/repositories/sync.repository';
import { SystemMetadataRepository } from 'src/repositories/system-metadata.repository'; import { SystemMetadataRepository } from 'src/repositories/system-metadata.repository';
import { TagRepository } from 'src/repositories/tag.repository'; import { TagRepository } from 'src/repositories/tag.repository';
@ -81,6 +82,7 @@ export const repositories = [
StackRepository, StackRepository,
StorageRepository, StorageRepository,
SyncRepository, SyncRepository,
SyncCheckpointRepository,
SystemMetadataRepository, SystemMetadataRepository,
TagRepository, TagRepository,
TelemetryRepository, TelemetryRepository,

41
server/src/repositories/sync-checkpoint.repository.ts Normal file
View File

@ -0,0 +1,41 @@
import { Injectable } from '@nestjs/common';
import { Insertable, Kysely } from 'kysely';
import { InjectKysely } from 'nestjs-kysely';
import { DB, SessionSyncCheckpoints } from 'src/db';
import { DummyValue, GenerateSql } from 'src/decorators';
import { SyncEntityType } from 'src/enum';
@Injectable()
export class SyncCheckpointRepository {
constructor(@InjectKysely() private db: Kysely<DB>) {}
@GenerateSql({ params: [DummyValue.UUID] })
getAll(sessionId: string) {
return this.db
.selectFrom('session_sync_checkpoints')
.select(['type', 'ack'])
.where('sessionId', '=', sessionId)
.execute();
}
upsertAll(items: Insertable<SessionSyncCheckpoints>[]) {
return this.db
.insertInto('session_sync_checkpoints')
.values(items)
.onConflict((oc) =>
oc.columns(['sessionId', 'type']).doUpdateSet((eb) => ({
ack: eb.ref('excluded.ack'),
})),
)
.execute();
}
@GenerateSql({ params: [DummyValue.UUID] })
deleteAll(sessionId: string, types?: SyncEntityType[]) {
return this.db
.deleteFrom('session_sync_checkpoints')
.where('sessionId', '=', sessionId)
.$if(!!types, (qb) => qb.where('type', 'in', types!))
.execute();
}
}

664
server/src/repositories/sync.repository.ts
View File

@ -1,10 +1,9 @@
import { Injectable } from '@nestjs/common'; import { Injectable } from '@nestjs/common';
import { Insertable, Kysely, SelectQueryBuilder, sql } from 'kysely'; import { Kysely, SelectQueryBuilder, sql } from 'kysely';
import { InjectKysely } from 'nestjs-kysely'; import { InjectKysely } from 'nestjs-kysely';
import { columns } from 'src/database'; import { columns } from 'src/database';
import { DB, SessionSyncCheckpoints } from 'src/db'; import { DB } from 'src/db';
import { DummyValue, GenerateSql } from 'src/decorators'; import { DummyValue, GenerateSql } from 'src/decorators';
import { SyncEntityType } from 'src/enum';
import { SyncAck } from 'src/types'; import { SyncAck } from 'src/types';
type AuditTables = type AuditTables =
@ -28,194 +27,78 @@ type UpsertTables =
@Injectable() @Injectable()
export class SyncRepository { export class SyncRepository {
constructor(@InjectKysely() private db: Kysely<DB>) {} album: AlbumSync;
albumAsset: AlbumAssetSync;
albumAssetExif: AlbumAssetExifSync;
albumToAsset: AlbumToAssetSync;
albumUser: AlbumUserSync;
asset: AssetSync;
assetExif: AssetExifSync;
memory: MemorySync;
memoryToAsset: MemoryToAssetSync;
partner: PartnerSync;
partnerAsset: PartnerAssetsSync;
partnerAssetExif: PartnerAssetExifsSync;
user: UserSync;
@GenerateSql({ params: [DummyValue.UUID] }) constructor(@InjectKysely() private db: Kysely<DB>) {
getCheckpoints(sessionId: string) { this.album = new AlbumSync(this.db);
return this.db this.albumAsset = new AlbumAssetSync(this.db);
.selectFrom('session_sync_checkpoints') this.albumAssetExif = new AlbumAssetExifSync(this.db);
.select(['type', 'ack']) this.albumToAsset = new AlbumToAssetSync(this.db);
.where('sessionId', '=', sessionId) this.albumUser = new AlbumUserSync(this.db);
.execute(); this.asset = new AssetSync(this.db);
this.assetExif = new AssetExifSync(this.db);
this.memory = new MemorySync(this.db);
this.memoryToAsset = new MemoryToAssetSync(this.db);
this.partner = new PartnerSync(this.db);
this.partnerAsset = new PartnerAssetsSync(this.db);
this.partnerAssetExif = new PartnerAssetExifsSync(this.db);
this.user = new UserSync(this.db);
}
}
class BaseSync {
constructor(protected db: Kysely<DB>) {}
protected auditTableFilters<T extends keyof Pick<DB, AuditTables>, D>(
qb: SelectQueryBuilder<DB, T, D>,
ack?: SyncAck,
) {
const builder = qb as SelectQueryBuilder<DB, AuditTables, D>;
return builder
.where('deletedAt', '<', sql.raw<Date>("now() - interval '1 millisecond'"))
.$if(!!ack, (qb) => qb.where('id', '>', ack!.updateId))
.orderBy('id', 'asc') as SelectQueryBuilder<DB, T, D>;
} }
upsertCheckpoints(items: Insertable<SessionSyncCheckpoints>[]) { protected upsertTableFilters<T extends keyof Pick<DB, UpsertTables>, D>(
return this.db qb: SelectQueryBuilder<DB, T, D>,
.insertInto('session_sync_checkpoints') ack?: SyncAck,
.values(items) ) {
.onConflict((oc) => const builder = qb as SelectQueryBuilder<DB, UpsertTables, D>;
oc.columns(['sessionId', 'type']).doUpdateSet((eb) => ({ return builder
ack: eb.ref('excluded.ack'), .where('updatedAt', '<', sql.raw<Date>("now() - interval '1 millisecond'"))
})), .$if(!!ack, (qb) => qb.where('updateId', '>', ack!.updateId))
) .orderBy('updateId', 'asc') as SelectQueryBuilder<DB, T, D>;
.execute();
}
@GenerateSql({ params: [DummyValue.UUID] })
deleteCheckpoints(sessionId: string, types?: SyncEntityType[]) {
return this.db
.deleteFrom('session_sync_checkpoints')
.where('sessionId', '=', sessionId)
.$if(!!types, (qb) => qb.where('type', 'in', types!))
.execute();
}
@GenerateSql({ params: [], stream: true })
getUserUpserts(ack?: SyncAck) {
return this.db
.selectFrom('users')
.select(['id', 'name', 'email', 'deletedAt', 'updateId'])
.$call((qb) => this.upsertTableFilters(qb, ack))
.stream();
}
@GenerateSql({ params: [], stream: true })
getUserDeletes(ack?: SyncAck) {
return this.db
.selectFrom('users_audit')
.select(['id', 'userId'])
.$call((qb) => this.auditTableFilters(qb, ack))
.stream();
}
@GenerateSql({ params: [DummyValue.UUID], stream: true })
getPartnerUpserts(userId: string, ack?: SyncAck) {
return this.db
.selectFrom('partners')
.select(['sharedById', 'sharedWithId', 'inTimeline', 'updateId'])
.where((eb) => eb.or([eb('sharedById', '=', userId), eb('sharedWithId', '=', userId)]))
.$call((qb) => this.upsertTableFilters(qb, ack))
.stream();
}
@GenerateSql({ params: [DummyValue.UUID], stream: true })
getPartnerDeletes(userId: string, ack?: SyncAck) {
return this.db
.selectFrom('partners_audit')
.select(['id', 'sharedById', 'sharedWithId'])
.where((eb) => eb.or([eb('sharedById', '=', userId), eb('sharedWithId', '=', userId)]))
.$call((qb) => this.auditTableFilters(qb, ack))
.stream();
}
@GenerateSql({ params: [DummyValue.UUID], stream: true })
getAssetUpserts(userId: string, ack?: SyncAck) {
return this.db
.selectFrom('assets')
.select(columns.syncAsset)
.select('assets.updateId')
.where('ownerId', '=', userId)
.$call((qb) => this.upsertTableFilters(qb, ack))
.stream();
} }
}
class AlbumSync extends BaseSync {
@GenerateSql({ params: [DummyValue.UUID, DummyValue.UUID] }) @GenerateSql({ params: [DummyValue.UUID, DummyValue.UUID] })
getPartnerBackfill(userId: string, afterCreateId?: string) { getCreatedAfter(userId: string, afterCreateId?: string) {
return this.db return this.db
.selectFrom('partners') .selectFrom('albums_shared_users_users')
.select(['sharedById', 'createId']) .select(['albumsId as id', 'createId'])
.where('sharedWithId', '=', userId) .where('usersId', '=', userId)
.$if(!!afterCreateId, (qb) => qb.where('createId', '>=', afterCreateId!)) .$if(!!afterCreateId, (qb) => qb.where('createId', '>=', afterCreateId!))
.where('createdAt', '<', sql.raw<Date>("now() - interval '1 millisecond'")) .where('createdAt', '<', sql.raw<Date>("now() - interval '1 millisecond'"))
.orderBy('partners.createId', 'asc') .orderBy('createId', 'asc')
.execute(); .execute();
} }
@GenerateSql({ params: [DummyValue.UUID, DummyValue.UUID, DummyValue.UUID], stream: true })
getPartnerAssetsBackfill(partnerId: string, afterUpdateId: string | undefined, beforeUpdateId: string) {
return this.db
.selectFrom('assets')
.select(columns.syncAsset)
.select('assets.updateId')
.where('ownerId', '=', partnerId)
.where('updatedAt', '<', sql.raw<Date>("now() - interval '1 millisecond'"))
.where('updateId', '<=', beforeUpdateId)
.$if(!!afterUpdateId, (eb) => eb.where('updateId', '>=', afterUpdateId!))
.orderBy('updateId', 'asc')
.stream();
}
@GenerateSql({ params: [DummyValue.UUID], stream: true }) @GenerateSql({ params: [DummyValue.UUID], stream: true })
getPartnerAssetsUpserts(userId: string, ack?: SyncAck) { getDeletes(userId: string, ack?: SyncAck) {
return this.db
.selectFrom('assets')
.select(columns.syncAsset)
.select('assets.updateId')
.where('ownerId', 'in', (eb) =>
eb.selectFrom('partners').select(['sharedById']).where('sharedWithId', '=', userId),
)
.$call((qb) => this.upsertTableFilters(qb, ack))
.stream();
}
@GenerateSql({ params: [DummyValue.UUID], stream: true })
getAssetDeletes(userId: string, ack?: SyncAck) {
return this.db
.selectFrom('assets_audit')
.select(['id', 'assetId'])
.where('ownerId', '=', userId)
.$call((qb) => this.auditTableFilters(qb, ack))
.stream();
}
@GenerateSql({ params: [DummyValue.UUID], stream: true })
getPartnerAssetDeletes(userId: string, ack?: SyncAck) {
return this.db
.selectFrom('assets_audit')
.select(['id', 'assetId'])
.where('ownerId', 'in', (eb) =>
eb.selectFrom('partners').select(['sharedById']).where('sharedWithId', '=', userId),
)
.$call((qb) => this.auditTableFilters(qb, ack))
.stream();
}
@GenerateSql({ params: [DummyValue.UUID], stream: true })
getAssetExifsUpserts(userId: string, ack?: SyncAck) {
return this.db
.selectFrom('exif')
.select(columns.syncAssetExif)
.select('exif.updateId')
.where('assetId', 'in', (eb) => eb.selectFrom('assets').select('id').where('ownerId', '=', userId))
.$call((qb) => this.upsertTableFilters(qb, ack))
.stream();
}
@GenerateSql({ params: [DummyValue.UUID, DummyValue.UUID, DummyValue.UUID], stream: true })
getPartnerAssetExifsBackfill(partnerId: string, afterUpdateId: string | undefined, beforeUpdateId: string) {
return this.db
.selectFrom('exif')
.select(columns.syncAssetExif)
.select('exif.updateId')
.innerJoin('assets', 'assets.id', 'exif.assetId')
.where('assets.ownerId', '=', partnerId)
.where('exif.updatedAt', '<', sql.raw<Date>("now() - interval '1 millisecond'"))
.where('exif.updateId', '<=', beforeUpdateId)
.$if(!!afterUpdateId, (eb) => eb.where('exif.updateId', '>=', afterUpdateId!))
.orderBy('exif.updateId', 'asc')
.stream();
}
@GenerateSql({ params: [DummyValue.UUID], stream: true })
getPartnerAssetExifsUpserts(userId: string, ack?: SyncAck) {
return this.db
.selectFrom('exif')
.select(columns.syncAssetExif)
.select('exif.updateId')
.where('assetId', 'in', (eb) =>
eb
.selectFrom('assets')
.select('id')
.where('ownerId', 'in', (eb) =>
eb.selectFrom('partners').select(['sharedById']).where('sharedWithId', '=', userId),
),
)
.$call((qb) => this.upsertTableFilters(qb, ack))
.stream();
}
@GenerateSql({ params: [DummyValue.UUID], stream: true })
getAlbumDeletes(userId: string, ack?: SyncAck) {
return this.db return this.db
.selectFrom('albums_audit') .selectFrom('albums_audit')
.select(['id', 'albumId']) .select(['id', 'albumId'])
@ -225,7 +108,7 @@ export class SyncRepository {
} }
@GenerateSql({ params: [DummyValue.UUID], stream: true }) @GenerateSql({ params: [DummyValue.UUID], stream: true })
getAlbumUpserts(userId: string, ack?: SyncAck) { getUpserts(userId: string, ack?: SyncAck) {
return this.db return this.db
.selectFrom('albums') .selectFrom('albums')
.distinctOn(['albums.id', 'albums.updateId']) .distinctOn(['albums.id', 'albums.updateId'])
@ -248,9 +131,90 @@ export class SyncRepository {
]) ])
.stream(); .stream();
} }
}
class AlbumAssetSync extends BaseSync {
@GenerateSql({ params: [DummyValue.UUID, DummyValue.UUID, DummyValue.UUID], stream: true })
getBackfill(albumId: string, afterUpdateId: string | undefined, beforeUpdateId: string) {
return this.db
.selectFrom('assets')
.innerJoin('albums_assets_assets as album_assets', 'album_assets.assetsId', 'assets.id')
.select(columns.syncAsset)
.select('assets.updateId')
.where('album_assets.albumsId', '=', albumId)
.where('assets.updatedAt', '<', sql.raw<Date>("now() - interval '1 millisecond'"))
.where('assets.updateId', '<=', beforeUpdateId)
.$if(!!afterUpdateId, (eb) => eb.where('assets.updateId', '>=', afterUpdateId!))
.orderBy('assets.updateId', 'asc')
.stream();
}
@GenerateSql({ params: [DummyValue.UUID], stream: true }) @GenerateSql({ params: [DummyValue.UUID], stream: true })
getAlbumToAssetDeletes(userId: string, ack?: SyncAck) { getUpserts(userId: string, ack?: SyncAck) {
return this.db
.selectFrom('assets')
.innerJoin('albums_assets_assets as album_assets', 'album_assets.assetsId', 'assets.id')
.select(columns.syncAsset)
.select('assets.updateId')
.where('assets.updatedAt', '<', sql.raw<Date>("now() - interval '1 millisecond'"))
.$if(!!ack, (qb) => qb.where('assets.updateId', '>', ack!.updateId))
.orderBy('assets.updateId', 'asc')
.innerJoin('albums', 'albums.id', 'album_assets.albumsId')
.leftJoin('albums_shared_users_users as album_users', 'album_users.albumsId', 'album_assets.albumsId')
.where((eb) => eb.or([eb('albums.ownerId', '=', userId), eb('album_users.usersId', '=', userId)]))
.stream();
}
}
class AlbumAssetExifSync extends BaseSync {
@GenerateSql({ params: [DummyValue.UUID, DummyValue.UUID, DummyValue.UUID], stream: true })
getBackfill(albumId: string, afterUpdateId: string | undefined, beforeUpdateId: string) {
return this.db
.selectFrom('exif')
.innerJoin('albums_assets_assets as album_assets', 'album_assets.assetsId', 'exif.assetId')
.select(columns.syncAssetExif)
.select('exif.updateId')
.where('album_assets.albumsId', '=', albumId)
.where('exif.updatedAt', '<', sql.raw<Date>("now() - interval '1 millisecond'"))
.where('exif.updateId', '<=', beforeUpdateId)
.$if(!!afterUpdateId, (eb) => eb.where('exif.updateId', '>=', afterUpdateId!))
.orderBy('exif.updateId', 'asc')
.stream();
}
@GenerateSql({ params: [DummyValue.UUID], stream: true })
getUpserts(userId: string, ack?: SyncAck) {
return this.db
.selectFrom('exif')
.innerJoin('albums_assets_assets as album_assets', 'album_assets.assetsId', 'exif.assetId')
.select(columns.syncAssetExif)
.select('exif.updateId')
.where('exif.updatedAt', '<', sql.raw<Date>("now() - interval '1 millisecond'"))
.$if(!!ack, (qb) => qb.where('exif.updateId', '>', ack!.updateId))
.orderBy('exif.updateId', 'asc')
.innerJoin('albums', 'albums.id', 'album_assets.albumsId')
.leftJoin('albums_shared_users_users as album_users', 'album_users.albumsId', 'album_assets.albumsId')
.where((eb) => eb.or([eb('albums.ownerId', '=', userId), eb('album_users.usersId', '=', userId)]))
.stream();
}
}
class AlbumToAssetSync extends BaseSync {
@GenerateSql({ params: [DummyValue.UUID, DummyValue.UUID, DummyValue.UUID], stream: true })
getBackfill(albumId: string, afterUpdateId: string | undefined, beforeUpdateId: string) {
return this.db
.selectFrom('albums_assets_assets as album_assets')
.select(['album_assets.assetsId as assetId', 'album_assets.albumsId as albumId', 'album_assets.updateId'])
.where('album_assets.albumsId', '=', albumId)
.where('album_assets.updatedAt', '<', sql.raw<Date>("now() - interval '1 millisecond'"))
.where('album_assets.updateId', '<=', beforeUpdateId)
.$if(!!afterUpdateId, (eb) => eb.where('album_assets.updateId', '>=', afterUpdateId!))
.orderBy('album_assets.updateId', 'asc')
.stream();
}
@GenerateSql({ params: [DummyValue.UUID], stream: true })
getDeletes(userId: string, ack?: SyncAck) {
return this.db return this.db
.selectFrom('album_assets_audit') .selectFrom('album_assets_audit')
.select(['id', 'assetId', 'albumId']) .select(['id', 'assetId', 'albumId'])
@ -277,7 +241,37 @@ export class SyncRepository {
} }
@GenerateSql({ params: [DummyValue.UUID], stream: true }) @GenerateSql({ params: [DummyValue.UUID], stream: true })
getAlbumUserDeletes(userId: string, ack?: SyncAck) { getUpserts(userId: string, ack?: SyncAck) {
return this.db
.selectFrom('albums_assets_assets as album_assets')
.select(['album_assets.assetsId as assetId', 'album_assets.albumsId as albumId', 'album_assets.updateId'])
.where('album_assets.updatedAt', '<', sql.raw<Date>("now() - interval '1 millisecond'"))
.$if(!!ack, (qb) => qb.where('album_assets.updateId', '>', ack!.updateId))
.orderBy('album_assets.updateId', 'asc')
.innerJoin('albums', 'albums.id', 'album_assets.albumsId')
.leftJoin('albums_shared_users_users as album_users', 'album_users.albumsId', 'album_assets.albumsId')
.where((eb) => eb.or([eb('albums.ownerId', '=', userId), eb('album_users.usersId', '=', userId)]))
.stream();
}
}
class AlbumUserSync extends BaseSync {
@GenerateSql({ params: [DummyValue.UUID, DummyValue.UUID, DummyValue.UUID, DummyValue.UUID], stream: true })
getBackfill(albumId: string, afterUpdateId: string | undefined, beforeUpdateId: string) {
return this.db
.selectFrom('albums_shared_users_users as album_users')
.select(columns.syncAlbumUser)
.select('album_users.updateId')
.where('albumsId', '=', albumId)
.where('updatedAt', '<', sql.raw<Date>("now() - interval '1 millisecond'"))
.where('updateId', '<=', beforeUpdateId)
.$if(!!afterUpdateId, (eb) => eb.where('updateId', '>=', afterUpdateId!))
.orderBy('updateId', 'asc')
.stream();
}
@GenerateSql({ params: [DummyValue.UUID], stream: true })
getDeletes(userId: string, ack?: SyncAck) {
return this.db return this.db
.selectFrom('album_users_audit') .selectFrom('album_users_audit')
.select(['id', 'userId', 'albumId']) .select(['id', 'userId', 'albumId'])
@ -303,34 +297,8 @@ export class SyncRepository {
.stream(); .stream();
} }
@GenerateSql({ params: [DummyValue.UUID, DummyValue.UUID] })
getAlbumBackfill(userId: string, afterCreateId?: string) {
return this.db
.selectFrom('albums_shared_users_users')
.select(['albumsId as id', 'createId'])
.where('usersId', '=', userId)
.$if(!!afterCreateId, (qb) => qb.where('createId', '>=', afterCreateId!))
.where('createdAt', '<', sql.raw<Date>("now() - interval '1 millisecond'"))
.orderBy('createId', 'asc')
.execute();
}
@GenerateSql({ params: [DummyValue.UUID, DummyValue.UUID, DummyValue.UUID, DummyValue.UUID], stream: true })
getAlbumUsersBackfill(albumId: string, afterUpdateId: string | undefined, beforeUpdateId: string) {
return this.db
.selectFrom('albums_shared_users_users as album_users')
.select(columns.syncAlbumUser)
.select('album_users.updateId')
.where('albumsId', '=', albumId)
.where('updatedAt', '<', sql.raw<Date>("now() - interval '1 millisecond'"))
.where('updateId', '<=', beforeUpdateId)
.$if(!!afterUpdateId, (eb) => eb.where('updateId', '>=', afterUpdateId!))
.orderBy('updateId', 'asc')
.stream();
}
@GenerateSql({ params: [DummyValue.UUID], stream: true }) @GenerateSql({ params: [DummyValue.UUID], stream: true })
getAlbumUserUpserts(userId: string, ack?: SyncAck) { getUpserts(userId: string, ack?: SyncAck) {
return this.db return this.db
.selectFrom('albums_shared_users_users as album_users') .selectFrom('albums_shared_users_users as album_users')
.select(columns.syncAlbumUser) .select(columns.syncAlbumUser)
@ -358,98 +326,57 @@ export class SyncRepository {
) )
.stream(); .stream();
} }
}
@GenerateSql({ params: [DummyValue.UUID, DummyValue.UUID, DummyValue.UUID], stream: true }) class AssetSync extends BaseSync {
getAlbumAssetsBackfill(albumId: string, afterUpdateId: string | undefined, beforeUpdateId: string) { @GenerateSql({ params: [DummyValue.UUID], stream: true })
getDeletes(userId: string, ack?: SyncAck) {
return this.db
.selectFrom('assets_audit')
.select(['id', 'assetId'])
.where('ownerId', '=', userId)
.$call((qb) => this.auditTableFilters(qb, ack))
.stream();
}
@GenerateSql({ params: [DummyValue.UUID], stream: true })
getUpserts(userId: string, ack?: SyncAck) {
return this.db return this.db
.selectFrom('assets') .selectFrom('assets')
.innerJoin('albums_assets_assets as album_assets', 'album_assets.assetsId', 'assets.id')
.select(columns.syncAsset) .select(columns.syncAsset)
.select('assets.updateId') .select('assets.updateId')
.where('album_assets.albumsId', '=', albumId) .where('ownerId', '=', userId)
.where('assets.updatedAt', '<', sql.raw<Date>("now() - interval '1 millisecond'")) .$call((qb) => this.upsertTableFilters(qb, ack))
.where('assets.updateId', '<=', beforeUpdateId)
.$if(!!afterUpdateId, (eb) => eb.where('assets.updateId', '>=', afterUpdateId!))
.orderBy('assets.updateId', 'asc')
.stream(); .stream();
} }
}
class AssetExifSync extends BaseSync {
@GenerateSql({ params: [DummyValue.UUID], stream: true }) @GenerateSql({ params: [DummyValue.UUID], stream: true })
getAlbumAssetsUpserts(userId: string, ack?: SyncAck) { getUpserts(userId: string, ack?: SyncAck) {
return this.db
.selectFrom('assets')
.innerJoin('albums_assets_assets as album_assets', 'album_assets.assetsId', 'assets.id')
.select(columns.syncAsset)
.select('assets.updateId')
.where('assets.updatedAt', '<', sql.raw<Date>("now() - interval '1 millisecond'"))
.$if(!!ack, (qb) => qb.where('assets.updateId', '>', ack!.updateId))
.orderBy('assets.updateId', 'asc')
.innerJoin('albums', 'albums.id', 'album_assets.albumsId')
.leftJoin('albums_shared_users_users as album_users', 'album_users.albumsId', 'album_assets.albumsId')
.where((eb) => eb.or([eb('albums.ownerId', '=', userId), eb('album_users.usersId', '=', userId)]))
.stream();
}
@GenerateSql({ params: [DummyValue.UUID, DummyValue.UUID, DummyValue.UUID], stream: true })
getAlbumToAssetBackfill(albumId: string, afterUpdateId: string | undefined, beforeUpdateId: string) {
return this.db
.selectFrom('albums_assets_assets as album_assets')
.select(['album_assets.assetsId as assetId', 'album_assets.albumsId as albumId', 'album_assets.updateId'])
.where('album_assets.albumsId', '=', albumId)
.where('album_assets.updatedAt', '<', sql.raw<Date>("now() - interval '1 millisecond'"))
.where('album_assets.updateId', '<=', beforeUpdateId)
.$if(!!afterUpdateId, (eb) => eb.where('album_assets.updateId', '>=', afterUpdateId!))
.orderBy('album_assets.updateId', 'asc')
.stream();
}
@GenerateSql({ params: [DummyValue.UUID], stream: true })
getAlbumToAssetUpserts(userId: string, ack?: SyncAck) {
return this.db
.selectFrom('albums_assets_assets as album_assets')
.select(['album_assets.assetsId as assetId', 'album_assets.albumsId as albumId', 'album_assets.updateId'])
.where('album_assets.updatedAt', '<', sql.raw<Date>("now() - interval '1 millisecond'"))
.$if(!!ack, (qb) => qb.where('album_assets.updateId', '>', ack!.updateId))
.orderBy('album_assets.updateId', 'asc')
.innerJoin('albums', 'albums.id', 'album_assets.albumsId')
.leftJoin('albums_shared_users_users as album_users', 'album_users.albumsId', 'album_assets.albumsId')
.where((eb) => eb.or([eb('albums.ownerId', '=', userId), eb('album_users.usersId', '=', userId)]))
.stream();
}
@GenerateSql({ params: [DummyValue.UUID, DummyValue.UUID, DummyValue.UUID], stream: true })
getAlbumAssetExifsBackfill(albumId: string, afterUpdateId: string | undefined, beforeUpdateId: string) {
return this.db return this.db
.selectFrom('exif') .selectFrom('exif')
.innerJoin('albums_assets_assets as album_assets', 'album_assets.assetsId', 'exif.assetId')
.select(columns.syncAssetExif) .select(columns.syncAssetExif)
.select('exif.updateId') .select('exif.updateId')
.where('album_assets.albumsId', '=', albumId) .where('assetId', 'in', (eb) => eb.selectFrom('assets').select('id').where('ownerId', '=', userId))
.where('exif.updatedAt', '<', sql.raw<Date>("now() - interval '1 millisecond'")) .$call((qb) => this.upsertTableFilters(qb, ack))
.where('exif.updateId', '<=', beforeUpdateId)
.$if(!!afterUpdateId, (eb) => eb.where('exif.updateId', '>=', afterUpdateId!))
.orderBy('exif.updateId', 'asc')
.stream(); .stream();
} }
}
class MemorySync extends BaseSync {
@GenerateSql({ params: [DummyValue.UUID], stream: true }) @GenerateSql({ params: [DummyValue.UUID], stream: true })
getAlbumAssetExifsUpserts(userId: string, ack?: SyncAck) { getDeletes(userId: string, ack?: SyncAck) {
return this.db return this.db
.selectFrom('exif') .selectFrom('memories_audit')
.innerJoin('albums_assets_assets as album_assets', 'album_assets.assetsId', 'exif.assetId') .select(['id', 'memoryId'])
.select(columns.syncAssetExif) .where('userId', '=', userId)
.select('exif.updateId') .$call((qb) => this.auditTableFilters(qb, ack))
.where('exif.updatedAt', '<', sql.raw<Date>("now() - interval '1 millisecond'"))
.$if(!!ack, (qb) => qb.where('exif.updateId', '>', ack!.updateId))
.orderBy('exif.updateId', 'asc')
.innerJoin('albums', 'albums.id', 'album_assets.albumsId')
.leftJoin('albums_shared_users_users as album_users', 'album_users.albumsId', 'album_assets.albumsId')
.where((eb) => eb.or([eb('albums.ownerId', '=', userId), eb('album_users.usersId', '=', userId)]))
.stream(); .stream();
} }
@GenerateSql({ params: [DummyValue.UUID], stream: true }) @GenerateSql({ params: [DummyValue.UUID], stream: true })
getMemoryUpserts(userId: string, ack?: SyncAck) { getUpserts(userId: string, ack?: SyncAck) {
return this.db return this.db
.selectFrom('memories') .selectFrom('memories')
.select([ .select([
@ -471,30 +398,11 @@ export class SyncRepository {
.$call((qb) => this.upsertTableFilters(qb, ack)) .$call((qb) => this.upsertTableFilters(qb, ack))
.stream(); .stream();
} }
}
class MemoryToAssetSync extends BaseSync {
@GenerateSql({ params: [DummyValue.UUID], stream: true }) @GenerateSql({ params: [DummyValue.UUID], stream: true })
getMemoryDeletes(userId: string, ack?: SyncAck) { getDeletes(userId: string, ack?: SyncAck) {
return this.db
.selectFrom('memories_audit')
.select(['id', 'memoryId'])
.where('userId', '=', userId)
.$call((qb) => this.auditTableFilters(qb, ack))
.stream();
}
@GenerateSql({ params: [DummyValue.UUID], stream: true })
getMemoryAssetUpserts(userId: string, ack?: SyncAck) {
return this.db
.selectFrom('memories_assets_assets')
.select(['memoriesId as memoryId', 'assetsId as assetId'])
.select('updateId')
.where('memoriesId', 'in', (eb) => eb.selectFrom('memories').select('id').where('ownerId', '=', userId))
.$call((qb) => this.upsertTableFilters(qb, ack))
.stream();
}
@GenerateSql({ params: [DummyValue.UUID], stream: true })
getMemoryAssetDeletes(userId: string, ack?: SyncAck) {
return this.db return this.db
.selectFrom('memory_assets_audit') .selectFrom('memory_assets_audit')
.select(['id', 'memoryId', 'assetId']) .select(['id', 'memoryId', 'assetId'])
@ -503,22 +411,144 @@ export class SyncRepository {
.stream(); .stream();
} }
private auditTableFilters<T extends keyof Pick<DB, AuditTables>, D>(qb: SelectQueryBuilder<DB, T, D>, ack?: SyncAck) { @GenerateSql({ params: [DummyValue.UUID], stream: true })
const builder = qb as SelectQueryBuilder<DB, AuditTables, D>; getUpserts(userId: string, ack?: SyncAck) {
return builder return this.db
.where('deletedAt', '<', sql.raw<Date>("now() - interval '1 millisecond'")) .selectFrom('memories_assets_assets')
.$if(!!ack, (qb) => qb.where('id', '>', ack!.updateId)) .select(['memoriesId as memoryId', 'assetsId as assetId'])
.orderBy('id', 'asc') as SelectQueryBuilder<DB, T, D>; .select('updateId')
} .where('memoriesId', 'in', (eb) => eb.selectFrom('memories').select('id').where('ownerId', '=', userId))
.$call((qb) => this.upsertTableFilters(qb, ack))
private upsertTableFilters<T extends keyof Pick<DB, UpsertTables>, D>( .stream();
qb: SelectQueryBuilder<DB, T, D>, }
ack?: SyncAck, }
) {
const builder = qb as SelectQueryBuilder<DB, UpsertTables, D>; class PartnerSync extends BaseSync {
return builder @GenerateSql({ params: [DummyValue.UUID, DummyValue.UUID] })
.where('updatedAt', '<', sql.raw<Date>("now() - interval '1 millisecond'")) getCreatedAfter(userId: string, afterCreateId?: string) {
.$if(!!ack, (qb) => qb.where('updateId', '>', ack!.updateId)) return this.db
.orderBy('updateId', 'asc') as SelectQueryBuilder<DB, T, D>; .selectFrom('partners')
.select(['sharedById', 'createId'])
.where('sharedWithId', '=', userId)
.$if(!!afterCreateId, (qb) => qb.where('createId', '>=', afterCreateId!))
.where('createdAt', '<', sql.raw<Date>("now() - interval '1 millisecond'"))
.orderBy('partners.createId', 'asc')
.execute();
}
@GenerateSql({ params: [DummyValue.UUID], stream: true })
getDeletes(userId: string, ack?: SyncAck) {
return this.db
.selectFrom('partners_audit')
.select(['id', 'sharedById', 'sharedWithId'])
.where((eb) => eb.or([eb('sharedById', '=', userId), eb('sharedWithId', '=', userId)]))
.$call((qb) => this.auditTableFilters(qb, ack))
.stream();
}
@GenerateSql({ params: [DummyValue.UUID], stream: true })
getUpserts(userId: string, ack?: SyncAck) {
return this.db
.selectFrom('partners')
.select(['sharedById', 'sharedWithId', 'inTimeline', 'updateId'])
.where((eb) => eb.or([eb('sharedById', '=', userId), eb('sharedWithId', '=', userId)]))
.$call((qb) => this.upsertTableFilters(qb, ack))
.stream();
}
}
class PartnerAssetsSync extends BaseSync {
@GenerateSql({ params: [DummyValue.UUID, DummyValue.UUID, DummyValue.UUID], stream: true })
getBackfill(partnerId: string, afterUpdateId: string | undefined, beforeUpdateId: string) {
return this.db
.selectFrom('assets')
.select(columns.syncAsset)
.select('assets.updateId')
.where('ownerId', '=', partnerId)
.where('updatedAt', '<', sql.raw<Date>("now() - interval '1 millisecond'"))
.where('updateId', '<=', beforeUpdateId)
.$if(!!afterUpdateId, (eb) => eb.where('updateId', '>=', afterUpdateId!))
.orderBy('updateId', 'asc')
.stream();
}
@GenerateSql({ params: [DummyValue.UUID], stream: true })
getDeletes(userId: string, ack?: SyncAck) {
return this.db
.selectFrom('assets_audit')
.select(['id', 'assetId'])
.where('ownerId', 'in', (eb) =>
eb.selectFrom('partners').select(['sharedById']).where('sharedWithId', '=', userId),
)
.$call((qb) => this.auditTableFilters(qb, ack))
.stream();
}
@GenerateSql({ params: [DummyValue.UUID], stream: true })
getUpserts(userId: string, ack?: SyncAck) {
return this.db
.selectFrom('assets')
.select(columns.syncAsset)
.select('assets.updateId')
.where('ownerId', 'in', (eb) =>
eb.selectFrom('partners').select(['sharedById']).where('sharedWithId', '=', userId),
)
.$call((qb) => this.upsertTableFilters(qb, ack))
.stream();
}
}
class PartnerAssetExifsSync extends BaseSync {
@GenerateSql({ params: [DummyValue.UUID, DummyValue.UUID, DummyValue.UUID], stream: true })
getBackfill(partnerId: string, afterUpdateId: string | undefined, beforeUpdateId: string) {
return this.db
.selectFrom('exif')
.select(columns.syncAssetExif)
.select('exif.updateId')
.innerJoin('assets', 'assets.id', 'exif.assetId')
.where('assets.ownerId', '=', partnerId)
.where('exif.updatedAt', '<', sql.raw<Date>("now() - interval '1 millisecond'"))
.where('exif.updateId', '<=', beforeUpdateId)
.$if(!!afterUpdateId, (eb) => eb.where('exif.updateId', '>=', afterUpdateId!))
.orderBy('exif.updateId', 'asc')
.stream();
}
@GenerateSql({ params: [DummyValue.UUID], stream: true })
getUpserts(userId: string, ack?: SyncAck) {
return this.db
.selectFrom('exif')
.select(columns.syncAssetExif)
.select('exif.updateId')
.where('assetId', 'in', (eb) =>
eb
.selectFrom('assets')
.select('id')
.where('ownerId', 'in', (eb) =>
eb.selectFrom('partners').select(['sharedById']).where('sharedWithId', '=', userId),
),
)
.$call((qb) => this.upsertTableFilters(qb, ack))
.stream();
}
}
class UserSync extends BaseSync {
@GenerateSql({ params: [], stream: true })
getDeletes(ack?: SyncAck) {
return this.db
.selectFrom('users_audit')
.select(['id', 'userId'])
.$call((qb) => this.auditTableFilters(qb, ack))
.stream();
}
@GenerateSql({ params: [], stream: true })
getUpserts(ack?: SyncAck) {
return this.db
.selectFrom('users')
.select(['id', 'name', 'email', 'deletedAt', 'updateId'])
.$call((qb) => this.upsertTableFilters(qb, ack))
.stream();
} }
} }

3
server/src/services/base.service.ts
View File

@ -41,6 +41,7 @@ import { SessionRepository } from 'src/repositories/session.repository';
import { SharedLinkRepository } from 'src/repositories/shared-link.repository'; import { SharedLinkRepository } from 'src/repositories/shared-link.repository';
import { StackRepository } from 'src/repositories/stack.repository'; import { StackRepository } from 'src/repositories/stack.repository';
import { StorageRepository } from 'src/repositories/storage.repository'; import { StorageRepository } from 'src/repositories/storage.repository';
import { SyncCheckpointRepository } from 'src/repositories/sync-checkpoint.repository';
import { SyncRepository } from 'src/repositories/sync.repository'; import { SyncRepository } from 'src/repositories/sync.repository';
import { SystemMetadataRepository } from 'src/repositories/system-metadata.repository'; import { SystemMetadataRepository } from 'src/repositories/system-metadata.repository';
import { TagRepository } from 'src/repositories/tag.repository'; import { TagRepository } from 'src/repositories/tag.repository';
@ -91,6 +92,7 @@ export const BASE_SERVICE_DEPENDENCIES = [
StackRepository, StackRepository,
StorageRepository, StorageRepository,
SyncRepository, SyncRepository,
SyncCheckpointRepository,
SystemMetadataRepository, SystemMetadataRepository,
TagRepository, TagRepository,
TelemetryRepository, TelemetryRepository,
@ -142,6 +144,7 @@ export class BaseService {
protected stackRepository: StackRepository, protected stackRepository: StackRepository,
protected storageRepository: StorageRepository, protected storageRepository: StorageRepository,
protected syncRepository: SyncRepository, protected syncRepository: SyncRepository,
protected syncCheckpointRepository: SyncCheckpointRepository,
protected systemMetadataRepository: SystemMetadataRepository, protected systemMetadataRepository: SystemMetadataRepository,
protected tagRepository: TagRepository, protected tagRepository: TagRepository,
protected telemetryRepository: TelemetryRepository, protected telemetryRepository: TelemetryRepository,

78
server/src/services/sync.service.ts
View File

@ -81,7 +81,7 @@ export class SyncService extends BaseService {
return throwSessionRequired(); return throwSessionRequired();
} }
return this.syncRepository.getCheckpoints(sessionId); return this.syncCheckpointRepository.getAll(sessionId);
} }
async setAcks(auth: AuthDto, dto: SyncAckSetDto) { async setAcks(auth: AuthDto, dto: SyncAckSetDto) {
@ -102,7 +102,7 @@ export class SyncService extends BaseService {
checkpoints[type] = { sessionId, type, ack }; checkpoints[type] = { sessionId, type, ack };
} }
await this.syncRepository.upsertCheckpoints(Object.values(checkpoints)); await this.syncCheckpointRepository.upsertAll(Object.values(checkpoints));
} }
async deleteAcks(auth: AuthDto, dto: SyncAckDeleteDto) { async deleteAcks(auth: AuthDto, dto: SyncAckDeleteDto) {
@ -111,7 +111,7 @@ export class SyncService extends BaseService {
return throwSessionRequired(); return throwSessionRequired();
} }
await this.syncRepository.deleteCheckpoints(sessionId, dto.types); await this.syncCheckpointRepository.deleteAll(sessionId, dto.types);
} }
async stream(auth: AuthDto, response: Writable, dto: SyncStreamDto) { async stream(auth: AuthDto, response: Writable, dto: SyncStreamDto) {
@ -120,7 +120,7 @@ export class SyncService extends BaseService {
return throwSessionRequired(); return throwSessionRequired();
} }
const checkpoints = await this.syncRepository.getCheckpoints(sessionId); const checkpoints = await this.syncCheckpointRepository.getAll(sessionId);
const checkpointMap: CheckpointMap = Object.fromEntries(checkpoints.map(({ type, ack }) => [type, fromAck(ack)])); const checkpointMap: CheckpointMap = Object.fromEntries(checkpoints.map(({ type, ack }) => [type, fromAck(ack)]));
const handlers: Record<SyncRequestType, () => Promise<void>> = { const handlers: Record<SyncRequestType, () => Promise<void>> = {
[SyncRequestType.UsersV1]: () => this.syncUsersV1(response, checkpointMap), [SyncRequestType.UsersV1]: () => this.syncUsersV1(response, checkpointMap),
@ -149,13 +149,13 @@ export class SyncService extends BaseService {
private async syncUsersV1(response: Writable, checkpointMap: CheckpointMap) { private async syncUsersV1(response: Writable, checkpointMap: CheckpointMap) {
const deleteType = SyncEntityType.UserDeleteV1; const deleteType = SyncEntityType.UserDeleteV1;
const deletes = this.syncRepository.getUserDeletes(checkpointMap[deleteType]); const deletes = this.syncRepository.user.getDeletes(checkpointMap[deleteType]);
for await (const { id, ...data } of deletes) { for await (const { id, ...data } of deletes) {
send(response, { type: deleteType, ids: [id], data }); send(response, { type: deleteType, ids: [id], data });
} }
const upsertType = SyncEntityType.UserV1; const upsertType = SyncEntityType.UserV1;
const upserts = this.syncRepository.getUserUpserts(checkpointMap[upsertType]); const upserts = this.syncRepository.user.getUpserts(checkpointMap[upsertType]);
for await (const { updateId, ...data } of upserts) { for await (const { updateId, ...data } of upserts) {
send(response, { type: upsertType, ids: [updateId], data }); send(response, { type: upsertType, ids: [updateId], data });
} }
@ -163,13 +163,13 @@ export class SyncService extends BaseService {
private async syncPartnersV1(response: Writable, checkpointMap: CheckpointMap, auth: AuthDto) { private async syncPartnersV1(response: Writable, checkpointMap: CheckpointMap, auth: AuthDto) {
const deleteType = SyncEntityType.PartnerDeleteV1; const deleteType = SyncEntityType.PartnerDeleteV1;
const deletes = this.syncRepository.getPartnerDeletes(auth.user.id, checkpointMap[deleteType]); const deletes = this.syncRepository.partner.getDeletes(auth.user.id, checkpointMap[deleteType]);
for await (const { id, ...data } of deletes) { for await (const { id, ...data } of deletes) {
send(response, { type: deleteType, ids: [id], data }); send(response, { type: deleteType, ids: [id], data });
} }
const upsertType = SyncEntityType.PartnerV1; const upsertType = SyncEntityType.PartnerV1;
const upserts = this.syncRepository.getPartnerUpserts(auth.user.id, checkpointMap[upsertType]); const upserts = this.syncRepository.partner.getUpserts(auth.user.id, checkpointMap[upsertType]);
for await (const { updateId, ...data } of upserts) { for await (const { updateId, ...data } of upserts) {
send(response, { type: upsertType, ids: [updateId], data }); send(response, { type: upsertType, ids: [updateId], data });
} }
@ -177,13 +177,13 @@ export class SyncService extends BaseService {
private async syncAssetsV1(response: Writable, checkpointMap: CheckpointMap, auth: AuthDto) { private async syncAssetsV1(response: Writable, checkpointMap: CheckpointMap, auth: AuthDto) {
const deleteType = SyncEntityType.AssetDeleteV1; const deleteType = SyncEntityType.AssetDeleteV1;
const deletes = this.syncRepository.getAssetDeletes(auth.user.id, checkpointMap[deleteType]); const deletes = this.syncRepository.asset.getDeletes(auth.user.id, checkpointMap[deleteType]);
for await (const { id, ...data } of deletes) { for await (const { id, ...data } of deletes) {
send(response, { type: deleteType, ids: [id], data }); send(response, { type: deleteType, ids: [id], data });
} }
const upsertType = SyncEntityType.AssetV1; const upsertType = SyncEntityType.AssetV1;
const upserts = this.syncRepository.getAssetUpserts(auth.user.id, checkpointMap[upsertType]); const upserts = this.syncRepository.asset.getUpserts(auth.user.id, checkpointMap[upsertType]);
for await (const { updateId, ...data } of upserts) { for await (const { updateId, ...data } of upserts) {
send(response, { type: upsertType, ids: [updateId], data: mapSyncAssetV1(data) }); send(response, { type: upsertType, ids: [updateId], data: mapSyncAssetV1(data) });
} }
@ -196,14 +196,14 @@ export class SyncService extends BaseService {
sessionId: string, sessionId: string,
) { ) {
const deleteType = SyncEntityType.PartnerAssetDeleteV1; const deleteType = SyncEntityType.PartnerAssetDeleteV1;
const deletes = this.syncRepository.getPartnerAssetDeletes(auth.user.id, checkpointMap[deleteType]); const deletes = this.syncRepository.partnerAsset.getDeletes(auth.user.id, checkpointMap[deleteType]);
for await (const { id, ...data } of deletes) { for await (const { id, ...data } of deletes) {
send(response, { type: deleteType, ids: [id], data }); send(response, { type: deleteType, ids: [id], data });
} }
const backfillType = SyncEntityType.PartnerAssetBackfillV1; const backfillType = SyncEntityType.PartnerAssetBackfillV1;
const backfillCheckpoint = checkpointMap[backfillType]; const backfillCheckpoint = checkpointMap[backfillType];
const partners = await this.syncRepository.getPartnerBackfill(auth.user.id, backfillCheckpoint?.updateId); const partners = await this.syncRepository.partner.getCreatedAfter(auth.user.id, backfillCheckpoint?.updateId);
const upsertType = SyncEntityType.PartnerAssetV1; const upsertType = SyncEntityType.PartnerAssetV1;
const upsertCheckpoint = checkpointMap[upsertType]; const upsertCheckpoint = checkpointMap[upsertType];
if (upsertCheckpoint) { if (upsertCheckpoint) {
@ -216,7 +216,7 @@ export class SyncService extends BaseService {
} }
const startId = getStartId(createId, backfillCheckpoint); const startId = getStartId(createId, backfillCheckpoint);
const backfill = this.syncRepository.getPartnerAssetsBackfill(partner.sharedById, startId, endId); const backfill = this.syncRepository.partnerAsset.getBackfill(partner.sharedById, startId, endId);
for await (const { updateId, ...data } of backfill) { for await (const { updateId, ...data } of backfill) {
send(response, { send(response, {
@ -236,7 +236,7 @@ export class SyncService extends BaseService {
}); });
} }
const upserts = this.syncRepository.getPartnerAssetsUpserts(auth.user.id, checkpointMap[upsertType]); const upserts = this.syncRepository.partnerAsset.getUpserts(auth.user.id, checkpointMap[upsertType]);
for await (const { updateId, ...data } of upserts) { for await (const { updateId, ...data } of upserts) {
send(response, { type: upsertType, ids: [updateId], data: mapSyncAssetV1(data) }); send(response, { type: upsertType, ids: [updateId], data: mapSyncAssetV1(data) });
} }
@ -244,7 +244,7 @@ export class SyncService extends BaseService {
private async syncAssetExifsV1(response: Writable, checkpointMap: CheckpointMap, auth: AuthDto) { private async syncAssetExifsV1(response: Writable, checkpointMap: CheckpointMap, auth: AuthDto) {
const upsertType = SyncEntityType.AssetExifV1; const upsertType = SyncEntityType.AssetExifV1;
const upserts = this.syncRepository.getAssetExifsUpserts(auth.user.id, checkpointMap[upsertType]); const upserts = this.syncRepository.assetExif.getUpserts(auth.user.id, checkpointMap[upsertType]);
for await (const { updateId, ...data } of upserts) { for await (const { updateId, ...data } of upserts) {
send(response, { type: upsertType, ids: [updateId], data }); send(response, { type: upsertType, ids: [updateId], data });
} }
@ -258,7 +258,7 @@ export class SyncService extends BaseService {
) { ) {
const backfillType = SyncEntityType.PartnerAssetExifBackfillV1; const backfillType = SyncEntityType.PartnerAssetExifBackfillV1;
const backfillCheckpoint = checkpointMap[backfillType]; const backfillCheckpoint = checkpointMap[backfillType];
const partners = await this.syncRepository.getPartnerBackfill(auth.user.id, backfillCheckpoint?.updateId); const partners = await this.syncRepository.partner.getCreatedAfter(auth.user.id, backfillCheckpoint?.updateId);
const upsertType = SyncEntityType.PartnerAssetExifV1; const upsertType = SyncEntityType.PartnerAssetExifV1;
const upsertCheckpoint = checkpointMap[upsertType]; const upsertCheckpoint = checkpointMap[upsertType];
@ -272,7 +272,7 @@ export class SyncService extends BaseService {
} }
const startId = getStartId(createId, backfillCheckpoint); const startId = getStartId(createId, backfillCheckpoint);
const backfill = this.syncRepository.getPartnerAssetExifsBackfill(partner.sharedById, startId, endId); const backfill = this.syncRepository.partnerAssetExif.getBackfill(partner.sharedById, startId, endId);
for await (const { updateId, ...data } of backfill) { for await (const { updateId, ...data } of backfill) {
send(response, { type: backfillType, ids: [partner.createId, updateId], data }); send(response, { type: backfillType, ids: [partner.createId, updateId], data });
@ -288,7 +288,7 @@ export class SyncService extends BaseService {
}); });
} }
const upserts = this.syncRepository.getPartnerAssetExifsUpserts(auth.user.id, checkpointMap[upsertType]); const upserts = this.syncRepository.partnerAssetExif.getUpserts(auth.user.id, checkpointMap[upsertType]);
for await (const { updateId, ...data } of upserts) { for await (const { updateId, ...data } of upserts) {
send(response, { type: upsertType, ids: [updateId], data }); send(response, { type: upsertType, ids: [updateId], data });
} }
@ -296,13 +296,13 @@ export class SyncService extends BaseService {
private async syncAlbumsV1(response: Writable, checkpointMap: CheckpointMap, auth: AuthDto) { private async syncAlbumsV1(response: Writable, checkpointMap: CheckpointMap, auth: AuthDto) {
const deleteType = SyncEntityType.AlbumDeleteV1; const deleteType = SyncEntityType.AlbumDeleteV1;
const deletes = this.syncRepository.getAlbumDeletes(auth.user.id, checkpointMap[deleteType]); const deletes = this.syncRepository.album.getDeletes(auth.user.id, checkpointMap[deleteType]);
for await (const { id, ...data } of deletes) { for await (const { id, ...data } of deletes) {
send(response, { type: deleteType, ids: [id], data }); send(response, { type: deleteType, ids: [id], data });
} }
const upsertType = SyncEntityType.AlbumV1; const upsertType = SyncEntityType.AlbumV1;
const upserts = this.syncRepository.getAlbumUpserts(auth.user.id, checkpointMap[upsertType]); const upserts = this.syncRepository.album.getUpserts(auth.user.id, checkpointMap[upsertType]);
for await (const { updateId, ...data } of upserts) { for await (const { updateId, ...data } of upserts) {
send(response, { type: upsertType, ids: [updateId], data }); send(response, { type: upsertType, ids: [updateId], data });
} }
@ -310,14 +310,14 @@ export class SyncService extends BaseService {
private async syncAlbumUsersV1(response: Writable, checkpointMap: CheckpointMap, auth: AuthDto, sessionId: string) { private async syncAlbumUsersV1(response: Writable, checkpointMap: CheckpointMap, auth: AuthDto, sessionId: string) {
const deleteType = SyncEntityType.AlbumUserDeleteV1; const deleteType = SyncEntityType.AlbumUserDeleteV1;
const deletes = this.syncRepository.getAlbumUserDeletes(auth.user.id, checkpointMap[deleteType]); const deletes = this.syncRepository.albumUser.getDeletes(auth.user.id, checkpointMap[deleteType]);
for await (const { id, ...data } of deletes) { for await (const { id, ...data } of deletes) {
send(response, { type: deleteType, ids: [id], data }); send(response, { type: deleteType, ids: [id], data });
} }
const backfillType = SyncEntityType.AlbumUserBackfillV1; const backfillType = SyncEntityType.AlbumUserBackfillV1;
const backfillCheckpoint = checkpointMap[backfillType]; const backfillCheckpoint = checkpointMap[backfillType];
const albums = await this.syncRepository.getAlbumBackfill(auth.user.id, backfillCheckpoint?.updateId); const albums = await this.syncRepository.album.getCreatedAfter(auth.user.id, backfillCheckpoint?.updateId);
const upsertType = SyncEntityType.AlbumUserV1; const upsertType = SyncEntityType.AlbumUserV1;
const upsertCheckpoint = checkpointMap[upsertType]; const upsertCheckpoint = checkpointMap[upsertType];
if (upsertCheckpoint) { if (upsertCheckpoint) {
@ -330,7 +330,7 @@ export class SyncService extends BaseService {
} }
const startId = getStartId(createId, backfillCheckpoint); const startId = getStartId(createId, backfillCheckpoint);
const backfill = this.syncRepository.getAlbumUsersBackfill(album.id, startId, endId); const backfill = this.syncRepository.albumUser.getBackfill(album.id, startId, endId);
for await (const { updateId, ...data } of backfill) { for await (const { updateId, ...data } of backfill) {
send(response, { type: backfillType, ids: [createId, updateId], data }); send(response, { type: backfillType, ids: [createId, updateId], data });
@ -346,7 +346,7 @@ export class SyncService extends BaseService {
}); });
} }
const upserts = this.syncRepository.getAlbumUserUpserts(auth.user.id, checkpointMap[upsertType]); const upserts = this.syncRepository.albumUser.getUpserts(auth.user.id, checkpointMap[upsertType]);
for await (const { updateId, ...data } of upserts) { for await (const { updateId, ...data } of upserts) {
send(response, { type: upsertType, ids: [updateId], data }); send(response, { type: upsertType, ids: [updateId], data });
} }
@ -355,7 +355,7 @@ export class SyncService extends BaseService {
private async syncAlbumAssetsV1(response: Writable, checkpointMap: CheckpointMap, auth: AuthDto, sessionId: string) { private async syncAlbumAssetsV1(response: Writable, checkpointMap: CheckpointMap, auth: AuthDto, sessionId: string) {
const backfillType = SyncEntityType.AlbumAssetBackfillV1; const backfillType = SyncEntityType.AlbumAssetBackfillV1;
const backfillCheckpoint = checkpointMap[backfillType]; const backfillCheckpoint = checkpointMap[backfillType];
const albums = await this.syncRepository.getAlbumBackfill(auth.user.id, backfillCheckpoint?.updateId); const albums = await this.syncRepository.album.getCreatedAfter(auth.user.id, backfillCheckpoint?.updateId);
const upsertType = SyncEntityType.AlbumAssetV1; const upsertType = SyncEntityType.AlbumAssetV1;
const upsertCheckpoint = checkpointMap[upsertType]; const upsertCheckpoint = checkpointMap[upsertType];
if (upsertCheckpoint) { if (upsertCheckpoint) {
@ -368,7 +368,7 @@ export class SyncService extends BaseService {
} }
const startId = getStartId(createId, backfillCheckpoint); const startId = getStartId(createId, backfillCheckpoint);
const backfill = this.syncRepository.getAlbumAssetsBackfill(album.id, startId, endId); const backfill = this.syncRepository.albumAsset.getBackfill(album.id, startId, endId);
for await (const { updateId, ...data } of backfill) { for await (const { updateId, ...data } of backfill) {
send(response, { type: backfillType, ids: [createId, updateId], data: mapSyncAssetV1(data) }); send(response, { type: backfillType, ids: [createId, updateId], data: mapSyncAssetV1(data) });
@ -384,7 +384,7 @@ export class SyncService extends BaseService {
}); });
} }
const upserts = this.syncRepository.getAlbumAssetsUpserts(auth.user.id, checkpointMap[upsertType]); const upserts = this.syncRepository.albumAsset.getUpserts(auth.user.id, checkpointMap[upsertType]);
for await (const { updateId, ...data } of upserts) { for await (const { updateId, ...data } of upserts) {
send(response, { type: upsertType, ids: [updateId], data: mapSyncAssetV1(data) }); send(response, { type: upsertType, ids: [updateId], data: mapSyncAssetV1(data) });
} }
@ -398,7 +398,7 @@ export class SyncService extends BaseService {
) { ) {
const backfillType = SyncEntityType.AlbumAssetExifBackfillV1; const backfillType = SyncEntityType.AlbumAssetExifBackfillV1;
const backfillCheckpoint = checkpointMap[backfillType]; const backfillCheckpoint = checkpointMap[backfillType];
const albums = await this.syncRepository.getAlbumBackfill(auth.user.id, backfillCheckpoint?.updateId); const albums = await this.syncRepository.album.getCreatedAfter(auth.user.id, backfillCheckpoint?.updateId);
const upsertType = SyncEntityType.AlbumAssetExifV1; const upsertType = SyncEntityType.AlbumAssetExifV1;
const upsertCheckpoint = checkpointMap[upsertType]; const upsertCheckpoint = checkpointMap[upsertType];
if (upsertCheckpoint) { if (upsertCheckpoint) {
@ -411,7 +411,7 @@ export class SyncService extends BaseService {
} }
const startId = getStartId(createId, backfillCheckpoint); const startId = getStartId(createId, backfillCheckpoint);
const backfill = this.syncRepository.getAlbumAssetExifsBackfill(album.id, startId, endId); const backfill = this.syncRepository.albumAssetExif.getBackfill(album.id, startId, endId);
for await (const { updateId, ...data } of backfill) { for await (const { updateId, ...data } of backfill) {
send(response, { type: backfillType, ids: [createId, updateId], data }); send(response, { type: backfillType, ids: [createId, updateId], data });
@ -427,7 +427,7 @@ export class SyncService extends BaseService {
}); });
} }
const upserts = this.syncRepository.getAlbumAssetExifsUpserts(auth.user.id, checkpointMap[upsertType]); const upserts = this.syncRepository.albumAssetExif.getUpserts(auth.user.id, checkpointMap[upsertType]);
for await (const { updateId, ...data } of upserts) { for await (const { updateId, ...data } of upserts) {
send(response, { type: upsertType, ids: [updateId], data }); send(response, { type: upsertType, ids: [updateId], data });
} }
@ -440,14 +440,14 @@ export class SyncService extends BaseService {
sessionId: string, sessionId: string,
) { ) {
const deleteType = SyncEntityType.AlbumToAssetDeleteV1; const deleteType = SyncEntityType.AlbumToAssetDeleteV1;
const deletes = this.syncRepository.getAlbumToAssetDeletes(auth.user.id, checkpointMap[deleteType]); const deletes = this.syncRepository.albumToAsset.getDeletes(auth.user.id, checkpointMap[deleteType]);
for await (const { id, ...data } of deletes) { for await (const { id, ...data } of deletes) {
send(response, { type: deleteType, ids: [id], data }); send(response, { type: deleteType, ids: [id], data });
} }
const backfillType = SyncEntityType.AlbumToAssetBackfillV1; const backfillType = SyncEntityType.AlbumToAssetBackfillV1;
const backfillCheckpoint = checkpointMap[backfillType]; const backfillCheckpoint = checkpointMap[backfillType];
const albums = await this.syncRepository.getAlbumBackfill(auth.user.id, backfillCheckpoint?.updateId); const albums = await this.syncRepository.album.getCreatedAfter(auth.user.id, backfillCheckpoint?.updateId);
const upsertType = SyncEntityType.AlbumToAssetV1; const upsertType = SyncEntityType.AlbumToAssetV1;
const upsertCheckpoint = checkpointMap[upsertType]; const upsertCheckpoint = checkpointMap[upsertType];
if (upsertCheckpoint) { if (upsertCheckpoint) {
@ -460,7 +460,7 @@ export class SyncService extends BaseService {
} }
const startId = getStartId(createId, backfillCheckpoint); const startId = getStartId(createId, backfillCheckpoint);
const backfill = this.syncRepository.getAlbumToAssetBackfill(album.id, startId, endId); const backfill = this.syncRepository.albumToAsset.getBackfill(album.id, startId, endId);
for await (const { updateId, ...data } of backfill) { for await (const { updateId, ...data } of backfill) {
send(response, { type: backfillType, ids: [createId, updateId], data }); send(response, { type: backfillType, ids: [createId, updateId], data });
@ -476,7 +476,7 @@ export class SyncService extends BaseService {
}); });
} }
const upserts = this.syncRepository.getAlbumToAssetUpserts(auth.user.id, checkpointMap[upsertType]); const upserts = this.syncRepository.albumToAsset.getUpserts(auth.user.id, checkpointMap[upsertType]);
for await (const { updateId, ...data } of upserts) { for await (const { updateId, ...data } of upserts) {
send(response, { type: upsertType, ids: [updateId], data }); send(response, { type: upsertType, ids: [updateId], data });
} }
@ -484,13 +484,13 @@ export class SyncService extends BaseService {
private async syncMemoriesV1(response: Writable, checkpointMap: CheckpointMap, auth: AuthDto) { private async syncMemoriesV1(response: Writable, checkpointMap: CheckpointMap, auth: AuthDto) {
const deleteType = SyncEntityType.MemoryDeleteV1; const deleteType = SyncEntityType.MemoryDeleteV1;
const deletes = this.syncRepository.getMemoryDeletes(auth.user.id, checkpointMap[SyncEntityType.MemoryDeleteV1]); const deletes = this.syncRepository.memory.getDeletes(auth.user.id, checkpointMap[SyncEntityType.MemoryDeleteV1]);
for await (const { id, ...data } of deletes) { for await (const { id, ...data } of deletes) {
send(response, { type: deleteType, ids: [id], data }); send(response, { type: deleteType, ids: [id], data });
} }
const upsertType = SyncEntityType.MemoryV1; const upsertType = SyncEntityType.MemoryV1;
const upserts = this.syncRepository.getMemoryUpserts(auth.user.id, checkpointMap[upsertType]); const upserts = this.syncRepository.memory.getUpserts(auth.user.id, checkpointMap[upsertType]);
for await (const { updateId, ...data } of upserts) { for await (const { updateId, ...data } of upserts) {
send(response, { type: upsertType, ids: [updateId], data }); send(response, { type: upsertType, ids: [updateId], data });
} }
@ -498,13 +498,13 @@ export class SyncService extends BaseService {
private async syncMemoryAssetsV1(response: Writable, checkpointMap: CheckpointMap, auth: AuthDto) { private async syncMemoryAssetsV1(response: Writable, checkpointMap: CheckpointMap, auth: AuthDto) {
const deleteType = SyncEntityType.MemoryToAssetDeleteV1; const deleteType = SyncEntityType.MemoryToAssetDeleteV1;
const deletes = this.syncRepository.getMemoryAssetDeletes(auth.user.id, checkpointMap[deleteType]); const deletes = this.syncRepository.memoryToAsset.getDeletes(auth.user.id, checkpointMap[deleteType]);
for await (const { id, ...data } of deletes) { for await (const { id, ...data } of deletes) {
send(response, { type: deleteType, ids: [id], data }); send(response, { type: deleteType, ids: [id], data });
} }
const upsertType = SyncEntityType.MemoryToAssetV1; const upsertType = SyncEntityType.MemoryToAssetV1;
const upserts = this.syncRepository.getMemoryAssetUpserts(auth.user.id, checkpointMap[upsertType]); const upserts = this.syncRepository.memoryToAsset.getUpserts(auth.user.id, checkpointMap[upsertType]);
for await (const { updateId, ...data } of upserts) { for await (const { updateId, ...data } of upserts) {
send(response, { type: upsertType, ids: [updateId], data }); send(response, { type: upsertType, ids: [updateId], data });
} }
@ -512,7 +512,7 @@ export class SyncService extends BaseService {
private async upsertBackfillCheckpoint(item: { type: SyncEntityType; sessionId: string; createId: string }) { private async upsertBackfillCheckpoint(item: { type: SyncEntityType; sessionId: string; createId: string }) {
const { type, sessionId, createId } = item; const { type, sessionId, createId } = item;
await this.syncRepository.upsertCheckpoints([ await this.syncCheckpointRepository.upsertAll([
{ {
type, type,
sessionId, sessionId,

9
server/test/medium.factory.ts
View File

@ -26,6 +26,7 @@ import { PersonRepository } from 'src/repositories/person.repository';
import { SearchRepository } from 'src/repositories/search.repository'; import { SearchRepository } from 'src/repositories/search.repository';
import { SessionRepository } from 'src/repositories/session.repository'; import { SessionRepository } from 'src/repositories/session.repository';
import { StorageRepository } from 'src/repositories/storage.repository'; import { StorageRepository } from 'src/repositories/storage.repository';
import { SyncCheckpointRepository } from 'src/repositories/sync-checkpoint.repository';
import { SyncRepository } from 'src/repositories/sync.repository'; import { SyncRepository } from 'src/repositories/sync.repository';
import { SystemMetadataRepository } from 'src/repositories/system-metadata.repository'; import { SystemMetadataRepository } from 'src/repositories/system-metadata.repository';
import { UserRepository } from 'src/repositories/user.repository'; import { UserRepository } from 'src/repositories/user.repository';
@ -202,7 +203,11 @@ export class MediumTestContext<S extends BaseService = BaseService> {
export class SyncTestContext extends MediumTestContext<SyncService> { export class SyncTestContext extends MediumTestContext<SyncService> {
constructor(database: Kysely<DB>) { constructor(database: Kysely<DB>) {
super(SyncService, { database, real: [SyncRepository, SessionRepository], mock: [LoggingRepository] }); super(SyncService, {
database,
real: [SyncRepository, SyncCheckpointRepository, SessionRepository],
mock: [LoggingRepository],
});
} }
async syncStream(auth: AuthDto, types: SyncRequestType[]) { async syncStream(auth: AuthDto, types: SyncRequestType[]) {
@ -239,6 +244,7 @@ const newRealRepository = <T>(key: ClassConstructor<T>, db: Kysely<DB>): T => {
case SearchRepository: case SearchRepository:
case SessionRepository: case SessionRepository:
case SyncRepository: case SyncRepository:
case SyncCheckpointRepository:
case SystemMetadataRepository: case SystemMetadataRepository:
case UserRepository: case UserRepository:
case VersionHistoryRepository: { case VersionHistoryRepository: {
@ -282,6 +288,7 @@ const newMockRepository = <T>(key: ClassConstructor<T>) => {
case PersonRepository: case PersonRepository:
case SessionRepository: case SessionRepository:
case SyncRepository: case SyncRepository:
case SyncCheckpointRepository:
case SystemMetadataRepository: case SystemMetadataRepository:
case UserRepository: case UserRepository:
case VersionHistoryRepository: { case VersionHistoryRepository: {

4
server/test/utils.ts
View File

@ -47,6 +47,7 @@ import { SessionRepository } from 'src/repositories/session.repository';
import { SharedLinkRepository } from 'src/repositories/shared-link.repository'; import { SharedLinkRepository } from 'src/repositories/shared-link.repository';
import { StackRepository } from 'src/repositories/stack.repository'; import { StackRepository } from 'src/repositories/stack.repository';
import { StorageRepository } from 'src/repositories/storage.repository'; import { StorageRepository } from 'src/repositories/storage.repository';
import { SyncCheckpointRepository } from 'src/repositories/sync-checkpoint.repository';
import { SyncRepository } from 'src/repositories/sync.repository'; import { SyncRepository } from 'src/repositories/sync.repository';
import { SystemMetadataRepository } from 'src/repositories/system-metadata.repository'; import { SystemMetadataRepository } from 'src/repositories/system-metadata.repository';
import { TagRepository } from 'src/repositories/tag.repository'; import { TagRepository } from 'src/repositories/tag.repository';
@ -217,6 +218,7 @@ export type ServiceOverrides = {
stack: StackRepository; stack: StackRepository;
storage: StorageRepository; storage: StorageRepository;
sync: SyncRepository; sync: SyncRepository;
syncCheckpoint: SyncCheckpointRepository;
systemMetadata: SystemMetadataRepository; systemMetadata: SystemMetadataRepository;
tag: TagRepository; tag: TagRepository;
telemetry: TelemetryRepository; telemetry: TelemetryRepository;
@ -287,6 +289,7 @@ export const newTestService = <T extends BaseService>(
stack: automock(StackRepository), stack: automock(StackRepository),
storage: newStorageRepositoryMock(), storage: newStorageRepositoryMock(),
sync: automock(SyncRepository), sync: automock(SyncRepository),
syncCheckpoint: automock(SyncCheckpointRepository),
systemMetadata: newSystemMetadataRepositoryMock(), systemMetadata: newSystemMetadataRepositoryMock(),
// systemMetadata: automock(SystemMetadataRepository, { strict: false }), // systemMetadata: automock(SystemMetadataRepository, { strict: false }),
// eslint-disable-next-line no-sparse-arrays // eslint-disable-next-line no-sparse-arrays
@ -336,6 +339,7 @@ export const newTestService = <T extends BaseService>(
overrides.stack || (mocks.stack as As<StackRepository>), overrides.stack || (mocks.stack as As<StackRepository>),
overrides.storage || (mocks.storage as As<StorageRepository>), overrides.storage || (mocks.storage as As<StorageRepository>),
overrides.sync || (mocks.sync as As<SyncRepository>), overrides.sync || (mocks.sync as As<SyncRepository>),
overrides.syncCheckpoint || (mocks.syncCheckpoint as As<SyncCheckpointRepository>),
overrides.systemMetadata || (mocks.systemMetadata as As<SystemMetadataRepository>), overrides.systemMetadata || (mocks.systemMetadata as As<SystemMetadataRepository>),
overrides.tag || (mocks.tag as As<TagRepository>), overrides.tag || (mocks.tag as As<TagRepository>),
overrides.telemetry || (mocks.telemetry as unknown as TelemetryRepository), overrides.telemetry || (mocks.telemetry as unknown as TelemetryRepository),