forked from Cutlery/immich
Compare commits
8 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| d8413ea2ce | |||
| 81d6f69a47 | |||
| 6feac7df20 | |||
| de7475f3fc | |||
| 7f5d554d1a | |||
| e033e9aad2 | |||
| 061c567468 | |||
| 4ce3676127 |
@@ -49,16 +49,23 @@ class AssetService {
|
||||
return changes;
|
||||
}
|
||||
|
||||
/// Returns `(null, null)` if changes are invalid -> requires full sync
|
||||
Future<(List<Asset>? toUpsert, List<String>? toDelete)>
|
||||
/// Returns `(null, null, time)` if changes are invalid -> requires full sync
|
||||
Future<(List<Asset>? toUpsert, List<String>? toDelete, DateTime? time)>
|
||||
_getRemoteAssetChanges(User user, DateTime since) async {
|
||||
final deleted = await _apiService.auditApi
|
||||
.getAuditDeletes(since, EntityType.ASSET, userId: user.id);
|
||||
if (deleted == null || deleted.needsFullSync) return (null, null);
|
||||
if (deleted == null || deleted.needsFullSync) {
|
||||
return (null, null, deleted?.requestedAt);
|
||||
}
|
||||
|
||||
final assetDto = await _apiService.assetApi
|
||||
.getAllAssets(userId: user.id, updatedAfter: since);
|
||||
if (assetDto == null) return (null, null);
|
||||
return (assetDto.map(Asset.remote).toList(), deleted.ids);
|
||||
if (assetDto == null) return (null, null, deleted.requestedAt);
|
||||
return (
|
||||
assetDto.map(Asset.remote).toList(),
|
||||
deleted.ids,
|
||||
deleted.requestedAt
|
||||
);
|
||||
}
|
||||
|
||||
/// Returns the list of people of the given asset id.
|
||||
@@ -83,10 +90,11 @@ class AssetService {
|
||||
}
|
||||
|
||||
/// Returns `null` if the server state did not change, else list of assets
|
||||
Future<List<Asset>?> _getRemoteAssets(User user) async {
|
||||
|
||||
Future<List<Asset>?> _getRemoteAssets(User user, DateTime now) async {
|
||||
const int chunkSize = 10000;
|
||||
|
||||
try {
|
||||
final DateTime now = DateTime.now().toUtc();
|
||||
final List<Asset> allAssets = [];
|
||||
for (int i = 0;; i += chunkSize) {
|
||||
final List<AssetResponseDto>? assets =
|
||||
|
||||
@@ -41,17 +41,20 @@ class SyncService {
|
||||
/// Returns `true` if there were any changes
|
||||
Future<bool> syncRemoteAssetsToDb(
|
||||
User user,
|
||||
Future<(List<Asset>? toUpsert, List<String>? toDelete)> Function(
|
||||
Future<(List<Asset>? toUpsert, List<String>? toDelete, DateTime? time)>
|
||||
Function(
|
||||
User user,
|
||||
DateTime since,
|
||||
) getChangedAssets,
|
||||
FutureOr<List<Asset>?> Function(User user) loadAssets,
|
||||
FutureOr<List<Asset>?> Function(User user, DateTime now) loadAssets,
|
||||
) =>
|
||||
_lock.run(
|
||||
() async =>
|
||||
await _syncRemoteAssetChanges(user, getChangedAssets) ??
|
||||
await _syncRemoteAssetsFull(user, loadAssets),
|
||||
);
|
||||
_lock.run(() async {
|
||||
final (changes, serverTime) =
|
||||
await _syncRemoteAssetChanges(user, getChangedAssets);
|
||||
if (changes != null) return changes;
|
||||
final time = serverTime ?? DateTime.now().toUtc();
|
||||
return await _syncRemoteAssetsFull(user, time, loadAssets);
|
||||
});
|
||||
|
||||
/// Syncs remote albums to the database
|
||||
/// returns `true` if there were any changes
|
||||
@@ -146,19 +149,22 @@ class SyncService {
|
||||
return true;
|
||||
}
|
||||
|
||||
/// Efficiently syncs assets via changes. Returns `null` when a full sync is required.
|
||||
Future<bool?> _syncRemoteAssetChanges(
|
||||
/// Efficiently syncs assets via changes. Returns `(null, serverTime)` when a full sync is required.
|
||||
Future<(bool?, DateTime? serverTime)> _syncRemoteAssetChanges(
|
||||
User user,
|
||||
Future<(List<Asset>? toUpsert, List<String>? toDelete)> Function(
|
||||
Future<(List<Asset>? toUpsert, List<String>? toDelete, DateTime? time)>
|
||||
Function(
|
||||
User user,
|
||||
DateTime since,
|
||||
) getChangedAssets,
|
||||
) async {
|
||||
final DateTime? since = _db.eTags.getByIdSync(user.id)?.time?.toUtc();
|
||||
if (since == null) return null;
|
||||
final DateTime now = DateTime.now();
|
||||
final (toUpsert, toDelete) = await getChangedAssets(user, since);
|
||||
if (toUpsert == null || toDelete == null) return null;
|
||||
final DateTime now = DateTime.now().toUtc();
|
||||
final (toUpsert, toDelete, serverTime) =
|
||||
await getChangedAssets(user, since ?? now);
|
||||
if (since == null || toUpsert == null || toDelete == null) {
|
||||
return (null, serverTime);
|
||||
}
|
||||
try {
|
||||
if (toDelete.isNotEmpty) {
|
||||
await handleRemoteAssetRemoval(toDelete);
|
||||
@@ -168,14 +174,14 @@ class SyncService {
|
||||
await upsertAssetsWithExif(updated);
|
||||
}
|
||||
if (toUpsert.isNotEmpty || toDelete.isNotEmpty) {
|
||||
await _updateUserAssetsETag(user, now);
|
||||
return true;
|
||||
await _updateUserAssetsETag(user, serverTime ?? now);
|
||||
return (true, serverTime);
|
||||
}
|
||||
return false;
|
||||
return (false, serverTime);
|
||||
} on IsarError catch (e) {
|
||||
_log.severe("Failed to sync remote assets to db", e);
|
||||
}
|
||||
return null;
|
||||
return (null, serverTime);
|
||||
}
|
||||
|
||||
/// Deletes remote-only assets, updates merged assets to be local-only
|
||||
@@ -202,11 +208,11 @@ class SyncService {
|
||||
|
||||
/// Syncs assets by loading and comparing all assets from the server.
|
||||
Future<bool> _syncRemoteAssetsFull(
|
||||
User user,
|
||||
FutureOr<List<Asset>?> Function(User user) loadAssets,
|
||||
final User user,
|
||||
final DateTime now,
|
||||
final FutureOr<List<Asset>?> Function(User user, DateTime now) loadAssets,
|
||||
) async {
|
||||
final DateTime now = DateTime.now().toUtc();
|
||||
final List<Asset>? remote = await loadAssets(user);
|
||||
final List<Asset>? remote = await loadAssets(user, now);
|
||||
if (remote == null) {
|
||||
return false;
|
||||
}
|
||||
|
||||
+1
@@ -10,6 +10,7 @@ Name | Type | Description | Notes
|
||||
------------ | ------------- | ------------- | -------------
|
||||
**ids** | **List<String>** | | [default to const []]
|
||||
**needsFullSync** | **bool** | |
|
||||
**requestedAt** | [**DateTime**](DateTime.md) | | [optional]
|
||||
|
||||
[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)
|
||||
|
||||
|
||||
+41
-13
@@ -15,30 +15,49 @@ class AuditDeletesResponseDto {
|
||||
AuditDeletesResponseDto({
|
||||
this.ids = const [],
|
||||
required this.needsFullSync,
|
||||
this.requestedAt,
|
||||
});
|
||||
|
||||
List<String> ids;
|
||||
|
||||
bool needsFullSync;
|
||||
|
||||
///
|
||||
/// Please note: This property should have been non-nullable! Since the specification file
|
||||
/// does not include a default value (using the "default:" property), however, the generated
|
||||
/// source code must fall back to having a nullable type.
|
||||
/// Consider adding a "default:" property in the specification file to hide this note.
|
||||
///
|
||||
DateTime? requestedAt;
|
||||
|
||||
@override
|
||||
bool operator ==(Object other) => identical(this, other) || other is AuditDeletesResponseDto &&
|
||||
_deepEquality.equals(other.ids, ids) &&
|
||||
other.needsFullSync == needsFullSync;
|
||||
bool operator ==(Object other) =>
|
||||
identical(this, other) ||
|
||||
other is AuditDeletesResponseDto &&
|
||||
other.ids == ids &&
|
||||
other.needsFullSync == needsFullSync &&
|
||||
other.requestedAt == requestedAt;
|
||||
|
||||
@override
|
||||
int get hashCode =>
|
||||
// ignore: unnecessary_parenthesis
|
||||
(ids.hashCode) +
|
||||
(needsFullSync.hashCode);
|
||||
// ignore: unnecessary_parenthesis
|
||||
(ids.hashCode) +
|
||||
(needsFullSync.hashCode) +
|
||||
(requestedAt == null ? 0 : requestedAt!.hashCode);
|
||||
|
||||
@override
|
||||
String toString() => 'AuditDeletesResponseDto[ids=$ids, needsFullSync=$needsFullSync]';
|
||||
String toString() =>
|
||||
'AuditDeletesResponseDto[ids=$ids, needsFullSync=$needsFullSync, requestedAt=$requestedAt]';
|
||||
|
||||
Map<String, dynamic> toJson() {
|
||||
final json = <String, dynamic>{};
|
||||
json[r'ids'] = this.ids;
|
||||
json[r'needsFullSync'] = this.needsFullSync;
|
||||
json[r'ids'] = this.ids;
|
||||
json[r'needsFullSync'] = this.needsFullSync;
|
||||
if (this.requestedAt != null) {
|
||||
json[r'requestedAt'] = this.requestedAt!.toUtc().toIso8601String();
|
||||
} else {
|
||||
// json[r'requestedAt'] = null;
|
||||
}
|
||||
return json;
|
||||
}
|
||||
|
||||
@@ -54,12 +73,16 @@ class AuditDeletesResponseDto {
|
||||
? (json[r'ids'] as Iterable).cast<String>().toList(growable: false)
|
||||
: const [],
|
||||
needsFullSync: mapValueOfType<bool>(json, r'needsFullSync')!,
|
||||
requestedAt: mapDateTime(json, r'requestedAt', ''),
|
||||
);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
static List<AuditDeletesResponseDto> listFromJson(dynamic json, {bool growable = false,}) {
|
||||
static List<AuditDeletesResponseDto> listFromJson(
|
||||
dynamic json, {
|
||||
bool growable = false,
|
||||
}) {
|
||||
final result = <AuditDeletesResponseDto>[];
|
||||
if (json is List && json.isNotEmpty) {
|
||||
for (final row in json) {
|
||||
@@ -87,13 +110,19 @@ class AuditDeletesResponseDto {
|
||||
}
|
||||
|
||||
// maps a json object with a list of AuditDeletesResponseDto-objects as value to a dart map
|
||||
static Map<String, List<AuditDeletesResponseDto>> mapListFromJson(dynamic json, {bool growable = false,}) {
|
||||
static Map<String, List<AuditDeletesResponseDto>> mapListFromJson(
|
||||
dynamic json, {
|
||||
bool growable = false,
|
||||
}) {
|
||||
final map = <String, List<AuditDeletesResponseDto>>{};
|
||||
if (json is Map && json.isNotEmpty) {
|
||||
// ignore: parameter_assignments
|
||||
json = json.cast<String, dynamic>();
|
||||
for (final entry in json.entries) {
|
||||
map[entry.key] = AuditDeletesResponseDto.listFromJson(entry.value, growable: growable,);
|
||||
map[entry.key] = AuditDeletesResponseDto.listFromJson(
|
||||
entry.value,
|
||||
growable: growable,
|
||||
);
|
||||
}
|
||||
}
|
||||
return map;
|
||||
@@ -105,4 +134,3 @@ class AuditDeletesResponseDto {
|
||||
'needsFullSync',
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -26,6 +26,11 @@ void main() {
|
||||
// TODO
|
||||
});
|
||||
|
||||
// DateTime requestedAt
|
||||
test('to test the property `requestedAt`', () async {
|
||||
// TODO
|
||||
});
|
||||
|
||||
|
||||
});
|
||||
|
||||
|
||||
@@ -7326,6 +7326,10 @@
|
||||
},
|
||||
"needsFullSync": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"requestedAt": {
|
||||
"format": "date-time",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
|
||||
@@ -29,6 +29,7 @@ export enum PathEntityType {
|
||||
export class AuditDeletesResponseDto {
|
||||
needsFullSync!: boolean;
|
||||
ids!: string[];
|
||||
requestedAt?: Date;
|
||||
}
|
||||
|
||||
export class FileReportDto {
|
||||
|
||||
@@ -61,6 +61,7 @@ describe(AuditService.name, () => {
|
||||
await expect(sut.getDeletes(authStub.admin, { after: date, entityType: EntityType.ASSET })).resolves.toEqual({
|
||||
needsFullSync: true,
|
||||
ids: [],
|
||||
requestedAt: expect.any(Date),
|
||||
});
|
||||
|
||||
expect(auditMock.getAfter).toHaveBeenCalledWith(date, {
|
||||
@@ -77,6 +78,7 @@ describe(AuditService.name, () => {
|
||||
await expect(sut.getDeletes(authStub.admin, { after: date, entityType: EntityType.ASSET })).resolves.toEqual({
|
||||
needsFullSync: false,
|
||||
ids: ['asset-deleted'],
|
||||
requestedAt: expect.any(Date),
|
||||
});
|
||||
|
||||
expect(auditMock.getAfter).toHaveBeenCalledWith(date, {
|
||||
|
||||
@@ -5,7 +5,7 @@ import { DateTime } from 'luxon';
|
||||
import { resolve } from 'node:path';
|
||||
import { AccessCore, Permission } from '../access';
|
||||
import { AuthDto } from '../auth';
|
||||
import { AUDIT_LOG_MAX_DURATION } from '../domain.constant';
|
||||
import { AUDIT_LOG_CLEANUP_DURATION, AUDIT_LOG_MAX_DURATION } from '../domain.constant';
|
||||
import { usePagination } from '../domain.util';
|
||||
import { JOBS_ASSET_PAGINATION_SIZE } from '../job';
|
||||
import {
|
||||
@@ -45,7 +45,7 @@ export class AuditService {
|
||||
}
|
||||
|
||||
async handleCleanup(): Promise<boolean> {
|
||||
await this.repository.removeBefore(DateTime.now().minus(AUDIT_LOG_MAX_DURATION).toJSDate());
|
||||
await this.repository.removeBefore(DateTime.now().minus(AUDIT_LOG_CLEANUP_DURATION).toJSDate());
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -53,15 +53,16 @@ export class AuditService {
|
||||
const userId = dto.userId || auth.user.id;
|
||||
await this.access.requirePermission(auth, Permission.TIMELINE_READ, userId);
|
||||
|
||||
const now = DateTime.utc();
|
||||
const duration = now.diff(DateTime.fromJSDate(dto.after));
|
||||
const audits = await this.repository.getAfter(dto.after, {
|
||||
ownerId: userId,
|
||||
entityType: dto.entityType,
|
||||
action: DatabaseAction.DELETE,
|
||||
});
|
||||
|
||||
const duration = DateTime.now().diff(DateTime.fromJSDate(dto.after));
|
||||
|
||||
return {
|
||||
requestedAt: now.toJSDate(),
|
||||
needsFullSync: duration > AUDIT_LOG_MAX_DURATION,
|
||||
ids: audits.map(({ entityId }) => entityId),
|
||||
};
|
||||
|
||||
@@ -4,6 +4,7 @@ import { readFileSync } from 'node:fs';
|
||||
import { extname, join } from 'node:path';
|
||||
|
||||
export const AUDIT_LOG_MAX_DURATION = Duration.fromObject({ days: 100 });
|
||||
export const AUDIT_LOG_CLEANUP_DURATION = Duration.fromObject({ days: 101 });
|
||||
export const ONE_HOUR = Duration.fromObject({ hours: 1 });
|
||||
|
||||
export interface IVersion {
|
||||
|
||||
Reference in New Issue
Block a user