1
0
forked from Cutlery/immich

Compare commits

...

8 Commits

Author SHA1 Message Date
Alex Tran d8413ea2ce fix 2024-03-08 10:50:03 -06:00
Alex Tran 81d6f69a47 resolve conflict 2024-03-08 10:48:28 -06:00
Alex 6feac7df20 Merge branch 'main' into dev/fix-sync-time-mismatch 2023-12-11 10:51:48 -06:00
Fynn Petersen-Frey de7475f3fc prettier 2023-11-29 20:57:59 +01:00
Fynn Petersen-Frey 7f5d554d1a fix tests 2023-11-29 20:38:13 +01:00
Fynn Petersen-Frey e033e9aad2 timeOfRequest -> requestedAt 2023-11-29 19:46:21 +01:00
Fynn Petersen-Frey 061c567468 prettier 2023-11-29 17:17:44 +01:00
Fynn Petersen-Frey 4ce3676127 fix(mobile): sync issue due to time mismatch 2023-11-29 17:08:03 +01:00
10 changed files with 103 additions and 46 deletions
+15 -7
View File
@@ -49,16 +49,23 @@ class AssetService {
return changes;
}
/// Returns `(null, null)` if changes are invalid -> requires full sync
Future<(List<Asset>? toUpsert, List<String>? toDelete)>
/// Returns `(null, null, time)` if changes are invalid -> requires full sync
Future<(List<Asset>? toUpsert, List<String>? toDelete, DateTime? time)>
_getRemoteAssetChanges(User user, DateTime since) async {
final deleted = await _apiService.auditApi
.getAuditDeletes(since, EntityType.ASSET, userId: user.id);
if (deleted == null || deleted.needsFullSync) return (null, null);
if (deleted == null || deleted.needsFullSync) {
return (null, null, deleted?.requestedAt);
}
final assetDto = await _apiService.assetApi
.getAllAssets(userId: user.id, updatedAfter: since);
if (assetDto == null) return (null, null);
return (assetDto.map(Asset.remote).toList(), deleted.ids);
if (assetDto == null) return (null, null, deleted.requestedAt);
return (
assetDto.map(Asset.remote).toList(),
deleted.ids,
deleted.requestedAt
);
}
/// Returns the list of people of the given asset id.
@@ -83,10 +90,11 @@ class AssetService {
}
/// Returns `null` if the server state did not change, else list of assets
Future<List<Asset>?> _getRemoteAssets(User user) async {
Future<List<Asset>?> _getRemoteAssets(User user, DateTime now) async {
const int chunkSize = 10000;
try {
final DateTime now = DateTime.now().toUtc();
final List<Asset> allAssets = [];
for (int i = 0;; i += chunkSize) {
final List<AssetResponseDto>? assets =
+28 -22
View File
@@ -41,17 +41,20 @@ class SyncService {
/// Returns `true` if there were any changes
Future<bool> syncRemoteAssetsToDb(
User user,
Future<(List<Asset>? toUpsert, List<String>? toDelete)> Function(
Future<(List<Asset>? toUpsert, List<String>? toDelete, DateTime? time)>
Function(
User user,
DateTime since,
) getChangedAssets,
FutureOr<List<Asset>?> Function(User user) loadAssets,
FutureOr<List<Asset>?> Function(User user, DateTime now) loadAssets,
) =>
_lock.run(
() async =>
await _syncRemoteAssetChanges(user, getChangedAssets) ??
await _syncRemoteAssetsFull(user, loadAssets),
);
_lock.run(() async {
final (changes, serverTime) =
await _syncRemoteAssetChanges(user, getChangedAssets);
if (changes != null) return changes;
final time = serverTime ?? DateTime.now().toUtc();
return await _syncRemoteAssetsFull(user, time, loadAssets);
});
/// Syncs remote albums to the database
/// returns `true` if there were any changes
@@ -146,19 +149,22 @@ class SyncService {
return true;
}
/// Efficiently syncs assets via changes. Returns `null` when a full sync is required.
Future<bool?> _syncRemoteAssetChanges(
/// Efficiently syncs assets via changes. Returns `(null, serverTime)` when a full sync is required.
Future<(bool?, DateTime? serverTime)> _syncRemoteAssetChanges(
User user,
Future<(List<Asset>? toUpsert, List<String>? toDelete)> Function(
Future<(List<Asset>? toUpsert, List<String>? toDelete, DateTime? time)>
Function(
User user,
DateTime since,
) getChangedAssets,
) async {
final DateTime? since = _db.eTags.getByIdSync(user.id)?.time?.toUtc();
if (since == null) return null;
final DateTime now = DateTime.now();
final (toUpsert, toDelete) = await getChangedAssets(user, since);
if (toUpsert == null || toDelete == null) return null;
final DateTime now = DateTime.now().toUtc();
final (toUpsert, toDelete, serverTime) =
await getChangedAssets(user, since ?? now);
if (since == null || toUpsert == null || toDelete == null) {
return (null, serverTime);
}
try {
if (toDelete.isNotEmpty) {
await handleRemoteAssetRemoval(toDelete);
@@ -168,14 +174,14 @@ class SyncService {
await upsertAssetsWithExif(updated);
}
if (toUpsert.isNotEmpty || toDelete.isNotEmpty) {
await _updateUserAssetsETag(user, now);
return true;
await _updateUserAssetsETag(user, serverTime ?? now);
return (true, serverTime);
}
return false;
return (false, serverTime);
} on IsarError catch (e) {
_log.severe("Failed to sync remote assets to db", e);
}
return null;
return (null, serverTime);
}
/// Deletes remote-only assets, updates merged assets to be local-only
@@ -202,11 +208,11 @@ class SyncService {
/// Syncs assets by loading and comparing all assets from the server.
Future<bool> _syncRemoteAssetsFull(
User user,
FutureOr<List<Asset>?> Function(User user) loadAssets,
final User user,
final DateTime now,
final FutureOr<List<Asset>?> Function(User user, DateTime now) loadAssets,
) async {
final DateTime now = DateTime.now().toUtc();
final List<Asset>? remote = await loadAssets(user);
final List<Asset>? remote = await loadAssets(user, now);
if (remote == null) {
return false;
}
+1
View File
@@ -10,6 +10,7 @@ Name | Type | Description | Notes
------------ | ------------- | ------------- | -------------
**ids** | **List<String>** | | [default to const []]
**needsFullSync** | **bool** | |
**requestedAt** | [**DateTime**](DateTime.md) | | [optional]
[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)
+41 -13
View File
@@ -15,30 +15,49 @@ class AuditDeletesResponseDto {
AuditDeletesResponseDto({
this.ids = const [],
required this.needsFullSync,
this.requestedAt,
});
List<String> ids;
bool needsFullSync;
///
/// Please note: This property should have been non-nullable! Since the specification file
/// does not include a default value (using the "default:" property), however, the generated
/// source code must fall back to having a nullable type.
/// Consider adding a "default:" property in the specification file to hide this note.
///
DateTime? requestedAt;
@override
bool operator ==(Object other) => identical(this, other) || other is AuditDeletesResponseDto &&
_deepEquality.equals(other.ids, ids) &&
other.needsFullSync == needsFullSync;
bool operator ==(Object other) =>
identical(this, other) ||
other is AuditDeletesResponseDto &&
other.ids == ids &&
other.needsFullSync == needsFullSync &&
other.requestedAt == requestedAt;
@override
int get hashCode =>
// ignore: unnecessary_parenthesis
(ids.hashCode) +
(needsFullSync.hashCode);
// ignore: unnecessary_parenthesis
(ids.hashCode) +
(needsFullSync.hashCode) +
(requestedAt == null ? 0 : requestedAt!.hashCode);
@override
String toString() => 'AuditDeletesResponseDto[ids=$ids, needsFullSync=$needsFullSync]';
String toString() =>
'AuditDeletesResponseDto[ids=$ids, needsFullSync=$needsFullSync, requestedAt=$requestedAt]';
Map<String, dynamic> toJson() {
final json = <String, dynamic>{};
json[r'ids'] = this.ids;
json[r'needsFullSync'] = this.needsFullSync;
json[r'ids'] = this.ids;
json[r'needsFullSync'] = this.needsFullSync;
if (this.requestedAt != null) {
json[r'requestedAt'] = this.requestedAt!.toUtc().toIso8601String();
} else {
// json[r'requestedAt'] = null;
}
return json;
}
@@ -54,12 +73,16 @@ class AuditDeletesResponseDto {
? (json[r'ids'] as Iterable).cast<String>().toList(growable: false)
: const [],
needsFullSync: mapValueOfType<bool>(json, r'needsFullSync')!,
requestedAt: mapDateTime(json, r'requestedAt', ''),
);
}
return null;
}
static List<AuditDeletesResponseDto> listFromJson(dynamic json, {bool growable = false,}) {
static List<AuditDeletesResponseDto> listFromJson(
dynamic json, {
bool growable = false,
}) {
final result = <AuditDeletesResponseDto>[];
if (json is List && json.isNotEmpty) {
for (final row in json) {
@@ -87,13 +110,19 @@ class AuditDeletesResponseDto {
}
// maps a json object with a list of AuditDeletesResponseDto-objects as value to a dart map
static Map<String, List<AuditDeletesResponseDto>> mapListFromJson(dynamic json, {bool growable = false,}) {
static Map<String, List<AuditDeletesResponseDto>> mapListFromJson(
dynamic json, {
bool growable = false,
}) {
final map = <String, List<AuditDeletesResponseDto>>{};
if (json is Map && json.isNotEmpty) {
// ignore: parameter_assignments
json = json.cast<String, dynamic>();
for (final entry in json.entries) {
map[entry.key] = AuditDeletesResponseDto.listFromJson(entry.value, growable: growable,);
map[entry.key] = AuditDeletesResponseDto.listFromJson(
entry.value,
growable: growable,
);
}
}
return map;
@@ -105,4 +134,3 @@ class AuditDeletesResponseDto {
'needsFullSync',
};
}
@@ -26,6 +26,11 @@ void main() {
// TODO
});
// DateTime requestedAt
test('to test the property `requestedAt`', () async {
// TODO
});
});
+4
View File
@@ -7326,6 +7326,10 @@
},
"needsFullSync": {
"type": "boolean"
},
"requestedAt": {
"format": "date-time",
"type": "string"
}
},
"required": [
+1
View File
@@ -29,6 +29,7 @@ export enum PathEntityType {
export class AuditDeletesResponseDto {
needsFullSync!: boolean;
ids!: string[];
requestedAt?: Date;
}
export class FileReportDto {
@@ -61,6 +61,7 @@ describe(AuditService.name, () => {
await expect(sut.getDeletes(authStub.admin, { after: date, entityType: EntityType.ASSET })).resolves.toEqual({
needsFullSync: true,
ids: [],
requestedAt: expect.any(Date),
});
expect(auditMock.getAfter).toHaveBeenCalledWith(date, {
@@ -77,6 +78,7 @@ describe(AuditService.name, () => {
await expect(sut.getDeletes(authStub.admin, { after: date, entityType: EntityType.ASSET })).resolves.toEqual({
needsFullSync: false,
ids: ['asset-deleted'],
requestedAt: expect.any(Date),
});
expect(auditMock.getAfter).toHaveBeenCalledWith(date, {
+5 -4
View File
@@ -5,7 +5,7 @@ import { DateTime } from 'luxon';
import { resolve } from 'node:path';
import { AccessCore, Permission } from '../access';
import { AuthDto } from '../auth';
import { AUDIT_LOG_MAX_DURATION } from '../domain.constant';
import { AUDIT_LOG_CLEANUP_DURATION, AUDIT_LOG_MAX_DURATION } from '../domain.constant';
import { usePagination } from '../domain.util';
import { JOBS_ASSET_PAGINATION_SIZE } from '../job';
import {
@@ -45,7 +45,7 @@ export class AuditService {
}
async handleCleanup(): Promise<boolean> {
await this.repository.removeBefore(DateTime.now().minus(AUDIT_LOG_MAX_DURATION).toJSDate());
await this.repository.removeBefore(DateTime.now().minus(AUDIT_LOG_CLEANUP_DURATION).toJSDate());
return true;
}
@@ -53,15 +53,16 @@ export class AuditService {
const userId = dto.userId || auth.user.id;
await this.access.requirePermission(auth, Permission.TIMELINE_READ, userId);
const now = DateTime.utc();
const duration = now.diff(DateTime.fromJSDate(dto.after));
const audits = await this.repository.getAfter(dto.after, {
ownerId: userId,
entityType: dto.entityType,
action: DatabaseAction.DELETE,
});
const duration = DateTime.now().diff(DateTime.fromJSDate(dto.after));
return {
requestedAt: now.toJSDate(),
needsFullSync: duration > AUDIT_LOG_MAX_DURATION,
ids: audits.map(({ entityId }) => entityId),
};
+1
View File
@@ -4,6 +4,7 @@ import { readFileSync } from 'node:fs';
import { extname, join } from 'node:path';
export const AUDIT_LOG_MAX_DURATION = Duration.fromObject({ days: 100 });
export const AUDIT_LOG_CLEANUP_DURATION = Duration.fromObject({ days: 101 });
export const ONE_HOUR = Duration.fromObject({ hours: 1 });
export interface IVersion {