Compare commits

...

8 Commits

Author SHA1 Message Date
Alex Tran d8413ea2ce fix 2024-03-08 10:50:03 -06:00
Alex Tran 81d6f69a47 resolve conflict 2024-03-08 10:48:28 -06:00
Alex 6feac7df20 Merge branch 'main' into dev/fix-sync-time-mismatch 2023-12-11 10:51:48 -06:00
Fynn Petersen-Frey de7475f3fc prettier 2023-11-29 20:57:59 +01:00
Fynn Petersen-Frey 7f5d554d1a fix tests 2023-11-29 20:38:13 +01:00
Fynn Petersen-Frey e033e9aad2 timeOfRequest -> requestedAt 2023-11-29 19:46:21 +01:00
Fynn Petersen-Frey 061c567468 prettier 2023-11-29 17:17:44 +01:00
Fynn Petersen-Frey 4ce3676127 fix(mobile): sync issue due to time mismatch 2023-11-29 17:08:03 +01:00
10 changed files with 103 additions and 46 deletions
+15 -7
View File
@@ -49,16 +49,23 @@ class AssetService {
return changes; return changes;
} }
/// Returns `(null, null)` if changes are invalid -> requires full sync /// Returns `(null, null, time)` if changes are invalid -> requires full sync
Future<(List<Asset>? toUpsert, List<String>? toDelete)> Future<(List<Asset>? toUpsert, List<String>? toDelete, DateTime? time)>
_getRemoteAssetChanges(User user, DateTime since) async { _getRemoteAssetChanges(User user, DateTime since) async {
final deleted = await _apiService.auditApi final deleted = await _apiService.auditApi
.getAuditDeletes(since, EntityType.ASSET, userId: user.id); .getAuditDeletes(since, EntityType.ASSET, userId: user.id);
if (deleted == null || deleted.needsFullSync) return (null, null); if (deleted == null || deleted.needsFullSync) {
return (null, null, deleted?.requestedAt);
}
final assetDto = await _apiService.assetApi final assetDto = await _apiService.assetApi
.getAllAssets(userId: user.id, updatedAfter: since); .getAllAssets(userId: user.id, updatedAfter: since);
if (assetDto == null) return (null, null); if (assetDto == null) return (null, null, deleted.requestedAt);
return (assetDto.map(Asset.remote).toList(), deleted.ids); return (
assetDto.map(Asset.remote).toList(),
deleted.ids,
deleted.requestedAt
);
} }
/// Returns the list of people of the given asset id. /// Returns the list of people of the given asset id.
@@ -83,10 +90,11 @@ class AssetService {
} }
/// Returns `null` if the server state did not change, else list of assets /// Returns `null` if the server state did not change, else list of assets
Future<List<Asset>?> _getRemoteAssets(User user) async {
Future<List<Asset>?> _getRemoteAssets(User user, DateTime now) async {
const int chunkSize = 10000; const int chunkSize = 10000;
try { try {
final DateTime now = DateTime.now().toUtc();
final List<Asset> allAssets = []; final List<Asset> allAssets = [];
for (int i = 0;; i += chunkSize) { for (int i = 0;; i += chunkSize) {
final List<AssetResponseDto>? assets = final List<AssetResponseDto>? assets =
+28 -22
View File
@@ -41,17 +41,20 @@ class SyncService {
/// Returns `true` if there were any changes /// Returns `true` if there were any changes
Future<bool> syncRemoteAssetsToDb( Future<bool> syncRemoteAssetsToDb(
User user, User user,
Future<(List<Asset>? toUpsert, List<String>? toDelete)> Function( Future<(List<Asset>? toUpsert, List<String>? toDelete, DateTime? time)>
Function(
User user, User user,
DateTime since, DateTime since,
) getChangedAssets, ) getChangedAssets,
FutureOr<List<Asset>?> Function(User user) loadAssets, FutureOr<List<Asset>?> Function(User user, DateTime now) loadAssets,
) => ) =>
_lock.run( _lock.run(() async {
() async => final (changes, serverTime) =
await _syncRemoteAssetChanges(user, getChangedAssets) ?? await _syncRemoteAssetChanges(user, getChangedAssets);
await _syncRemoteAssetsFull(user, loadAssets), if (changes != null) return changes;
); final time = serverTime ?? DateTime.now().toUtc();
return await _syncRemoteAssetsFull(user, time, loadAssets);
});
/// Syncs remote albums to the database /// Syncs remote albums to the database
/// returns `true` if there were any changes /// returns `true` if there were any changes
@@ -146,19 +149,22 @@ class SyncService {
return true; return true;
} }
/// Efficiently syncs assets via changes. Returns `null` when a full sync is required. /// Efficiently syncs assets via changes. Returns `(null, serverTime)` when a full sync is required.
Future<bool?> _syncRemoteAssetChanges( Future<(bool?, DateTime? serverTime)> _syncRemoteAssetChanges(
User user, User user,
Future<(List<Asset>? toUpsert, List<String>? toDelete)> Function( Future<(List<Asset>? toUpsert, List<String>? toDelete, DateTime? time)>
Function(
User user, User user,
DateTime since, DateTime since,
) getChangedAssets, ) getChangedAssets,
) async { ) async {
final DateTime? since = _db.eTags.getByIdSync(user.id)?.time?.toUtc(); final DateTime? since = _db.eTags.getByIdSync(user.id)?.time?.toUtc();
if (since == null) return null; final DateTime now = DateTime.now().toUtc();
final DateTime now = DateTime.now(); final (toUpsert, toDelete, serverTime) =
final (toUpsert, toDelete) = await getChangedAssets(user, since); await getChangedAssets(user, since ?? now);
if (toUpsert == null || toDelete == null) return null; if (since == null || toUpsert == null || toDelete == null) {
return (null, serverTime);
}
try { try {
if (toDelete.isNotEmpty) { if (toDelete.isNotEmpty) {
await handleRemoteAssetRemoval(toDelete); await handleRemoteAssetRemoval(toDelete);
@@ -168,14 +174,14 @@ class SyncService {
await upsertAssetsWithExif(updated); await upsertAssetsWithExif(updated);
} }
if (toUpsert.isNotEmpty || toDelete.isNotEmpty) { if (toUpsert.isNotEmpty || toDelete.isNotEmpty) {
await _updateUserAssetsETag(user, now); await _updateUserAssetsETag(user, serverTime ?? now);
return true; return (true, serverTime);
} }
return false; return (false, serverTime);
} on IsarError catch (e) { } on IsarError catch (e) {
_log.severe("Failed to sync remote assets to db", e); _log.severe("Failed to sync remote assets to db", e);
} }
return null; return (null, serverTime);
} }
/// Deletes remote-only assets, updates merged assets to be local-only /// Deletes remote-only assets, updates merged assets to be local-only
@@ -202,11 +208,11 @@ class SyncService {
/// Syncs assets by loading and comparing all assets from the server. /// Syncs assets by loading and comparing all assets from the server.
Future<bool> _syncRemoteAssetsFull( Future<bool> _syncRemoteAssetsFull(
User user, final User user,
FutureOr<List<Asset>?> Function(User user) loadAssets, final DateTime now,
final FutureOr<List<Asset>?> Function(User user, DateTime now) loadAssets,
) async { ) async {
final DateTime now = DateTime.now().toUtc(); final List<Asset>? remote = await loadAssets(user, now);
final List<Asset>? remote = await loadAssets(user);
if (remote == null) { if (remote == null) {
return false; return false;
} }
+1
View File
@@ -10,6 +10,7 @@ Name | Type | Description | Notes
------------ | ------------- | ------------- | ------------- ------------ | ------------- | ------------- | -------------
**ids** | **List<String>** | | [default to const []] **ids** | **List<String>** | | [default to const []]
**needsFullSync** | **bool** | | **needsFullSync** | **bool** | |
**requestedAt** | [**DateTime**](DateTime.md) | | [optional]
[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) [[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)
+41 -13
View File
@@ -15,30 +15,49 @@ class AuditDeletesResponseDto {
AuditDeletesResponseDto({ AuditDeletesResponseDto({
this.ids = const [], this.ids = const [],
required this.needsFullSync, required this.needsFullSync,
this.requestedAt,
}); });
List<String> ids; List<String> ids;
bool needsFullSync; bool needsFullSync;
///
/// Please note: This property should have been non-nullable! Since the specification file
/// does not include a default value (using the "default:" property), however, the generated
/// source code must fall back to having a nullable type.
/// Consider adding a "default:" property in the specification file to hide this note.
///
DateTime? requestedAt;
@override @override
bool operator ==(Object other) => identical(this, other) || other is AuditDeletesResponseDto && bool operator ==(Object other) =>
_deepEquality.equals(other.ids, ids) && identical(this, other) ||
other.needsFullSync == needsFullSync; other is AuditDeletesResponseDto &&
other.ids == ids &&
other.needsFullSync == needsFullSync &&
other.requestedAt == requestedAt;
@override @override
int get hashCode => int get hashCode =>
// ignore: unnecessary_parenthesis // ignore: unnecessary_parenthesis
(ids.hashCode) + (ids.hashCode) +
(needsFullSync.hashCode); (needsFullSync.hashCode) +
(requestedAt == null ? 0 : requestedAt!.hashCode);
@override @override
String toString() => 'AuditDeletesResponseDto[ids=$ids, needsFullSync=$needsFullSync]'; String toString() =>
'AuditDeletesResponseDto[ids=$ids, needsFullSync=$needsFullSync, requestedAt=$requestedAt]';
Map<String, dynamic> toJson() { Map<String, dynamic> toJson() {
final json = <String, dynamic>{}; final json = <String, dynamic>{};
json[r'ids'] = this.ids; json[r'ids'] = this.ids;
json[r'needsFullSync'] = this.needsFullSync; json[r'needsFullSync'] = this.needsFullSync;
if (this.requestedAt != null) {
json[r'requestedAt'] = this.requestedAt!.toUtc().toIso8601String();
} else {
// json[r'requestedAt'] = null;
}
return json; return json;
} }
@@ -54,12 +73,16 @@ class AuditDeletesResponseDto {
? (json[r'ids'] as Iterable).cast<String>().toList(growable: false) ? (json[r'ids'] as Iterable).cast<String>().toList(growable: false)
: const [], : const [],
needsFullSync: mapValueOfType<bool>(json, r'needsFullSync')!, needsFullSync: mapValueOfType<bool>(json, r'needsFullSync')!,
requestedAt: mapDateTime(json, r'requestedAt', ''),
); );
} }
return null; return null;
} }
static List<AuditDeletesResponseDto> listFromJson(dynamic json, {bool growable = false,}) { static List<AuditDeletesResponseDto> listFromJson(
dynamic json, {
bool growable = false,
}) {
final result = <AuditDeletesResponseDto>[]; final result = <AuditDeletesResponseDto>[];
if (json is List && json.isNotEmpty) { if (json is List && json.isNotEmpty) {
for (final row in json) { for (final row in json) {
@@ -87,13 +110,19 @@ class AuditDeletesResponseDto {
} }
// maps a json object with a list of AuditDeletesResponseDto-objects as value to a dart map // maps a json object with a list of AuditDeletesResponseDto-objects as value to a dart map
static Map<String, List<AuditDeletesResponseDto>> mapListFromJson(dynamic json, {bool growable = false,}) { static Map<String, List<AuditDeletesResponseDto>> mapListFromJson(
dynamic json, {
bool growable = false,
}) {
final map = <String, List<AuditDeletesResponseDto>>{}; final map = <String, List<AuditDeletesResponseDto>>{};
if (json is Map && json.isNotEmpty) { if (json is Map && json.isNotEmpty) {
// ignore: parameter_assignments // ignore: parameter_assignments
json = json.cast<String, dynamic>(); json = json.cast<String, dynamic>();
for (final entry in json.entries) { for (final entry in json.entries) {
map[entry.key] = AuditDeletesResponseDto.listFromJson(entry.value, growable: growable,); map[entry.key] = AuditDeletesResponseDto.listFromJson(
entry.value,
growable: growable,
);
} }
} }
return map; return map;
@@ -105,4 +134,3 @@ class AuditDeletesResponseDto {
'needsFullSync', 'needsFullSync',
}; };
} }
@@ -26,6 +26,11 @@ void main() {
// TODO // TODO
}); });
// DateTime requestedAt
test('to test the property `requestedAt`', () async {
// TODO
});
}); });
+4
View File
@@ -7326,6 +7326,10 @@
}, },
"needsFullSync": { "needsFullSync": {
"type": "boolean" "type": "boolean"
},
"requestedAt": {
"format": "date-time",
"type": "string"
} }
}, },
"required": [ "required": [
+1
View File
@@ -29,6 +29,7 @@ export enum PathEntityType {
export class AuditDeletesResponseDto { export class AuditDeletesResponseDto {
needsFullSync!: boolean; needsFullSync!: boolean;
ids!: string[]; ids!: string[];
requestedAt?: Date;
} }
export class FileReportDto { export class FileReportDto {
@@ -61,6 +61,7 @@ describe(AuditService.name, () => {
await expect(sut.getDeletes(authStub.admin, { after: date, entityType: EntityType.ASSET })).resolves.toEqual({ await expect(sut.getDeletes(authStub.admin, { after: date, entityType: EntityType.ASSET })).resolves.toEqual({
needsFullSync: true, needsFullSync: true,
ids: [], ids: [],
requestedAt: expect.any(Date),
}); });
expect(auditMock.getAfter).toHaveBeenCalledWith(date, { expect(auditMock.getAfter).toHaveBeenCalledWith(date, {
@@ -77,6 +78,7 @@ describe(AuditService.name, () => {
await expect(sut.getDeletes(authStub.admin, { after: date, entityType: EntityType.ASSET })).resolves.toEqual({ await expect(sut.getDeletes(authStub.admin, { after: date, entityType: EntityType.ASSET })).resolves.toEqual({
needsFullSync: false, needsFullSync: false,
ids: ['asset-deleted'], ids: ['asset-deleted'],
requestedAt: expect.any(Date),
}); });
expect(auditMock.getAfter).toHaveBeenCalledWith(date, { expect(auditMock.getAfter).toHaveBeenCalledWith(date, {
+5 -4
View File
@@ -5,7 +5,7 @@ import { DateTime } from 'luxon';
import { resolve } from 'node:path'; import { resolve } from 'node:path';
import { AccessCore, Permission } from '../access'; import { AccessCore, Permission } from '../access';
import { AuthDto } from '../auth'; import { AuthDto } from '../auth';
import { AUDIT_LOG_MAX_DURATION } from '../domain.constant'; import { AUDIT_LOG_CLEANUP_DURATION, AUDIT_LOG_MAX_DURATION } from '../domain.constant';
import { usePagination } from '../domain.util'; import { usePagination } from '../domain.util';
import { JOBS_ASSET_PAGINATION_SIZE } from '../job'; import { JOBS_ASSET_PAGINATION_SIZE } from '../job';
import { import {
@@ -45,7 +45,7 @@ export class AuditService {
} }
async handleCleanup(): Promise<boolean> { async handleCleanup(): Promise<boolean> {
await this.repository.removeBefore(DateTime.now().minus(AUDIT_LOG_MAX_DURATION).toJSDate()); await this.repository.removeBefore(DateTime.now().minus(AUDIT_LOG_CLEANUP_DURATION).toJSDate());
return true; return true;
} }
@@ -53,15 +53,16 @@ export class AuditService {
const userId = dto.userId || auth.user.id; const userId = dto.userId || auth.user.id;
await this.access.requirePermission(auth, Permission.TIMELINE_READ, userId); await this.access.requirePermission(auth, Permission.TIMELINE_READ, userId);
const now = DateTime.utc();
const duration = now.diff(DateTime.fromJSDate(dto.after));
const audits = await this.repository.getAfter(dto.after, { const audits = await this.repository.getAfter(dto.after, {
ownerId: userId, ownerId: userId,
entityType: dto.entityType, entityType: dto.entityType,
action: DatabaseAction.DELETE, action: DatabaseAction.DELETE,
}); });
const duration = DateTime.now().diff(DateTime.fromJSDate(dto.after));
return { return {
requestedAt: now.toJSDate(),
needsFullSync: duration > AUDIT_LOG_MAX_DURATION, needsFullSync: duration > AUDIT_LOG_MAX_DURATION,
ids: audits.map(({ entityId }) => entityId), ids: audits.map(({ entityId }) => entityId),
}; };
+1
View File
@@ -4,6 +4,7 @@ import { readFileSync } from 'node:fs';
import { extname, join } from 'node:path'; import { extname, join } from 'node:path';
export const AUDIT_LOG_MAX_DURATION = Duration.fromObject({ days: 100 }); export const AUDIT_LOG_MAX_DURATION = Duration.fromObject({ days: 100 });
export const AUDIT_LOG_CLEANUP_DURATION = Duration.fromObject({ days: 101 });
export const ONE_HOUR = Duration.fromObject({ hours: 1 }); export const ONE_HOUR = Duration.fromObject({ hours: 1 });
export interface IVersion { export interface IVersion {