diff --git a/docs/docs/features/libraries.md b/docs/docs/features/libraries.md index 6dcb9982bfbe8..995d65249278f 100644 --- a/docs/docs/features/libraries.md +++ b/docs/docs/features/libraries.md @@ -42,23 +42,24 @@ Finally, files can be deleted from Immich via the `Remove Offline Files` job. Th ### Import Paths -External libraries use import paths to determine which files to scan. Each library can have multiple import paths so that files from different locations can be added to the same library. Import paths are scanned recursively, and if a file is in multiple import paths, it will only be added once. If the import paths are edited in a way that an external file is no longer in any import path, it will be removed from the library in the same way a deleted file would. If the file is moved back to an import path, it will be added again as if it was a new file. +External libraries use import paths to determine which files to scan. Each library can have multiple import paths so that files from different locations can be added to the same library. Import paths are scanned recursively, and if a file is in multiple import paths, it will only be added once. Each import file must be a readable directory that exists on the filesystem; the import path dialog will alert you of any paths that are not accessible. + +If the import paths are edited in a way that an external file is no longer in any import path, it will be removed from the library in the same way a deleted file would. If the file is moved back to an import path, it will be added again as if it was a new file. ### Troubleshooting -Sometimes, an external library will not scan correctly. This can happen if the immich_server or immich_microservices can't access the files. Here are some things to check: +Sometimes, an external library will not scan correctly. This can happen if immich_server or immich_microservices can't access the files. Here are some things to check: -- Is the external path set correctly? +- Is the external path set correctly? Each import path must be contained in the external path. +- Make sure the external path does not contain spaces - In the docker-compose file, are the volumes mounted correctly? - Are the volumes identical between the `server` and `microservices` container? - Are the import paths set correctly, and do they match the path set in docker-compose file? -- Are you using symbolic link in your import library? +- Make sure you don't use symlinks in your import libraries, and that you aren't linking across docker mounts. - Are the permissions set correctly? -- Are you using forward slashes everywhere? (`/`) -- Are you using symlink across docker mounts? -- Are you using [spaces in the internal path](/docs/features/libraries#:~:text=can%20be%20accessed.-,NOTE,-Spaces%20in%20the)? +- Make sure you are using forward slashes (`/`) and not backward slashes. -If all else fails, you can always start a shell inside the container and check if the path is accessible. For example, `docker exec -it immich_microservices /bin/bash` will start a bash shell. If your import path, for instance, is `/data/import/photos`, you can check if the files are accessible by running `ls /data/import/photos`. Also check the `immich_server` container in the same way. +To validate that Immich can reach your external library, start a shell inside the container. Run `docker exec -it immich_microservices /bin/bash` to a bash shell. If your import path is `/data/import/photos`, check it with `ls /data/import/photos`. Do the same check for the `immich_server` container. If you cannot access this directory in both the `microservices` and `server` containers, Immich won't be able to import files. ### Security Considerations diff --git a/mobile/openapi/.openapi-generator/FILES b/mobile/openapi/.openapi-generator/FILES index f42a9c82ca521..0679a1749d756 100644 --- a/mobile/openapi/.openapi-generator/FILES +++ b/mobile/openapi/.openapi-generator/FILES @@ -182,6 +182,9 @@ doc/UserAvatarColor.md doc/UserDto.md doc/UserResponseDto.md doc/ValidateAccessTokenResponseDto.md +doc/ValidateLibraryDto.md +doc/ValidateLibraryImportPathResponseDto.md +doc/ValidateLibraryResponseDto.md doc/VideoCodec.md git_push.sh lib/api.dart @@ -372,6 +375,9 @@ lib/model/user_avatar_color.dart lib/model/user_dto.dart lib/model/user_response_dto.dart lib/model/validate_access_token_response_dto.dart +lib/model/validate_library_dto.dart +lib/model/validate_library_import_path_response_dto.dart +lib/model/validate_library_response_dto.dart lib/model/video_codec.dart pubspec.yaml test/activity_api_test.dart @@ -553,4 +559,7 @@ test/user_avatar_color_test.dart test/user_dto_test.dart test/user_response_dto_test.dart test/validate_access_token_response_dto_test.dart +test/validate_library_dto_test.dart +test/validate_library_import_path_response_dto_test.dart +test/validate_library_response_dto_test.dart test/video_codec_test.dart diff --git a/mobile/openapi/README.md b/mobile/openapi/README.md index 9e241b0a44e30..8cb0919df181c 100644 --- a/mobile/openapi/README.md +++ b/mobile/openapi/README.md @@ -141,6 +141,7 @@ Class | Method | HTTP request | Description *LibraryApi* | [**removeOfflineFiles**](doc//LibraryApi.md#removeofflinefiles) | **POST** /library/{id}/removeOffline | *LibraryApi* | [**scanLibrary**](doc//LibraryApi.md#scanlibrary) | **POST** /library/{id}/scan | *LibraryApi* | [**updateLibrary**](doc//LibraryApi.md#updatelibrary) | **PUT** /library/{id} | +*LibraryApi* | [**validate**](doc//LibraryApi.md#validate) | **POST** /library/{id}/validate | *OAuthApi* | [**finishOAuth**](doc//OAuthApi.md#finishoauth) | **POST** /oauth/callback | *OAuthApi* | [**linkOAuthAccount**](doc//OAuthApi.md#linkoauthaccount) | **POST** /oauth/link | *OAuthApi* | [**redirectOAuthToMobile**](doc//OAuthApi.md#redirectoauthtomobile) | **GET** /oauth/mobile-redirect | @@ -372,6 +373,9 @@ Class | Method | HTTP request | Description - [UserDto](doc//UserDto.md) - [UserResponseDto](doc//UserResponseDto.md) - [ValidateAccessTokenResponseDto](doc//ValidateAccessTokenResponseDto.md) + - [ValidateLibraryDto](doc//ValidateLibraryDto.md) + - [ValidateLibraryImportPathResponseDto](doc//ValidateLibraryImportPathResponseDto.md) + - [ValidateLibraryResponseDto](doc//ValidateLibraryResponseDto.md) - [VideoCodec](doc//VideoCodec.md) diff --git a/mobile/openapi/doc/LibraryApi.md b/mobile/openapi/doc/LibraryApi.md index 5ecd46503a648..ecaa73d86be9e 100644 --- a/mobile/openapi/doc/LibraryApi.md +++ b/mobile/openapi/doc/LibraryApi.md @@ -17,6 +17,7 @@ Method | HTTP request | Description [**removeOfflineFiles**](LibraryApi.md#removeofflinefiles) | **POST** /library/{id}/removeOffline | [**scanLibrary**](LibraryApi.md#scanlibrary) | **POST** /library/{id}/scan | [**updateLibrary**](LibraryApi.md#updatelibrary) | **PUT** /library/{id} | +[**validate**](LibraryApi.md#validate) | **POST** /library/{id}/validate | # **createLibrary** @@ -456,3 +457,60 @@ Name | Type | Description | Notes [[Back to top]](#) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to Model list]](../README.md#documentation-for-models) [[Back to README]](../README.md) +# **validate** +> ValidateLibraryResponseDto validate(id, validateLibraryDto) + + + +### Example +```dart +import 'package:openapi/api.dart'; +// TODO Configure API key authorization: cookie +//defaultApiClient.getAuthentication('cookie').apiKey = 'YOUR_API_KEY'; +// uncomment below to setup prefix (e.g. Bearer) for API key, if needed +//defaultApiClient.getAuthentication('cookie').apiKeyPrefix = 'Bearer'; +// TODO Configure API key authorization: api_key +//defaultApiClient.getAuthentication('api_key').apiKey = 'YOUR_API_KEY'; +// uncomment below to setup prefix (e.g. Bearer) for API key, if needed +//defaultApiClient.getAuthentication('api_key').apiKeyPrefix = 'Bearer'; +// TODO Configure HTTP Bearer authorization: bearer +// Case 1. Use String Token +//defaultApiClient.getAuthentication('bearer').setAccessToken('YOUR_ACCESS_TOKEN'); +// Case 2. Use Function which generate token. +// String yourTokenGeneratorFunction() { ... } +//defaultApiClient.getAuthentication('bearer').setAccessToken(yourTokenGeneratorFunction); + +final api_instance = LibraryApi(); +final id = 38400000-8cf0-11bd-b23e-10b96e4ef00d; // String | +final validateLibraryDto = ValidateLibraryDto(); // ValidateLibraryDto | + +try { + final result = api_instance.validate(id, validateLibraryDto); + print(result); +} catch (e) { + print('Exception when calling LibraryApi->validate: $e\n'); +} +``` + +### Parameters + +Name | Type | Description | Notes +------------- | ------------- | ------------- | ------------- + **id** | **String**| | + **validateLibraryDto** | [**ValidateLibraryDto**](ValidateLibraryDto.md)| | + +### Return type + +[**ValidateLibraryResponseDto**](ValidateLibraryResponseDto.md) + +### Authorization + +[cookie](../README.md#cookie), [api_key](../README.md#api_key), [bearer](../README.md#bearer) + +### HTTP request headers + + - **Content-Type**: application/json + - **Accept**: application/json + +[[Back to top]](#) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to Model list]](../README.md#documentation-for-models) [[Back to README]](../README.md) + diff --git a/mobile/openapi/doc/ValidateLibraryDto.md b/mobile/openapi/doc/ValidateLibraryDto.md new file mode 100644 index 0000000000000..13c74a6a66d73 --- /dev/null +++ b/mobile/openapi/doc/ValidateLibraryDto.md @@ -0,0 +1,16 @@ +# openapi.model.ValidateLibraryDto + +## Load the model package +```dart +import 'package:openapi/api.dart'; +``` + +## Properties +Name | Type | Description | Notes +------------ | ------------- | ------------- | ------------- +**exclusionPatterns** | **List** | | [optional] [default to const []] +**importPaths** | **List** | | [optional] [default to const []] + +[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) + + diff --git a/mobile/openapi/doc/ValidateLibraryImportPathResponseDto.md b/mobile/openapi/doc/ValidateLibraryImportPathResponseDto.md new file mode 100644 index 0000000000000..4601d8d2f2547 --- /dev/null +++ b/mobile/openapi/doc/ValidateLibraryImportPathResponseDto.md @@ -0,0 +1,17 @@ +# openapi.model.ValidateLibraryImportPathResponseDto + +## Load the model package +```dart +import 'package:openapi/api.dart'; +``` + +## Properties +Name | Type | Description | Notes +------------ | ------------- | ------------- | ------------- +**importPath** | **String** | | +**isValid** | **bool** | | [optional] [default to false] +**message** | **String** | | [optional] + +[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) + + diff --git a/mobile/openapi/doc/ValidateLibraryResponseDto.md b/mobile/openapi/doc/ValidateLibraryResponseDto.md new file mode 100644 index 0000000000000..7b22fdfe90d6e --- /dev/null +++ b/mobile/openapi/doc/ValidateLibraryResponseDto.md @@ -0,0 +1,15 @@ +# openapi.model.ValidateLibraryResponseDto + +## Load the model package +```dart +import 'package:openapi/api.dart'; +``` + +## Properties +Name | Type | Description | Notes +------------ | ------------- | ------------- | ------------- +**importPaths** | [**List**](ValidateLibraryImportPathResponseDto.md) | | [optional] [default to const []] + +[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) + + diff --git a/mobile/openapi/lib/api.dart b/mobile/openapi/lib/api.dart index 3816a05b3b2e1..72a6567648de8 100644 --- a/mobile/openapi/lib/api.dart +++ b/mobile/openapi/lib/api.dart @@ -209,6 +209,9 @@ part 'model/user_avatar_color.dart'; part 'model/user_dto.dart'; part 'model/user_response_dto.dart'; part 'model/validate_access_token_response_dto.dart'; +part 'model/validate_library_dto.dart'; +part 'model/validate_library_import_path_response_dto.dart'; +part 'model/validate_library_response_dto.dart'; part 'model/video_codec.dart'; diff --git a/mobile/openapi/lib/api/library_api.dart b/mobile/openapi/lib/api/library_api.dart index 23de593eac9f8..21cec23eba99c 100644 --- a/mobile/openapi/lib/api/library_api.dart +++ b/mobile/openapi/lib/api/library_api.dart @@ -378,4 +378,56 @@ class LibraryApi { } return null; } + + /// Performs an HTTP 'POST /library/{id}/validate' operation and returns the [Response]. + /// Parameters: + /// + /// * [String] id (required): + /// + /// * [ValidateLibraryDto] validateLibraryDto (required): + Future validateWithHttpInfo(String id, ValidateLibraryDto validateLibraryDto,) async { + // ignore: prefer_const_declarations + final path = r'/library/{id}/validate' + .replaceAll('{id}', id); + + // ignore: prefer_final_locals + Object? postBody = validateLibraryDto; + + final queryParams = []; + final headerParams = {}; + final formParams = {}; + + const contentTypes = ['application/json']; + + + return apiClient.invokeAPI( + path, + 'POST', + queryParams, + postBody, + headerParams, + formParams, + contentTypes.isEmpty ? null : contentTypes.first, + ); + } + + /// Parameters: + /// + /// * [String] id (required): + /// + /// * [ValidateLibraryDto] validateLibraryDto (required): + Future validate(String id, ValidateLibraryDto validateLibraryDto,) async { + final response = await validateWithHttpInfo(id, validateLibraryDto,); + if (response.statusCode >= HttpStatus.badRequest) { + throw ApiException(response.statusCode, await _decodeBodyBytes(response)); + } + // When a remote server returns no body with a status of 204, we shall not decode it. + // At the time of writing this, `dart:convert` will throw an "Unexpected end of input" + // FormatException when trying to decode an empty string. + if (response.body.isNotEmpty && response.statusCode != HttpStatus.noContent) { + return await apiClient.deserializeAsync(await _decodeBodyBytes(response), 'ValidateLibraryResponseDto',) as ValidateLibraryResponseDto; + + } + return null; + } } diff --git a/mobile/openapi/lib/api_client.dart b/mobile/openapi/lib/api_client.dart index 3a0e535e73df5..2df5e67119073 100644 --- a/mobile/openapi/lib/api_client.dart +++ b/mobile/openapi/lib/api_client.dart @@ -500,6 +500,12 @@ class ApiClient { return UserResponseDto.fromJson(value); case 'ValidateAccessTokenResponseDto': return ValidateAccessTokenResponseDto.fromJson(value); + case 'ValidateLibraryDto': + return ValidateLibraryDto.fromJson(value); + case 'ValidateLibraryImportPathResponseDto': + return ValidateLibraryImportPathResponseDto.fromJson(value); + case 'ValidateLibraryResponseDto': + return ValidateLibraryResponseDto.fromJson(value); case 'VideoCodec': return VideoCodecTypeTransformer().decode(value); default: diff --git a/mobile/openapi/lib/model/validate_library_dto.dart b/mobile/openapi/lib/model/validate_library_dto.dart new file mode 100644 index 0000000000000..a29a622079b82 --- /dev/null +++ b/mobile/openapi/lib/model/validate_library_dto.dart @@ -0,0 +1,108 @@ +// +// AUTO-GENERATED FILE, DO NOT MODIFY! +// +// @dart=2.12 + +// ignore_for_file: unused_element, unused_import +// ignore_for_file: always_put_required_named_parameters_first +// ignore_for_file: constant_identifier_names +// ignore_for_file: lines_longer_than_80_chars + +part of openapi.api; + +class ValidateLibraryDto { + /// Returns a new [ValidateLibraryDto] instance. + ValidateLibraryDto({ + this.exclusionPatterns = const [], + this.importPaths = const [], + }); + + List exclusionPatterns; + + List importPaths; + + @override + bool operator ==(Object other) => identical(this, other) || other is ValidateLibraryDto && + _deepEquality.equals(other.exclusionPatterns, exclusionPatterns) && + _deepEquality.equals(other.importPaths, importPaths); + + @override + int get hashCode => + // ignore: unnecessary_parenthesis + (exclusionPatterns.hashCode) + + (importPaths.hashCode); + + @override + String toString() => 'ValidateLibraryDto[exclusionPatterns=$exclusionPatterns, importPaths=$importPaths]'; + + Map toJson() { + final json = {}; + json[r'exclusionPatterns'] = this.exclusionPatterns; + json[r'importPaths'] = this.importPaths; + return json; + } + + /// Returns a new [ValidateLibraryDto] instance and imports its values from + /// [value] if it's a [Map], null otherwise. + // ignore: prefer_constructors_over_static_methods + static ValidateLibraryDto? fromJson(dynamic value) { + if (value is Map) { + final json = value.cast(); + + return ValidateLibraryDto( + exclusionPatterns: json[r'exclusionPatterns'] is Iterable + ? (json[r'exclusionPatterns'] as Iterable).cast().toList(growable: false) + : const [], + importPaths: json[r'importPaths'] is Iterable + ? (json[r'importPaths'] as Iterable).cast().toList(growable: false) + : const [], + ); + } + return null; + } + + static List listFromJson(dynamic json, {bool growable = false,}) { + final result = []; + if (json is List && json.isNotEmpty) { + for (final row in json) { + final value = ValidateLibraryDto.fromJson(row); + if (value != null) { + result.add(value); + } + } + } + return result.toList(growable: growable); + } + + static Map mapFromJson(dynamic json) { + final map = {}; + if (json is Map && json.isNotEmpty) { + json = json.cast(); // ignore: parameter_assignments + for (final entry in json.entries) { + final value = ValidateLibraryDto.fromJson(entry.value); + if (value != null) { + map[entry.key] = value; + } + } + } + return map; + } + + // maps a json object with a list of ValidateLibraryDto-objects as value to a dart map + static Map> mapListFromJson(dynamic json, {bool growable = false,}) { + final map = >{}; + if (json is Map && json.isNotEmpty) { + // ignore: parameter_assignments + json = json.cast(); + for (final entry in json.entries) { + map[entry.key] = ValidateLibraryDto.listFromJson(entry.value, growable: growable,); + } + } + return map; + } + + /// The list of required keys that must be present in a JSON. + static const requiredKeys = { + }; +} + diff --git a/mobile/openapi/lib/model/validate_library_import_path_response_dto.dart b/mobile/openapi/lib/model/validate_library_import_path_response_dto.dart new file mode 100644 index 0000000000000..1297c824c21c5 --- /dev/null +++ b/mobile/openapi/lib/model/validate_library_import_path_response_dto.dart @@ -0,0 +1,122 @@ +// +// AUTO-GENERATED FILE, DO NOT MODIFY! +// +// @dart=2.12 + +// ignore_for_file: unused_element, unused_import +// ignore_for_file: always_put_required_named_parameters_first +// ignore_for_file: constant_identifier_names +// ignore_for_file: lines_longer_than_80_chars + +part of openapi.api; + +class ValidateLibraryImportPathResponseDto { + /// Returns a new [ValidateLibraryImportPathResponseDto] instance. + ValidateLibraryImportPathResponseDto({ + required this.importPath, + this.isValid = false, + this.message, + }); + + String importPath; + + bool isValid; + + /// + /// Please note: This property should have been non-nullable! Since the specification file + /// does not include a default value (using the "default:" property), however, the generated + /// source code must fall back to having a nullable type. + /// Consider adding a "default:" property in the specification file to hide this note. + /// + String? message; + + @override + bool operator ==(Object other) => identical(this, other) || other is ValidateLibraryImportPathResponseDto && + other.importPath == importPath && + other.isValid == isValid && + other.message == message; + + @override + int get hashCode => + // ignore: unnecessary_parenthesis + (importPath.hashCode) + + (isValid.hashCode) + + (message == null ? 0 : message!.hashCode); + + @override + String toString() => 'ValidateLibraryImportPathResponseDto[importPath=$importPath, isValid=$isValid, message=$message]'; + + Map toJson() { + final json = {}; + json[r'importPath'] = this.importPath; + json[r'isValid'] = this.isValid; + if (this.message != null) { + json[r'message'] = this.message; + } else { + // json[r'message'] = null; + } + return json; + } + + /// Returns a new [ValidateLibraryImportPathResponseDto] instance and imports its values from + /// [value] if it's a [Map], null otherwise. + // ignore: prefer_constructors_over_static_methods + static ValidateLibraryImportPathResponseDto? fromJson(dynamic value) { + if (value is Map) { + final json = value.cast(); + + return ValidateLibraryImportPathResponseDto( + importPath: mapValueOfType(json, r'importPath')!, + isValid: mapValueOfType(json, r'isValid') ?? false, + message: mapValueOfType(json, r'message'), + ); + } + return null; + } + + static List listFromJson(dynamic json, {bool growable = false,}) { + final result = []; + if (json is List && json.isNotEmpty) { + for (final row in json) { + final value = ValidateLibraryImportPathResponseDto.fromJson(row); + if (value != null) { + result.add(value); + } + } + } + return result.toList(growable: growable); + } + + static Map mapFromJson(dynamic json) { + final map = {}; + if (json is Map && json.isNotEmpty) { + json = json.cast(); // ignore: parameter_assignments + for (final entry in json.entries) { + final value = ValidateLibraryImportPathResponseDto.fromJson(entry.value); + if (value != null) { + map[entry.key] = value; + } + } + } + return map; + } + + // maps a json object with a list of ValidateLibraryImportPathResponseDto-objects as value to a dart map + static Map> mapListFromJson(dynamic json, {bool growable = false,}) { + final map = >{}; + if (json is Map && json.isNotEmpty) { + // ignore: parameter_assignments + json = json.cast(); + for (final entry in json.entries) { + map[entry.key] = ValidateLibraryImportPathResponseDto.listFromJson(entry.value, growable: growable,); + } + } + return map; + } + + /// The list of required keys that must be present in a JSON. + static const requiredKeys = { + 'importPath', + }; +} + diff --git a/mobile/openapi/lib/model/validate_library_response_dto.dart b/mobile/openapi/lib/model/validate_library_response_dto.dart new file mode 100644 index 0000000000000..bb975da23aa20 --- /dev/null +++ b/mobile/openapi/lib/model/validate_library_response_dto.dart @@ -0,0 +1,97 @@ +// +// AUTO-GENERATED FILE, DO NOT MODIFY! +// +// @dart=2.12 + +// ignore_for_file: unused_element, unused_import +// ignore_for_file: always_put_required_named_parameters_first +// ignore_for_file: constant_identifier_names +// ignore_for_file: lines_longer_than_80_chars + +part of openapi.api; + +class ValidateLibraryResponseDto { + /// Returns a new [ValidateLibraryResponseDto] instance. + ValidateLibraryResponseDto({ + this.importPaths = const [], + }); + + List importPaths; + + @override + bool operator ==(Object other) => identical(this, other) || other is ValidateLibraryResponseDto && + _deepEquality.equals(other.importPaths, importPaths); + + @override + int get hashCode => + // ignore: unnecessary_parenthesis + (importPaths.hashCode); + + @override + String toString() => 'ValidateLibraryResponseDto[importPaths=$importPaths]'; + + Map toJson() { + final json = {}; + json[r'importPaths'] = this.importPaths; + return json; + } + + /// Returns a new [ValidateLibraryResponseDto] instance and imports its values from + /// [value] if it's a [Map], null otherwise. + // ignore: prefer_constructors_over_static_methods + static ValidateLibraryResponseDto? fromJson(dynamic value) { + if (value is Map) { + final json = value.cast(); + + return ValidateLibraryResponseDto( + importPaths: ValidateLibraryImportPathResponseDto.listFromJson(json[r'importPaths']), + ); + } + return null; + } + + static List listFromJson(dynamic json, {bool growable = false,}) { + final result = []; + if (json is List && json.isNotEmpty) { + for (final row in json) { + final value = ValidateLibraryResponseDto.fromJson(row); + if (value != null) { + result.add(value); + } + } + } + return result.toList(growable: growable); + } + + static Map mapFromJson(dynamic json) { + final map = {}; + if (json is Map && json.isNotEmpty) { + json = json.cast(); // ignore: parameter_assignments + for (final entry in json.entries) { + final value = ValidateLibraryResponseDto.fromJson(entry.value); + if (value != null) { + map[entry.key] = value; + } + } + } + return map; + } + + // maps a json object with a list of ValidateLibraryResponseDto-objects as value to a dart map + static Map> mapListFromJson(dynamic json, {bool growable = false,}) { + final map = >{}; + if (json is Map && json.isNotEmpty) { + // ignore: parameter_assignments + json = json.cast(); + for (final entry in json.entries) { + map[entry.key] = ValidateLibraryResponseDto.listFromJson(entry.value, growable: growable,); + } + } + return map; + } + + /// The list of required keys that must be present in a JSON. + static const requiredKeys = { + }; +} + diff --git a/mobile/openapi/test/library_api_test.dart b/mobile/openapi/test/library_api_test.dart index 85882edbd1f8b..20abcffe7036b 100644 --- a/mobile/openapi/test/library_api_test.dart +++ b/mobile/openapi/test/library_api_test.dart @@ -57,5 +57,10 @@ void main() { // TODO }); + //Future validate(String id, ValidateLibraryDto validateLibraryDto) async + test('test validate', () async { + // TODO + }); + }); } diff --git a/mobile/openapi/test/validate_library_dto_test.dart b/mobile/openapi/test/validate_library_dto_test.dart new file mode 100644 index 0000000000000..8d4922ee4a295 --- /dev/null +++ b/mobile/openapi/test/validate_library_dto_test.dart @@ -0,0 +1,32 @@ +// +// AUTO-GENERATED FILE, DO NOT MODIFY! +// +// @dart=2.12 + +// ignore_for_file: unused_element, unused_import +// ignore_for_file: always_put_required_named_parameters_first +// ignore_for_file: constant_identifier_names +// ignore_for_file: lines_longer_than_80_chars + +import 'package:openapi/api.dart'; +import 'package:test/test.dart'; + +// tests for ValidateLibraryDto +void main() { + // final instance = ValidateLibraryDto(); + + group('test ValidateLibraryDto', () { + // List exclusionPatterns (default value: const []) + test('to test the property `exclusionPatterns`', () async { + // TODO + }); + + // List importPaths (default value: const []) + test('to test the property `importPaths`', () async { + // TODO + }); + + + }); + +} diff --git a/mobile/openapi/test/validate_library_import_path_response_dto_test.dart b/mobile/openapi/test/validate_library_import_path_response_dto_test.dart new file mode 100644 index 0000000000000..a118698a13adc --- /dev/null +++ b/mobile/openapi/test/validate_library_import_path_response_dto_test.dart @@ -0,0 +1,37 @@ +// +// AUTO-GENERATED FILE, DO NOT MODIFY! +// +// @dart=2.12 + +// ignore_for_file: unused_element, unused_import +// ignore_for_file: always_put_required_named_parameters_first +// ignore_for_file: constant_identifier_names +// ignore_for_file: lines_longer_than_80_chars + +import 'package:openapi/api.dart'; +import 'package:test/test.dart'; + +// tests for ValidateLibraryImportPathResponseDto +void main() { + // final instance = ValidateLibraryImportPathResponseDto(); + + group('test ValidateLibraryImportPathResponseDto', () { + // String importPath + test('to test the property `importPath`', () async { + // TODO + }); + + // bool isValid (default value: false) + test('to test the property `isValid`', () async { + // TODO + }); + + // String message + test('to test the property `message`', () async { + // TODO + }); + + + }); + +} diff --git a/mobile/openapi/test/validate_library_response_dto_test.dart b/mobile/openapi/test/validate_library_response_dto_test.dart new file mode 100644 index 0000000000000..6750809066693 --- /dev/null +++ b/mobile/openapi/test/validate_library_response_dto_test.dart @@ -0,0 +1,27 @@ +// +// AUTO-GENERATED FILE, DO NOT MODIFY! +// +// @dart=2.12 + +// ignore_for_file: unused_element, unused_import +// ignore_for_file: always_put_required_named_parameters_first +// ignore_for_file: constant_identifier_names +// ignore_for_file: lines_longer_than_80_chars + +import 'package:openapi/api.dart'; +import 'package:test/test.dart'; + +// tests for ValidateLibraryResponseDto +void main() { + // final instance = ValidateLibraryResponseDto(); + + group('test ValidateLibraryResponseDto', () { + // List importPaths (default value: const []) + test('to test the property `importPaths`', () async { + // TODO + }); + + + }); + +} diff --git a/open-api/immich-openapi-specs.json b/open-api/immich-openapi-specs.json index 5dd394108ee06..e15073e1394bf 100644 --- a/open-api/immich-openapi-specs.json +++ b/open-api/immich-openapi-specs.json @@ -3618,6 +3618,58 @@ ] } }, + "/library/{id}/validate": { + "post": { + "operationId": "validate", + "parameters": [ + { + "name": "id", + "required": true, + "in": "path", + "schema": { + "format": "uuid", + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ValidateLibraryDto" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ValidateLibraryResponseDto" + } + } + }, + "description": "" + } + }, + "security": [ + { + "bearer": [] + }, + { + "cookie": [] + }, + { + "api_key": [] + } + ], + "tags": [ + "Library" + ] + } + }, "/oauth/authorize": { "post": { "operationId": "startOAuth", @@ -10406,6 +10458,52 @@ ], "type": "object" }, + "ValidateLibraryDto": { + "properties": { + "exclusionPatterns": { + "items": { + "type": "string" + }, + "type": "array" + }, + "importPaths": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "type": "object" + }, + "ValidateLibraryImportPathResponseDto": { + "properties": { + "importPath": { + "type": "string" + }, + "isValid": { + "default": false, + "type": "boolean" + }, + "message": { + "type": "string" + } + }, + "required": [ + "importPath" + ], + "type": "object" + }, + "ValidateLibraryResponseDto": { + "properties": { + "importPaths": { + "items": { + "$ref": "#/components/schemas/ValidateLibraryImportPathResponseDto" + }, + "type": "array" + } + }, + "type": "object" + }, "VideoCodec": { "enum": [ "h264", diff --git a/open-api/typescript-sdk/axios-client/api.ts b/open-api/typescript-sdk/axios-client/api.ts index 5d37706904929..49cf286f3b2a7 100644 --- a/open-api/typescript-sdk/axios-client/api.ts +++ b/open-api/typescript-sdk/axios-client/api.ts @@ -5316,6 +5316,63 @@ export interface ValidateAccessTokenResponseDto { */ 'authStatus': boolean; } +/** + * + * @export + * @interface ValidateLibraryDto + */ +export interface ValidateLibraryDto { + /** + * + * @type {Array} + * @memberof ValidateLibraryDto + */ + 'exclusionPatterns'?: Array; + /** + * + * @type {Array} + * @memberof ValidateLibraryDto + */ + 'importPaths'?: Array; +} +/** + * + * @export + * @interface ValidateLibraryImportPathResponseDto + */ +export interface ValidateLibraryImportPathResponseDto { + /** + * + * @type {string} + * @memberof ValidateLibraryImportPathResponseDto + */ + 'importPath': string; + /** + * + * @type {boolean} + * @memberof ValidateLibraryImportPathResponseDto + */ + 'isValid'?: boolean; + /** + * + * @type {string} + * @memberof ValidateLibraryImportPathResponseDto + */ + 'message'?: string; +} +/** + * + * @export + * @interface ValidateLibraryResponseDto + */ +export interface ValidateLibraryResponseDto { + /** + * + * @type {Array} + * @memberof ValidateLibraryResponseDto + */ + 'importPaths'?: Array; +} /** * * @export @@ -12813,6 +12870,54 @@ export const LibraryApiAxiosParamCreator = function (configuration?: Configurati localVarRequestOptions.headers = {...localVarHeaderParameter, ...headersFromBaseOptions, ...options.headers}; localVarRequestOptions.data = serializeDataIfNeeded(updateLibraryDto, localVarRequestOptions, configuration) + return { + url: toPathString(localVarUrlObj), + options: localVarRequestOptions, + }; + }, + /** + * + * @param {string} id + * @param {ValidateLibraryDto} validateLibraryDto + * @param {*} [options] Override http request option. + * @throws {RequiredError} + */ + validate: async (id: string, validateLibraryDto: ValidateLibraryDto, options: RawAxiosRequestConfig = {}): Promise => { + // verify required parameter 'id' is not null or undefined + assertParamExists('validate', 'id', id) + // verify required parameter 'validateLibraryDto' is not null or undefined + assertParamExists('validate', 'validateLibraryDto', validateLibraryDto) + const localVarPath = `/library/{id}/validate` + .replace(`{${"id"}}`, encodeURIComponent(String(id))); + // use dummy base URL string because the URL constructor only accepts absolute URLs. + const localVarUrlObj = new URL(localVarPath, DUMMY_BASE_URL); + let baseOptions; + if (configuration) { + baseOptions = configuration.baseOptions; + } + + const localVarRequestOptions = { method: 'POST', ...baseOptions, ...options}; + const localVarHeaderParameter = {} as any; + const localVarQueryParameter = {} as any; + + // authentication cookie required + + // authentication api_key required + await setApiKeyToObject(localVarHeaderParameter, "x-api-key", configuration) + + // authentication bearer required + // http bearer authentication required + await setBearerAuthToObject(localVarHeaderParameter, configuration) + + + + localVarHeaderParameter['Content-Type'] = 'application/json'; + + setSearchParams(localVarUrlObj, localVarQueryParameter); + let headersFromBaseOptions = baseOptions && baseOptions.headers ? baseOptions.headers : {}; + localVarRequestOptions.headers = {...localVarHeaderParameter, ...headersFromBaseOptions, ...options.headers}; + localVarRequestOptions.data = serializeDataIfNeeded(validateLibraryDto, localVarRequestOptions, configuration) + return { url: toPathString(localVarUrlObj), options: localVarRequestOptions, @@ -12925,6 +13030,19 @@ export const LibraryApiFp = function(configuration?: Configuration) { const operationBasePath = operationServerMap['LibraryApi.updateLibrary']?.[index]?.url; return (axios, basePath) => createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration)(axios, operationBasePath || basePath); }, + /** + * + * @param {string} id + * @param {ValidateLibraryDto} validateLibraryDto + * @param {*} [options] Override http request option. + * @throws {RequiredError} + */ + async validate(id: string, validateLibraryDto: ValidateLibraryDto, options?: RawAxiosRequestConfig): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise> { + const localVarAxiosArgs = await localVarAxiosParamCreator.validate(id, validateLibraryDto, options); + const index = configuration?.serverIndex ?? 0; + const operationBasePath = operationServerMap['LibraryApi.validate']?.[index]?.url; + return (axios, basePath) => createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration)(axios, operationBasePath || basePath); + }, } }; @@ -13006,6 +13124,15 @@ export const LibraryApiFactory = function (configuration?: Configuration, basePa updateLibrary(requestParameters: LibraryApiUpdateLibraryRequest, options?: RawAxiosRequestConfig): AxiosPromise { return localVarFp.updateLibrary(requestParameters.id, requestParameters.updateLibraryDto, options).then((request) => request(axios, basePath)); }, + /** + * + * @param {LibraryApiValidateRequest} requestParameters Request parameters. + * @param {*} [options] Override http request option. + * @throws {RequiredError} + */ + validate(requestParameters: LibraryApiValidateRequest, options?: RawAxiosRequestConfig): AxiosPromise { + return localVarFp.validate(requestParameters.id, requestParameters.validateLibraryDto, options).then((request) => request(axios, basePath)); + }, }; }; @@ -13121,6 +13248,27 @@ export interface LibraryApiUpdateLibraryRequest { readonly updateLibraryDto: UpdateLibraryDto } +/** + * Request parameters for validate operation in LibraryApi. + * @export + * @interface LibraryApiValidateRequest + */ +export interface LibraryApiValidateRequest { + /** + * + * @type {string} + * @memberof LibraryApiValidate + */ + readonly id: string + + /** + * + * @type {ValidateLibraryDto} + * @memberof LibraryApiValidate + */ + readonly validateLibraryDto: ValidateLibraryDto +} + /** * LibraryApi - object-oriented interface * @export @@ -13214,6 +13362,17 @@ export class LibraryApi extends BaseAPI { public updateLibrary(requestParameters: LibraryApiUpdateLibraryRequest, options?: RawAxiosRequestConfig) { return LibraryApiFp(this.configuration).updateLibrary(requestParameters.id, requestParameters.updateLibraryDto, options).then((request) => request(this.axios, this.basePath)); } + + /** + * + * @param {LibraryApiValidateRequest} requestParameters Request parameters. + * @param {*} [options] Override http request option. + * @throws {RequiredError} + * @memberof LibraryApi + */ + public validate(requestParameters: LibraryApiValidateRequest, options?: RawAxiosRequestConfig) { + return LibraryApiFp(this.configuration).validate(requestParameters.id, requestParameters.validateLibraryDto, options).then((request) => request(this.axios, this.basePath)); + } } diff --git a/open-api/typescript-sdk/fetch-client.ts b/open-api/typescript-sdk/fetch-client.ts index 0f42ff9178126..40600e56fa68e 100644 --- a/open-api/typescript-sdk/fetch-client.ts +++ b/open-api/typescript-sdk/fetch-client.ts @@ -480,6 +480,18 @@ export type LibraryStatsResponseDto = { usage: number; videos: number; }; +export type ValidateLibraryDto = { + exclusionPatterns?: string[]; + importPaths?: string[]; +}; +export type ValidateLibraryImportPathResponseDto = { + importPath: string; + isValid?: boolean; + message?: string; +}; +export type ValidateLibraryResponseDto = { + importPaths?: ValidateLibraryImportPathResponseDto[]; +}; export type OAuthConfigDto = { redirectUri: string; }; @@ -1901,6 +1913,19 @@ export function getLibraryStatistics({ id }: { ...opts })); } +export function validate({ id, validateLibraryDto }: { + id: string; + validateLibraryDto: ValidateLibraryDto; +}, opts?: Oazapfts.RequestOpts) { + return oazapfts.ok(oazapfts.fetchJson<{ + status: 200; + data: ValidateLibraryResponseDto; + }>(`/library/${encodeURIComponent(id)}/validate`, oazapfts.json({ + ...opts, + method: "POST", + body: validateLibraryDto + }))); +} export function startOAuth({ oAuthConfigDto }: { oAuthConfigDto: OAuthConfigDto; }, opts?: Oazapfts.RequestOpts) { diff --git a/server/e2e/api/setup.ts b/server/e2e/api/setup.ts index 8d44b07cbbca3..88f2f598bdbb0 100644 --- a/server/e2e/api/setup.ts +++ b/server/e2e/api/setup.ts @@ -1,6 +1,16 @@ import { PostgreSqlContainer } from '@testcontainers/postgresql'; +import path from 'node:path'; export default async () => { + let IMMICH_TEST_ASSET_PATH: string = ''; + + if (process.env.IMMICH_TEST_ASSET_PATH === undefined) { + IMMICH_TEST_ASSET_PATH = path.normalize(`${__dirname}/../../test/assets/`); + process.env.IMMICH_TEST_ASSET_PATH = IMMICH_TEST_ASSET_PATH; + } else { + IMMICH_TEST_ASSET_PATH = process.env.IMMICH_TEST_ASSET_PATH; + } + const pg = await new PostgreSqlContainer('tensorchord/pgvecto-rs:pg14-v0.2.0') .withDatabase('immich') .withUsername('postgres') @@ -11,6 +21,9 @@ export default async () => { process.env.DB_URL = pg.getConnectionUri(); process.env.NODE_ENV = 'development'; - process.env.LOG_LEVEL = 'fatal'; process.env.TZ = 'Z'; + + if (process.env.LOG_LEVEL === undefined) { + process.env.LOG_LEVEL = 'fatal'; + } }; diff --git a/server/e2e/api/specs/library.e2e-spec.ts b/server/e2e/api/specs/library.e2e-spec.ts index 75c973466e2d6..5be9e3035b3f3 100644 --- a/server/e2e/api/specs/library.e2e-spec.ts +++ b/server/e2e/api/specs/library.e2e-spec.ts @@ -2,6 +2,7 @@ import { LibraryResponseDto, LoginResponseDto } from '@app/domain'; import { LibraryController } from '@app/immich'; import { LibraryType } from '@app/infra/entities'; import { errorStub, userDto, uuidStub } from '@test/fixtures'; +import { IMMICH_TEST_ASSET_TEMP_PATH, restoreTempFolder } from 'src/test-utils/utils'; import request from 'supertest'; import { api } from '../../client'; import { testApp } from '../utils'; @@ -20,6 +21,7 @@ describe(`${LibraryController.name} (e2e)`, () => { }); beforeEach(async () => { + await restoreTempFolder(); await testApp.reset(); await api.authApi.adminSignUp(server); admin = await api.authApi.adminLogin(server); @@ -247,15 +249,16 @@ describe(`${LibraryController.name} (e2e)`, () => { }); it('should change the import paths', async () => { + await api.userApi.setExternalPath(server, admin.accessToken, admin.userId, IMMICH_TEST_ASSET_TEMP_PATH); const { status, body } = await request(server) .put(`/library/${library.id}`) .set('Authorization', `Bearer ${admin.accessToken}`) - .send({ importPaths: ['/path/to/import'] }); + .send({ importPaths: [IMMICH_TEST_ASSET_TEMP_PATH] }); expect(status).toBe(200); expect(body).toEqual( expect.objectContaining({ - importPaths: ['/path/to/import'], + importPaths: [IMMICH_TEST_ASSET_TEMP_PATH], }), ); }); @@ -435,4 +438,93 @@ describe(`${LibraryController.name} (e2e)`, () => { expect(body).toEqual(errorStub.unauthorized); }); }); + + describe('POST /library/:id/validate', () => { + it('should require authentication', async () => { + const { status, body } = await request(server).post(`/library/${uuidStub.notFound}/validate`).send({}); + + expect(status).toBe(401); + expect(body).toEqual(errorStub.unauthorized); + }); + + describe('Validate import path', () => { + let library: LibraryResponseDto; + + beforeEach(async () => { + // Create an external library with default settings + library = await api.libraryApi.create(server, admin.accessToken, { type: LibraryType.EXTERNAL }); + }); + + it('should fail with no external path set', async () => { + const { status, body } = await request(server) + .post(`/library/${library.id}/validate`) + .set('Authorization', `Bearer ${admin.accessToken}`) + + .send({ importPaths: [] }); + + expect(status).toBe(400); + expect(body).toEqual(errorStub.badRequest('User has no external path set')); + }); + + describe('With external path set', () => { + beforeEach(async () => { + await api.userApi.setExternalPath(server, admin.accessToken, admin.userId, IMMICH_TEST_ASSET_TEMP_PATH); + }); + + it('should pass with no import paths', async () => { + const response = await api.libraryApi.validate(server, admin.accessToken, library.id, { importPaths: [] }); + expect(response.importPaths).toEqual([]); + }); + + it('should not allow paths outside of the external path', async () => { + const pathToTest = `${IMMICH_TEST_ASSET_TEMP_PATH}/../`; + const response = await api.libraryApi.validate(server, admin.accessToken, library.id, { + importPaths: [pathToTest], + }); + expect(response.importPaths?.length).toEqual(1); + const pathResponse = response?.importPaths?.at(0); + + expect(pathResponse).toEqual({ + importPath: pathToTest, + isValid: false, + message: `Not contained in user's external path`, + }); + }); + + it('should fail if path does not exist', async () => { + const pathToTest = `${IMMICH_TEST_ASSET_TEMP_PATH}/does/not/exist`; + + const response = await api.libraryApi.validate(server, admin.accessToken, library.id, { + importPaths: [pathToTest], + }); + + expect(response.importPaths?.length).toEqual(1); + const pathResponse = response?.importPaths?.at(0); + + expect(pathResponse).toEqual({ + importPath: pathToTest, + isValid: false, + message: `Path does not exist (ENOENT)`, + }); + }); + + it('should fail if path is a file', async () => { + const pathToTest = `${IMMICH_TEST_ASSET_TEMP_PATH}/does/not/exist`; + + const response = await api.libraryApi.validate(server, admin.accessToken, library.id, { + importPaths: [pathToTest], + }); + + expect(response.importPaths?.length).toEqual(1); + const pathResponse = response?.importPaths?.at(0); + + expect(pathResponse).toEqual({ + importPath: pathToTest, + isValid: false, + message: `Path does not exist (ENOENT)`, + }); + }); + }); + }); + }); }); diff --git a/server/e2e/client/library-api.ts b/server/e2e/client/library-api.ts index c40f005457c38..9f2d0b77efa25 100644 --- a/server/e2e/client/library-api.ts +++ b/server/e2e/client/library-api.ts @@ -4,6 +4,8 @@ import { LibraryStatsResponseDto, ScanLibraryDto, UpdateLibraryDto, + ValidateLibraryDto, + ValidateLibraryResponseDto, } from '@app/domain'; import request from 'supertest'; @@ -58,4 +60,12 @@ export const libraryApi = { expect(status).toBe(200); return body as LibraryResponseDto; }, + validate: async (server: any, accessToken: string, id: string, data: ValidateLibraryDto) => { + const { body, status } = await request(server) + .post(`/library/${id}/validate`) + .set('Authorization', `Bearer ${accessToken}`) + .send(data); + expect(status).toBe(200); + return body as ValidateLibraryResponseDto; + }, }; diff --git a/server/src/domain/library/library.dto.ts b/server/src/domain/library/library.dto.ts index 638841ec63800..db6119d4ded27 100644 --- a/server/src/domain/library/library.dto.ts +++ b/server/src/domain/library/library.dto.ts @@ -1,6 +1,6 @@ import { LibraryEntity, LibraryType } from '@app/infra/entities'; import { ApiProperty } from '@nestjs/swagger'; -import { ArrayUnique, IsBoolean, IsEnum, IsNotEmpty, IsOptional, IsString } from 'class-validator'; +import { ArrayMaxSize, ArrayUnique, IsBoolean, IsEnum, IsNotEmpty, IsOptional, IsString } from 'class-validator'; import { ValidateUUID } from '../domain.util'; export class CreateLibraryDto { @@ -21,12 +21,14 @@ export class CreateLibraryDto { @IsString({ each: true }) @IsNotEmpty({ each: true }) @ArrayUnique() + @ArrayMaxSize(128) importPaths?: string[]; @IsOptional() @IsString({ each: true }) @IsNotEmpty({ each: true }) @ArrayUnique() + @ArrayMaxSize(128) exclusionPatterns?: string[]; @IsOptional() @@ -48,12 +50,14 @@ export class UpdateLibraryDto { @IsString({ each: true }) @IsNotEmpty({ each: true }) @ArrayUnique() + @ArrayMaxSize(128) importPaths?: string[]; @IsOptional() @IsNotEmpty({ each: true }) @IsString({ each: true }) @ArrayUnique() + @ArrayMaxSize(128) exclusionPatterns?: string[]; } @@ -63,6 +67,32 @@ export class CrawlOptionsDto { exclusionPatterns?: string[]; } +export class ValidateLibraryDto { + @IsOptional() + @IsString({ each: true }) + @IsNotEmpty({ each: true }) + @ArrayUnique() + @ArrayMaxSize(128) + importPaths?: string[]; + + @IsOptional() + @IsNotEmpty({ each: true }) + @IsString({ each: true }) + @ArrayUnique() + @ArrayMaxSize(128) + exclusionPatterns?: string[]; +} + +export class ValidateLibraryResponseDto { + importPaths?: ValidateLibraryImportPathResponseDto[]; +} + +export class ValidateLibraryImportPathResponseDto { + importPath!: string; + isValid?: boolean = false; + message?: string; +} + export class LibrarySearchDto { @ValidateUUID({ optional: true }) userId?: string; diff --git a/server/src/domain/library/library.service.spec.ts b/server/src/domain/library/library.service.spec.ts index c86aa4a4e70d9..cafe70b4d742b 100644 --- a/server/src/domain/library/library.service.spec.ts +++ b/server/src/domain/library/library.service.spec.ts @@ -54,12 +54,6 @@ describe(LibraryService.name, () => { cryptoMock = newCryptoRepositoryMock(); storageMock = newStorageRepositoryMock(); - storageMock.stat.mockResolvedValue({ - size: 100, - mtime: new Date('2023-01-01'), - ctime: new Date('2023-01-01'), - } as Stats); - // Always validate owner access for library. accessMock.library.checkOwnerAccess.mockImplementation(async (_, libraryIds) => libraryIds); @@ -270,6 +264,39 @@ describe(LibraryService.name, () => { await expect(sut.handleQueueAssetRefresh(mockLibraryJob)).resolves.toBe(false); }); + + it('should ignore import paths that do not exist', async () => { + storageMock.stat.mockImplementation((path): Promise => { + if (path === libraryStub.externalLibraryWithImportPaths1.importPaths[0]) { + const error = { code: 'ENOENT' } as any; + throw error; + } + return Promise.resolve({ + isDirectory: () => true, + } as Stats); + }); + + storageMock.checkFileExists.mockResolvedValue(true); + + const mockLibraryJob: ILibraryRefreshJob = { + id: libraryStub.externalLibraryWithImportPaths1.id, + refreshModifiedFiles: false, + refreshAllFiles: false, + }; + + libraryMock.get.mockResolvedValue(libraryStub.externalLibraryWithImportPaths1); + storageMock.crawl.mockResolvedValue([]); + assetMock.getByLibraryId.mockResolvedValue([]); + libraryMock.getOnlineAssetPaths.mockResolvedValue([]); + userMock.get.mockResolvedValue(userStub.externalPathRoot); + + await sut.handleQueueAssetRefresh(mockLibraryJob); + + expect(storageMock.crawl).toHaveBeenCalledWith({ + pathsToCrawl: [libraryStub.externalLibraryWithImportPaths1.importPaths[1]], + exclusionPatterns: [], + }); + }); }); describe('handleAssetRefresh', () => { @@ -278,6 +305,12 @@ describe(LibraryService.name, () => { beforeEach(() => { mockUser = userStub.externalPath1; userMock.get.mockResolvedValue(mockUser); + + storageMock.stat.mockResolvedValue({ + size: 100, + mtime: new Date('2023-01-01'), + ctime: new Date('2023-01-01'), + } as Stats); }); it('should reject an unknown file extension', async () => { @@ -1104,13 +1137,19 @@ describe(LibraryService.name, () => { libraryMock.update.mockResolvedValue(libraryStub.externalLibraryWithImportPaths1); libraryMock.get.mockResolvedValue(libraryStub.externalLibraryWithImportPaths1); - await expect(sut.update(authStub.admin, authStub.admin.user.id, { importPaths: ['/foo'] })).resolves.toEqual( - mapLibrary(libraryStub.externalLibraryWithImportPaths1), - ); + storageMock.stat.mockResolvedValue({ + isDirectory: () => true, + } as Stats); + + storageMock.checkFileExists.mockResolvedValue(true); + + await expect( + sut.update(authStub.external1, authStub.external1.user.id, { importPaths: ['/data/user1/foo'] }), + ).resolves.toEqual(mapLibrary(libraryStub.externalLibraryWithImportPaths1)); expect(libraryMock.update).toHaveBeenCalledWith( expect.objectContaining({ - id: authStub.admin.user.id, + id: authStub.external1.user.id, }), ); expect(storageMock.watch).toHaveBeenCalledWith( @@ -1142,7 +1181,7 @@ describe(LibraryService.name, () => { }); }); - describe('watchAll new', () => { + describe('watchAll', () => { describe('watching disabled', () => { beforeEach(async () => { configMock.load.mockResolvedValue(systemConfigStub.libraryWatchDisabled); @@ -1523,4 +1562,121 @@ describe(LibraryService.name, () => { ]); }); }); + + describe('validate', () => { + it('should validate directory', async () => { + storageMock.stat.mockResolvedValue({ + isDirectory: () => true, + } as Stats); + + storageMock.checkFileExists.mockResolvedValue(true); + + const result = await sut.validate(authStub.external1, libraryStub.externalLibraryWithImportPaths1.id, { + importPaths: ['/data/user1/'], + }); + + expect(result.importPaths).toEqual([ + { + importPath: '/data/user1/', + isValid: true, + message: undefined, + }, + ]); + }); + + it('should error when no external path is set', async () => { + await expect( + sut.validate(authStub.admin, libraryStub.externalLibrary1.id, { importPaths: ['/photos'] }), + ).rejects.toBeInstanceOf(BadRequestException); + }); + + it('should detect when path is outside external path', async () => { + const result = await sut.validate(authStub.external1, libraryStub.externalLibraryWithImportPaths1.id, { + importPaths: ['/data/user2'], + }); + + expect(result.importPaths).toEqual([ + { + importPath: '/data/user2', + isValid: false, + message: "Not contained in user's external path", + }, + ]); + }); + + it('should detect when path does not exist', async () => { + storageMock.stat.mockImplementation(() => { + const error = { code: 'ENOENT' } as any; + throw error; + }); + + const result = await sut.validate(authStub.external1, libraryStub.externalLibraryWithImportPaths1.id, { + importPaths: ['/data/user1/'], + }); + + expect(result.importPaths).toEqual([ + { + importPath: '/data/user1/', + isValid: false, + message: 'Path does not exist (ENOENT)', + }, + ]); + }); + + it('should detect when path is not a directory', async () => { + storageMock.stat.mockResolvedValue({ + isDirectory: () => false, + } as Stats); + + const result = await sut.validate(authStub.external1, libraryStub.externalLibraryWithImportPaths1.id, { + importPaths: ['/data/user1/file'], + }); + + expect(result.importPaths).toEqual([ + { + importPath: '/data/user1/file', + isValid: false, + message: 'Not a directory', + }, + ]); + }); + + it('should return an unknown exception from stat', async () => { + storageMock.stat.mockImplementation(() => { + throw new Error('Unknown error'); + }); + + const result = await sut.validate(authStub.external1, libraryStub.externalLibraryWithImportPaths1.id, { + importPaths: ['/data/user1/'], + }); + + expect(result.importPaths).toEqual([ + { + importPath: '/data/user1/', + isValid: false, + message: 'Error: Unknown error', + }, + ]); + }); + + it('should detect when access rights are missing', async () => { + storageMock.stat.mockResolvedValue({ + isDirectory: () => true, + } as Stats); + + storageMock.checkFileExists.mockResolvedValue(false); + + const result = await sut.validate(authStub.external1, libraryStub.externalLibraryWithImportPaths1.id, { + importPaths: ['/data/user1/'], + }); + + expect(result.importPaths).toEqual([ + { + importPath: '/data/user1/', + isValid: false, + message: 'Lacking read permission for folder', + }, + ]); + }); + }); }); diff --git a/server/src/domain/library/library.service.ts b/server/src/domain/library/library.service.ts index 3454f3547b242..7c961963ecab5 100644 --- a/server/src/domain/library/library.service.ts +++ b/server/src/domain/library/library.service.ts @@ -30,6 +30,9 @@ import { LibraryStatsResponseDto, ScanLibraryDto, UpdateLibraryDto, + ValidateLibraryDto, + ValidateLibraryImportPathResponseDto, + ValidateLibraryResponseDto, mapLibrary, } from './library.dto'; @@ -270,10 +273,81 @@ export class LibraryService extends EventEmitter { ); } + private async validateImportPath(importPath: string): Promise { + const validation = new ValidateLibraryImportPathResponseDto(); + validation.importPath = importPath; + + try { + const stat = await this.storageRepository.stat(importPath); + + if (!stat.isDirectory()) { + validation.message = 'Not a directory'; + return validation; + } + } catch (error: any) { + if (error.code === 'ENOENT') { + validation.message = 'Path does not exist (ENOENT)'; + return validation; + } + validation.message = String(error); + return validation; + } + + const access = await this.storageRepository.checkFileExists(importPath, R_OK); + + if (!access) { + validation.message = 'Lacking read permission for folder'; + return validation; + } + + validation.isValid = true; + return validation; + } + + public async validate(auth: AuthDto, id: string, dto: ValidateLibraryDto): Promise { + await this.access.requirePermission(auth, Permission.LIBRARY_UPDATE, id); + + if (!auth.user.externalPath) { + throw new BadRequestException('User has no external path set'); + } + + const response = new ValidateLibraryResponseDto(); + + if (dto.importPaths) { + response.importPaths = await Promise.all( + dto.importPaths.map(async (importPath) => { + const normalizedPath = path.normalize(importPath); + + if (!this.isInExternalPath(normalizedPath, auth.user.externalPath)) { + const validation = new ValidateLibraryImportPathResponseDto(); + validation.importPath = importPath; + validation.message = `Not contained in user's external path`; + return validation; + } + + return await this.validateImportPath(importPath); + }), + ); + } + + return response; + } + async update(auth: AuthDto, id: string, dto: UpdateLibraryDto): Promise { await this.access.requirePermission(auth, Permission.LIBRARY_UPDATE, id); const library = await this.repository.update({ id, ...dto }); + if (dto.importPaths) { + const validation = await this.validate(auth, id, { importPaths: dto.importPaths }); + if (validation.importPaths) { + for (const path of validation.importPaths) { + if (!path.isValid) { + throw new BadRequestException(`Invalid import path: ${path.message}`); + } + } + } + } + if (dto.importPaths || dto.exclusionPatterns) { // Re-watch library to use new paths and/or exclusion patterns await this.watch(id); @@ -509,6 +583,14 @@ export class LibraryService extends EventEmitter { return true; } + // Check if a given path is in a user's external path. Both arguments are assumed to be normalized + private isInExternalPath(filePath: string, externalPath: string | null): boolean { + if (externalPath === null) { + return false; + } + return filePath.startsWith(externalPath); + } + async handleQueueAssetRefresh(job: ILibraryRefreshJob): Promise { const library = await this.repository.get(job.id); if (!library || library.type !== LibraryType.EXTERNAL) { @@ -523,17 +605,31 @@ export class LibraryService extends EventEmitter { } this.logger.verbose(`Refreshing library: ${job.id}`); + + const pathValidation = await Promise.all( + library.importPaths.map(async (importPath) => await this.validateImportPath(importPath)), + ); + + const validImportPaths = pathValidation + .map((validation) => { + if (!validation.isValid) { + this.logger.error(`Skipping invalid import path: ${validation.importPath}. Reason: ${validation.message}`); + } + return validation; + }) + .filter((validation) => validation.isValid) + .map((validation) => validation.importPath); + const rawPaths = await this.storageRepository.crawl({ - pathsToCrawl: library.importPaths, + pathsToCrawl: validImportPaths, exclusionPatterns: library.exclusionPatterns, }); const crawledAssetPaths = rawPaths + // Normalize file paths. This is important to prevent security issues like path traversal .map((filePath) => path.normalize(filePath)) - .filter((assetPath) => - // Filter out paths that are not within the user's external path - assetPath.match(new RegExp(`^${user.externalPath}`)), - ) as string[]; + // Filter out paths that are not within the user's external path + .filter((assetPath) => this.isInExternalPath(assetPath, user.externalPath)) as string[]; this.logger.debug(`Found ${crawledAssetPaths.length} asset(s) when crawling import paths ${library.importPaths}`); const assetsInLibrary = await this.assetRepository.getByLibraryId([job.id]); diff --git a/server/src/immich/controllers/library.controller.ts b/server/src/immich/controllers/library.controller.ts index 56dd5d8e76bd7..173e632110a56 100644 --- a/server/src/immich/controllers/library.controller.ts +++ b/server/src/immich/controllers/library.controller.ts @@ -6,8 +6,10 @@ import { LibraryResponseDto as ResponseDto, ScanLibraryDto, UpdateLibraryDto as UpdateDto, + ValidateLibraryDto, + ValidateLibraryResponseDto, } from '@app/domain'; -import { Body, Controller, Delete, Get, Param, Post, Put } from '@nestjs/common'; +import { Body, Controller, Delete, Get, HttpCode, Param, Post, Put } from '@nestjs/common'; import { ApiTags } from '@nestjs/swagger'; import { Auth, Authenticated } from '../app.guard'; import { UseValidation } from '../app.utils'; @@ -40,6 +42,16 @@ export class LibraryController { return this.service.get(auth, id); } + @Post(':id/validate') + @HttpCode(200) + validate( + @Auth() auth: AuthDto, + @Param() { id }: UUIDParamDto, + @Body() dto: ValidateLibraryDto, + ): Promise { + return this.service.validate(auth, id, dto); + } + @Delete(':id') deleteLibrary(@Auth() auth: AuthDto, @Param() { id }: UUIDParamDto): Promise { return this.service.delete(auth, id); diff --git a/server/src/infra/infra.utils.ts b/server/src/infra/infra.utils.ts index d9eb823639a64..ab7d744317c8d 100644 --- a/server/src/infra/infra.utils.ts +++ b/server/src/infra/infra.utils.ts @@ -164,7 +164,21 @@ export function searchAssetBuilder( builder.andWhere(_.omitBy(path, _.isUndefined)); const status = _.pick(options, ['isExternal', 'isFavorite', 'isOffline', 'isReadOnly', 'isVisible', 'type']); - const { isArchived, isEncoded, isMotion, withArchived } = options; + const { + isArchived, + isEncoded, + isMotion, + withArchived, + isNotInAlbum, + withFaces, + withPeople, + withSmartInfo, + personIds, + withExif, + withStacked, + trashedAfter, + trashedBefore, + } = options; builder.andWhere( _.omitBy( { @@ -177,38 +191,38 @@ export function searchAssetBuilder( ), ); - if (options.isNotInAlbum) { + if (isNotInAlbum) { builder .leftJoin(`${builder.alias}.albums`, 'albums') .andWhere('albums.id IS NULL') .andWhere(`${builder.alias}.isVisible = true`); } - if (options.withFaces || options.withPeople) { + if (withFaces || withPeople) { builder.leftJoinAndSelect(`${builder.alias}.faces`, 'faces'); } - if (options.withPeople) { + if (withPeople) { builder.leftJoinAndSelect(`${builder.alias}.person`, 'person'); } - if (options.withSmartInfo) { + if (withSmartInfo) { builder.leftJoinAndSelect(`${builder.alias}.smartInfo`, 'smartInfo'); } - if (options.personIds && options.personIds.length > 0) { + if (personIds && personIds.length > 0) { builder .leftJoin(`${builder.alias}.faces`, 'faces') - .andWhere('faces.personId IN (:...personIds)', { personIds: options.personIds }) + .andWhere('faces.personId IN (:...personIds)', { personIds: personIds }) .addGroupBy(`${builder.alias}.id`) - .having('COUNT(faces.id) = :personCount', { personCount: options.personIds.length }); + .having('COUNT(faces.id) = :personCount', { personCount: personIds.length }); - if (options.withExif) { + if (withExif) { builder.addGroupBy('exifInfo.assetId'); } } - if (options.withStacked) { + if (withStacked) { builder .leftJoinAndSelect(`${builder.alias}.stack`, 'stack') .leftJoinAndSelect('stack.assets', 'stackedAssets') @@ -217,8 +231,7 @@ export function searchAssetBuilder( ); } - const withDeleted = - options.withDeleted ?? (options.trashedAfter !== undefined || options.trashedBefore !== undefined); + const withDeleted = options.withDeleted ?? (trashedAfter !== undefined || trashedBefore !== undefined); if (withDeleted) { builder.withDeleted(); } diff --git a/server/src/infra/repositories/filesystem.provider.ts b/server/src/infra/repositories/filesystem.provider.ts index ed009da76f6af..3ffcd8111fd66 100644 --- a/server/src/infra/repositories/filesystem.provider.ts +++ b/server/src/infra/repositories/filesystem.provider.ts @@ -12,12 +12,24 @@ import archiver from 'archiver'; import chokidar, { WatchOptions } from 'chokidar'; import { glob } from 'glob'; import { constants, createReadStream, existsSync, mkdirSync } from 'node:fs'; -import fs, { copyFile, readdir, rename, utimes, writeFile } from 'node:fs/promises'; +import fs, { copyFile, readdir, rename, stat, utimes, writeFile } from 'node:fs/promises'; import path from 'node:path'; export class FilesystemProvider implements IStorageRepository { private logger = new ImmichLogger(FilesystemProvider.name); + readdir = readdir; + + copyFile = copyFile; + + stat = stat; + + writeFile = writeFile; + + rename = rename; + + utimes = utimes; + createZipStream(): ImmichZipStream { const archive = archiver('zip', { store: true }); @@ -50,14 +62,6 @@ export class FilesystemProvider implements IStorageRepository { } } - writeFile = writeFile; - - rename = rename; - - copyFile = copyFile; - - utimes = utimes; - async checkFileExists(filepath: string, mode = constants.F_OK): Promise { try { await fs.access(filepath, mode); @@ -79,8 +83,6 @@ export class FilesystemProvider implements IStorageRepository { } } - stat = fs.stat; - async unlinkDir(folder: string, options: { recursive?: boolean; force?: boolean }) { await fs.rm(folder, options); } @@ -146,6 +148,4 @@ export class FilesystemProvider implements IStorageRepository { return () => watcher.close(); } - - readdir = readdir; } diff --git a/server/test/fixtures/user.stub.ts b/server/test/fixtures/user.stub.ts index 770e615306dc0..1b8a3b3fed939 100644 --- a/server/test/fixtures/user.stub.ts +++ b/server/test/fixtures/user.stub.ts @@ -140,6 +140,25 @@ export const userStub = { quotaSizeInBytes: null, quotaUsageInBytes: 0, }), + externalPathRoot: Object.freeze({ + ...authStub.user1.user, + password: 'immich_password', + name: 'immich_name', + storageLabel: 'label-1', + externalPath: '/', + oauthId: '', + shouldChangePassword: false, + profileImagePath: '', + createdAt: new Date('2021-01-01'), + deletedAt: null, + updatedAt: new Date('2021-01-01'), + tags: [], + assets: [], + memoriesEnabled: true, + avatarColor: UserAvatarColor.PRIMARY, + quotaSizeInBytes: null, + quotaUsageInBytes: 0, + }), profilePath: Object.freeze({ ...authStub.user1.user, password: 'immich_password', diff --git a/web/src/lib/components/forms/library-import-paths-form.svelte b/web/src/lib/components/forms/library-import-paths-form.svelte index fddb4b9867bbf..17903526be652 100644 --- a/web/src/lib/components/forms/library-import-paths-form.svelte +++ b/web/src/lib/components/forms/library-import-paths-form.svelte @@ -1,13 +1,15 @@