From b862c20e8e38b56f2c26fc0a1af974cf0970f6ba Mon Sep 17 00:00:00 2001 From: Alex Date: Fri, 24 Mar 2023 23:22:02 -0500 Subject: [PATCH 1/2] chore: fix api (#2079) --- .../openapi/lib/model/asset_response_dto.dart | 139 ++++++++---------- 1 file changed, 64 insertions(+), 75 deletions(-) diff --git a/mobile/openapi/lib/model/asset_response_dto.dart b/mobile/openapi/lib/model/asset_response_dto.dart index e6b728566d..013ac00ff4 100644 --- a/mobile/openapi/lib/model/asset_response_dto.dart +++ b/mobile/openapi/lib/model/asset_response_dto.dart @@ -85,79 +85,76 @@ class AssetResponseDto { List tags; @override - bool operator ==(Object other) => - identical(this, other) || - other is AssetResponseDto && - other.type == type && - other.id == id && - other.deviceAssetId == deviceAssetId && - other.ownerId == ownerId && - other.deviceId == deviceId && - other.originalPath == originalPath && - other.resizePath == resizePath && - other.fileCreatedAt == fileCreatedAt && - other.fileModifiedAt == fileModifiedAt && - other.updatedAt == updatedAt && - other.isFavorite == isFavorite && - other.mimeType == mimeType && - other.duration == duration && - other.webpPath == webpPath && - other.encodedVideoPath == encodedVideoPath && - other.exifInfo == exifInfo && - other.smartInfo == smartInfo && - other.livePhotoVideoId == livePhotoVideoId && - other.tags == tags; + bool operator ==(Object other) => identical(this, other) || other is AssetResponseDto && + other.type == type && + other.id == id && + other.deviceAssetId == deviceAssetId && + other.ownerId == ownerId && + other.deviceId == deviceId && + other.originalPath == originalPath && + other.resizePath == resizePath && + other.fileCreatedAt == fileCreatedAt && + other.fileModifiedAt == fileModifiedAt && + other.updatedAt == updatedAt && + other.isFavorite == isFavorite && + other.mimeType == mimeType && + other.duration == duration && + other.webpPath == webpPath && + other.encodedVideoPath == encodedVideoPath && + other.exifInfo == exifInfo && + other.smartInfo == smartInfo && + other.livePhotoVideoId == livePhotoVideoId && + other.tags == tags; @override int get hashCode => - // ignore: unnecessary_parenthesis - (type.hashCode) + - (id.hashCode) + - (deviceAssetId.hashCode) + - (ownerId.hashCode) + - (deviceId.hashCode) + - (originalPath.hashCode) + - (resizePath == null ? 0 : resizePath!.hashCode) + - (fileCreatedAt.hashCode) + - (fileModifiedAt.hashCode) + - (updatedAt.hashCode) + - (isFavorite.hashCode) + - (mimeType == null ? 0 : mimeType!.hashCode) + - (duration.hashCode) + - (webpPath == null ? 0 : webpPath!.hashCode) + - (encodedVideoPath == null ? 0 : encodedVideoPath!.hashCode) + - (exifInfo == null ? 0 : exifInfo!.hashCode) + - (smartInfo == null ? 0 : smartInfo!.hashCode) + - (livePhotoVideoId == null ? 0 : livePhotoVideoId!.hashCode) + - (tags.hashCode); + // ignore: unnecessary_parenthesis + (type.hashCode) + + (id.hashCode) + + (deviceAssetId.hashCode) + + (ownerId.hashCode) + + (deviceId.hashCode) + + (originalPath.hashCode) + + (resizePath == null ? 0 : resizePath!.hashCode) + + (fileCreatedAt.hashCode) + + (fileModifiedAt.hashCode) + + (updatedAt.hashCode) + + (isFavorite.hashCode) + + (mimeType == null ? 0 : mimeType!.hashCode) + + (duration.hashCode) + + (webpPath == null ? 0 : webpPath!.hashCode) + + (encodedVideoPath == null ? 0 : encodedVideoPath!.hashCode) + + (exifInfo == null ? 0 : exifInfo!.hashCode) + + (smartInfo == null ? 0 : smartInfo!.hashCode) + + (livePhotoVideoId == null ? 0 : livePhotoVideoId!.hashCode) + + (tags.hashCode); @override - String toString() => - 'AssetResponseDto[type=$type, id=$id, deviceAssetId=$deviceAssetId, ownerId=$ownerId, deviceId=$deviceId, originalPath=$originalPath, resizePath=$resizePath, fileCreatedAt=$fileCreatedAt, fileModifiedAt=$fileModifiedAt, updatedAt=$updatedAt, isFavorite=$isFavorite, mimeType=$mimeType, duration=$duration, webpPath=$webpPath, encodedVideoPath=$encodedVideoPath, exifInfo=$exifInfo, smartInfo=$smartInfo, livePhotoVideoId=$livePhotoVideoId, tags=$tags]'; + String toString() => 'AssetResponseDto[type=$type, id=$id, deviceAssetId=$deviceAssetId, ownerId=$ownerId, deviceId=$deviceId, originalPath=$originalPath, resizePath=$resizePath, fileCreatedAt=$fileCreatedAt, fileModifiedAt=$fileModifiedAt, updatedAt=$updatedAt, isFavorite=$isFavorite, mimeType=$mimeType, duration=$duration, webpPath=$webpPath, encodedVideoPath=$encodedVideoPath, exifInfo=$exifInfo, smartInfo=$smartInfo, livePhotoVideoId=$livePhotoVideoId, tags=$tags]'; Map toJson() { final json = {}; - json[r'type'] = this.type; - json[r'id'] = this.id; - json[r'deviceAssetId'] = this.deviceAssetId; - json[r'ownerId'] = this.ownerId; - json[r'deviceId'] = this.deviceId; - json[r'originalPath'] = this.originalPath; + json[r'type'] = this.type; + json[r'id'] = this.id; + json[r'deviceAssetId'] = this.deviceAssetId; + json[r'ownerId'] = this.ownerId; + json[r'deviceId'] = this.deviceId; + json[r'originalPath'] = this.originalPath; if (this.resizePath != null) { json[r'resizePath'] = this.resizePath; } else { // json[r'resizePath'] = null; } - json[r'fileCreatedAt'] = this.fileCreatedAt; - json[r'fileModifiedAt'] = this.fileModifiedAt; - json[r'updatedAt'] = this.updatedAt; - json[r'isFavorite'] = this.isFavorite; + json[r'fileCreatedAt'] = this.fileCreatedAt; + json[r'fileModifiedAt'] = this.fileModifiedAt; + json[r'updatedAt'] = this.updatedAt; + json[r'isFavorite'] = this.isFavorite; if (this.mimeType != null) { json[r'mimeType'] = this.mimeType; } else { // json[r'mimeType'] = null; } - json[r'duration'] = this.duration; + json[r'duration'] = this.duration; if (this.webpPath != null) { json[r'webpPath'] = this.webpPath; } else { @@ -183,7 +180,7 @@ class AssetResponseDto { } else { // json[r'livePhotoVideoId'] = null; } - json[r'tags'] = this.tags; + json[r'tags'] = this.tags; return json; } @@ -197,13 +194,13 @@ class AssetResponseDto { // Ensure that the map contains the required keys. // Note 1: the values aren't checked for validity beyond being non-null. // Note 2: this code is stripped in release mode! - // assert(() { - // requiredKeys.forEach((key) { - // assert(json.containsKey(key), 'Required key "AssetResponseDto[$key]" is missing from JSON.'); - // assert(json[key] != null, 'Required key "AssetResponseDto[$key]" has a null value in JSON.'); - // }); - // return true; - // }()); + assert(() { + requiredKeys.forEach((key) { + assert(json.containsKey(key), 'Required key "AssetResponseDto[$key]" is missing from JSON.'); + assert(json[key] != null, 'Required key "AssetResponseDto[$key]" has a null value in JSON.'); + }); + return true; + }()); return AssetResponseDto( type: AssetTypeEnum.fromJson(json[r'type'])!, @@ -230,10 +227,7 @@ class AssetResponseDto { return null; } - static List? listFromJson( - dynamic json, { - bool growable = false, - }) { + static List? listFromJson(dynamic json, {bool growable = false,}) { final result = []; if (json is List && json.isNotEmpty) { for (final row in json) { @@ -261,18 +255,12 @@ class AssetResponseDto { } // maps a json object with a list of AssetResponseDto-objects as value to a dart map - static Map> mapListFromJson( - dynamic json, { - bool growable = false, - }) { + static Map> mapListFromJson(dynamic json, {bool growable = false,}) { final map = >{}; if (json is Map && json.isNotEmpty) { json = json.cast(); // ignore: parameter_assignments for (final entry in json.entries) { - final value = AssetResponseDto.listFromJson( - entry.value, - growable: growable, - ); + final value = AssetResponseDto.listFromJson(entry.value, growable: growable,); if (value != null) { map[entry.key] = value; } @@ -299,3 +287,4 @@ class AssetResponseDto { 'webpPath', }; } + From 2400004f41c339475ed8ef8a17a8dd22ae99664d Mon Sep 17 00:00:00 2001 From: Jason Rasmussen Date: Sat, 25 Mar 2023 10:50:57 -0400 Subject: [PATCH 2/2] feat(server): split generated content into a separate folder (#2047) * feat: organize media folders * fix: tests --- .../src/api-v1/asset/asset.service.spec.ts | 2 +- .../src/config/asset-upload.config.spec.ts | 11 +----- .../immich/src/config/asset-upload.config.ts | 17 +++++----- .../profile-image-upload.config.spec.ts | 2 +- .../src/config/profile-image-upload.config.ts | 8 ++--- .../processors/video-transcode.processor.ts | 19 ++++++----- server/libs/domain/src/domain.constant.ts | 2 +- .../domain/src/media/media.service.spec.ts | 34 +++++++++---------- server/libs/domain/src/media/media.service.ts | 11 +++--- .../src/server-info/server-info.service.ts | 4 +-- .../storage-template/storage-template.core.ts | 20 +++++------ .../storage-template.service.spec.ts | 22 ++++++------ .../storage-template.service.ts | 4 +-- server/libs/domain/src/storage/index.ts | 1 + .../libs/domain/src/storage/storage.core.ts | 16 +++++++++ .../libs/domain/src/user/user.service.spec.ts | 8 ++++- server/libs/domain/src/user/user.service.ts | 20 ++++++++--- server/libs/domain/test/fixtures.ts | 2 +- server/package.json | 1 + 19 files changed, 113 insertions(+), 91 deletions(-) create mode 100644 server/libs/domain/src/storage/storage.core.ts diff --git a/server/apps/immich/src/api-v1/asset/asset.service.spec.ts b/server/apps/immich/src/api-v1/asset/asset.service.spec.ts index 40d2b23da1..e88f2651ae 100644 --- a/server/apps/immich/src/api-v1/asset/asset.service.spec.ts +++ b/server/apps/immich/src/api-v1/asset/asset.service.spec.ts @@ -275,7 +275,7 @@ describe('AssetService', () => { expect(assetRepositoryMock.create).toHaveBeenCalled(); expect(assetRepositoryMock.save).toHaveBeenCalledWith({ id: 'id_1', - originalPath: 'upload/user_id_1/2022/2022-06-19/asset_1.jpeg', + originalPath: 'upload/library/user_id_1/2022/2022-06-19/asset_1.jpeg', }); }); diff --git a/server/apps/immich/src/config/asset-upload.config.spec.ts b/server/apps/immich/src/config/asset-upload.config.spec.ts index fbe4d20f91..a01a0983d4 100644 --- a/server/apps/immich/src/config/asset-upload.config.spec.ts +++ b/server/apps/immich/src/config/asset-upload.config.spec.ts @@ -137,16 +137,7 @@ describe('assetUploadOption', () => { destination(mock.userRequest, mock.file, callback); expect(mkdirSync).not.toHaveBeenCalled(); - expect(callback).toHaveBeenCalledWith(null, 'upload/test-user/original/test-device'); - }); - - it('should sanitize the deviceId', () => { - const request = { ...mock.userRequest, body: { deviceId: 'test-devi\u0000ce' } } as Request; - destination(request, mock.file, callback); - - const [folderName] = existsSync.mock.calls[0]; - expect(folderName.endsWith('test-device')).toBeTruthy(); - expect(callback).toHaveBeenCalledWith(null, 'upload/test-user/original/test-device'); + expect(callback).toHaveBeenCalledWith(null, 'upload/upload/test-user'); }); }); diff --git a/server/apps/immich/src/config/asset-upload.config.ts b/server/apps/immich/src/config/asset-upload.config.ts index a262b3023e..6a4f6804e5 100644 --- a/server/apps/immich/src/config/asset-upload.config.ts +++ b/server/apps/immich/src/config/asset-upload.config.ts @@ -1,11 +1,11 @@ -import { APP_UPLOAD_LOCATION } from '@app/domain/domain.constant'; +import { StorageCore, StorageFolder } from '@app/domain/storage'; import { BadRequestException, Logger, UnauthorizedException } from '@nestjs/common'; import { MulterOptions } from '@nestjs/platform-express/multer/interfaces/multer-options.interface'; import { createHash, randomUUID } from 'crypto'; import { Request } from 'express'; import { existsSync, mkdirSync } from 'fs'; import { diskStorage, StorageEngine } from 'multer'; -import { extname, join } from 'path'; +import { extname } from 'path'; import sanitize from 'sanitize-filename'; import { AuthUserDto } from '../decorators/auth-user.decorator'; import { patchFormData } from '../utils/path-form-data.util'; @@ -20,6 +20,8 @@ export const assetUploadOption: MulterOptions = { storage: customStorage(), }; +const storageCore = new StorageCore(); + export function customStorage(): StorageEngine { const storage = diskStorage({ destination, filename }); @@ -71,16 +73,13 @@ function destination(req: Request, file: Express.Multer.File, cb: any) { const user = req.user as AuthUserDto; - const basePath = APP_UPLOAD_LOCATION; - const sanitizedDeviceId = sanitize(String(req.body['deviceId'])); - const originalUploadFolder = join(basePath, user.id, 'original', sanitizedDeviceId); - - if (!existsSync(originalUploadFolder)) { - mkdirSync(originalUploadFolder, { recursive: true }); + const uploadFolder = storageCore.getFolderLocation(StorageFolder.UPLOAD, user.id); + if (!existsSync(uploadFolder)) { + mkdirSync(uploadFolder, { recursive: true }); } // Save original to disk - cb(null, originalUploadFolder); + cb(null, uploadFolder); } function filename(req: Request, file: Express.Multer.File, cb: any) { diff --git a/server/apps/immich/src/config/profile-image-upload.config.spec.ts b/server/apps/immich/src/config/profile-image-upload.config.spec.ts index b483c7d0d0..e2006a1510 100644 --- a/server/apps/immich/src/config/profile-image-upload.config.spec.ts +++ b/server/apps/immich/src/config/profile-image-upload.config.spec.ts @@ -85,7 +85,7 @@ describe('profileImageUploadOption', () => { destination(mock.userRequest, mock.file, callback); expect(mkdirSync).not.toHaveBeenCalled(); - expect(callback).toHaveBeenCalledWith(null, './upload/test-user/profile'); + expect(callback).toHaveBeenCalledWith(null, 'upload/profile/test-user'); }); }); diff --git a/server/apps/immich/src/config/profile-image-upload.config.ts b/server/apps/immich/src/config/profile-image-upload.config.ts index 6a970020d4..8ce14c4ed6 100644 --- a/server/apps/immich/src/config/profile-image-upload.config.ts +++ b/server/apps/immich/src/config/profile-image-upload.config.ts @@ -1,4 +1,4 @@ -import { APP_UPLOAD_LOCATION } from '@app/domain/domain.constant'; +import { StorageCore, StorageFolder } from '@app/domain/storage'; import { BadRequestException, UnauthorizedException } from '@nestjs/common'; import { MulterOptions } from '@nestjs/platform-express/multer/interfaces/multer-options.interface'; import { Request } from 'express'; @@ -19,6 +19,8 @@ export const profileImageUploadOption: MulterOptions = { export const multerUtils = { fileFilter, filename, destination }; +const storageCore = new StorageCore(); + function fileFilter(req: Request, file: any, cb: any) { if (!req.user) { return cb(new UnauthorizedException()); @@ -38,9 +40,7 @@ function destination(req: Request, file: Express.Multer.File, cb: any) { const user = req.user as AuthUserDto; - const basePath = APP_UPLOAD_LOCATION; - const profileImageLocation = `${basePath}/${user.id}/profile`; - + const profileImageLocation = storageCore.getFolderLocation(StorageFolder.PROFILE, user.id); if (!existsSync(profileImageLocation)) { mkdirSync(profileImageLocation, { recursive: true }); } diff --git a/server/apps/microservices/src/processors/video-transcode.processor.ts b/server/apps/microservices/src/processors/video-transcode.processor.ts index 8dc84a2b8e..e02009fcd0 100644 --- a/server/apps/microservices/src/processors/video-transcode.processor.ts +++ b/server/apps/microservices/src/processors/video-transcode.processor.ts @@ -1,11 +1,13 @@ import { - APP_UPLOAD_LOCATION, IAssetJob, IAssetRepository, IBaseJob, IJobRepository, + IStorageRepository, JobName, QueueName, + StorageCore, + StorageFolder, SystemConfigService, WithoutProperty, } from '@app/domain'; @@ -14,15 +16,18 @@ import { Process, Processor } from '@nestjs/bull'; import { Inject, Logger } from '@nestjs/common'; import { Job } from 'bull'; import ffmpeg, { FfprobeData } from 'fluent-ffmpeg'; -import { existsSync, mkdirSync } from 'fs'; +import { join } from 'path'; @Processor(QueueName.VIDEO_CONVERSION) export class VideoTranscodeProcessor { readonly logger = new Logger(VideoTranscodeProcessor.name); + private storageCore = new StorageCore(); + constructor( @Inject(IAssetRepository) private assetRepository: IAssetRepository, @Inject(IJobRepository) private jobRepository: IJobRepository, private systemConfigService: SystemConfigService, + @Inject(IStorageRepository) private storageRepository: IStorageRepository, ) {} @Process({ name: JobName.QUEUE_VIDEO_CONVERSION, concurrency: 1 }) @@ -43,14 +48,12 @@ export class VideoTranscodeProcessor { @Process({ name: JobName.VIDEO_CONVERSION, concurrency: 2 }) async handleVideoConversion(job: Job) { const { asset } = job.data; - const basePath = APP_UPLOAD_LOCATION; - const encodedVideoPath = `${basePath}/${asset.ownerId}/encoded-video`; - if (!existsSync(encodedVideoPath)) { - mkdirSync(encodedVideoPath, { recursive: true }); - } + const encodedVideoPath = this.storageCore.getFolderLocation(StorageFolder.ENCODED_VIDEO, asset.ownerId); - const savedEncodedPath = `${encodedVideoPath}/${asset.id}.mp4`; + this.storageRepository.mkdirSync(encodedVideoPath); + + const savedEncodedPath = join(encodedVideoPath, `${asset.id}.mp4`); await this.runVideoEncode(asset, savedEncodedPath); } diff --git a/server/libs/domain/src/domain.constant.ts b/server/libs/domain/src/domain.constant.ts index 2dd0c1e578..b0e0759773 100644 --- a/server/libs/domain/src/domain.constant.ts +++ b/server/libs/domain/src/domain.constant.ts @@ -17,7 +17,7 @@ export const serverVersion: IServerVersion = { export const SERVER_VERSION = `${serverVersion.major}.${serverVersion.minor}.${serverVersion.patch}`; -export const APP_UPLOAD_LOCATION = './upload'; +export const APP_MEDIA_LOCATION = process.env.IMMICH_MEDIA_LOCATION || './upload'; export const MACHINE_LEARNING_URL = process.env.IMMICH_MACHINE_LEARNING_URL || 'http://immich-machine-learning:3003'; export const MACHINE_LEARNING_ENABLED = MACHINE_LEARNING_URL !== 'false'; diff --git a/server/libs/domain/src/media/media.service.spec.ts b/server/libs/domain/src/media/media.service.spec.ts index 979b37c7d3..6de82faee3 100644 --- a/server/libs/domain/src/media/media.service.spec.ts +++ b/server/libs/domain/src/media/media.service.spec.ts @@ -75,16 +75,15 @@ describe(MediaService.name, () => { it('should generate a thumbnail for an image', async () => { await sut.handleGenerateJpegThumbnail({ asset: _.cloneDeep(assetEntityStub.image) }); - expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/user-id/thumb/device-id'); - expect(mediaMock.resize).toHaveBeenCalledWith( - '/original/path.ext', - 'upload/user-id/thumb/device-id/asset-id.jpeg', - { size: 1440, format: 'jpeg' }, - ); + expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/thumbs/user-id'); + expect(mediaMock.resize).toHaveBeenCalledWith('/original/path.ext', 'upload/thumbs/user-id/asset-id.jpeg', { + size: 1440, + format: 'jpeg', + }); expect(mediaMock.extractThumbnailFromExif).not.toHaveBeenCalled(); expect(assetMock.save).toHaveBeenCalledWith({ id: 'asset-id', - resizePath: 'upload/user-id/thumb/device-id/asset-id.jpeg', + resizePath: 'upload/thumbs/user-id/asset-id.jpeg', }); }); @@ -93,33 +92,32 @@ describe(MediaService.name, () => { await sut.handleGenerateJpegThumbnail({ asset: _.cloneDeep(assetEntityStub.image) }); - expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/user-id/thumb/device-id'); - expect(mediaMock.resize).toHaveBeenCalledWith( - '/original/path.ext', - 'upload/user-id/thumb/device-id/asset-id.jpeg', - { size: 1440, format: 'jpeg' }, - ); + expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/thumbs/user-id'); + expect(mediaMock.resize).toHaveBeenCalledWith('/original/path.ext', 'upload/thumbs/user-id/asset-id.jpeg', { + size: 1440, + format: 'jpeg', + }); expect(mediaMock.extractThumbnailFromExif).toHaveBeenCalledWith( '/original/path.ext', - 'upload/user-id/thumb/device-id/asset-id.jpeg', + 'upload/thumbs/user-id/asset-id.jpeg', ); expect(assetMock.save).toHaveBeenCalledWith({ id: 'asset-id', - resizePath: 'upload/user-id/thumb/device-id/asset-id.jpeg', + resizePath: 'upload/thumbs/user-id/asset-id.jpeg', }); }); it('should generate a thumbnail for a video', async () => { await sut.handleGenerateJpegThumbnail({ asset: _.cloneDeep(assetEntityStub.video) }); - expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/user-id/thumb/device-id'); + expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/thumbs/user-id'); expect(mediaMock.extractVideoThumbnail).toHaveBeenCalledWith( '/original/path.ext', - 'upload/user-id/thumb/device-id/asset-id.jpeg', + 'upload/thumbs/user-id/asset-id.jpeg', ); expect(assetMock.save).toHaveBeenCalledWith({ id: 'asset-id', - resizePath: 'upload/user-id/thumb/device-id/asset-id.jpeg', + resizePath: 'upload/thumbs/user-id/asset-id.jpeg', }); }); diff --git a/server/libs/domain/src/media/media.service.ts b/server/libs/domain/src/media/media.service.ts index 1da60ad2a3..8d470928c9 100644 --- a/server/libs/domain/src/media/media.service.ts +++ b/server/libs/domain/src/media/media.service.ts @@ -1,17 +1,16 @@ import { AssetType } from '@app/infra/db/entities'; import { Inject, Injectable, Logger } from '@nestjs/common'; import { join } from 'path'; -import sanitize from 'sanitize-filename'; import { IAssetRepository, mapAsset, WithoutProperty } from '../asset'; import { CommunicationEvent, ICommunicationRepository } from '../communication'; -import { APP_UPLOAD_LOCATION } from '../domain.constant'; import { IAssetJob, IBaseJob, IJobRepository, JobName } from '../job'; -import { IStorageRepository } from '../storage'; +import { IStorageRepository, StorageCore, StorageFolder } from '../storage'; import { IMediaRepository } from './media.repository'; @Injectable() export class MediaService { private logger = new Logger(MediaService.name); + private storageCore = new StorageCore(); constructor( @Inject(IAssetRepository) private assetRepository: IAssetRepository, @@ -41,11 +40,9 @@ export class MediaService { const { asset } = data; try { - const basePath = APP_UPLOAD_LOCATION; - const sanitizedDeviceId = sanitize(String(asset.deviceId)); - const resizePath = join(basePath, asset.ownerId, 'thumb', sanitizedDeviceId); - const jpegThumbnailPath = join(resizePath, `${asset.id}.jpeg`); + const resizePath = this.storageCore.getFolderLocation(StorageFolder.THUMBNAILS, asset.ownerId); this.storageRepository.mkdirSync(resizePath); + const jpegThumbnailPath = join(resizePath, `${asset.id}.jpeg`); if (asset.type == AssetType.IMAGE) { try { diff --git a/server/libs/domain/src/server-info/server-info.service.ts b/server/libs/domain/src/server-info/server-info.service.ts index 53f8ecfc69..5e49668204 100644 --- a/server/libs/domain/src/server-info/server-info.service.ts +++ b/server/libs/domain/src/server-info/server-info.service.ts @@ -1,5 +1,5 @@ import { Inject, Injectable } from '@nestjs/common'; -import { APP_UPLOAD_LOCATION, serverVersion } from '../domain.constant'; +import { APP_MEDIA_LOCATION, serverVersion } from '../domain.constant'; import { asHumanReadable } from '../domain.util'; import { IStorageRepository } from '../storage'; import { IUserRepository, UserStatsQueryResponse } from '../user'; @@ -13,7 +13,7 @@ export class ServerInfoService { ) {} async getInfo(): Promise { - const diskInfo = await this.storageRepository.checkDiskUsage(APP_UPLOAD_LOCATION); + const diskInfo = await this.storageRepository.checkDiskUsage(APP_MEDIA_LOCATION); const usagePercentage = (((diskInfo.total - diskInfo.free) / diskInfo.total) * 100).toFixed(2); diff --git a/server/libs/domain/src/storage-template/storage-template.core.ts b/server/libs/domain/src/storage-template/storage-template.core.ts index a08a512690..902df6ec4d 100644 --- a/server/libs/domain/src/storage-template/storage-template.core.ts +++ b/server/libs/domain/src/storage-template/storage-template.core.ts @@ -1,5 +1,11 @@ +import { AssetEntity, AssetType, SystemConfig } from '@app/infra/db/entities'; +import { Logger } from '@nestjs/common'; +import handlebar from 'handlebars'; +import * as luxon from 'luxon'; +import path from 'node:path'; +import sanitize from 'sanitize-filename'; +import { IStorageRepository, StorageCore, StorageFolder } from '../storage'; import { - IStorageRepository, ISystemConfigRepository, supportedDayTokens, supportedHourTokens, @@ -7,20 +13,14 @@ import { supportedMonthTokens, supportedSecondTokens, supportedYearTokens, -} from '@app/domain'; -import { AssetEntity, AssetType, SystemConfig } from '@app/infra/db/entities'; -import { Logger } from '@nestjs/common'; -import handlebar from 'handlebars'; -import * as luxon from 'luxon'; -import path from 'node:path'; -import sanitize from 'sanitize-filename'; -import { APP_UPLOAD_LOCATION } from '../domain.constant'; +} from '../system-config'; import { SystemConfigCore } from '../system-config/system-config.core'; export class StorageTemplateCore { private logger = new Logger(StorageTemplateCore.name); private configCore: SystemConfigCore; private storageTemplate: HandlebarsTemplateDelegate; + private storageCore = new StorageCore(); constructor( configRepository: ISystemConfigRepository, @@ -38,7 +38,7 @@ export class StorageTemplateCore { const source = asset.originalPath; const ext = path.extname(source).split('.').pop() as string; const sanitized = sanitize(path.basename(filename, `.${ext}`)); - const rootPath = path.join(APP_UPLOAD_LOCATION, asset.ownerId); + const rootPath = this.storageCore.getFolderLocation(StorageFolder.LIBRARY, asset.ownerId); const storagePath = this.render(this.storageTemplate, asset, sanitized, ext); const fullPath = path.normalize(path.join(rootPath, storagePath)); let destination = `${fullPath}.${ext}`; diff --git a/server/libs/domain/src/storage-template/storage-template.service.spec.ts b/server/libs/domain/src/storage-template/storage-template.service.spec.ts index 6401b7ab46..d14f23d7a4 100644 --- a/server/libs/domain/src/storage-template/storage-template.service.spec.ts +++ b/server/libs/domain/src/storage-template/storage-template.service.spec.ts @@ -42,11 +42,11 @@ describe(StorageTemplateService.name, () => { assetMock.save.mockResolvedValue(assetEntityStub.image); when(storageMock.checkFileExists) - .calledWith('upload/user-id/2023/2023-02-23/asset-id.ext') + .calledWith('upload/library/user-id/2023/2023-02-23/asset-id.ext') .mockResolvedValue(true); when(storageMock.checkFileExists) - .calledWith('upload/user-id/2023/2023-02-23/asset-id+1.ext') + .calledWith('upload/library/user-id/2023/2023-02-23/asset-id+1.ext') .mockResolvedValue(false); await sut.handleTemplateMigration(); @@ -55,7 +55,7 @@ describe(StorageTemplateService.name, () => { expect(storageMock.checkFileExists).toHaveBeenCalledTimes(2); expect(assetMock.save).toHaveBeenCalledWith({ id: assetEntityStub.image.id, - originalPath: 'upload/user-id/2023/2023-02-23/asset-id+1.ext', + originalPath: 'upload/library/user-id/2023/2023-02-23/asset-id+1.ext', }); }); @@ -63,7 +63,7 @@ describe(StorageTemplateService.name, () => { assetMock.getAll.mockResolvedValue([ { ...assetEntityStub.image, - originalPath: 'upload/user-id/2023/2023-02-23/asset-id.ext', + originalPath: 'upload/library/user-id/2023/2023-02-23/asset-id.ext', }, ]); @@ -79,7 +79,7 @@ describe(StorageTemplateService.name, () => { assetMock.getAll.mockResolvedValue([ { ...assetEntityStub.image, - originalPath: 'upload/user-id/2023/2023-02-23/asset-id+1.ext', + originalPath: 'upload/library/user-id/2023/2023-02-23/asset-id+1.ext', }, ]); @@ -100,11 +100,11 @@ describe(StorageTemplateService.name, () => { expect(assetMock.getAll).toHaveBeenCalled(); expect(storageMock.moveFile).toHaveBeenCalledWith( '/original/path.ext', - 'upload/user-id/2023/2023-02-23/asset-id.ext', + 'upload/library/user-id/2023/2023-02-23/asset-id.ext', ); expect(assetMock.save).toHaveBeenCalledWith({ id: assetEntityStub.image.id, - originalPath: 'upload/user-id/2023/2023-02-23/asset-id.ext', + originalPath: 'upload/library/user-id/2023/2023-02-23/asset-id.ext', }); }); @@ -117,7 +117,7 @@ describe(StorageTemplateService.name, () => { expect(assetMock.getAll).toHaveBeenCalled(); expect(storageMock.moveFile).toHaveBeenCalledWith( '/original/path.ext', - 'upload/user-id/2023/2023-02-23/asset-id.ext', + 'upload/library/user-id/2023/2023-02-23/asset-id.ext', ); expect(assetMock.save).not.toHaveBeenCalled(); }); @@ -131,11 +131,11 @@ describe(StorageTemplateService.name, () => { expect(assetMock.getAll).toHaveBeenCalled(); expect(assetMock.save).toHaveBeenCalledWith({ id: assetEntityStub.image.id, - originalPath: 'upload/user-id/2023/2023-02-23/asset-id.ext', + originalPath: 'upload/library/user-id/2023/2023-02-23/asset-id.ext', }); expect(storageMock.moveFile.mock.calls).toEqual([ - ['/original/path.ext', 'upload/user-id/2023/2023-02-23/asset-id.ext'], - ['upload/user-id/2023/2023-02-23/asset-id.ext', '/original/path.ext'], + ['/original/path.ext', 'upload/library/user-id/2023/2023-02-23/asset-id.ext'], + ['upload/library/user-id/2023/2023-02-23/asset-id.ext', '/original/path.ext'], ]); }); }); diff --git a/server/libs/domain/src/storage-template/storage-template.service.ts b/server/libs/domain/src/storage-template/storage-template.service.ts index 9cf225fd43..947e1b8070 100644 --- a/server/libs/domain/src/storage-template/storage-template.service.ts +++ b/server/libs/domain/src/storage-template/storage-template.service.ts @@ -1,7 +1,7 @@ import { AssetEntity, SystemConfig } from '@app/infra/db/entities'; import { Inject, Injectable, Logger } from '@nestjs/common'; import { IAssetRepository } from '../asset/asset.repository'; -import { APP_UPLOAD_LOCATION } from '../domain.constant'; +import { APP_MEDIA_LOCATION } from '../domain.constant'; import { IStorageRepository } from '../storage/storage.repository'; import { INITIAL_SYSTEM_CONFIG, ISystemConfigRepository } from '../system-config'; import { StorageTemplateCore } from './storage-template.core'; @@ -41,7 +41,7 @@ export class StorageTemplateService { } this.logger.debug('Cleaning up empty directories...'); - await this.storageRepository.removeEmptyDirs(APP_UPLOAD_LOCATION); + await this.storageRepository.removeEmptyDirs(APP_MEDIA_LOCATION); } catch (error: any) { this.logger.error('Error running template migration', error); } finally { diff --git a/server/libs/domain/src/storage/index.ts b/server/libs/domain/src/storage/index.ts index 72e76e38ac..7a28d7af8f 100644 --- a/server/libs/domain/src/storage/index.ts +++ b/server/libs/domain/src/storage/index.ts @@ -1,2 +1,3 @@ +export * from './storage.core'; export * from './storage.repository'; export * from './storage.service'; diff --git a/server/libs/domain/src/storage/storage.core.ts b/server/libs/domain/src/storage/storage.core.ts new file mode 100644 index 0000000000..87827d0c2d --- /dev/null +++ b/server/libs/domain/src/storage/storage.core.ts @@ -0,0 +1,16 @@ +import { join } from 'node:path'; +import { APP_MEDIA_LOCATION } from '../domain.constant'; + +export enum StorageFolder { + ENCODED_VIDEO = 'encoded-video', + LIBRARY = 'library', + UPLOAD = 'upload', + PROFILE = 'profile', + THUMBNAILS = 'thumbs', +} + +export class StorageCore { + getFolderLocation(folder: StorageFolder, userId: string) { + return join(APP_MEDIA_LOCATION, folder, userId); + } +} diff --git a/server/libs/domain/src/user/user.service.spec.ts b/server/libs/domain/src/user/user.service.spec.ts index 5973b4da07..ac2c6db7e1 100644 --- a/server/libs/domain/src/user/user.service.spec.ts +++ b/server/libs/domain/src/user/user.service.spec.ts @@ -467,7 +467,13 @@ describe(UserService.name, () => { await sut.handleUserDelete({ user }); - expect(storageMock.unlinkDir).toHaveBeenCalledWith('upload/deleted-user', { force: true, recursive: true }); + const options = { force: true, recursive: true }; + + expect(storageMock.unlinkDir).toHaveBeenCalledWith('upload/library/deleted-user', options); + expect(storageMock.unlinkDir).toHaveBeenCalledWith('upload/upload/deleted-user', options); + expect(storageMock.unlinkDir).toHaveBeenCalledWith('upload/profile/deleted-user', options); + expect(storageMock.unlinkDir).toHaveBeenCalledWith('upload/thumbs/deleted-user', options); + expect(storageMock.unlinkDir).toHaveBeenCalledWith('upload/encoded-video/deleted-user', options); expect(tokenMock.deleteAll).toHaveBeenCalledWith(user.id); expect(keyMock.deleteAll).toHaveBeenCalledWith(user.id); expect(albumMock.deleteAll).toHaveBeenCalledWith(user.id); diff --git a/server/libs/domain/src/user/user.service.ts b/server/libs/domain/src/user/user.service.ts index acb1c87f18..e999f12b3c 100644 --- a/server/libs/domain/src/user/user.service.ts +++ b/server/libs/domain/src/user/user.service.ts @@ -2,14 +2,13 @@ import { UserEntity } from '@app/infra/db/entities'; import { BadRequestException, Inject, Injectable, Logger, NotFoundException } from '@nestjs/common'; import { randomBytes } from 'crypto'; import { ReadStream } from 'fs'; -import { join } from 'path'; import { IAlbumRepository } from '../album/album.repository'; import { IKeyRepository } from '../api-key/api-key.repository'; import { IAssetRepository } from '../asset/asset.repository'; import { AuthUserDto } from '../auth'; import { ICryptoRepository } from '../crypto/crypto.repository'; -import { APP_UPLOAD_LOCATION } from '../domain.constant'; import { IJobRepository, IUserDeletionJob, JobName } from '../job'; +import { StorageCore, StorageFolder } from '../storage'; import { IStorageRepository } from '../storage/storage.repository'; import { IUserTokenRepository } from '../user-token/user-token.repository'; import { IUserRepository } from '../user/user.repository'; @@ -28,6 +27,8 @@ import { UserCore } from './user.core'; export class UserService { private logger = new Logger(UserService.name); private userCore: UserCore; + private storageCore = new StorageCore(); + constructor( @Inject(IUserRepository) private userRepository: IUserRepository, @Inject(ICryptoRepository) cryptoRepository: ICryptoRepository, @@ -162,9 +163,18 @@ export class UserService { this.logger.log(`Deleting user: ${user.id}`); try { - const userAssetDir = join(APP_UPLOAD_LOCATION, user.id); - this.logger.warn(`Removing user from filesystem: ${userAssetDir}`); - await this.storageRepository.unlinkDir(userAssetDir, { recursive: true, force: true }); + const folders = [ + this.storageCore.getFolderLocation(StorageFolder.LIBRARY, user.id), + this.storageCore.getFolderLocation(StorageFolder.UPLOAD, user.id), + this.storageCore.getFolderLocation(StorageFolder.PROFILE, user.id), + this.storageCore.getFolderLocation(StorageFolder.THUMBNAILS, user.id), + this.storageCore.getFolderLocation(StorageFolder.ENCODED_VIDEO, user.id), + ]; + + for (const folder of folders) { + this.logger.warn(`Removing user from filesystem: ${folder}`); + await this.storageRepository.unlinkDir(folder, { recursive: true, force: true }); + } this.logger.warn(`Removing user from database: ${user.id}`); diff --git a/server/libs/domain/test/fixtures.ts b/server/libs/domain/test/fixtures.ts index 82b0729feb..9e3e3a286d 100644 --- a/server/libs/domain/test/fixtures.ts +++ b/server/libs/domain/test/fixtures.ts @@ -119,7 +119,7 @@ export const assetEntityStub = { owner: userEntityStub.user1, ownerId: 'user-id', deviceId: 'device-id', - originalPath: '/original/path.ext', + originalPath: 'upload/upload/path.ext', resizePath: null, type: AssetType.IMAGE, webpPath: null, diff --git a/server/package.json b/server/package.json index ef16490881..e886b7c576 100644 --- a/server/package.json +++ b/server/package.json @@ -29,6 +29,7 @@ "test:debug": "node --inspect-brk -r tsconfig-paths/register -r ts-node/register node_modules/.bin/jest --runInBand", "test:e2e": "jest --config ./apps/immich/test/jest-e2e.json --runInBand", "typeorm": "node --require ts-node/register ./node_modules/typeorm/cli.js", + "typeorm:migrations:create": "node --require ts-node/register ./node_modules/typeorm/cli.js migration:create", "typeorm:migrations:generate": "node --require ts-node/register ./node_modules/typeorm/cli.js migration:generate -d ./libs/infra/src/db/config/database.config.ts", "typeorm:migrations:run": "node --require ts-node/register ./node_modules/typeorm/cli.js migration:run -d ./libs/infra/src/db/config/database.config.ts", "typeorm:migrations:revert": "node --require ts-node/register ./node_modules/typeorm/cli.js migration:revert -d ./libs/infra/src/db/config/database.config.ts",