refactor(server): storage template core (#3059)

server/src/domain/storage-template/index.ts

@@ -1,2 +1 @@
-export * from './storage-template.core';
 export * from './storage-template.service';

server/src/domain/storage-template/storage-template.core.ts

@@ -1,160 +0,0 @@
-import { AssetEntity, AssetType, SystemConfig } from '@app/infra/entities';
-import { Logger } from '@nestjs/common';
-import handlebar from 'handlebars';
-import * as luxon from 'luxon';
-import path from 'node:path';
-import sanitize from 'sanitize-filename';
-import { IStorageRepository, StorageCore } from '../storage';
-import {
-  ISystemConfigRepository,
-  supportedDayTokens,
-  supportedHourTokens,
-  supportedMinuteTokens,
-  supportedMonthTokens,
-  supportedSecondTokens,
-  supportedYearTokens,
-} from '../system-config';
-import { SystemConfigCore } from '../system-config/system-config.core';
-import { MoveAssetMetadata } from './storage-template.service';
-
-export class StorageTemplateCore {
-  private logger = new Logger(StorageTemplateCore.name);
-  private configCore: SystemConfigCore;
-  private storageTemplate: HandlebarsTemplateDelegate<any>;
-  private storageCore = new StorageCore();
-
-  constructor(
-    configRepository: ISystemConfigRepository,
-    config: SystemConfig,
-    private storageRepository: IStorageRepository,
-  ) {
-    this.storageTemplate = this.compile(config.storageTemplate.template);
-    this.configCore = new SystemConfigCore(configRepository);
-    this.configCore.addValidator((config) => this.validateConfig(config));
-    this.configCore.config$.subscribe((config) => this.onConfig(config));
-  }
-
-  public async getTemplatePath(asset: AssetEntity, metadata: MoveAssetMetadata): Promise<string> {
-    const { storageLabel, filename } = metadata;
-
-    try {
-      const source = asset.originalPath;
-      const ext = path.extname(source).split('.').pop() as string;
-      const sanitized = sanitize(path.basename(filename, `.${ext}`));
-      const rootPath = this.storageCore.getLibraryFolder({ id: asset.ownerId, storageLabel });
-      const storagePath = this.render(this.storageTemplate, asset, sanitized, ext);
-      const fullPath = path.normalize(path.join(rootPath, storagePath));
-      let destination = `${fullPath}.${ext}`;
-
-      if (!fullPath.startsWith(rootPath)) {
-        this.logger.warn(`Skipped attempt to access an invalid path: ${fullPath}. Path should start with ${rootPath}`);
-        return source;
-      }
-
-      if (source === destination) {
-        return source;
-      }
-
-      /**
-       * In case of migrating duplicate filename to a new path, we need to check if it is already migrated
-       * Due to the mechanism of appending +1, +2, +3, etc to the filename
-       *
-       * Example:
-       * Source = upload/abc/def/FullSizeRender+7.heic
-       * Expected Destination = upload/abc/def/FullSizeRender.heic
-       *
-       * The file is already at the correct location, but since there are other FullSizeRender.heic files in the
-       * destination, it was renamed to FullSizeRender+7.heic.
-       *
-       * The lines below will be used to check if the differences between the source and destination is only the
-       * +7 suffix, and if so, it will be considered as already migrated.
-       */
-      if (source.startsWith(fullPath) && source.endsWith(`.${ext}`)) {
-        const diff = source.replace(fullPath, '').replace(`.${ext}`, '');
-        const hasDuplicationAnnotation = /^\+\d+$/.test(diff);
-        if (hasDuplicationAnnotation) {
-          return source;
-        }
-      }
-
-      let duplicateCount = 0;
-
-      while (true) {
-        const exists = await this.storageRepository.checkFileExists(destination);
-        if (!exists) {
-          break;
-        }
-
-        duplicateCount++;
-        destination = `${fullPath}+${duplicateCount}.${ext}`;
-      }
-
-      return destination;
-    } catch (error: any) {
-      this.logger.error(`Unable to get template path for ${filename}`, error);
-      return asset.originalPath;
-    }
-  }
-
-  private validateConfig(config: SystemConfig) {
-    this.validateStorageTemplate(config.storageTemplate.template);
-  }
-
-  private validateStorageTemplate(templateString: string) {
-    try {
-      const template = this.compile(templateString);
-      // test render an asset
-      this.render(
-        template,
-        {
-          fileCreatedAt: new Date(),
-          originalPath: '/upload/test/IMG_123.jpg',
-          type: AssetType.IMAGE,
-        } as AssetEntity,
-        'IMG_123',
-        'jpg',
-      );
-    } catch (e) {
-      this.logger.warn(`Storage template validation failed: ${JSON.stringify(e)}`);
-      throw new Error(`Invalid storage template: ${e}`);
-    }
-  }
-
-  private onConfig(config: SystemConfig) {
-    this.logger.debug(`Received new config, recompiling storage template: ${config.storageTemplate.template}`);
-    this.storageTemplate = this.compile(config.storageTemplate.template);
-  }
-
-  private compile(template: string) {
-    return handlebar.compile(template, {
-      knownHelpers: undefined,
-      strict: true,
-    });
-  }
-
-  private render(template: HandlebarsTemplateDelegate<any>, asset: AssetEntity, filename: string, ext: string) {
-    const substitutions: Record<string, string> = {
-      filename,
-      ext,
-      filetype: asset.type == AssetType.IMAGE ? 'IMG' : 'VID',
-      filetypefull: asset.type == AssetType.IMAGE ? 'IMAGE' : 'VIDEO',
-    };
-
-    const dt = luxon.DateTime.fromJSDate(asset.fileCreatedAt);
-
-    const dateTokens = [
-      ...supportedYearTokens,
-      ...supportedMonthTokens,
-      ...supportedDayTokens,
-      ...supportedHourTokens,
-      ...supportedMinuteTokens,
-      ...supportedSecondTokens,
-    ];
-
-    for (const token of dateTokens) {
-      substitutions[token] = dt.toFormat(token);
-    }
-
-    return template(substitutions);
-  }
-}

server/src/domain/storage-template/storage-template.service.ts

@@ -1,13 +1,25 @@
-import { AssetEntity, SystemConfig } from '@app/infra/entities';
+import { AssetEntity, AssetType, SystemConfig } from '@app/infra/entities';
 import { Inject, Injectable, Logger } from '@nestjs/common';
+import handlebar from 'handlebars';
+import * as luxon from 'luxon';
+import path from 'node:path';
+import sanitize from 'sanitize-filename';
 import { IAssetRepository } from '../asset/asset.repository';
-import { APP_MEDIA_LOCATION } from '../domain.constant';
 import { getLivePhotoMotionFilename, usePagination } from '../domain.util';
 import { IEntityJob, JOBS_ASSET_PAGINATION_SIZE } from '../job';
-import { IStorageRepository } from '../storage/storage.repository';
+import { IStorageRepository, StorageCore, StorageFolder } from '../storage';
-import { INITIAL_SYSTEM_CONFIG, ISystemConfigRepository } from '../system-config';
+import {
+  INITIAL_SYSTEM_CONFIG,
+  ISystemConfigRepository,
+  supportedDayTokens,
+  supportedHourTokens,
+  supportedMinuteTokens,
+  supportedMonthTokens,
+  supportedSecondTokens,
+  supportedYearTokens,
+} from '../system-config';
+import { SystemConfigCore } from '../system-config/system-config.core';
 import { IUserRepository } from '../user/user.repository';
-import { StorageTemplateCore } from './storage-template.core';
 
 export interface MoveAssetMetadata {
   storageLabel: string | null;
@@ -17,7 +29,9 @@ export interface MoveAssetMetadata {
 @Injectable()
 export class StorageTemplateService {
   private logger = new Logger(StorageTemplateService.name);
-  private core: StorageTemplateCore;
+  private configCore: SystemConfigCore;
+  private storageCore = new StorageCore();
+  private storageTemplate: HandlebarsTemplateDelegate<any>;
 
   constructor(
     @Inject(IAssetRepository) private assetRepository: IAssetRepository,
@@ -26,7 +40,10 @@ export class StorageTemplateService {
     @Inject(IStorageRepository) private storageRepository: IStorageRepository,
     @Inject(IUserRepository) private userRepository: IUserRepository,
   ) {
-    this.core = new StorageTemplateCore(configRepository, config, storageRepository);
+    this.storageTemplate = this.compile(config.storageTemplate.template);
+    this.configCore = new SystemConfigCore(configRepository);
+    this.configCore.addValidator((config) => this.validate(config));
+    this.configCore.config$.subscribe((config) => this.onConfig(config));
   }
 
   async handleMigrationSingle({ id }: IEntityJob) {
@@ -48,29 +65,27 @@ export class StorageTemplateService {
   }
 
   async handleMigration() {
-    try {
+    this.logger.log('Starting storage template migration');
-      console.time('migrating-time');
+    const assetPagination = usePagination(JOBS_ASSET_PAGINATION_SIZE, (pagination) =>
+      this.assetRepository.getAll(pagination),
+    );
+    const users = await this.userRepository.getList();
 
-      const assetPagination = usePagination(JOBS_ASSET_PAGINATION_SIZE, (pagination) =>
+    for await (const assets of assetPagination) {
-        this.assetRepository.getAll(pagination),
+      for (const asset of assets) {
-      );
+        const user = users.find((user) => user.id === asset.ownerId);
-      const users = await this.userRepository.getList();
+        const storageLabel = user?.storageLabel || null;
-
+        const filename = asset.originalFileName || asset.id;
-      for await (const assets of assetPagination) {
+        await this.moveAsset(asset, { storageLabel, filename });
-        for (const asset of assets) {
-          const user = users.find((user) => user.id === asset.ownerId);
-          const storageLabel = user?.storageLabel || null;
-          const filename = asset.originalFileName || asset.id;
-          await this.moveAsset(asset, { storageLabel, filename });
-        }
       }
-
-      this.logger.debug('Cleaning up empty directories...');
-      await this.storageRepository.removeEmptyDirs(APP_MEDIA_LOCATION);
-    } finally {
-      console.timeEnd('migrating-time');
     }
 
+    this.logger.debug('Cleaning up empty directories...');
+    const libraryFolder = this.storageCore.getBaseFolder(StorageFolder.LIBRARY);
+    await this.storageRepository.removeEmptyDirs(libraryFolder);
+
+    this.logger.log('Finished storage template migration');
+
     return true;
   }
 
@@ -81,7 +96,7 @@ export class StorageTemplateService {
       return;
     }
 
-    const destination = await this.core.getTemplatePath(asset, metadata);
+    const destination = await this.getTemplatePath(asset, metadata);
     if (asset.originalPath !== destination) {
       const source = asset.originalPath;
 
@@ -121,4 +136,118 @@ export class StorageTemplateService {
     }
     return asset;
   }
+
+  private async getTemplatePath(asset: AssetEntity, metadata: MoveAssetMetadata): Promise<string> {
+    const { storageLabel, filename } = metadata;
+
+    try {
+      const source = asset.originalPath;
+      const ext = path.extname(source).split('.').pop() as string;
+      const sanitized = sanitize(path.basename(filename, `.${ext}`));
+      const rootPath = this.storageCore.getLibraryFolder({ id: asset.ownerId, storageLabel });
+      const storagePath = this.render(this.storageTemplate, asset, sanitized, ext);
+      const fullPath = path.normalize(path.join(rootPath, storagePath));
+      let destination = `${fullPath}.${ext}`;
+
+      if (!fullPath.startsWith(rootPath)) {
+        this.logger.warn(`Skipped attempt to access an invalid path: ${fullPath}. Path should start with ${rootPath}`);
+        return source;
+      }
+
+      if (source === destination) {
+        return source;
+      }
+
+      /**
+       * In case of migrating duplicate filename to a new path, we need to check if it is already migrated
+       * Due to the mechanism of appending +1, +2, +3, etc to the filename
+       *
+       * Example:
+       * Source = upload/abc/def/FullSizeRender+7.heic
+       * Expected Destination = upload/abc/def/FullSizeRender.heic
+       *
+       * The file is already at the correct location, but since there are other FullSizeRender.heic files in the
+       * destination, it was renamed to FullSizeRender+7.heic.
+       *
+       * The lines below will be used to check if the differences between the source and destination is only the
+       * +7 suffix, and if so, it will be considered as already migrated.
+       */
+      if (source.startsWith(fullPath) && source.endsWith(`.${ext}`)) {
+        const diff = source.replace(fullPath, '').replace(`.${ext}`, '');
+        const hasDuplicationAnnotation = /^\+\d+$/.test(diff);
+        if (hasDuplicationAnnotation) {
+          return source;
+        }
+      }
+
+      let duplicateCount = 0;
+
+      while (true) {
+        const exists = await this.storageRepository.checkFileExists(destination);
+        if (!exists) {
+          break;
+        }
+
+        duplicateCount++;
+        destination = `${fullPath}+${duplicateCount}.${ext}`;
+      }
+
+      return destination;
+    } catch (error: any) {
+      this.logger.error(`Unable to get template path for ${filename}`, error);
+      return asset.originalPath;
+    }
+  }
+
+  private validate(config: SystemConfig) {
+    const testAsset = {
+      fileCreatedAt: new Date(),
+      originalPath: '/upload/test/IMG_123.jpg',
+      type: AssetType.IMAGE,
+    } as AssetEntity;
+    try {
+      this.render(this.compile(config.storageTemplate.template), testAsset, 'IMG_123', 'jpg');
+    } catch (e) {
+      this.logger.warn(`Storage template validation failed: ${JSON.stringify(e)}`);
+      throw new Error(`Invalid storage template: ${e}`);
+    }
+  }
+
+  private onConfig(config: SystemConfig) {
+    this.logger.debug(`Received new config, recompiling storage template: ${config.storageTemplate.template}`);
+    this.storageTemplate = this.compile(config.storageTemplate.template);
+  }
+
+  private compile(template: string) {
+    return handlebar.compile(template, {
+      knownHelpers: undefined,
+      strict: true,
+    });
+  }
+
+  private render(template: HandlebarsTemplateDelegate<any>, asset: AssetEntity, filename: string, ext: string) {
+    const substitutions: Record<string, string> = {
+      filename,
+      ext,
+      filetype: asset.type == AssetType.IMAGE ? 'IMG' : 'VID',
+      filetypefull: asset.type == AssetType.IMAGE ? 'IMAGE' : 'VIDEO',
+    };
+
+    const dt = luxon.DateTime.fromJSDate(asset.fileCreatedAt);
+
+    const dateTokens = [
+      ...supportedYearTokens,
+      ...supportedMonthTokens,
+      ...supportedDayTokens,
+      ...supportedHourTokens,
+      ...supportedMinuteTokens,
+      ...supportedSecondTokens,
+    ];
+
+    for (const token of dateTokens) {
+      substitutions[token] = dt.toFormat(token);
+    }
+
+    return template(substitutions);
+  }
 }