From 902d4d027515017d594e7f4919797def91ba3e9f Mon Sep 17 00:00:00 2001 From: Daniel Dietzler Date: Thu, 28 Dec 2023 19:06:35 +0100 Subject: [PATCH] settings for metrics --- server/immich-openapi-specs.json | 111 ++++++++++++ server/src/domain/metrics/metrics.dto.ts | 66 +++++-- server/src/domain/metrics/metrics.service.ts | 74 +++++--- .../src/domain/repositories/job.repository.ts | 3 +- .../domain/repositories/metrics.repository.ts | 9 +- .../src/domain/server-info/server-info.dto.ts | 1 + server/src/domain/system-config/dto/index.ts | 1 + .../dto/system-config-metrics.dto.ts | 18 ++ .../system-config/dto/system-config.dto.ts | 6 + .../system-config/system-config.core.ts | 16 ++ server/src/immich/app.module.ts | 2 + server/src/immich/controllers/index.ts | 1 + .../immich/controllers/metrics.controller.ts | 18 ++ .../infra/entities/system-config.entity.ts | 23 +++ .../infra/repositories/metrics.repository.ts | 4 +- server/src/microservices/app.service.ts | 2 +- .../metrics-settings/metrics-settings.svelte | 168 ++++++++++++++++++ .../routes/admin/system-settings/+page.svelte | 8 + 18 files changed, 486 insertions(+), 45 deletions(-) create mode 100644 server/src/domain/system-config/dto/system-config-metrics.dto.ts create mode 100644 server/src/immich/controllers/metrics.controller.ts create mode 100644 web/src/lib/components/admin-page/settings/metrics-settings/metrics-settings.svelte diff --git a/server/immich-openapi-specs.json b/server/immich-openapi-specs.json index 99779609bb..45e072576c 100644 --- a/server/immich-openapi-specs.json +++ b/server/immich-openapi-specs.json @@ -3716,6 +3716,48 @@ ] } }, + "/metrics": { + "put": { + "operationId": "getMetrics", + "parameters": [], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SystemConfigMetricsDto" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + }, + "description": "" + } + }, + "security": [ + { + "bearer": [] + }, + { + "cookie": [] + }, + { + "api_key": [] + } + ], + "tags": [ + "Metrics" + ] + } + }, "/oauth/authorize": { "post": { "operationId": "startOAuth", @@ -8061,6 +8103,48 @@ ], "type": "object" }, + "MetricServerInfoConfig": { + "properties": { + "cpuCount": { + "type": "boolean" + }, + "cpuModel": { + "type": "boolean" + }, + "memory": { + "type": "boolean" + }, + "version": { + "type": "boolean" + } + }, + "required": [ + "cpuCount", + "cpuModel", + "memory", + "version" + ], + "type": "object" + }, + "MetricsAssetCountConfig": { + "properties": { + "image": { + "type": "boolean" + }, + "total": { + "type": "boolean" + }, + "video": { + "type": "boolean" + } + }, + "required": [ + "image", + "video", + "total" + ], + "type": "object" + }, "ModelType": { "enum": [ "facial-recognition", @@ -8628,6 +8712,9 @@ "map": { "type": "boolean" }, + "metrics": { + "type": "boolean" + }, "oauth": { "type": "boolean" }, @@ -8655,6 +8742,7 @@ "configFile", "facialRecognition", "map", + "metrics", "trash", "reverseGeocoding", "oauth", @@ -9027,6 +9115,9 @@ "map": { "$ref": "#/components/schemas/SystemConfigMapDto" }, + "metrics": { + "$ref": "#/components/schemas/SystemConfigMetricsDto" + }, "newVersionCheck": { "$ref": "#/components/schemas/SystemConfigNewVersionCheckDto" }, @@ -9057,6 +9148,7 @@ "logging", "machineLearning", "map", + "metrics", "newVersionCheck", "oauth", "passwordLogin", @@ -9279,6 +9371,25 @@ ], "type": "object" }, + "SystemConfigMetricsDto": { + "properties": { + "assetCount": { + "$ref": "#/components/schemas/MetricsAssetCountConfig" + }, + "enabled": { + "type": "boolean" + }, + "serverInfo": { + "$ref": "#/components/schemas/MetricServerInfoConfig" + } + }, + "required": [ + "enabled", + "serverInfo", + "assetCount" + ], + "type": "object" + }, "SystemConfigNewVersionCheckDto": { "properties": { "enabled": { diff --git a/server/src/domain/metrics/metrics.dto.ts b/server/src/domain/metrics/metrics.dto.ts index 7f70330a99..80261b12b2 100644 --- a/server/src/domain/metrics/metrics.dto.ts +++ b/server/src/domain/metrics/metrics.dto.ts @@ -1,17 +1,59 @@ -export class MetricsServerInfoDto { - cpuCount?: number; - cpuModel?: string; - memoryCount?: number; - version?: string; +import { IsBoolean } from 'class-validator'; + +// TODO I feel like it must be possible to generate those from MetricsServerInfo and MetricsAssetCount +export class MetricServerInfoConfig { + @IsBoolean() + cpuCount!: boolean; + + @IsBoolean() + cpuModel!: boolean; + + @IsBoolean() + memory!: boolean; + + @IsBoolean() + version!: boolean; } -export class MetricsAssetCountDto { - image?: number; - video?: number; - total?: number; +export class MetricsAssetCountConfig { + @IsBoolean() + image!: boolean; + + @IsBoolean() + video!: boolean; + + @IsBoolean() + total!: boolean; } -export class MetricsDto { - serverInfo!: MetricsServerInfoDto; - assetCount!: MetricsAssetCountDto; +class MetricsServerInfo { + cpuCount!: number; + cpuModel!: string; + memory!: number; + version!: string; +} + +class MetricsAssetCount { + image!: number; + video!: number; + total!: number; +} + +export interface Metrics { + serverInfo: { + cpuCount: number; + cpuModel: string; + memory: number; + version: string; + }; + assetCount: { + image: number; + video: number; + total: number; + }; +} + +export class MetricsDto implements Metrics { + serverInfo!: MetricsServerInfo; + assetCount!: MetricsAssetCount; } diff --git a/server/src/domain/metrics/metrics.service.ts b/server/src/domain/metrics/metrics.service.ts index caef82e8d5..d1f816e873 100644 --- a/server/src/domain/metrics/metrics.service.ts +++ b/server/src/domain/metrics/metrics.service.ts @@ -1,39 +1,71 @@ import { Inject, Injectable } from '@nestjs/common'; +import _ from 'lodash'; import { serverVersion } from '../domain.constant'; import { JobName } from '../job'; +import { ISystemConfigRepository } from '../repositories'; import { IJobRepository } from '../repositories/job.repository'; -import { IMetricsRepository, SharedMetrics } from '../repositories/metrics.repository'; -import { MetricsDto } from './metrics.dto'; +import { IMetricsRepository } from '../repositories/metrics.repository'; +import { FeatureFlag, SystemConfigCore, SystemConfigMetricsDto } from '../system-config'; +import { Metrics, MetricsDto } from './metrics.dto'; @Injectable() export class MetricsService { + private configCore: SystemConfigCore; + constructor( @Inject(IJobRepository) private jobRepository: IJobRepository, @Inject(IMetricsRepository) private repository: IMetricsRepository, - ) {} + @Inject(ISystemConfigRepository) systemConfigRepository: ISystemConfigRepository, + ) { + this.configCore = SystemConfigCore.create(systemConfigRepository); + } async handleQueueMetrics() { - // TODO config for what metrics should be fetched and if any at all - - await this.jobRepository.queue({ name: JobName.METRICS, data: { assetCount: true, serverInfo: true } }); + if (await this.configCore.hasFeature(FeatureFlag.METRICS)) { + await this.jobRepository.queue({ name: JobName.METRICS }); + } } - async handleMetrics(metrics: SharedMetrics) { - const metricsPayload = new MetricsDto(); - if (metrics.serverInfo) { - metricsPayload.serverInfo.version = serverVersion.toString(); - metricsPayload.serverInfo.cpuCount = this.repository.getCpuCount(); - metricsPayload.serverInfo.cpuModel = this.repository.getCpuModel(); - metricsPayload.serverInfo.memoryCount = this.repository.getMemoryCount(); - } + async handleSendMetrics() { + const metricsConfig = await this.configCore.getConfig().then((config) => config.metrics); + const metrics = await this.getMetrics(metricsConfig); - if (metrics.assetCount) { - metricsPayload.assetCount.image = await this.repository.getImageCount(); - metricsPayload.assetCount.video = await this.repository.getVideoCount(); - metricsPayload.assetCount.total = await this.repository.getAssetCount(); - } - - await this.repository.sendMetrics(metricsPayload); + await this.repository.sendMetrics(metrics); return true; } + + async getMetrics(config: SystemConfigMetricsDto) { + const metrics: Metrics = new MetricsDto(); + + metrics.serverInfo = { + cpuCount: this.repository.getCpuCount(), + cpuModel: this.repository.getCpuModel(), + memory: this.repository.getMemory(), + version: serverVersion.toString(), + }; + + metrics.assetCount = { + image: await this.repository.getImageCount(), + video: await this.repository.getVideoCount(), + total: await this.repository.getAssetCount(), + }; + + return _.pick(metrics, this.getKeys(config)); + } + + private getKeys(config: SystemConfigMetricsDto) { + const result = []; + const keys = _.keys(config) as Array; + for (const key of keys) { + const subConfig = _.get(config, key); + if (typeof subConfig === 'boolean') { + continue; + } + + const keys = _.keys(_.pickBy(subConfig)).map((value) => `${key}.${value}`); + result.push(...keys); + } + + return result; + } } diff --git a/server/src/domain/repositories/job.repository.ts b/server/src/domain/repositories/job.repository.ts index e2075eabe7..00ed6a1634 100644 --- a/server/src/domain/repositories/job.repository.ts +++ b/server/src/domain/repositories/job.repository.ts @@ -9,7 +9,6 @@ import { ILibraryRefreshJob, ISidecarWriteJob, } from '../job/job.interface'; -import { SharedMetrics } from './metrics.repository'; export interface JobCounts { active: number; @@ -93,7 +92,7 @@ export type JobItem = | { name: JobName.LIBRARY_QUEUE_CLEANUP; data: IBaseJob } // Metrics - | { name: JobName.METRICS; data: SharedMetrics }; + | { name: JobName.METRICS; data?: IBaseJob }; export type JobHandler = (data: T) => boolean | Promise; export type JobItemHandler = (item: JobItem) => Promise; diff --git a/server/src/domain/repositories/metrics.repository.ts b/server/src/domain/repositories/metrics.repository.ts index 7ea46f059d..b8f29a85da 100644 --- a/server/src/domain/repositories/metrics.repository.ts +++ b/server/src/domain/repositories/metrics.repository.ts @@ -2,17 +2,12 @@ import { MetricsDto } from '../metrics'; export const IMetricsRepository = 'IMetricsRepository'; -export interface SharedMetrics { - serverInfo: boolean; - assetCount: boolean; -} - export interface IMetricsRepository { getAssetCount(): Promise; getCpuCount(): number; getCpuModel(): string; - getMemoryCount(): number; + getMemory(): number; getImageCount(): Promise; getVideoCount(): Promise; - sendMetrics(payload: MetricsDto): Promise; + sendMetrics(payload: Partial): Promise; } diff --git a/server/src/domain/server-info/server-info.dto.ts b/server/src/domain/server-info/server-info.dto.ts index e8c68d559b..4b6f0b607c 100644 --- a/server/src/domain/server-info/server-info.dto.ts +++ b/server/src/domain/server-info/server-info.dto.ts @@ -93,6 +93,7 @@ export class ServerFeaturesDto implements FeatureFlags { configFile!: boolean; facialRecognition!: boolean; map!: boolean; + metrics!: boolean; trash!: boolean; reverseGeocoding!: boolean; oauth!: boolean; diff --git a/server/src/domain/system-config/dto/index.ts b/server/src/domain/system-config/dto/index.ts index 652e34cc50..c8cbcc70c1 100644 --- a/server/src/domain/system-config/dto/index.ts +++ b/server/src/domain/system-config/dto/index.ts @@ -1,5 +1,6 @@ export * from './system-config-ffmpeg.dto'; export * from './system-config-library.dto'; +export * from './system-config-metrics.dto'; export * from './system-config-oauth.dto'; export * from './system-config-password-login.dto'; export * from './system-config-storage-template.dto'; diff --git a/server/src/domain/system-config/dto/system-config-metrics.dto.ts b/server/src/domain/system-config/dto/system-config-metrics.dto.ts new file mode 100644 index 0000000000..f011343cab --- /dev/null +++ b/server/src/domain/system-config/dto/system-config-metrics.dto.ts @@ -0,0 +1,18 @@ +import { MetricServerInfoConfig, MetricsAssetCountConfig } from '@app/domain'; +import { Type } from 'class-transformer'; +import { IsBoolean, IsObject, ValidateNested } from 'class-validator'; + +export class SystemConfigMetricsDto { + @IsBoolean() + enabled!: boolean; + + @Type(() => MetricServerInfoConfig) + @ValidateNested() + @IsObject() + serverInfo!: MetricServerInfoConfig; + + @Type(() => MetricsAssetCountConfig) + @ValidateNested() + @IsObject() + assetCount!: MetricsAssetCountConfig; +} diff --git a/server/src/domain/system-config/dto/system-config.dto.ts b/server/src/domain/system-config/dto/system-config.dto.ts index 6fbfeced2b..871e26c0b1 100644 --- a/server/src/domain/system-config/dto/system-config.dto.ts +++ b/server/src/domain/system-config/dto/system-config.dto.ts @@ -1,6 +1,7 @@ import { SystemConfig } from '@app/infra/entities'; import { Type } from 'class-transformer'; import { IsObject, ValidateNested } from 'class-validator'; +import { SystemConfigMetricsDto } from '.'; import { SystemConfigFFmpegDto } from './system-config-ffmpeg.dto'; import { SystemConfigJobDto } from './system-config-job.dto'; import { SystemConfigLibraryDto } from './system-config-library.dto'; @@ -37,6 +38,11 @@ export class SystemConfigDto implements SystemConfig { @IsObject() map!: SystemConfigMapDto; + @Type(() => SystemConfigMetricsDto) + @ValidateNested() + @IsObject() + metrics!: SystemConfigMetricsDto; + @Type(() => SystemConfigNewVersionCheckDto) @ValidateNested() @IsObject() diff --git a/server/src/domain/system-config/system-config.core.ts b/server/src/domain/system-config/system-config.core.ts index e110c76d0d..02a4fb73ed 100644 --- a/server/src/domain/system-config/system-config.core.ts +++ b/server/src/domain/system-config/system-config.core.ts @@ -82,6 +82,20 @@ export const defaults = Object.freeze({ lightStyle: '', darkStyle: '', }, + metrics: { + enabled: false, + serverInfo: { + cpuCount: true, + cpuModel: true, + memory: true, + version: true, + }, + assetCount: { + image: true, + video: true, + total: true, + }, + }, reverseGeocoding: { enabled: true, }, @@ -132,6 +146,7 @@ export enum FeatureFlag { CLIP_ENCODE = 'clipEncode', FACIAL_RECOGNITION = 'facialRecognition', MAP = 'map', + METRICS = 'metrics', REVERSE_GEOCODING = 'reverseGeocoding', SIDECAR = 'sidecar', SEARCH = 'search', @@ -204,6 +219,7 @@ export class SystemConfigCore { [FeatureFlag.CLIP_ENCODE]: mlEnabled && config.machineLearning.clip.enabled, [FeatureFlag.FACIAL_RECOGNITION]: mlEnabled && config.machineLearning.facialRecognition.enabled, [FeatureFlag.MAP]: config.map.enabled, + [FeatureFlag.METRICS]: config.metrics.enabled, [FeatureFlag.REVERSE_GEOCODING]: config.reverseGeocoding.enabled, [FeatureFlag.SIDECAR]: true, [FeatureFlag.SEARCH]: true, diff --git a/server/src/immich/app.module.ts b/server/src/immich/app.module.ts index 3694626f26..df12ce580f 100644 --- a/server/src/immich/app.module.ts +++ b/server/src/immich/app.module.ts @@ -22,6 +22,7 @@ import { FaceController, JobController, LibraryController, + MetricsController, OAuthController, PartnerController, PersonController, @@ -54,6 +55,7 @@ import { ErrorInterceptor, FileUploadInterceptor } from './interceptors'; FaceController, JobController, LibraryController, + MetricsController, OAuthController, PartnerController, SearchController, diff --git a/server/src/immich/controllers/index.ts b/server/src/immich/controllers/index.ts index c177144c3d..75b2afdcd9 100644 --- a/server/src/immich/controllers/index.ts +++ b/server/src/immich/controllers/index.ts @@ -8,6 +8,7 @@ export * from './auth.controller'; export * from './face.controller'; export * from './job.controller'; export * from './library.controller'; +export * from './metrics.controller'; export * from './oauth.controller'; export * from './partner.controller'; export * from './person.controller'; diff --git a/server/src/immich/controllers/metrics.controller.ts b/server/src/immich/controllers/metrics.controller.ts new file mode 100644 index 0000000000..96ade28f47 --- /dev/null +++ b/server/src/immich/controllers/metrics.controller.ts @@ -0,0 +1,18 @@ +import { Metrics, MetricsService, SystemConfigMetricsDto } from '@app/domain'; +import { Body, Controller, Put } from '@nestjs/common'; +import { ApiTags } from '@nestjs/swagger'; +import { Authenticated } from '../app.guard'; +import { UseValidation } from '../app.utils'; + +@ApiTags('Metrics') +@Controller('metrics') +@Authenticated() +@UseValidation() +export class MetricsController { + constructor(private service: MetricsService) {} + + @Put() + getMetrics(@Body() dto: SystemConfigMetricsDto): Promise> { + return this.service.getMetrics(dto); + } +} diff --git a/server/src/infra/entities/system-config.entity.ts b/server/src/infra/entities/system-config.entity.ts index 16b00c9606..5898e18a36 100644 --- a/server/src/infra/entities/system-config.entity.ts +++ b/server/src/infra/entities/system-config.entity.ts @@ -66,6 +66,15 @@ export enum SystemConfigKey { MAP_LIGHT_STYLE = 'map.lightStyle', MAP_DARK_STYLE = 'map.darkStyle', + METRICS_ENABLED = 'metrics.enabled', + METRICS_SERVER_INFO_CPU_COUNT = 'metrics.serverInfo.cpuCount', + METRICS_SERVER_INFO_CPU_MODEL = 'metrics.serverInfo.cpuModel', + METRICS_SERVER_INFO_MEMORY = 'metrics.serverInfo.memory', + METRICS_SERVER_INFO_VERSION = 'metrics.serverInfo.version', + METRICS_ASSET_COUNT_IMAGE = 'metrics.assetCount.image', + METRICS_ASSET_COUNT_VIDEO = 'metrics.assetCount.video', + METRICS_ASSET_COUNT_TOTAL = 'metrics.assetCount.total', + REVERSE_GEOCODING_ENABLED = 'reverseGeocoding.enabled', NEW_VERSION_CHECK_ENABLED = 'newVersionCheck.enabled', @@ -196,6 +205,20 @@ export interface SystemConfig { lightStyle: string; darkStyle: string; }; + metrics: { + enabled: boolean; + serverInfo: { + cpuCount: boolean; + cpuModel: boolean; + memory: boolean; + version: boolean; + }; + assetCount: { + image: boolean; + video: boolean; + total: boolean; + }; + }; reverseGeocoding: { enabled: boolean; }; diff --git a/server/src/infra/repositories/metrics.repository.ts b/server/src/infra/repositories/metrics.repository.ts index 8081945e00..37b8cef4e7 100644 --- a/server/src/infra/repositories/metrics.repository.ts +++ b/server/src/infra/repositories/metrics.repository.ts @@ -10,7 +10,7 @@ import { AssetEntity, AssetType } from '../entities'; @Injectable() export class MetricsRepository implements IMetricsRepository { constructor(@InjectRepository(AssetEntity) private assetRepository: Repository) {} - async sendMetrics(payload: MetricsDto): Promise { + async sendMetrics(payload: Partial): Promise { await axios.post('IMMICH-DATA-DOMAIN', payload); } @@ -26,7 +26,7 @@ export class MetricsRepository implements IMetricsRepository { return os.cpus()[0].model; } - getMemoryCount() { + getMemory() { return os.totalmem(); } diff --git a/server/src/microservices/app.service.ts b/server/src/microservices/app.service.ts index 29c3e7ce46..4006b0c1f3 100644 --- a/server/src/microservices/app.service.ts +++ b/server/src/microservices/app.service.ts @@ -62,7 +62,7 @@ export class AppService { [JobName.VIDEO_CONVERSION]: (data) => this.mediaService.handleVideoConversion(data), [JobName.QUEUE_METADATA_EXTRACTION]: (data) => this.metadataService.handleQueueMetadataExtraction(data), [JobName.METADATA_EXTRACTION]: (data) => this.metadataService.handleMetadataExtraction(data), - [JobName.METRICS]: (data) => this.metricsService.handleMetrics(data), + [JobName.METRICS]: () => this.metricsService.handleSendMetrics(), [JobName.LINK_LIVE_PHOTOS]: (data) => this.metadataService.handleLivePhotoLinking(data), [JobName.QUEUE_RECOGNIZE_FACES]: (data) => this.personService.handleQueueRecognizeFaces(data), [JobName.RECOGNIZE_FACES]: (data) => this.personService.handleRecognizeFaces(data), diff --git a/web/src/lib/components/admin-page/settings/metrics-settings/metrics-settings.svelte b/web/src/lib/components/admin-page/settings/metrics-settings/metrics-settings.svelte new file mode 100644 index 0000000000..ec04ad0b73 --- /dev/null +++ b/web/src/lib/components/admin-page/settings/metrics-settings/metrics-settings.svelte @@ -0,0 +1,168 @@ + + +
+ {#await refreshConfig() then} +
+
+
+ + + +
+ + + + + + + +
+ + +
+ + + + + +
+ + {#if config.enabled} + {#await sharedMetrics} + + {:then metrics} +
+
{JSON.stringify(metrics, null, 2)}
+
+ {/await} + {/if} + + handleReset(detail)} + on:save={saveSetting} + showResetToDefault={!isEqual(savedConfig, defaultConfig)} + {disabled} + /> +
+
+
+ {/await} +
diff --git a/web/src/routes/admin/system-settings/+page.svelte b/web/src/routes/admin/system-settings/+page.svelte index 2a6fce5aec..11ca0fd283 100644 --- a/web/src/routes/admin/system-settings/+page.svelte +++ b/web/src/routes/admin/system-settings/+page.svelte @@ -4,6 +4,7 @@ import JobSettings from '$lib/components/admin-page/settings/job-settings/job-settings.svelte'; import MachineLearningSettings from '$lib/components/admin-page/settings/machine-learning-settings/machine-learning-settings.svelte'; import MapSettings from '$lib/components/admin-page/settings/map-settings/map-settings.svelte'; + import MetricsSettings from '$lib/components/admin-page/settings/metrics-settings/metrics-settings.svelte'; import OAuthSettings from '$lib/components/admin-page/settings/oauth/oauth-settings.svelte'; import PasswordLoginSettings from '$lib/components/admin-page/settings/password-login/password-login-settings.svelte'; import SettingAccordion from '$lib/components/admin-page/settings/setting-accordion.svelte'; @@ -87,6 +88,13 @@ + + + +