feat: user upload heatmap (#28593)

* Feat - Heatmap

* Implemented Comments to prettify and code cleanup

* fixing code to pass cases.

* fixing errors for OpenAPI Clients

* Improving the code.

* Fix code

* Rerun generated client check

* Rerun generated client

* feat: command for user pages (#28554)

* fix(web): timeline stuttering with many assets in 1 day (#28509)

* fix(web): timeline stuttering with many assets in 1 day

* cache isInOrNearViewport per day

* skip inOrNearViewport check on first run

* chore(ml): allow insightface 1.x (#28595)

* chore(ml): allow insightface 1.x

The new insightface 1.0 release appears to have no breaking code changes nor relevant license changes ([before](https://github.com/deepinsight/insightface/blob/2a78baec428354883e0cda39c54b555a5ed8358a/README.md), [after](https://github.com/deepinsight/insightface/blob/70f3269ea628d0658c5723976944c9de414e96f8/README.md), c.f. https://github.com/immich-app/immich/blob/fd7ddfef54cdf2b6256c4fc08bc5ff3f86176775/machine-learning/README.md), and it works on my machine.

* Update uv.lock

* please excuse my incompetence

* Triggering the actions.

* bad merge

* Fix code

* Code clear

* Resolve conflict

* Resolve conflict

* Resolve conflict

* Resolve errors

* Resolve errors

* Resolve errors more

* chore: clean up

---------

Co-authored-by: Alex <alex.tran1502@gmail.com>
Co-authored-by: Ben Beckford <ben@benjaminbeckford.com>
Co-authored-by: Aaron Liu <aaronliu0130@gmail.com>
Co-authored-by: Jason Rasmussen <jason@rasm.me>

server/src/controllers/user-admin.controller.ts

@@ -3,6 +3,7 @@ import { ApiTags } from '@nestjs/swagger';
 import { Endpoint, HistoryBuilder } from 'src/decorators';
 import { AssetStatsDto, AssetStatsResponseDto } from 'src/dtos/asset.dto';
 import { AuthDto } from 'src/dtos/auth.dto';
+import { CalendarHeatmapDto, CalendarHeatmapResponseDto } from 'src/dtos/calendar-heatmap.dto';
 import { SessionResponseDto } from 'src/dtos/session.dto';
 import { UserPreferencesResponseDto, UserPreferencesUpdateDto } from 'src/dtos/user-preferences.dto';
 import {
@@ -85,6 +86,21 @@ export class UserAdminController {
     return this.service.delete(auth, id, dto);
   }
 
+  @Get(':id/calendar-heatmap')
+  @Authenticated({ permission: Permission.UserRead })
+  @Endpoint({
+    summary: 'Retrieve calendar heatmap activity',
+    description: 'Retrieve activity counts for a specified period, in a calendar heatmap format.',
+    history: new HistoryBuilder().added('v3').stable('v3'),
+  })
+  getUserCalendarHeatmapAdmin(
+    @Auth() auth: AuthDto,
+    @Param() { id }: UUIDParamDto,
+    @Query() dto: CalendarHeatmapDto,
+  ): Promise<CalendarHeatmapResponseDto> {
+    return this.service.getCalendarHeatmap(auth, id, dto);
+  }
+
   @Get(':id/sessions')
   @Authenticated({ permission: Permission.AdminSessionRead, admin: true })
   @Endpoint({

server/src/controllers/user.controller.ts

@@ -9,6 +9,7 @@ import {
   Param,
   Post,
   Put,
+  Query,
   Res,
   UploadedFile,
   UseInterceptors,
@@ -17,6 +18,7 @@ import { ApiBody, ApiConsumes, ApiTags } from '@nestjs/swagger';
 import { NextFunction, Response } from 'express';
 import { Endpoint, HistoryBuilder } from 'src/decorators';
 import { AuthDto } from 'src/dtos/auth.dto';
+import { CalendarHeatmapDto, CalendarHeatmapResponseDto } from 'src/dtos/calendar-heatmap.dto';
 import { LicenseKeyDto, LicenseResponseDto } from 'src/dtos/license.dto';
 import { OnboardingDto, OnboardingResponseDto } from 'src/dtos/onboarding.dto';
 import { UserPreferencesResponseDto, UserPreferencesUpdateDto } from 'src/dtos/user-preferences.dto';
@@ -60,6 +62,17 @@ export class UserController {
     return this.service.getMe(auth);
   }
 
+  @Get('me/calendar-heatmap')
+  @Authenticated({ permission: Permission.UserRead })
+  @Endpoint({
+    summary: 'Retrieve calendar heatmap activity',
+    description: 'Retrieve activity counts for a specified period, in a calendar heatmap format.',
+    history: new HistoryBuilder().added('v3').stable('v3'),
+  })
+  getMyCalendarHeatmap(@Auth() auth: AuthDto, @Query() dto: CalendarHeatmapDto): Promise<CalendarHeatmapResponseDto> {
+    return this.service.getCalendarHeatmap(auth, dto);
+  }
+
   @Put('me')
   @Authenticated({ permission: Permission.UserUpdate })
   @Endpoint({

server/src/dtos/calendar-heatmap.dto.ts

@@ -0,0 +1,36 @@
+import { createZodDto } from 'nestjs-zod';
+import { CalendarHeatmapType } from 'src/enum';
+import { isoDateToDate } from 'src/validation';
+import z from 'zod';
+
+const CalendarHeatmapTypeSchema = z
+  .enum(CalendarHeatmapType)
+  .describe('Type of calendar heatmap')
+  .meta({ id: 'CalendarHeatmapType' });
+
+const CalendarHeatmapSchema = z
+  .object({
+    from: isoDateToDate.optional().describe('Start date in UTC'),
+    to: isoDateToDate.optional().describe('End date in UTC'),
+    type: CalendarHeatmapTypeSchema.optional().default(CalendarHeatmapType.Upload),
+  })
+  .refine((dto) => !dto.from || !dto.to || dto.from <= dto.to, { message: 'from must be before to', path: ['from'] })
+  .meta({ id: 'CalendarHeatmapDto' });
+
+export class CalendarHeatmapDto extends createZodDto(CalendarHeatmapSchema) {}
+
+const CalendarHeatmapResponseSchema = z
+  .object({
+    from: z.string().describe('Start date in UTC').meta({ example: '2024-01-01' }),
+    to: z.string().describe('End date in UTC').meta({ example: '2024-12-31' }),
+    series: z.array(
+      z.object({
+        date: z.string().describe('Date in UTC').meta({ example: '2024-01-01' }),
+        count: z.int().nonnegative().describe('Activity count'),
+      }),
+    ),
+    totalCount: z.int().nonnegative().describe('Total activity count over the period'),
+  })
+  .meta({ id: 'CalendarHeatmapResponseDto' });
+
+export class CalendarHeatmapResponseDto extends createZodDto(CalendarHeatmapResponseSchema) {}

server/src/enum.ts

@@ -1175,3 +1175,8 @@ export enum WorkflowType {
 }
 
 export const WorkflowTypeSchema = z.enum(WorkflowType).describe('Workflow type').meta({ id: 'WorkflowType' });
+
+export enum CalendarHeatmapType {
+  Upload = 'Upload',
+  Taken = 'Taken',
+}

server/src/queries/asset.repository.sql

@@ -339,6 +339,22 @@ where
 limit
   $3
 
+-- AssetRepository.getCalendarHeatmap
+select
+  date_trunc('DAY', "asset"."createdAt" AT TIME ZONE 'UTC') AT TIME ZONE 'UTC' as "date",
+  count(*) as "count"
+from
+  "asset"
+where
+  "ownerId" = $1::uuid
+  and "createdAt" >= $2
+  and "createdAt" < $3
+  and "deletedAt" is null
+group by
+  date_trunc('DAY', "asset"."createdAt" AT TIME ZONE 'UTC') AT TIME ZONE 'UTC'
+order by
+  "date" asc
+
 -- AssetRepository.getTimeBuckets
 with
   "asset" as (

server/src/repositories/asset.repository.ts

@@ -17,7 +17,15 @@ import { InjectKysely } from 'nestjs-kysely';
 import { LockableProperty, Stack } from 'src/database';
 import { Chunked, ChunkedArray, DummyValue, GenerateSql } from 'src/decorators';
 import { AuthDto } from 'src/dtos/auth.dto';
-import { AssetFileType, AssetOrder, AssetOrderBy, AssetStatus, AssetType, AssetVisibility } from 'src/enum';
+import {
+  AssetFileType,
+  AssetOrder,
+  AssetOrderBy,
+  AssetStatus,
+  AssetType,
+  AssetVisibility,
+  CalendarHeatmapType,
+} from 'src/enum';
 import { DB } from 'src/schema';
 import { AssetAudioTable, AssetKeyframeTable, AssetVideoTable } from 'src/schema/tables/asset-av.table';
 import { AssetExifTable } from 'src/schema/tables/asset-exif.table';
@@ -706,6 +714,32 @@ export class AssetRepository {
       .executeTakeFirstOrThrow();
   }
 
+  @GenerateSql({
+    params: [DummyValue.UUID, { from: DummyValue.DATE, to: DummyValue.DATE, type: CalendarHeatmapType.Upload }],
+  })
+  getCalendarHeatmap(ownerId: string, dto: { from: Date; to: Date; type: CalendarHeatmapType }) {
+    const dateColumns: Record<CalendarHeatmapType, { order: AssetOrderBy; column: 'createdAt' | 'localDateTime' }> = {
+      [CalendarHeatmapType.Upload]: { order: AssetOrderBy.CreatedAt, column: 'createdAt' },
+      [CalendarHeatmapType.Taken]: { order: AssetOrderBy.TakenAt, column: 'localDateTime' },
+    } as const;
+
+    const { order, column } = dateColumns[dto.type];
+
+    const date = truncatedDate<Date>(order, 'DAY');
+
+    return this.db
+      .selectFrom('asset')
+      .select(date.as('date'))
+      .select((eb) => eb.fn.countAll<number>().as('count'))
+      .where('ownerId', '=', asUuid(ownerId))
+      .where(column, '>=', dto.from)
+      .where(column, '<', dto.to)
+      .where('deletedAt', 'is', null)
+      .groupBy(date)
+      .orderBy('date', 'asc')
+      .execute();
+  }
+
   @GenerateSql({ params: [{}] })
   async getTimeBuckets(options: TimeBucketOptions): Promise<TimeBucketItem[]> {
     return this.db

server/src/services/shared/user-methods.ts

@@ -0,0 +1,36 @@
+import { DateTime } from 'luxon';
+import { CalendarHeatmapDto } from 'src/dtos/calendar-heatmap.dto';
+import { CalendarHeatmapType } from 'src/enum';
+import { AssetRepository } from 'src/repositories/asset.repository';
+import { asDateString } from 'src/utils/date';
+
+export const getCalendarHeatmap = async (
+  userId: string,
+  dto: CalendarHeatmapDto,
+  repos: { asset: AssetRepository },
+) => {
+  const toDate = DateTime.fromJSDate(dto.to ?? new Date(), { zone: 'utc' }).startOf('day');
+  const fromDate = (
+    dto.from ? DateTime.fromJSDate(dto.from, { zone: 'utc' }) : toDate.minus({ weeks: 52 }).plus({ days: 1 })
+  ).startOf('day');
+
+  const counts = await repos.asset.getCalendarHeatmap(userId, {
+    from: fromDate.toJSDate(),
+    to: toDate.plus({ days: 1 }).toJSDate(),
+    type: dto.type ?? CalendarHeatmapType.Upload,
+  });
+  const countsMap = new Map(counts.map((item) => [asDateString(item.date)!, item.count]));
+
+  const series: Array<{ date: string; count: number }> = [];
+  for (let date = fromDate; date <= toDate; date = date.plus({ days: 1 })) {
+    const key = date.toISODate()!;
+    series.push({ date: key, count: countsMap.get(key) ?? 0 });
+  }
+
+  return {
+    from: fromDate.toISODate()!,
+    to: toDate.toISODate()!,
+    series,
+    totalCount: series.reduce((totalCount, item) => totalCount + item.count, 0),
+  };
+};

server/src/services/user-admin.service.ts

@@ -2,6 +2,7 @@ import { BadRequestException, ForbiddenException, Injectable } from '@nestjs/com
 import { SALT_ROUNDS } from 'src/constants';
 import { AssetStatsDto, AssetStatsResponseDto, mapStats } from 'src/dtos/asset.dto';
 import { AuthDto } from 'src/dtos/auth.dto';
+import { CalendarHeatmapDto, CalendarHeatmapResponseDto } from 'src/dtos/calendar-heatmap.dto';
 import { SessionResponseDto, mapSession } from 'src/dtos/session.dto';
 import { UserPreferencesResponseDto, UserPreferencesUpdateDto, mapPreferences } from 'src/dtos/user-preferences.dto';
 import {
@@ -15,6 +16,7 @@ import {
 import { JobName, UserMetadataKey, UserStatus } from 'src/enum';
 import { UserFindOptions } from 'src/repositories/user.repository';
 import { BaseService } from 'src/services/base.service';
+import { getCalendarHeatmap } from 'src/services/shared/user-methods';
 import { getPreferences, getPreferencesPartial, mergePreferences } from 'src/utils/preferences';
 
 @Injectable()
@@ -122,6 +124,11 @@ export class UserAdminService extends BaseService {
     return mapUserAdmin(user);
   }
 
+  async getCalendarHeatmap(auth: AuthDto, id: string, dto: CalendarHeatmapDto): Promise<CalendarHeatmapResponseDto> {
+    await this.findOrFail(id, { withDeleted: false });
+    return getCalendarHeatmap(id, dto, { asset: this.assetRepository });
+  }
+
   async getSessions(auth: AuthDto, id: string): Promise<SessionResponseDto[]> {
     const sessions = await this.sessionRepository.getByUserId(id);
     return sessions.map((session) => mapSession(session));

server/src/services/user.service.ts

@@ -5,6 +5,7 @@ import { SALT_ROUNDS } from 'src/constants';
 import { StorageCore } from 'src/cores/storage.core';
 import { OnEvent, OnJob } from 'src/decorators';
 import { AuthDto } from 'src/dtos/auth.dto';
+import { CalendarHeatmapDto, CalendarHeatmapResponseDto } from 'src/dtos/calendar-heatmap.dto';
 import { LicenseKeyDto, LicenseResponseDto } from 'src/dtos/license.dto';
 import { OnboardingDto, OnboardingResponseDto } from 'src/dtos/onboarding.dto';
 import { UserPreferencesResponseDto, UserPreferencesUpdateDto, mapPreferences } from 'src/dtos/user-preferences.dto';
@@ -15,6 +16,7 @@ import { ArgOf } from 'src/repositories/event.repository';
 import { UserFindOptions } from 'src/repositories/user.repository';
 import { UserTable } from 'src/schema/tables/user.table';
 import { BaseService } from 'src/services/base.service';
+import { getCalendarHeatmap } from 'src/services/shared/user-methods';
 import { JobOf, UserMetadataItem } from 'src/types';
 import { ImmichFileResponse } from 'src/utils/file';
 import { mimeTypes } from 'src/utils/mime-types';
@@ -46,6 +48,10 @@ export class UserService extends BaseService {
     return mapUserAdmin(user);
   }
 
+  getCalendarHeatmap(auth: AuthDto, dto: CalendarHeatmapDto): Promise<CalendarHeatmapResponseDto> {
+    return getCalendarHeatmap(auth.user.id, dto, { asset: this.assetRepository });
+  }
+
   async updateMe({ user }: AuthDto, dto: UserUpdateMeDto): Promise<UserAdminResponseDto> {
     if (dto.email) {
       const duplicate = await this.userRepository.getByEmail(dto.email);

server/src/utils/database.ts

@@ -301,8 +301,8 @@ export function withTags(eb: ExpressionBuilder<DB, 'asset'>) {
   ).as('tags');
 }
 
-export function truncatedDate<O>(order: AssetOrderBy = AssetOrderBy.TakenAt) {
-  return sql<O>`date_trunc(${sql.lit('MONTH')}, ${sql.ref(order === AssetOrderBy.CreatedAt ? 'asset.createdAt' : 'localDateTime')} AT TIME ZONE 'UTC') AT TIME ZONE 'UTC'`;
+export function truncatedDate<O>(order: AssetOrderBy = AssetOrderBy.TakenAt, size?: 'DAY' | 'MONTH') {
+  return sql<O>`date_trunc(${sql.lit(size ?? 'MONTH')}, ${sql.ref(order === AssetOrderBy.CreatedAt ? 'asset.createdAt' : 'localDateTime')} AT TIME ZONE 'UTC') AT TIME ZONE 'UTC'`;
 }
 
 export function withTagId<O>(qb: SelectQueryBuilder<DB, 'asset', O>, tagId: string) {