Added machine learning microservice and object detection (#76)

Makefile

@@ -1,8 +1,8 @@
 dev:
-	docker-compose -f ./docker/docker-compose.yml up
+	docker-compose -f ./docker/docker-compose.yml up --remove-orphans
 
 dev-update:
-	docker-compose -f ./docker/docker-compose.yml up --build -V 
+	docker-compose -f ./docker/docker-compose.yml up --build -V  --remove-orphans
 
 dev-scale:
 	docker-compose -f ./docker/docker-compose.yml up --build -V  --scale immich_server=3 --remove-orphans 

docker/docker-compose.gpu.yml

@@ -22,6 +22,34 @@ services:
     networks:
       - immich_network
 
+  immich_microservices:
+    image: immich-microservices-dev:1.3.2
+    build:
+      context: ../microservices
+      target: development
+      dockerfile: ../microservices/Dockerfile
+    command: npm run start:dev
+    deploy:
+      resources:
+        reservations:
+          devices:
+            - driver: nvidia
+              count: 1
+              capabilities: [ gpu ]
+    expose:
+      - "3001"
+    volumes:
+      - ../microservices:/usr/src/app
+      - ${UPLOAD_LOCATION}:/usr/src/app/upload
+      - /usr/src/app/node_modules
+    env_file:
+      - .env
+    depends_on:
+      - database
+      - immich_server
+    networks:
+      - immich_network
+
   redis:
     container_name: immich_redis
     image: redis:6.2
@@ -60,35 +88,6 @@ services:
     depends_on:
       - immich_server
 
-  immich_tf_fastapi:
-    container_name: immich_tf_fastapi
-    image: tensor_flow_fastapi:1.0.0
-    restart: always
-    command: uvicorn app.main:app --proxy-headers --host 0.0.0.0 --port 8000 --reload
-    build:
-      context: ../machine_learning
-      target: gpu
-      dockerfile: ../machine_learning/Dockerfile
-    deploy:
-      resources:
-        reservations:
-          devices:
-            - driver: nvidia
-              count: 1
-              capabilities: [gpu]
-    volumes:
-      - ../machine_learning/app:/code/app
-      - ${UPLOAD_LOCATION}:/code/app/upload
-    ports:
-      - 2285:8000
-    expose:
-      - "8000"
-    depends_on:
-      - database
-    networks:
-      - immich_network
-
-      
 networks:
   immich_network:
 volumes:

docker/docker-compose.yml

@@ -23,6 +23,27 @@ services:
     networks:
       - immich_network
 
+  immich_microservices:
+    image: immich-microservices-dev:1.3.2
+    build:
+      context: ../microservices
+      target: development
+      dockerfile: ../microservices/Dockerfile
+    command: npm run start:dev
+    expose:
+      - "3001"
+    volumes:
+      - ../microservices:/usr/src/app
+      - ${UPLOAD_LOCATION}:/usr/src/app/upload
+      - /usr/src/app/node_modules
+    env_file:
+      - .env
+    depends_on:
+      - database
+    networks:
+      - immich_network
+
+
   redis:
     container_name: immich_redis
     image: redis:6.2
@@ -61,26 +82,26 @@ services:
     depends_on:
       - immich_server
 
-  immich_tf_fastapi:
+  # immich_tf_fastapi:
-    container_name: immich_tf_fastapi
+  #   container_name: immich_tf_fastapi
-    image: tensor_flow_fastapi:1.0.0
+  #   image: tensor_flow_fastapi:1.0.0
-    restart: always
+  #   restart: always
-    command: uvicorn app.main:app --proxy-headers --host 0.0.0.0 --port 8000 --reload
+  #   command: uvicorn app.main:app --proxy-headers --host 0.0.0.0 --port 8000 --reload
-    build:
+  #   build:
-      context: ../machine_learning
+  #     context: ../machine_learning
-      target: cpu
+  #     target: cpu
-      dockerfile: ../machine_learning/Dockerfile
+  #     dockerfile: ../machine_learning/Dockerfile
-    volumes:
+  #   volumes:
-      - ../machine_learning/app:/code/app
+  #     - ../machine_learning/app:/code/app
-      - ${UPLOAD_LOCATION}:/code/app/upload
+  #     - ${UPLOAD_LOCATION}:/code/app/upload
-    ports:
+  #   ports:
-      - 2285:8000
+  #     - 2285:8000
-    expose:
+  #   expose:
-      - "8000"
+  #     - "8000"
-    depends_on:
+  #   depends_on:
-      - database
+  #     - database
-    networks:
+  #   networks:
-      - immich_network
+  #     - immich_network
 
 networks:
   immich_network:

docker/settings/nginx-conf/nginx.conf

@@ -13,7 +13,7 @@ server {
   client_max_body_size 50000M;
 
   listen 80;
-
+  access_log off;
   location / {
     proxy_buffering off;
     proxy_buffer_size 16k;

microservices/.dockerignore

@@ -0,0 +1,4 @@
+node_modules/
+upload/
+dist/
+

microservices/Dockerfile

@@ -0,0 +1,43 @@
+##################################
+# DEVELOPMENT
+##################################
+FROM node:16-bullseye-slim AS development
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+WORKDIR /usr/src/app
+
+COPY package.json package-lock.json ./
+
+RUN apt-get update
+RUN apt-get install gcc g++ make cmake python3 python3-pip ffmpeg -y
+
+RUN npm install
+
+COPY . .
+
+RUN npm run build
+
+#################################
+# PRODUCTION
+#################################
+FROM node:16-bullseye-slim AS production
+
+ARG DEBIAN_FRONTEND=noninteractive
+ARG NODE_ENV=production
+ENV NODE_ENV=${NODE_ENV}
+
+WORKDIR /usr/src/app
+
+COPY package.json package-lock.json ./
+
+RUN apt-get update
+RUN apt-get install gcc g++ make cmake python3 python3-pip ffmpeg -y
+
+RUN npm install --only=production
+
+COPY . .
+
+COPY --from=development /usr/src/app/dist ./dist
+
+CMD ["node", "dist/main"]

microservices/README.md

@@ -1,73 +1,4 @@
-<p align="center">
-  <a href="http://nestjs.com/" target="blank"><img src="https://nestjs.com/img/logo_text.svg" width="320" alt="Nest Logo" /></a>
-</p>
 
-[circleci-image]: https://img.shields.io/circleci/build/github/nestjs/nest/master?token=abc123def456
+# Microservices for Immich
-[circleci-url]: https://circleci.com/gh/nestjs/nest
 
-  <p align="center">A progressive <a href="http://nodejs.org" target="_blank">Node.js</a> framework for building efficient and scalable server-side applications.</p>
+## Image Classifier
-    <p align="center">
-<a href="https://www.npmjs.com/~nestjscore" target="_blank"><img src="https://img.shields.io/npm/v/@nestjs/core.svg" alt="NPM Version" /></a>
-<a href="https://www.npmjs.com/~nestjscore" target="_blank"><img src="https://img.shields.io/npm/l/@nestjs/core.svg" alt="Package License" /></a>
-<a href="https://www.npmjs.com/~nestjscore" target="_blank"><img src="https://img.shields.io/npm/dm/@nestjs/common.svg" alt="NPM Downloads" /></a>
-<a href="https://circleci.com/gh/nestjs/nest" target="_blank"><img src="https://img.shields.io/circleci/build/github/nestjs/nest/master" alt="CircleCI" /></a>
-<a href="https://coveralls.io/github/nestjs/nest?branch=master" target="_blank"><img src="https://coveralls.io/repos/github/nestjs/nest/badge.svg?branch=master#9" alt="Coverage" /></a>
-<a href="https://discord.gg/G7Qnnhy" target="_blank"><img src="https://img.shields.io/badge/discord-online-brightgreen.svg" alt="Discord"/></a>
-<a href="https://opencollective.com/nest#backer" target="_blank"><img src="https://opencollective.com/nest/backers/badge.svg" alt="Backers on Open Collective" /></a>
-<a href="https://opencollective.com/nest#sponsor" target="_blank"><img src="https://opencollective.com/nest/sponsors/badge.svg" alt="Sponsors on Open Collective" /></a>
-  <a href="https://paypal.me/kamilmysliwiec" target="_blank"><img src="https://img.shields.io/badge/Donate-PayPal-ff3f59.svg"/></a>
-    <a href="https://opencollective.com/nest#sponsor"  target="_blank"><img src="https://img.shields.io/badge/Support%20us-Open%20Collective-41B883.svg" alt="Support us"></a>
-  <a href="https://twitter.com/nestframework" target="_blank"><img src="https://img.shields.io/twitter/follow/nestframework.svg?style=social&label=Follow"></a>
-</p>
-  <!--[![Backers on Open Collective](https://opencollective.com/nest/backers/badge.svg)](https://opencollective.com/nest#backer)
-  [![Sponsors on Open Collective](https://opencollective.com/nest/sponsors/badge.svg)](https://opencollective.com/nest#sponsor)-->
-
-## Description
-
-[Nest](https://github.com/nestjs/nest) framework TypeScript starter repository.
-
-## Installation
-
-```bash
-$ npm install
-```
-
-## Running the app
-
-```bash
-# development
-$ npm run start
-
-# watch mode
-$ npm run start:dev
-
-# production mode
-$ npm run start:prod
-```
-
-## Test
-
-```bash
-# unit tests
-$ npm run test
-
-# e2e tests
-$ npm run test:e2e
-
-# test coverage
-$ npm run test:cov
-```
-
-## Support
-
-Nest is an MIT-licensed open source project. It can grow thanks to the sponsors and support by the amazing backers. If you'd like to join them, please [read more here](https://docs.nestjs.com/support).
-
-## Stay in touch
-
-- Author - [Kamil Myśliwiec](https://kamilmysliwiec.com)
-- Website - [https://nestjs.com](https://nestjs.com/)
-- Twitter - [@nestframework](https://twitter.com/nestframework)
-
-## License
-
-Nest is [MIT licensed](LICENSE).

microservices/package.json

@@ -23,13 +23,25 @@
   "dependencies": {
     "@nestjs/common": "^8.0.0",
     "@nestjs/core": "^8.0.0",
+    "@nestjs/mapped-types": "^1.0.1",
     "@nestjs/platform-express": "^8.0.0",
+    "@nestjs/typeorm": "^8.0.3",
+    "@tensorflow-models/coco-ssd": "^2.2.2",
+    "@tensorflow-models/mobilenet": "^2.1.0",
+    "@tensorflow/tfjs": "^3.15.0",
+    "@tensorflow/tfjs-converter": "^3.15.0",
+    "@tensorflow/tfjs-core": "^3.15.0",
+    "@tensorflow/tfjs-node": "^3.15.0",
+    "@tensorflow/tfjs-node-gpu": "^3.15.0",
+    "@trpc/server": "^9.20.3",
+    "pg": "^8.7.3",
     "reflect-metadata": "^0.1.13",
     "rimraf": "^3.0.2",
-    "rxjs": "^7.2.0"
+    "rxjs": "^7.2.0",
+    "typeorm": "^0.2.45"
   },
   "devDependencies": {
-    "@nestjs/cli": "^8.0.0",
+    "@nestjs/cli": "^8.2.4",
     "@nestjs/schematics": "^8.0.0",
     "@nestjs/testing": "^8.0.0",
     "@types/express": "^4.17.13",

microservices/src/app.controller.ts

@@ -1,12 +0,0 @@
-import { Controller, Get } from '@nestjs/common';
-import { AppService } from './app.service';
-
-@Controller()
-export class AppController {
-  constructor(private readonly appService: AppService) {}
-
-  @Get()
-  getHello(): string {
-    return this.appService.getHello();
-  }
-}

microservices/src/app.module.ts

@@ -1,10 +1,16 @@
 import { Module } from '@nestjs/common';
-import { AppController } from './app.controller';
+import { ImageClassifierModule } from './image-classifier/image-classifier.module';
-import { AppService } from './app.service';
+import { databaseConfig } from './config/database.config';
+import { TypeOrmModule } from '@nestjs/typeorm';
+import { ObjectDetectionModule } from './object-detection/object-detection.module';
 
 @Module({
-  imports: [],
+  imports: [
-  controllers: [AppController],
+    TypeOrmModule.forRoot(databaseConfig),
-  providers: [AppService],
+    ImageClassifierModule,
+    ObjectDetectionModule,
+  ],
+  controllers: [],
+  providers: [],
 })
 export class AppModule {}

microservices/src/app.service.ts

@@ -1,9 +0,0 @@
-import { Injectable } from '@nestjs/common';
-
-@Injectable()
-export class AppService {
-  getHello(): string {
-    console.log('Hello World 123');
-    return 'Hello World!';
-  }
-}

microservices/src/config/database.config.ts

@@ -0,0 +1,11 @@
+import { TypeOrmModuleOptions } from '@nestjs/typeorm';
+
+export const databaseConfig: TypeOrmModuleOptions = {
+  type: 'postgres',
+  host: 'immich_postgres',
+  port: 5432,
+  username: process.env.DB_USERNAME,
+  password: process.env.DB_PASSWORD,
+  database: process.env.DB_DATABASE_NAME,
+  synchronize: false,
+};

microservices/src/image-classifier/image-classifier.controller.ts

@@ -0,0 +1,14 @@
+import { Body, Controller, Post } from '@nestjs/common';
+import { ImageClassifierService } from './image-classifier.service';
+
+@Controller('image-classifier')
+export class ImageClassifierController {
+  constructor(
+    private readonly imageClassifierService: ImageClassifierService,
+  ) {}
+
+  @Post('/tagImage')
+  async tagImage(@Body('thumbnailPath') thumbnailPath: string) {
+    return await this.imageClassifierService.tagImage(thumbnailPath);
+  }
+}

microservices/src/image-classifier/image-classifier.module.ts

@@ -0,0 +1,9 @@
+import { Module } from '@nestjs/common';
+import { ImageClassifierService } from './image-classifier.service';
+import { ImageClassifierController } from './image-classifier.controller';
+
+@Module({
+  controllers: [ImageClassifierController],
+  providers: [ImageClassifierService],
+})
+export class ImageClassifierModule {}

microservices/src/image-classifier/image-classifier.service.ts

@@ -0,0 +1,48 @@
+import { Injectable, Logger } from '@nestjs/common';
+import * as mobilenet from '@tensorflow-models/mobilenet';
+import * as cocoSsd from '@tensorflow-models/coco-ssd';
+import * as tf from '@tensorflow/tfjs-node';
+import * as fs from 'fs';
+
+@Injectable()
+export class ImageClassifierService {
+  private readonly MOBILENET_VERSION = 2;
+  private readonly MOBILENET_ALPHA = 1.0;
+
+  private mobileNetModel: mobilenet.MobileNet;
+
+  constructor() {
+    Logger.log(
+      `Running Node TensorFlow Version : ${tf.version['tfjs']}`,
+      'ImageClassifier',
+    );
+    mobilenet
+      .load({
+        version: this.MOBILENET_VERSION,
+        alpha: this.MOBILENET_ALPHA,
+      })
+      .then((mobilenetModel) => (this.mobileNetModel = mobilenetModel));
+  }
+
+  async tagImage(thumbnailPath: string) {
+    try {
+      const isExist = fs.existsSync(thumbnailPath);
+      if (isExist) {
+        const tags = [];
+        const image = fs.readFileSync(thumbnailPath);
+        const decodedImage = tf.node.decodeImage(image, 3) as tf.Tensor3D;
+        const predictions = await this.mobileNetModel.classify(decodedImage);
+
+        for (const prediction of predictions) {
+          if (prediction.probability >= 0.1) {
+            tags.push(...prediction.className.split(',').map((e) => e.trim()));
+          }
+        }
+
+        return tags;
+      }
+    } catch (e) {
+      console.log('Error reading file ', e);
+    }
+  }
+}

microservices/src/main.ts

@@ -1,15 +1,10 @@
 import { NestFactory } from '@nestjs/core';
 import { AppModule } from './app.module';
-import { AppService } from './app.service';
 
 async function bootstrap() {
-  const app = await NestFactory.createApplicationContext(AppModule);
+  const app = await NestFactory.create(AppModule);
 
-  const appService = app.get(AppService);
+  await app.listen(3001);
-
-  appService.getHello();
-
-  await app.close();
 }
 
 bootstrap();

microservices/src/object-detection/object-detection.controller.ts

@@ -0,0 +1,14 @@
+import { Body, Controller, Post } from '@nestjs/common';
+import { ObjectDetectionService } from './object-detection.service';
+
+@Controller('object-detection')
+export class ObjectDetectionController {
+  constructor(
+    private readonly objectDetectionService: ObjectDetectionService,
+  ) {}
+
+  @Post('/detectObject')
+  async detectObject(@Body('thumbnailPath') thumbnailPath: string) {
+    return await this.objectDetectionService.detectObject(thumbnailPath);
+  }
+}

microservices/src/object-detection/object-detection.module.ts

@@ -0,0 +1,9 @@
+import { Module } from '@nestjs/common';
+import { ObjectDetectionService } from './object-detection.service';
+import { ObjectDetectionController } from './object-detection.controller';
+
+@Module({
+  controllers: [ObjectDetectionController],
+  providers: [ObjectDetectionService],
+})
+export class ObjectDetectionModule {}

microservices/src/object-detection/object-detection.service.ts

@@ -0,0 +1,38 @@
+import { Injectable, Logger } from '@nestjs/common';
+import * as cocoSsd from '@tensorflow-models/coco-ssd';
+import * as tf from '@tensorflow/tfjs-node';
+import * as fs from 'fs';
+
+@Injectable()
+export class ObjectDetectionService {
+  private cocoSsdModel: cocoSsd.ObjectDetection;
+
+  constructor() {
+    Logger.log(
+      `Running Node TensorFlow Version : ${tf.version['tfjs']}`,
+      'ObjectDetection',
+    );
+    cocoSsd.load().then((model) => (this.cocoSsdModel = model));
+  }
+  async detectObject(thumbnailPath: string) {
+    try {
+      const isExist = fs.existsSync(thumbnailPath);
+      if (isExist) {
+        const tags = new Set();
+        const image = fs.readFileSync(thumbnailPath);
+        const decodedImage = tf.node.decodeImage(image, 3) as tf.Tensor3D;
+        const predictions = await this.cocoSsdModel.detect(decodedImage);
+
+        for (const result of predictions) {
+          if (result.score > 0.5) {
+            tags.add(result.class);
+          }
+        }
+
+        return [...tags];
+      }
+    } catch (e) {
+      console.log('Error reading file ', e);
+    }
+  }
+}

microservices/test/app.e2e-spec.ts

@@ -1,8 +1,9 @@
 import { Test, TestingModule } from '@nestjs/testing';
 import { INestApplication } from '@nestjs/common';
 import * as request from 'supertest';
-import { AppModule } from './../src/app.module';
+import { AppModule } from '../src/app.module';
 
+// End to End test
 describe('AppController (e2e)', () => {
   let app: INestApplication;
 

mobile/ios/Podfile.lock

@@ -79,4 +79,4 @@ SPEC CHECKSUMS:
 
 PODFILE CHECKSUM: 05c3056158482c567a3e0cdab1351ceeee238a07
 
-COCOAPODS: 1.10.1
+COCOAPODS: 1.11.3

mobile/ios/Runner.xcodeproj/project.pbxproj

@@ -341,7 +341,7 @@
 				GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
 				GCC_WARN_UNUSED_FUNCTION = YES;
 				GCC_WARN_UNUSED_VARIABLE = YES;
-				IPHONEOS_DEPLOYMENT_TARGET = 9.0;
+				IPHONEOS_DEPLOYMENT_TARGET = 11.0;
 				MTL_ENABLE_DEBUG_INFO = NO;
 				NEW_SETTING = "";
 				SDKROOT = iphoneos;
@@ -425,7 +425,7 @@
 				GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
 				GCC_WARN_UNUSED_FUNCTION = YES;
 				GCC_WARN_UNUSED_VARIABLE = YES;
-				IPHONEOS_DEPLOYMENT_TARGET = 9.0;
+				IPHONEOS_DEPLOYMENT_TARGET = 11.0;
 				MTL_ENABLE_DEBUG_INFO = YES;
 				NEW_SETTING = "";
 				ONLY_ACTIVE_ARCH = YES;
@@ -475,7 +475,7 @@
 				GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
 				GCC_WARN_UNUSED_FUNCTION = YES;
 				GCC_WARN_UNUSED_VARIABLE = YES;
-				IPHONEOS_DEPLOYMENT_TARGET = 9.0;
+				IPHONEOS_DEPLOYMENT_TARGET = 11.0;
 				MTL_ENABLE_DEBUG_INFO = NO;
 				NEW_SETTING = "";
 				SDKROOT = iphoneos;

mobile/lib/modules/login/ui/login_form.dart

@@ -15,7 +15,7 @@ class LoginForm extends HookConsumerWidget {
   Widget build(BuildContext context, WidgetRef ref) {
     final usernameController = useTextEditingController(text: 'testuser@email.com');
     final passwordController = useTextEditingController(text: 'password');
-    final serverEndpointController = useTextEditingController(text: 'http://192.168.1.103:2283');
+    final serverEndpointController = useTextEditingController(text: 'http://192.168.1.216:2283');
 
     return Center(
       child: ConstrainedBox(

mobile/lib/modules/search/models/curated_object.model.dart

@@ -0,0 +1,80 @@
+import 'dart:convert';
+
+class CuratedObject {
+  final String id;
+  final String object;
+  final String resizePath;
+  final String deviceAssetId;
+  final String deviceId;
+  CuratedObject({
+    required this.id,
+    required this.object,
+    required this.resizePath,
+    required this.deviceAssetId,
+    required this.deviceId,
+  });
+
+  CuratedObject copyWith({
+    String? id,
+    String? object,
+    String? resizePath,
+    String? deviceAssetId,
+    String? deviceId,
+  }) {
+    return CuratedObject(
+      id: id ?? this.id,
+      object: object ?? this.object,
+      resizePath: resizePath ?? this.resizePath,
+      deviceAssetId: deviceAssetId ?? this.deviceAssetId,
+      deviceId: deviceId ?? this.deviceId,
+    );
+  }
+
+  Map<String, dynamic> toMap() {
+    final result = <String, dynamic>{};
+
+    result.addAll({'id': id});
+    result.addAll({'object': object});
+    result.addAll({'resizePath': resizePath});
+    result.addAll({'deviceAssetId': deviceAssetId});
+    result.addAll({'deviceId': deviceId});
+
+    return result;
+  }
+
+  factory CuratedObject.fromMap(Map<String, dynamic> map) {
+    return CuratedObject(
+      id: map['id'] ?? '',
+      object: map['object'] ?? '',
+      resizePath: map['resizePath'] ?? '',
+      deviceAssetId: map['deviceAssetId'] ?? '',
+      deviceId: map['deviceId'] ?? '',
+    );
+  }
+
+  String toJson() => json.encode(toMap());
+
+  factory CuratedObject.fromJson(String source) => CuratedObject.fromMap(json.decode(source));
+
+  @override
+  String toString() {
+    return 'CuratedObject(id: $id, object: $object, resizePath: $resizePath, deviceAssetId: $deviceAssetId, deviceId: $deviceId)';
+  }
+
+  @override
+  bool operator ==(Object other) {
+    if (identical(this, other)) return true;
+
+    return other is CuratedObject &&
+        other.id == id &&
+        other.object == object &&
+        other.resizePath == resizePath &&
+        other.deviceAssetId == deviceAssetId &&
+        other.deviceId == deviceId;
+  }
+
+  @override
+  int get hashCode {
+    return id.hashCode ^ object.hashCode ^ resizePath.hashCode ^ deviceAssetId.hashCode ^ deviceId.hashCode;
+  }
+}

mobile/lib/modules/search/providers/search_page_state.provider.dart

@@ -1,5 +1,6 @@
 import 'package:hooks_riverpod/hooks_riverpod.dart';
 import 'package:immich_mobile/modules/search/models/curated_location.model.dart';
+import 'package:immich_mobile/modules/search/models/curated_object.model.dart';
 import 'package:immich_mobile/modules/search/models/search_page_state.model.dart';
 
 import 'package:immich_mobile/modules/search/services/search.service.dart';
@@ -64,3 +65,14 @@ final getCuratedLocationProvider = FutureProvider.autoDispose<List<CuratedLocati
     return [];
   }
 });
+
+final getCuratedObjectProvider = FutureProvider.autoDispose<List<CuratedObject>>((ref) async {
+  final SearchService _searchService = SearchService();
+
+  var curatedObject = await _searchService.getCuratedObjects();
+  if (curatedObject != null) {
+    return curatedObject;
+  } else {
+    return [];
+  }
+});

mobile/lib/modules/search/services/search.service.dart

@@ -2,6 +2,7 @@ import 'dart:convert';
 
 import 'package:flutter/material.dart';
 import 'package:immich_mobile/modules/search/models/curated_location.model.dart';
+import 'package:immich_mobile/modules/search/models/curated_object.model.dart';
 import 'package:immich_mobile/shared/models/immich_asset.model.dart';
 import 'package:immich_mobile/shared/services/network.service.dart';
 
@@ -52,4 +53,19 @@ class SearchService {
       throw Error();
     }
   }
+
+  Future<List<CuratedObject>?> getCuratedObjects() async {
+    try {
+      var res = await _networkService.getRequest(url: "asset/allObjects");
+
+      List<dynamic> decodedData = jsonDecode(res.toString());
+
+      List<CuratedObject> result = List.from(decodedData.map((a) => CuratedObject.fromMap(a)));
+
+      return result;
+    } catch (e) {
+      debugPrint("[ERROR] [CuratedObject] ${e.toString()}");
+      throw Error();
+    }
+  }
 }

mobile/lib/modules/search/views/search_page.dart

@@ -6,10 +6,12 @@ import 'package:hive_flutter/hive_flutter.dart';
 import 'package:hooks_riverpod/hooks_riverpod.dart';
 import 'package:immich_mobile/constants/hive_box.dart';
 import 'package:immich_mobile/modules/search/models/curated_location.model.dart';
+import 'package:immich_mobile/modules/search/models/curated_object.model.dart';
 import 'package:immich_mobile/modules/search/providers/search_page_state.provider.dart';
 import 'package:immich_mobile/modules/search/ui/search_bar.dart';
 import 'package:immich_mobile/modules/search/ui/search_suggestion_list.dart';
 import 'package:immich_mobile/routing/router.dart';
+import 'package:immich_mobile/utils/capitalize_first_letter.dart';
 
 // ignore: must_be_immutable
 class SearchPage extends HookConsumerWidget {
@@ -22,6 +24,7 @@ class SearchPage extends HookConsumerWidget {
     var box = Hive.box(userInfoBox);
     final isSearchEnabled = ref.watch(searchPageStateProvider).isSearchEnabled;
     AsyncValue<List<CuratedLocation>> curatedLocation = ref.watch(getCuratedLocationProvider);
+    AsyncValue<List<CuratedObject>> curatedObjects = ref.watch(getCuratedObjectProvider);
 
     useEffect(() {
       searchFocusNode = FocusNode();
@@ -82,6 +85,54 @@ class SearchPage extends HookConsumerWidget {
       );
     }
 
+    _buildThings() {
+      return curatedObjects.when(
+        loading: () => const CircularProgressIndicator(),
+        error: (err, stack) => Text('Error: $err'),
+        data: (objects) {
+          return objects.isNotEmpty
+              ? SizedBox(
+                  height: MediaQuery.of(context).size.width / 3,
+                  child: ListView.builder(
+                    padding: const EdgeInsets.only(left: 16),
+                    scrollDirection: Axis.horizontal,
+                    itemCount: curatedObjects.value?.length,
+                    itemBuilder: ((context, index) {
+                      CuratedObject curatedObjectInfo = objects[index];
+                      var thumbnailRequestUrl =
+                          '${box.get(serverEndpointKey)}/asset/file?aid=${curatedObjectInfo.deviceAssetId}&did=${curatedObjectInfo.deviceId}&isThumb=true';
+
+                      return ThumbnailWithInfo(
+                        imageUrl: thumbnailRequestUrl,
+                        textInfo: curatedObjectInfo.object,
+                        onTap: () {
+                          AutoRouter.of(context)
+                              .push(SearchResultRoute(searchTerm: curatedObjectInfo.object.capitalizeFirstLetter()));
+                        },
+                      );
+                    }),
+                  ),
+                )
+              : SizedBox(
+                  height: MediaQuery.of(context).size.width / 3,
+                  child: ListView.builder(
+                    padding: const EdgeInsets.only(left: 16),
+                    scrollDirection: Axis.horizontal,
+                    itemCount: 1,
+                    itemBuilder: ((context, index) {
+                      return ThumbnailWithInfo(
+                        imageUrl:
+                            'https://images.unsplash.com/photo-1612178537253-bccd437b730e?ixlib=rb-1.2.1&ixid=MnwxMjA3fDB8MHxzZWFyY2h8NXx8Ymxhbmt8ZW58MHx8MHx8&auto=format&fit=crop&w=700&q=60',
+                        textInfo: 'No Object Info Available',
+                        onTap: () {},
+                      );
+                    }),
+                  ),
+                );
+        },
+      );
+    }
+
     return Scaffold(
       appBar: SearchBar(
         searchFocusNode: searchFocusNode,
@@ -104,6 +155,14 @@ class SearchPage extends HookConsumerWidget {
                   ),
                 ),
                 _buildPlaces(),
+                const Padding(
+                  padding: EdgeInsets.all(16.0),
+                  child: Text(
+                    "Things",
+                    style: TextStyle(fontWeight: FontWeight.bold, fontSize: 16),
+                  ),
+                ),
+                _buildThings()
               ],
             ),
             isSearchEnabled ? SearchSuggestionList(onSubmitted: _onSearchSubmitted) : Container(),
@@ -160,7 +219,7 @@ class ThumbnailWithInfo extends StatelessWidget {
                 child: SizedBox(
                   width: MediaQuery.of(context).size.width / 3,
                   child: Text(
-                    textInfo,
+                    textInfo.capitalizeFirstLetter(),
                     style: const TextStyle(
                       color: Colors.white,
                       fontWeight: FontWeight.bold,

mobile/lib/routing/tab_navigation_observer.dart

@@ -26,6 +26,7 @@ class TabNavigationObserver extends AutoRouterObserver {
     if (route.name == 'SearchRoute') {
       // Refresh Location State
       ref.refresh(getCuratedLocationProvider);
+      ref.refresh(getCuratedObjectProvider);
     }
 
     ref.watch(serverInfoProvider.notifier).getServerVersion();

mobile/lib/utils/capitalize_first_letter.dart

@@ -0,0 +1,5 @@
+extension StringExtension on String {
+  String capitalizeFirstLetter() {
+    return "${this[0].toUpperCase()}${substring(1).toLowerCase()}";
+  }
+}

server/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "immich",
-  "version": "0.0.1",
+  "version": "1.3.2",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
     "": {
       "name": "immich",
-      "version": "0.0.1",
+      "version": "1.3.2",
       "license": "UNLICENSED",
       "dependencies": {
         "@mapbox/mapbox-sdk": "^0.13.3",
@@ -1547,6 +1547,66 @@
         }
       }
     },
+    "node_modules/@nestjs/microservices": {
+      "version": "8.4.3",
+      "resolved": "https://registry.npmjs.org/@nestjs/microservices/-/microservices-8.4.3.tgz",
+      "integrity": "sha512-/ZT5wo1s65J9Cqp2g5eNrYO34VH7/qUkDu4jJyZCT61I9UqpO49J3+1YIAHfmJJzHcrenjgt1sBtlFhwPR3Lgg==",
+      "optional": true,
+      "peer": true,
+      "dependencies": {
+        "iterare": "1.2.1",
+        "json-socket": "0.3.0",
+        "tslib": "2.3.1"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/nest"
+      },
+      "peerDependencies": {
+        "@grpc/grpc-js": "*",
+        "@nestjs/common": "^8.0.0",
+        "@nestjs/core": "^8.0.0",
+        "@nestjs/websockets": "^8.0.0",
+        "amqp-connection-manager": "*",
+        "amqplib": "*",
+        "cache-manager": "*",
+        "kafkajs": "*",
+        "mqtt": "*",
+        "nats": "*",
+        "redis": "*",
+        "reflect-metadata": "^0.1.12",
+        "rxjs": "^7.1.0"
+      },
+      "peerDependenciesMeta": {
+        "@grpc/grpc-js": {
+          "optional": true
+        },
+        "@nestjs/websockets": {
+          "optional": true
+        },
+        "amqp-connection-manager": {
+          "optional": true
+        },
+        "amqplib": {
+          "optional": true
+        },
+        "cache-manager": {
+          "optional": true
+        },
+        "kafkajs": {
+          "optional": true
+        },
+        "mqtt": {
+          "optional": true
+        },
+        "nats": {
+          "optional": true
+        },
+        "redis": {
+          "optional": true
+        }
+      }
+    },
     "node_modules/@nestjs/passport": {
       "version": "8.1.0",
       "resolved": "https://registry.npmjs.org/@nestjs/passport/-/passport-8.1.0.tgz",
@@ -7053,6 +7113,13 @@
       "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz",
       "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug=="
     },
+    "node_modules/json-socket": {
+      "version": "0.3.0",
+      "resolved": "https://registry.npmjs.org/json-socket/-/json-socket-0.3.0.tgz",
+      "integrity": "sha512-jc8ZbUnYIWdxERFWQKVgwSLkGSe+kyzvmYxwNaRgx/c8NNyuHes4UHnPM3LUrAFXUx1BhNJ94n1h/KCRlbvV0g==",
+      "optional": true,
+      "peer": true
+    },
     "node_modules/json-stable-stringify-without-jsonify": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz",
@@ -11943,6 +12010,18 @@
       "integrity": "sha512-NFvofzSinp00j5rzUd4tf+xi9od6383iY0JP7o0Bnu1fuItAUkWBgc4EKuIQ3D+c2QI3i9pG1kDWAeY27EMGtg==",
       "requires": {}
     },
+    "@nestjs/microservices": {
+      "version": "8.4.3",
+      "resolved": "https://registry.npmjs.org/@nestjs/microservices/-/microservices-8.4.3.tgz",
+      "integrity": "sha512-/ZT5wo1s65J9Cqp2g5eNrYO34VH7/qUkDu4jJyZCT61I9UqpO49J3+1YIAHfmJJzHcrenjgt1sBtlFhwPR3Lgg==",
+      "optional": true,
+      "peer": true,
+      "requires": {
+        "iterare": "1.2.1",
+        "json-socket": "0.3.0",
+        "tslib": "2.3.1"
+      }
+    },
     "@nestjs/passport": {
       "version": "8.1.0",
       "resolved": "https://registry.npmjs.org/@nestjs/passport/-/passport-8.1.0.tgz",
@@ -16243,6 +16322,13 @@
       "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz",
       "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug=="
     },
+    "json-socket": {
+      "version": "0.3.0",
+      "resolved": "https://registry.npmjs.org/json-socket/-/json-socket-0.3.0.tgz",
+      "integrity": "sha512-jc8ZbUnYIWdxERFWQKVgwSLkGSe+kyzvmYxwNaRgx/c8NNyuHes4UHnPM3LUrAFXUx1BhNJ94n1h/KCRlbvV0g==",
+      "optional": true,
+      "peer": true
+    },
     "json-stable-stringify-without-jsonify": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz",

server/src/api-v1/asset/asset.controller.ts

@@ -16,13 +16,12 @@ import {
 } from '@nestjs/common';
 import { JwtAuthGuard } from '../../modules/immich-jwt/guards/jwt-auth.guard';
 import { AssetService } from './asset.service';
-import { FileFieldsInterceptor, FilesInterceptor } from '@nestjs/platform-express';
+import { FileFieldsInterceptor } from '@nestjs/platform-express';
 import { multerOption } from '../../config/multer-option.config';
 import { AuthUserDto, GetAuthUser } from '../../decorators/auth-user.decorator';
 import { CreateAssetDto } from './dto/create-asset.dto';
 import { ServeFileDto } from './dto/serve-file.dto';
-import { AssetOptimizeService } from '../../modules/image-optimize/image-optimize.service';
+import { AssetEntity } from './entities/asset.entity';
-import { AssetEntity, AssetType } from './entities/asset.entity';
 import { GetAllAssetQueryDto } from './dto/get-all-asset-query.dto';
 import { Response as Res } from 'express';
 import { GetNewAssetQueryDto } from './dto/get-new-asset-query.dto';
@@ -61,6 +60,7 @@ export class AssetController {
       if (uploadFiles.thumbnailData != null) {
         await this.assetService.updateThumbnailInfo(savedAsset.id, uploadFiles.thumbnailData[0].path);
         await this.backgroundTaskService.tagImage(uploadFiles.thumbnailData[0].path, savedAsset);
+        await this.backgroundTaskService.detectObject(uploadFiles.thumbnailData[0].path, savedAsset);
       }
 
       await this.backgroundTaskService.extractExif(savedAsset, file.originalname, file.size);
@@ -81,6 +81,11 @@ export class AssetController {
     return this.assetService.serveFile(authUser, query, res, headers);
   }
 
+  @Get('/allObjects')
+  async getCuratedObject(@GetAuthUser() authUser: AuthUserDto) {
+    return this.assetService.getCuratedObject(authUser);
+  }
+
   @Get('/allLocation')
   async getCuratedLocation(@GetAuthUser() authUser: AuthUserDto) {
     return this.assetService.getCuratedLocation(authUser);

server/src/api-v1/asset/asset.service.ts

@@ -3,9 +3,8 @@ import { InjectRepository } from '@nestjs/typeorm';
 import { MoreThan, Repository } from 'typeorm';
 import { AuthUserDto } from '../../decorators/auth-user.decorator';
 import { CreateAssetDto } from './dto/create-asset.dto';
-import { UpdateAssetDto } from './dto/update-asset.dto';
 import { AssetEntity, AssetType } from './entities/asset.entity';
-import _, { result } from 'lodash';
+import _ from 'lodash';
 import { GetAllAssetQueryDto } from './dto/get-all-asset-query.dto';
 import { GetAllAssetReponseDto } from './dto/get-all-asset-response.dto';
 import { createReadStream, stat } from 'fs';
@@ -44,9 +43,7 @@ export class AssetService {
     asset.duration = assetInfo.duration;
 
     try {
-      const res = await this.assetRepository.save(asset);
+      return await this.assetRepository.save(asset);
-
-      return res;
     } catch (e) {
       Logger.error(`Error Create New Asset ${e}`, 'createUserAsset');
     }
@@ -68,13 +65,11 @@ export class AssetService {
 
   public async getAllAssetsNoPagination(authUser: AuthUserDto) {
     try {
-      const assets = await this.assetRepository
+      return await this.assetRepository
         .createQueryBuilder('a')
         .where('a."userId" = :userId', { userId: authUser.id })
         .orderBy('a."createdAt"::date', 'DESC')
         .getMany();
-
-      return assets;
     } catch (e) {
       Logger.error(e, 'getAllAssets');
     }
@@ -226,10 +221,10 @@ export class AssetService {
   }
 
   public async deleteAssetById(authUser: AuthUserDto, assetIds: DeleteAssetDto) {
-    let result = [];
+    const result = [];
 
     const target = assetIds.ids;
-    for (let assetId of target) {
+    for (const assetId of target) {
       const res = await this.assetRepository.delete({
         id: assetId,
         userId: authUser.id,
@@ -251,11 +246,11 @@ export class AssetService {
     return result;
   }
 
-  async getAssetSearchTerm(authUser: AuthUserDto): Promise<String[]> {
+  async getAssetSearchTerm(authUser: AuthUserDto): Promise<string[]> {
-    const possibleSearchTerm = new Set<String>();
+    const possibleSearchTerm = new Set<string>();
     const rows = await this.assetRepository.query(
       `
-      select distinct si.tags, e.orientation, e."lensModel", e.make, e.model , a.type, e.city, e.state, e.country
+      select distinct si.tags, si.objects, e.orientation, e."lensModel", e.make, e.model , a.type, e.city, e.state, e.country
       from assets a
       left join exif e on a.id = e."assetId"
       left join smart_info si on a.id = si."assetId"
@@ -268,6 +263,9 @@ export class AssetService {
       // tags
       row['tags']?.map((tag) => possibleSearchTerm.add(tag?.toLowerCase()));
 
+      // objects
+      row['objects']?.map((object) => possibleSearchTerm.add(object?.toLowerCase()));
+
       // asset's tyoe
       possibleSearchTerm.add(row['type']?.toLowerCase());
 
@@ -301,17 +299,16 @@ export class AssetService {
        AND 
        (
          TO_TSVECTOR('english', ARRAY_TO_STRING(si.tags, ',')) @@ PLAINTO_TSQUERY('english', $2) OR
+         TO_TSVECTOR('english', ARRAY_TO_STRING(si.objects, ',')) @@ PLAINTO_TSQUERY('english', $2) OR
          e.exif_text_searchable_column @@ PLAINTO_TSQUERY('english', $2)
         );
     `;
 
-    const rows = await this.assetRepository.query(query, [authUser.id, searchAssetDto.searchTerm]);
+    return await this.assetRepository.query(query, [authUser.id, searchAssetDto.searchTerm]);
-
-    return rows;
   }
 
   async getCuratedLocation(authUser: AuthUserDto) {
-    const rows = await this.assetRepository.query(
+    return await this.assetRepository.query(
       `
         select distinct on (e.city) a.id, e.city, a."resizePath", a."deviceAssetId", a."deviceId"
         from assets a
@@ -322,7 +319,18 @@ export class AssetService {
       `,
       [authUser.id],
     );
+  }
 
-    return rows;
+  async getCuratedObject(authUser: AuthUserDto) {
+    return await this.assetRepository.query(
+      `
+        select distinct on (unnest(si.objects)) a.id, unnest(si.objects) as "object", a."resizePath", a."deviceAssetId", a."deviceId"
+        from assets a
+        left join smart_info si on a.id = si."assetId"
+        where a."userId" = $1 
+        and si.objects is not null
+      `,
+      [authUser.id],
+    );
   }
 }

server/src/api-v1/asset/entities/smart-info.entity.ts

@@ -13,6 +13,9 @@ export class SmartInfoEntity {
   @Column({ type: 'text', array: true, nullable: true })
   tags: string[];
 
+  @Column({ type: 'text', array: true, nullable: true })
+  objects: string[];
+
   @OneToOne(() => AssetEntity, { onDelete: 'CASCADE', nullable: true })
   @JoinColumn({ name: 'assetId', referencedColumnName: 'id' })
   asset: SmartInfoEntity;

server/src/app.module.ts

@@ -5,7 +5,6 @@ import { UserModule } from './api-v1/user/user.module';
 import { AssetModule } from './api-v1/asset/asset.module';
 import { AuthModule } from './api-v1/auth/auth.module';
 import { ImmichJwtModule } from './modules/immich-jwt/immich-jwt.module';
-import { JwtModule } from '@nestjs/jwt';
 import { DeviceInfoModule } from './api-v1/device-info/device-info.module';
 import { AppLoggerMiddleware } from './middlewares/app-logger.middleware';
 import { ConfigModule, ConfigService } from '@nestjs/config';
@@ -26,14 +25,12 @@ import { CommunicationModule } from './api-v1/communication/communication.module
     ImmichJwtModule,
     DeviceInfoModule,
     BullModule.forRootAsync({
-      imports: [ConfigModule],
+      useFactory: async () => ({
-      useFactory: async (configService: ConfigService) => ({
         redis: {
           host: 'immich_redis',
           port: 6379,
         },
       }),
-      inject: [ConfigService],
     }),
 
     ImageOptimizeModule,

server/src/migration/1648317474768-AddObjectColumnToSmartInfo.ts

@@ -0,0 +1,20 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+export class AddObjectColumnToSmartInfo1648317474768
+  implements MigrationInterface
+{
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`
+      ALTER TABLE smart_info
+        ADD COLUMN objects text[];
+
+    `);
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`
+      ALTER TABLE smart_info
+        DROP COLUMN objects;
+    `);
+  }
+}

server/src/modules/background-task/background-task.processor.ts

@@ -6,7 +6,7 @@ import { AssetEntity } from '../../api-v1/asset/entities/asset.entity';
 import { ConfigService } from '@nestjs/config';
 import exifr from 'exifr';
 import { readFile } from 'fs/promises';
-import fs, { rmSync } from 'fs';
+import fs from 'fs';
 import { Logger } from '@nestjs/common';
 import { ExifEntity } from '../../api-v1/asset/entities/exif.entity';
 import axios from 'axios';
@@ -114,14 +114,37 @@ export class BackgroundTaskProcessor {
   @Process('tag-image')
   async tagImage(job) {
     const { thumbnailPath, asset }: { thumbnailPath: string; asset: AssetEntity } = job.data;
-    const res = await axios.post('http://immich_tf_fastapi:8000/tagImage', { thumbnail_path: thumbnailPath });
 
-    if (res.status == 200) {
+    const res = await axios.post('http://immich_microservices:3001/image-classifier/tagImage', {
+      thumbnailPath: thumbnailPath,
+    });
+
+    if (res.status == 201 && res.data.length > 0) {
       const smartInfo = new SmartInfoEntity();
       smartInfo.assetId = asset.id;
       smartInfo.tags = [...res.data];
 
-      await this.smartInfoRepository.save(smartInfo);
+      await this.smartInfoRepository.upsert(smartInfo, {
+        conflictPaths: ['assetId'],
+      });
+    }
+  }
+
+  @Process('detect-object')
+  async detectObject(job) {
+    const { thumbnailPath, asset }: { thumbnailPath: string; asset: AssetEntity } = job.data;
+
+    const res = await axios.post('http://immich_microservices:3001/object-detection/detectObject', {
+      thumbnailPath: thumbnailPath,
+    });
+
+    if (res.status == 201 && res.data.length > 0) {
+      const smartInfo = new SmartInfoEntity();
+      smartInfo.assetId = asset.id;
+      smartInfo.objects = [...res.data];
+      await this.smartInfoRepository.upsert(smartInfo, {
+        conflictPaths: ['assetId'],
+      });
     }
   }
 }

server/src/modules/background-task/background-task.service.ts

@@ -43,4 +43,15 @@ export class BackgroundTaskService {
       { jobId: randomUUID() },
     );
   }
+
+  async detectObject(thumbnailPath: string, asset: AssetEntity) {
+    await this.backgroundTaskQueue.add(
+      'detect-object',
+      {
+        thumbnailPath,
+        asset,
+      },
+      { jobId: randomUUID() },
+    );
+  }
 }