feat(server): preserve caption fields and extract mediainfo for video (#505)

* feat(server): preserve caption fields and extract mediainfo for video

* Fixed Geocoding missing info leads to fail EXIF extraction for the whole file

Co-authored-by: Alex <alex.tran1502@gmail.com>

server/apps/microservices/src/processors/metadata-extraction.processor.ts

@@ -85,9 +85,21 @@ export class MetadataExtractionProcessor {
 
         const res: [] = geoCodeInfo.body['features'];
 
-        const city = res.filter((geoInfo) => geoInfo['place_type'][0] == 'place')[0]['text'];
+        let city = '';
-        const state = res.filter((geoInfo) => geoInfo['place_type'][0] == 'region')[0]['text'];
+        let state = '';
-        const country = res.filter((geoInfo) => geoInfo['place_type'][0] == 'country')[0]['text'];
+        let country = '';
+
+        if (res.filter((geoInfo) => geoInfo['place_type'][0] == 'place')[0]) {
+          city = res.filter((geoInfo) => geoInfo['place_type'][0] == 'place')[0]['text'];
+        }
+
+        if (res.filter((geoInfo) => geoInfo['place_type'][0] == 'region')[0]) {
+          state = res.filter((geoInfo) => geoInfo['place_type'][0] == 'region')[0]['text'];
+        }
+
+        if (res.filter((geoInfo) => geoInfo['place_type'][0] == 'country')[0]) {
+          country = res.filter((geoInfo) => geoInfo['place_type'][0] == 'country')[0]['text'];
+        }
 
         newExif.city = city || null;
         newExif.state = state || null;
@@ -114,9 +126,21 @@ export class MetadataExtractionProcessor {
 
       const res: [] = geoCodeInfo.body['features'];
 
-      const city = res.filter((geoInfo) => geoInfo['place_type'][0] == 'place')[0]['text'];
+      let city = '';
-      const state = res.filter((geoInfo) => geoInfo['place_type'][0] == 'region')[0]['text'];
+      let state = '';
-      const country = res.filter((geoInfo) => geoInfo['place_type'][0] == 'country')[0]['text'];
+      let country = '';
+
+      if (res.filter((geoInfo) => geoInfo['place_type'][0] == 'place')[0]) {
+        city = res.filter((geoInfo) => geoInfo['place_type'][0] == 'place')[0]['text'];
+      }
+
+      if (res.filter((geoInfo) => geoInfo['place_type'][0] == 'region')[0]) {
+        state = res.filter((geoInfo) => geoInfo['place_type'][0] == 'region')[0]['text'];
+      }
+
+      if (res.filter((geoInfo) => geoInfo['place_type'][0] == 'country')[0]) {
+        country = res.filter((geoInfo) => geoInfo['place_type'][0] == 'country')[0]['text'];
+      }
 
       await this.exifRepository.update({ id: exif.id }, { city, state, country });
     }
@@ -168,31 +192,126 @@ export class MetadataExtractionProcessor {
   async extractVideoMetadata(job: Job<IVideoLengthExtractionProcessor>) {
     const { asset } = job.data;
 
-    ffmpeg.ffprobe(asset.originalPath, async (err, data) => {
+    try {
-      if (!err) {
+      const data = await new Promise<ffmpeg.FfprobeData>((resolve, reject) =>
-        let durationString = asset.duration;
+        ffmpeg.ffprobe(asset.originalPath, (err, data) => {
-        let createdAt = asset.createdAt;
+          if (err) return reject(err);
+          return resolve(data);
+        }),
+      );
+      let durationString = asset.duration;
+      let createdAt = asset.createdAt;
 
-        if (data.format.duration) {
+      if (data.format.duration) {
-          durationString = this.extractDuration(data.format.duration);
+        durationString = this.extractDuration(data.format.duration);
-        }
+      }
 
-        const videoTags = data.format.tags;
+      const videoTags = data.format.tags;
-        if (videoTags) {
+      if (videoTags) {
-          if (videoTags['com.apple.quicktime.creationdate']) {
+        if (videoTags['com.apple.quicktime.creationdate']) {
-            createdAt = String(videoTags['com.apple.quicktime.creationdate']);
+          createdAt = String(videoTags['com.apple.quicktime.creationdate']);
-          } else if (videoTags['creation_time']) {
+        } else if (videoTags['creation_time']) {
-            createdAt = String(videoTags['creation_time']);
+          createdAt = String(videoTags['creation_time']);
-          } else {
-            createdAt = asset.createdAt;
-          }
         } else {
           createdAt = asset.createdAt;
         }
-
+      } else {
-        await this.assetRepository.update({ id: asset.id }, { duration: durationString, createdAt: createdAt });
+        createdAt = asset.createdAt;
       }
-    });
+
+      const newExif = new ExifEntity();
+      newExif.assetId = asset.id;
+      newExif.description = '';
+      newExif.fileSizeInByte = data.format.size || null;
+      newExif.dateTimeOriginal = createdAt ? new Date(createdAt) : null;
+      newExif.modifyDate = null;
+      newExif.latitude = null;
+      newExif.longitude = null;
+      newExif.city = null;
+      newExif.state = null;
+      newExif.country = null;
+      newExif.fps = null;
+
+      if (videoTags && videoTags['location']) {
+        const location = videoTags['location'] as string;
+        const locationRegex = /([+-][0-9]+\.[0-9]+)([+-][0-9]+\.[0-9]+)\/$/;
+        const match = location.match(locationRegex);
+
+        if (match?.length === 3) {
+          newExif.latitude = parseFloat(match[0]);
+          newExif.longitude = parseFloat(match[1]);
+        }
+      } else if (videoTags && videoTags['com.apple.quicktime.location.ISO6709']) {
+        const location = videoTags['com.apple.quicktime.location.ISO6709'] as string;
+        const locationRegex = /([+-][0-9]+\.[0-9]+)([+-][0-9]+\.[0-9]+)([+-][0-9]+\.[0-9]+)\/$/;
+        const match = location.match(locationRegex);
+        if (match?.length === 4) {
+          newExif.latitude = parseFloat(match[1]);
+          newExif.longitude = parseFloat(match[2]);
+        }
+      }
+
+      // Reverse GeoCoding
+      if (this.geocodingClient && newExif.longitude && newExif.latitude) {
+        const geoCodeInfo: MapiResponse = await this.geocodingClient
+          .reverseGeocode({
+            query: [newExif.longitude, newExif.latitude],
+            types: ['country', 'region', 'place'],
+          })
+          .send();
+
+        const res: [] = geoCodeInfo.body['features'];
+
+        let city = '';
+        let state = '';
+        let country = '';
+
+        if (res.filter((geoInfo) => geoInfo['place_type'][0] == 'place')[0]) {
+          city = res.filter((geoInfo) => geoInfo['place_type'][0] == 'place')[0]['text'];
+        }
+
+        if (res.filter((geoInfo) => geoInfo['place_type'][0] == 'region')[0]) {
+          state = res.filter((geoInfo) => geoInfo['place_type'][0] == 'region')[0]['text'];
+        }
+
+        if (res.filter((geoInfo) => geoInfo['place_type'][0] == 'country')[0]) {
+          country = res.filter((geoInfo) => geoInfo['place_type'][0] == 'country')[0]['text'];
+        }
+
+        newExif.city = city || null;
+        newExif.state = state || null;
+        newExif.country = country || null;
+      }
+
+      for (const stream of data.streams) {
+        if (stream.codec_type === 'video') {
+          newExif.exifImageWidth = stream.width || null;
+          newExif.exifImageHeight = stream.height || null;
+
+          if (typeof stream.rotation === 'string') {
+            newExif.orientation = stream.rotation;
+          } else if (typeof stream.rotation === 'number') {
+            newExif.orientation = `${stream.rotation}`;
+          } else {
+            newExif.orientation = null;
+          }
+
+          if (stream.r_frame_rate) {
+            let fpsParts = stream.r_frame_rate.split('/');
+
+            if (fpsParts.length === 2) {
+              newExif.fps = Math.round(parseInt(fpsParts[0]) / parseInt(fpsParts[1]));
+            }
+          }
+        }
+      }
+
+      await this.exifRepository.save(newExif);
+      await this.assetRepository.update({ id: asset.id }, { duration: durationString, createdAt: createdAt });
+    } catch (err) {
+      // do nothing
+      console.log('Error in video metadata extraction', err);
+    }
   }
 
   private extractDuration(duration: number) {
@@ -202,8 +321,6 @@ export class MetadataExtractionProcessor {
     const minutes = Math.floor((videoDurationInSecond - hours * 3600) / 60);
     const seconds = videoDurationInSecond - hours * 3600 - minutes * 60;
 
-    return `${hours}:${minutes < 10 ? '0' + minutes.toString() : minutes}:${
+    return `${hours}:${minutes.toString().padStart(2, '0')}:${seconds.toString().padStart(2, '0')}.000000`;
-      seconds < 10 ? '0' + seconds.toString() : seconds
-    }.000000`;
   }
 }

server/libs/database/src/entities/exif.entity.ts

@@ -13,14 +13,9 @@ export class ExifEntity {
   @Column({ type: 'uuid' })
   assetId!: string;
 
-  @Column({ type: 'varchar', nullable: true })
+  /* General info */
-  make!: string | null;
+  @Column({ type: 'text', nullable: true, default: '' })
-
+  description!: string; // or caption
-  @Column({ type: 'varchar', nullable: true })
-  model!: string | null;
-
-  @Column({ type: 'varchar', nullable: true })
-  imageName!: string | null;
 
   @Column({ type: 'integer', nullable: true })
   exifImageWidth!: number | null;
@@ -40,21 +35,6 @@ export class ExifEntity {
   @Column({ type: 'timestamptz', nullable: true })
   modifyDate!: Date | null;
 
-  @Column({ type: 'varchar', nullable: true })
-  lensModel!: string | null;
-
-  @Column({ type: 'float8', nullable: true })
-  fNumber!: number | null;
-
-  @Column({ type: 'float8', nullable: true })
-  focalLength!: number | null;
-
-  @Column({ type: 'integer', nullable: true })
-  iso!: number | null;
-
-  @Column({ type: 'float', nullable: true })
-  exposureTime!: number | null;
-
   @Column({ type: 'float', nullable: true })
   latitude!: number | null;
 
@@ -70,9 +50,38 @@ export class ExifEntity {
   @Column({ type: 'varchar', nullable: true })
   country!: string | null;
 
+  /* Image info */
+  @Column({ type: 'varchar', nullable: true })
+  make!: string | null;
+
+  @Column({ type: 'varchar', nullable: true })
+  model!: string | null;
+
+  @Column({ type: 'varchar', nullable: true })
+  imageName!: string | null;
+
+  @Column({ type: 'varchar', nullable: true })
+  lensModel!: string | null;
+
+  @Column({ type: 'float8', nullable: true })
+  fNumber!: number | null;
+
+  @Column({ type: 'float8', nullable: true })
+  focalLength!: number | null;
+
+  @Column({ type: 'integer', nullable: true })
+  iso!: number | null;
+
+  @Column({ type: 'float', nullable: true })
+  exposureTime!: number | null;
+
+  /* Video info */
+  @Column({ type: 'float8', nullable: true })
+  fps?: number | null;
+
   @OneToOne(() => AssetEntity, { onDelete: 'CASCADE', nullable: true })
   @JoinColumn({ name: 'assetId', referencedColumnName: 'id' })
-  asset?: ExifEntity;
+  asset?: AssetEntity;
 
   @Index('exif_text_searchable', { synchronize: false })
   @Column({

server/libs/database/src/entities/smart-info.entity.ts

@@ -18,5 +18,5 @@ export class SmartInfoEntity {
 
   @OneToOne(() => AssetEntity, { onDelete: 'CASCADE', nullable: true })
   @JoinColumn({ name: 'assetId', referencedColumnName: 'id' })
-  asset?: SmartInfoEntity;
+  asset?: AssetEntity;
 }

server/libs/database/src/migrations/1661011331242-AddCaption.ts

@@ -0,0 +1,18 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+export class AddCaption1661011331242 implements MigrationInterface {
+    name = 'AddCaption1661011331242'
+
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE "exif" ADD "description" text DEFAULT ''`);
+        await queryRunner.query(`ALTER TABLE "exif" ADD "fps" double precision`);
+        // await queryRunner.query(`ALTER TABLE "exif" ALTER COLUMN "exifTextSearchableColumn" SET NOT NULL`);
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        // await queryRunner.query(`ALTER TABLE "exif" ALTER COLUMN "exifTextSearchableColumn" DROP NOT NULL`);
+        await queryRunner.query(`ALTER TABLE "exif" DROP COLUMN "fps"`);
+        await queryRunner.query(`ALTER TABLE "exif" DROP COLUMN "description"`);
+    }
+
+}