feat: object storage

server/package.json

@@ -34,6 +34,8 @@
     "sql:generate": "node ./dist/infra/sql-generator/"
   },
   "dependencies": {
+    "@aws-sdk/client-s3": "^3.515.0",
+    "@aws-sdk/lib-storage": "^3.515.0",
     "@babel/runtime": "^7.22.11",
     "@immich/cli": "^2.0.7",
     "@nestjs/bullmq": "^10.0.1",

server/src/domain/fs/fs.ts

@@ -0,0 +1,27 @@
+import { Readable, Writable } from "node:stream";
+
+export interface FS {
+  // create creates an object with the given name.
+  create(name: string): Promise<Writable>;
+
+  // open opens the named object.
+  open(name: string): Promise<Readable>;
+
+  // remove removes the named object.
+  remove(name: string): Promise<void>;
+}
+
+// export interface FS {
+//   // create creates an object with the given name.
+//   create(name: string): Promise<WritableFile>;
+
+//   // open opens the object with the given name.
+//   open(name: string): Promise<ReadableFile>;
+
+//   // remove removes the named object.
+//   remove(name: string): Promise<void>;
+// }
+
+// export interface File {
+//   createReadableStream(): Promise<Readable>;
+// }

server/src/domain/fs/local.ts

@@ -0,0 +1,21 @@
+import { constants, open, unlink } from "node:fs/promises";
+import { join } from "node:path";
+import { Readable, Writable } from "node:stream";
+
+export class LocalFS {
+        constructor(private dir: string) { }
+
+        async create(name: string): Promise<Writable> {
+                const file = await open(join(this.dir, name), constants.O_WRONLY);
+                return file.createWriteStream();
+        }
+
+        async open(name: string): Promise<Readable> {
+                const file = await open(join(this.dir, name), constants.O_RDONLY);
+                return file.createReadStream();
+        }
+
+        async remove(name: string): Promise<void> {
+                await unlink(join(this.dir, name));
+        }
+}

server/src/domain/fs/s3.ts

@@ -0,0 +1,65 @@
+import { PassThrough, Readable, Writable } from "node:stream";
+import { S3 } from "@aws-sdk/client-s3";
+import { FS } from "./fs";
+import { Upload } from "@aws-sdk/lib-storage";
+
+export class S3FS implements FS {
+        s3: S3;
+
+        constructor(private bucket: string) {
+                this.s3 = new S3();
+        }
+
+        async create(name: string): Promise<Writable> {
+                const stream = new PassThrough();
+                const upload = new Upload({
+                        client: this.s3,
+                        params: {
+                                Body: stream,
+                                Bucket: this.bucket,
+                                Key: name,
+                        },
+                });
+
+                // Abort the upload if the stream has finished. Should be a
+                // no-op if the upload has already finished.
+                stream.on('close', () => upload.abort());
+
+                // Close the stream when the upload is finished, or if it
+                // failed.
+                //
+                // TODO: Find a way to bubble up this error.
+                upload.done().then(() => void stream.end(), error => {
+                        console.log(`s3 upload failed: ${error}`);
+                        stream.end();
+                });
+
+                return stream;
+        }
+
+        async open(name: string): Promise<Readable> {
+                const obj = await this.s3.getObject({
+                        Bucket: this.bucket,
+                        Key: name,
+                });
+                return obj.Body as Readable;
+                // const stream = obj.Body?.transformToWebStream();
+                // if (!stream) {
+                //         throw new Error("no body");
+                // }
+                // return Readable.fromWeb(new ReadableStream(stream));
+        }
+
+        async remove(name: string): Promise<void> {
+                await this.s3.deleteObject({
+                        Bucket: this.bucket,
+                        Key: name,
+                })
+        }
+}
+
+// class ObjectReadable extends Readable {
+//         constructor(private s3: S3, private bucket: string) { }
+
+
+// }

server/src/domain/system-config/system-config.core.ts

@@ -140,6 +140,12 @@ export const defaults = Object.freeze<SystemConfig>({
     externalDomain: '',
     loginPageMessage: '',
   },
+  storage: {
+    kind: 'local',
+    options: {
+      path: process.env.UPLOAD_LOCATION ?? '',
+    }
+  }
 });
 
 export enum FeatureFlag {

server/src/infra/entities/system-config.entity.ts

@@ -172,6 +172,18 @@ export enum LogLevel {
   FATAL = 'fatal',
 }
 
+export interface StorageOptionsLocal {
+  path: string;
+}
+
+export interface StorageOptionsS3 {
+  bucket: string;
+  region: string;
+  endpoint: string;
+  accessKeyId: string;
+  secretAccessKey: string;
+}
+
 export interface SystemConfig {
   ffmpeg: {
     crf: number;
@@ -276,4 +288,10 @@ export interface SystemConfig {
     externalDomain: string;
     loginPageMessage: string;
   };
+  // TODO(uhthomas): Is this definitely the approach we want to take for
+  // configuring storage?
+  storage: {
+    kind: string;
+    options: StorageOptionsLocal | StorageOptionsS3;
+  }
 }