From 5837b9875d922afea718346fe5e2c643f4174755 Mon Sep 17 00:00:00 2001 From: Arlan Lloyd Date: Fri, 14 Nov 2025 16:46:51 +0000 Subject: [PATCH 1/4] update project structure --- DIAGRAMS.md | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/DIAGRAMS.md b/DIAGRAMS.md index 8b834aa3..089f29a5 100644 --- a/DIAGRAMS.md +++ b/DIAGRAMS.md @@ -5,27 +5,26 @@ These diagrams are created with Mermaid and rendered locally. For the best expe Kyoo is a monorepo that consists of several projects each in their own directory. Diagram below shows an outline of kyoo, projects, and artifacts. ```mermaid -block-beta +block columns 1 block:proj1:1 proj_name["Kyoo"]:1 end block:proj2:1 - dir_1["autosync/"] - dir_2["back/"] + dir_1["api/"] + dir_2["auth/"] dir_3["front/"] dir_4["transcoder/"] dir_5["scanner/"] end block:proj3:1 %% columns auto (default) - block:autosync_b:1 - autosync_i1("kyoo_autosync") + block:api_b:1 + autosync_i1("kyoo_api") end - block:back_b:1 + block:auth_b:1 columns 1 - back_i1("kyoo_back") - back_i2("kyoo_migrations") + back_i1("kyoo_auth") end block:front_b:1 front_i1("kyoo_front") @@ -36,7 +35,6 @@ block-beta block:scanner_b:1 columns 1 scanner_i1("kyoo_scanner") - scanner_i2("kyoo_scanner*") end end @@ -51,19 +49,17 @@ block-beta style dir_4 fill:#438dd5,stroke-width:0px style dir_5 fill:#438dd5,stroke-width:0px - style autosync_b fill:#438dd5,stroke-width:0px - style back_b fill:#438dd5,stroke-width:0px + style api_b fill:#438dd5,stroke-width:0px + style auth_b fill:#438dd5,stroke-width:0px style front_b fill:#438dd5,stroke-width:0px style transcoder_b fill:#438dd5,stroke-width:0px style scanner_b fill:#438dd5,stroke-width:0px style autosync_i1 fill:#85bbf0,stroke-width:0px style back_i1 fill:#85bbf0,stroke-width:0px - style back_i2 fill:#85bbf0,stroke-width:0px style front_i1 fill:#85bbf0,stroke-width:0px style transcoder_i1 fill:#85bbf0,stroke-width:0px style scanner_i1 fill:#85bbf0,stroke-width:0px - style scanner_i2 fill:#85bbf0,stroke-width:0px ``` # C4 Diagrams From efec489c96d2c6c4fe6ab7b5dc0e6d55cbea95ab Mon Sep 17 00:00:00 2001 From: Arlan Lloyd Date: Sat, 15 Nov 2025 20:11:24 +0000 Subject: [PATCH 2/4] update diagrams for v5 --- DIAGRAMS.md | 291 +++++++++++++++++++++------------------------------- 1 file changed, 119 insertions(+), 172 deletions(-) diff --git a/DIAGRAMS.md b/DIAGRAMS.md index 089f29a5..4dc88b32 100644 --- a/DIAGRAMS.md +++ b/DIAGRAMS.md @@ -88,209 +88,156 @@ C4Context Messaging is middleware. EnterpriseMessageBus is for any messaging handled between different projects. ```mermaid C4Container - UpdateLayoutConfig($c4ShapeInRow="3", $c4BoundaryInRow="3") + UpdateLayoutConfig($c4ShapeInRow="3", $c4BoundaryInRow="1") title Container diagram for Kyoo System Person(user, "User") - System_Boundary(internal, "Kyoo") { - Container(frontend, "front/") - Container(backend, "back/") - ContainerQueue(emb, "emb", "", "EnterpriseMessageBus") - Container(transcoder, "transcoder/") - Container(scanner, "scanner/") - Container(autosync, "autosync/") - } - System_Boundary(external, "") { - System_Ext(content, "ContentDatabase", "") - } - System_Boundary(external2, "") { - System_Ext(tracker, "ActivityTracker", "") - } - System_Boundary(external3, "") { - System_Ext(media, "MediaLibrary", "") - } + Container(apigateway, "API Gateway") + Container(auth, "auth") + Container(transcoder, "transcoder") + Container(scanner, "scanner") + Container(frontend, "front") + System_Ext(media, "MediaLibrary", "") + System_Ext(content, "ContentDatabase", "") + Container(api, "api") + System_Ext(tracker, "ActivityTracker", "") - Rel(user, frontend, "") - Rel(user, backend, "") - Rel(frontend, backend, "") - Rel(backend, emb, "") - Rel(backend, transcoder, "") - Rel_Back(autosync, emb, "") - Rel(autosync, tracker, "") - Rel_Back(scanner, emb, "") - Rel(scanner, backend, "") + + Rel(user, apigateway, "") + Rel(apigateway, frontend, "") + Rel(apigateway, scanner, "") + Rel(apigateway, transcoder, "") + Rel(apigateway, api, "") + Rel(apigateway, auth, "") + Rel(frontend, api, "") + Rel(api, tracker, "") + Rel(scanner, api, "") Rel(scanner, media, "") Rel(scanner, content, "") Rel(transcoder, media, "") ``` - - ## Component -### Autosync -```mermaid -C4Component - UpdateLayoutConfig($c4ShapeInRow="4", $c4BoundaryInRow="2") - - title Component Diagram for Autosync - - Container_Boundary(autosync, "autosync") { - Component(autosync_c1, "kyoo_autosync", "python, python3.12", "") - } - Container_Boundary(emb, "emb") { - ComponentQueue(emb_q1, "autosync", "RabbitMQ, Queue", "") - ComponentQueue(emb_e1, "events.watched", "RabbitMQ, Exchange", "") - - } - Container_Boundary(tracker, "ActivityTracker") { - Component_Ext(tracker_c1, "TrackerProvider", "API", "simkl") - } - Container_Boundary(backend, "back") { - Component(backend_c2, "kyoo_back", "C#, .NET 8.0", "API Backend") - } +#### Auth +Kyoo leverages the [API Gateway](https://learn.microsoft.com/en-us/azure/architecture/microservices/design/gateway) approach to microservices and [offloads](https://learn.microsoft.com/en-us/azure/architecture/patterns/gateway-offloading) authentication at the gateway. Auth microservice is implicitly used by each other microservice for both end user authentication and microservice to microservice communications. - Rel(emb_e1, emb_q1, "bound") - Rel_Back(autosync_c1, emb_q1, "consumes") - Rel(backend_c2, emb_e1, "produces") - Rel(autosync_c1, tracker_c1, "updates") -``` +*Auth microservice will not be directly represented in the other component diagrams. Instead in their relationsihp, they will specify "auth via middleware". -### Back ```mermaid C4Component UpdateLayoutConfig($c4ShapeInRow="5", $c4BoundaryInRow="2") - title Component Diagram for Back + title Auth Component Diagram - Person(user, "User") - - Container_Boundary(frontend, "front") { - Component(frontend_c1, "kyoo_front", "typescript, node.js", "Static Content") + Container_Boundary(auth, "auth") { + Component(auth_c1, "kyoo_auth", "Go", "") + ComponentDb(auth_db1, "kelbi", "Postgres", "") } - Container_Boundary(backend, "back") { - ComponentDb(backend_db2, "search", "Meilisearch", "search resource") - Component(backend_c3, "BackendMetadata", "Volume", "Persistent. Distributed Metadata") - ComponentDb(backend_db1, "backend", "Postgres", "user data and session state") - Component(backend_c1, "kyoo_migrations", "C#, .NET 8.0", "Postgres Migration") - Component(backend_c2, "kyoo_back", "C#, .NET 8.0", "API Backend") - } - - Container_Boundary(emb, "emb") { - ComponentQueue(emb_e1, "events.watched", "RabbitMQ, Exchange", "") - ComponentQueue(emb_q1, "autosync", "RabbitMQ, Queue", "") - ComponentQueue(emb_q2, "scanner.rescan", "RabbitMQ, Queue", "") - ComponentQueue(emb_e2, "events.resource", "RabbitMQ, Exchange", "unused") - } - - Container_Boundary(scanner, "scanner") { - Component(scanner_c2, "kyoo_scanner", "python, python3.12", "matcher") - Component(scanner_c1, "kyoo_scanner", "python, python3.12", "scanner") - } - - Container_Boundary(autosync, "autosync") { - Component(autosync_c1, "kyoo_autosync", "python, python3.12", "") - } - - Container_Boundary(transcoder, "transcoder") { - Component(transcoder_c1, "kyoo_transcoder", "go, go", "Video Transcoder") - } - - Rel(user, backend_c2, "") - Rel(backend_c1, backend_db1, "") - Rel(backend_c2, backend_db1, "") - Rel(backend_c2, backend_db2, "") - Rel(backend_c2, transcoder_c1, "") - Rel(backend_c2, backend_c3, "") - Rel(backend_c2, emb_q2, "produces") - Rel(backend_c2, emb_e1, "produces") - Rel(backend_c2, emb_e2, "produces") - Rel(emb_e1, emb_q1, "bound") - Rel_Back(autosync_c1, emb_q1, "consumes") - Rel_Back(scanner_c1, emb_q2, "consumes") - Rel(scanner_c1, backend_c2, "") - Rel(scanner_c2, backend_c2, "") - Rel(frontend_c1, backend_c2, "") + Rel(auth_c1, auth_db1, "") ``` -### Front -```mermaid -C4Component - UpdateLayoutConfig($c4ShapeInRow="4", $c4BoundaryInRow="2") - - title Component Diagram for Front - - Person(user, "User") - Container_Boundary(frontend, "front") { - Component(frontend_c1, "kyoo_front", "typescript, node.js", "Static Content") - } - Container_Boundary(backend, "back") { - Component(backend_c2, "kyoo_back", "C#, .NET 8.0", "API Backend") - } - - Rel(frontend_c1, backend_c2, "ssr") - Rel(user, frontend_c1, "") -``` - -### Scanner -```mermaid -C4Component - UpdateLayoutConfig($c4ShapeInRow="5", $c4BoundaryInRow="3") - - title Component Diagram for Scanner - - Container_Boundary(media, "MediaLibrary") { - Component_Ext(media_c1, "MediaShare", "Volume", "Read Only") - } - - Container_Boundary(content, "ContentDatabase") { - Component_Ext(content_c1, "ContentProvider", "API", "tmdb or tvdb") - } - - Container_Boundary(scanner, "scanner") { - Component(scanner_c2, "kyoo_scanner", "python, python3.12", "matcher") - ComponentQueue(scanner_q1, "scanner", "RabbitMQ, Queue", "") - Component(scanner_c1, "kyoo_scanner", "python, python3.12", "scanner") - } - - Container_Boundary(backend, "back") { - Component(backend_c2, "kyoo_back", "C#, .NET 8.0", "API Backend") - } - - Container_Boundary(emb, "emb") { - ComponentQueue(emb_q2, "scanner.rescan", "RabbitMQ, Queue", "") - } - - Rel(scanner_c1, scanner_q1, "produces") - Rel(scanner_c1, media_c1, "watches") - Rel(scanner_c1, backend_c2, "Fetch existing scans") - Rel(scanner_c2, content_c1, "Fetch media data") - Rel(scanner_c2, backend_c2, "Pushes media data") - Rel_Back(scanner_c2, scanner_q1, "consumes") - Rel_Back(scanner_c1, emb_q2, "consumes") - Rel(backend_c2, emb_q2, "produces") -``` - -### Transcoder +#### Api ```mermaid C4Component UpdateLayoutConfig($c4ShapeInRow="2", $c4BoundaryInRow="2") - title Component Diagram for Transcoder + title Api Component Diagram + + Person(user, "User") + Container_Boundary(api, "api") { + ComponentDb(api_db1, "kyoo", "Postgres", "") + Component(api_c1, "kyoo_api", "TypeScript", "") + Component(api_c2, "ApiMetadata", "Volume", "Persistent. Distributed Metadata") + } + Container_Boundary(scanner, "scanner") { + Component(scanner_c1, "kyoo_scanner", "Python", "") + } + Container_Boundary(front, "front") { + Component(front_c1, "kyoo_front", "TypeScript", "") + } + + System_Boundary(external, "") { + System_Ext(tracker, "ActivityTracker", "") + } + + Rel(user, api_c1, "auth via middleware") + Rel(api_c1, api_db1, "") + Rel(api_c1, api_c2, "") + Rel(api_c1, tracker, "") + Rel(scanner_c1, api_c1, "auth via middleware") + Rel(front_c1, api_c1, "") +``` + +#### Front +```mermaid +C4Component + UpdateLayoutConfig($c4ShapeInRow="5", $c4BoundaryInRow="2") + + title Front Component Diagram + + Person(user, "User") + Container_Boundary(front, "front") { + Component(front_c1, "kyoo_front", "TypeScript", "") + } + + Container_Boundary(api, "api") { + Component(api_c1, "kyoo_api", "TypeScript", "") + } + Rel(user, front_c1, "") + Rel(front_c1, api_c1, "") +``` + + +#### Transcoder +```mermaid +C4Component + UpdateLayoutConfig($c4ShapeInRow="2", $c4BoundaryInRow="1") + + title Transcoder Component Diagram + + Person(user, "User") Container_Boundary(transcoder, "transcoder") { + ComponentDb(transcoder_db1, "gocoder", "Postgres", "") + Component(transcoder_c1, "kyoo_transcoder", "Go", "") Component(transcoder_c2, "TranscodeMetadata", "Volume", "Persistent. Distributed Metadata") - Component(transcoder_c1, "kyoo_transcoder", "go, go", "Video Transcoder") Component(transcoder_c3, "TranscodeCache", "Volume", "Volatile. Local cache") } - Container_Boundary(media, "MediaLibrary") { - Component_Ext(media_c1, "MediaShare", "Volume", "Read Only") - } - Container_Boundary(backend, "back") { - Component(backend_c2, "kyoo_back", "C#, .NET 8.0", "API Backend") + + System_Boundary(external, "") { + System_Ext(media, "MediaLibrary", "") } - Rel(transcoder_c1, media_c1, "mounts") + Rel(user, transcoder_c1, "auth via middleware") + Rel(transcoder_c1, media, "mounted to filesystem
reads") + Rel(transcoder_c1, transcoder_db1, "") Rel(transcoder_c1, transcoder_c2, "") Rel(transcoder_c1, transcoder_c3, "") - Rel(backend_c2, transcoder_c1, "") +``` + + +#### Scanner +```mermaid +C4Component + UpdateLayoutConfig($c4ShapeInRow="5", $c4BoundaryInRow="2") + + title Scanner Component Diagram + + Container_Boundary(api, "api") { + Component(api_c1, "kyoo_api", "TypeScript", "") + } + + Container_Boundary(scanner, "scanner") { + Component(scanner_c1, "kyoo_scanner", "Python", "") + ComponentDb(scanner_db1, "scanner", "Postgres", "") + } + System_Boundary(external, "") { + System_Ext(content, "ContentDatabase", "") + System_Ext(media, "MediaLibrary", "") + } + + Rel(scanner_c1, api_c1, "http(s)
auth via middleware") + Rel(scanner_c1, scanner_db1, "") + Rel(scanner_c1, content, "http(s)
gathers media info & images") + Rel(scanner_c1, media, "mounted to filesystem
watches") ``` \ No newline at end of file From 8aaf786400a73a9de97456eba9d98e88b36cceb5 Mon Sep 17 00:00:00 2001 From: Arlan Lloyd Date: Tue, 18 Nov 2025 16:04:18 +0000 Subject: [PATCH 3/4] update metadata storage + scanner docs --- DIAGRAMS.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/DIAGRAMS.md b/DIAGRAMS.md index 4dc88b32..587f401d 100644 --- a/DIAGRAMS.md +++ b/DIAGRAMS.md @@ -85,7 +85,6 @@ C4Context ``` ## Container -Messaging is middleware. EnterpriseMessageBus is for any messaging handled between different projects. ```mermaid C4Container UpdateLayoutConfig($c4ShapeInRow="3", $c4BoundaryInRow="1") @@ -121,7 +120,7 @@ C4Container #### Auth Kyoo leverages the [API Gateway](https://learn.microsoft.com/en-us/azure/architecture/microservices/design/gateway) approach to microservices and [offloads](https://learn.microsoft.com/en-us/azure/architecture/patterns/gateway-offloading) authentication at the gateway. Auth microservice is implicitly used by each other microservice for both end user authentication and microservice to microservice communications. -*Auth microservice will not be directly represented in the other component diagrams. Instead in their relationsihp, they will specify "auth via middleware". +*Auth microservice will not be directly represented in the other component diagrams. Instead in their relationsihp will specify "auth via middleware". ```mermaid C4Component @@ -147,7 +146,7 @@ C4Component Container_Boundary(api, "api") { ComponentDb(api_db1, "kyoo", "Postgres", "") Component(api_c1, "kyoo_api", "TypeScript", "") - Component(api_c2, "ApiMetadata", "Volume", "Persistent. Distributed Metadata") + Component(api_c2, "ApiMetadata", "S3/Volume", "Persistent. Distributed Metadata") } Container_Boundary(scanner, "scanner") { Component(scanner_c1, "kyoo_scanner", "Python", "") @@ -200,7 +199,7 @@ C4Component Container_Boundary(transcoder, "transcoder") { ComponentDb(transcoder_db1, "gocoder", "Postgres", "") Component(transcoder_c1, "kyoo_transcoder", "Go", "") - Component(transcoder_c2, "TranscodeMetadata", "Volume", "Persistent. Distributed Metadata") + Component(transcoder_c2, "TranscodeMetadata", "S3/Volume", "Persistent. Distributed Metadata") Component(transcoder_c3, "TranscodeCache", "Volume", "Volatile. Local cache") } @@ -223,6 +222,8 @@ C4Component title Scanner Component Diagram + Person(user, "User") + Container_Boundary(api, "api") { Component(api_c1, "kyoo_api", "TypeScript", "") } @@ -236,6 +237,7 @@ C4Component System_Ext(media, "MediaLibrary", "") } + Rel(user, scanner_c1, "http(s)
auth via middleware") Rel(scanner_c1, api_c1, "http(s)
auth via middleware") Rel(scanner_c1, scanner_db1, "") Rel(scanner_c1, content, "http(s)
gathers media info & images") From df26dbca63f5d2edbfb28f4f82f1c411e0be06a6 Mon Sep 17 00:00:00 2001 From: Arlan Lloyd Date: Tue, 18 Nov 2025 18:10:13 +0000 Subject: [PATCH 4/4] update SSR notes + api downloads media images --- DIAGRAMS.md | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/DIAGRAMS.md b/DIAGRAMS.md index 587f401d..ecec1e19 100644 --- a/DIAGRAMS.md +++ b/DIAGRAMS.md @@ -85,6 +85,8 @@ C4Context ``` ## Container +Kyoo leverages the [API Gateway](https://learn.microsoft.com/en-us/azure/architecture/microservices/design/gateway) approach to microservices and [offloads](https://learn.microsoft.com/en-us/azure/architecture/patterns/gateway-offloading) authentication at the gateway. + ```mermaid C4Container UpdateLayoutConfig($c4ShapeInRow="3", $c4BoundaryInRow="1") @@ -111,6 +113,7 @@ C4Container Rel(apigateway, auth, "") Rel(frontend, api, "") Rel(api, tracker, "") + Rel(api, content, "") Rel(scanner, api, "") Rel(scanner, media, "") Rel(scanner, content, "") @@ -118,9 +121,7 @@ C4Container ``` ## Component #### Auth -Kyoo leverages the [API Gateway](https://learn.microsoft.com/en-us/azure/architecture/microservices/design/gateway) approach to microservices and [offloads](https://learn.microsoft.com/en-us/azure/architecture/patterns/gateway-offloading) authentication at the gateway. Auth microservice is implicitly used by each other microservice for both end user authentication and microservice to microservice communications. - -*Auth microservice will not be directly represented in the other component diagrams. Instead in their relationsihp will specify "auth via middleware". +Auth microservice is implicitly used by each other microservice for both end user authentication and microservice to microservice communications. Auth microservice will not be directly represented in the other component diagrams. Instead in their relationsihp will specify "auth via middleware". ```mermaid C4Component @@ -151,20 +152,23 @@ C4Component Container_Boundary(scanner, "scanner") { Component(scanner_c1, "kyoo_scanner", "Python", "") } - Container_Boundary(front, "front") { - Component(front_c1, "kyoo_front", "TypeScript", "") - } System_Boundary(external, "") { System_Ext(tracker, "ActivityTracker", "") + System_Ext(content, "ContentDatabase", "") + } + + Container_Boundary(front, "front") { + Component(front_c1, "kyoo_front", "TypeScript", "") } Rel(user, api_c1, "auth via middleware") Rel(api_c1, api_db1, "") Rel(api_c1, api_c2, "") + Rel(api_c1, content, "http(s)
retries media images") Rel(api_c1, tracker, "") Rel(scanner_c1, api_c1, "auth via middleware") - Rel(front_c1, api_c1, "") + Rel(front_c1, api_c1, "http(s) SSR
auth via middleware") ``` #### Front @@ -183,7 +187,7 @@ C4Component Component(api_c1, "kyoo_api", "TypeScript", "") } Rel(user, front_c1, "") - Rel(front_c1, api_c1, "") + Rel(front_c1, api_c1, "http(s) SSR
auth via middleware") ``` @@ -240,6 +244,6 @@ C4Component Rel(user, scanner_c1, "http(s)
auth via middleware") Rel(scanner_c1, api_c1, "http(s)
auth via middleware") Rel(scanner_c1, scanner_db1, "") - Rel(scanner_c1, content, "http(s)
gathers media info & images") + Rel(scanner_c1, content, "http(s)
gathers media metadata") Rel(scanner_c1, media, "mounted to filesystem
watches") ``` \ No newline at end of file