diff --git a/api/bun.lock b/api/bun.lock index 5842b41a..a1afe77a 100644 --- a/api/bun.lock +++ b/api/bun.lock @@ -5,7 +5,7 @@ "": { "name": "api", "dependencies": { - "@elysiajs/opentelemetry": "^1.4.7", + "@elysiajs/opentelemetry": "^1.4.8", "@elysiajs/swagger": "zoriya/elysia-swagger#build", "@kubiks/otel-drizzle": "zoriya/drizzle-otel#build", "@types/bun": "^1.3.1", @@ -49,7 +49,7 @@ "@drizzle-team/brocli": ["@drizzle-team/brocli@0.10.2", "", {}, "sha512-z33Il7l5dKjUgGULTqBsQBQwckHh5AbIuxhdsIxDDiZAzBOrZO6q9ogcWC65kU382AfynTfgNumVcNIjuIua6w=="], - "@elysiajs/opentelemetry": ["@elysiajs/opentelemetry@1.4.7", "", { "dependencies": { "@opentelemetry/api": "^1.9.0", "@opentelemetry/instrumentation": "^0.200.0", "@opentelemetry/sdk-node": "^0.200.0" }, "peerDependencies": { "elysia": ">= 1.4.0" } }, "sha512-biJfj3bCHf7aYPB8EygvN90sEKR/qgPn8Cziq2ebJSGyY8cpmskTTP6zbUMkMk6R6rfpoP7ECZbXlTZz+7BfJA=="], + "@elysiajs/opentelemetry": ["@elysiajs/opentelemetry@1.4.8", "", { "dependencies": { "@opentelemetry/api": "^1.9.0", "@opentelemetry/instrumentation": "^0.200.0", "@opentelemetry/sdk-node": "^0.200.0" }, "peerDependencies": { "elysia": ">= 1.4.0" } }, "sha512-c9unbcdXfehExCv1GsiTCfos5SyIAyDwP7apcMeXmUMBaJZiAYMfiEH8RFFFIfIHJHC/xlNJzUPodkcUaaoJJQ=="], "@elysiajs/swagger": ["@elysiajs/swagger@github:zoriya/elysia-swagger#f88fbc7", { "dependencies": { "@scalar/themes": "^0.9.81", "@scalar/types": "^0.1.3", "openapi-types": "^12.1.3", "pathe": "^1.1.2" }, "peerDependencies": { "elysia": ">= 1.3.0" } }, "zoriya-elysia-swagger-f88fbc7"], diff --git a/api/drizzle/0023_mqueue-priority.sql b/api/drizzle/0023_mqueue-priority.sql new file mode 100644 index 00000000..d08b6a32 --- /dev/null +++ b/api/drizzle/0023_mqueue-priority.sql @@ -0,0 +1,3 @@ +ALTER TABLE "kyoo"."history" ALTER COLUMN "time" SET DEFAULT 0;--> statement-breakpoint +ALTER TABLE "kyoo"."history" ALTER COLUMN "time" SET NOT NULL;--> statement-breakpoint +ALTER TABLE "kyoo"."mqueue" ADD COLUMN "priority" integer DEFAULT 0 NOT NULL; \ No newline at end of file diff --git a/api/drizzle/meta/0023_snapshot.json b/api/drizzle/meta/0023_snapshot.json new file mode 100644 index 00000000..9fe88ef2 --- /dev/null +++ b/api/drizzle/meta/0023_snapshot.json @@ -0,0 +1,2036 @@ +{ + "id": "5d875c17-7d8b-4aa1-a4a8-137df6703537", + "prevId": "40840aa3-3b37-4dc0-97fa-efb8adc4da02", + "version": "7", + "dialect": "postgresql", + "tables": { + "kyoo.entries": { + "name": "entries", + "schema": "kyoo", + "columns": { + "pk": { + "name": "pk", + "type": "integer", + "primaryKey": true, + "notNull": true, + "identity": { + "type": "always", + "name": "entries_pk_seq", + "schema": "kyoo", + "increment": "1", + "startWith": "1", + "minValue": "1", + "maxValue": "2147483647", + "cache": "1", + "cycle": false + } + }, + "id": { + "name": "id", + "type": "uuid", + "primaryKey": false, + "notNull": true, + "default": "gen_random_uuid()" + }, + "slug": { + "name": "slug", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "show_pk": { + "name": "show_pk", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "order": { + "name": "order", + "type": "real", + "primaryKey": false, + "notNull": false + }, + "season_number": { + "name": "season_number", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "episode_number": { + "name": "episode_number", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "kind": { + "name": "kind", + "type": "entry_type", + "typeSchema": "kyoo", + "primaryKey": false, + "notNull": true + }, + "extra_kind": { + "name": "extra_kind", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "air_date": { + "name": "air_date", + "type": "date", + "primaryKey": false, + "notNull": false + }, + "runtime": { + "name": "runtime", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "thumbnail": { + "name": "thumbnail", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "external_id": { + "name": "external_id", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "available_since": { + "name": "available_since", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "next_refresh": { + "name": "next_refresh", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "entry_kind": { + "name": "entry_kind", + "columns": [ + { + "expression": "kind", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "hash", + "with": {} + }, + "entry_order": { + "name": "entry_order", + "columns": [ + { + "expression": "order", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "entries_show_pk_shows_pk_fk": { + "name": "entries_show_pk_shows_pk_fk", + "tableFrom": "entries", + "tableTo": "shows", + "schemaTo": "kyoo", + "columnsFrom": [ + "show_pk" + ], + "columnsTo": [ + "pk" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "entries_id_unique": { + "name": "entries_id_unique", + "nullsNotDistinct": false, + "columns": [ + "id" + ] + }, + "entries_slug_unique": { + "name": "entries_slug_unique", + "nullsNotDistinct": false, + "columns": [ + "slug" + ] + }, + "entries_showPk_seasonNumber_episodeNumber_unique": { + "name": "entries_showPk_seasonNumber_episodeNumber_unique", + "nullsNotDistinct": false, + "columns": [ + "show_pk", + "season_number", + "episode_number" + ] + } + }, + "policies": {}, + "checkConstraints": { + "order_positive": { + "name": "order_positive", + "value": "\"kyoo\".\"entries\".\"order\" >= 0" + } + }, + "isRLSEnabled": false + }, + "kyoo.entry_translations": { + "name": "entry_translations", + "schema": "kyoo", + "columns": { + "pk": { + "name": "pk", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "language": { + "name": "language", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tagline": { + "name": "tagline", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "poster": { + "name": "poster", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "entry_name_trgm": { + "name": "entry_name_trgm", + "columns": [ + { + "expression": "\"name\" gin_trgm_ops", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "gin", + "with": {} + } + }, + "foreignKeys": { + "entry_translations_pk_entries_pk_fk": { + "name": "entry_translations_pk_entries_pk_fk", + "tableFrom": "entry_translations", + "tableTo": "entries", + "schemaTo": "kyoo", + "columnsFrom": [ + "pk" + ], + "columnsTo": [ + "pk" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "entry_translations_pk_language_pk": { + "name": "entry_translations_pk_language_pk", + "columns": [ + "pk", + "language" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "kyoo.history": { + "name": "history", + "schema": "kyoo", + "columns": { + "pk": { + "name": "pk", + "type": "integer", + "primaryKey": true, + "notNull": true, + "identity": { + "type": "always", + "name": "history_pk_seq", + "schema": "kyoo", + "increment": "1", + "startWith": "1", + "minValue": "1", + "maxValue": "2147483647", + "cache": "1", + "cycle": false + } + }, + "profile_pk": { + "name": "profile_pk", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "entry_pk": { + "name": "entry_pk", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "video_pk": { + "name": "video_pk", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "percent": { + "name": "percent", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "time": { + "name": "time", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "played_date": { + "name": "played_date", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "history_play_date": { + "name": "history_play_date", + "columns": [ + { + "expression": "played_date", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "history_profile_pk_profiles_pk_fk": { + "name": "history_profile_pk_profiles_pk_fk", + "tableFrom": "history", + "tableTo": "profiles", + "schemaTo": "kyoo", + "columnsFrom": [ + "profile_pk" + ], + "columnsTo": [ + "pk" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "history_entry_pk_entries_pk_fk": { + "name": "history_entry_pk_entries_pk_fk", + "tableFrom": "history", + "tableTo": "entries", + "schemaTo": "kyoo", + "columnsFrom": [ + "entry_pk" + ], + "columnsTo": [ + "pk" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "history_video_pk_videos_pk_fk": { + "name": "history_video_pk_videos_pk_fk", + "tableFrom": "history", + "tableTo": "videos", + "schemaTo": "kyoo", + "columnsFrom": [ + "video_pk" + ], + "columnsTo": [ + "pk" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "percent_valid": { + "name": "percent_valid", + "value": "\"kyoo\".\"history\".\"percent\" between 0 and 100" + } + }, + "isRLSEnabled": false + }, + "kyoo.mqueue": { + "name": "mqueue", + "schema": "kyoo", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "kind": { + "name": "kind", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "message": { + "name": "message", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "priority": { + "name": "priority", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "attempt": { + "name": "attempt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "mqueue_created": { + "name": "mqueue_created", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "kyoo.profiles": { + "name": "profiles", + "schema": "kyoo", + "columns": { + "pk": { + "name": "pk", + "type": "integer", + "primaryKey": true, + "notNull": true, + "identity": { + "type": "always", + "name": "profiles_pk_seq", + "schema": "kyoo", + "increment": "1", + "startWith": "1", + "minValue": "1", + "maxValue": "2147483647", + "cache": "1", + "cycle": false + } + }, + "id": { + "name": "id", + "type": "uuid", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "profiles_id_unique": { + "name": "profiles_id_unique", + "nullsNotDistinct": false, + "columns": [ + "id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "kyoo.season_translations": { + "name": "season_translations", + "schema": "kyoo", + "columns": { + "pk": { + "name": "pk", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "language": { + "name": "language", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "poster": { + "name": "poster", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "thumbnail": { + "name": "thumbnail", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "banner": { + "name": "banner", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "season_name_trgm": { + "name": "season_name_trgm", + "columns": [ + { + "expression": "\"name\" gin_trgm_ops", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "gin", + "with": {} + } + }, + "foreignKeys": { + "season_translations_pk_seasons_pk_fk": { + "name": "season_translations_pk_seasons_pk_fk", + "tableFrom": "season_translations", + "tableTo": "seasons", + "schemaTo": "kyoo", + "columnsFrom": [ + "pk" + ], + "columnsTo": [ + "pk" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "season_translations_pk_language_pk": { + "name": "season_translations_pk_language_pk", + "columns": [ + "pk", + "language" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "kyoo.seasons": { + "name": "seasons", + "schema": "kyoo", + "columns": { + "pk": { + "name": "pk", + "type": "integer", + "primaryKey": true, + "notNull": true, + "identity": { + "type": "always", + "name": "seasons_pk_seq", + "schema": "kyoo", + "increment": "1", + "startWith": "1", + "minValue": "1", + "maxValue": "2147483647", + "cache": "1", + "cycle": false + } + }, + "id": { + "name": "id", + "type": "uuid", + "primaryKey": false, + "notNull": true, + "default": "gen_random_uuid()" + }, + "slug": { + "name": "slug", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "show_pk": { + "name": "show_pk", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "season_number": { + "name": "season_number", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "start_air": { + "name": "start_air", + "type": "date", + "primaryKey": false, + "notNull": false + }, + "end_air": { + "name": "end_air", + "type": "date", + "primaryKey": false, + "notNull": false + }, + "entries_count": { + "name": "entries_count", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "available_count": { + "name": "available_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "external_id": { + "name": "external_id", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "next_refresh": { + "name": "next_refresh", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "show_fk": { + "name": "show_fk", + "columns": [ + { + "expression": "show_pk", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "hash", + "with": {} + }, + "season_nbr": { + "name": "season_nbr", + "columns": [ + { + "expression": "season_number", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "seasons_show_pk_shows_pk_fk": { + "name": "seasons_show_pk_shows_pk_fk", + "tableFrom": "seasons", + "tableTo": "shows", + "schemaTo": "kyoo", + "columnsFrom": [ + "show_pk" + ], + "columnsTo": [ + "pk" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "seasons_id_unique": { + "name": "seasons_id_unique", + "nullsNotDistinct": false, + "columns": [ + "id" + ] + }, + "seasons_slug_unique": { + "name": "seasons_slug_unique", + "nullsNotDistinct": false, + "columns": [ + "slug" + ] + }, + "seasons_showPk_seasonNumber_unique": { + "name": "seasons_showPk_seasonNumber_unique", + "nullsNotDistinct": false, + "columns": [ + "show_pk", + "season_number" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "kyoo.show_translations": { + "name": "show_translations", + "schema": "kyoo", + "columns": { + "pk": { + "name": "pk", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "language": { + "name": "language", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tagline": { + "name": "tagline", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "aliases": { + "name": "aliases", + "type": "text[]", + "primaryKey": false, + "notNull": true + }, + "tags": { + "name": "tags", + "type": "text[]", + "primaryKey": false, + "notNull": true + }, + "poster": { + "name": "poster", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "thumbnail": { + "name": "thumbnail", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "banner": { + "name": "banner", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "logo": { + "name": "logo", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "trailer_url": { + "name": "trailer_url", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "name_trgm": { + "name": "name_trgm", + "columns": [ + { + "expression": "\"name\" gin_trgm_ops", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "gin", + "with": {} + }, + "tags": { + "name": "tags", + "columns": [ + { + "expression": "tags", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "show_translations_pk_shows_pk_fk": { + "name": "show_translations_pk_shows_pk_fk", + "tableFrom": "show_translations", + "tableTo": "shows", + "schemaTo": "kyoo", + "columnsFrom": [ + "pk" + ], + "columnsTo": [ + "pk" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "show_translations_pk_language_pk": { + "name": "show_translations_pk_language_pk", + "columns": [ + "pk", + "language" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "kyoo.shows": { + "name": "shows", + "schema": "kyoo", + "columns": { + "pk": { + "name": "pk", + "type": "integer", + "primaryKey": true, + "notNull": true, + "identity": { + "type": "always", + "name": "shows_pk_seq", + "schema": "kyoo", + "increment": "1", + "startWith": "1", + "minValue": "1", + "maxValue": "2147483647", + "cache": "1", + "cycle": false + } + }, + "id": { + "name": "id", + "type": "uuid", + "primaryKey": false, + "notNull": true, + "default": "gen_random_uuid()" + }, + "slug": { + "name": "slug", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "show_kind", + "typeSchema": "kyoo", + "primaryKey": false, + "notNull": true + }, + "genres": { + "name": "genres", + "type": "genres[]", + "typeSchema": "kyoo", + "primaryKey": false, + "notNull": true + }, + "rating": { + "name": "rating", + "type": "smallint", + "primaryKey": false, + "notNull": false + }, + "runtime": { + "name": "runtime", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "show_status", + "typeSchema": "kyoo", + "primaryKey": false, + "notNull": true + }, + "start_air": { + "name": "start_air", + "type": "date", + "primaryKey": false, + "notNull": false + }, + "end_air": { + "name": "end_air", + "type": "date", + "primaryKey": false, + "notNull": false + }, + "original": { + "name": "original", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "collection_pk": { + "name": "collection_pk", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "entries_count": { + "name": "entries_count", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "available_count": { + "name": "available_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "external_id": { + "name": "external_id", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "next_refresh": { + "name": "next_refresh", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "kind": { + "name": "kind", + "columns": [ + { + "expression": "kind", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "hash", + "with": {} + }, + "rating": { + "name": "rating", + "columns": [ + { + "expression": "rating", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "startAir": { + "name": "startAir", + "columns": [ + { + "expression": "start_air", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "shows_collection_pk_shows_pk_fk": { + "name": "shows_collection_pk_shows_pk_fk", + "tableFrom": "shows", + "tableTo": "shows", + "schemaTo": "kyoo", + "columnsFrom": [ + "collection_pk" + ], + "columnsTo": [ + "pk" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "shows_id_unique": { + "name": "shows_id_unique", + "nullsNotDistinct": false, + "columns": [ + "id" + ] + }, + "shows_slug_unique": { + "name": "shows_slug_unique", + "nullsNotDistinct": false, + "columns": [ + "slug" + ] + } + }, + "policies": {}, + "checkConstraints": { + "rating_valid": { + "name": "rating_valid", + "value": "\"kyoo\".\"shows\".\"rating\" between 0 and 100" + }, + "runtime_valid": { + "name": "runtime_valid", + "value": "\"kyoo\".\"shows\".\"runtime\" >= 0" + } + }, + "isRLSEnabled": false + }, + "kyoo.roles": { + "name": "roles", + "schema": "kyoo", + "columns": { + "pk": { + "name": "pk", + "type": "integer", + "primaryKey": true, + "notNull": true, + "identity": { + "type": "always", + "name": "roles_pk_seq", + "schema": "kyoo", + "increment": "1", + "startWith": "1", + "minValue": "1", + "maxValue": "2147483647", + "cache": "1", + "cycle": false + } + }, + "show_pk": { + "name": "show_pk", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "staff_pk": { + "name": "staff_pk", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "role_kind", + "typeSchema": "kyoo", + "primaryKey": false, + "notNull": true + }, + "order": { + "name": "order", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "character": { + "name": "character", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "role_kind": { + "name": "role_kind", + "columns": [ + { + "expression": "kind", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "hash", + "with": {} + }, + "role_order": { + "name": "role_order", + "columns": [ + { + "expression": "order", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "roles_show_pk_shows_pk_fk": { + "name": "roles_show_pk_shows_pk_fk", + "tableFrom": "roles", + "tableTo": "shows", + "schemaTo": "kyoo", + "columnsFrom": [ + "show_pk" + ], + "columnsTo": [ + "pk" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "roles_staff_pk_staff_pk_fk": { + "name": "roles_staff_pk_staff_pk_fk", + "tableFrom": "roles", + "tableTo": "staff", + "schemaTo": "kyoo", + "columnsFrom": [ + "staff_pk" + ], + "columnsTo": [ + "pk" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "kyoo.staff": { + "name": "staff", + "schema": "kyoo", + "columns": { + "pk": { + "name": "pk", + "type": "integer", + "primaryKey": true, + "notNull": true, + "identity": { + "type": "always", + "name": "staff_pk_seq", + "schema": "kyoo", + "increment": "1", + "startWith": "1", + "minValue": "1", + "maxValue": "2147483647", + "cache": "1", + "cycle": false + } + }, + "id": { + "name": "id", + "type": "uuid", + "primaryKey": false, + "notNull": true, + "default": "gen_random_uuid()" + }, + "slug": { + "name": "slug", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "latin_name": { + "name": "latin_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "image": { + "name": "image", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "external_id": { + "name": "external_id", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "staff_id_unique": { + "name": "staff_id_unique", + "nullsNotDistinct": false, + "columns": [ + "id" + ] + }, + "staff_slug_unique": { + "name": "staff_slug_unique", + "nullsNotDistinct": false, + "columns": [ + "slug" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "kyoo.show_studio_join": { + "name": "show_studio_join", + "schema": "kyoo", + "columns": { + "show_pk": { + "name": "show_pk", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "studio_pk": { + "name": "studio_pk", + "type": "integer", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "show_studio_join_show_pk_shows_pk_fk": { + "name": "show_studio_join_show_pk_shows_pk_fk", + "tableFrom": "show_studio_join", + "tableTo": "shows", + "schemaTo": "kyoo", + "columnsFrom": [ + "show_pk" + ], + "columnsTo": [ + "pk" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "show_studio_join_studio_pk_studios_pk_fk": { + "name": "show_studio_join_studio_pk_studios_pk_fk", + "tableFrom": "show_studio_join", + "tableTo": "studios", + "schemaTo": "kyoo", + "columnsFrom": [ + "studio_pk" + ], + "columnsTo": [ + "pk" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "show_studio_join_show_pk_studio_pk_pk": { + "name": "show_studio_join_show_pk_studio_pk_pk", + "columns": [ + "show_pk", + "studio_pk" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "kyoo.studio_translations": { + "name": "studio_translations", + "schema": "kyoo", + "columns": { + "pk": { + "name": "pk", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "language": { + "name": "language", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "logo": { + "name": "logo", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "studio_name_trgm": { + "name": "studio_name_trgm", + "columns": [ + { + "expression": "\"name\" gin_trgm_ops", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "gin", + "with": {} + } + }, + "foreignKeys": { + "studio_translations_pk_studios_pk_fk": { + "name": "studio_translations_pk_studios_pk_fk", + "tableFrom": "studio_translations", + "tableTo": "studios", + "schemaTo": "kyoo", + "columnsFrom": [ + "pk" + ], + "columnsTo": [ + "pk" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "studio_translations_pk_language_pk": { + "name": "studio_translations_pk_language_pk", + "columns": [ + "pk", + "language" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "kyoo.studios": { + "name": "studios", + "schema": "kyoo", + "columns": { + "pk": { + "name": "pk", + "type": "integer", + "primaryKey": true, + "notNull": true, + "identity": { + "type": "always", + "name": "studios_pk_seq", + "schema": "kyoo", + "increment": "1", + "startWith": "1", + "minValue": "1", + "maxValue": "2147483647", + "cache": "1", + "cycle": false + } + }, + "id": { + "name": "id", + "type": "uuid", + "primaryKey": false, + "notNull": true, + "default": "gen_random_uuid()" + }, + "slug": { + "name": "slug", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "external_id": { + "name": "external_id", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "studios_id_unique": { + "name": "studios_id_unique", + "nullsNotDistinct": false, + "columns": [ + "id" + ] + }, + "studios_slug_unique": { + "name": "studios_slug_unique", + "nullsNotDistinct": false, + "columns": [ + "slug" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "kyoo.entry_video_join": { + "name": "entry_video_join", + "schema": "kyoo", + "columns": { + "entry_pk": { + "name": "entry_pk", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "video_pk": { + "name": "video_pk", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "slug": { + "name": "slug", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "entry_video_join_entry_pk_entries_pk_fk": { + "name": "entry_video_join_entry_pk_entries_pk_fk", + "tableFrom": "entry_video_join", + "tableTo": "entries", + "schemaTo": "kyoo", + "columnsFrom": [ + "entry_pk" + ], + "columnsTo": [ + "pk" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "entry_video_join_video_pk_videos_pk_fk": { + "name": "entry_video_join_video_pk_videos_pk_fk", + "tableFrom": "entry_video_join", + "tableTo": "videos", + "schemaTo": "kyoo", + "columnsFrom": [ + "video_pk" + ], + "columnsTo": [ + "pk" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "entry_video_join_entry_pk_video_pk_pk": { + "name": "entry_video_join_entry_pk_video_pk_pk", + "columns": [ + "entry_pk", + "video_pk" + ] + } + }, + "uniqueConstraints": { + "entry_video_join_slug_unique": { + "name": "entry_video_join_slug_unique", + "nullsNotDistinct": false, + "columns": [ + "slug" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "kyoo.videos": { + "name": "videos", + "schema": "kyoo", + "columns": { + "pk": { + "name": "pk", + "type": "integer", + "primaryKey": true, + "notNull": true, + "identity": { + "type": "always", + "name": "videos_pk_seq", + "schema": "kyoo", + "increment": "1", + "startWith": "1", + "minValue": "1", + "maxValue": "2147483647", + "cache": "1", + "cycle": false + } + }, + "id": { + "name": "id", + "type": "uuid", + "primaryKey": false, + "notNull": true, + "default": "gen_random_uuid()" + }, + "path": { + "name": "path", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "rendering": { + "name": "rendering", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "part": { + "name": "part", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "version": { + "name": "version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "guess": { + "name": "guess", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "videos_id_unique": { + "name": "videos_id_unique", + "nullsNotDistinct": false, + "columns": [ + "id" + ] + }, + "videos_path_unique": { + "name": "videos_path_unique", + "nullsNotDistinct": false, + "columns": [ + "path" + ] + }, + "rendering_unique": { + "name": "rendering_unique", + "nullsNotDistinct": true, + "columns": [ + "rendering", + "part", + "version" + ] + } + }, + "policies": {}, + "checkConstraints": { + "part_pos": { + "name": "part_pos", + "value": "\"kyoo\".\"videos\".\"part\" >= 0" + }, + "version_pos": { + "name": "version_pos", + "value": "\"kyoo\".\"videos\".\"version\" >= 0" + } + }, + "isRLSEnabled": false + }, + "kyoo.watchlist": { + "name": "watchlist", + "schema": "kyoo", + "columns": { + "profile_pk": { + "name": "profile_pk", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "show_pk": { + "name": "show_pk", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "watchlist_status", + "typeSchema": "kyoo", + "primaryKey": false, + "notNull": true + }, + "seen_count": { + "name": "seen_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "next_entry": { + "name": "next_entry", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "score": { + "name": "score", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "started_at": { + "name": "started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_played_at": { + "name": "last_played_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "watchlist_profile_pk_profiles_pk_fk": { + "name": "watchlist_profile_pk_profiles_pk_fk", + "tableFrom": "watchlist", + "tableTo": "profiles", + "schemaTo": "kyoo", + "columnsFrom": [ + "profile_pk" + ], + "columnsTo": [ + "pk" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "watchlist_show_pk_shows_pk_fk": { + "name": "watchlist_show_pk_shows_pk_fk", + "tableFrom": "watchlist", + "tableTo": "shows", + "schemaTo": "kyoo", + "columnsFrom": [ + "show_pk" + ], + "columnsTo": [ + "pk" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "watchlist_next_entry_entries_pk_fk": { + "name": "watchlist_next_entry_entries_pk_fk", + "tableFrom": "watchlist", + "tableTo": "entries", + "schemaTo": "kyoo", + "columnsFrom": [ + "next_entry" + ], + "columnsTo": [ + "pk" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "watchlist_profile_pk_show_pk_pk": { + "name": "watchlist_profile_pk_show_pk_pk", + "columns": [ + "profile_pk", + "show_pk" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "score_percent": { + "name": "score_percent", + "value": "\"kyoo\".\"watchlist\".\"score\" between 0 and 100" + } + }, + "isRLSEnabled": false + } + }, + "enums": { + "kyoo.entry_type": { + "name": "entry_type", + "schema": "kyoo", + "values": [ + "episode", + "movie", + "special", + "extra" + ] + }, + "kyoo.genres": { + "name": "genres", + "schema": "kyoo", + "values": [ + "action", + "adventure", + "animation", + "comedy", + "crime", + "documentary", + "drama", + "family", + "fantasy", + "history", + "horror", + "music", + "mystery", + "romance", + "science-fiction", + "thriller", + "war", + "western", + "kids", + "reality", + "politics", + "soap", + "talk" + ] + }, + "kyoo.show_kind": { + "name": "show_kind", + "schema": "kyoo", + "values": [ + "serie", + "movie", + "collection" + ] + }, + "kyoo.show_status": { + "name": "show_status", + "schema": "kyoo", + "values": [ + "unknown", + "finished", + "airing", + "planned" + ] + }, + "kyoo.role_kind": { + "name": "role_kind", + "schema": "kyoo", + "values": [ + "actor", + "director", + "writter", + "producer", + "music", + "crew", + "other" + ] + }, + "kyoo.watchlist_status": { + "name": "watchlist_status", + "schema": "kyoo", + "values": [ + "watching", + "rewatching", + "completed", + "dropped", + "planned" + ] + } + }, + "schemas": { + "kyoo": "kyoo" + }, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} diff --git a/api/drizzle/meta/_journal.json b/api/drizzle/meta/_journal.json index cde3d2b7..b29a0679 100644 --- a/api/drizzle/meta/_journal.json +++ b/api/drizzle/meta/_journal.json @@ -162,6 +162,13 @@ "when": 1752446736231, "tag": "0022_seasons-count", "breakpoints": true + }, + { + "idx": 23, + "version": "7", + "when": 1763924097229, + "tag": "0023_mqueue-priority", + "breakpoints": true } ] } diff --git a/api/package.json b/api/package.json index a056b893..bb910af3 100644 --- a/api/package.json +++ b/api/package.json @@ -9,7 +9,7 @@ "format": "biome check --write ." }, "dependencies": { - "@elysiajs/opentelemetry": "^1.4.7", + "@elysiajs/opentelemetry": "^1.4.8", "@elysiajs/swagger": "zoriya/elysia-swagger#build", "@kubiks/otel-drizzle": "zoriya/drizzle-otel#build", "@types/bun": "^1.3.1", diff --git a/api/src/controllers/seed/images.ts b/api/src/controllers/seed/images.ts index 106047b8..7bfdc9d2 100644 --- a/api/src/controllers/seed/images.ts +++ b/api/src/controllers/seed/images.ts @@ -15,7 +15,7 @@ import { getFile } from "~/utils"; export const imageDir = process.env.IMAGES_PATH ?? "/images"; export const defaultBlurhash = "000000"; -type ImageTask = { +export type ImageTask = { id: string; url: string; table: string; @@ -25,12 +25,12 @@ type ImageTask = { // this will only push a task to the image downloader service and not download it instantly. // this is both done to prevent too many requests to be sent at once and to make sure POST // requests are not blocked by image downloading or blurhash calculation -export const enqueueOptImage = async ( - tx: Transaction, +export const enqueueOptImage = ( + imgQueue: ImageTask[], img: | { url: string | null; column: PgColumn } | { url: string | null; table: PgTable; column: SQL }, -): Promise => { +): Image | null => { if (!img.url) return null; const hasher = new Bun.CryptoHasher("sha256"); @@ -66,11 +66,8 @@ export const enqueueOptImage = async ( table: db.dialect.sqlToQuery(sql`${img.column.table}`).sql, column: sql.identifier(img.column.name).value, }; - await tx.insert(mqueue).values({ - kind: "image", - message, - }); - await tx.execute(sql`notify kyoo_image`); + + imgQueue.push(message); return { id, @@ -79,6 +76,20 @@ export const enqueueOptImage = async ( }; }; +export const flushImageQueue = async ( + tx: Transaction, + imgQueue: ImageTask[], + priority: number, +) => { + if (!imgQueue.length) return; + record("enqueue images", async () => { + await tx + .insert(mqueue) + .values(imgQueue.map((x) => ({ kind: "image", message: x, priority }))); + await tx.execute(sql`notify kyoo_image`); + }); +}; + export const processImages = async () => { return record("download images", async () => { let running = false; @@ -114,7 +125,7 @@ async function processOne() { .from(mqueue) .for("update", { skipLocked: true }) .where(and(eq(mqueue.kind, "image"), lt(mqueue.attempt, 5))) - .orderBy(mqueue.attempt, mqueue.createdAt) + .orderBy(mqueue.priority, mqueue.attempt, mqueue.createdAt) .limit(1); if (!item) return false; diff --git a/api/src/controllers/seed/insert/collection.ts b/api/src/controllers/seed/insert/collection.ts index 024c3307..0b1a9b9e 100644 --- a/api/src/controllers/seed/insert/collection.ts +++ b/api/src/controllers/seed/insert/collection.ts @@ -5,7 +5,7 @@ import { conflictUpdateAllExcept } from "~/db/utils"; import type { SeedCollection } from "~/models/collections"; import type { SeedMovie } from "~/models/movie"; import type { SeedSerie } from "~/models/serie"; -import { enqueueOptImage } from "../images"; +import { enqueueOptImage, flushImageQueue, type ImageTask } from "../images"; type ShowTrans = typeof showTranslations.$inferInsert; @@ -19,6 +19,7 @@ export const insertCollection = async ( const { translations, ...col } = collection; return await db.transaction(async (tx) => { + const imgQueue: ImageTask[] = []; const [ret] = await tx .insert(shows) .values({ @@ -48,29 +49,30 @@ export const insertCollection = async ( }) .returning({ pk: shows.pk, id: shows.id, slug: shows.slug }); - const trans: ShowTrans[] = await Promise.all( - Object.entries(translations).map(async ([lang, tr]) => ({ + const trans: ShowTrans[] = Object.entries(translations).map( + ([lang, tr]) => ({ pk: ret.pk, language: lang, ...tr, - poster: await enqueueOptImage(tx, { + poster: enqueueOptImage(imgQueue, { url: tr.poster, column: showTranslations.poster, }), - thumbnail: await enqueueOptImage(tx, { + thumbnail: enqueueOptImage(imgQueue, { url: tr.thumbnail, column: showTranslations.thumbnail, }), - logo: await enqueueOptImage(tx, { + logo: enqueueOptImage(imgQueue, { url: tr.logo, column: showTranslations.logo, }), - banner: await enqueueOptImage(tx, { + banner: enqueueOptImage(imgQueue, { url: tr.banner, column: showTranslations.banner, }), - })), + }), ); + await flushImageQueue(tx, imgQueue, 100); await tx .insert(showTranslations) .values(trans) diff --git a/api/src/controllers/seed/insert/entries.ts b/api/src/controllers/seed/insert/entries.ts index f6c0561c..c53590c4 100644 --- a/api/src/controllers/seed/insert/entries.ts +++ b/api/src/controllers/seed/insert/entries.ts @@ -8,7 +8,7 @@ import { } from "~/db/schema"; import { conflictUpdateAllExcept, values } from "~/db/utils"; import type { SeedEntry as SEntry, SeedExtra as SExtra } from "~/models/entry"; -import { enqueueOptImage } from "../images"; +import { enqueueOptImage, flushImageQueue, type ImageTask } from "../images"; import { guessNextRefresh } from "../refresh"; import { updateAvailableCount, updateAvailableSince } from "./shows"; @@ -50,30 +50,29 @@ export const insertEntries = async ( if (!items.length) return []; const retEntries = await db.transaction(async (tx) => { - const vals: EntryI[] = await Promise.all( - items.map(async (seed) => { - const { translations, videos, video, ...entry } = seed; - return { - ...entry, - showPk: show.pk, - slug: generateSlug(show.slug, seed), - thumbnail: await enqueueOptImage(tx, { - url: seed.thumbnail, - column: entries.thumbnail, - }), - nextRefresh: - entry.kind !== "extra" - ? guessNextRefresh(entry.airDate ?? new Date()) - : guessNextRefresh(new Date()), - episodeNumber: - entry.kind === "episode" - ? entry.episodeNumber - : entry.kind === "special" - ? entry.number - : undefined, - }; - }), - ); + const imgQueue: ImageTask[] = []; + const vals: EntryI[] = items.map((seed) => { + const { translations, videos, video, ...entry } = seed; + return { + ...entry, + showPk: show.pk, + slug: generateSlug(show.slug, seed), + thumbnail: enqueueOptImage(imgQueue, { + url: seed.thumbnail, + column: entries.thumbnail, + }), + nextRefresh: + entry.kind !== "extra" + ? guessNextRefresh(entry.airDate ?? new Date()) + : guessNextRefresh(new Date()), + episodeNumber: + entry.kind === "episode" + ? entry.episodeNumber + : entry.kind === "special" + ? entry.number + : undefined, + }; + }); const ret = await tx .insert(entries) .values(vals) @@ -89,41 +88,36 @@ export const insertEntries = async ( }) .returning({ pk: entries.pk, id: entries.id, slug: entries.slug }); - const trans: EntryTransI[] = ( - await Promise.all( - items.map(async (seed, i) => { - if (seed.kind === "extra") { - return [ - { - pk: ret[i].pk, - // yeah we hardcode the language to extra because if we want to support - // translations one day it won't be awkward - language: "extra", - name: seed.name, - description: null, - poster: undefined, - }, - ]; - } + const trans: EntryTransI[] = items.flatMap((seed, i) => { + if (seed.kind === "extra") { + return [ + { + pk: ret[i].pk, + // yeah we hardcode the language to extra because if we want to support + // translations one day it won't be awkward + language: "extra", + name: seed.name, + description: null, + poster: undefined, + }, + ]; + } - return await Promise.all( - Object.entries(seed.translations).map(async ([lang, tr]) => ({ - // assumes ret is ordered like items. - pk: ret[i].pk, - language: lang, - ...tr, - poster: - seed.kind === "movie" - ? await enqueueOptImage(tx, { - url: (tr as any).poster, - column: entryTranslations.poster, - }) - : undefined, - })), - ); - }), - ) - ).flat(); + return Object.entries(seed.translations).map(([lang, tr]) => ({ + // assumes ret is ordered like items. + pk: ret[i].pk, + language: lang, + ...tr, + poster: + seed.kind === "movie" + ? enqueueOptImage(imgQueue, { + url: (tr as any).poster, + column: entryTranslations.poster, + }) + : undefined, + })); + }); + await flushImageQueue(tx, imgQueue, 0); await tx .insert(entryTranslations) .values(trans) diff --git a/api/src/controllers/seed/insert/seasons.ts b/api/src/controllers/seed/insert/seasons.ts index 04342c8f..c0520da4 100644 --- a/api/src/controllers/seed/insert/seasons.ts +++ b/api/src/controllers/seed/insert/seasons.ts @@ -2,7 +2,7 @@ import { db } from "~/db"; import { seasons, seasonTranslations } from "~/db/schema"; import { conflictUpdateAllExcept } from "~/db/utils"; import type { SeedSeason } from "~/models/season"; -import { enqueueOptImage } from "../images"; +import { enqueueOptImage, flushImageQueue, type ImageTask } from "../images"; import { guessNextRefresh } from "../refresh"; type SeasonI = typeof seasons.$inferInsert; @@ -15,6 +15,7 @@ export const insertSeasons = async ( if (!items.length) return []; return db.transaction(async (tx) => { + const imgQueue: ImageTask[] = []; const vals: SeasonI[] = items.map((x) => { const { translations, ...season } = x; return { @@ -42,33 +43,27 @@ export const insertSeasons = async ( }) .returning({ pk: seasons.pk, id: seasons.id, slug: seasons.slug }); - const trans: SeasonTransI[] = ( - await Promise.all( - items.map( - async (seed, i) => - await Promise.all( - Object.entries(seed.translations).map(async ([lang, tr]) => ({ - // assumes ret is ordered like items. - pk: ret[i].pk, - language: lang, - ...tr, - poster: await enqueueOptImage(tx, { - url: tr.poster, - column: seasonTranslations.poster, - }), - thumbnail: await enqueueOptImage(tx, { - url: tr.thumbnail, - column: seasonTranslations.thumbnail, - }), - banner: await enqueueOptImage(tx, { - url: tr.banner, - column: seasonTranslations.banner, - }), - })), - ), - ), - ) - ).flat(); + const trans: SeasonTransI[] = items.flatMap((seed, i) => + Object.entries(seed.translations).map(([lang, tr]) => ({ + // assumes ret is ordered like items. + pk: ret[i].pk, + language: lang, + ...tr, + poster: enqueueOptImage(imgQueue, { + url: tr.poster, + column: seasonTranslations.poster, + }), + thumbnail: enqueueOptImage(imgQueue, { + url: tr.thumbnail, + column: seasonTranslations.thumbnail, + }), + banner: enqueueOptImage(imgQueue, { + url: tr.banner, + column: seasonTranslations.banner, + }), + })), + ); + await flushImageQueue(tx, imgQueue, -10); await tx .insert(seasonTranslations) .values(trans) diff --git a/api/src/controllers/seed/insert/shows.ts b/api/src/controllers/seed/insert/shows.ts index ac6dda37..88cc8eab 100644 --- a/api/src/controllers/seed/insert/shows.ts +++ b/api/src/controllers/seed/insert/shows.ts @@ -22,7 +22,7 @@ import type { SeedMovie } from "~/models/movie"; import type { SeedSerie } from "~/models/serie"; import type { Original } from "~/models/utils"; import { getYear } from "~/utils"; -import { enqueueOptImage } from "../images"; +import { enqueueOptImage, flushImageQueue, type ImageTask } from "../images"; type Show = typeof shows.$inferInsert; type ShowTrans = typeof showTranslations.$inferInsert; @@ -41,24 +41,25 @@ export const insertShow = async ( | SeedCollection["translations"], ) => { return await db.transaction(async (tx) => { + const imgQueue: ImageTask[] = []; const orig = { ...original, - poster: await enqueueOptImage(tx, { + poster: enqueueOptImage(imgQueue, { url: original.poster, table: shows, column: sql`${shows.original}['poster']`, }), - thumbnail: await enqueueOptImage(tx, { + thumbnail: enqueueOptImage(imgQueue, { url: original.thumbnail, table: shows, column: sql`${shows.original}['thumbnail']`, }), - banner: await enqueueOptImage(tx, { + banner: enqueueOptImage(imgQueue, { url: original.banner, table: shows, column: sql`${shows.original}['banner']`, }), - logo: await enqueueOptImage(tx, { + logo: enqueueOptImage(imgQueue, { url: original.logo, table: shows, column: sql`${shows.original}['logo']`, @@ -67,30 +68,31 @@ export const insertShow = async ( const ret = await insertBaseShow(tx, { ...show, original: orig }); if ("status" in ret) return ret; - const trans: ShowTrans[] = await Promise.all( - Object.entries(translations).map(async ([lang, tr]) => ({ + const trans: ShowTrans[] = Object.entries(translations).map( + ([lang, tr]) => ({ pk: ret.pk, language: lang, ...tr, latinName: tr.latinName ?? null, - poster: await enqueueOptImage(tx, { + poster: enqueueOptImage(imgQueue, { url: tr.poster, column: showTranslations.poster, }), - thumbnail: await enqueueOptImage(tx, { + thumbnail: enqueueOptImage(imgQueue, { url: tr.thumbnail, column: showTranslations.thumbnail, }), - logo: await enqueueOptImage(tx, { + logo: enqueueOptImage(imgQueue, { url: tr.logo, column: showTranslations.logo, }), - banner: await enqueueOptImage(tx, { + banner: enqueueOptImage(imgQueue, { url: tr.banner, column: showTranslations.banner, }), - })), + }), ); + await flushImageQueue(tx, imgQueue, 200); await tx .insert(showTranslations) .values(trans) diff --git a/api/src/controllers/seed/insert/staff.ts b/api/src/controllers/seed/insert/staff.ts index 50b15467..11cd689b 100644 --- a/api/src/controllers/seed/insert/staff.ts +++ b/api/src/controllers/seed/insert/staff.ts @@ -3,7 +3,7 @@ import { db } from "~/db"; import { roles, staff } from "~/db/schema"; import { conflictUpdateAllExcept } from "~/db/utils"; import type { SeedStaff } from "~/models/staff"; -import { enqueueOptImage } from "../images"; +import { enqueueOptImage, flushImageQueue, type ImageTask } from "../images"; export const insertStaff = async ( seed: SeedStaff[] | undefined, @@ -12,15 +12,14 @@ export const insertStaff = async ( if (!seed?.length) return []; return await db.transaction(async (tx) => { - const people = await Promise.all( - seed.map(async (x) => ({ - ...x.staff, - image: await enqueueOptImage(tx, { - url: x.staff.image, - column: staff.image, - }), - })), - ); + const imgQueue: ImageTask[] = []; + const people = seed.map((x) => ({ + ...x.staff, + image: enqueueOptImage(imgQueue, { + url: x.staff.image, + column: staff.image, + }), + })); const ret = await tx .insert(staff) .values(people) @@ -30,22 +29,22 @@ export const insertStaff = async ( }) .returning({ pk: staff.pk, id: staff.id, slug: staff.slug }); - const rval = await Promise.all( - seed.map(async (x, i) => ({ - showPk, - staffPk: ret[i].pk, - kind: x.kind, - order: i, - character: { - ...x.character, - image: await enqueueOptImage(tx, { - url: x.character.image, - table: roles, - column: sql`${roles.character}['image']`, - }), - }, - })), - ); + const rval = seed.map((x, i) => ({ + showPk, + staffPk: ret[i].pk, + kind: x.kind, + order: i, + character: { + ...x.character, + image: enqueueOptImage(imgQueue, { + url: x.character.image, + table: roles, + column: sql`${roles.character}['image']`, + }), + }, + })); + + await flushImageQueue(tx, imgQueue, -200); // always replace all roles. this is because: // - we want `order` to stay in sync (& without duplicates) diff --git a/api/src/controllers/seed/insert/studios.ts b/api/src/controllers/seed/insert/studios.ts index e8a856c9..0c52b2c5 100644 --- a/api/src/controllers/seed/insert/studios.ts +++ b/api/src/controllers/seed/insert/studios.ts @@ -2,7 +2,7 @@ import { db } from "~/db"; import { showStudioJoin, studios, studioTranslations } from "~/db/schema"; import { conflictUpdateAllExcept } from "~/db/utils"; import type { SeedStudio } from "~/models/studio"; -import { enqueueOptImage } from "../images"; +import { enqueueOptImage, flushImageQueue, ImageTask } from "../images"; type StudioI = typeof studios.$inferInsert; type StudioTransI = typeof studioTranslations.$inferInsert; @@ -33,24 +33,19 @@ export const insertStudios = async ( }) .returning({ pk: studios.pk, id: studios.id, slug: studios.slug }); - const trans: StudioTransI[] = ( - await Promise.all( - seed.map( - async (x, i) => - await Promise.all( - Object.entries(x.translations).map(async ([lang, tr]) => ({ - pk: ret[i].pk, - language: lang, - name: tr.name, - logo: await enqueueOptImage(tx, { - url: tr.logo, - column: studioTranslations.logo, - }), - })), - ), - ), - ) - ).flat(); + const imgQueue: ImageTask[] = []; + const trans: StudioTransI[] = seed.flatMap((x, i) => + Object.entries(x.translations).map(([lang, tr]) => ({ + pk: ret[i].pk, + language: lang, + name: tr.name, + logo: enqueueOptImage(imgQueue, { + url: tr.logo, + column: studioTranslations.logo, + }), + })), + ); + await flushImageQueue(tx, imgQueue, -100); await tx .insert(studioTranslations) .values(trans) diff --git a/api/src/db/index.ts b/api/src/db/index.ts index d19588ec..ae67439f 100644 --- a/api/src/db/index.ts +++ b/api/src/db/index.ts @@ -8,18 +8,18 @@ import { migrate as migrateDb } from "drizzle-orm/node-postgres/migrator"; import type { PoolConfig } from "pg"; import * as schema from "./schema"; -async function getPostgresConfig(): Promise { - const config: PoolConfig = { - connectionString: process.env.POSTGRES_URL, - host: process.env.PGHOST ?? "postgres", - port: Number(process.env.PGPORT) || 5432, - database: process.env.PGDATABASE ?? "kyoo", - user: process.env.PGUSER ?? "kyoo", - password: process.env.PGPASSWORD ?? "password", - options: process.env.PGOPTIONS, - application_name: process.env.PGAPPNAME ?? "kyoo", - }; +const config: PoolConfig = { + connectionString: process.env.POSTGRES_URL, + host: process.env.PGHOST ?? "postgres", + port: Number(process.env.PGPORT) || 5432, + database: process.env.PGDATABASE ?? "kyoo", + user: process.env.PGUSER ?? "kyoo", + password: process.env.PGPASSWORD ?? "password", + options: process.env.PGOPTIONS, + application_name: process.env.PGAPPNAME ?? "kyoo", +}; +async function parseSslConfig(): Promise { // Due to an upstream bug, if `ssl` is not falsey, an SSL connection will always be attempted. This means // that non-SSL connection options under `ssl` (which is incorrectly named) cannot be set unless SSL is enabled. if (!process.env.PGSSLMODE || process.env.PGSSLMODE === "disable") @@ -108,7 +108,9 @@ async function getPostgresConfig(): Promise { return config; } -const postgresConfig = await getPostgresConfig(); +const postgresConfig = await parseSslConfig(); +// use this when using drizzle-kit since it can't parse await statements +// const postgresConfig = config; export const db = drizzle({ schema, diff --git a/api/src/db/schema/entries.ts b/api/src/db/schema/entries.ts index 5425367d..1cd05de7 100644 --- a/api/src/db/schema/entries.ts +++ b/api/src/db/schema/entries.ts @@ -12,9 +12,8 @@ import { uuid, varchar, } from "drizzle-orm/pg-core"; -import { timestamp } from "../utils"; import { shows } from "./shows"; -import { image, language, schema } from "./utils"; +import { image, language, schema, timestamp } from "./utils"; import { entryVideoJoin } from "./videos"; export const entryType = schema.enum("entry_type", [ diff --git a/api/src/db/schema/history.ts b/api/src/db/schema/history.ts index de2a4420..a2c6f685 100644 --- a/api/src/db/schema/history.ts +++ b/api/src/db/schema/history.ts @@ -1,9 +1,8 @@ import { sql } from "drizzle-orm"; import { check, index, integer } from "drizzle-orm/pg-core"; -import { timestamp } from "../utils"; import { entries } from "./entries"; import { profiles } from "./profiles"; -import { schema } from "./utils"; +import { schema, timestamp } from "./utils"; import { videos } from "./videos"; export const history = schema.table( diff --git a/api/src/db/schema/mqueue.ts b/api/src/db/schema/mqueue.ts index 676bcc5e..c2498146 100644 --- a/api/src/db/schema/mqueue.ts +++ b/api/src/db/schema/mqueue.ts @@ -1,7 +1,6 @@ import { sql } from "drizzle-orm"; import { index, integer, jsonb, uuid, varchar } from "drizzle-orm/pg-core"; -import { timestamp } from "../utils"; -import { schema } from "./utils"; +import { schema, timestamp } from "./utils"; export const mqueue = schema.table( "mqueue", @@ -9,6 +8,7 @@ export const mqueue = schema.table( id: uuid().notNull().primaryKey().defaultRandom(), kind: varchar({ length: 255 }).notNull(), message: jsonb().notNull(), + priority: integer().notNull().default(0), attempt: integer().notNull().default(0), createdAt: timestamp({ withTimezone: true, mode: "iso" }) .notNull() diff --git a/api/src/db/schema/seasons.ts b/api/src/db/schema/seasons.ts index 111ffb0d..6121c57e 100644 --- a/api/src/db/schema/seasons.ts +++ b/api/src/db/schema/seasons.ts @@ -10,9 +10,8 @@ import { uuid, varchar, } from "drizzle-orm/pg-core"; -import { timestamp } from "../utils"; import { shows } from "./shows"; -import { image, language, schema } from "./utils"; +import { image, language, schema, timestamp } from "./utils"; export const season_extid = () => jsonb() diff --git a/api/src/db/schema/shows.ts b/api/src/db/schema/shows.ts index 697e8ba3..2fbf466a 100644 --- a/api/src/db/schema/shows.ts +++ b/api/src/db/schema/shows.ts @@ -13,12 +13,11 @@ import { varchar, } from "drizzle-orm/pg-core"; import type { Image, Original } from "~/models/utils"; -import { timestamp } from "../utils"; import { entries } from "./entries"; import { seasons } from "./seasons"; import { roles } from "./staff"; import { showStudioJoin } from "./studios"; -import { externalid, image, language, schema } from "./utils"; +import { externalid, image, language, schema, timestamp } from "./utils"; export const showKind = schema.enum("show_kind", [ "serie", diff --git a/api/src/db/schema/staff.ts b/api/src/db/schema/staff.ts index 375e2fdb..469bb516 100644 --- a/api/src/db/schema/staff.ts +++ b/api/src/db/schema/staff.ts @@ -8,9 +8,8 @@ import { varchar, } from "drizzle-orm/pg-core"; import type { Character } from "~/models/staff"; -import { timestamp } from "../utils"; import { shows } from "./shows"; -import { externalid, image, schema } from "./utils"; +import { externalid, image, schema, timestamp } from "./utils"; export const roleKind = schema.enum("role_kind", [ "actor", diff --git a/api/src/db/schema/studios.ts b/api/src/db/schema/studios.ts index cb4f0f9c..ff32323e 100644 --- a/api/src/db/schema/studios.ts +++ b/api/src/db/schema/studios.ts @@ -7,9 +7,8 @@ import { uuid, varchar, } from "drizzle-orm/pg-core"; -import { timestamp } from "../utils"; import { shows } from "./shows"; -import { externalid, image, language, schema } from "./utils"; +import { externalid, image, language, schema, timestamp } from "./utils"; export const studios = schema.table("studios", { pk: integer().primaryKey().generatedAlwaysAsIdentity(), diff --git a/api/src/db/schema/utils.ts b/api/src/db/schema/utils.ts index 3f6e92e1..ae113bfc 100644 --- a/api/src/db/schema/utils.ts +++ b/api/src/db/schema/utils.ts @@ -1,4 +1,4 @@ -import { jsonb, pgSchema, varchar } from "drizzle-orm/pg-core"; +import { customType, jsonb, pgSchema, varchar } from "drizzle-orm/pg-core"; import type { Image } from "~/models/utils"; export const schema = pgSchema("kyoo"); @@ -20,3 +20,19 @@ export const externalid = () => >() .notNull() .default({}); + +export const timestamp = customType<{ + data: string; + driverData: string; + config: { withTimezone: boolean; precision?: number; mode: "iso" }; +}>({ + dataType(config) { + const precision = config?.precision ? ` (${config.precision})` : ""; + return `timestamp${precision}${config?.withTimezone ? " with time zone" : ""}`; + }, + fromDriver(value: string): string { + // postgres format: 2025-06-22 16:13:37.489301+00 + // what we want: 2025-06-22T16:13:37Z + return `${value.substring(0, 10)}T${value.substring(11, 19)}Z`; + }, +}); diff --git a/api/src/db/schema/videos.ts b/api/src/db/schema/videos.ts index f5f18340..045e07c4 100644 --- a/api/src/db/schema/videos.ts +++ b/api/src/db/schema/videos.ts @@ -10,9 +10,8 @@ import { varchar, } from "drizzle-orm/pg-core"; import type { Guess } from "~/models/video"; -import { timestamp } from "../utils"; import { entries } from "./entries"; -import { schema } from "./utils"; +import { schema, timestamp } from "./utils"; export const videos = schema.table( "videos", diff --git a/api/src/db/schema/watchlist.ts b/api/src/db/schema/watchlist.ts index 4c8e59d6..48e699f1 100644 --- a/api/src/db/schema/watchlist.ts +++ b/api/src/db/schema/watchlist.ts @@ -1,10 +1,9 @@ import { sql } from "drizzle-orm"; import { check, integer, primaryKey } from "drizzle-orm/pg-core"; -import { timestamp } from "../utils"; import { entries } from "./entries"; import { profiles } from "./profiles"; import { shows } from "./shows"; -import { schema } from "./utils"; +import { schema, timestamp } from "./utils"; export const watchlistStatus = schema.enum("watchlist_status", [ "watching", diff --git a/api/src/db/utils.ts b/api/src/db/utils.ts index e255de2f..bab0eedf 100644 --- a/api/src/db/utils.ts +++ b/api/src/db/utils.ts @@ -13,11 +13,7 @@ import { } from "drizzle-orm"; import type { CasingCache } from "drizzle-orm/casing"; import type { AnyMySqlSelect } from "drizzle-orm/mysql-core"; -import { - type AnyPgSelect, - customType, - type SelectedFieldsFlat, -} from "drizzle-orm/pg-core"; +import type { AnyPgSelect, SelectedFieldsFlat } from "drizzle-orm/pg-core"; import type { AnySQLiteSelect } from "drizzle-orm/sqlite-core"; import type { WithSubquery } from "drizzle-orm/subquery"; import { db } from "./index"; @@ -157,19 +153,3 @@ export const isUniqueConstraint = (e: unknown): boolean => { cause.code === "23505" ); }; - -export const timestamp = customType<{ - data: string; - driverData: string; - config: { withTimezone: boolean; precision?: number; mode: "iso" }; -}>({ - dataType(config) { - const precision = config?.precision ? ` (${config.precision})` : ""; - return `timestamp${precision}${config?.withTimezone ? " with time zone" : ""}`; - }, - fromDriver(value: string): string { - // postgres format: 2025-06-22 16:13:37.489301+00 - // what we want: 2025-06-22T16:13:37Z - return `${value.substring(0, 10)}T${value.substring(11, 19)}Z`; - }, -});