diff --git a/api/src/controllers/seed/images.ts b/api/src/controllers/seed/images.ts index 18e5faa2..9bce15e1 100644 --- a/api/src/controllers/seed/images.ts +++ b/api/src/controllers/seed/images.ts @@ -2,10 +2,11 @@ import { mkdir, writeFile } from "node:fs/promises"; import path from "node:path"; import { encode } from "blurhash"; import { eq, sql } from "drizzle-orm"; +import type { PgColumn } from "drizzle-orm/pg-core"; import { version } from "package.json"; import type { PoolClient } from "pg"; import sharp from "sharp"; -import { db } from "~/db"; +import { type Transaction, db } from "~/db"; import * as schema from "~/db/schema"; import { mqueue } from "~/db/schema/queue"; import type { Image } from "~/models/utils"; @@ -20,18 +21,31 @@ type ImageTask = { column: string; }; +type ImageTaskC = { + url: string; + column: PgColumn; +}; + // this will only push a task to the image downloader service and not download it instantly. // this is both done to prevent to many requests to be sent at once and to make sure POST // requests are not blocked by image downloading or blurhash calculation export const enqueueImage = async ( - tx: typeof db, - img: Omit, + tx: Transaction, + img: ImageTaskC, ): Promise => { const hasher = new Bun.CryptoHasher("sha256"); hasher.update(img.url); const id = hasher.digest().toString("hex"); - await tx.insert(mqueue).values({ kind: "image", message: { id, ...img } }); + await tx.insert(mqueue).values({ + kind: "image", + message: { + id, + url: img.url, + table: img.column.table._.name, + column: img.column.name, + } satisfies ImageTask, + }); await tx.execute(sql`notify image`); return { @@ -42,11 +56,11 @@ export const enqueueImage = async ( }; export const enqueueOptImage = async ( - tx: typeof db, - img: Omit, + tx: Transaction, + img: { url: string | null; column: PgColumn }, ): Promise => { if (!img.url) return null; - return await enqueueImage(tx, img); + return await enqueueImage(tx, { url: img.url, column: img.column }); }; export const processImages = async () => { @@ -107,6 +121,7 @@ export const processImages = async () => { }; async function downloadImage(id: string, url: string): Promise { + // TODO: check if file exists before downloading const resp = await fetch(url, { headers: { "User-Agent": `Kyoo v${version}` }, }); diff --git a/api/src/controllers/seed/insert/collection.ts b/api/src/controllers/seed/insert/collection.ts index 495a3f33..b5a1e65a 100644 --- a/api/src/controllers/seed/insert/collection.ts +++ b/api/src/controllers/seed/insert/collection.ts @@ -48,16 +48,28 @@ export const insertCollection = async ( }) .returning({ pk: shows.pk, id: shows.id, slug: shows.slug }); - const trans: ShowTrans[] = Object.entries(translations).map( - ([lang, tr]) => ({ + const trans: ShowTrans[] = await Promise.all( + Object.entries(translations).map(async ([lang, tr]) => ({ pk: ret.pk, language: lang, ...tr, - poster: enqueueOptImage(tr.poster), - thumbnail: enqueueOptImage(tr.thumbnail), - logo: enqueueOptImage(tr.logo), - banner: enqueueOptImage(tr.banner), - }), + poster: await enqueueOptImage(tx, { + url: tr.poster, + column: showTranslations.poster, + }), + thumbnail: await enqueueOptImage(tx, { + url: tr.thumbnail, + column: showTranslations.thumbnail, + }), + logo: await enqueueOptImage(tx, { + url: tr.logo, + column: showTranslations.logo, + }), + banner: await enqueueOptImage(tx, { + url: tr.banner, + column: showTranslations.banner, + }), + })), ); await tx .insert(showTranslations) diff --git a/api/src/controllers/seed/insert/entries.ts b/api/src/controllers/seed/insert/entries.ts index d57a5b4a..10ce07fa 100644 --- a/api/src/controllers/seed/insert/entries.ts +++ b/api/src/controllers/seed/insert/entries.ts @@ -23,6 +23,7 @@ type SeedExtra = Omit & { }; type EntryI = typeof entries.$inferInsert; +type EntryTransI = typeof entryTranslations.$inferInsert; const generateSlug = ( showSlug: string, @@ -49,25 +50,30 @@ export const insertEntries = async ( if (!items) return []; const retEntries = await db.transaction(async (tx) => { - const vals: EntryI[] = items.map((seed) => { - const { translations, videos, video, ...entry } = seed; - return { - ...entry, - showPk: show.pk, - slug: generateSlug(show.slug, seed), - thumbnail: enqueueOptImage(seed.thumbnail), - nextRefresh: - entry.kind !== "extra" - ? guessNextRefresh(entry.airDate ?? new Date()) - : guessNextRefresh(new Date()), - episodeNumber: - entry.kind === "episode" - ? entry.episodeNumber - : entry.kind === "special" - ? entry.number - : undefined, - }; - }); + const vals: EntryI[] = await Promise.all( + items.map(async (seed) => { + const { translations, videos, video, ...entry } = seed; + return { + ...entry, + showPk: show.pk, + slug: generateSlug(show.slug, seed), + thumbnail: await enqueueOptImage(tx, { + url: seed.thumbnail, + column: entries.thumbnail, + }), + nextRefresh: + entry.kind !== "extra" + ? guessNextRefresh(entry.airDate ?? new Date()) + : guessNextRefresh(new Date()), + episodeNumber: + entry.kind === "episode" + ? entry.episodeNumber + : entry.kind === "special" + ? entry.number + : undefined, + }; + }), + ); const ret = await tx .insert(entries) .values(vals) @@ -83,30 +89,41 @@ export const insertEntries = async ( }) .returning({ pk: entries.pk, id: entries.id, slug: entries.slug }); - const trans = items.flatMap((seed, i) => { - if (seed.kind === "extra") { - return { - pk: ret[i].pk, - // yeah we hardcode the language to extra because if we want to support - // translations one day it won't be awkward - language: "extra", - name: seed.name, - description: null, - poster: undefined, - }; - } + const trans: EntryTransI[] = ( + await Promise.all( + items.map(async (seed, i) => { + if (seed.kind === "extra") { + return [ + { + pk: ret[i].pk, + // yeah we hardcode the language to extra because if we want to support + // translations one day it won't be awkward + language: "extra", + name: seed.name, + description: null, + poster: undefined, + }, + ]; + } - return Object.entries(seed.translations).map(([lang, tr]) => ({ - // assumes ret is ordered like items. - pk: ret[i].pk, - language: lang, - ...tr, - poster: - seed.kind === "movie" - ? enqueueOptImage((tr as any).poster) - : undefined, - })); - }); + return await Promise.all( + Object.entries(seed.translations).map(async ([lang, tr]) => ({ + // assumes ret is ordered like items. + pk: ret[i].pk, + language: lang, + ...tr, + poster: + seed.kind === "movie" + ? await enqueueOptImage(tx, { + url: (tr as any).poster, + column: entryTranslations.poster, + }) + : undefined, + })), + ); + }), + ) + ).flat(); await tx .insert(entryTranslations) .values(trans) diff --git a/api/src/controllers/seed/insert/seasons.ts b/api/src/controllers/seed/insert/seasons.ts index bdb5ee39..5f43b8a0 100644 --- a/api/src/controllers/seed/insert/seasons.ts +++ b/api/src/controllers/seed/insert/seasons.ts @@ -37,17 +37,33 @@ export const insertSeasons = async ( }) .returning({ pk: seasons.pk, id: seasons.id, slug: seasons.slug }); - const trans: SeasonTransI[] = items.flatMap((seed, i) => - Object.entries(seed.translations).map(([lang, tr]) => ({ - // assumes ret is ordered like items. - pk: ret[i].pk, - language: lang, - ...tr, - poster: enqueueOptImage(tr.poster), - thumbnail: enqueueOptImage(tr.thumbnail), - banner: enqueueOptImage(tr.banner), - })), - ); + const trans: SeasonTransI[] = ( + await Promise.all( + items.map( + async (seed, i) => + await Promise.all( + Object.entries(seed.translations).map(async ([lang, tr]) => ({ + // assumes ret is ordered like items. + pk: ret[i].pk, + language: lang, + ...tr, + poster: await enqueueOptImage(tx, { + url: tr.poster, + column: seasonTranslations.poster, + }), + thumbnail: await enqueueOptImage(tx, { + url: tr.thumbnail, + column: seasonTranslations.thumbnail, + }), + banner: await enqueueOptImage(tx, { + url: tr.banner, + column: seasonTranslations.banner, + }), + })), + ), + ), + ) + ).flat(); await tx .insert(seasonTranslations) .values(trans) diff --git a/api/src/controllers/seed/insert/shows.ts b/api/src/controllers/seed/insert/shows.ts index 580738b3..f84e72eb 100644 --- a/api/src/controllers/seed/insert/shows.ts +++ b/api/src/controllers/seed/insert/shows.ts @@ -1,5 +1,5 @@ import { and, count, eq, exists, ne, sql } from "drizzle-orm"; -import { db } from "~/db"; +import { type Transaction, db } from "~/db"; import { entries, entryVideoJoin, showTranslations, shows } from "~/db/schema"; import { conflictUpdateAllExcept, sqlarr } from "~/db/utils"; import type { SeedCollection } from "~/models/collections"; @@ -12,30 +12,53 @@ type Show = typeof shows.$inferInsert; type ShowTrans = typeof showTranslations.$inferInsert; export const insertShow = async ( - show: Show, + show: Omit & { originalLanguage: string }, translations: | SeedMovie["translations"] | SeedSerie["translations"] | SeedCollection["translations"], ) => { return await db.transaction(async (tx) => { - const ret = await insertBaseShow(tx, show); + const trans: (Omit & { latinName: string | null })[] = + await Promise.all( + Object.entries(translations).map(async ([lang, tr]) => ({ + language: lang, + ...tr, + latinName: tr.latinName ?? null, + poster: await enqueueOptImage(tx, { + url: tr.poster, + column: showTranslations.poster, + }), + thumbnail: await enqueueOptImage(tx, { + url: tr.thumbnail, + column: showTranslations.thumbnail, + }), + logo: await enqueueOptImage(tx, { + url: tr.logo, + column: showTranslations.logo, + }), + banner: await enqueueOptImage(tx, { + url: tr.banner, + column: showTranslations.banner, + }), + })), + ); + const original = trans.find((x) => x.language === show.originalLanguage); + + if (!original) { + tx.rollback(); + return { + status: 422 as const, + message: "No translation available in the original language.", + }; + } + + const ret = await insertBaseShow(tx, { ...show, original }); if ("status" in ret) return ret; - const trans: ShowTrans[] = Object.entries(translations).map( - ([lang, tr]) => ({ - pk: ret.pk, - language: lang, - ...tr, - poster: enqueueOptImage(tr.poster), - thumbnail: enqueueOptImage(tr.thumbnail), - logo: enqueueOptImage(tr.logo), - banner: enqueueOptImage(tr.banner), - }), - ); await tx .insert(showTranslations) - .values(trans) + .values(trans.map((x) => ({ ...x, pk: ret.pk }))) .onConflictDoUpdate({ target: [showTranslations.pk, showTranslations.language], set: conflictUpdateAllExcept(showTranslations, ["pk", "language"]), @@ -44,10 +67,7 @@ export const insertShow = async ( }); }; -async function insertBaseShow( - tx: Parameters[0]>[0], - show: Show, -) { +async function insertBaseShow(tx: Transaction, show: Show) { function insert() { return tx .insert(shows) @@ -97,7 +117,7 @@ async function insertBaseShow( } export async function updateAvailableCount( - tx: typeof db | Parameters[0]>[0], + tx: Transaction, showPks: number[], updateEntryCount = true, ) { diff --git a/api/src/controllers/seed/insert/staff.ts b/api/src/controllers/seed/insert/staff.ts index d84092d3..8fa96d25 100644 --- a/api/src/controllers/seed/insert/staff.ts +++ b/api/src/controllers/seed/insert/staff.ts @@ -12,10 +12,15 @@ export const insertStaff = async ( if (!seed?.length) return []; return await db.transaction(async (tx) => { - const people = seed.map((x) => ({ - ...x.staff, - image: enqueueOptImage(x.staff.image), - })); + const people = await Promise.all( + seed.map(async (x) => ({ + ...x.staff, + image: await enqueueOptImage(tx, { + url: x.staff.image, + column: staff.image, + }), + })), + ); const ret = await tx .insert(staff) .values(people) @@ -25,16 +30,21 @@ export const insertStaff = async ( }) .returning({ pk: staff.pk, id: staff.id, slug: staff.slug }); - const rval = seed.map((x, i) => ({ - showPk, - staffPk: ret[i].pk, - kind: x.kind, - order: i, - character: { - ...x.character, - image: enqueueOptImage(x.character.image), - }, - })); + const rval = await Promise.all( + seed.map(async (x, i) => ({ + showPk, + staffPk: ret[i].pk, + kind: x.kind, + order: i, + character: { + ...x.character, + image: await enqueueOptImage(tx, { + url: x.character.image, + column: roles.character.image, + }), + }, + })), + ); // always replace all roles. this is because: // - we want `order` to stay in sync (& without duplicates) diff --git a/api/src/controllers/seed/insert/studios.ts b/api/src/controllers/seed/insert/studios.ts index 85d18403..7eef1bd1 100644 --- a/api/src/controllers/seed/insert/studios.ts +++ b/api/src/controllers/seed/insert/studios.ts @@ -33,14 +33,24 @@ export const insertStudios = async ( }) .returning({ pk: studios.pk, id: studios.id, slug: studios.slug }); - const trans: StudioTransI[] = seed.flatMap((x, i) => - Object.entries(x.translations).map(([lang, tr]) => ({ - pk: ret[i].pk, - language: lang, - name: tr.name, - logo: enqueueOptImage(tr.logo), - })), - ); + const trans: StudioTransI[] = ( + await Promise.all( + seed.map( + async (x, i) => + await Promise.all( + Object.entries(x.translations).map(async ([lang, tr]) => ({ + pk: ret[i].pk, + language: lang, + name: tr.name, + logo: await enqueueOptImage(tx, { + url: tr.logo, + column: studioTranslations.logo, + }), + })), + ), + ), + ) + ).flat(); await tx .insert(studioTranslations) .values(trans) diff --git a/api/src/controllers/seed/movies.ts b/api/src/controllers/seed/movies.ts index 9ab4ac85..2102b6ce 100644 --- a/api/src/controllers/seed/movies.ts +++ b/api/src/controllers/seed/movies.ts @@ -1,7 +1,6 @@ import { t } from "elysia"; import type { SeedMovie } from "~/models/movie"; import { getYear } from "~/utils"; -import { enqueueOptImage } from "./images"; import { insertCollection } from "./insert/collection"; import { insertEntries } from "./insert/entries"; import { insertShow, updateAvailableCount } from "./insert/shows"; @@ -55,13 +54,6 @@ export const seedMovie = async ( const { translations, videos, collection, studios, staff, ...movie } = seed; const nextRefresh = guessNextRefresh(movie.airDate ?? new Date()); - const original = translations[movie.originalLanguage]; - if (!original) { - return { - status: 422, - message: "No translation available in the original language.", - }; - } const col = await insertCollection(collection, { kind: "movie", @@ -76,15 +68,6 @@ export const seedMovie = async ( nextRefresh, collectionPk: col?.pk, entriesCount: 1, - original: { - language: movie.originalLanguage, - name: original.name, - latinName: original.latinName ?? null, - poster: enqueueOptImage(original.poster), - thumbnail: enqueueOptImage(original.thumbnail), - logo: enqueueOptImage(original.logo), - banner: enqueueOptImage(original.banner), - }, ...movie, }, translations, diff --git a/api/src/controllers/seed/series.ts b/api/src/controllers/seed/series.ts index 62b6d453..d207439f 100644 --- a/api/src/controllers/seed/series.ts +++ b/api/src/controllers/seed/series.ts @@ -5,7 +5,7 @@ import { enqueueOptImage } from "./images"; import { insertCollection } from "./insert/collection"; import { insertEntries } from "./insert/entries"; import { insertSeasons } from "./insert/seasons"; -import { insertShow, updateAvailableCount } from "./insert/shows"; +import { insertShow } from "./insert/shows"; import { insertStaff } from "./insert/staff"; import { insertStudios } from "./insert/studios"; import { guessNextRefresh } from "./refresh"; @@ -91,13 +91,6 @@ export const seedSerie = async ( ...serie } = seed; const nextRefresh = guessNextRefresh(serie.startAir ?? new Date()); - const original = translations[serie.originalLanguage]; - if (!original) { - return { - status: 422, - message: "No translation available in the original language.", - }; - } const col = await insertCollection(collection, { kind: "serie", @@ -111,15 +104,6 @@ export const seedSerie = async ( nextRefresh, collectionPk: col?.pk, entriesCount: entries.length, - original: { - language: serie.originalLanguage, - name: original.name, - latinName: original.latinName ?? null, - poster: enqueueOptImage(original.poster), - thumbnail: enqueueOptImage(original.thumbnail), - logo: enqueueOptImage(original.logo), - banner: enqueueOptImage(original.banner), - }, ...serie, }, translations, diff --git a/api/src/db/index.ts b/api/src/db/index.ts index 0745761f..0935a8c9 100644 --- a/api/src/db/index.ts +++ b/api/src/db/index.ts @@ -31,3 +31,7 @@ export const migrate = async () => { }); console.log(`Database ${dbConfig.database} migrated!`); }; + +export type Transaction = + | typeof db + | Parameters[0]>[0]; diff --git a/api/src/db/schema/shows.ts b/api/src/db/schema/shows.ts index 6729071f..35a75bec 100644 --- a/api/src/db/schema/shows.ts +++ b/api/src/db/schema/shows.ts @@ -58,10 +58,10 @@ export const genres = schema.enum("genres", [ ]); type OriginalWithImages = Original & { - poster: Image | null; - thumbnail: Image | null; - banner: Image | null; - logo: Image | null; + poster?: Image | null; + thumbnail?: Image | null; + banner?: Image | null; + logo?: Image | null; }; export const shows = schema.table(