Create history APIs (#881)

api/drizzle/0018_history.sql

@@ -0,0 +1,5 @@
+ALTER TABLE "kyoo"."history" ALTER COLUMN "video_pk" DROP NOT NULL;--> statement-breakpoint
+ALTER TABLE "kyoo"."watchlist" ALTER COLUMN "status" SET DATA TYPE text;--> statement-breakpoint
+DROP TYPE "kyoo"."watchlist_status";--> statement-breakpoint
+CREATE TYPE "kyoo"."watchlist_status" AS ENUM('watching', 'rewatching', 'completed', 'dropped', 'planned');--> statement-breakpoint
+ALTER TABLE "kyoo"."watchlist" ALTER COLUMN "status" SET DATA TYPE "kyoo"."watchlist_status" USING "status"::"kyoo"."watchlist_status";

api/drizzle/meta/_journal.json

@@ -127,6 +127,13 @@
 			"when": 1743944773824,
 			"tag": "0017_watchlist",
 			"breakpoints": true
+		},
+		{
+			"idx": 18,
+			"version": "7",
+			"when": 1744053556621,
+			"tag": "0018_history",
+			"breakpoints": true
 		}
 	]
 }

api/patches/drizzle-orm@0.39.0.patch

@@ -1,9 +1,90 @@
+diff --git a/node_modules/drizzle-orm/.bun-tag-3622ae30f31c0d9a b/.bun-tag-3622ae30f31c0d9a
+new file mode 100644
+index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
 diff --git a/node_modules/drizzle-orm/.bun-tag-36446a2521398ee8 b/.bun-tag-36446a2521398ee8
 new file mode 100644
 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
+diff --git a/node_modules/drizzle-orm/.bun-tag-844efc51a55b820c b/.bun-tag-844efc51a55b820c
+new file mode 100644
+index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
 diff --git a/node_modules/drizzle-orm/.bun-tag-9fae835e61d5cc75 b/.bun-tag-9fae835e61d5cc75
 new file mode 100644
 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
+diff --git a/node_modules/drizzle-orm/.bun-tag-ce8efc9a806990a3 b/.bun-tag-ce8efc9a806990a3
+new file mode 100644
+index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
+diff --git a/pg-core/dialect.cjs b/pg-core/dialect.cjs
+index 52acbfb6038fb1bbba4e34115d75a22bb0f9ab1a..1f10884caf05329ab98b06a68c8e7803e5283d32 100644
+--- a/pg-core/dialect.cjs
++++ b/pg-core/dialect.cjs
+@@ -347,7 +347,14 @@ class PgDialect {
+   buildInsertQuery({ table, values: valuesOrSelect, onConflict, returning, withList, select, overridingSystemValue_ }) {
+     const valuesSqlList = [];
+     const columns = table[import_table2.Table.Symbol.Columns];
+-    const colEntries = Object.entries(columns).filter(([_, col]) => !col.shouldDisableInsert());
++    let colEntries = Object.entries(columns);
++    colEntries = select && !is(valuesOrSelect, SQL)
++        ? Object
++            .keys(valuesOrSelect._.selectedFields)
++            .map((key) => [key, columns[key]])
++        : overridingSystemValue_
++            ? colEntries
++            : colEntries.filter(([_, col]) => !col.shouldDisableInsert());
+     const insertOrder = colEntries.map(
+       ([, column]) => import_sql2.sql.identifier(this.casing.getColumnCasing(column))
+     );
+diff --git a/pg-core/dialect.js b/pg-core/dialect.js
+index d7985c81f3d224f7671efe72e79b14153d5ca8ce..91d99ccd2ebda807a7d45c76f7164e571b922159 100644
+--- a/pg-core/dialect.js
++++ b/pg-core/dialect.js
+@@ -345,7 +345,14 @@ class PgDialect {
+   buildInsertQuery({ table, values: valuesOrSelect, onConflict, returning, withList, select, overridingSystemValue_ }) {
+     const valuesSqlList = [];
+     const columns = table[Table.Symbol.Columns];
+-    const colEntries = Object.entries(columns).filter(([_, col]) => !col.shouldDisableInsert());
++    let colEntries = Object.entries(columns);
++    colEntries = select && !is(valuesOrSelect, SQL)
++        ? Object
++            .keys(valuesOrSelect._.selectedFields)
++            .map((key) => [key, columns[key]])
++        : overridingSystemValue_
++            ? colEntries
++            : colEntries.filter(([_, col]) => !col.shouldDisableInsert());
+     const insertOrder = colEntries.map(
+       ([, column]) => sql.identifier(this.casing.getColumnCasing(column))
+     );
+diff --git a/pg-core/query-builders/insert.cjs b/pg-core/query-builders/insert.cjs
+index 08bb0d7485ebf997e3f081e2254ea8fd8bc20f65..341d2513d4377acc33ee0606d05580566fd4b88c 100644
+--- a/pg-core/query-builders/insert.cjs
++++ b/pg-core/query-builders/insert.cjs
+@@ -75,11 +75,6 @@ class PgInsertBuilder {
+   }
+   select(selectQuery) {
+     const select = typeof selectQuery === "function" ? selectQuery(new import_query_builder.QueryBuilder()) : selectQuery;
+-    if (!(0, import_entity.is)(select, import_sql.SQL) && !(0, import_utils.haveSameKeys)(this.table[import_table.Columns], select._.selectedFields)) {
+-      throw new Error(
+-        "Insert select error: selected fields are not the same or are in a different order compared to the table definition"
+-      );
+-    }
+     return new PgInsertBase(this.table, select, this.session, this.dialect, this.withList, true);
+   }
+ }
+diff --git a/pg-core/query-builders/insert.js b/pg-core/query-builders/insert.js
+index 0fc8eeb80f4a5512f6c84f3d596832623a33b748..b993f226daf16f423db012dff828d89c522603c3 100644
+--- a/pg-core/query-builders/insert.js
++++ b/pg-core/query-builders/insert.js
+@@ -51,11 +51,6 @@ class PgInsertBuilder {
+   }
+   select(selectQuery) {
+     const select = typeof selectQuery === "function" ? selectQuery(new QueryBuilder()) : selectQuery;
+-    if (!is(select, SQL) && !haveSameKeys(this.table[Columns], select._.selectedFields)) {
+-      throw new Error(
+-        "Insert select error: selected fields are not the same or are in a different order compared to the table definition"
+-      );
+-    }
+     return new PgInsertBase(this.table, select, this.session, this.dialect, this.withList, true);
+   }
+ }
 diff --git a/pg-core/query-builders/select.d.cts b/pg-core/query-builders/select.d.cts
 index b968ebb3f563f37c8c36221dd17cc6f3603270ec..3fda6d0a97997f6bd07ec6a0c83397c0fdd2e97e 100644
 --- a/pg-core/query-builders/select.d.cts

api/src/base.ts

@@ -2,6 +2,8 @@ import { Elysia, t } from "elysia";
 import { auth } from "./auth";
 import { entriesH } from "./controllers/entries";
 import { imagesH } from "./controllers/images";
+import { historyH } from "./controllers/profiles/history";
+import { watchlistH } from "./controllers/profiles/watchlist";
 import { seasonsH } from "./controllers/seasons";
 import { seed } from "./controllers/seed";
 import { collections } from "./controllers/shows/collections";
@@ -11,7 +13,6 @@ import { showsH } from "./controllers/shows/shows";
 import { staffH } from "./controllers/staff";
 import { studiosH } from "./controllers/studios";
 import { videosH } from "./controllers/videos";
-import { watchlistH } from "./controllers/watchlist";
 import type { KError } from "./models/error";
 
 export const base = new Elysia({ name: "base" })
@@ -95,4 +96,5 @@ export const app = new Elysia({ prefix })
 		},
 		(app) => app.use(videosH).use(seed),
 	)
-	.use(watchlistH);
+	.use(watchlistH)
+	.use(historyH);

api/src/controllers/entries.ts

@@ -45,7 +45,22 @@ import {
 import { desc as description } from "~/models/utils/descriptions";
 import type { EmbeddedVideo } from "~/models/video";
 
-const entryFilters: FilterDef = {
+export const entryProgressQ = db
+	.selectDistinctOn([history.entryPk], {
+		percent: history.percent,
+		time: history.time,
+		entryPk: history.entryPk,
+		playedDate: history.playedDate,
+		videoId: videos.id,
+	})
+	.from(history)
+	.leftJoin(videos, eq(history.videoPk, videos.pk))
+	.leftJoin(profiles, eq(history.profilePk, profiles.pk))
+	.where(eq(profiles.id, sql.placeholder("userId")))
+	.orderBy(history.entryPk, desc(history.playedDate))
+	.as("progress");
+
+export const entryFilters: FilterDef = {
 	kind: {
 		column: entries.kind,
 		type: "enum",
@@ -57,18 +72,21 @@ const entryFilters: FilterDef = {
 	order: { column: entries.order, type: "float" },
 	runtime: { column: entries.runtime, type: "float" },
 	airDate: { column: entries.airDate, type: "date" },
+	playedDate: { column: entryProgressQ.playedDate, type: "date" },
 };
 
 const extraFilters: FilterDef = {
 	kind: { column: entries.extraKind, type: "enum", values: ExtraType.enum },
 	runtime: { column: entries.runtime, type: "float" },
+	playedDate: { column: entryProgressQ.playedDate, type: "date" },
 };
 
 const unknownFilters: FilterDef = {
 	runtime: { column: entries.runtime, type: "float" },
+	playedDate: { column: entryProgressQ.playedDate, type: "date" },
 };
 
-const entrySort = Sort(
+export const entrySort = Sort(
 	{
 		order: entries.order,
 		seasonNumber: entries.seasonNumber,
@@ -76,6 +94,7 @@ const entrySort = Sort(
 		number: entries.episodeNumber,
 		airDate: entries.airDate,
 		nextRefresh: entries.nextRefresh,
+		playedDate: entryProgressQ.playedDate,
 	},
 	{
 		default: ["order"],
@@ -89,6 +108,7 @@ const extraSort = Sort(
 		name: entryTranslations.name,
 		runtime: entries.runtime,
 		createdAt: entries.createdAt,
+		playedDate: entryProgressQ.playedDate,
 	},
 	{
 		default: ["slug"],
@@ -126,36 +146,19 @@ export const entryVideosQ = db
 	.leftJoin(videos, eq(videos.pk, entryVideoJoin.videoPk))
 	.as("videos");
 
-export const getEntryProgressQ = (userId: string) =>
+export const mapProgress = ({ aliased }: { aliased: boolean }) => {
-	db
+	const { time, percent, playedDate, videoId } = getColumns(entryProgressQ);
-		.selectDistinctOn([history.entryPk], {
-			percent: history.percent,
-			time: history.time,
-			entryPk: history.entryPk,
-			videoId: videos.id,
-		})
-		.from(history)
-		.leftJoin(videos, eq(history.videoPk, videos.pk))
-		.leftJoin(profiles, eq(history.profilePk, profiles.pk))
-		.where(eq(profiles.id, userId))
-		.orderBy(history.entryPk, desc(history.playedDate))
-		.as("progress");
-
-export const mapProgress = (
-	progressQ: ReturnType<typeof getEntryProgressQ>,
-	{ aliased }: { aliased: boolean } = { aliased: false },
-) => {
-	const { time, percent, videoId } = getColumns(progressQ);
 	const ret = {
 		time: coalesce(time, sql`0`),
 		percent: coalesce(percent, sql`0`),
+		playedDate: sql`${playedDate}`,
 		videoId: sql`${videoId}`,
 	};
 	if (!aliased) return ret;
 	return Object.fromEntries(Object.entries(ret).map(([k, v]) => [k, v.as(k)]));
 };
 
-async function getEntries({
+export async function getEntries({
 	after,
 	limit,
 	query,
@@ -163,6 +166,7 @@ async function getEntries({
 	filter,
 	languages,
 	userId,
+	progressQ = entryProgressQ,
 }: {
 	after: string | undefined;
 	limit: number;
@@ -171,6 +175,7 @@ async function getEntries({
 	filter: SQL | undefined;
 	languages: string[];
 	userId: string;
+	progressQ?: typeof entryProgressQ;
 }): Promise<(Entry | Extra | UnknownEntry)[]> {
 	const transQ = db
 		.selectDistinctOn([entryTranslations.pk])
@@ -182,8 +187,6 @@ async function getEntries({
 		.as("t");
 	const { pk, name, ...transCol } = getColumns(transQ);
 
-	const entryProgressQ = getEntryProgressQ(userId);
-
 	const {
 		kind,
 		externalId,
@@ -198,7 +201,7 @@ async function getEntries({
 			...entryCol,
 			...transCol,
 			videos: entryVideosQ.videos,
-			progress: mapProgress(entryProgressQ, { aliased: true }),
+			progress: mapProgress({ aliased: true }),
 			// specials don't have an `episodeNumber` but a `number` field.
 			number: episodeNumber,
 
@@ -217,7 +220,7 @@ async function getEntries({
 		.from(entries)
 		.innerJoin(transQ, eq(entries.pk, transQ.pk))
 		.leftJoinLateral(entryVideosQ, sql`true`)
-		.leftJoin(entryProgressQ, eq(entries.pk, entryProgressQ.entryPk))
+		.leftJoin(progressQ, eq(entries.pk, progressQ.entryPk))
 		.where(
 			and(
 				filter,
@@ -231,7 +234,8 @@ async function getEntries({
 				: sortToSql(sort)),
 			entries.pk,
 		)
-		.limit(limit);
+		.limit(limit)
+		.execute({ userId });
 }
 
 export const entriesH = new Elysia({ tags: ["series"] })

api/src/controllers/profiles/history.ts

@@ -0,0 +1,347 @@
+import {
+	and,
+	count,
+	eq,
+	exists,
+	gt,
+	isNotNull,
+	ne,
+	not,
+	or,
+	sql,
+} from "drizzle-orm";
+import { alias } from "drizzle-orm/pg-core";
+import Elysia, { t } from "elysia";
+import { auth, getUserInfo } from "~/auth";
+import { db } from "~/db";
+import { entries, history, profiles, shows, videos } from "~/db/schema";
+import { watchlist } from "~/db/schema/watchlist";
+import { coalesce, values } from "~/db/utils";
+import { Entry } from "~/models/entry";
+import { KError } from "~/models/error";
+import { SeedHistory } from "~/models/history";
+import {
+	AcceptLanguage,
+	Filter,
+	Page,
+	createPage,
+	isUuid,
+	processLanguages,
+} from "~/models/utils";
+import { desc } from "~/models/utils/descriptions";
+import {
+	entryFilters,
+	entryProgressQ,
+	entrySort,
+	getEntries,
+} from "../entries";
+import { getOrCreateProfile } from "./profile";
+
+const historyProgressQ: typeof entryProgressQ = db
+	.select({
+		percent: history.percent,
+		time: history.time,
+		entryPk: history.entryPk,
+		playedDate: history.playedDate,
+		videoId: videos.id,
+	})
+	.from(history)
+	.leftJoin(videos, eq(history.videoPk, videos.pk))
+	.leftJoin(profiles, eq(history.profilePk, profiles.pk))
+	.where(eq(profiles.id, sql.placeholder("userId")))
+	.as("progress");
+
+export const historyH = new Elysia({ tags: ["profiles"] })
+	.use(auth)
+	.guard(
+		{
+			query: t.Object({
+				sort: {
+					...entrySort,
+					default: ["-playedDate"],
+				},
+				filter: t.Optional(Filter({ def: entryFilters })),
+				query: t.Optional(t.String({ description: desc.query })),
+				limit: t.Integer({
+					minimum: 1,
+					maximum: 250,
+					default: 50,
+					description: "Max page size.",
+				}),
+				after: t.Optional(t.String({ description: desc.after })),
+			}),
+		},
+		(app) =>
+			app
+				.get(
+					"/profiles/me/history",
+					async ({
+						query: { sort, filter, query, limit, after },
+						headers: { "accept-language": languages },
+						request: { url },
+						jwt: { sub },
+					}) => {
+						const langs = processLanguages(languages);
+						const items = (await getEntries({
+							limit,
+							after,
+							query,
+							sort,
+							filter: and(
+								isNotNull(entryProgressQ.playedDate),
+								ne(entries.kind, "extra"),
+								ne(entries.kind, "unknown"),
+								filter,
+							),
+							languages: langs,
+							userId: sub,
+							progressQ: historyProgressQ,
+						})) as Entry[];
+
+						return createPage(items, { url, sort, limit });
+					},
+					{
+						detail: {
+							description: "List your watch history (episodes/movies seen)",
+						},
+						headers: t.Object(
+							{
+								"accept-language": AcceptLanguage({ autoFallback: true }),
+							},
+							{ additionalProperties: true },
+						),
+						response: {
+							200: Page(Entry),
+						},
+					},
+				)
+				.get(
+					"/profiles/:id/history",
+					async ({
+						params: { id },
+						query: { sort, filter, query, limit, after },
+						headers: { "accept-language": languages, authorization },
+						request: { url },
+						error,
+					}) => {
+						const uInfo = await getUserInfo(id, { authorization });
+						if ("status" in uInfo) return error(uInfo.status as 404, uInfo);
+
+						const langs = processLanguages(languages);
+						const items = (await getEntries({
+							limit,
+							after,
+							query,
+							sort,
+							filter: and(
+								isNotNull(entryProgressQ.playedDate),
+								ne(entries.kind, "extra"),
+								ne(entries.kind, "unknown"),
+								filter,
+							),
+							languages: langs,
+							userId: uInfo.id,
+							progressQ: historyProgressQ,
+						})) as Entry[];
+
+						return createPage(items, { url, sort, limit });
+					},
+					{
+						detail: {
+							description: "List your watch history (episodes/movies seen)",
+						},
+						params: t.Object({
+							id: t.String({
+								description:
+									"The id or username of the user to read the watchlist of",
+								example: "zoriya",
+							}),
+						}),
+						headers: t.Object({
+							authorization: t.TemplateLiteral("Bearer ${string}"),
+							"accept-language": AcceptLanguage({ autoFallback: true }),
+						}),
+						response: {
+							200: Page(Entry),
+							403: KError,
+							404: {
+								...KError,
+								description: "No user found with the specified id/username.",
+							},
+							422: KError,
+						},
+					},
+				),
+	)
+	.post(
+		"/profiles/me/history",
+		async ({ body, jwt: { sub }, error }) => {
+			const profilePk = await getOrCreateProfile(sub);
+
+			const vals = values(
+				body.map((x) => ({ ...x, entryUseId: isUuid(x.entry) })),
+			).as("hist");
+
+			const rows = await db
+				.insert(history)
+				.select(
+					db
+						.select({
+							profilePk: sql`${profilePk}`,
+							entryPk: entries.pk,
+							videoPk: videos.pk,
+							percent: sql`hist.percent::integer`,
+							time: sql`hist.time::integer`,
+							playedDate: sql`hist.playedDate::timestamptz`,
+						})
+						.from(vals)
+						.innerJoin(
+							entries,
+							or(
+								and(
+									sql`hist.entryUseId::boolean`,
+									eq(entries.id, sql`hist.entry::uuid`),
+								),
+								and(
+									not(sql`hist.entryUseId::boolean`),
+									eq(entries.slug, sql`hist.entry`),
+								),
+							),
+						)
+						.leftJoin(videos, eq(videos.id, sql`hist.videoId::uuid`)),
+				)
+				.returning({ pk: history.pk });
+
+			// automatically update watchlist with this new info
+
+			const nextEntry = alias(entries, "next_entry");
+			const nextEntryQ = db
+				.select({
+					pk: nextEntry.pk,
+				})
+				.from(nextEntry)
+				.where(
+					and(
+						eq(nextEntry.showPk, entries.showPk),
+						gt(nextEntry.order, entries.order),
+					),
+				)
+				.orderBy(nextEntry.showPk, entries.order)
+				.limit(1)
+				.as("nextEntryQ");
+
+			const seenCountQ = db
+				.select({ c: count() })
+				.from(entries)
+				.where(
+					and(
+						eq(entries.showPk, sql`excluded.show_pk`),
+						exists(
+							db
+								.select()
+								.from(history)
+								.where(
+									and(
+										eq(history.profilePk, profilePk),
+										eq(history.entryPk, entries.pk),
+									),
+								),
+						),
+					),
+				);
+
+			await db
+				.insert(watchlist)
+				.select(
+					db
+						.select({
+							profilePk: sql`${profilePk}`,
+							showPk: entries.showPk,
+							status: sql`
+								case
+									when
+										hist.percent::integer >= 95
+										and ${nextEntryQ.pk} is null
+									then 'completed'::watchlist_status
+									else 'watching'::watchlist_status
+								end
+							`,
+							seenCount: sql`
+								case
+									when ${entries.kind} = 'movie' then hist.percent::integer
+									when hist.percent::integer >= 95 then 1
+									else 0
+								end
+							`,
+							nextEntry: nextEntryQ.pk,
+							score: sql`null`,
+							startedAt: sql`hist.playedDate::timestamptz`,
+							completedAt: sql`
+								case
+									when ${nextEntryQ.pk} is null then hist.playedDate::timestamptz
+									else null
+								end
+							`,
+							// see https://github.com/drizzle-team/drizzle-orm/issues/3608
+							updatedAt: sql`now()`,
+						})
+						.from(vals)
+						.leftJoin(
+							entries,
+							or(
+								and(
+									sql`hist.entryUseId::boolean`,
+									eq(entries.id, sql`hist.entry::uuid`),
+								),
+								and(
+									not(sql`hist.entryUseId::boolean`),
+									eq(entries.slug, sql`hist.entry`),
+								),
+							),
+						)
+						.leftJoinLateral(nextEntryQ, sql`true`),
+				)
+				.onConflictDoUpdate({
+					target: [watchlist.profilePk, watchlist.showPk],
+					set: {
+						status: sql`
+							case
+								when excluded.status = 'completed' then excluded.status
+								when
+									${watchlist.status} != 'completed'
+									and ${watchlist.status} != 'rewatching'
+								then excluded.status
+								else ${watchlist.status}
+							end
+						`,
+						seenCount: sql`${seenCountQ}`,
+						nextEntry: sql`
+							case
+								when ${watchlist.status} = 'completed' then null
+								else excluded.next_entry
+							end
+						`,
+						completedAt: coalesce(
+							watchlist.completedAt,
+							sql`excluded.completed_at`,
+						),
+					},
+				});
+
+			return error(201, { status: 201, inserted: rows.length });
+		},
+		{
+			detail: { description: "Bulk add entries/movies to your watch history." },
+			body: t.Array(SeedHistory),
+			permissions: ["core.read"],
+			response: {
+				201: t.Object({
+					status: t.Literal(201),
+					inserted: t.Integer({
+						description: "The number of history entry inserted",
+					}),
+				}),
+				422: KError,
+			},
+		},
+	);

api/src/controllers/profiles/profile.ts

@@ -0,0 +1,24 @@
+import { eq, sql } from "drizzle-orm";
+import { db } from "~/db";
+import { profiles } from "~/db/schema";
+
+export async function getOrCreateProfile(userId: string) {
+	let [profile] = await db
+		.select({ pk: profiles.pk })
+		.from(profiles)
+		.where(eq(profiles.id, userId))
+		.limit(1);
+	if (profile) return profile.pk;
+
+	[profile] = await db
+		.insert(profiles)
+		.values({ id: userId })
+		.onConflictDoUpdate({
+			// we can't do `onConflictDoNothing` because on race conditions
+			// we still want the profile to be returned.
+			target: [profiles.id],
+			set: { id: sql`excluded.id` },
+		})
+		.returning({ pk: profiles.pk });
+	return profile.pk;
+}

api/src/controllers/profiles/watchlist.ts

@@ -1,8 +1,14 @@
-import { type SQL, and, eq, isNotNull, isNull, sql } from "drizzle-orm";
+import { and, eq, isNotNull, isNull, sql } from "drizzle-orm";
 import Elysia, { t } from "elysia";
 import { auth, getUserInfo } from "~/auth";
+import {
+	getShows,
+	showFilters,
+	showSort,
+	watchStatusQ,
+} from "~/controllers/shows/logic";
 import { db } from "~/db";
-import { profiles, shows } from "~/db/schema";
+import { shows } from "~/db/schema";
 import { watchlist } from "~/db/schema/watchlist";
 import { conflictUpdateAllExcept, getColumns } from "~/db/utils";
 import { KError } from "~/models/error";
@@ -19,7 +25,7 @@ import {
 } from "~/models/utils";
 import { desc } from "~/models/utils/descriptions";
 import { MovieWatchStatus, SerieWatchStatus } from "~/models/watchlist";
-import { getShows, showFilters, showSort, watchStatusQ } from "./shows/logic";
+import { getOrCreateProfile } from "./profile";
 
 async function setWatchStatus({
 	show,
@@ -30,29 +36,13 @@ async function setWatchStatus({
 	status: SerieWatchStatus;
 	userId: string;
 }) {
-	let [profile] = await db
+	const profilePk = await getOrCreateProfile(userId);
-		.select({ pk: profiles.pk })
-		.from(profiles)
-		.where(eq(profiles.id, userId))
-		.limit(1);
-	if (!profile) {
-		[profile] = await db
-			.insert(profiles)
-			.values({ id: userId })
-			.onConflictDoUpdate({
-				// we can't do `onConflictDoNothing` because on race conditions
-				// we still want the profile to be returned.
-				target: [profiles.id],
-				set: { id: sql`excluded.id` },
-			})
-			.returning({ pk: profiles.pk });
-	}
 
 	const [ret] = await db
 		.insert(watchlist)
 		.values({
 			...status,
-			profilePk: profile.pk,
+			profilePk: profilePk,
 			showPk: show.pk,
 		})
 		.onConflictDoUpdate({
@@ -72,7 +62,7 @@ async function setWatchStatus({
 		})
 		.returning({
 			...getColumns(watchlist),
-			percent: sql`${watchlist.seenCount}`.as("percent"),
+			percent: sql<number>`${watchlist.seenCount}`.as("percent"),
 		});
 	return ret;
 }
@@ -82,7 +72,10 @@ export const watchlistH = new Elysia({ tags: ["profiles"] })
 	.guard(
 		{
 			query: t.Object({
-				sort: showSort,
+				sort: {
+					...showSort,
+					default: ["watchStatus", ...showSort.default],
+				},
 				filter: t.Optional(Filter({ def: showFilters })),
 				query: t.Optional(t.String({ description: desc.query })),
 				limit: t.Integer({
@@ -150,7 +143,6 @@ export const watchlistH = new Elysia({ tags: ["profiles"] })
 						error,
 					}) => {
 						const uInfo = await getUserInfo(id, { authorization });
-
 						if ("status" in uInfo) return error(uInfo.status as 404, uInfo);
 
 						const langs = processLanguages(languages);
@@ -233,7 +225,7 @@ export const watchlistH = new Elysia({ tags: ["profiles"] })
 			}),
 			body: SerieWatchStatus,
 			response: {
-				200: t.Union([SerieWatchStatus, DbMetadata]),
+				200: t.Intersect([SerieWatchStatus, DbMetadata]),
 				404: KError,
 			},
 			permissions: ["core.read"],
@@ -280,7 +272,7 @@ export const watchlistH = new Elysia({ tags: ["profiles"] })
 			}),
 			body: t.Omit(MovieWatchStatus, ["percent"]),
 			response: {
-				200: t.Union([MovieWatchStatus, DbMetadata]),
+				200: t.Intersect([MovieWatchStatus, DbMetadata]),
 				404: KError,
 			},
 			permissions: ["core.read"],

api/src/controllers/shows/logic.ts

@@ -36,7 +36,7 @@ import {
 } from "~/models/utils";
 import type { EmbeddedVideo } from "~/models/video";
 import { WatchlistStatus } from "~/models/watchlist";
-import { entryVideosQ, getEntryProgressQ, mapProgress } from "../entries";
+import { entryProgressQ, entryVideosQ, mapProgress } from "../entries";
 
 export const watchStatusQ = db
 	.select({
@@ -75,6 +75,7 @@ export const showFilters: FilterDef = {
 		type: "enum",
 		values: WatchlistStatus.enum,
 	},
+	score: { column: watchStatusQ.score, type: "int" },
 };
 export const showSort = Sort(
 	{
@@ -86,6 +87,7 @@ export const showSort = Sort(
 		createdAt: shows.createdAt,
 		nextRefresh: shows.nextRefresh,
 		watchStatus: watchStatusQ.status,
+		score: watchStatusQ.score,
 	},
 	{
 		default: ["slug"],
@@ -164,10 +166,7 @@ const showRelations = {
 			.leftJoin(videos, eq(videos.pk, entryVideoJoin.videoPk))
 			.as("videos");
 	},
-	firstEntry: ({
+	firstEntry: ({ languages }: { languages: string[] }) => {
-		languages,
-		userId,
-	}: { languages: string[]; userId: string }) => {
 		const transQ = db
 			.selectDistinctOn([entryTranslations.pk])
 			.from(entryTranslations)
@@ -178,8 +177,6 @@ const showRelations = {
 			.as("t");
 		const { pk, ...transCol } = getColumns(transQ);
 
-		const progressQ = getEntryProgressQ(userId);
-
 		return db
 			.select({
 				firstEntry: jsonbBuildObject<Entry>({
@@ -187,12 +184,12 @@ const showRelations = {
 					...transCol,
 					number: entries.episodeNumber,
 					videos: entryVideosQ.videos,
-					progress: mapProgress(progressQ),
+					progress: mapProgress({ aliased: false }),
 				}).as("firstEntry"),
 			})
 			.from(entries)
 			.innerJoin(transQ, eq(entries.pk, transQ.pk))
-			.leftJoin(progressQ, eq(entries.pk, progressQ.entryPk))
+			.leftJoin(entryProgressQ, eq(entries.pk, entryProgressQ.entryPk))
 			.leftJoinLateral(entryVideosQ, sql`true`)
 			.where(and(eq(entries.showPk, shows.pk), ne(entries.kind, "extra")))
 			.orderBy(entries.order)
@@ -201,10 +198,8 @@ const showRelations = {
 	},
 	nextEntry: ({
 		languages,
-		userId,
 	}: {
 		languages: string[];
-		userId: string;
 	}) => {
 		const transQ = db
 			.selectDistinctOn([entryTranslations.pk])
@@ -216,8 +211,6 @@ const showRelations = {
 			.as("t");
 		const { pk, ...transCol } = getColumns(transQ);
 
-		const progressQ = getEntryProgressQ(userId);
-
 		return db
 			.select({
 				nextEntry: jsonbBuildObject<Entry>({
@@ -225,12 +218,12 @@ const showRelations = {
 					...transCol,
 					number: entries.episodeNumber,
 					videos: entryVideosQ.videos,
-					progress: mapProgress(progressQ),
+					progress: mapProgress({ aliased: false }),
 				}).as("nextEntry"),
 			})
 			.from(entries)
 			.innerJoin(transQ, eq(entries.pk, transQ.pk))
-			.leftJoin(progressQ, eq(entries.pk, progressQ.entryPk))
+			.leftJoin(entryProgressQ, eq(entries.pk, entryProgressQ.entryPk))
 			.leftJoinLateral(entryVideosQ, sql`true`)
 			.where(eq(watchStatusQ.nextEntry, entries.pk))
 			.as("nextEntry");
@@ -294,7 +287,7 @@ export async function getShows({
 
 			watchStatus: getColumns(watchStatusQ),
 
-			...buildRelations(relations, showRelations, { languages, userId }),
+			...buildRelations(relations, showRelations, { languages }),
 		})
 		.from(shows)
 		.leftJoin(watchStatusQ, eq(shows.pk, watchStatusQ.showPk))

api/src/db/schema/history.ts

@@ -15,9 +15,7 @@ export const history = schema.table(
 		entryPk: integer()
 			.notNull()
 			.references(() => entries.pk, { onDelete: "cascade" }),
-		videoPk: integer()
+		videoPk: integer().references(() => videos.pk, { onDelete: "set null" }),
-			.notNull()
-			.references(() => videos.pk, { onDelete: "set null" }),
 		percent: integer().notNull().default(0),
 		time: integer(),
 		playedDate: timestamp({ withTimezone: true, mode: "string" })

api/src/db/utils.ts

@@ -95,10 +95,14 @@ export function values(items: Record<string, unknown>[]) {
 	};
 }
 
-export const coalesce = <T>(val: SQL<T> | Column, def: SQL<T>) => {
+export const coalesce = <T>(val: SQL<T> | Column, def: SQL<T> | Column) => {
 	return sql<T>`coalesce(${val}, ${def})`;
 };
 
+export const nullif = <T>(val: SQL<T> | Column, eq: SQL<T>) => {
+	return sql<T>`nullif(${val}, ${eq})`;
+};
+
 export const jsonbObjectAgg = <T>(key: SQLWrapper, value: SQL<T>) => {
 	return sql<
 		Record<string, T>

api/src/models/entry/episode.ts

@@ -1,6 +1,7 @@
 import { t } from "elysia";
 import type { Prettify } from "~/utils";
 import { bubbleImages, madeInAbyss, registerExamples } from "../examples";
+import { Progress } from "../history";
 import {
 	DbMetadata,
 	EpisodeId,
@@ -9,7 +10,6 @@ import {
 	TranslationRecord,
 } from "../utils";
 import { EmbeddedVideo } from "../video";
-import { Progress } from "../watchlist";
 import { BaseEntry, EntryTranslation } from "./base-entry";
 
 export const BaseEpisode = t.Intersect([

api/src/models/entry/extra.ts

@@ -1,9 +1,9 @@
 import { t } from "elysia";
 import { type Prettify, comment } from "~/utils";
 import { madeInAbyss, registerExamples } from "../examples";
+import { Progress } from "../history";
 import { DbMetadata, SeedImage } from "../utils";
 import { Resource } from "../utils/resource";
-import { Progress } from "../watchlist";
 import { BaseEntry } from "./base-entry";
 
 export const ExtraType = t.UnionEnum([

api/src/models/entry/movie-entry.ts

@@ -1,6 +1,7 @@
 import { t } from "elysia";
 import { type Prettify, comment } from "~/utils";
 import { bubbleImages, madeInAbyss, registerExamples } from "../examples";
+import { Progress } from "../history";
 import {
 	DbMetadata,
 	ExternalId,
@@ -10,7 +11,6 @@ import {
 	TranslationRecord,
 } from "../utils";
 import { EmbeddedVideo } from "../video";
-import { Progress } from "../watchlist";
 import { BaseEntry, EntryTranslation } from "./base-entry";
 
 export const BaseMovieEntry = t.Intersect(

api/src/models/entry/special.ts

@@ -1,6 +1,7 @@
 import { t } from "elysia";
 import { type Prettify, comment } from "~/utils";
 import { bubbleImages, madeInAbyss, registerExamples } from "../examples";
+import { Progress } from "../history";
 import {
 	DbMetadata,
 	EpisodeId,
@@ -9,7 +10,6 @@ import {
 	TranslationRecord,
 } from "../utils";
 import { EmbeddedVideo } from "../video";
-import { Progress } from "../watchlist";
 import { BaseEntry, EntryTranslation } from "./base-entry";
 
 export const BaseSpecial = t.Intersect(

api/src/models/entry/unknown-entry.ts

@@ -1,8 +1,8 @@
 import { t } from "elysia";
 import { type Prettify, comment } from "~/utils";
 import { bubbleImages, registerExamples, youtubeExample } from "../examples";
+import { Progress } from "../history";
 import { DbMetadata, Resource } from "../utils";
-import { Progress } from "../watchlist";
 import { BaseEntry, EntryTranslation } from "./base-entry";
 
 export const BaseUnknownEntry = t.Intersect(

api/src/models/history.ts

@@ -0,0 +1,40 @@
+import { t } from "elysia";
+import { comment } from "~/utils";
+
+export const Progress = t.Object({
+	percent: t.Integer({ minimum: 0, maximum: 100 }),
+	time: t.Nullable(
+		t.Integer({
+			minimum: 0,
+			description: comment`
+				When this episode was stopped (in seconds since the start).
+				This value is null if the entry was never watched or is finished.
+			`,
+		}),
+	),
+	playedDate: t.Nullable(t.String({ format: "date-time" })),
+	videoId: t.Nullable(
+		t.String({
+			format: "uuid",
+			description: comment`
+				Id of the video the user watched.
+				This can be used to resume playback in the correct video file
+				without asking the user what video to play.
+
+				This will be null if the user did not watch the entry or
+				if the video was deleted since.
+			`,
+		}),
+	),
+});
+export type Progress = typeof Progress.static;
+
+export const SeedHistory = t.Intersect([
+	t.Object({
+		entry: t.String({
+			description: "Id or slug of the entry/movie you watched",
+		}),
+	}),
+	Progress,
+]);
+export type SeedHistory = typeof SeedHistory.static;

api/src/models/watchlist.ts

@@ -1,32 +1,4 @@
 import { t } from "elysia";
-import { comment } from "~/utils";
-
-export const Progress = t.Object({
-	percent: t.Integer({ minimum: 0, maximum: 100 }),
-	time: t.Nullable(
-		t.Integer({
-			minimum: 0,
-			description: comment`
-				When this episode was stopped (in seconds since the start).
-				This value is null if the entry was never watched or is finished.
-			`,
-		}),
-	),
-	videoId: t.Nullable(
-		t.String({
-			format: "uuid",
-			description: comment`
-				Id of the video the user watched.
-				This can be used to resume playback in the correct video file
-				without asking the user what video to play.
-
-				This will be null if the user did not watch the entry or
-				if the video was deleted since.
-			`,
-		}),
-	),
-});
-export type Progress = typeof Progress.static;
 
 export const WatchlistStatus = t.UnionEnum([
 	"completed",

api/tests/helpers/series-helper.ts

@@ -1,5 +1,6 @@
 import { buildUrl } from "tests/utils";
 import { app } from "~/base";
+import type { SeedHistory } from "~/models/history";
 import type { SeedSerie } from "~/models/serie";
 import type { SerieWatchStatus } from "~/models/watchlist";
 import { getJwtHeaders } from "./jwt";
@@ -197,3 +198,47 @@ export const setSerieStatus = async (id: string, status: SerieWatchStatus) => {
 	const body = await resp.json();
 	return [resp, body] as const;
 };
+
+export const getHistory = async (
+	profile: string,
+	{
+		langs,
+		...opts
+	}: {
+		filter?: string;
+		limit?: number;
+		after?: string;
+		query?: string;
+		langs?: string;
+		preferOriginal?: boolean;
+	},
+) => {
+	const resp = await app.handle(
+		new Request(buildUrl(`profiles/${profile}/history`, opts), {
+			method: "GET",
+			headers: langs
+				? {
+						"Accept-Language": langs,
+						...(await getJwtHeaders()),
+					}
+				: await getJwtHeaders(),
+		}),
+	);
+	const body = await resp.json();
+	return [resp, body] as const;
+};
+
+export const addToHistory = async (profile: string, seed: SeedHistory[]) => {
+	const resp = await app.handle(
+		new Request(buildUrl(`profiles/${profile}/history`), {
+			method: "POST",
+			body: JSON.stringify(seed),
+			headers: {
+				"Content-Type": "application/json",
+				...(await getJwtHeaders()),
+			},
+		}),
+	);
+	const body = await resp.json();
+	return [resp, body] as const;
+};

api/tests/movies/watchstatus.test.ts

@@ -67,6 +67,21 @@ describe("Set & get watch status", () => {
 		});
 	});
 
+	it("Can filter watchlist", async () => {
+		let [resp, body] = await getWatchlist("me", {
+			filter: "watchStatus eq rewatching",
+		});
+		expectStatus(resp, body).toBe(200);
+		expect(body.items).toBeArrayOfSize(1);
+		expect(body.items[0].slug).toBe(bubble.slug);
+
+		[resp, body] = await getWatchlist("me", {
+			filter: "watchStatus eq completed",
+		});
+		expectStatus(resp, body).toBe(200);
+		expect(body.items).toBeArrayOfSize(0);
+	});
+
 	it("Return watchstatus in /shows", async () => {
 		const [resp, body] = await getShows({});
 		expectStatus(resp, body).toBe(200);

api/tests/series/history.test.ts

@@ -0,0 +1,149 @@
+import { beforeAll, describe, expect, it } from "bun:test";
+import {
+	addToHistory,
+	createMovie,
+	createSerie,
+	getEntries,
+	getHistory,
+	getNews,
+	getWatchlist,
+} from "tests/helpers";
+import { expectStatus } from "tests/utils";
+import { db } from "~/db";
+import { entries, shows, videos } from "~/db/schema";
+import { bubble, madeInAbyss, madeInAbyssVideo } from "~/models/examples";
+
+beforeAll(async () => {
+	await db.delete(shows);
+	await db.delete(entries);
+	await db.delete(videos);
+	// create video beforehand to test linking
+	await db.insert(videos).values(madeInAbyssVideo);
+	let [ret, body] = await createSerie(madeInAbyss);
+	expectStatus(ret, body).toBe(201);
+	[ret, body] = await createMovie(bubble);
+	expectStatus(ret, body).toBe(201);
+});
+
+const miaEntrySlug = `${madeInAbyss.slug}-s1e13`;
+
+describe("Set & get history", () => {
+	it("Add episodes & movie to history", async () => {
+		let [resp, body] = await getHistory("me", {});
+		expectStatus(resp, body).toBe(200);
+		expect(body.items).toBeArrayOfSize(0);
+
+		const [r, b] = await addToHistory("me", [
+			{
+				entry: miaEntrySlug,
+				videoId: madeInAbyssVideo.id,
+				percent: 58,
+				time: 28 * 60 + 12,
+				playedDate: "2025-02-01",
+			},
+			{
+				entry: bubble.slug,
+				videoId: null,
+				percent: 100,
+				time: 2 * 60,
+				playedDate: "2025-02-02",
+			},
+		]);
+		expectStatus(r, b).toBe(201);
+		expect(b.inserted).toBe(2);
+
+		[resp, body] = await getHistory("me", {});
+		expectStatus(resp, body).toBe(200);
+		expect(body.items).toBeArrayOfSize(2);
+		expect(body.items[0].slug).toBe(bubble.slug);
+		expect(body.items[0].progress).toMatchObject({
+			percent: 100,
+			time: 2 * 60,
+		});
+		expect(body.items[1].slug).toBe(miaEntrySlug);
+		expect(body.items[1].progress).toMatchObject({
+			percent: 58,
+			videoId: madeInAbyssVideo.id,
+		});
+	});
+
+	it("Create duplicated history entry", async () => {
+		const [r, b] = await addToHistory("me", [
+			{
+				entry: miaEntrySlug!,
+				videoId: madeInAbyssVideo.id,
+				percent: 100,
+				time: 38 * 60,
+				playedDate: "2025-02-03",
+			},
+		]);
+		expectStatus(r, b).toBe(201);
+		expect(b.inserted).toBe(1);
+
+		const [resp, body] = await getHistory("me", {});
+		expectStatus(resp, body).toBe(200);
+		expect(body.items).toBeArrayOfSize(3);
+		expect(body.items[0].slug).toBe(miaEntrySlug);
+		expect(body.items[0].progress).toMatchObject({
+			percent: 100,
+			videoId: madeInAbyssVideo.id,
+		});
+		expect(body.items[1].slug).toBe(bubble.slug);
+		expect(body.items[1].progress).toMatchObject({
+			percent: 100,
+			time: 2 * 60,
+		});
+		expect(body.items[2].slug).toBe(miaEntrySlug);
+		expect(body.items[2].progress).toMatchObject({
+			percent: 58,
+			videoId: madeInAbyssVideo.id,
+		});
+	});
+
+	it("Return progress in /shows/:id/entries", async () => {
+		const [resp, body] = await getEntries(madeInAbyss.slug, { langs: "en" });
+
+		expectStatus(resp, body).toBe(200);
+		expect(body.items).toBeArrayOfSize(madeInAbyss.entries.length);
+		expect(body.items[0].progress).toMatchObject({
+			percent: 100,
+			time: 38 * 60,
+			videoId: madeInAbyssVideo.id,
+			playedDate: "2025-02-03 00:00:00+00",
+		});
+	});
+
+	it("Return progress in /news", async () => {
+		const [resp, body] = await getNews({ langs: "en" });
+
+		expectStatus(resp, body).toBe(200);
+		const entry = body.items.find((x: any) => x.slug === miaEntrySlug);
+		expect(entry.progress).toMatchObject({
+			percent: 100,
+			time: 38 * 60,
+			videoId: madeInAbyssVideo.id,
+			playedDate: "2025-02-03 00:00:00+00",
+		});
+	});
+
+	// extras, unknowns
+
+	it("Update watchlist", async () => {
+		const [resp, body] = await getWatchlist("me", {});
+		expectStatus(resp, body).toBe(200);
+		expect(body.items).toBeArrayOfSize(2);
+		// watching items before completed ones
+		expect(body.items[0].slug).toBe(madeInAbyss.slug);
+		expect(body.items[0].watchStatus).toMatchObject({
+			status: "watching",
+			seenCount: 1,
+			startedAt: "2025-02-01 00:00:00+00",
+		});
+		expect(body.items[1].slug).toBe(bubble.slug);
+		expect(body.items[1].watchStatus).toMatchObject({
+			status: "completed",
+			percent: 100,
+			completedAt: "2025-02-02 00:00:00+00",
+		});
+	});
+});