Create keyset pagination function

api/src/controllers/movies.ts

@@ -1,20 +1,23 @@
 import { and, desc, eq, sql } from "drizzle-orm";
 import { Elysia, t } from "elysia";
 import { KError } from "~/models/error";
-import {
-	type FilterDef,
-	Genre,
-	isUuid,
-	processLanguages,
-} from "~/models/utils";
 import { comment } from "~/utils";
 import { db } from "../db";
 import { shows, showTranslations } from "../db/schema/shows";
 import { getColumns } from "../db/schema/utils";
 import { bubble } from "../models/examples";
 import { Movie, MovieStatus, MovieTranslation } from "../models/movie";
-import { Filter, type Page } from "~/models/utils";
+import {
-import { Sort } from "~/models/utils/sort";
+	Filter,
+	Sort,
+	type FilterDef,
+	Genre,
+	isUuid,
+	keysetPaginate,
+	processLanguages,
+	type Page,
+	createPage,
+} from "~/models/utils";
 
 // drizzle is bugged and doesn't allow js arrays to be used in raw sql.
 export function sqlarr(array: unknown[]) {
@@ -156,6 +159,7 @@ export const movies = new Elysia({ prefix: "/movies", tags: ["movies"] })
 		async ({
 			query: { limit, after, sort, filter },
 			headers: { "accept-language": languages },
+			request: { url },
 		}) => {
 			const langs = processLanguages(languages);
 			const [transQ, transCol] = getTranslationQuery(langs);
@@ -171,33 +175,24 @@ export const movies = new Elysia({ prefix: "/movies", tags: ["movies"] })
 				})
 				.from(shows)
 				.innerJoin(transQ, eq(shows.pk, transQ.pk))
-				.where(filter)
+				.where(and(filter, keysetPaginate({ table: shows, after, sort })))
 				.orderBy(
 					...sort.map((x) => (x.desc ? desc(shows[x.key]) : shows[x.key])),
 					shows.pk,
 				)
 				.limit(limit);
 
-			return { items, next: "", prev: "", this: "" };
+			return createPage(items, { url, sort });
 		},
 		{
 			detail: { description: "Get all movies" },
 			query: t.Object({
-				sort: Sort(
+				sort: Sort(["slug", "rating", "airDate", "createdAt", "nextRefresh"], {
-					[
+					// TODO: Add random
-						"slug",
+					remap: { airDate: "startAir" },
-						"rating",
+					default: ["slug"],
-						"airDate",
+					description: "How to sort the query",
-						"createdAt",
+				}),
-						"nextRefresh",
-					],
-					{
-						// TODO: Add random
-						remap: { airDate: "startAir" },
-						default: ["slug"],
-						description: "How to sort the query",
-					},
-				),
 				filter: t.Optional(Filter({ def: movieFilters })),
 				limit: t.Integer({
 					minimum: 1,
@@ -207,7 +202,7 @@ export const movies = new Elysia({ prefix: "/movies", tags: ["movies"] })
 				}),
 				after: t.Optional(
 					t.String({
-						format: "uuid",
+						format: "byte",
 						description: comment`
 							Id of the cursor in the pagination.
 							You can ignore this and only use the prev/next field in the response.

api/src/db/schema/utils.ts

@@ -69,7 +69,7 @@ export function conflictUpdateAllExcept<
 
 	return updateColumns.reduce(
 		(acc, [colName, col]) => {
-			// @ts-ignore: drizzle internal
+			// @ts-expect-error: drizzle internal
 			const name = (db.dialect.casing as CasingCache).getColumnCasing(col);
 			acc[colName as keyof typeof acc] = sql.raw(`excluded."${name}"`);
 			return acc;

api/src/models/utils/index.ts

@@ -5,3 +5,5 @@ export * from "./language";
 export * from "./resource";
 export * from "./filters";
 export * from "./page";
+export * from "./sort";
+export * from "./keyset-paginate";

api/src/models/utils/keyset-paginate.ts

@@ -0,0 +1,47 @@
+import type { NonEmptyArray, Sort } from "./sort";
+import { eq, or, type Column, and, gt, lt } from "drizzle-orm";
+
+type Table<Name extends string> = Record<Name, Column>;
+
+// Create a filter (where) expression on the query to skip everything before/after the referenceID.
+// The generalized expression for this in pseudocode is:
+//   (x > a) OR
+//   (x = a AND y > b) OR
+//   (x = a AND y = b AND z > c) OR...
+//
+// Of course, this will be a bit more complex when ASC and DESC are mixed.
+// Assume x is ASC, y is DESC, and z is ASC:
+//   (x > a) OR
+//   (x = a AND y < b) OR
+//   (x = a AND y = b AND z > c) OR...
+export const keysetPaginate = <
+	const T extends NonEmptyArray<string>,
+	const Remap extends Partial<Record<T[number], string>>,
+>({
+	table,
+	sort,
+	after,
+}: {
+	table: Table<"pk" | Sort<T, Remap>[number]["key"]>;
+	after: string | undefined;
+	sort: Sort<T, Remap>;
+}) => {
+	if (!after) return undefined;
+	const cursor: Record<string, string | number> = JSON.parse(
+		Buffer.from(after, "base64").toString("utf-8"),
+	);
+
+	// TODO: Add an outer query >= for perf
+	// PERF: See https://use-the-index-luke.com/sql/partial-results/fetch-next-page#sb-equivalent-logic
+	let where = undefined;
+	let previous = undefined;
+	for (const by of [...sort, { key: "pk" as const, desc: false }]) {
+		const cmp = by.desc ? lt : gt;
+		where = or(where, and(previous, cmp(table[by.key], cursor[by.key])));
+		previous = and(previous, eq(table[by.key], cursor[by.key]));
+	}
+
+	return where;
+};
+
+

api/src/models/utils/sort.ts

@@ -1,14 +1,14 @@
 import { t } from "elysia";
 
-type Sort<
+export type Sort<
 	T extends string[],
 	Remap extends Partial<Record<T[number], string>>,
 > = {
-	key: Exclude<T[number], keyof Remap> | Remap[keyof Remap];
+	key: Exclude<T[number], keyof Remap> | NonNullable<Remap[keyof Remap]>;
 	desc: boolean;
 }[];
 
-type NonEmptyArray<T> = [T, ...T[]];
+export type NonEmptyArray<T> = [T, ...T[]];
 
 export const Sort = <
 	const T extends NonEmptyArray<string>,
@@ -44,7 +44,7 @@ export const Sort = <
 			return sort.map((x) => {
 				const desc = x[0] === "-";
 				const key = (desc ? x.substring(1) : x) as T[number];
-				if (key in remap) return { key: remap[key], desc };
+				if (key in remap) return { key: remap[key]!, desc };
 				return { key: key as Exclude<typeof key, keyof Remap>, desc };
 			});
 		})