From 0e230114a7b83495068309e014e46ff89aa9624e Mon Sep 17 00:00:00 2001 From: Arthur Jamet Date: Sat, 11 Jan 2025 13:05:25 +0000 Subject: [PATCH 01/12] v5 api: Sort Movies Randomly, passing seed as query parameter --- api/package.json | 3 +- api/src/controllers/movies.ts | 15 +++++++-- api/src/models/utils/sort.ts | 2 +- api/tests/movies/get-all-movies.test.ts | 43 +++++++++++++++++++++++++ api/tests/movies/movies-helper.ts | 1 + 5 files changed, 60 insertions(+), 4 deletions(-) diff --git a/api/package.json b/api/package.json index feffaf22..465af2a8 100644 --- a/api/package.json +++ b/api/package.json @@ -20,5 +20,6 @@ "@types/pg": "^8.11.10", "bun-types": "^1.1.42" }, - "module": "src/index.js" + "module": "src/index.js", + "packageManager": "yarn@1.22.21+sha1.1959a18351b811cdeedbd484a8f86c3cc3bbaf72" } diff --git a/api/src/controllers/movies.ts b/api/src/controllers/movies.ts index 21683381..30c5494b 100644 --- a/api/src/controllers/movies.ts +++ b/api/src/controllers/movies.ts @@ -157,7 +157,7 @@ export const movies = new Elysia({ prefix: "/movies", tags: ["movies"] }) .get( "", async ({ - query: { limit, after, sort, filter }, + query: { limit, after, sort, filter, random }, headers: { "accept-language": languages }, request: { url }, }) => { @@ -177,7 +177,12 @@ export const movies = new Elysia({ prefix: "/movies", tags: ["movies"] }) .innerJoin(transQ, eq(shows.pk, transQ.pk)) .where(and(filter, keysetPaginate({ table: shows, after, sort }))) .orderBy( - ...sort.map((x) => (x.desc ? sql`${shows[x.key]} desc nulls last` : shows[x.key])), + ...(random !== undefined + ? [sql`md5(${random} || ${shows.pk} )`] + : []), + ...sort.map((x) => + x.desc ? sql`${shows[x.key]} desc nulls last` : shows[x.key], + ), shows.pk, ) .limit(limit); @@ -193,6 +198,12 @@ export const movies = new Elysia({ prefix: "/movies", tags: ["movies"] }) default: ["slug"], description: "How to sort the query", }), + random: t.Optional( + t.Integer({ + minimum: 0, + description: "Seed to shuffle items", + }), + ), filter: t.Optional(Filter({ def: movieFilters })), limit: t.Integer({ minimum: 1, diff --git a/api/src/models/utils/sort.ts b/api/src/models/utils/sort.ts index 650ffb79..e452688a 100644 --- a/api/src/models/utils/sort.ts +++ b/api/src/models/utils/sort.ts @@ -1,4 +1,4 @@ -import { t } from "elysia"; +import { t, TSchema } from "elysia"; export type Sort< T extends string[], diff --git a/api/tests/movies/get-all-movies.test.ts b/api/tests/movies/get-all-movies.test.ts index 09adc457..76da7e14 100644 --- a/api/tests/movies/get-all-movies.test.ts +++ b/api/tests/movies/get-all-movies.test.ts @@ -7,6 +7,7 @@ import { bubble } from "~/models/examples"; import { dune1984 } from "~/models/examples/dune-1984"; import { dune } from "~/models/examples/dune-2021"; import { getMovies, movieApp } from "./movies-helper"; +import { Movie } from "~/models/movie"; beforeAll(async () => { await db.delete(shows); @@ -120,4 +121,46 @@ describe("Get all movies", () => { next: null, }); }); + + describe("Random sort", () => { + it("No limit, compare order with same seeds", async () => { + // First query + let [resp1, body1] = await getMovies({ + random: 100, + }); + expectStatus(resp1, body1).toBe(200); + const items1: Movie[] = body1.items; + const items1Ids = items1.map(({ id }) => id); + + // Second query + let [resp2, body2] = await getMovies({ + random: 100, + }); + expectStatus(resp2, body2).toBe(200); + const items2: Movie[] = body2.items; + const items2Ids = items2.map(({ id }) => id); + + expect(items1Ids).toEqual(items2Ids); + }); + it("No limit, compare order with different seeds", async () => { + // First query + let [resp1, body1] = await getMovies({ + random: 100, + }); + expectStatus(resp1, body1).toBe(200); + const items1: Movie[] = body1.items; + const items1Ids = items1.map(({ id }) => id); + + // Second query + let [resp2, body2] = await getMovies({ + random: 1, + }); + expectStatus(resp2, body2).toBe(200); + const items2: Movie[] = body2.items; + const items2Ids = items2.map(({ id }) => id); + + console.log(items1Ids, items2Ids); + expect(items1Ids).not.toEqual(items2Ids); + }); + }); }); diff --git a/api/tests/movies/movies-helper.ts b/api/tests/movies/movies-helper.ts index 9d160b15..825da29e 100644 --- a/api/tests/movies/movies-helper.ts +++ b/api/tests/movies/movies-helper.ts @@ -30,6 +30,7 @@ export const getMovies = async ({ limit?: number; after?: string; sort?: string | string[]; + random?: number; langs?: string; }) => { const resp = await movieApp.handle( From 2afccaa8136d982df191b91ae37a07477e8a27ae Mon Sep 17 00:00:00 2001 From: Arthur Jamet Date: Sat, 11 Jan 2025 17:52:56 +0000 Subject: [PATCH 02/12] v5 api: Random query parameter becomes sort value --- api/package.json | 3 +- api/src/controllers/movies.ts | 20 ++++----- api/src/models/utils/keyset-paginate.ts | 6 +-- api/src/models/utils/sort.ts | 60 ++++++++++++++++++++----- api/tests/movies/get-all-movies.test.ts | 38 +++++++++++++--- api/tests/movies/movies-helper.ts | 1 - 6 files changed, 93 insertions(+), 35 deletions(-) diff --git a/api/package.json b/api/package.json index 465af2a8..feffaf22 100644 --- a/api/package.json +++ b/api/package.json @@ -20,6 +20,5 @@ "@types/pg": "^8.11.10", "bun-types": "^1.1.42" }, - "module": "src/index.js", - "packageManager": "yarn@1.22.21+sha1.1959a18351b811cdeedbd484a8f86c3cc3bbaf72" + "module": "src/index.js" } diff --git a/api/src/controllers/movies.ts b/api/src/controllers/movies.ts index 30c5494b..5951d966 100644 --- a/api/src/controllers/movies.ts +++ b/api/src/controllers/movies.ts @@ -157,13 +157,12 @@ export const movies = new Elysia({ prefix: "/movies", tags: ["movies"] }) .get( "", async ({ - query: { limit, after, sort, filter, random }, + query: { limit, after, sort, filter }, headers: { "accept-language": languages }, request: { url }, }) => { const langs = processLanguages(languages); const [transQ, transCol] = getTranslationQuery(langs, true); - // TODO: Add sql indexes on sort keys const items = await db @@ -177,10 +176,14 @@ export const movies = new Elysia({ prefix: "/movies", tags: ["movies"] }) .innerJoin(transQ, eq(shows.pk, transQ.pk)) .where(and(filter, keysetPaginate({ table: shows, after, sort }))) .orderBy( - ...(random !== undefined - ? [sql`md5(${random} || ${shows.pk} )`] + ...(sort.random !== undefined + ? [ + sort.random.desc + ? sql`md5(${sort.random.seed} || ${shows.pk}) desc` + : sql`md5(${sort.random.seed} || ${shows.pk})`, + ] : []), - ...sort.map((x) => + ...sort.sort.map((x) => x.desc ? sql`${shows[x.key]} desc nulls last` : shows[x.key], ), shows.pk, @@ -193,17 +196,10 @@ export const movies = new Elysia({ prefix: "/movies", tags: ["movies"] }) detail: { description: "Get all movies" }, query: t.Object({ sort: Sort(["slug", "rating", "airDate", "createdAt", "nextRefresh"], { - // TODO: Add random remap: { airDate: "startAir" }, default: ["slug"], description: "How to sort the query", }), - random: t.Optional( - t.Integer({ - minimum: 0, - description: "Seed to shuffle items", - }), - ), filter: t.Optional(Filter({ def: movieFilters })), limit: t.Integer({ minimum: 1, diff --git a/api/src/models/utils/keyset-paginate.ts b/api/src/models/utils/keyset-paginate.ts index 6bcba223..29aa14ba 100644 --- a/api/src/models/utils/keyset-paginate.ts +++ b/api/src/models/utils/keyset-paginate.ts @@ -24,7 +24,7 @@ export const keysetPaginate = < sort, after, }: { - table: Table<"pk" | Sort[number]["key"]>; + table: Table<"pk" | Sort["sort"][number]["key"]>; after: string | undefined; sort: Sort; }) => { @@ -39,7 +39,7 @@ export const keysetPaginate = < // PERF: See https://use-the-index-luke.com/sql/partial-results/fetch-next-page#sb-equivalent-logic let where = undefined; let previous = undefined; - for (const [i, by] of [...sort, pkSort].entries()) { + for (const [i, by] of [...sort.sort, pkSort].entries()) { const cmp = by.desc ? lt : gt; where = or( where, @@ -62,7 +62,7 @@ export const keysetPaginate = < export const generateAfter = (cursor: any, sort: Sort) => { const ret = [ - ...sort.map((by) => cursor[by.remmapedKey ?? by.key]), + ...sort.sort.map((by) => cursor[by.remmapedKey ?? by.key]), cursor.pk, ]; return Buffer.from(JSON.stringify(ret), "utf-8").toString("base64url"); diff --git a/api/src/models/utils/sort.ts b/api/src/models/utils/sort.ts index e452688a..cc841997 100644 --- a/api/src/models/utils/sort.ts +++ b/api/src/models/utils/sort.ts @@ -1,13 +1,16 @@ -import { t, TSchema } from "elysia"; +import { t } from "elysia"; export type Sort< T extends string[], Remap extends Partial>, > = { - key: Exclude | NonNullable; - remmapedKey?: keyof Remap; - desc: boolean; -}[]; + sort: { + key: Exclude | NonNullable; + remmapedKey?: keyof Remap; + desc: boolean; + }[]; + random?: { desc: boolean; seed: number }; +}; export type NonEmptyArray = [T, ...T[]]; @@ -29,9 +32,15 @@ export const Sort = < t .Transform( t.Array( - t.UnionEnum([ - ...values, - ...values.map((x: T[number]) => `-${x}` as const), + t.Union([ + t.UnionEnum([ + ...values, + ...values.map((x: T[number]) => `-${x}` as const), + ]), + t.Union([ + t.TemplateLiteral("random:${number}"), + t.TemplateLiteral("-random:${number}"), + ]), ]), { // TODO: support explode: true (allow sort=slug,-createdAt). needs a pr to elysia @@ -42,12 +51,39 @@ export const Sort = < ), ) .Decode((sort): Sort => { - return sort.map((x) => { + const sortItems: Sort["sort"] = []; + let random: Sort["random"] = undefined; + for (const x of sort) { const desc = x[0] === "-"; const key = (desc ? x.substring(1) : x) as T[number]; - if (key in remap) return { key: remap[key]!, remmapedKey: key, desc }; - return { key: key as Exclude, desc }; - }); + if (key == "random") { + random = { + seed: Math.floor(Math.random() * Number.MAX_SAFE_INTEGER), + desc, + }; + continue; + } else if (key.startsWith("random:")) { + const strSeed = key.replace("random:", ""); + random = { + seed: parseInt(strSeed), + desc, + }; + continue; + } + + if (key in remap) { + sortItems.push({ key: remap[key]!, remmapedKey: key, desc }); + } else { + sortItems.push({ + key: key as Exclude, + desc, + }); + } + } + return { + sort: sortItems, + random, + }; }) .Encode(() => { throw new Error("Encode not supported for sort"); diff --git a/api/tests/movies/get-all-movies.test.ts b/api/tests/movies/get-all-movies.test.ts index 76da7e14..9b25526f 100644 --- a/api/tests/movies/get-all-movies.test.ts +++ b/api/tests/movies/get-all-movies.test.ts @@ -126,7 +126,7 @@ describe("Get all movies", () => { it("No limit, compare order with same seeds", async () => { // First query let [resp1, body1] = await getMovies({ - random: 100, + sort: "random:100", }); expectStatus(resp1, body1).toBe(200); const items1: Movie[] = body1.items; @@ -134,7 +134,7 @@ describe("Get all movies", () => { // Second query let [resp2, body2] = await getMovies({ - random: 100, + sort: "random:100", }); expectStatus(resp2, body2).toBe(200); const items2: Movie[] = body2.items; @@ -145,7 +145,7 @@ describe("Get all movies", () => { it("No limit, compare order with different seeds", async () => { // First query let [resp1, body1] = await getMovies({ - random: 100, + sort: "random:100", }); expectStatus(resp1, body1).toBe(200); const items1: Movie[] = body1.items; @@ -153,14 +153,42 @@ describe("Get all movies", () => { // Second query let [resp2, body2] = await getMovies({ - random: 1, + sort: "random:5", }); expectStatus(resp2, body2).toBe(200); const items2: Movie[] = body2.items; const items2Ids = items2.map(({ id }) => id); - console.log(items1Ids, items2Ids); expect(items1Ids).not.toEqual(items2Ids); }); + + it("Limit 1, pages 1 and 2 ", async () => { + // First query fetches all + // use the result to know what is expected + let [resp, body] = await getMovies({ + sort: "random:1234", + }); + expectStatus(resp, body).toBe(200); + let items: Movie[] = body.items; + const expectedIds = items.map(({ id }) => id); + + // Get First Page + [resp, body] = await getMovies({ + sort: "random:1234", + limit: 1, + }); + expectStatus(resp, body).toBe(200); + items = body.items; + expect(items.length).toBe(1); + expect(items[0].id).toBe(expectedIds[0]); + // Get Second Page + resp = await movieApp.handle(new Request(body.next)); + body = await resp.json(); + + expectStatus(resp, body).toBe(200); + items = body.items; + expect(items.length).toBe(1); + expect(items[0].id).toBe(expectedIds[1]); + }); }); }); diff --git a/api/tests/movies/movies-helper.ts b/api/tests/movies/movies-helper.ts index 825da29e..9d160b15 100644 --- a/api/tests/movies/movies-helper.ts +++ b/api/tests/movies/movies-helper.ts @@ -30,7 +30,6 @@ export const getMovies = async ({ limit?: number; after?: string; sort?: string | string[]; - random?: number; langs?: string; }) => { const resp = await movieApp.handle( From 67ea86a955cd504e7f754c64c2e5e6791d2e16e0 Mon Sep 17 00:00:00 2001 From: Arthur Jamet Date: Sun, 12 Jan 2025 10:13:32 +0000 Subject: [PATCH 03/12] v5 api: Random sort query parameter cannot be reversed --- api/src/controllers/movies.ts | 6 +----- api/src/models/utils/sort.ts | 10 +++------- 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/api/src/controllers/movies.ts b/api/src/controllers/movies.ts index 5951d966..a4922a72 100644 --- a/api/src/controllers/movies.ts +++ b/api/src/controllers/movies.ts @@ -177,11 +177,7 @@ export const movies = new Elysia({ prefix: "/movies", tags: ["movies"] }) .where(and(filter, keysetPaginate({ table: shows, after, sort }))) .orderBy( ...(sort.random !== undefined - ? [ - sort.random.desc - ? sql`md5(${sort.random.seed} || ${shows.pk}) desc` - : sql`md5(${sort.random.seed} || ${shows.pk})`, - ] + ? [sql`md5(${sort.random.seed} || ${shows.pk})`] : []), ...sort.sort.map((x) => x.desc ? sql`${shows[x.key]} desc nulls last` : shows[x.key], diff --git a/api/src/models/utils/sort.ts b/api/src/models/utils/sort.ts index cc841997..7a7a1ee5 100644 --- a/api/src/models/utils/sort.ts +++ b/api/src/models/utils/sort.ts @@ -9,7 +9,7 @@ export type Sort< remmapedKey?: keyof Remap; desc: boolean; }[]; - random?: { desc: boolean; seed: number }; + random?: { seed: number }; }; export type NonEmptyArray = [T, ...T[]]; @@ -33,14 +33,12 @@ export const Sort = < .Transform( t.Array( t.Union([ + t.Literal("random"), + t.TemplateLiteral("random:${number}"), t.UnionEnum([ ...values, ...values.map((x: T[number]) => `-${x}` as const), ]), - t.Union([ - t.TemplateLiteral("random:${number}"), - t.TemplateLiteral("-random:${number}"), - ]), ]), { // TODO: support explode: true (allow sort=slug,-createdAt). needs a pr to elysia @@ -59,14 +57,12 @@ export const Sort = < if (key == "random") { random = { seed: Math.floor(Math.random() * Number.MAX_SAFE_INTEGER), - desc, }; continue; } else if (key.startsWith("random:")) { const strSeed = key.replace("random:", ""); random = { seed: parseInt(strSeed), - desc, }; continue; } From f490faa796257e053c397a6cefdfe8c497ea2769 Mon Sep 17 00:00:00 2001 From: Arthur Jamet Date: Sun, 12 Jan 2025 11:07:56 +0000 Subject: [PATCH 04/12] v5 api: Handle random in keyset paginate --- api/src/controllers/movies.ts | 2 +- api/src/models/utils/keyset-paginate.ts | 22 ++++++++++++++++++++-- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/api/src/controllers/movies.ts b/api/src/controllers/movies.ts index a4922a72..34928261 100644 --- a/api/src/controllers/movies.ts +++ b/api/src/controllers/movies.ts @@ -1,4 +1,4 @@ -import { and, desc, eq, sql } from "drizzle-orm"; +import { and, eq, sql } from "drizzle-orm"; import { Elysia, t } from "elysia"; import { KError } from "~/models/error"; import { comment } from "~/utils"; diff --git a/api/src/models/utils/keyset-paginate.ts b/api/src/models/utils/keyset-paginate.ts index 29aa14ba..76714dce 100644 --- a/api/src/models/utils/keyset-paginate.ts +++ b/api/src/models/utils/keyset-paginate.ts @@ -1,5 +1,5 @@ import type { NonEmptyArray, Sort } from "./sort"; -import { eq, or, type Column, and, gt, lt, isNull } from "drizzle-orm"; +import { eq, or, type Column, and, gt, lt, isNull, sql } from "drizzle-orm"; type Table = Record; @@ -29,7 +29,7 @@ export const keysetPaginate = < sort: Sort; }) => { if (!after) return undefined; - const cursor: After = JSON.parse( + let cursor: After = JSON.parse( Buffer.from(after, "base64").toString("utf-8"), ); @@ -39,6 +39,23 @@ export const keysetPaginate = < // PERF: See https://use-the-index-luke.com/sql/partial-results/fetch-next-page#sb-equivalent-logic let where = undefined; let previous = undefined; + + if (sort.random) { + const lastCursor = cursor.slice(-1)[0]; + where = or( + where, + and( + previous, + gt( + sql`md5(${sort.random.seed} || ${table[pkSort.key]})`, + sql`md5(${sort.random.seed} || ${lastCursor})`, + ), + ), + ); + + previous = and(previous, eq(table[pkSort.key], lastCursor)); + cursor = cursor.slice(1); + } for (const [i, by] of [...sort.sort, pkSort].entries()) { const cmp = by.desc ? lt : gt; where = or( @@ -62,6 +79,7 @@ export const keysetPaginate = < export const generateAfter = (cursor: any, sort: Sort) => { const ret = [ + ...(sort.random ? [`random:${sort.random.seed}`] : []), ...sort.sort.map((by) => cursor[by.remmapedKey ?? by.key]), cursor.pk, ]; From 86d37514dcd631819000000eea2f73699b489418 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Sun, 12 Jan 2025 15:55:02 +0100 Subject: [PATCH 05/12] Fix after handling --- api/src/models/utils/keyset-paginate.ts | 35 ++++++++++++------------- api/tests/movies/get-all-movies.test.ts | 22 +--------------- 2 files changed, 18 insertions(+), 39 deletions(-) diff --git a/api/src/models/utils/keyset-paginate.ts b/api/src/models/utils/keyset-paginate.ts index 76714dce..44e96a89 100644 --- a/api/src/models/utils/keyset-paginate.ts +++ b/api/src/models/utils/keyset-paginate.ts @@ -29,33 +29,33 @@ export const keysetPaginate = < sort: Sort; }) => { if (!after) return undefined; - let cursor: After = JSON.parse( + const cursor: After = JSON.parse( Buffer.from(after, "base64").toString("utf-8"), ); const pkSort = { key: "pk" as const, desc: false }; + if (sort.random) { + return or( + gt( + sql`md5(${sort.random.seed} || ${table[pkSort.key]})`, + sql`md5(${sort.random.seed} || ${cursor[0]})`, + ), + and( + eq( + sql`md5(${sort.random.seed} || ${table[pkSort.key]})`, + sql`md5(${sort.random.seed} || ${cursor[0]})`, + ), + gt(table[pkSort.key], cursor[0]), + ), + ); + } + // TODO: Add an outer query >= for perf // PERF: See https://use-the-index-luke.com/sql/partial-results/fetch-next-page#sb-equivalent-logic let where = undefined; let previous = undefined; - if (sort.random) { - const lastCursor = cursor.slice(-1)[0]; - where = or( - where, - and( - previous, - gt( - sql`md5(${sort.random.seed} || ${table[pkSort.key]})`, - sql`md5(${sort.random.seed} || ${lastCursor})`, - ), - ), - ); - - previous = and(previous, eq(table[pkSort.key], lastCursor)); - cursor = cursor.slice(1); - } for (const [i, by] of [...sort.sort, pkSort].entries()) { const cmp = by.desc ? lt : gt; where = or( @@ -79,7 +79,6 @@ export const keysetPaginate = < export const generateAfter = (cursor: any, sort: Sort) => { const ret = [ - ...(sort.random ? [`random:${sort.random.seed}`] : []), ...sort.sort.map((by) => cursor[by.remmapedKey ?? by.key]), cursor.pk, ]; diff --git a/api/tests/movies/get-all-movies.test.ts b/api/tests/movies/get-all-movies.test.ts index 9b25526f..c725d47e 100644 --- a/api/tests/movies/get-all-movies.test.ts +++ b/api/tests/movies/get-all-movies.test.ts @@ -7,7 +7,7 @@ import { bubble } from "~/models/examples"; import { dune1984 } from "~/models/examples/dune-1984"; import { dune } from "~/models/examples/dune-2021"; import { getMovies, movieApp } from "./movies-helper"; -import { Movie } from "~/models/movie"; +import type { Movie } from "~/models/movie"; beforeAll(async () => { await db.delete(shows); @@ -142,26 +142,6 @@ describe("Get all movies", () => { expect(items1Ids).toEqual(items2Ids); }); - it("No limit, compare order with different seeds", async () => { - // First query - let [resp1, body1] = await getMovies({ - sort: "random:100", - }); - expectStatus(resp1, body1).toBe(200); - const items1: Movie[] = body1.items; - const items1Ids = items1.map(({ id }) => id); - - // Second query - let [resp2, body2] = await getMovies({ - sort: "random:5", - }); - expectStatus(resp2, body2).toBe(200); - const items2: Movie[] = body2.items; - const items2Ids = items2.map(({ id }) => id); - - expect(items1Ids).not.toEqual(items2Ids); - }); - it("Limit 1, pages 1 and 2 ", async () => { // First query fetches all // use the result to know what is expected From 8b7e109be3fd5f5ee42d3153c3ebbee2fa3ff6da Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Sun, 12 Jan 2025 16:02:19 +0100 Subject: [PATCH 06/12] Cleanup sort parsing --- api/src/models/utils/sort.ts | 40 +++++++++++------------------------- 1 file changed, 12 insertions(+), 28 deletions(-) diff --git a/api/src/models/utils/sort.ts b/api/src/models/utils/sort.ts index 7a7a1ee5..83e22d39 100644 --- a/api/src/models/utils/sort.ts +++ b/api/src/models/utils/sort.ts @@ -49,36 +49,20 @@ export const Sort = < ), ) .Decode((sort): Sort => { - const sortItems: Sort["sort"] = []; - let random: Sort["random"] = undefined; - for (const x of sort) { - const desc = x[0] === "-"; - const key = (desc ? x.substring(1) : x) as T[number]; - if (key == "random") { - random = { - seed: Math.floor(Math.random() * Number.MAX_SAFE_INTEGER), - }; - continue; - } else if (key.startsWith("random:")) { - const strSeed = key.replace("random:", ""); - random = { - seed: parseInt(strSeed), - }; - continue; - } - - if (key in remap) { - sortItems.push({ key: remap[key]!, remmapedKey: key, desc }); - } else { - sortItems.push({ - key: key as Exclude, - desc, - }); - } + const random = sort.find((x) => x.startsWith("random")); + if (random) { + const seed = random.includes(":") + ? Number.parseInt(random.substring("random:".length)) + : Math.floor(Math.random() * Number.MAX_SAFE_INTEGER); + return { random: { seed }, sort: [] }; } return { - sort: sortItems, - random, + sort: sort.map((x) => { + const desc = x[0] === "-"; + const key = (desc ? x.substring(1) : x) as T[number]; + if (key in remap) return { key: remap[key]!, remmapedKey: key, desc }; + return { key: key as Exclude, desc }; + }), }; }) .Encode(() => { From 9a54266967a662cf46fb64fa34f67fff445f48e5 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Sun, 12 Jan 2025 16:18:06 +0100 Subject: [PATCH 07/12] Add /movies/random route --- api/src/controllers/movies.ts | 37 ++++++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/api/src/controllers/movies.ts b/api/src/controllers/movies.ts index 34928261..13cc221c 100644 --- a/api/src/controllers/movies.ts +++ b/api/src/controllers/movies.ts @@ -1,5 +1,5 @@ import { and, eq, sql } from "drizzle-orm"; -import { Elysia, t } from "elysia"; +import { Elysia, redirect, t } from "elysia"; import { KError } from "~/models/error"; import { comment } from "~/utils"; import { db } from "../db"; @@ -154,6 +154,41 @@ export const movies = new Elysia({ prefix: "/movies", tags: ["movies"] }) }, }, ) + .get( + "random", + async ({ error, redirect }) => { + const [id] = await db + .select({ id: shows.id }) + .from(shows) + .where(eq(shows.kind, "movie")) + .orderBy(sql`random()`) + .limit(1); + if (!id) + return error(404, { + status: 404, + message: "No movies in the database", + }); + return redirect(`/movies/${id}`); + }, + { + detail: { + description: "Get a random movie", + }, + response: { + 302: t.Void({ + description: + "Redirected to the [/movies/id](#tag/movies/GET/movies/{id}) route.", + }), + 404: { + ...KError, + description: "No movie found with the given id or slug.", + examples: [ + { status: 404, message: "Movie not found", details: undefined }, + ], + }, + }, + }, + ) .get( "", async ({ From b9da57fd884730f6e321b6759bcfbb981b0628c7 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Sun, 12 Jan 2025 16:22:51 +0100 Subject: [PATCH 08/12] Cleanups --- api/src/controllers/movies.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/api/src/controllers/movies.ts b/api/src/controllers/movies.ts index 13cc221c..de20518b 100644 --- a/api/src/controllers/movies.ts +++ b/api/src/controllers/movies.ts @@ -95,14 +95,12 @@ export const movies = new Elysia({ prefix: "/movies", tags: ["movies"] }) return error(404, { status: 404, message: "Movie not found", - details: undefined, }); } if (!ret.translation) { return error(422, { status: 422, message: "Accept-Language header could not be satisfied.", - details: undefined, }); } set.headers["content-language"] = ret.translation.language; @@ -211,7 +209,7 @@ export const movies = new Elysia({ prefix: "/movies", tags: ["movies"] }) .innerJoin(transQ, eq(shows.pk, transQ.pk)) .where(and(filter, keysetPaginate({ table: shows, after, sort }))) .orderBy( - ...(sort.random !== undefined + ...(sort.random ? [sql`md5(${sort.random.seed} || ${shows.pk})`] : []), ...sort.sort.map((x) => From 46570410ea8e70333e258dfbfd0c125514274f7a Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Sun, 12 Jan 2025 16:31:43 +0100 Subject: [PATCH 09/12] Add seed in response if it was not specified --- api/src/models/utils/page.ts | 7 ++++++- api/tests/movies/get-all-movies.test.ts | 20 ++++++++++++++++++-- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/api/src/models/utils/page.ts b/api/src/models/utils/page.ts index daad081e..c7a18d1b 100644 --- a/api/src/models/utils/page.ts +++ b/api/src/models/utils/page.ts @@ -18,12 +18,17 @@ export const createPage = ( { url, sort, limit }: { url: string; sort: Sort; limit: number }, ) => { let next: string | null = null; + const uri = new URL(url); + + if (sort.random) { + uri.searchParams.set("sort", `random:${sort.random.seed}`); + url = uri.toString(); + } // we can't know for sure if there's a next page when the current page is full. // maybe the next page is empty, this is a bit weird but it allows us to handle pages // without making a new request to the db so it's fine. if (items.length === limit && limit > 0) { - const uri = new URL(url); uri.searchParams.set("after", generateAfter(items[items.length - 1], sort)); next = uri.toString(); } diff --git a/api/tests/movies/get-all-movies.test.ts b/api/tests/movies/get-all-movies.test.ts index c725d47e..e91c7be8 100644 --- a/api/tests/movies/get-all-movies.test.ts +++ b/api/tests/movies/get-all-movies.test.ts @@ -125,7 +125,7 @@ describe("Get all movies", () => { describe("Random sort", () => { it("No limit, compare order with same seeds", async () => { // First query - let [resp1, body1] = await getMovies({ + const [resp1, body1] = await getMovies({ sort: "random:100", }); expectStatus(resp1, body1).toBe(200); @@ -133,7 +133,7 @@ describe("Get all movies", () => { const items1Ids = items1.map(({ id }) => id); // Second query - let [resp2, body2] = await getMovies({ + const [resp2, body2] = await getMovies({ sort: "random:100", }); expectStatus(resp2, body2).toBe(200); @@ -170,5 +170,21 @@ describe("Get all movies", () => { expect(items.length).toBe(1); expect(items[0].id).toBe(expectedIds[1]); }); + it("Limit 1, pages 1 and 2, no seed ", async () => { + const [resp, body] = await getMovies({ + sort: "random", + limit: 2, + }); + expectStatus(resp, body).toBe(200); + + const resp2 = await movieApp.handle(new Request(body.next)); + const body2 = await resp2.json(); + expectStatus(resp2, body).toBe(200); + + expect(body2.items.length).toBe(1); + expect(body.items.map((x: Movie) => x.slug)).not.toContain( + body2.items[0].slug, + ); + }); }); }); From b6f996139fc9b693ad600286a10583398b0a2a5e Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Sun, 12 Jan 2025 16:39:27 +0100 Subject: [PATCH 10/12] Reserve the `random` slug --- api/src/controllers/seed/movies.ts | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/api/src/controllers/seed/movies.ts b/api/src/controllers/seed/movies.ts index 45b3b8a4..d4b36771 100644 --- a/api/src/controllers/seed/movies.ts +++ b/api/src/controllers/seed/movies.ts @@ -13,6 +13,7 @@ import { conflictUpdateAllExcept } from "~/db/schema/utils"; import type { SeedMovie } from "~/models/movie"; import { processOptImage } from "./images"; import { guessNextRefresh } from "./refresh"; +import { KErrorT } from "~/models/error"; type Show = typeof shows.$inferInsert; type ShowTrans = typeof showTranslations.$inferInsert; @@ -29,11 +30,18 @@ export type SeedMovieResponse = typeof SeedMovieResponse.static; export const seedMovie = async ( seed: SeedMovie, -): Promise< - SeedMovieResponse & { status: "Created" | "OK" | "Conflict" } -> => { +): Promise => { const { translations, videos: vids, ...bMovie } = seed; + if (seed.slug === "random") { + if (seed.airDate === null) { + throw new KErrorT("`random` is a reserved slug. Use something else."); + } + seed.slug = `random-${getYear(seed.airDate)}`; + } + const ret = await db.transaction(async (tx) => { const movie: Show = { kind: "movie", From 1cdb3720795456df2a0ad6bc4d630344f37c0b6a Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Mon, 13 Jan 2025 12:52:39 +0100 Subject: [PATCH 11/12] Fix & test /movies/random --- api/src/controllers/movies.ts | 8 ++++---- api/tests/movies/get-all-movies.test.ts | 14 +++++++++++++- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/api/src/controllers/movies.ts b/api/src/controllers/movies.ts index de20518b..c6e3485f 100644 --- a/api/src/controllers/movies.ts +++ b/api/src/controllers/movies.ts @@ -155,18 +155,18 @@ export const movies = new Elysia({ prefix: "/movies", tags: ["movies"] }) .get( "random", async ({ error, redirect }) => { - const [id] = await db + const [movie] = await db .select({ id: shows.id }) .from(shows) .where(eq(shows.kind, "movie")) .orderBy(sql`random()`) .limit(1); - if (!id) + if (!movie) return error(404, { status: 404, message: "No movies in the database", }); - return redirect(`/movies/${id}`); + return redirect(`/movies/${movie.id}`); }, { detail: { @@ -175,7 +175,7 @@ export const movies = new Elysia({ prefix: "/movies", tags: ["movies"] }) response: { 302: t.Void({ description: - "Redirected to the [/movies/id](#tag/movies/GET/movies/{id}) route.", + "Redirected to the [/movies/{id}](#tag/movies/GET/movies/{id}) route.", }), 404: { ...KError, diff --git a/api/tests/movies/get-all-movies.test.ts b/api/tests/movies/get-all-movies.test.ts index e91c7be8..d7f28a4e 100644 --- a/api/tests/movies/get-all-movies.test.ts +++ b/api/tests/movies/get-all-movies.test.ts @@ -6,8 +6,9 @@ import { shows } from "~/db/schema"; import { bubble } from "~/models/examples"; import { dune1984 } from "~/models/examples/dune-1984"; import { dune } from "~/models/examples/dune-2021"; -import { getMovies, movieApp } from "./movies-helper"; +import { getMovie, getMovies, movieApp } from "./movies-helper"; import type { Movie } from "~/models/movie"; +import { isUuid } from "~/models/utils"; beforeAll(async () => { await db.delete(shows); @@ -186,5 +187,16 @@ describe("Get all movies", () => { body2.items[0].slug, ); }); + + it("Get /random", async () => { + const resp = await movieApp.handle( + new Request("http://localhost/movies/random"), + ); + expect(resp.status).toBe(302); + const location = resp.headers.get("location")!; + expect(location).toStartWith("/movies/"); + const id = location.substring("/movies/".length); + expect(isUuid(id)).toBe(true); + }); }); }); From d1609ddfbf2fa5b3c18a7e4f38a09092b3869bb4 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Mon, 13 Jan 2025 14:15:16 +0100 Subject: [PATCH 12/12] Fix & test random slug reservation --- api/src/controllers/seed/index.ts | 5 +++-- api/src/controllers/seed/movies.ts | 18 +++++++++++------- api/tests/movies/seed-movies.test.ts | 10 ++++++++++ 3 files changed, 24 insertions(+), 9 deletions(-) diff --git a/api/src/controllers/seed/index.ts b/api/src/controllers/seed/index.ts index 8a5dac11..9926f37d 100644 --- a/api/src/controllers/seed/index.ts +++ b/api/src/controllers/seed/index.ts @@ -17,8 +17,9 @@ export const seed = new Elysia() const err = validateTranslations(body.translations); if (err) return error(400, err); - const { status, ...ret } = await seedMovie(body); - return error(status, ret); + const ret = await seedMovie(body); + if (ret.status === 422) return error(422, ret); + return error(ret.status, ret); }, { body: "seed-movie", diff --git a/api/src/controllers/seed/movies.ts b/api/src/controllers/seed/movies.ts index d4b36771..396fcd27 100644 --- a/api/src/controllers/seed/movies.ts +++ b/api/src/controllers/seed/movies.ts @@ -30,18 +30,22 @@ export type SeedMovieResponse = typeof SeedMovieResponse.static; export const seedMovie = async ( seed: SeedMovie, -): Promise => { - const { translations, videos: vids, ...bMovie } = seed; - +): Promise< + | (SeedMovieResponse & { status: "Created" | "OK" | "Conflict" }) + | { status: 422; message: string } +> => { if (seed.slug === "random") { - if (seed.airDate === null) { - throw new KErrorT("`random` is a reserved slug. Use something else."); + if (!seed.airDate) { + return { + status: 422, + message: "`random` is a reserved slug. Use something else.", + }; } seed.slug = `random-${getYear(seed.airDate)}`; } + const { translations, videos: vids, ...bMovie } = seed; + const ret = await db.transaction(async (tx) => { const movie: Show = { kind: "movie", diff --git a/api/tests/movies/seed-movies.test.ts b/api/tests/movies/seed-movies.test.ts index cb32428a..827072d3 100644 --- a/api/tests/movies/seed-movies.test.ts +++ b/api/tests/movies/seed-movies.test.ts @@ -192,6 +192,16 @@ describe("Movie seeding", () => { }); }); + it("Refuses random as a slug", async () => { + const [resp, body] = await createMovie({ ...bubble, slug: "random", airDate: null }); + expectStatus(resp, body).toBe(422); + }); + it("Refuses random as a slug but fallback w/ airDate", async () => { + const [resp, body] = await createMovie({ ...bubble, slug: "random" }); + expectStatus(resp, body).toBe(201); + expect(body.slug).toBe("random-2022"); + }); + test.todo("Create correct video slug (version)", async () => {}); test.todo("Create correct video slug (part)", async () => {}); test.todo("Create correct video slug (rendering)", async () => {});