From 4dc34641ecbedeb5c691b134e033b122fa88a02c Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Sun, 2 Nov 2025 18:11:25 +0100 Subject: [PATCH 01/13] Handle cookies in keibi --- auth/jwt.go | 18 ++++++++++++++---- docker-compose.dev.yml | 6 +++--- docker-compose.yml | 6 +++--- 3 files changed, 20 insertions(+), 10 deletions(-) diff --git a/auth/jwt.go b/auth/jwt.go index a0977593..7e78f875 100644 --- a/auth/jwt.go +++ b/auth/jwt.go @@ -40,13 +40,23 @@ func (h *Handler) CreateJwt(c echo.Context) error { } auth := c.Request().Header.Get("Authorization") - var jwt *string + var token string - if !strings.HasPrefix(auth, "Bearer ") { + if auth == "" { + c, _ := c.Request().Cookie("X-Bearer") + if c != nil { + token = c.Value + } + } else if strings.HasPrefix(auth, "Bearer ") { + token = auth[len("Bearer "):] + } else if auth != "" { + return echo.NewHTTPError(http.StatusUnauthorized, "Invalid bearer format.") + } + + var jwt *string + if token == "" { jwt = h.createGuestJwt() } else { - token := auth[len("Bearer "):] - tkn, err := h.createJwt(token) if err != nil { return err diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 802e05f1..b02600d6 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -23,7 +23,7 @@ x-transcoder: &transcoder-base - "traefik.http.routers.transcoder.rule=PathPrefix(`/video`)" - "traefik.http.routers.transcoder.middlewares=phantom-token" - "traefik.http.middlewares.phantom-token.forwardauth.address=http://auth:4568/auth/jwt" - - "traefik.http.middlewares.phantom-token.forwardauth.authRequestHeaders=Authorization,X-Api-Key" + - "traefik.http.middlewares.phantom-token.forwardauth.authRequestHeaders=Authorization,Cookie,X-Api-Key" - "traefik.http.middlewares.phantom-token.forwardauth.authResponseHeaders=Authorization" develop: watch: @@ -94,7 +94,7 @@ services: - "traefik.http.routers.api.rule=PathPrefix(`/api/`) || PathPrefix(`/swagger`)" - "traefik.http.routers.api.middlewares=phantom-token" - "traefik.http.middlewares.phantom-token.forwardauth.address=http://auth:4568/auth/jwt" - - "traefik.http.middlewares.phantom-token.forwardauth.authRequestHeaders=Authorization,X-Api-Key" + - "traefik.http.middlewares.phantom-token.forwardauth.authRequestHeaders=Authorization,Cookie,X-Api-Key" - "traefik.http.middlewares.phantom-token.forwardauth.authResponseHeaders=Authorization" develop: watch: @@ -129,7 +129,7 @@ services: - "traefik.http.routers.scanner.rule=PathPrefix(`/scanner/`)" - "traefik.http.routers.scanner.middlewares=phantom-token" - "traefik.http.middlewares.phantom-token.forwardauth.address=http://auth:4568/auth/jwt" - - "traefik.http.middlewares.phantom-token.forwardauth.authRequestHeaders=Authorization,X-Api-Key" + - "traefik.http.middlewares.phantom-token.forwardauth.authRequestHeaders=Authorization,Cookie,X-Api-Key" - "traefik.http.middlewares.phantom-token.forwardauth.authResponseHeaders=Authorization" command: fastapi dev scanner --host 0.0.0.0 --port 4389 develop: diff --git a/docker-compose.yml b/docker-compose.yml index 1c561b17..b8cd7c5e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -19,7 +19,7 @@ x-transcoder: &transcoder-base - "traefik.http.routers.transcoder.rule=PathPrefix(`/video`)" - "traefik.http.routers.transcoder.middlewares=phantom-token" - "traefik.http.middlewares.phantom-token.forwardauth.address=http://auth:4568/auth/jwt" - - "traefik.http.middlewares.phantom-token.forwardauth.authRequestHeaders=Authorization,X-Api-Key" + - "traefik.http.middlewares.phantom-token.forwardauth.authRequestHeaders=Authorization,Cookie,X-Api-Key" - "traefik.http.middlewares.phantom-token.forwardauth.authResponseHeaders=Authorization" services: @@ -64,7 +64,7 @@ services: - "traefik.http.routers.api.rule=PathPrefix(`/api/`) || PathPrefix(`/swagger`)" - "traefik.http.routers.api.middlewares=phantom-token" - "traefik.http.middlewares.phantom-token.forwardauth.address=http://auth:4568/auth/jwt" - - "traefik.http.middlewares.phantom-token.forwardauth.authRequestHeaders=Authorization,X-Api-Key" + - "traefik.http.middlewares.phantom-token.forwardauth.authRequestHeaders=Authorization,Cookie,X-Api-Key" - "traefik.http.middlewares.phantom-token.forwardauth.authResponseHeaders=Authorization" scanner: @@ -86,7 +86,7 @@ services: - "traefik.http.routers.scanner.rule=PathPrefix(`/scanner/`)" - "traefik.http.routers.scanner.middlewares=phantom-token" - "traefik.http.middlewares.phantom-token.forwardauth.address=http://auth:4568/auth/jwt" - - "traefik.http.middlewares.phantom-token.forwardauth.authRequestHeaders=Authorization,X-Api-Key" + - "traefik.http.middlewares.phantom-token.forwardauth.authRequestHeaders=Authorization,Cookie,X-Api-Key" - "traefik.http.middlewares.phantom-token.forwardauth.authResponseHeaders=Authorization" transcoder: From 5827cc32e8fb99db012b44d0cc8c38a1bdbef440 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Sun, 2 Nov 2025 18:17:18 +0100 Subject: [PATCH 02/13] Hard code keibi postgres schema --- auth/.env.example | 5 ----- auth/dbc/apikeys.sql.go | 12 +++++------ auth/dbc/db.go | 2 +- auth/dbc/models.go | 2 +- auth/dbc/sessions.sql.go | 18 ++++++++-------- auth/dbc/users.sql.go | 22 ++++++++++---------- auth/main.go | 17 ++++----------- auth/sql/migrations/000001_users.down.sql | 4 ++-- auth/sql/migrations/000001_users.up.sql | 8 ++++--- auth/sql/migrations/000002_sessions.down.sql | 2 +- auth/sql/migrations/000002_sessions.up.sql | 4 ++-- auth/sql/migrations/000003_apikeys.down.sql | 2 +- auth/sql/migrations/000003_apikeys.up.sql | 4 ++-- auth/sql/queries/apikeys.sql | 10 ++++----- auth/sql/queries/sessions.sql | 16 +++++++------- auth/sql/queries/users.sql | 20 +++++++++--------- auth/sqlc.yaml | 13 ++++++++---- 17 files changed, 77 insertions(+), 84 deletions(-) diff --git a/auth/.env.example b/auth/.env.example index 39220724..d842cb5a 100644 --- a/auth/.env.example +++ b/auth/.env.example @@ -43,8 +43,3 @@ PGPORT=5432 # PGSSLROOTCERT=/my/serving.crt # PGSSLCERT=/my/client.crt # PGSSLKEY=/my/client.key - -# Default is keibi, you can specify "disabled" to use the default search_path of the user. -# If this is not "disabled", the schema will be created (if it does not exists) and -# the search_path of the user will be ignored (only the schema specified will be used). -POSTGRES_SCHEMA=keibi diff --git a/auth/dbc/apikeys.sql.go b/auth/dbc/apikeys.sql.go index fa4d1643..698f8733 100644 --- a/auth/dbc/apikeys.sql.go +++ b/auth/dbc/apikeys.sql.go @@ -1,6 +1,6 @@ // Code generated by sqlc. DO NOT EDIT. // versions: -// sqlc v1.28.0 +// sqlc v1.30.0 // source: apikeys.sql package dbc @@ -13,7 +13,7 @@ import ( ) const createApiKey = `-- name: CreateApiKey :one -insert into apikeys(name, token, claims, created_by) +insert into keibi.apikeys(name, token, claims, created_by) values ($1, $2, $3, $4) returning pk, id, name, token, claims, created_by, created_at, last_used @@ -48,7 +48,7 @@ func (q *Queries) CreateApiKey(ctx context.Context, arg CreateApiKeyParams) (Api } const deleteApiKey = `-- name: DeleteApiKey :one -delete from apikeys +delete from keibi.apikeys where id = $1 returning pk, id, name, token, claims, created_by, created_at, last_used @@ -74,7 +74,7 @@ const getApiKey = `-- name: GetApiKey :one select pk, id, name, token, claims, created_by, created_at, last_used from - apikeys + keibi.apikeys where name = $1 and token = $2 @@ -105,7 +105,7 @@ const listApiKeys = `-- name: ListApiKeys :many select pk, id, name, token, claims, created_by, created_at, last_used from - apikeys + keibi.apikeys order by last_used ` @@ -141,7 +141,7 @@ func (q *Queries) ListApiKeys(ctx context.Context) ([]Apikey, error) { const touchApiKey = `-- name: TouchApiKey :exec update - apikeys + keibi.apikeys set last_used = now()::timestamptz where diff --git a/auth/dbc/db.go b/auth/dbc/db.go index babe8e31..dcfc5dbe 100644 --- a/auth/dbc/db.go +++ b/auth/dbc/db.go @@ -1,6 +1,6 @@ // Code generated by sqlc. DO NOT EDIT. // versions: -// sqlc v1.28.0 +// sqlc v1.30.0 package dbc diff --git a/auth/dbc/models.go b/auth/dbc/models.go index 7bf7c38f..fcd423c2 100644 --- a/auth/dbc/models.go +++ b/auth/dbc/models.go @@ -1,6 +1,6 @@ // Code generated by sqlc. DO NOT EDIT. // versions: -// sqlc v1.28.0 +// sqlc v1.30.0 package dbc diff --git a/auth/dbc/sessions.sql.go b/auth/dbc/sessions.sql.go index bcb81869..3c6ef7ae 100644 --- a/auth/dbc/sessions.sql.go +++ b/auth/dbc/sessions.sql.go @@ -1,6 +1,6 @@ // Code generated by sqlc. DO NOT EDIT. // versions: -// sqlc v1.28.0 +// sqlc v1.30.0 // source: sessions.sql package dbc @@ -13,7 +13,7 @@ import ( ) const clearOtherSessions = `-- name: ClearOtherSessions :exec -delete from sessions as s using users as u +delete from keibi.sessions as s using keibi.users as u where s.user_pk = u.pk and s.id != $1 and u.id = $2 @@ -30,7 +30,7 @@ func (q *Queries) ClearOtherSessions(ctx context.Context, arg ClearOtherSessions } const createSession = `-- name: CreateSession :one -insert into sessions(token, user_pk, device) +insert into keibi.sessions(token, user_pk, device) values ($1, $2, $3) returning pk, id, token, user_pk, created_date, last_used, device @@ -58,7 +58,7 @@ func (q *Queries) CreateSession(ctx context.Context, arg CreateSessionParams) (S } const deleteSession = `-- name: DeleteSession :one -delete from sessions as s using users as u +delete from keibi.sessions as s using keibi.users as u where s.user_pk = u.pk and s.id = $1 and u.id = $2 @@ -93,8 +93,8 @@ select s.last_used, u.pk, u.id, u.username, u.email, u.password, u.claims, u.created_date, u.last_seen from - users as u - inner join sessions as s on u.pk = s.user_pk + keibi.users as u + inner join keibi.sessions as s on u.pk = s.user_pk where s.token = $1 limit 1 @@ -130,8 +130,8 @@ const getUserSessions = `-- name: GetUserSessions :many select s.pk, s.id, s.token, s.user_pk, s.created_date, s.last_used, s.device from - sessions as s - inner join users as u on u.pk = s.user_pk + keibi.sessions as s + inner join keibi.users as u on u.pk = s.user_pk where u.pk = $1 order by @@ -168,7 +168,7 @@ func (q *Queries) GetUserSessions(ctx context.Context, pk int32) ([]Session, err const touchSession = `-- name: TouchSession :exec update - sessions + keibi.sessions set last_used = now()::timestamptz where diff --git a/auth/dbc/users.sql.go b/auth/dbc/users.sql.go index 02964f00..3d8bfbb4 100644 --- a/auth/dbc/users.sql.go +++ b/auth/dbc/users.sql.go @@ -1,6 +1,6 @@ // Code generated by sqlc. DO NOT EDIT. // versions: -// sqlc v1.28.0 +// sqlc v1.30.0 // source: users.sql package dbc @@ -13,12 +13,12 @@ import ( ) const createUser = `-- name: CreateUser :one -insert into users(username, email, password, claims) +insert into keibi.users(username, email, password, claims) values ($1, $2, $3, case when not exists ( select pk, id, username, email, password, claims, created_date, last_seen from - users) then + keibi.users) then $4::jsonb else $5::jsonb @@ -58,7 +58,7 @@ func (q *Queries) CreateUser(ctx context.Context, arg CreateUserParams) (User, e } const deleteUser = `-- name: DeleteUser :one -delete from users +delete from keibi.users where id = $1 returning pk, id, username, email, password, claims, created_date, last_seen @@ -84,7 +84,7 @@ const getAllUsers = `-- name: GetAllUsers :many select pk, id, username, email, password, claims, created_date, last_seen from - users + keibi.users order by id limit $1 @@ -123,7 +123,7 @@ const getAllUsersAfter = `-- name: GetAllUsersAfter :many select pk, id, username, email, password, claims, created_date, last_seen from - users + keibi.users where id >= $2 order by @@ -173,8 +173,8 @@ select h.username, h.profile_url from - users as u - left join oidc_handle as h on u.pk = h.user_pk + keibi.users as u + left join keibi.oidc_handle as h on u.pk = h.user_pk where ($1::boolean and u.id = $2) or (not $1 @@ -232,7 +232,7 @@ const getUserByLogin = `-- name: GetUserByLogin :one select pk, id, username, email, password, claims, created_date, last_seen from - users + keibi.users where email = $1 or username = $1 @@ -257,7 +257,7 @@ func (q *Queries) GetUserByLogin(ctx context.Context, login string) (User, error const touchUser = `-- name: TouchUser :exec update - users + keibi.users set last_used = now()::timestamptz where @@ -271,7 +271,7 @@ func (q *Queries) TouchUser(ctx context.Context, pk int32) error { const updateUser = `-- name: UpdateUser :one update - users + keibi.users set username = coalesce($2, username), email = coalesce($3, email), diff --git a/auth/main.go b/auth/main.go index 4360df5c..b56a6f67 100644 --- a/auth/main.go +++ b/auth/main.go @@ -106,29 +106,20 @@ func OpenDatabase() (*pgxpool.Pool, error) { config.ConnConfig.RuntimeParams["application_name"] = "keibi" } - schema := GetenvOr("POSTGRES_SCHEMA", "keibi") - if _, ok := config.ConnConfig.RuntimeParams["search_path"]; !ok { - config.ConnConfig.RuntimeParams["search_path"] = schema - } - db, err := pgxpool.NewWithConfig(ctx, config) if err != nil { fmt.Printf("Could not connect to database, check your env variables!\n") return nil, err } - if schema != "disabled" { - _, err = db.Exec(ctx, fmt.Sprintf("create schema if not exists %s", schema)) - if err != nil { - return nil, err - } - } - fmt.Println("Migrating database") dbi := stdlib.OpenDBFromPool(db) defer dbi.Close() - driver, err := pgxd.WithInstance(dbi, &pgxd.Config{}) + dbi.Exec("create schema if not exists keibi") + driver, err := pgxd.WithInstance(dbi, &pgxd.Config{ + SchemaName: "keibi", + }) if err != nil { return nil, err } diff --git a/auth/sql/migrations/000001_users.down.sql b/auth/sql/migrations/000001_users.down.sql index d70106fc..160485ef 100644 --- a/auth/sql/migrations/000001_users.down.sql +++ b/auth/sql/migrations/000001_users.down.sql @@ -1,6 +1,6 @@ begin; -drop table oidc_handle; -drop table users; +drop table keibi.oidc_handle; +drop table keibi.users; commit; diff --git a/auth/sql/migrations/000001_users.up.sql b/auth/sql/migrations/000001_users.up.sql index 7076151d..9123f396 100644 --- a/auth/sql/migrations/000001_users.up.sql +++ b/auth/sql/migrations/000001_users.up.sql @@ -1,6 +1,8 @@ begin; -create table users( +create schema if not exists keibi; + +create table keibi.users( pk serial primary key, id uuid not null default gen_random_uuid(), username varchar(256) not null unique, @@ -12,8 +14,8 @@ create table users( last_seen timestamptz not null default now()::timestamptz ); -create table oidc_handle( - user_pk integer not null references users(pk) on delete cascade, +create table keibi.oidc_handle( + user_pk integer not null references keibi.users(pk) on delete cascade, provider varchar(256) not null, id text not null, diff --git a/auth/sql/migrations/000002_sessions.down.sql b/auth/sql/migrations/000002_sessions.down.sql index 7a17cccd..0a1c6e5f 100644 --- a/auth/sql/migrations/000002_sessions.down.sql +++ b/auth/sql/migrations/000002_sessions.down.sql @@ -1,5 +1,5 @@ begin; -drop table sessions; +drop table keibi.sessions; commit; diff --git a/auth/sql/migrations/000002_sessions.up.sql b/auth/sql/migrations/000002_sessions.up.sql index 61e11291..8728be1c 100644 --- a/auth/sql/migrations/000002_sessions.up.sql +++ b/auth/sql/migrations/000002_sessions.up.sql @@ -1,10 +1,10 @@ begin; -create table sessions( +create table keibi.sessions( pk serial primary key, id uuid not null default gen_random_uuid(), token varchar(128) not null unique, - user_pk integer not null references users(pk) on delete cascade, + user_pk integer not null references keibi.users(pk) on delete cascade, created_date timestamptz not null default now()::timestamptz, last_used timestamptz not null default now()::timestamptz, device varchar(1024) diff --git a/auth/sql/migrations/000003_apikeys.down.sql b/auth/sql/migrations/000003_apikeys.down.sql index 3bdbcde3..637af454 100644 --- a/auth/sql/migrations/000003_apikeys.down.sql +++ b/auth/sql/migrations/000003_apikeys.down.sql @@ -1,5 +1,5 @@ begin; -drop table apikeys; +drop table keibi.apikeys; commit; diff --git a/auth/sql/migrations/000003_apikeys.up.sql b/auth/sql/migrations/000003_apikeys.up.sql index 22ddbd86..a301abc5 100644 --- a/auth/sql/migrations/000003_apikeys.up.sql +++ b/auth/sql/migrations/000003_apikeys.up.sql @@ -1,13 +1,13 @@ begin; -create table apikeys( +create table keibi.apikeys( pk serial primary key, id uuid not null default gen_random_uuid(), name varchar(256) not null unique, token varchar(128) not null unique, claims jsonb not null, - created_by integer references users(pk) on delete cascade, + created_by integer references keibi.users(pk) on delete cascade, created_at timestamptz not null default now()::timestamptz, last_used timestamptz not null default now()::timestamptz ); diff --git a/auth/sql/queries/apikeys.sql b/auth/sql/queries/apikeys.sql index 634038c1..f340d9fe 100644 --- a/auth/sql/queries/apikeys.sql +++ b/auth/sql/queries/apikeys.sql @@ -2,14 +2,14 @@ select * from - apikeys + keibi.apikeys where name = $1 and token = $2; -- name: TouchApiKey :exec update - apikeys + keibi.apikeys set last_used = now()::timestamptz where @@ -19,18 +19,18 @@ where select * from - apikeys + keibi.apikeys order by last_used; -- name: CreateApiKey :one -insert into apikeys(name, token, claims, created_by) +insert into keibi.apikeys(name, token, claims, created_by) values ($1, $2, $3, $4) returning *; -- name: DeleteApiKey :one -delete from apikeys +delete from keibi.apikeys where id = $1 returning *; diff --git a/auth/sql/queries/sessions.sql b/auth/sql/queries/sessions.sql index a2a06727..187f1627 100644 --- a/auth/sql/queries/sessions.sql +++ b/auth/sql/queries/sessions.sql @@ -5,15 +5,15 @@ select s.last_used, sqlc.embed(u) from - users as u - inner join sessions as s on u.pk = s.user_pk + keibi.users as u + inner join keibi.sessions as s on u.pk = s.user_pk where s.token = $1 limit 1; -- name: TouchSession :exec update - sessions + keibi.sessions set last_used = now()::timestamptz where @@ -23,21 +23,21 @@ where select s.* from - sessions as s - inner join users as u on u.pk = s.user_pk + keibi.sessions as s + inner join keibi.users as u on u.pk = s.user_pk where u.pk = $1 order by last_used; -- name: CreateSession :one -insert into sessions(token, user_pk, device) +insert into keibi.sessions(token, user_pk, device) values ($1, $2, $3) returning *; -- name: DeleteSession :one -delete from sessions as s using users as u +delete from keibi.sessions as s using keibi.users as u where s.user_pk = u.pk and s.id = $1 and u.id = sqlc.arg(user_id) @@ -45,7 +45,7 @@ returning s.*; -- name: ClearOtherSessions :exec -delete from sessions as s using users as u +delete from keibi.sessions as s using keibi.users as u where s.user_pk = u.pk and s.id != @session_id and u.id = @user_id; diff --git a/auth/sql/queries/users.sql b/auth/sql/queries/users.sql index b80181c2..f73dce75 100644 --- a/auth/sql/queries/users.sql +++ b/auth/sql/queries/users.sql @@ -2,7 +2,7 @@ select * from - users + keibi.users order by id limit $1; @@ -11,7 +11,7 @@ limit $1; select * from - users + keibi.users where id >= sqlc.arg(after_id) order by @@ -26,8 +26,8 @@ select h.username, h.profile_url from - users as u - left join oidc_handle as h on u.pk = h.user_pk + keibi.users as u + left join keibi.oidc_handle as h on u.pk = h.user_pk where (@use_id::boolean and u.id = @id) or (not @use_id @@ -37,7 +37,7 @@ where (@use_id::boolean select * from - users + keibi.users where email = sqlc.arg(login) or username = sqlc.arg(login) @@ -45,19 +45,19 @@ limit 1; -- name: TouchUser :exec update - users + keibi.users set last_used = now()::timestamptz where pk = $1; -- name: CreateUser :one -insert into users(username, email, password, claims) +insert into keibi.users(username, email, password, claims) values ($1, $2, $3, case when not exists ( select * from - users) then + keibi.users) then sqlc.arg(first_claims)::jsonb else sqlc.arg(claims)::jsonb @@ -67,7 +67,7 @@ returning -- name: UpdateUser :one update - users + keibi.users set username = coalesce(sqlc.narg(username), username), email = coalesce(sqlc.narg(email), email), @@ -79,7 +79,7 @@ returning *; -- name: DeleteUser :one -delete from users +delete from keibi.users where id = $1 returning *; diff --git a/auth/sqlc.yaml b/auth/sqlc.yaml index 638f61b5..611f2a36 100644 --- a/auth/sqlc.yaml +++ b/auth/sqlc.yaml @@ -30,15 +30,20 @@ sql: - db_type: "jsonb" go_type: type: "interface{}" - - column: "users.claims" + - column: "keibi.users.claims" go_type: import: "github.com/golang-jwt/jwt/v5" package: "jwt" type: "MapClaims" - - column: "apikeys.claims" + - column: "keibi.apikeys.claims" go_type: import: "github.com/golang-jwt/jwt/v5" package: "jwt" type: "MapClaims" - - +overrides: + go: + rename: + keibi_apikey: Apikey + keibi_oidc_handle: OidcHandle + keibi_session: Session + keibi_user: User From f1ddc7e7b91588180db1c1cc606e78dbf9fb593b Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Sun, 2 Nov 2025 18:21:26 +0100 Subject: [PATCH 03/13] Hardcode gocoder db schema --- transcoder/.env.example | 4 ---- transcoder/src/extract.go | 2 +- transcoder/src/keyframes.go | 6 +++--- transcoder/src/metadata.go | 32 ++++++++++---------------------- transcoder/src/thumbnails.go | 2 +- 5 files changed, 15 insertions(+), 31 deletions(-) diff --git a/transcoder/.env.example b/transcoder/.env.example index 577830c4..d744c30f 100644 --- a/transcoder/.env.example +++ b/transcoder/.env.example @@ -34,10 +34,6 @@ POSTGRES_SERVER= POSTGRES_PORT=5432 # can also be "require" ("prefer" is not supported) POSTGRES_SSLMODE="disable" -# Default is gocoder, you can specify "disabled" to use the default search_path of the user. -# If this is not "disabled", the schema will be created (if it does not exists) and -# the search_path of the user will be ignored (only the schema specified will be used). -POSTGRES_SCHEMA=gocoder # Storage backend # There are two currently supported backends: local filesystem and s3. diff --git a/transcoder/src/extract.go b/transcoder/src/extract.go index 91040704..354dc791 100644 --- a/transcoder/src/extract.go +++ b/transcoder/src/extract.go @@ -27,7 +27,7 @@ func (s *MetadataService) ExtractSubs(ctx context.Context, info *MediaInfo) (any log.Printf("Couldn't extract subs: %v", err) return set(nil, err) } - _, err = s.database.Exec(`update info set ver_extract = $2 where sha = $1`, info.Sha, ExtractVersion) + _, err = s.database.Exec(`update gocoder.info set ver_extract = $2 where sha = $1`, info.Sha, ExtractVersion) return set(nil, err) } diff --git a/transcoder/src/keyframes.go b/transcoder/src/keyframes.go index 1c08ac6f..1234fe00 100644 --- a/transcoder/src/keyframes.go +++ b/transcoder/src/keyframes.go @@ -134,10 +134,10 @@ func (s *MetadataService) GetKeyframes(info *MediaInfo, isVideo bool, idx uint32 var table string var err error if isVideo { - table = "videos" + table = "gocoder.videos" err = getVideoKeyframes(info.Path, idx, kf) } else { - table = "audios" + table = "gocoder.audios" err = getAudioKeyframes(info, idx, kf) } @@ -154,7 +154,7 @@ func (s *MetadataService) GetKeyframes(info *MediaInfo, isVideo bool, idx uint32 idx, pq.Array(kf.Keyframes), ) - tx.Exec(`update info set ver_keyframes = $2 where sha = $1`, info.Sha, KeyframeVersion) + tx.Exec(`update gocoder.info set ver_keyframes = $2 where sha = $1`, info.Sha, KeyframeVersion) err = tx.Commit() if err != nil { log.Printf("Couldn't store keyframes on database: %v", err) diff --git a/transcoder/src/metadata.go b/transcoder/src/metadata.go index ffaa009a..7e2ff60c 100644 --- a/transcoder/src/metadata.go +++ b/transcoder/src/metadata.go @@ -78,8 +78,6 @@ func (s *MetadataService) Close() error { } func (s *MetadataService) setupDb() (*sql.DB, error) { - schema := GetEnvOr("POSTGRES_SCHEMA", "gocoder") - connectionString := os.Getenv("POSTGRES_URL") if connectionString == "" { connectionString = fmt.Sprintf( @@ -91,9 +89,6 @@ func (s *MetadataService) setupDb() (*sql.DB, error) { url.QueryEscape(os.Getenv("POSTGRES_DB")), url.QueryEscape(GetEnvOr("POSTGRES_SSLMODE", "disable")), ) - if schema != "disabled" { - connectionString = fmt.Sprintf("%s&search_path=%s", connectionString, url.QueryEscape(schema)) - } } db, err := sql.Open("postgres", connectionString) @@ -102,13 +97,6 @@ func (s *MetadataService) setupDb() (*sql.DB, error) { return nil, err } - if schema != "disabled" { - _, err = db.Exec(fmt.Sprintf("create schema if not exists %s", schema)) - if err != nil { - return nil, err - } - } - driver, err := postgres.WithInstance(db, &postgres.Config{}) if err != nil { return nil, err @@ -169,9 +157,9 @@ func (s *MetadataService) GetMetadata(ctx context.Context, path string, sha stri if err != nil { return nil, err } - tx.Exec(`update videos set keyframes = null where sha = $1`, sha) - tx.Exec(`update audios set keyframes = null where sha = $1`, sha) - tx.Exec(`update info set ver_keyframes = 0 where sha = $1`, sha) + tx.Exec(`update gocoder.videos set keyframes = null where sha = $1`, sha) + tx.Exec(`update gocoder.audios set keyframes = null where sha = $1`, sha) + tx.Exec(`update gocoder.info set ver_keyframes = 0 where sha = $1`, sha) err = tx.Commit() if err != nil { fmt.Printf("error deleting old keyframes from database: %v", err) @@ -187,7 +175,7 @@ func (s *MetadataService) getMetadata(path string, sha string) (*MediaInfo, erro err := s.database.QueryRow( `select i.sha, i.path, i.extension, i.mime_codec, i.size, i.duration, i.container, i.fonts, i.ver_info, i.ver_extract, i.ver_thumbs, i.ver_keyframes - from info as i where i.sha=$1`, + from gocoder.info as i where i.sha=$1`, sha, ).Scan( &ret.Sha, &ret.Path, &ret.Extension, &ret.MimeCodec, &ret.Size, &ret.Duration, &ret.Container, @@ -307,9 +295,9 @@ func (s *MetadataService) storeFreshMetadata(path string, sha string) (*MediaInf // it needs to be a delete instead of a on conflict do update because we want to trigger delete casquade for // videos/audios & co. - tx.Exec(`delete from info where path = $1`, path) + tx.Exec(`delete from gocoder.info where path = $1`, path) tx.Exec(` - insert into info(sha, path, extension, mime_codec, size, duration, container, + insert into gocoder.info(sha, path, extension, mime_codec, size, duration, container, fonts, ver_info, ver_extract, ver_thumbs, ver_keyframes) values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) `, @@ -319,7 +307,7 @@ func (s *MetadataService) storeFreshMetadata(path string, sha string) (*MediaInf ) for _, v := range ret.Videos { tx.Exec(` - insert into videos(sha, idx, title, language, codec, mime_codec, width, height, is_default, bitrate) + insert into gocoder.videos(sha, idx, title, language, codec, mime_codec, width, height, is_default, bitrate) values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) on conflict (sha, idx) do update set sha = excluded.sha, @@ -338,7 +326,7 @@ func (s *MetadataService) storeFreshMetadata(path string, sha string) (*MediaInf } for _, a := range ret.Audios { tx.Exec(` - insert into audios(sha, idx, title, language, codec, mime_codec, is_default, bitrate) + insert into gocoder.audios(sha, idx, title, language, codec, mime_codec, is_default, bitrate) values ($1, $2, $3, $4, $5, $6, $7, $8) on conflict (sha, idx) do update set sha = excluded.sha, @@ -355,7 +343,7 @@ func (s *MetadataService) storeFreshMetadata(path string, sha string) (*MediaInf } for _, s := range ret.Subtitles { tx.Exec(` - insert into subtitles(sha, idx, title, language, codec, mime_codec, extension, is_default, is_forced, is_hearing_impaired) + insert into gocoder.subtitles(sha, idx, title, language, codec, mime_codec, extension, is_default, is_forced, is_hearing_impaired) values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) on conflict (sha, idx) do update set sha = excluded.sha, @@ -374,7 +362,7 @@ func (s *MetadataService) storeFreshMetadata(path string, sha string) (*MediaInf } for _, c := range ret.Chapters { tx.Exec(` - insert into chapters(sha, start_time, end_time, name, type) + insert into gocoder.chapters(sha, start_time, end_time, name, type) values ($1, $2, $3, $4, $5) on conflict (sha, start_time) do update set sha = excluded.sha, diff --git a/transcoder/src/thumbnails.go b/transcoder/src/thumbnails.go index 4b8edea8..d9252646 100644 --- a/transcoder/src/thumbnails.go +++ b/transcoder/src/thumbnails.go @@ -77,7 +77,7 @@ func (s *MetadataService) ExtractThumbs(ctx context.Context, path string, sha st if err != nil { return set(nil, err) } - _, err = s.database.Exec(`update info set ver_thumbs = $2 where sha = $1`, sha, ThumbsVersion) + _, err = s.database.Exec(`update gocoder.info set ver_thumbs = $2 where sha = $1`, sha, ThumbsVersion) return set(nil, err) } From 04171af3e34ed5a09b4497d0713067ddc1885b4c Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Sun, 2 Nov 2025 18:37:07 +0100 Subject: [PATCH 04/13] Require `core.play` to play videos in gocoder --- transcoder/main.go | 34 +++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/transcoder/main.go b/transcoder/main.go index 84de3650..4601c10a 100644 --- a/transcoder/main.go +++ b/transcoder/main.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "net/http" + "slices" _ "github.com/zoriya/kyoo/transcoder/docs" @@ -37,6 +38,35 @@ func ErrorHandler(err error, c echo.Context) { }{Errors: []string{message}}) } +func RequireCorePlayPermission(next echo.HandlerFunc) echo.HandlerFunc { + return func(c echo.Context) error { + user := c.Get("user") + if user == nil { + return echo.NewHTTPError(http.StatusForbidden, "missing jwt") + } + token, ok := user.(*jwt.Token) + if !ok { + return echo.NewHTTPError(http.StatusForbidden, "invalid jwt") + } + claims, ok := token.Claims.(jwt.MapClaims) + if !ok { + return echo.NewHTTPError(http.StatusForbidden, "invalid jwt claims") + } + permissions, ok := claims["permissions"] + if !ok { + return echo.NewHTTPError(http.StatusForbidden, "missing permissions claim") + } + perms, ok := permissions.([]any) + if !ok { + return echo.NewHTTPError(http.StatusForbidden, "permissions claim is not an array") + } + if !slices.Contains(perms, "core.play") { + return echo.NewHTTPError(http.StatusForbidden, "missing core.play permission") + } + return next(c) + } +} + // @title gocoder - Kyoo's transcoder // @version 1.0 // @description Real time transcoder. @@ -103,7 +133,7 @@ func main() { return nil, fmt.Errorf("unable to find key %q", kid) } - var pubkey interface{} + var pubkey any if err := jwk.Export(key, &pubkey); err != nil { return nil, fmt.Errorf("Unable to get the public key. Error: %s", err.Error()) } @@ -111,6 +141,8 @@ func main() { return pubkey, nil }, })) + + g.Use(RequireCorePlayPermission) } api.RegisterStreamHandlers(g, transcoder) From 165d9e8f31e6db4730ccdfced0b79072832c056c Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Sun, 2 Nov 2025 18:40:23 +0100 Subject: [PATCH 05/13] Update .env.example --- .env.example | 62 ++++++++++++++++++++++++++++------------------------ 1 file changed, 33 insertions(+), 29 deletions(-) diff --git a/.env.example b/.env.example index a361dd8a..7e5647d5 100644 --- a/.env.example +++ b/.env.example @@ -21,8 +21,8 @@ GOCODER_PRESET=fast # Keep those empty to use kyoo's default api key. You can also specify a custom API key if you want. -# go to https://www.themoviedb.org/settings/api and copy the api key (not the read access token, the api key) -THEMOVIEDB_APIKEY= +# go to https://www.themoviedb.org/settings/api and copy the read access token (not the api key) +THEMOVIEDB_API_ACCESS_TOKEN="" # go to https://thetvdb.com/api-information/signup and copy the api key TVDB_APIKEY= # you can also input your subscriber's pin to support TVDB @@ -32,41 +32,45 @@ TVDB_PIN= # The url you can use to reach your kyoo instance. This is used during oidc to redirect users to your instance. PUBLIC_URL=http://localhost:8901 -# Use a builtin oidc service (google, discord, trakt, or simkl): -# When you create a client_id, secret combo you may be asked for a redirect url. You need to specify https://YOUR-PUBLIC-URL/api/auth/logged/YOUR-SERVICE-NAME -OIDC_DISCORD_CLIENTID= -OIDC_DISCORD_SECRET= -# Or add your custom one: -OIDC_SERVICE_NAME=YourPrettyName -OIDC_SERVICE_LOGO=https://url-of-your-logo.com -OIDC_SERVICE_CLIENTID= -OIDC_SERVICE_SECRET= -OIDC_SERVICE_AUTHORIZATION=https://url-of-the-authorization-endpoint-of-the-oidc-service.com/auth -OIDC_SERVICE_TOKEN=https://url-of-the-token-endpoint-of-the-oidc-service.com/token -OIDC_SERVICE_PROFILE=https://url-of-the-profile-endpoint-of-the-oidc-service.com/userinfo -OIDC_SERVICE_SCOPE="the list of scopes space separeted like email identity" -# Token authentication method as seen in https://openid.net/specs/openid-connect-core-1_0.html#ClientAuthentication -# Supported values: ClientSecretBasic (default) or ClientSecretPost -# If in doubt, leave this empty. -OIDC_SERVICE_AUTHMETHOD=ClientSecretBasic -# on the previous list, service is the internal name of your service, you can add as many as you want. +# Default permissions of new users. They are able to browse & play videos. +# Set `verified` to true if you don't wanna manually verify users. +EXTRA_CLAIMS='{"permissions": ["core.read", "core.play"], "verified": false}' +# This is the permissions of the first user (aka the first user is admin) +FIRST_USER_CLAIMS='{"permissions": ["users.read", "users.write", "apikeys.read", "apikeys.write", "users.delete", "core.read", "core.write", "core.play", "scanner.trigger"], "verified": true}' + +# Guest (meaning unlogged in users) can be: +# unauthorized (they need to connect before doing anything) +# GUEST_CLAIMS="" +# able to browse & see what you have but not able to play +GUEST_CLAIMS='{"permissions": ["core.read"], "verified": true}' +# or have browse & play permissions +GUEST_CLAIMS='{"permissions": ["core.read", "core.play"], "verified": true}' + +# DO NOT change this. +PROTECTED_CLAIMS="permissions,verified" -# Following options are optional and only useful for debugging. +# You can create apikeys at runtime via POST /apikey but you can also have some defined in the env. +# Replace $YOURNAME with the name of the key you want (only alpha are valid) +# The value will be the apikey (max 128 bytes) +# KEIBI_APIKEY_$YOURNAME=oaeushtaoesunthoaensuth +# KEIBI_APIKEY_$YOURNAME_CLAIMS='{"permissions": ["users.read"]}' # To debug the front end, you can set the following to an external backend KYOO_URL= -# Database things +# It is recommended to use the below PG environment variables when possible. +# POSTGRES_URL=postgres://user:password@hostname:port/dbname?sslmode=verify-full&sslrootcert=/path/to/server.crt&sslcert=/path/to/client.crt&sslkey=/path/to/client.key +# The behavior of the below variables match what is documented here: +# https://www.postgresql.org/docs/current/libpq-envars.html PGUSER=kyoo PGPASSWORD=password PGDATABASE=kyoo PGHOST=postgres PGPORT=5432 - -# v5 stuff, does absolutely nothing on master (aka: you can delete this) -EXTRA_CLAIMS='{"permissions": ["core.read"], "verified": false}' -FIRST_USER_CLAIMS='{"permissions": ["users.read", "users.write", "apikeys.read", "apikeys.write", "users.delete", "core.read", "core.write", "scanner.trigger"], "verified": true}' -GUEST_CLAIMS='{"permissions": ["users.read", "users.write", "apikeys.read", "apikeys.write", "users.delete", "core.read", "core.write", "scanner.trigger"], "verified": true}' -# GUEST_CLAIMS='{"permissions": ["core.read"]}' -PROTECTED_CLAIMS="permissions,verified" +# PGOPTIONS=-c search_path=kyoo,public +# PGPASSFILE=/my/password # Takes precedence over PGPASSWORD. New line characters are not trimmed. +# PGSSLMODE=verify-full +# PGSSLROOTCERT=/my/serving.crt +# PGSSLCERT=/my/client.crt +# PGSSLKEY=/my/client.key=password From 509e7c08cd37b4df7bfa024069876f76b9feb064 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Sun, 2 Nov 2025 22:29:55 +0100 Subject: [PATCH 06/13] Switch transcoder to pgx --- transcoder/go.mod | 76 ++++--- transcoder/go.sum | 160 +++++++------ transcoder/migrations/000001_init_db.down.sql | 12 +- transcoder/migrations/000001_init_db.up.sql | 22 +- ...00002_add_hearing_impaired_column.down.sql | 2 +- .../000002_add_hearing_impaired_column.up.sql | 2 +- .../migrations/000003_subtitle_mime.down.sql | 2 +- .../migrations/000003_subtitle_mime.up.sql | 2 +- transcoder/src/extract.go | 2 +- transcoder/src/info.go | 110 ++++----- transcoder/src/keyframes.go | 27 ++- transcoder/src/metadata.go | 215 +++++++++--------- transcoder/src/thumbnails.go | 2 +- 13 files changed, 348 insertions(+), 286 deletions(-) diff --git a/transcoder/go.mod b/transcoder/go.mod index fce4051e..31509eac 100644 --- a/transcoder/go.mod +++ b/transcoder/go.mod @@ -3,14 +3,14 @@ module github.com/zoriya/kyoo/transcoder go 1.24.2 require ( - github.com/asticode/go-astisub v0.35.0 - github.com/aws/aws-sdk-go-v2 v1.39.3 - github.com/aws/aws-sdk-go-v2/service/s3 v1.88.6 + github.com/asticode/go-astisub v0.36.0 + github.com/aws/aws-sdk-go-v2 v1.39.5 + github.com/aws/aws-sdk-go-v2/service/s3 v1.89.1 github.com/disintegration/imaging v1.6.2 github.com/golang-migrate/migrate/v4 v4.19.0 + github.com/jackc/pgx/v5 v5.7.6 github.com/labstack/echo-jwt/v4 v4.3.1 github.com/labstack/echo/v4 v4.13.4 - github.com/lib/pq v1.10.9 github.com/swaggo/echo-swagger v1.4.1 github.com/swaggo/swag v1.16.6 gitlab.com/opennota/screengen v1.0.2 @@ -20,38 +20,46 @@ require ( require ( github.com/KyleBanks/depth v1.2.1 // indirect - github.com/asticode/go-astikit v0.20.0 // indirect - github.com/asticode/go-astits v1.8.0 // indirect + github.com/asticode/go-astikit v0.56.0 // indirect + github.com/asticode/go-astits v1.14.0 // indirect github.com/ghodss/yaml v1.0.0 // indirect - github.com/go-openapi/jsonpointer v0.21.1 // indirect - github.com/go-openapi/jsonreference v0.21.0 // indirect - github.com/go-openapi/spec v0.21.0 // indirect - github.com/go-openapi/swag v0.23.1 // indirect - github.com/josharian/intern v1.0.0 // indirect - github.com/mailru/easyjson v0.9.0 // indirect + github.com/go-openapi/jsonpointer v0.22.1 // indirect + github.com/go-openapi/jsonreference v0.21.2 // indirect + github.com/go-openapi/spec v0.22.0 // indirect + github.com/go-openapi/swag/conv v0.25.1 // indirect + github.com/go-openapi/swag/jsonname v0.25.1 // indirect + github.com/go-openapi/swag/jsonutils v0.25.1 // indirect + github.com/go-openapi/swag/loading v0.25.1 // indirect + github.com/go-openapi/swag/stringutils v0.25.1 // indirect + github.com/go-openapi/swag/typeutils v0.25.1 // indirect + github.com/go-openapi/swag/yamlutils v0.25.1 // indirect + github.com/jackc/pgerrcode v0.0.0-20250907135507-afb5586c32a6 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/swaggo/files/v2 v2.0.2 // indirect - golang.org/x/mod v0.28.0 // indirect - golang.org/x/tools v0.37.0 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/mod v0.29.0 // indirect + golang.org/x/tools v0.38.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect ) require ( github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.2 // indirect - github.com/aws/aws-sdk-go-v2/config v1.31.14 - github.com/aws/aws-sdk-go-v2/credentials v1.18.18 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.10 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.10 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.10 // indirect + github.com/aws/aws-sdk-go-v2/config v1.31.16 + github.com/aws/aws-sdk-go-v2/credentials v1.18.20 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.12 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.12 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.12 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.10 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.12 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.2 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.1 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.10 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.10 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.29.7 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.2 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.38.8 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.3 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.12 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.12 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.30.0 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.4 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.39.0 // indirect github.com/aws/smithy-go v1.23.1 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 // indirect github.com/goccy/go-json v0.10.5 // indirect @@ -62,18 +70,18 @@ require ( github.com/lestrrat-go/blackmagic v1.0.4 // indirect github.com/lestrrat-go/httpcc v1.0.1 // indirect github.com/lestrrat-go/httprc/v3 v3.0.1 - github.com/lestrrat-go/jwx/v3 v3.0.10 + github.com/lestrrat-go/jwx/v3 v3.0.12 github.com/lestrrat-go/option v1.0.1 // indirect github.com/lestrrat-go/option/v2 v2.0.0 // indirect github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-isatty v0.0.20 // indirect - github.com/segmentio/asm v1.2.0 // indirect + github.com/segmentio/asm v1.2.1 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/valyala/fasttemplate v1.2.2 // indirect - golang.org/x/crypto v0.42.0 // indirect - golang.org/x/image v0.29.0 // indirect - golang.org/x/net v0.44.0 // indirect - golang.org/x/sys v0.36.0 // indirect + golang.org/x/crypto v0.43.0 // indirect + golang.org/x/image v0.32.0 // indirect + golang.org/x/net v0.46.0 // indirect + golang.org/x/sys v0.37.0 // indirect golang.org/x/text v0.30.0 - golang.org/x/time v0.12.0 // indirect + golang.org/x/time v0.14.0 // indirect ) diff --git a/transcoder/go.sum b/transcoder/go.sum index b2792a85..48ce609b 100644 --- a/transcoder/go.sum +++ b/transcoder/go.sum @@ -4,46 +4,49 @@ github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= -github.com/asticode/go-astikit v0.20.0 h1:+7N+J4E4lWx2QOkRdOf6DafWJMv6O4RRfgClwQokrH8= github.com/asticode/go-astikit v0.20.0/go.mod h1:h4ly7idim1tNhaVkdVBeXQZEE3L0xblP7fCWbgwipF0= -github.com/asticode/go-astisub v0.35.0 h1:wnELGJMeJbavW//X7nLTy97L3iblub7tO1VSeHnZBdA= -github.com/asticode/go-astisub v0.35.0/go.mod h1:WTkuSzFB+Bp7wezuSf2Oxulj5A8zu2zLRVFf6bIFQK8= -github.com/asticode/go-astits v1.8.0 h1:rf6aiiGn/QhlFjNON1n5plqF3Fs025XLUwiQ0NB6oZg= +github.com/asticode/go-astikit v0.30.0/go.mod h1:h4ly7idim1tNhaVkdVBeXQZEE3L0xblP7fCWbgwipF0= +github.com/asticode/go-astikit v0.56.0 h1:DmD2p7YnvxiPdF0h+dRmos3bsejNEXbycENsY5JfBqw= +github.com/asticode/go-astikit v0.56.0/go.mod h1:fV43j20UZYfXzP9oBn33udkvCvDvCDhzjVqoLFuuYZE= +github.com/asticode/go-astisub v0.36.0 h1:AatpRp9xZSv/pUoCnsx/NmKEhyjkyHFwrkkon4kgDBI= +github.com/asticode/go-astisub v0.36.0/go.mod h1:WTkuSzFB+Bp7wezuSf2Oxulj5A8zu2zLRVFf6bIFQK8= github.com/asticode/go-astits v1.8.0/go.mod h1:DkOWmBNQpnr9mv24KfZjq4JawCFX1FCqjLVGvO0DygQ= -github.com/aws/aws-sdk-go-v2 v1.39.3 h1:h7xSsanJ4EQJXG5iuW4UqgP7qBopLpj84mpkNx3wPjM= -github.com/aws/aws-sdk-go-v2 v1.39.3/go.mod h1:yWSxrnioGUZ4WVv9TgMrNUeLV3PFESn/v+6T/Su8gnM= +github.com/asticode/go-astits v1.14.0 h1:zkgnZzipx2XX5mWycqsSBeEyDH58+i4HtyF4j2ROb00= +github.com/asticode/go-astits v1.14.0/go.mod h1:QSHmknZ51pf6KJdHKZHJTLlMegIrhega3LPWz3ND/iI= +github.com/aws/aws-sdk-go-v2 v1.39.5 h1:e/SXuia3rkFtapghJROrydtQpfQaaUgd1cUvyO1mp2w= +github.com/aws/aws-sdk-go-v2 v1.39.5/go.mod h1:yWSxrnioGUZ4WVv9TgMrNUeLV3PFESn/v+6T/Su8gnM= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.2 h1:t9yYsydLYNBk9cJ73rgPhPWqOh/52fcWDQB5b1JsKSY= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.2/go.mod h1:IusfVNTmiSN3t4rhxWFaBAqn+mcNdwKtPcV16eYdgko= -github.com/aws/aws-sdk-go-v2/config v1.31.14 h1:kj/KpDqvt0UqcEL3WOvCykE9QUpBb6b23hQdnXe+elo= -github.com/aws/aws-sdk-go-v2/config v1.31.14/go.mod h1:X5PaY6QCzViihn/ru7VxnIamcJQrG9NSeTxuSKm2YtU= -github.com/aws/aws-sdk-go-v2/credentials v1.18.18 h1:5AfxTvDN0AJoA7rg/yEc0sHhl6/B9fZ+NtiQuOjWGQM= -github.com/aws/aws-sdk-go-v2/credentials v1.18.18/go.mod h1:m9mE1mJ1s7zI6rrt7V3RQU2SCgUbNaphlfqEksLp+Fs= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.10 h1:UuGVOX48oP4vgQ36oiKmW9RuSeT8jlgQgBFQD+HUiHY= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.10/go.mod h1:vM/Ini41PzvudT4YkQyE/+WiQJiQ6jzeDyU8pQKwCac= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.10 h1:mj/bdWleWEh81DtpdHKkw41IrS+r3uw1J/VQtbwYYp8= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.10/go.mod h1:7+oEMxAZWP8gZCyjcm9VicI0M61Sx4DJtcGfKYv2yKQ= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.10 h1:wh+/mn57yhUrFtLIxyFPh2RgxgQz/u+Yrf7hiHGHqKY= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.10/go.mod h1:7zirD+ryp5gitJJ2m1BBux56ai8RIRDykXZrJSp540w= +github.com/aws/aws-sdk-go-v2/config v1.31.16 h1:E4Tz+tJiPc7kGnXwIfCyUj6xHJNpENlY11oKpRTgsjc= +github.com/aws/aws-sdk-go-v2/config v1.31.16/go.mod h1:2S9hBElpCyGMifv14WxQ7EfPumgoeCPZUpuPX8VtW34= +github.com/aws/aws-sdk-go-v2/credentials v1.18.20 h1:KFndAnHd9NUuzikHjQ8D5CfFVO+bgELkmcGY8yAw98Q= +github.com/aws/aws-sdk-go-v2/credentials v1.18.20/go.mod h1:9mCi28a+fmBHSQ0UM79omkz6JtN+PEsvLrnG36uoUv0= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.12 h1:VO3FIM2TDbm0kqp6sFNR0PbioXJb/HzCDW6NtIZpIWE= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.12/go.mod h1:6C39gB8kg82tx3r72muZSrNhHia9rjGkX7ORaS2GKNE= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.12 h1:p/9flfXdoAnwJnuW9xHEAFY22R3A6skYkW19JFF9F+8= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.12/go.mod h1:ZTLHakoVCTtW8AaLGSwJ3LXqHD9uQKnOcv1TrpO6u2k= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.12 h1:2lTWFvRcnWFFLzHWmtddu5MTchc5Oj2OOey++99tPZ0= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.12/go.mod h1:hI92pK+ho8HVcWMHKHrK3Uml4pfG7wvL86FzO0LVtQQ= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.10 h1:FHw90xCTsofzk6vjU808TSuDtDfOOKPNdz5Weyc3tUI= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.10/go.mod h1:n8jdIE/8F3UYkg8O4IGkQpn2qUmapg/1K1yl29/uf/c= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.12 h1:itu4KHu8JK/N6NcLIISlf3LL1LccMqruLUXZ9y7yBZw= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.12/go.mod h1:i+6vTU3xziikTY3vcox23X8pPGW5X3wVgd1VZ7ha+x8= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.2 h1:xtuxji5CS0JknaXoACOunXOYOQzgfTvGAc9s2QdCJA4= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.2/go.mod h1:zxwi0DIR0rcRcgdbl7E2MSOvxDyyXGBlScvBkARFaLQ= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.1 h1:ne+eepnDB2Wh5lHKzELgEncIqeVlQ1rSF9fEa4r5I+A= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.1/go.mod h1:u0Jkg0L+dcG1ozUq21uFElmpbmjBnhHR5DELHIme4wg= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.10 h1:DRND0dkCKtJzCj4Xl4OpVbXZgfttY5q712H9Zj7qc/0= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.10/go.mod h1:tGGNmJKOTernmR2+VJ0fCzQRurcPZj9ut60Zu5Fi6us= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.10 h1:DA+Hl5adieRyFvE7pCvBWm3VOZTRexGVkXw33SUqNoY= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.10/go.mod h1:L+A89dH3/gr8L4ecrdzuXUYd1znoko6myzndVGZx/DA= -github.com/aws/aws-sdk-go-v2/service/s3 v1.88.6 h1:Hcb4yllr4GTOHC/BKjEklxWhciWMHIqzeCI9oYf1OIk= -github.com/aws/aws-sdk-go-v2/service/s3 v1.88.6/go.mod h1:N/iojY+8bW3MYol9NUMuKimpSbPEur75cuI1SmtonFM= -github.com/aws/aws-sdk-go-v2/service/sso v1.29.7 h1:fspVFg6qMx0svs40YgRmE7LZXh9VRZvTT35PfdQR6FM= -github.com/aws/aws-sdk-go-v2/service/sso v1.29.7/go.mod h1:BQTKL3uMECaLaUV3Zc2L4Qybv8C6BIXjuu1dOPyxTQs= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.2 h1:scVnW+NLXasGOhy7HhkdT9AGb6kjgW7fJ5xYkUaqHs0= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.2/go.mod h1:FRNCY3zTEWZXBKm2h5UBUPvCVDOecTad9KhynDyGBc0= -github.com/aws/aws-sdk-go-v2/service/sts v1.38.8 h1:xSL4IV19pKDASL2fjWXRfTGmZddPiPPZNPpbv6uZQZY= -github.com/aws/aws-sdk-go-v2/service/sts v1.38.8/go.mod h1:L1xxV3zAdB+qVrVW/pBIrIAnHFWHo6FBbFe4xOGsG/o= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.3 h1:NEe7FaViguRQEm8zl8Ay/kC/QRsMtWUiCGZajQIsLdc= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.3/go.mod h1:JLuCKu5VfiLBBBl/5IzZILU7rxS0koQpHzMOCzycOJU= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.12 h1:MM8imH7NZ0ovIVX7D2RxfMDv7Jt9OiUXkcQ+GqywA7M= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.12/go.mod h1:gf4OGwdNkbEsb7elw2Sy76odfhwNktWII3WgvQgQQ6w= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.12 h1:R3uW0iKl8rgNEXNjVGliW/oMEh9fO/LlUEV8RvIFr1I= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.12/go.mod h1:XEttbEr5yqsw8ebi7vlDoGJJjMXRez4/s9pibpJyL5s= +github.com/aws/aws-sdk-go-v2/service/s3 v1.89.1 h1:Dq82AV+Qxpno/fG162eAhnD8d48t9S+GZCfz7yv1VeA= +github.com/aws/aws-sdk-go-v2/service/s3 v1.89.1/go.mod h1:MbKLznDKpf7PnSonNRUVYZzfP0CeLkRIUexeblgKcU4= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.0 h1:xHXvxst78wBpJFgDW07xllOx0IAzbryrSdM4nMVQ4Dw= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.0/go.mod h1:/e8m+AO6HNPPqMyfKRtzZ9+mBF5/x1Wk8QiDva4m07I= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.4 h1:tBw2Qhf0kj4ZwtsVpDiVRU3zKLvjvjgIjHMKirxXg8M= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.4/go.mod h1:Deq4B7sRM6Awq/xyOBlxBdgW8/Z926KYNNaGMW2lrkA= +github.com/aws/aws-sdk-go-v2/service/sts v1.39.0 h1:C+BRMnasSYFcgDw8o9H5hzehKzXyAb9GY5v/8bP9DUY= +github.com/aws/aws-sdk-go-v2/service/sts v1.39.0/go.mod h1:4EjU+4mIx6+JqKQkruye+CaigV7alL3thVPfDd9VlMs= github.com/aws/smithy-go v1.23.1 h1:sLvcH6dfAFwGkHLZ7dGiYF7aK6mg4CgKA/iDKjLDt9M= github.com/aws/smithy-go v1.23.1/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0= github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= @@ -75,14 +78,29 @@ github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/go-openapi/jsonpointer v0.21.1 h1:whnzv/pNXtK2FbX/W9yJfRmE2gsmkfahjMKB0fZvcic= -github.com/go-openapi/jsonpointer v0.21.1/go.mod h1:50I1STOfbY1ycR8jGz8DaMeLCdXiI6aDteEdRNNzpdk= -github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF2+tg1TRrwQ= -github.com/go-openapi/jsonreference v0.21.0/go.mod h1:LmZmgsrTkVg9LG4EaHeY8cBDslNPMo06cago5JNLkm4= -github.com/go-openapi/spec v0.21.0 h1:LTVzPc3p/RzRnkQqLRndbAzjY0d0BCL72A6j3CdL9ZY= -github.com/go-openapi/spec v0.21.0/go.mod h1:78u6VdPw81XU44qEWGhtr982gJ5BWg2c0I5XwVMotYk= -github.com/go-openapi/swag v0.23.1 h1:lpsStH0n2ittzTnbaSloVZLuB5+fvSY/+hnagBjSNZU= -github.com/go-openapi/swag v0.23.1/go.mod h1:STZs8TbRvEQQKUA+JZNAm3EWlgaOBGpyFDqQnDHMef0= +github.com/go-openapi/jsonpointer v0.22.1 h1:sHYI1He3b9NqJ4wXLoJDKmUmHkWy/L7rtEo92JUxBNk= +github.com/go-openapi/jsonpointer v0.22.1/go.mod h1:pQT9OsLkfz1yWoMgYFy4x3U5GY5nUlsOn1qSBH5MkCM= +github.com/go-openapi/jsonreference v0.21.2 h1:Wxjda4M/BBQllegefXrY/9aq1fxBA8sI5M/lFU6tSWU= +github.com/go-openapi/jsonreference v0.21.2/go.mod h1:pp3PEjIsJ9CZDGCNOyXIQxsNuroxm8FAJ/+quA0yKzQ= +github.com/go-openapi/spec v0.22.0 h1:xT/EsX4frL3U09QviRIZXvkh80yibxQmtoEvyqug0Tw= +github.com/go-openapi/spec v0.22.0/go.mod h1:K0FhKxkez8YNS94XzF8YKEMULbFrRw4m15i2YUht4L0= +github.com/go-openapi/swag v0.19.15 h1:D2NRCBzS9/pEY3gP9Nl8aDqGUcPFrwG2p+CNFrLyrCM= +github.com/go-openapi/swag/conv v0.25.1 h1:+9o8YUg6QuqqBM5X6rYL/p1dpWeZRhoIt9x7CCP+he0= +github.com/go-openapi/swag/conv v0.25.1/go.mod h1:Z1mFEGPfyIKPu0806khI3zF+/EUXde+fdeksUl2NiDs= +github.com/go-openapi/swag/jsonname v0.25.1 h1:Sgx+qbwa4ej6AomWC6pEfXrA6uP2RkaNjA9BR8a1RJU= +github.com/go-openapi/swag/jsonname v0.25.1/go.mod h1:71Tekow6UOLBD3wS7XhdT98g5J5GR13NOTQ9/6Q11Zo= +github.com/go-openapi/swag/jsonutils v0.25.1 h1:AihLHaD0brrkJoMqEZOBNzTLnk81Kg9cWr+SPtxtgl8= +github.com/go-openapi/swag/jsonutils v0.25.1/go.mod h1:JpEkAjxQXpiaHmRO04N1zE4qbUEg3b7Udll7AMGTNOo= +github.com/go-openapi/swag/jsonutils/fixtures_test v0.25.1 h1:DSQGcdB6G0N9c/KhtpYc71PzzGEIc/fZ1no35x4/XBY= +github.com/go-openapi/swag/jsonutils/fixtures_test v0.25.1/go.mod h1:kjmweouyPwRUEYMSrbAidoLMGeJ5p6zdHi9BgZiqmsg= +github.com/go-openapi/swag/loading v0.25.1 h1:6OruqzjWoJyanZOim58iG2vj934TysYVptyaoXS24kw= +github.com/go-openapi/swag/loading v0.25.1/go.mod h1:xoIe2EG32NOYYbqxvXgPzne989bWvSNoWoyQVWEZicc= +github.com/go-openapi/swag/stringutils v0.25.1 h1:Xasqgjvk30eUe8VKdmyzKtjkVjeiXx1Iz0zDfMNpPbw= +github.com/go-openapi/swag/stringutils v0.25.1/go.mod h1:JLdSAq5169HaiDUbTvArA2yQxmgn4D6h4A+4HqVvAYg= +github.com/go-openapi/swag/typeutils v0.25.1 h1:rD/9HsEQieewNt6/k+JBwkxuAHktFtH3I3ysiFZqukA= +github.com/go-openapi/swag/typeutils v0.25.1/go.mod h1:9McMC/oCdS4BKwk2shEB7x17P6HmMmA6dQRtAkSnNb8= +github.com/go-openapi/swag/yamlutils v0.25.1 h1:mry5ez8joJwzvMbaTGLhw8pXUnhDK91oSJLDPF1bmGk= +github.com/go-openapi/swag/yamlutils v0.25.1/go.mod h1:cm9ywbzncy3y6uPm/97ysW8+wZ09qsks+9RS8fLWKqg= github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= @@ -98,8 +116,16 @@ github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= -github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= -github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/jackc/pgerrcode v0.0.0-20250907135507-afb5586c32a6 h1:D/V0gu4zQ3cL2WKeVNVM4r2gLxGGf6McLwgXzRTo2RQ= +github.com/jackc/pgerrcode v0.0.0-20250907135507-afb5586c32a6/go.mod h1:a/s9Lp5W7n/DD0VrVoyJ00FbP2ytTPDVOivvn2bMlds= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.7.6 h1:rWQc5FwZSPX58r1OQmkuaNicxdmExaEz5A2DO2hUuTk= +github.com/jackc/pgx/v5 v5.7.6/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -112,20 +138,22 @@ github.com/labstack/gommon v0.4.2 h1:F8qTUNXgG1+6WQmqoUWnz8WiEU60mXVVw0P4ht1WRA0 github.com/labstack/gommon v0.4.2/go.mod h1:QlUFxVM+SNXhDL/Z7YhocGIBYOiwB0mXm1+1bAPHPyU= github.com/lestrrat-go/blackmagic v1.0.4 h1:IwQibdnf8l2KoO+qC3uT4OaTWsW7tuRQXy9TRN9QanA= github.com/lestrrat-go/blackmagic v1.0.4/go.mod h1:6AWFyKNNj0zEXQYfTMPfZrAXUWUfTIZ5ECEUEJaijtw= +github.com/lestrrat-go/dsig v1.0.0 h1:OE09s2r9Z81kxzJYRn07TFM9XA4akrUdoMwr0L8xj38= +github.com/lestrrat-go/dsig v1.0.0/go.mod h1:dEgoOYYEJvW6XGbLasr8TFcAxoWrKlbQvmJgCR0qkDo= +github.com/lestrrat-go/dsig-secp256k1 v1.0.0 h1:JpDe4Aybfl0soBvoVwjqDbp+9S1Y2OM7gcrVVMFPOzY= +github.com/lestrrat-go/dsig-secp256k1 v1.0.0/go.mod h1:CxUgAhssb8FToqbL8NjSPoGQlnO4w3LG1P0qPWQm/NU= github.com/lestrrat-go/httpcc v1.0.1 h1:ydWCStUeJLkpYyjLDHihupbn2tYmZ7m22BGkcvZZrIE= github.com/lestrrat-go/httpcc v1.0.1/go.mod h1:qiltp3Mt56+55GPVCbTdM9MlqhvzyuL6W/NMDA8vA5E= github.com/lestrrat-go/httprc/v3 v3.0.1 h1:3n7Es68YYGZb2Jf+k//llA4FTZMl3yCwIjFIk4ubevI= github.com/lestrrat-go/httprc/v3 v3.0.1/go.mod h1:2uAvmbXE4Xq8kAUjVrZOq1tZVYYYs5iP62Cmtru00xk= -github.com/lestrrat-go/jwx/v3 v3.0.10 h1:XuoCBhZBncRIjMQ32HdEc76rH0xK/Qv2wq5TBouYJDw= -github.com/lestrrat-go/jwx/v3 v3.0.10/go.mod h1:kNMedLgTpHvPJkK5EMVa1JFz+UVyY2dMmZKu3qjl/Pk= +github.com/lestrrat-go/jwx/v3 v3.0.12 h1:p25r68Y4KrbBdYjIsQweYxq794CtGCzcrc5dGzJIRjg= +github.com/lestrrat-go/jwx/v3 v3.0.12/go.mod h1:HiUSaNmMLXgZ08OmGBaPVvoZQgJVOQphSrGr5zMamS8= github.com/lestrrat-go/option v1.0.1 h1:oAzP2fvZGQKWkvHa1/SAcFolBEca1oN+mQ7eooNBEYU= github.com/lestrrat-go/option v1.0.1/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I= github.com/lestrrat-go/option/v2 v2.0.0 h1:XxrcaJESE1fokHy3FpaQ/cXW8ZsIdWcdFzzLOcID3Ss= github.com/lestrrat-go/option/v2 v2.0.0/go.mod h1:oSySsmzMoR0iRzCDCaUfsCzxQHUEuhOViQObyy7S6Vg= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4= -github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU= github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= @@ -147,14 +175,16 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= -github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys= -github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= +github.com/segmentio/asm v1.2.1 h1:DTNbBqs57ioxAD4PrArqftgypG4/qNpXoJx8TVXxPR0= +github.com/segmentio/asm v1.2.1/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/swaggo/echo-swagger v1.4.1 h1:Yf0uPaJWp1uRtDloZALyLnvdBeoEL5Kc7DtnjzO/TUk= github.com/swaggo/echo-swagger v1.4.1/go.mod h1:C8bSi+9yH2FLZsnhqMZLIZddpUxZdBYuNHbtaS1Hljc= github.com/swaggo/files/v2 v2.0.2 h1:Bq4tgS/yxLB/3nwOMcul5oLEUKa877Ykgz3CJMVbQKU= @@ -179,36 +209,38 @@ go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/Wgbsd go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E= go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4= go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.42.0 h1:chiH31gIWm57EkTXpwnqf8qeuMUi0yekh6mT2AvFlqI= -golang.org/x/crypto v0.42.0/go.mod h1:4+rDnOTJhQCx2q7/j6rAN5XDw8kPjeaXEUR2eL94ix8= +golang.org/x/crypto v0.43.0 h1:dduJYIi3A3KOfdGOHX8AVZ/jGiyPa3IbBozJ5kNuE04= +golang.org/x/crypto v0.43.0/go.mod h1:BFbav4mRNlXJL4wNeejLpWxB7wMbc79PdRGhWKncxR0= golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/image v0.29.0 h1:HcdsyR4Gsuys/Axh0rDEmlBmB68rW1U9BUdB3UVHsas= -golang.org/x/image v0.29.0/go.mod h1:RVJROnf3SLK8d26OW91j4FrIHGbsJ8QnbEocVTOWQDA= -golang.org/x/mod v0.28.0 h1:gQBtGhjxykdjY9YhZpSlZIsbnaE2+PgjfLWUQTnoZ1U= -golang.org/x/mod v0.28.0/go.mod h1:yfB/L0NOf/kmEbXjzCPOx1iK1fRutOydrCMsqRhEBxI= +golang.org/x/image v0.32.0 h1:6lZQWq75h7L5IWNk0r+SCpUJ6tUVd3v4ZHnbRKLkUDQ= +golang.org/x/image v0.32.0/go.mod h1:/R37rrQmKXtO6tYXAjtDLwQgFLHmhW+V6ayXlxzP2Pc= +golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA= +golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.44.0 h1:evd8IRDyfNBMBTTY5XRF1vaZlD+EmWx6x8PkhR04H/I= -golang.org/x/net v0.44.0/go.mod h1:ECOoLqd5U3Lhyeyo/QDCEVQ4sNgYsqvCZ722XogGieY= +golang.org/x/net v0.46.0 h1:giFlY12I07fugqwPuWJi68oOnpfqFnJIJzaIIm2JVV4= +golang.org/x/net v0.46.0/go.mod h1:Q9BGdFy1y4nkUwiLvT5qtyhAnEHgnQ/zd8PfU6nc210= golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k= -golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ= +golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.30.0 h1:yznKA/E9zq54KzlzBEAWn1NXSQ8DIp/NYMy88xJjl4k= golang.org/x/text v0.30.0/go.mod h1:yDdHFIX9t+tORqspjENWgzaCVXgk0yYnYuSZ8UzzBVM= -golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= -golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= +golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= +golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.37.0 h1:DVSRzp7FwePZW356yEAChSdNcQo6Nsp+fex1SUW09lE= -golang.org/x/tools v0.37.0/go.mod h1:MBN5QPQtLMHVdvsbtarmTNukZDdgwdwlO5qGacAzF0w= +golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ= +golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/transcoder/migrations/000001_init_db.down.sql b/transcoder/migrations/000001_init_db.down.sql index 89e4b552..bdaf8765 100644 --- a/transcoder/migrations/000001_init_db.down.sql +++ b/transcoder/migrations/000001_init_db.down.sql @@ -1,10 +1,10 @@ begin; -drop table info; -drop table videos; -drop table audios; -drop table subtitles; -drop table chapters; -drop type chapter_type; +drop table gocoder.info; +drop table gocoder.videos; +drop table gocoder.audios; +drop table gocoder.subtitles; +drop table gocoder.chapters; +drop type gocoder.chapter_type; commit; diff --git a/transcoder/migrations/000001_init_db.up.sql b/transcoder/migrations/000001_init_db.up.sql index 30b983b6..18f10610 100644 --- a/transcoder/migrations/000001_init_db.up.sql +++ b/transcoder/migrations/000001_init_db.up.sql @@ -1,6 +1,6 @@ begin; -create table info( +create table gocoder.info( sha varchar(40) not null primary key, path varchar(4096) not null unique, extension varchar(16), @@ -15,8 +15,8 @@ create table info( ver_keyframes integer not null ); -create table videos( - sha varchar(40) not null references info(sha) on delete cascade, +create table gocoder.videos( + sha varchar(40) not null references gocoder.info(sha) on delete cascade, idx integer not null, title varchar(1024), language varchar(256), @@ -32,8 +32,8 @@ create table videos( constraint videos_pk primary key (sha, idx) ); -create table audios( - sha varchar(40) not null references info(sha) on delete cascade, +create table gocoder.audios( + sha varchar(40) not null references gocoder.info(sha) on delete cascade, idx integer not null, title varchar(1024), language varchar(256), @@ -47,8 +47,8 @@ create table audios( constraint audios_pk primary key (sha, idx) ); -create table subtitles( - sha varchar(40) not null references info(sha) on delete cascade, +create table gocoder.subtitles( + sha varchar(40) not null references gocoder.info(sha) on delete cascade, idx integer not null, title varchar(1024), language varchar(256), @@ -60,14 +60,14 @@ create table subtitles( constraint subtitle_pk primary key (sha, idx) ); -create type chapter_type as enum('content', 'recap', 'intro', 'credits', 'preview'); +create type gocoder.chapter_type as enum('content', 'recap', 'intro', 'credits', 'preview'); -create table chapters( - sha varchar(40) not null references info(sha) on delete cascade, +create table gocoder.chapters( + sha varchar(40) not null references gocoder.info(sha) on delete cascade, start_time real not null, end_time real not null, name varchar(1024), - type chapter_type, + type gocoder.chapter_type, constraint chapter_pk primary key (sha, start_time) ); diff --git a/transcoder/migrations/000002_add_hearing_impaired_column.down.sql b/transcoder/migrations/000002_add_hearing_impaired_column.down.sql index 9b5e0058..00f5aa55 100644 --- a/transcoder/migrations/000002_add_hearing_impaired_column.down.sql +++ b/transcoder/migrations/000002_add_hearing_impaired_column.down.sql @@ -1,5 +1,5 @@ begin; -alter table subtitles drop column is_hearing_impaired; +alter table gocoder.subtitles drop column is_hearing_impaired; commit; diff --git a/transcoder/migrations/000002_add_hearing_impaired_column.up.sql b/transcoder/migrations/000002_add_hearing_impaired_column.up.sql index 36192861..4ccb01c3 100644 --- a/transcoder/migrations/000002_add_hearing_impaired_column.up.sql +++ b/transcoder/migrations/000002_add_hearing_impaired_column.up.sql @@ -1,5 +1,5 @@ begin; -alter table subtitles add column is_hearing_impaired boolean not null default false; +alter table gocoder.subtitles add column is_hearing_impaired boolean not null default false; commit; diff --git a/transcoder/migrations/000003_subtitle_mime.down.sql b/transcoder/migrations/000003_subtitle_mime.down.sql index a1e9e93a..df66910a 100644 --- a/transcoder/migrations/000003_subtitle_mime.down.sql +++ b/transcoder/migrations/000003_subtitle_mime.down.sql @@ -1,5 +1,5 @@ begin; -alter table subtitles drop column mime_codec; +alter table gocoder.subtitles drop column mime_codec; commit; diff --git a/transcoder/migrations/000003_subtitle_mime.up.sql b/transcoder/migrations/000003_subtitle_mime.up.sql index fdbf4382..6f57f36f 100644 --- a/transcoder/migrations/000003_subtitle_mime.up.sql +++ b/transcoder/migrations/000003_subtitle_mime.up.sql @@ -1,5 +1,5 @@ begin; -alter table subtitles add column mime_codec varchar(256) default null; +alter table gocoder.subtitles add column mime_codec varchar(256) default null; commit; diff --git a/transcoder/src/extract.go b/transcoder/src/extract.go index 354dc791..a06a0314 100644 --- a/transcoder/src/extract.go +++ b/transcoder/src/extract.go @@ -27,7 +27,7 @@ func (s *MetadataService) ExtractSubs(ctx context.Context, info *MediaInfo) (any log.Printf("Couldn't extract subs: %v", err) return set(nil, err) } - _, err = s.database.Exec(`update gocoder.info set ver_extract = $2 where sha = $1`, info.Sha, ExtractVersion) + _, err = s.database.Exec(ctx, `update gocoder.info set ver_extract = $2 where sha = $1`, info.Sha, ExtractVersion) return set(nil, err) } diff --git a/transcoder/src/info.go b/transcoder/src/info.go index 5a3f2610..4c84e5b6 100644 --- a/transcoder/src/info.go +++ b/transcoder/src/info.go @@ -20,125 +20,133 @@ import ( const InfoVersion = 3 type Versions struct { - Info int32 `json:"info"` - Extract int32 `json:"extract"` - Thumbs int32 `json:"thumbs"` - Keyframes int32 `json:"keyframes"` + Info int32 `json:"info" db:"ver_info"` + Extract int32 `json:"extract" db:"ver_extract"` + Thumbs int32 `json:"thumbs" db:"ver_thumbs"` + Keyframes int32 `json:"keyframes" db:"ver_keyframes"` } type MediaInfo struct { // The sha1 of the video file. - Sha string `json:"sha"` + Sha string `json:"sha" db:"sha"` /// The internal path of the video file. - Path string `json:"path"` + Path string `json:"path" db:"path"` /// The extension currently used to store this video file - Extension string `json:"extension"` - /// The whole mimetype (defined as the RFC 6381). ex: `video/mp4; codecs="avc1.640028, mp4a.40.2"` - MimeCodec *string `json:"mimeCodec"` + Extension string `json:"extension" db:"extension"` + /// The whole mimetype (defined as the RFC 6381). ex: `video/mp4; codecs=\"avc1.640028, mp4a.40.2\"` + MimeCodec *string `json:"mimeCodec" db:"mime_codec"` /// The file size of the video file. - Size int64 `json:"size"` + Size int64 `json:"size" db:"size"` /// The length of the media in seconds. - Duration float64 `json:"duration"` + Duration float64 `json:"duration" db:"duration"` /// The container of the video file of this episode. - Container *string `json:"container"` + Container *string `json:"container" db:"container"` /// Version of the metadata. This can be used to invalidate older metadata from db if the extraction code has changed. - Versions Versions `json:"versions"` + Versions Versions `json:"versions" db:"versions"` /// The list of videos if there are multiples. - Videos []Video `json:"videos"` + Videos []Video `json:"videos" db:"-"` /// The list of audio tracks. - Audios []Audio `json:"audios"` + Audios []Audio `json:"audios" db:"-"` /// The list of subtitles tracks. - Subtitles []Subtitle `json:"subtitles"` + Subtitles []Subtitle `json:"subtitles" db:"-"` /// The list of fonts that can be used to display subtitles. - Fonts []string `json:"fonts"` + Fonts []string `json:"fonts" db:"fonts"` /// The list of chapters. See Chapter for more information. - Chapters []Chapter `json:"chapters"` + Chapters []Chapter `json:"chapters" db:"-"` /// lock used to read/set keyframes of video/audio - lock sync.Mutex + lock sync.Mutex `json:"-" db:"-"` } type Video struct { + Sha string `json:"-" db:"sha"` + /// The index of this track on the media. - Index uint32 `json:"index"` + Index uint32 `json:"index" db:"idx"` /// The title of the stream. - Title *string `json:"title"` + Title *string `json:"title" db:"title"` /// The language of this stream (as a ISO-639-2 language code) - Language *string `json:"language"` + Language *string `json:"language" db:"language"` /// The human readable codec name. - Codec string `json:"codec"` + Codec string `json:"codec" db:"codec"` /// The codec of this stream (defined as the RFC 6381). - MimeCodec *string `json:"mimeCodec"` + MimeCodec *string `json:"mimeCodec" db:"mime_codec"` /// The width of the video stream - Width uint32 `json:"width"` + Width uint32 `json:"width" db:"width"` /// The height of the video stream - Height uint32 `json:"height"` + Height uint32 `json:"height" db:"height"` /// The average bitrate of the video in bytes/s - Bitrate uint32 `json:"bitrate"` + Bitrate uint32 `json:"bitrate" db:"bitrate"` /// Is this stream the default one of it's type? - IsDefault bool `json:"isDefault"` + IsDefault bool `json:"isDefault" db:"is_default"` /// Keyframes of this video Keyframes *Keyframe `json:"-"` } type Audio struct { + Sha string `json:"-" db:"sha"` + /// The index of this track on the media. - Index uint32 `json:"index"` + Index uint32 `json:"index" db:"idx"` /// The title of the stream. - Title *string `json:"title"` + Title *string `json:"title" db:"title"` /// The language of this stream (as a IETF-BCP-47 language code) - Language *string `json:"language"` + Language *string `json:"language" db:"language"` /// The human readable codec name. - Codec string `json:"codec"` + Codec string `json:"codec" db:"codec"` /// The codec of this stream (defined as the RFC 6381). - MimeCodec *string `json:"mimeCodec"` + MimeCodec *string `json:"mimeCodec" db:"mime_codec"` /// The average bitrate of the audio in bytes/s - Bitrate uint32 `json:"bitrate"` + Bitrate uint32 `json:"bitrate" db:"bitrate"` /// Is this stream the default one of it's type? - IsDefault bool `json:"isDefault"` + IsDefault bool `json:"isDefault" db:"is_default"` /// Keyframes of this video Keyframes *Keyframe `json:"-"` } type Subtitle struct { + Sha string `json:"-" db:"sha"` + /// The index of this track on the media. - Index *uint32 `json:"index"` + Index *uint32 `json:"index" db:"idx"` /// The title of the stream. - Title *string `json:"title"` + Title *string `json:"title" db:"title"` /// The language of this stream (as a IETF-BCP-47 language code) - Language *string `json:"language"` + Language *string `json:"language" db:"language"` /// The codec of this stream. - Codec string `json:"codec"` + Codec string `json:"codec" db:"codec"` /// The codec of this stream (defined as the RFC 6381). - MimeCodec *string `json:"mimeCodec"` + MimeCodec *string `json:"mimeCodec" db:"mime_codec"` /// The extension for the codec. - Extension *string `json:"extension"` + Extension *string `json:"extension" db:"extension"` /// Is this stream the default one of it's type? - IsDefault bool `json:"isDefault"` + IsDefault bool `json:"isDefault" db:"is_default"` /// Is this stream tagged as forced? - IsForced bool `json:"isForced"` + IsForced bool `json:"isForced" db:"is_forced"` /// Is this stream tagged as hearing impaired? - IsHearingImpaired bool `json:"isHearingImpaired"` + IsHearingImpaired bool `json:"isHearingImpaired" db:"is_hearing_impaired"` /// Is this an external subtitle (as in stored in a different file) - IsExternal bool `json:"isExternal"` + IsExternal bool `json:"isExternal" db:"-"` /// Where the subtitle is stored (null if stored inside the video) - Path *string `json:"path"` + Path *string `json:"path" db:"-"` /// The link to access this subtitle. - Link *string `json:"link"` + Link *string `json:"link" db:"-"` } type Chapter struct { + Sha string `json:"-" db:"sha"` + /// The start time of the chapter (in second from the start of the episode). - StartTime float32 `json:"startTime"` + StartTime float32 `json:"startTime" db:"start_time"` /// The end time of the chapter (in second from the start of the episode). - EndTime float32 `json:"endTime"` + EndTime float32 `json:"endTime" db:"end_time"` /// The name of this chapter. This should be a human-readable name that could be presented to the user. - Name string `json:"name"` + Name string `json:"name" db:"name"` /// The type value is used to mark special chapters (openning/credits...) - Type ChapterType `json:"type"` + Type ChapterType `json:"type" db:"type"` } type ChapterType string diff --git a/transcoder/src/keyframes.go b/transcoder/src/keyframes.go index 1234fe00..c747296c 100644 --- a/transcoder/src/keyframes.go +++ b/transcoder/src/keyframes.go @@ -2,6 +2,7 @@ package src import ( "bufio" + "context" "errors" "fmt" "log" @@ -10,7 +11,7 @@ import ( "strings" "sync" - "github.com/lib/pq" + "github.com/jackc/pgx/v5/pgtype" "github.com/zoriya/kyoo/transcoder/src/utils" ) @@ -75,12 +76,20 @@ func (kf *Keyframe) AddListener(callback func(keyframes []float64)) { kf.info.listeners = append(kf.info.listeners, callback) } -func (kf *Keyframe) Scan(src interface{}) error { - var arr pq.Float64Array - err := arr.Scan(src) +func (kf *Keyframe) Scan(src any) error { + var arr []float64 + + m := pgtype.NewMap() + t, ok := m.TypeForValue(&arr) + + if !ok { + return errors.New("failed to parse keyframes") + } + err := m.Scan(t.OID, pgtype.BinaryFormatCode, src.([]byte), &arr) if err != nil { return err } + kf.Keyframes = arr kf.IsDone = true kf.info = &KeyframeInfo{} @@ -131,6 +140,7 @@ func (s *MetadataService) GetKeyframes(info *MediaInfo, isVideo bool, idx uint32 info.lock.Unlock() go func() { + ctx := context.Background() var table string var err error if isVideo { @@ -147,15 +157,16 @@ func (s *MetadataService) GetKeyframes(info *MediaInfo, isVideo bool, idx uint32 } kf.info.ready.Wait() - tx, _ := s.database.Begin() + tx, _ := s.database.Begin(ctx) tx.Exec( + ctx, fmt.Sprintf(`update %s set keyframes = $3 where sha = $1 and idx = $2`, table), info.Sha, idx, - pq.Array(kf.Keyframes), + kf.Keyframes, ) - tx.Exec(`update gocoder.info set ver_keyframes = $2 where sha = $1`, info.Sha, KeyframeVersion) - err = tx.Commit() + tx.Exec(ctx, `update gocoder.info set ver_keyframes = $2 where sha = $1`, info.Sha, KeyframeVersion) + err = tx.Commit(ctx) if err != nil { log.Printf("Couldn't store keyframes on database: %v", err) } diff --git a/transcoder/src/metadata.go b/transcoder/src/metadata.go index 7e2ff60c..a4512b5b 100644 --- a/transcoder/src/metadata.go +++ b/transcoder/src/metadata.go @@ -2,24 +2,26 @@ package src import ( "context" - "database/sql" "encoding/base64" "errors" "fmt" - "net/url" "os" + "os/user" + "strings" "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/golang-migrate/migrate/v4" - "github.com/golang-migrate/migrate/v4/database/postgres" + pgxd "github.com/golang-migrate/migrate/v4/database/pgx/v5" _ "github.com/golang-migrate/migrate/v4/source/file" - "github.com/lib/pq" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/jackc/pgx/v5/stdlib" "github.com/zoriya/kyoo/transcoder/src/storage" ) type MetadataService struct { - database *sql.DB + database *pgxpool.Pool lock RunLock[string, *MediaInfo] thumbLock RunLock[string, any] extractLock RunLock[string, any] @@ -53,51 +55,69 @@ func NewMetadataService() (*MetadataService, error) { } func (s *MetadataService) Close() error { - cleanupErrs := make([]error, 0, 2) if s.database != nil { - err := s.database.Close() - if err != nil { - cleanupErrs = append(cleanupErrs, fmt.Errorf("failed to close database: %w", err)) - } + s.database.Close() } if s.storage != nil { if storageCloser, ok := s.storage.(storage.StorageBackendCloser); ok { err := storageCloser.Close() if err != nil { - cleanupErrs = append(cleanupErrs, fmt.Errorf("failed to close storage: %w", err)) + return err } } } - if err := errors.Join(cleanupErrs...); err != nil { - return fmt.Errorf("failed to cleanup resources: %w", err) - } - return nil } -func (s *MetadataService) setupDb() (*sql.DB, error) { +func (s *MetadataService) setupDb() (*pgxpool.Pool, error) { + ctx := context.Background() + connectionString := os.Getenv("POSTGRES_URL") - if connectionString == "" { - connectionString = fmt.Sprintf( - "postgresql://%v:%v@%v:%v/%v?application_name=gocoder&sslmode=%s", - url.QueryEscape(os.Getenv("POSTGRES_USER")), - url.QueryEscape(os.Getenv("POSTGRES_PASSWORD")), - url.QueryEscape(os.Getenv("POSTGRES_SERVER")), - url.QueryEscape(os.Getenv("POSTGRES_PORT")), - url.QueryEscape(os.Getenv("POSTGRES_DB")), - url.QueryEscape(GetEnvOr("POSTGRES_SSLMODE", "disable")), - ) + config, err := pgxpool.ParseConfig(connectionString) + if err != nil { + return nil, errors.New("failed to create postgres config from environment variables") } - db, err := sql.Open("postgres", connectionString) + // Set default values + if config.ConnConfig.Host == "/tmp" { + config.ConnConfig.Host = "postgres" + } + if config.ConnConfig.Database == "" { + config.ConnConfig.Database = "kyoo" + } + // The pgx library will set the username to the name of the current user if not provided via + // environment variable or connection string. Make a best-effort attempt to see if the user + // was explicitly specified, without implementing full connection string parsing. If not, set + // the username to the default value of "kyoo". + if os.Getenv("PGUSER") == "" { + currentUserName, _ := user.Current() + // If the username matches the current user and it's not in the connection string, then it was set + // by the pgx library. This doesn't cover the case where the system username happens to be in some other part + // of the connection string, but this cannot be checked without full connection string parsing. + if currentUserName.Username == config.ConnConfig.User && !strings.Contains(connectionString, currentUserName.Username) { + config.ConnConfig.User = "kyoo" + } + } + if _, ok := config.ConnConfig.RuntimeParams["application_name"]; !ok { + config.ConnConfig.RuntimeParams["application_name"] = "gocoder" + } + + db, err := pgxpool.NewWithConfig(ctx, config) if err != nil { - fmt.Printf("Could not connect to database, check your env variables!") + fmt.Printf("Could not connect to database, check your env variables!\n") return nil, err } - driver, err := postgres.WithInstance(db, &postgres.Config{}) + fmt.Println("Migrating database") + dbi := stdlib.OpenDBFromPool(db) + defer dbi.Close() + + dbi.Exec("create schema if not exists gocoder") + driver, err := pgxd.WithInstance(dbi, &pgxd.Config{ + SchemaName: "gocoder", + }) if err != nil { return nil, err } @@ -106,6 +126,7 @@ func (s *MetadataService) setupDb() (*sql.DB, error) { return nil, err } m.Up() + fmt.Println("Migrating finished") return db, nil } @@ -135,7 +156,7 @@ func (s *MetadataService) setupStorage(ctx context.Context) (storage.StorageBack } func (s *MetadataService) GetMetadata(ctx context.Context, path string, sha string) (*MediaInfo, error) { - ret, err := s.getMetadata(path, sha) + ret, err := s.getMetadata(ctx, path, sha) if err != nil { return nil, err } @@ -153,14 +174,14 @@ func (s *MetadataService) GetMetadata(ctx context.Context, path string, sha stri for _, audio := range ret.Audios { audio.Keyframes = nil } - tx, err := s.database.Begin() + tx, err := s.database.Begin(ctx) if err != nil { return nil, err } - tx.Exec(`update gocoder.videos set keyframes = null where sha = $1`, sha) - tx.Exec(`update gocoder.audios set keyframes = null where sha = $1`, sha) - tx.Exec(`update gocoder.info set ver_keyframes = 0 where sha = $1`, sha) - err = tx.Commit() + tx.Exec(ctx, `update gocoder.videos set keyframes = null where sha = $1`, sha) + tx.Exec(ctx, `update gocoder.audios set keyframes = null where sha = $1`, sha) + tx.Exec(ctx, `update gocoder.info set ver_keyframes = 0 where sha = $1`, sha) + err = tx.Commit(ctx) if err != nil { fmt.Printf("error deleting old keyframes from database: %v", err) } @@ -169,79 +190,60 @@ func (s *MetadataService) GetMetadata(ctx context.Context, path string, sha stri return ret, nil } -func (s *MetadataService) getMetadata(path string, sha string) (*MediaInfo, error) { - var ret MediaInfo - var fonts pq.StringArray - err := s.database.QueryRow( - `select i.sha, i.path, i.extension, i.mime_codec, i.size, i.duration, i.container, - i.fonts, i.ver_info, i.ver_extract, i.ver_thumbs, i.ver_keyframes - from gocoder.info as i where i.sha=$1`, +func (s *MetadataService) getMetadata(ctx context.Context, path string, sha string) (*MediaInfo, error) { + rows, _ := s.database.Query( + ctx, + `select + i.sha, i.path, i.extension, i.mime_codec, i.size, i.duration, i.container, i.fonts, + jsonb_build_object( + 'info', i.ver_info, + 'extract', i.ver_extract, + 'thumbs', i.ver_thumbs, + 'keyframes', i.ver_keyframes + ) as versions + from gocoder.info as i + where i.sha=$1 limit 1`, sha, - ).Scan( - &ret.Sha, &ret.Path, &ret.Extension, &ret.MimeCodec, &ret.Size, &ret.Duration, &ret.Container, - &fonts, &ret.Versions.Info, &ret.Versions.Extract, &ret.Versions.Thumbs, &ret.Versions.Keyframes, ) - ret.Fonts = fonts - ret.Videos = make([]Video, 0) - ret.Audios = make([]Audio, 0) - ret.Subtitles = make([]Subtitle, 0) - ret.Chapters = make([]Chapter, 0) + ret, err := pgx.CollectOneRow(rows, pgx.RowToStructByName[MediaInfo]) - if err == sql.ErrNoRows || (ret.Versions.Info < InfoVersion && ret.Versions.Info != 0) { - return s.storeFreshMetadata(path, sha) + if errors.Is(err, pgx.ErrNoRows) || (ret.Versions.Info < InfoVersion && ret.Versions.Info != 0) { + return s.storeFreshMetadata(ctx, path, sha) } if err != nil { return nil, err } - rows, err := s.database.Query( - `select v.idx, v.title, v.language, v.codec, v.mime_codec, v.width, v.height, v.bitrate, v.is_default, v.keyframes - from videos as v where v.sha=$1`, + rows, _ = s.database.Query( + ctx, + `select * from gocoder.videos as v where v.sha=$1`, sha, ) + ret.Videos, err = pgx.CollectRows(rows, pgx.RowToStructByName[Video]) if err != nil { return nil, err } - for rows.Next() { - var v Video - err := rows.Scan(&v.Index, &v.Title, &v.Language, &v.Codec, &v.MimeCodec, &v.Width, &v.Height, &v.Bitrate, &v.IsDefault, &v.Keyframes) - if err != nil { - return nil, err - } - ret.Videos = append(ret.Videos, v) - } - rows, err = s.database.Query( - `select a.idx, a.title, a.language, a.codec, a.mime_codec, a.bitrate, a.is_default, a.keyframes - from audios as a where a.sha=$1`, + rows, _ = s.database.Query( + ctx, + `select * from gocoder.audios as a where a.sha=$1`, sha, ) + ret.Audios, err = pgx.CollectRows(rows, pgx.RowToStructByName[Audio]) if err != nil { return nil, err } - for rows.Next() { - var a Audio - err := rows.Scan(&a.Index, &a.Title, &a.Language, &a.Codec, &a.MimeCodec, &a.Bitrate, &a.IsDefault, &a.Keyframes) - if err != nil { - return nil, err - } - ret.Audios = append(ret.Audios, a) - } - rows, err = s.database.Query( - `select s.idx, s.title, s.language, s.codec, s.mime_codec, s.extension, s.is_default, s.is_forced, s.is_hearing_impaired - from subtitles as s where s.sha=$1`, + rows, _ = s.database.Query( + ctx, + `select * from gocoder.subtitles as s where s.sha=$1`, sha, ) + ret.Subtitles, err = pgx.CollectRows(rows, pgx.RowToStructByName[Subtitle]) if err != nil { return nil, err } - for rows.Next() { - var s Subtitle - err := rows.Scan(&s.Index, &s.Title, &s.Language, &s.Codec, &s.MimeCodec, &s.Extension, &s.IsDefault, &s.IsForced, &s.IsHearingImpaired) - if err != nil { - return nil, err - } + for i, s := range ret.Subtitles { if s.Extension != nil { link := fmt.Sprintf( "/video/%s/subtitle/%d.%s", @@ -249,35 +251,27 @@ func (s *MetadataService) getMetadata(path string, sha string) (*MediaInfo, erro *s.Index, *s.Extension, ) - s.Link = &link + ret.Subtitles[i].Link = &link } - ret.Subtitles = append(ret.Subtitles, s) } err = ret.SearchExternalSubtitles() if err != nil { fmt.Printf("Couldn't find external subtitles: %v", err) } - rows, err = s.database.Query( - `select c.start_time, c.end_time, c.name, c.type - from chapters as c where c.sha=$1`, + rows, _ = s.database.Query( + ctx, + `select * from gocoder.chapters as c where c.sha=$1`, sha, ) + ret.Chapters, err = pgx.CollectRows(rows, pgx.RowToStructByName[Chapter]) if err != nil { return nil, err } - for rows.Next() { - var c Chapter - err := rows.Scan(&c.StartTime, &c.EndTime, &c.Name, &c.Type) - if err != nil { - return nil, err - } - ret.Chapters = append(ret.Chapters, c) - } return &ret, nil } -func (s *MetadataService) storeFreshMetadata(path string, sha string) (*MediaInfo, error) { +func (s *MetadataService) storeFreshMetadata(ctx context.Context, path string, sha string) (*MediaInfo, error) { get_running, set := s.lock.Start(sha) if get_running != nil { return get_running() @@ -288,25 +282,28 @@ func (s *MetadataService) storeFreshMetadata(path string, sha string) (*MediaInf return set(nil, err) } - tx, err := s.database.Begin() + tx, err := s.database.Begin(ctx) if err != nil { return set(ret, err) } // it needs to be a delete instead of a on conflict do update because we want to trigger delete casquade for // videos/audios & co. - tx.Exec(`delete from gocoder.info where path = $1`, path) - tx.Exec(` + tx.Exec(ctx, `delete from gocoder.info where path = $1`, path) + tx.Exec(ctx, + ` insert into gocoder.info(sha, path, extension, mime_codec, size, duration, container, fonts, ver_info, ver_extract, ver_thumbs, ver_keyframes) values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) `, // on conflict do not update versions of extract/thumbs/keyframes ret.Sha, ret.Path, ret.Extension, ret.MimeCodec, ret.Size, ret.Duration, ret.Container, - pq.Array(ret.Fonts), ret.Versions.Info, ret.Versions.Extract, ret.Versions.Thumbs, ret.Versions.Keyframes, + ret.Fonts, ret.Versions.Info, ret.Versions.Extract, ret.Versions.Thumbs, ret.Versions.Keyframes, ) for _, v := range ret.Videos { - tx.Exec(` + tx.Exec( + ctx, + ` insert into gocoder.videos(sha, idx, title, language, codec, mime_codec, width, height, is_default, bitrate) values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) on conflict (sha, idx) do update set @@ -325,7 +322,9 @@ func (s *MetadataService) storeFreshMetadata(path string, sha string) (*MediaInf ) } for _, a := range ret.Audios { - tx.Exec(` + tx.Exec( + ctx, + ` insert into gocoder.audios(sha, idx, title, language, codec, mime_codec, is_default, bitrate) values ($1, $2, $3, $4, $5, $6, $7, $8) on conflict (sha, idx) do update set @@ -342,7 +341,9 @@ func (s *MetadataService) storeFreshMetadata(path string, sha string) (*MediaInf ) } for _, s := range ret.Subtitles { - tx.Exec(` + tx.Exec( + ctx, + ` insert into gocoder.subtitles(sha, idx, title, language, codec, mime_codec, extension, is_default, is_forced, is_hearing_impaired) values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) on conflict (sha, idx) do update set @@ -361,7 +362,9 @@ func (s *MetadataService) storeFreshMetadata(path string, sha string) (*MediaInf ) } for _, c := range ret.Chapters { - tx.Exec(` + tx.Exec( + ctx, + ` insert into gocoder.chapters(sha, start_time, end_time, name, type) values ($1, $2, $3, $4, $5) on conflict (sha, start_time) do update set @@ -374,7 +377,7 @@ func (s *MetadataService) storeFreshMetadata(path string, sha string) (*MediaInf ret.Sha, c.StartTime, c.EndTime, c.Name, c.Type, ) } - err = tx.Commit() + err = tx.Commit(ctx) if err != nil { return set(ret, err) } diff --git a/transcoder/src/thumbnails.go b/transcoder/src/thumbnails.go index d9252646..199ab33a 100644 --- a/transcoder/src/thumbnails.go +++ b/transcoder/src/thumbnails.go @@ -77,7 +77,7 @@ func (s *MetadataService) ExtractThumbs(ctx context.Context, path string, sha st if err != nil { return set(nil, err) } - _, err = s.database.Exec(`update gocoder.info set ver_thumbs = $2 where sha = $1`, sha, ThumbsVersion) + _, err = s.database.Exec(ctx, `update gocoder.info set ver_thumbs = $2 where sha = $1`, sha, ThumbsVersion) return set(nil, err) } From 31500dc3c529c7d54f461990ebe59f01edf4606e Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Sun, 2 Nov 2025 23:40:54 +0100 Subject: [PATCH 07/13] Use an api key for the scanner --- .env.example | 6 +++--- auth/.env.example | 2 +- auth/README.md | 4 ++-- auth/jwt.go | 1 + docker-compose.dev.yml | 1 + docker-compose.yml | 1 + scanner/.env.example | 2 +- 7 files changed, 10 insertions(+), 7 deletions(-) diff --git a/.env.example b/.env.example index 7e5647d5..3cd7a0ef 100644 --- a/.env.example +++ b/.env.example @@ -50,11 +50,11 @@ GUEST_CLAIMS='{"permissions": ["core.read", "core.play"], "verified": true}' PROTECTED_CLAIMS="permissions,verified" -# You can create apikeys at runtime via POST /apikey but you can also have some defined in the env. +# You can create apikeys at runtime via POST /keys but you can also have some defined in the env. # Replace $YOURNAME with the name of the key you want (only alpha are valid) # The value will be the apikey (max 128 bytes) -# KEIBI_APIKEY_$YOURNAME=oaeushtaoesunthoaensuth -# KEIBI_APIKEY_$YOURNAME_CLAIMS='{"permissions": ["users.read"]}' +KEIBI_APIKEY_SCANNER=EJqUB8robwKwLNt37SuHqdcsNGrtwpfYxeExfiAbokpxZVd4WctWr7gnSZ +KEIBI_APIKEY_SCANNER_CLAIMS='{"permissions": ["core.write"]}' # To debug the front end, you can set the following to an external backend KYOO_URL= diff --git a/auth/.env.example b/auth/.env.example index d842cb5a..e0bb34a8 100644 --- a/auth/.env.example +++ b/auth/.env.example @@ -19,7 +19,7 @@ PROTECTED_CLAIMS="permissions" # The url you can use to reach your kyoo instance. This is used during oidc to redirect users to your instance. PUBLIC_URL=http://localhost:8901 -# You can create apikeys at runtime via POST /apikey but you can also have some defined in the env. +# You can create apikeys at runtime via POST /key but you can also have some defined in the env. # Replace $YOURNAME with the name of the key you want (only alpha are valid) # The value will be the apikey (max 128 bytes) # KEIBI_APIKEY_$YOURNAME=oaeushtaoesunthoaensuth diff --git a/auth/README.md b/auth/README.md index 0b446249..0e5feaf8 100644 --- a/auth/README.md +++ b/auth/README.md @@ -60,8 +60,8 @@ GET `/users/$id/sessions` can be used by admins to list others session ### Api keys ``` -Get `/apikeys` -Post `/apikeys` {...claims} Create a new api keys with given claims +Get `/keys` +Post `/keys` {...claims} Create a new api keys with given claims ``` An api key can be used like an opaque token, calling /jwt with it will return a valid jwt with the claims you specified during the post request to create it. diff --git a/auth/jwt.go b/auth/jwt.go index 7e78f875..4337bca9 100644 --- a/auth/jwt.go +++ b/auth/jwt.go @@ -34,6 +34,7 @@ func (h *Handler) CreateJwt(c echo.Context) error { if err != nil { return err } + c.Response().Header().Add("Authorization", fmt.Sprintf("Bearer %s", token)) return c.JSON(http.StatusOK, Jwt{ Token: &token, }) diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index b02600d6..db6631d4 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -120,6 +120,7 @@ services: # Use this env var once we use mTLS for auth # - KYOO_URL=${KYOO_URL:-http://api:3567/api} - KYOO_URL=${KYOO_URL:-http://traefik:8901/api} + - KYOO_APIKEY=scanner-$KEIBI_APIKEY_SCANNER - JWKS_URL=http://auth:4568/.well-known/jwks.json - JWT_ISSUER=${PUBLIC_URL} volumes: diff --git a/docker-compose.yml b/docker-compose.yml index b8cd7c5e..c167991c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -77,6 +77,7 @@ services: # Use this env var once we use mTLS for auth # - KYOO_URL=${KYOO_URL:-http://api:3567/api} - KYOO_URL=${KYOO_URL:-http://traefik:8901/api} + - KYOO_APIKEY=scanner-$KEIBI_APIKEY_SCANNER - JWKS_URL=http://auth:4568/.well-known/jwks.json - JWT_ISSUER=${PUBLIC_URL} volumes: diff --git a/scanner/.env.example b/scanner/.env.example index 15b1a847..c547a8aa 100644 --- a/scanner/.env.example +++ b/scanner/.env.example @@ -11,7 +11,7 @@ LIBRARY_IGNORE_PATTERN=".*/[dD]ownloads?/.*" THEMOVIEDB_API_ACCESS_TOKEN="" KYOO_URL="http://api:3567/api" -KYOO_APIKEY="" +KYOO_APIKEY=scanner-$KEIBI_APIKEY_SCANNER JWKS_URL="http://auth:4568/.well-known/jwks.json" JWT_ISSUER=$PUBLIC_URL From c1b243df9c86f5d9231b39e58a58be5cfbbbc54e Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Mon, 3 Nov 2025 12:08:52 +0100 Subject: [PATCH 08/13] Fix image downloading error handling --- api/src/controllers/seed/images.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/api/src/controllers/seed/images.ts b/api/src/controllers/seed/images.ts index fd31f838..7eaefcc4 100644 --- a/api/src/controllers/seed/images.ts +++ b/api/src/controllers/seed/images.ts @@ -99,13 +99,15 @@ export const processImages = async () => { const column = sql.raw(img.column); await tx.execute(sql` - update ${table} set ${column} = ${ret} where ${column}->'id' = ${sql.raw(`'"${img.id}"'::jsonb`)} - `); + update ${table} set ${column} = ${ret} + where ${column}->'id' = ${sql.raw(`'"${img.id}"'::jsonb`)} + `); await tx.delete(mqueue).where(eq(mqueue.id, item.id)); } catch (err: any) { console.error("Failed to download image", img.url, err.message); - await tx + // don't use the transaction here, it can be aborted. + await db .update(mqueue) .set({ attempt: sql`${mqueue.attempt}+1` }) .where(eq(mqueue.id, item.id)); From 572ddc69add41e32a8b345ff3eddada2f8e9dbd2 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Mon, 3 Nov 2025 12:27:30 +0100 Subject: [PATCH 09/13] Fix video controller permissions --- .env.example | 2 +- api/src/base.ts | 9 +++++---- api/src/controllers/videos.ts | 13 ++++++++++--- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/.env.example b/.env.example index 3cd7a0ef..d1ea0a24 100644 --- a/.env.example +++ b/.env.example @@ -54,7 +54,7 @@ PROTECTED_CLAIMS="permissions,verified" # Replace $YOURNAME with the name of the key you want (only alpha are valid) # The value will be the apikey (max 128 bytes) KEIBI_APIKEY_SCANNER=EJqUB8robwKwLNt37SuHqdcsNGrtwpfYxeExfiAbokpxZVd4WctWr7gnSZ -KEIBI_APIKEY_SCANNER_CLAIMS='{"permissions": ["core.write"]}' +KEIBI_APIKEY_SCANNER_CLAIMS='{"permissions": ["core.read", "core.write"]}' # To debug the front end, you can set the following to an external backend KYOO_URL= diff --git a/api/src/base.ts b/api/src/base.ts index a4e58355..95994a0f 100644 --- a/api/src/base.ts +++ b/api/src/base.ts @@ -13,8 +13,8 @@ import { series } from "./controllers/shows/series"; import { showsH } from "./controllers/shows/shows"; import { staffH } from "./controllers/staff"; import { studiosH } from "./controllers/studios"; -import { videosH } from "./controllers/videos"; -import type { KError } from "./models/error"; +import { videosReadH, videosWriteH } from "./controllers/videos"; +import { KError } from "./models/error"; export const base = new Elysia({ name: "base" }) .onError(({ code, error }) => { @@ -90,7 +90,8 @@ export const handlers = new Elysia({ prefix }) .use(imagesH) .use(watchlistH) .use(historyH) - .use(nextup), + .use(nextup) + .use(videosReadH), ) .guard( { @@ -104,5 +105,5 @@ export const handlers = new Elysia({ prefix }) // }, permissions: ["core.write"], }, - (app) => app.use(videosH).use(seed), + (app) => app.use(videosWriteH).use(seed), ); diff --git a/api/src/controllers/videos.ts b/api/src/controllers/videos.ts index 2409ed2e..57b987f0 100644 --- a/api/src/controllers/videos.ts +++ b/api/src/controllers/videos.ts @@ -439,10 +439,9 @@ function getNextVideoEntry({ .as("next"); } -export const videosH = new Elysia({ prefix: "/videos", tags: ["videos"] }) +export const videosReadH = new Elysia({ prefix: "/videos", tags: ["videos"] }) .model({ video: Video, - "created-videos": t.Array(CreatedVideo), error: t.Object({}), }) .use(auth) @@ -483,7 +482,7 @@ export const videosH = new Elysia({ prefix: "/videos", tags: ["videos"] }) message: `No video found with id or slug '${id}'`, }); } - return video; + return video as any; }, { detail: { @@ -806,6 +805,14 @@ export const videosH = new Elysia({ prefix: "/videos", tags: ["videos"] }) }, }, ) + +export const videosWriteH = new Elysia({ prefix: "/videos", tags: ["videos"] }) + .model({ + video: Video, + "created-videos": t.Array(CreatedVideo), + error: t.Object({}), + }) + .use(auth) .post( "", async ({ body, status }) => { From bc6c93c9c7069ea754ac9565401132a5ae4706ec Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Mon, 3 Nov 2025 12:39:27 +0100 Subject: [PATCH 10/13] Fix bearer cookie being base64ed --- front/src/providers/account-store.ts | 2 +- front/src/providers/settings.ts | 10 ++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/front/src/providers/account-store.ts b/front/src/providers/account-store.ts index 343bcafe..e77cd843 100644 --- a/front/src/providers/account-store.ts +++ b/front/src/providers/account-store.ts @@ -10,7 +10,7 @@ const writeAccounts = (accounts: Account[]) => { if (!selected) return; setCookie("account", selected); // cookie used for images and videos since we can't add Authorization headers in img or video tags. - setCookie("X-Bearer", selected?.token); + setCookie("X-Bearer", selected?.token, { skipBase64: true }); } }; diff --git a/front/src/providers/settings.ts b/front/src/providers/settings.ts index a03691c3..6ce5dd70 100644 --- a/front/src/providers/settings.ts +++ b/front/src/providers/settings.ts @@ -15,8 +15,14 @@ function fromBase64(b64: string) { return Buffer.from(b64, "base64").toString("utf8"); } -export const setCookie = (key: string, val?: unknown) => { - const value = toBase64(typeof val !== "string" ? JSON.stringify(val) : val); +export const setCookie = ( + key: string, + val?: unknown, + opts?: { skipBase64?: boolean }, +) => { + const value = opts?.skipBase64 + ? val + : toBase64(typeof val !== "string" ? JSON.stringify(val) : val); const d = new Date(); // A year d.setTime(d.getTime() + 365 * 24 * 60 * 60 * 1000); From a86cd969a3f5fa9dc0f2f1c565c02363112b9f3d Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Mon, 3 Nov 2025 13:01:54 +0100 Subject: [PATCH 11/13] Properly handle rate limits in the scanner --- scanner/scanner/__init__.py | 10 +++++++--- scanner/scanner/providers/themoviedatabase.py | 18 ++++++++++++++++-- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/scanner/scanner/__init__.py b/scanner/scanner/__init__.py index 30c6126d..62107f20 100644 --- a/scanner/scanner/__init__.py +++ b/scanner/scanner/__init__.py @@ -1,5 +1,5 @@ import logging -from asyncio import CancelledError, TaskGroup, create_task +from asyncio import CancelledError, TaskGroup, create_task, sleep from contextlib import asynccontextmanager from fastapi import FastAPI @@ -48,12 +48,16 @@ async def background_startup( processor: RequestProcessor, is_master: bool | None, ): + async def scan(): + # wait for everything to startup & resume before scanning + await sleep(30) + await scanner.scan(remove_deleted=True) + async with TaskGroup() as tg: _ = tg.create_task(processor.listen(tg)) if is_master: _ = tg.create_task(scanner.monitor()) - _ = tg.create_task(scanner.scan(remove_deleted=True)) - + _ = tg.create_task(scan()) async def cancel(): raise CancelledError() diff --git a/scanner/scanner/providers/themoviedatabase.py b/scanner/scanner/providers/themoviedatabase.py index bb214512..680b66b4 100644 --- a/scanner/scanner/providers/themoviedatabase.py +++ b/scanner/scanner/providers/themoviedatabase.py @@ -7,7 +7,7 @@ from statistics import mean from types import TracebackType from typing import Any, cast, override -from aiohttp import ClientSession +from aiohttp import ClientResponseError, ClientSession from langcodes import Language from ..models.collection import Collection, CollectionTranslation @@ -643,7 +643,21 @@ class TheMovieDatabase(Provider): async with self._client.get(path, params=params) as r: if not_found_fail and r.status == 404: raise ProviderError(not_found_fail) - r.raise_for_status() + if r.status == 429: + retry_after = r.headers.get("Retry-After") + delay = float(retry_after) if retry_after else 2.0 + await asyncio.sleep(delay) + return await self._get( + path, params=params, not_found_fail=not_found_fail + ) + if r.status >= 400: + raise ClientResponseError( + r.request_info, + r.history, + status=r.status, + message=await r.text(), + headers=r.headers, + ) return await r.json() def _map_genres(self, genres: Generator[int]) -> list[Genre]: From 01177c2489d7a871c0bf041e93bc61d187590f53 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Mon, 3 Nov 2025 13:05:01 +0100 Subject: [PATCH 12/13] Proprely remove cookies on logout --- front/src/providers/account-store.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/front/src/providers/account-store.ts b/front/src/providers/account-store.ts index e77cd843..f42b87d5 100644 --- a/front/src/providers/account-store.ts +++ b/front/src/providers/account-store.ts @@ -7,7 +7,6 @@ const writeAccounts = (accounts: Account[]) => { storeValue("accounts", accounts); if (Platform.OS === "web") { const selected = accounts.find((x) => x.selected); - if (!selected) return; setCookie("account", selected); // cookie used for images and videos since we can't add Authorization headers in img or video tags. setCookie("X-Bearer", selected?.token, { skipBase64: true }); From 03bb51661a4e1e720e2c9ed30ade47c64a1f46bd Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Tue, 4 Nov 2025 09:48:14 +0100 Subject: [PATCH 13/13] Format stuff --- api/src/base.ts | 2 +- api/src/controllers/videos.ts | 2 +- scanner/scanner/__init__.py | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/api/src/base.ts b/api/src/base.ts index 95994a0f..5f8d0dc6 100644 --- a/api/src/base.ts +++ b/api/src/base.ts @@ -14,7 +14,7 @@ import { showsH } from "./controllers/shows/shows"; import { staffH } from "./controllers/staff"; import { studiosH } from "./controllers/studios"; import { videosReadH, videosWriteH } from "./controllers/videos"; -import { KError } from "./models/error"; +import type { KError } from "./models/error"; export const base = new Elysia({ name: "base" }) .onError(({ code, error }) => { diff --git a/api/src/controllers/videos.ts b/api/src/controllers/videos.ts index 57b987f0..c151cec9 100644 --- a/api/src/controllers/videos.ts +++ b/api/src/controllers/videos.ts @@ -804,7 +804,7 @@ export const videosReadH = new Elysia({ prefix: "/videos", tags: ["videos"] }) 422: KError, }, }, - ) + ); export const videosWriteH = new Elysia({ prefix: "/videos", tags: ["videos"] }) .model({ diff --git a/scanner/scanner/__init__.py b/scanner/scanner/__init__.py index 62107f20..152b470a 100644 --- a/scanner/scanner/__init__.py +++ b/scanner/scanner/__init__.py @@ -59,6 +59,7 @@ async def background_startup( _ = tg.create_task(scanner.monitor()) _ = tg.create_task(scan()) + async def cancel(): raise CancelledError()