From 35909632068e9d3b5e1d17ac8f5759c496accf8d Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Mon, 20 Oct 2025 18:57:42 +0200 Subject: [PATCH] Add subtitle conversion (from srt to vtt for example) --- front/bun.lock | 2 +- front/src/ui/player/index.tsx | 5 +- transcoder/Dockerfile | 28 +------- transcoder/Dockerfile.dev | 25 +------- transcoder/go.mod | 3 + transcoder/go.sum | 82 ++---------------------- transcoder/src/api/metadata.go | 40 +++++++++--- transcoder/src/extract.go | 59 ++++++++--------- transcoder/src/metadata.go | 8 +-- transcoder/src/storage/filestorage.go | 92 +++++---------------------- transcoder/src/storage/storage.go | 26 +++----- transcoder/src/subtitles.go | 59 ++++++++++++++++- transcoder/src/thumbnails.go | 51 +++------------ 13 files changed, 171 insertions(+), 309 deletions(-) diff --git a/front/bun.lock b/front/bun.lock index 34b7d835..7d439862 100644 --- a/front/bun.lock +++ b/front/bun.lock @@ -1265,7 +1265,7 @@ "react-native-svg-transformer": ["react-native-svg-transformer@1.5.1", "", { "dependencies": { "@svgr/core": "^8.1.0", "@svgr/plugin-jsx": "^8.1.0", "@svgr/plugin-svgo": "^8.1.0", "path-dirname": "^1.0.2" }, "peerDependencies": { "react-native": ">=0.59.0", "react-native-svg": ">=12.0.0" } }, "sha512-dFvBNR8A9VPum9KCfh+LE49YiJEF8zUSnEFciKQroR/bEOhlPoZA0SuQ0qNk7m2iZl2w59FYjdRe0pMHWMDl0Q=="], - "react-native-video": ["react-native-video@github:zoriya/react-native-video#ad69842", { "peerDependencies": { "react": "*", "react-native": "*", "react-native-nitro-modules": ">=0.27.2" } }, "zoriya-react-native-video-ad69842"], + "react-native-video": ["react-native-video@github:zoriya/react-native-video#19925f8", { "peerDependencies": { "react": "*", "react-native": "*", "react-native-nitro-modules": ">=0.27.2" } }, "zoriya-react-native-video-19925f8"], "react-native-web": ["react-native-web@0.21.2", "", { "dependencies": { "@babel/runtime": "^7.18.6", "@react-native/normalize-colors": "^0.74.1", "fbjs": "^3.0.4", "inline-style-prefixer": "^7.0.1", "memoize-one": "^6.0.0", "nullthrows": "^1.1.1", "postcss-value-parser": "^4.2.0", "styleq": "^0.1.3" }, "peerDependencies": { "react": "^18.0.0 || ^19.0.0", "react-dom": "^18.0.0 || ^19.0.0" } }, "sha512-SO2t9/17zM4iEnFvlu2DA9jqNbzNhoUP+AItkoCOyFmDMOhUnBBznBDCYN92fGdfAkfQlWzPoez6+zLxFNsZEg=="], diff --git a/front/src/ui/player/index.tsx b/front/src/ui/player/index.tsx index 0146ee86..7d5f43d8 100644 --- a/front/src/ui/player/index.tsx +++ b/front/src/ui/player/index.tsx @@ -68,7 +68,10 @@ export const Player = () => { .map((x) => ({ // we also add those without link to prevent the order from getting out of sync with `info.subtitles`. // since we never actually play those this is fine - uri: x.link!, + uri: + x.codec === "subrip" && x.link && Platform.OS === "web" + ? `${x.link}?format=vtt` + : x.link!, label: x.title ?? "Unknown", language: x.language ?? "und", type: x.codec, diff --git a/transcoder/Dockerfile b/transcoder/Dockerfile index a9c98f0e..3deebbc2 100644 --- a/transcoder/Dockerfile +++ b/transcoder/Dockerfile @@ -1,24 +1,4 @@ -# FROM golang:1.23 as build -FROM debian:trixie-slim AS build -# those were copied from https://github.com/docker-library/golang/blob/master/Dockerfile-linux.template -ENV GOTOOLCHAIN=local -ENV GOPATH=/go -ENV PATH=$GOPATH/bin:/usr/local/go/bin:$PATH -RUN set -eux; \ - apt-get update; \ - apt-get install -y --no-install-recommends \ - ca-certificates openssl \ - golang\ - g++ \ - gcc \ - libc6-dev \ - make \ - pkg-config - -# https://github.com/golang/go/issues/54400 -ENV SSL_CERT_DIR=/etc/ssl/certs -RUN update-ca-certificates - +FROM golang:1.25 as build RUN apt-get update \ && apt-get install --no-install-recommends --no-install-suggests -y \ ffmpeg libavformat-dev libavutil-dev libswscale-dev \ @@ -32,10 +12,8 @@ RUN go mod download COPY . . RUN GOOS=linux go build -o ./transcoder -# debian is required for nvidia hardware acceleration -# we use trixie (debian's testing because ffmpeg on latest is v5 and we need v6) -# https://packages.debian.org/bookworm/ffmpeg for version tracking -FROM debian:trixie-slim +# https://packages.debian.org/trixie/ffmpeg for version tracking +FROM debian:trixie # read target arch from buildx or default to amd64 if using legacy builder. ARG TARGETARCH diff --git a/transcoder/Dockerfile.dev b/transcoder/Dockerfile.dev index b3ccf9b7..6946e183 100644 --- a/transcoder/Dockerfile.dev +++ b/transcoder/Dockerfile.dev @@ -1,27 +1,4 @@ -# we use trixie (debian's testing because ffmpeg on latest is v5 and we need v6) -# https://packages.debian.org/bookworm/ffmpeg for version tracking -# FROM golang:1.21 -# trixie's golang is also 1.21 -FROM debian:trixie-slim -# those were copied from https://github.com/docker-library/golang/blob/master/Dockerfile-linux.template -ENV GOTOOLCHAIN=local -ENV GOPATH=/go -ENV PATH=$GOPATH/bin:/usr/local/go/bin:$PATH -RUN set -eux; \ - apt-get update; \ - apt-get install -y --no-install-recommends \ - ca-certificates openssl \ - golang\ - g++ \ - gcc \ - libc6-dev \ - make \ - pkg-config - -# https://github.com/golang/go/issues/54400 -ENV SSL_CERT_DIR=/etc/ssl/certs -RUN update-ca-certificates - +FROM golang:1.25 # read target arch from buildx or default to amd64 if using legacy builder. ARG TARGETARCH ENV TARGETARCH=${TARGETARCH:-amd64} diff --git a/transcoder/go.mod b/transcoder/go.mod index 3fc51871..2310f9cc 100644 --- a/transcoder/go.mod +++ b/transcoder/go.mod @@ -3,6 +3,7 @@ module github.com/zoriya/kyoo/transcoder go 1.24.2 require ( + github.com/asticode/go-astisub v0.35.0 github.com/aws/aws-sdk-go-v2 v1.39.3 github.com/aws/aws-sdk-go-v2/service/s3 v1.88.5 github.com/disintegration/imaging v1.6.2 @@ -19,6 +20,8 @@ require ( require ( github.com/KyleBanks/depth v1.2.1 // indirect + github.com/asticode/go-astikit v0.20.0 // indirect + github.com/asticode/go-astits v1.8.0 // indirect github.com/ghodss/yaml v1.0.0 // indirect github.com/go-openapi/jsonpointer v0.21.1 // indirect github.com/go-openapi/jsonreference v0.21.0 // indirect diff --git a/transcoder/go.sum b/transcoder/go.sum index 81c2b219..22e40a7b 100644 --- a/transcoder/go.sum +++ b/transcoder/go.sum @@ -1,9 +1,11 @@ -github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= -github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc= github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE= -github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= -github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/asticode/go-astikit v0.20.0 h1:+7N+J4E4lWx2QOkRdOf6DafWJMv6O4RRfgClwQokrH8= +github.com/asticode/go-astikit v0.20.0/go.mod h1:h4ly7idim1tNhaVkdVBeXQZEE3L0xblP7fCWbgwipF0= +github.com/asticode/go-astisub v0.35.0 h1:wnELGJMeJbavW//X7nLTy97L3iblub7tO1VSeHnZBdA= +github.com/asticode/go-astisub v0.35.0/go.mod h1:WTkuSzFB+Bp7wezuSf2Oxulj5A8zu2zLRVFf6bIFQK8= +github.com/asticode/go-astits v1.8.0 h1:rf6aiiGn/QhlFjNON1n5plqF3Fs025XLUwiQ0NB6oZg= +github.com/asticode/go-astits v1.8.0/go.mod h1:DkOWmBNQpnr9mv24KfZjq4JawCFX1FCqjLVGvO0DygQ= github.com/aws/aws-sdk-go-v2 v1.39.3 h1:h7xSsanJ4EQJXG5iuW4UqgP7qBopLpj84mpkNx3wPjM= github.com/aws/aws-sdk-go-v2 v1.39.3/go.mod h1:yWSxrnioGUZ4WVv9TgMrNUeLV3PFESn/v+6T/Su8gnM= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.2 h1:t9yYsydLYNBk9cJ73rgPhPWqOh/52fcWDQB5b1JsKSY= @@ -40,35 +42,12 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.38.7 h1:VEO5dqFkMsl8QZ2yHsFDJAIZLAkE github.com/aws/aws-sdk-go-v2/service/sts v1.38.7/go.mod h1:L1xxV3zAdB+qVrVW/pBIrIAnHFWHo6FBbFe4xOGsG/o= github.com/aws/smithy-go v1.23.1 h1:sLvcH6dfAFwGkHLZ7dGiYF7aK6mg4CgKA/iDKjLDt9M= github.com/aws/smithy-go v1.23.1/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0= -github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= -github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= -github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= -github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 h1:NMZiJj8QnKe1LgsbDayM4UoHwbvwDRwnI3hwNaAHRnc= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0/go.mod h1:ZXNYxsqcloTdSy/rNShjYzMhyjf0LaoftYK0p+A3h40= -github.com/dhui/dktest v0.4.6 h1:+DPKyScKSEp3VLtbMDHcUq6V5Lm5zfZZVb0Sk7Ahom4= -github.com/dhui/dktest v0.4.6/go.mod h1:JHTSYDtKkvFNFHJKqCzVzqXecyv+tKt8EzceOmQOgbU= github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1ei82L+c= github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4= -github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= -github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= -github.com/docker/docker v28.3.3+incompatible h1:Dypm25kh4rmk49v1eiVbsAtpAsYURjYkaKubwuBdxEI= -github.com/docker/docker v28.3.3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= -github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= -github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= -github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= -github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= -github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= -github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= -github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-openapi/jsonpointer v0.21.1 h1:whnzv/pNXtK2FbX/W9yJfRmE2gsmkfahjMKB0fZvcic= github.com/go-openapi/jsonpointer v0.21.1/go.mod h1:50I1STOfbY1ycR8jGz8DaMeLCdXiI6aDteEdRNNzpdk= github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF2+tg1TRrwQ= @@ -79,25 +58,16 @@ github.com/go-openapi/swag v0.23.1 h1:lpsStH0n2ittzTnbaSloVZLuB5+fvSY/+hnagBjSNZ github.com/go-openapi/swag v0.23.1/go.mod h1:STZs8TbRvEQQKUA+JZNAm3EWlgaOBGpyFDqQnDHMef0= github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= -github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= -github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo= github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/golang-migrate/migrate/v4 v4.19.0 h1:RcjOnCGz3Or6HQYEJ/EEVLfWnmw9KnoigPSjzhCuaSE= github.com/golang-migrate/migrate/v4 v4.19.0/go.mod h1:9dyEcu+hO+G9hPSw8AIg50yg622pXJsoHItQnDGZkI0= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= -github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= -github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= -github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/labstack/echo-jwt/v4 v4.3.1 h1:d8+/qf8nx7RxeL46LtoIwHJsH2PNN8xXCQ/jDianycE= github.com/labstack/echo-jwt/v4 v4.3.1/go.mod h1:yJi83kN8S/5vePVPd+7ID75P4PqPNVRs2HVeuvYJH00= github.com/labstack/echo/v4 v4.13.4 h1:oTZZW+T3s9gAu5L8vmzihV7/lkXGZuITzTQkTEhcXEA= @@ -124,29 +94,8 @@ github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHP github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= -github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= -github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= -github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= -github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= -github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= -github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= -github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= -github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= -github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys= github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/swaggo/echo-swagger v1.4.1 h1:Yf0uPaJWp1uRtDloZALyLnvdBeoEL5Kc7DtnjzO/TUk= github.com/swaggo/echo-swagger v1.4.1/go.mod h1:C8bSi+9yH2FLZsnhqMZLIZddpUxZdBYuNHbtaS1Hljc= github.com/swaggo/files/v2 v2.0.2 h1:Bq4tgS/yxLB/3nwOMcul5oLEUKa877Ykgz3CJMVbQKU= @@ -155,25 +104,12 @@ github.com/swaggo/swag v1.16.6 h1:qBNcx53ZaX+M5dxVyTrgQ0PJ/ACK+NzhwcbieTt+9yI= github.com/swaggo/swag v1.16.6/go.mod h1:ngP2etMK5a0P3QBizic5MEwpRmluJZPHjXcMoj4Xesg= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/valyala/fastjson v1.6.4 h1:uAUNq9Z6ymTgGhcm0UynUAB6tlbakBrz6CQFax3BXVQ= -github.com/valyala/fastjson v1.6.4/go.mod h1:CLCAqky6SMuOcxStkYQvblddUtoRxhYMGLrsQns1aXY= github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQD0Loo= github.com/valyala/fasttemplate v1.2.2/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= gitlab.com/opennota/screengen v1.0.2 h1:GxYTJdAPEzmg5v5CV4dgn45JVW+EcXXAvCxhE7w6UDw= gitlab.com/opennota/screengen v1.0.2/go.mod h1:4kED4yriw2zslwYmXFCa5qCvEKwleBA7l5OE+d94NTU= -go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= -go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 h1:TT4fX+nBOA/+LUkobKGW1ydGcn+G3vRw9+g5HwCphpk= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0/go.mod h1:L7UH0GbB0p47T4Rri3uHjbpCFYrVrwc1I25QhNPiGK8= -go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ= -go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I= -go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE= -go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E= -go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4= -go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0= golang.org/x/crypto v0.42.0 h1:chiH31gIWm57EkTXpwnqf8qeuMUi0yekh6mT2AvFlqI= golang.org/x/crypto v0.42.0/go.mod h1:4+rDnOTJhQCx2q7/j6rAN5XDw8kPjeaXEUR2eL94ix8= -golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.29.0 h1:HcdsyR4Gsuys/Axh0rDEmlBmB68rW1U9BUdB3UVHsas= golang.org/x/image v0.29.0/go.mod h1:RVJROnf3SLK8d26OW91j4FrIHGbsJ8QnbEocVTOWQDA= golang.org/x/mod v0.28.0 h1:gQBtGhjxykdjY9YhZpSlZIsbnaE2+PgjfLWUQTnoZ1U= @@ -182,23 +118,17 @@ golang.org/x/net v0.44.0 h1:evd8IRDyfNBMBTTY5XRF1vaZlD+EmWx6x8PkhR04H/I= golang.org/x/net v0.44.0/go.mod h1:ECOoLqd5U3Lhyeyo/QDCEVQ4sNgYsqvCZ722XogGieY= golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k= golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.30.0 h1:yznKA/E9zq54KzlzBEAWn1NXSQ8DIp/NYMy88xJjl4k= golang.org/x/text v0.30.0/go.mod h1:yDdHFIX9t+tORqspjENWgzaCVXgk0yYnYuSZ8UzzBVM= golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= golang.org/x/tools v0.37.0 h1:DVSRzp7FwePZW356yEAChSdNcQo6Nsp+fex1SUW09lE= golang.org/x/tools v0.37.0/go.mod h1:MBN5QPQtLMHVdvsbtarmTNukZDdgwdwlO5qGacAzF0w= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/vansante/go-ffprobe.v2 v2.2.1 h1:sFV08OT1eZ1yroLCZVClIVd9YySgCh9eGjBWO0oRayI= gopkg.in/vansante/go-ffprobe.v2 v2.2.1/go.mod h1:qF0AlAjk7Nqzqf3y333Ly+KxN3cKF2JqA3JT5ZheUGE= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/transcoder/src/api/metadata.go b/transcoder/src/api/metadata.go index 81e35b59..5879cb56 100644 --- a/transcoder/src/api/metadata.go +++ b/transcoder/src/api/metadata.go @@ -7,6 +7,7 @@ import ( "net/http" "path/filepath" + "github.com/asticode/go-astisub" "github.com/labstack/echo/v4" "github.com/zoriya/kyoo/transcoder/src" "github.com/zoriya/kyoo/transcoder/src/utils" @@ -55,11 +56,12 @@ func (h *mhandler) GetInfo(c echo.Context) error { // @Tags metadata // @Param path path string true "Base64 of a video's path" format(base64) example(L3ZpZGVvL2J1YmJsZS5ta3YK) // @Param name path string true "Name of the subtitle" example(en.srt) +// @Param format query string false "Output format to convert the subtitle into" example(vtt) // // @Success 200 file "Requested subtitle" // @Router /:path/subtitle/:name [get] func (h *mhandler) GetSubtitle(c echo.Context) (err error) { - _, sha, err := getPath(c) + path, sha, err := getPath(c) if err != nil { return err } @@ -68,23 +70,41 @@ func (h *mhandler) GetSubtitle(c echo.Context) (err error) { return err } - subtitleStream, err := h.metadata.GetSubtitle(c.Request().Context(), sha, name) + stream, err := h.metadata.GetSubtitle(c.Request().Context(), path, sha, name) if err != nil { return err } - defer utils.CleanupWithErr(&err, subtitleStream.Close, "failed to close subtitle reader") + defer utils.CleanupWithErr(&err, stream.Close, "failed to close subtitle reader") - mimeType, err := guessMimeType(name, subtitleStream) - if err != nil { - return fmt.Errorf("failed to guess mime type: %w", err) + outFmt := c.QueryParam("format") + if outFmt == "" { + mimeType, err := guessMimeType(name, stream) + if err != nil { + return fmt.Errorf("failed to guess mime type: %w", err) + } + + // Default the mime type to text/plain if it is not recognized + if mimeType == "" { + mimeType = "text/plain" + } + return c.Stream(200, mimeType, stream) } - // Default the mime type to text/plain if it is not recognized - if mimeType == "" { - mimeType = "text/plain" + pr, pw := io.Pipe() + outExt := fmt.Sprintf(".%s", outFmt) + err = src.ConvertSubtitle( + filepath.Ext(name), + stream, + outExt, + pw, + ) + if err == astisub.ErrInvalidExtension { + return echo.NewHTTPError(http.StatusBadRequest, "Input or output format not supported. Conversion failed.") + } else if err != nil { + return err } - return c.Stream(200, mimeType, subtitleStream) + return c.Stream(200, mime.TypeByExtension(outExt), pr) } // @Summary Get attachments diff --git a/transcoder/src/extract.go b/transcoder/src/extract.go index 2543d1dd..91040704 100644 --- a/transcoder/src/extract.go +++ b/transcoder/src/extract.go @@ -8,10 +8,10 @@ import ( "os" "os/exec" "path/filepath" + "strings" "github.com/zoriya/kyoo/transcoder/src/storage" "github.com/zoriya/kyoo/transcoder/src/utils" - "golang.org/x/sync/errgroup" ) const ExtractVersion = 1 @@ -24,6 +24,7 @@ func (s *MetadataService) ExtractSubs(ctx context.Context, info *MediaInfo) (any err := s.extractSubs(ctx, info) if err != nil { + log.Printf("Couldn't extract subs: %v", err) return set(nil, err) } _, err = s.database.Exec(`update info set ver_extract = $2 where sha = $1`, info.Sha, ExtractVersion) @@ -45,13 +46,24 @@ func (s *MetadataService) GetAttachment(ctx context.Context, sha string, name st return item, nil } -func (s *MetadataService) GetSubtitle(ctx context.Context, sha string, name string) (io.ReadCloser, error) { +func (s *MetadataService) GetSubtitle( + ctx context.Context, + path string, + sha string, + name string, +) (io.ReadCloser, error) { + // name is either `{index}.{extension}` or `ext-{filename}.{extension}` + if strings.HasPrefix(name, "ext") { + // external subtitle already available on the fs + return os.Open(path) + } + _, err := s.extractLock.WaitFor(sha) if err != nil { return nil, err } - itemPath := fmt.Sprintf("%s/%s/%s", sha, "sub", name) + itemPath := fmt.Sprintf("%s/sub/%s", sha, name) item, err := s.storage.GetItem(ctx, itemPath) if err != nil { @@ -75,24 +87,24 @@ func (s *MetadataService) extractSubs(ctx context.Context, info *MediaInfo) (err return nil } - tempWorkingDirectory := filepath.Join(os.TempDir(), info.Sha) - if err := os.MkdirAll(tempWorkingDirectory, 0660); err != nil { + workdir := filepath.Join(os.TempDir(), info.Sha) + if err := os.MkdirAll(workdir, 0660); err != nil { return fmt.Errorf("failed to create temporary directory: %w", err) } - attachmentWorkingDirectory := filepath.Join(tempWorkingDirectory, "att") - if err := os.MkdirAll(attachmentWorkingDirectory, 0660); err != nil { + attDir := filepath.Join(workdir, "att") + if err := os.MkdirAll(attDir, 0660); err != nil { return fmt.Errorf("failed to create attachment directory: %w", err) } - subtitlesWorkingDirectory := filepath.Join(tempWorkingDirectory, "sub") - if err := os.MkdirAll(subtitlesWorkingDirectory, 0660); err != nil { + subDir := filepath.Join(workdir, "sub") + if err := os.MkdirAll(subDir, 0660); err != nil { return fmt.Errorf("failed to create subtitles directory: %w", err) } defer utils.CleanupWithErr( &err, func() error { - return os.RemoveAll(attachmentWorkingDirectory) + return os.RemoveAll(workdir) }, "failed to remove attachment working directory", ) @@ -105,7 +117,7 @@ func (s *MetadataService) extractSubs(ctx context.Context, info *MediaInfo) (err "-y", "-i", info.Path, ) - cmd.Dir = attachmentWorkingDirectory + cmd.Dir = attDir for _, sub := range info.Subtitles { if ext := sub.Extension; ext != nil { @@ -117,7 +129,7 @@ func (s *MetadataService) extractSubs(ctx context.Context, info *MediaInfo) (err cmd.Args, "-map", fmt.Sprintf("0:s:%d", *sub.Index), "-c:s", "copy", - fmt.Sprintf("%s/%d.%s", subtitlesWorkingDirectory, *sub.Index, *ext), + fmt.Sprintf("%s/%d.%s", subDir, *sub.Index, *ext), ) } } @@ -129,27 +141,12 @@ func (s *MetadataService) extractSubs(ctx context.Context, info *MediaInfo) (err return err } - // Move attachments and subtitles to the storage backend - var saveGroup errgroup.Group - saveGroup.Go(func() error { - err := storage.SaveFilesToBackend(ctx, s.storage, attachmentWorkingDirectory, filepath.Join(info.Sha, "att")) - if err != nil { - return fmt.Errorf("failed to save attachments to backend: %w", err) - } - return nil - }) - - saveGroup.Go(func() error { - err := storage.SaveFilesToBackend(ctx, s.storage, subtitlesWorkingDirectory, filepath.Join(info.Sha, "sub")) - if err != nil { - return fmt.Errorf("failed to save subtitles to backend: %w", err) - } - return nil - }) - - if err := saveGroup.Wait(); err != nil { + err = storage.SaveFilesToBackend(ctx, s.storage, workdir, info.Sha) + if err != nil { return fmt.Errorf("failed while saving files to backend: %w", err) } + log.Printf("Extraction finished for %s", info.Path) + return nil } diff --git a/transcoder/src/metadata.go b/transcoder/src/metadata.go index 5d33875a..ffaa009a 100644 --- a/transcoder/src/metadata.go +++ b/transcoder/src/metadata.go @@ -21,8 +21,8 @@ import ( type MetadataService struct { database *sql.DB lock RunLock[string, *MediaInfo] - thumbLock RunLock[string, interface{}] - extractLock RunLock[string, interface{}] + thumbLock RunLock[string, any] + extractLock RunLock[string, any] keyframeLock RunLock[KeyframeKey, *Keyframe] storage storage.StorageBackend } @@ -32,8 +32,8 @@ func NewMetadataService() (*MetadataService, error) { s := &MetadataService{ lock: NewRunLock[string, *MediaInfo](), - thumbLock: NewRunLock[string, interface{}](), - extractLock: NewRunLock[string, interface{}](), + thumbLock: NewRunLock[string, any](), + extractLock: NewRunLock[string, any](), keyframeLock: NewRunLock[KeyframeKey, *Keyframe](), } diff --git a/transcoder/src/storage/filestorage.go b/transcoder/src/storage/filestorage.go index b8617e40..43205688 100644 --- a/transcoder/src/storage/filestorage.go +++ b/transcoder/src/storage/filestorage.go @@ -7,18 +7,15 @@ import ( "io/fs" "os" "path/filepath" - "strings" "github.com/zoriya/kyoo/transcoder/src/utils" ) type FileStorageBackend struct { - // Base directory for the storage backend - baseDirectory string - baseDirectoryRoot *os.Root + baseDirectory string + root *os.Root } -// NewFileStorageBackend creates a new FileStorageBackend with the specified base directory. func NewFileStorageBackend(baseDirectory string) (*FileStorageBackend, error) { // Attempt to create the directory if it doesn't exist // This should be the only filesystem call in this file that does not use os.Root. @@ -33,21 +30,20 @@ func NewFileStorageBackend(baseDirectory string) (*FileStorageBackend, error) { } return &FileStorageBackend{ - baseDirectory: baseDirectory, - baseDirectoryRoot: root, + baseDirectory: baseDirectory, + root: root, }, nil } func (fsb *FileStorageBackend) Close() error { - if fsb.baseDirectoryRoot != nil { - return fsb.baseDirectoryRoot.Close() + if fsb.root != nil { + return fsb.root.Close() } return nil } -// DoesItemExist checks if an item exists in the file storage backend. func (fsb *FileStorageBackend) DoesItemExist(_ context.Context, path string) (bool, error) { - _, err := fsb.baseDirectoryRoot.Stat(path) + _, err := fsb.root.Stat(path) if err != nil { if os.IsNotExist(err) { return false, nil @@ -59,93 +55,45 @@ func (fsb *FileStorageBackend) DoesItemExist(_ context.Context, path string) (bo return true, nil } -// ListItemsWithPrefix returns a list of items in the storage backend that match the given prefix. func (fsb *FileStorageBackend) ListItemsWithPrefix(_ context.Context, pathPrefix string) ([]string, error) { var items []string - rootFS := fsb.baseDirectoryRoot.FS() - // This is split so that a smaller subset of files are checked, rather than literally everything under the base directory. - // All matching files for the path prefix will also have the prefixDirPath as their parent directory. - prefixDirPath := filepath.Dir(pathPrefix) - err := fs.WalkDir(rootFS, prefixDirPath, func(path string, d fs.DirEntry, err error) error { + err := fs.WalkDir(fsb.root.FS(), pathPrefix, func(path string, d fs.DirEntry, err error) error { if err != nil { - // This can happen if prefixDirPath does not exist. The walk function will handle - // checking this. - if os.IsNotExist(err) { - return fs.SkipDir - } - return fmt.Errorf("failed on %q while walking directory %q: %w", path, pathPrefix, err) + return err } - - // If the path does not start with the prefix, skip it. - if !strings.HasPrefix(path, pathPrefix) { - if d.IsDir() { - // Skip directories that do not match the prefix - return fs.SkipDir - } - return nil - } - - // Collect matching non-directory items if !d.IsDir() { items = append(items, path) } - return nil }) if err != nil { - return nil, fmt.Errorf("failed to walk directory %q: %w", pathPrefix, err) + return nil, err } return items, nil } -// DeleteItem deletes an item from the storage backend. If the item does not exist, it returns nil. func (fsb *FileStorageBackend) DeleteItem(_ context.Context, path string) error { - err := fsb.baseDirectoryRoot.Remove(path) + err := fsb.root.Remove(path) if err != nil && !os.IsNotExist(err) { return fmt.Errorf("failed to delete item %q: %w", path, err) } return nil } -// DeleteItemsWithPrefix deletes all items in the storage backend that match the given prefix. -// Deletion should be "syncronous" (i.e. the function should block until the write is complete). func (fsb *FileStorageBackend) DeleteItemsWithPrefix(ctx context.Context, pathPrefix string) error { - // Unfortunately this implementation is needed until https://go-review.googlesource.com/c/go/+/661595 is released. - // The new os.Root type does not yet have a RemoveAll method. The next Go release will have this. - // Once RemoveAll is available, this function can be reduced to a single ReadDir call, a filter, and a RemoveAll call. - - // Get all items with the prefix - items, err := fsb.ListItemsWithPrefix(ctx, pathPrefix) - if err != nil { - return fmt.Errorf("failed to list items with prefix %q: %w", pathPrefix, err) - } - - // Delete all items. This will leave behind empty directories, but that shouldn't really matter. A future - // implementation that uses os.Root.RemoveAll will handle this. - for _, item := range items { - err = fsb.DeleteItem(ctx, item) - if err != nil { - return fmt.Errorf("failed to delete item %q: %w", item, err) - } - } - - return nil + return fsb.root.RemoveAll(pathPrefix) } -// SaveItemWithCallback saves an item to the storage backend. If the item already exists, it overwrites it. -// The writeContents function is called with a writer to write the contents of the item. func (fsb *FileStorageBackend) SaveItemWithCallback(ctx context.Context, path string, writeContents ContentsWriterCallback) (err error) { - // Open the file for writing file, err := fsb.openFileForWriting(path) if err != nil { return fmt.Errorf("failed to open %q for writing: %w", path, err) } defer utils.CleanupWithErr(&err, file.Close, "failed to close file %q", path) - // Write the contents using the provided callback if err := writeContents(ctx, file); err != nil { return fmt.Errorf("failed to write contents to file %q: %w", path, err) } @@ -153,16 +101,13 @@ func (fsb *FileStorageBackend) SaveItemWithCallback(ctx context.Context, path st return nil } -// SaveItem saves an item to the storage backend. If the item already exists, it overwrites it. func (fsb *FileStorageBackend) SaveItem(ctx context.Context, path string, contents io.Reader) (err error) { - // Open the file for writing file, err := fsb.openFileForWriting(path) if err != nil { return fmt.Errorf("failed to open %q for writing: %w", path, err) } defer utils.CleanupWithErr(&err, file.Close, "failed to close file %q", path) - // Copy the contents to the file if _, err := io.Copy(file, contents); err != nil { return fmt.Errorf("failed to copy contents to file %q: %w", path, err) } @@ -170,19 +115,13 @@ func (fsb *FileStorageBackend) SaveItem(ctx context.Context, path string, conten return nil } -// openFileForWriting opens a file for writing. If the file already exists, it overwrites it. -// The parent directory is created if it doesn't exist. -// This function is used internally to create files in the storage backend. -// The returned file should be closed by the caller. func (fsb *FileStorageBackend) openFileForWriting(path string) (*os.File, error) { - // Create the parent directory if it doesn't exist dir := filepath.Dir(path) - if err := fsb.baseDirectoryRoot.Mkdir(dir, 0770); err != nil { + if err := fsb.root.MkdirAll(dir, 0770); err != nil { return nil, fmt.Errorf("failed to create directory %q: %w", dir, err) } - // Open the file for writing - file, err := fsb.baseDirectoryRoot.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0660) + file, err := fsb.root.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0660) if err != nil { return nil, fmt.Errorf("failed to create file %q: %w", path, err) } @@ -190,9 +129,8 @@ func (fsb *FileStorageBackend) openFileForWriting(path string) (*os.File, error) return file, nil } -// GetItem retrieves an item from the storage backend. func (fsb *FileStorageBackend) GetItem(_ context.Context, path string) (io.ReadCloser, error) { - file, err := fsb.baseDirectoryRoot.Open(path) + file, err := fsb.root.Open(path) if err != nil { return nil, fmt.Errorf("failed to open file %q: %w", path, err) } diff --git a/transcoder/src/storage/storage.go b/transcoder/src/storage/storage.go index 661ff5d9..883b4afe 100644 --- a/transcoder/src/storage/storage.go +++ b/transcoder/src/storage/storage.go @@ -8,7 +8,6 @@ import ( "os" "path/filepath" - "github.com/zoriya/kyoo/transcoder/src/utils" "golang.org/x/sync/errgroup" ) @@ -18,9 +17,7 @@ type ContentsWriterCallback func(ctx context.Context, writer io.Writer) error // "Items" are pieces of data that can be stored and retrieved. // Paths with ".." are not allowed, and may result in unexpected behavior. type StorageBackend interface { - // DoesItemExist checks if an item exists in the storage backend. DoesItemExist(ctx context.Context, path string) (bool, error) - // ListItemsWithPrefix returns a list of items in the storage backend that match the given prefix. // Note: returned items may have a "/" in them, e.g. "foo/bar/baz". ListItemsWithPrefix(ctx context.Context, pathPrefix string) ([]string, error) // DeleteItem deletes an item from the storage backend. If the item does not exist, it returns nil. @@ -45,37 +42,34 @@ type StorageBackendCloser interface { Close() error } -// SaveFilesToBackend is a helper function that saves files from the source directory to the backend storage. -func SaveFilesToBackend(ctx context.Context, backend StorageBackend, sourceDirectory, destinationBasePath string) error { +func SaveFilesToBackend(ctx context.Context, backend StorageBackend, source, dest string) error { // Allow for save in parallel. Running in series can be slow with many files. var saveGroup errgroup.Group - err := filepath.WalkDir(sourceDirectory, func(sourcePath string, d fs.DirEntry, err error) error { + err := filepath.WalkDir(source, func(path string, d fs.DirEntry, err error) error { if err != nil { - return fmt.Errorf("error walking the path %q: %v", sourcePath, err) + return err } if d.IsDir() { return nil } - // Path relative to the source directory - relativePath, err := filepath.Rel(sourceDirectory, sourcePath) + rel, err := filepath.Rel(source, path) if err != nil { return err } + dest := filepath.Join(dest, rel) - // Destination path in the backend storage - destinationPath := filepath.Join(destinationBasePath, relativePath) saveGroup.Go(func() (err error) { - file, err := os.Open(sourcePath) + file, err := os.Open(path) if err != nil { - return fmt.Errorf("failed to open file %q: %w", sourcePath, err) + return err } - utils.CleanupWithErr(&err, file.Close, "failed to close file %q", sourcePath) + defer file.Close() - if err := backend.SaveItem(ctx, destinationPath, file); err != nil { - return fmt.Errorf("failed to save file %q to backend: %w", sourcePath, err) + if err := backend.SaveItem(ctx, dest, file); err != nil { + return fmt.Errorf("failed to save file %q to backend: %w", path, err) } return nil diff --git a/transcoder/src/subtitles.go b/transcoder/src/subtitles.go index 69fc4b63..d32a8bfd 100644 --- a/transcoder/src/subtitles.go +++ b/transcoder/src/subtitles.go @@ -3,11 +3,13 @@ package src import ( "encoding/base64" "fmt" + "io" "os" "path/filepath" "regexp" "strings" + "github.com/asticode/go-astisub" "github.com/zoriya/kyoo/transcoder/src/utils" "golang.org/x/text/language" ) @@ -31,7 +33,7 @@ outer: for codec, ext := range SubtitleExtensions { if strings.HasSuffix(match, ext) { link := fmt.Sprintf( - "/video/%s/direct/%s", + "/video/%s/subtitle/ext-%s", base64.RawURLEncoding.EncodeToString([]byte(match)), filepath.Base(match), ) @@ -90,3 +92,58 @@ outer: } return nil } + +func ConvertSubtitle( + format string, + stream io.ReadCloser, + outFmt string, + out io.WriteCloser, +) error { + var s *astisub.Subtitles + var err error + + switch format { + case ".srt": + s, err = astisub.ReadFromSRT(stream) + case ".ssa", ".ass": + s, err = astisub.ReadFromSSA(stream) + case ".stl": + s, err = astisub.ReadFromSTL(stream, astisub.STLOptions{}) + case ".ts": + s, err = astisub.ReadFromTeletext(stream, astisub.TeletextOptions{}) + case ".ttml": + s, err = astisub.ReadFromTTML(stream) + case ".vtt": + s, err = astisub.ReadFromWebVTT(stream) + default: + err = astisub.ErrInvalidExtension + } + if err != nil { + return err + } + if len(s.Items) == 0 { + return astisub.ErrNoSubtitlesToWrite + } + + var convert func(io.Writer) error + switch outFmt { + case ".srt": + convert = s.WriteToSRT + case ".ssa", ".ass": + convert = s.WriteToSSA + case ".stl": + convert = s.WriteToSTL + case ".ttml": + convert = func(out io.Writer) error { return s.WriteToTTML(out) } + case ".vtt": + convert = s.WriteToWebVTT + default: + return astisub.ErrInvalidExtension + } + + go func() { + convert(out) + out.Close() + }() + return nil +} diff --git a/transcoder/src/thumbnails.go b/transcoder/src/thumbnails.go index 9e8e17eb..4b8edea8 100644 --- a/transcoder/src/thumbnails.go +++ b/transcoder/src/thumbnails.go @@ -31,11 +31,6 @@ type Thumbnail struct { const ThumbsVersion = 1 -// getThumbGlob returns the path prefix for all thumbnail files for a given sha. -func getThumbPrefix(sha string) string { - return sha -} - func getThumbPath(sha string) string { return fmt.Sprintf("%s/thumbs-v%d.png", sha, ThumbsVersion) } @@ -91,13 +86,9 @@ func (s *MetadataService) extractThumbnail(ctx context.Context, path string, sha vttPath := getThumbVttPath(sha) spritePath := getThumbPath(sha) - newItemPaths := []string{spritePath, vttPath} - doAllExist, err := s.doAllThumbnailFilesExist(ctx, newItemPaths) - if err != nil { - return fmt.Errorf("failed to check if thumbnail files exist: %w", err) - } - if doAllExist { + alreadyOk, _ := s.storage.DoesItemExist(ctx, spritePath) + if alreadyOk { return nil } @@ -106,7 +97,7 @@ func (s *MetadataService) extractThumbnail(ctx context.Context, path string, sha log.Printf("Error reading video file: %v", err) return err } - defer utils.CleanupWithErr(&err, gen.Close, "failed to close screengen generator") + defer gen.Close() gen.Fast = true @@ -156,47 +147,21 @@ func (s *MetadataService) extractThumbnail(ctx context.Context, path string, sha ) } - // Cleanup old thumbnails - if err := s.storage.DeleteItemsWithPrefix(ctx, getThumbPrefix(sha)); err != nil { - return fmt.Errorf("failed to delete old thumbnails: %w", err) - } + _ = s.storage.DeleteItem(ctx, spritePath) + _ = s.storage.DeleteItem(ctx, vttPath) - // Store the new items - // Thumbnail vtt - if err = s.storage.SaveItem(ctx, vttPath, strings.NewReader(vtt)); err != nil { - return fmt.Errorf("failed to save thumbnail vtt with path %q: %w", vttPath, err) - } - - // Thumbnail sprite spriteFormat, err := imaging.FormatFromFilename(spritePath) if err != nil { return err } err = s.storage.SaveItemWithCallback(ctx, spritePath, func(_ context.Context, writer io.Writer) error { - if err := imaging.Encode(writer, sprite, spriteFormat); err != nil { - return fmt.Errorf("failed to encode thumbnail sprite: %w", err) - } - return nil + return imaging.Encode(writer, sprite, spriteFormat) }) if err != nil { - return fmt.Errorf("failed to save thumbnail sprite with path %q: %w", spritePath, err) + return err } - - return nil -} - -func (s *MetadataService) doAllThumbnailFilesExist(ctx context.Context, filePaths []string) (bool, error) { - for _, filePath := range filePaths { - doesExist, err := s.storage.DoesItemExist(ctx, filePath) - if err != nil { - return false, fmt.Errorf("failed to check if thumbnail file %q exists: %w", filePath, err) - } - if !doesExist { - return false, nil - } - } - return true, nil + return s.storage.SaveItem(ctx, vttPath, strings.NewReader(vtt)) } func tsToVttTime(ts int) string {