diff --git a/transcoder/Dockerfile b/transcoder/Dockerfile index 19161b87..11cb9653 100644 --- a/transcoder/Dockerfile +++ b/transcoder/Dockerfile @@ -1,4 +1,4 @@ -# FROM golang:1.21 as build +# FROM golang:1.22 as build FROM debian:trixie-slim as build # those were copied from https://github.com/docker-library/golang/blob/master/Dockerfile-linux.template ENV GOTOOLCHAIN=local @@ -53,6 +53,7 @@ RUN set -x && apt-get update \ WORKDIR /app COPY --from=build /app/transcoder /app/transcoder +COPY ./migrations /app/migrations # flags for nvidia acceleration on docker < 25.0 ENV NVIDIA_VISIBLE_DEVICES="all" diff --git a/transcoder/go.mod b/transcoder/go.mod index 78669920..c55fb851 100644 --- a/transcoder/go.mod +++ b/transcoder/go.mod @@ -4,7 +4,6 @@ go 1.22 require ( github.com/golang-migrate/migrate/v4 v4.17.1 - github.com/jmoiron/sqlx v1.4.0 github.com/labstack/echo/v4 v4.12.0 github.com/lib/pq v1.10.9 gopkg.in/vansante/go-ffprobe.v2 v2.2.0 @@ -29,5 +28,3 @@ require ( golang.org/x/text v0.14.0 golang.org/x/time v0.5.0 // indirect ) - -replace github.com/jmoiron/sqlx v1.4.0 => github.com/kmpm/sqlx v1.3.5-0.20220614102404-845a9a7f1301 diff --git a/transcoder/go.sum b/transcoder/go.sum index 2bb48202..6741575f 100644 --- a/transcoder/go.sum +++ b/transcoder/go.sum @@ -17,8 +17,6 @@ github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKoh github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= -github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= @@ -30,13 +28,10 @@ github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= -github.com/kmpm/sqlx v1.3.5-0.20220614102404-845a9a7f1301 h1:BX43DPpHzm4o6tYs3W+92d0F3iCD5/n4zKjv1FV8Ews= -github.com/kmpm/sqlx v1.3.5-0.20220614102404-845a9a7f1301/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ= github.com/labstack/echo/v4 v4.12.0 h1:IKpw49IMryVB2p1a4dzwlhP1O2Tf2E0Ir/450lH+kI0= github.com/labstack/echo/v4 v4.12.0/go.mod h1:UP9Cr2DJXbOK3Kr9ONYzNowSh7HP0aG0ShAyycHSJvM= github.com/labstack/gommon v0.4.2 h1:F8qTUNXgG1+6WQmqoUWnz8WiEU60mXVVw0P4ht1WRA0= github.com/labstack/gommon v0.4.2/go.mod h1:QlUFxVM+SNXhDL/Z7YhocGIBYOiwB0mXm1+1bAPHPyU= -github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= @@ -44,9 +39,6 @@ github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovk github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= -github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwpU1Y= -github.com/mattn/go-sqlite3 v1.14.16/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= diff --git a/transcoder/src/extract.go b/transcoder/src/extract.go index 864fb439..2a0db02e 100644 --- a/transcoder/src/extract.go +++ b/transcoder/src/extract.go @@ -21,13 +21,7 @@ func (s *MetadataService) ExtractSubs(info *MediaInfo) (interface{}, error) { if err != nil { return set(nil, err) } - _, err = s.database.NamedExec( - `update info set ver_extract = :version where sha = :sha`, - map[string]interface{}{ - "sha": info.Sha, - "version": ExtractVersion, - }, - ) + _, err = s.database.Exec(`update info set ver_extract = $2 where sha = $1`, info.Sha, ExtractVersion) return set(nil, err) } diff --git a/transcoder/src/info.go b/transcoder/src/info.go index 87e4d453..d15ef4b6 100644 --- a/transcoder/src/info.go +++ b/transcoder/src/info.go @@ -18,10 +18,10 @@ import ( const InfoVersion = 1 type Versions struct { - Info int32 `db:"ver_info"` - Extract int32 `db:"ver_extract"` - Thumbs int32 `db:"ver_thumbs"` - Keyframes int32 `db:"ver_keyframes"` + Info int32 + Extract int32 + Thumbs int32 + Keyframes int32 } type MediaInfo struct { @@ -32,7 +32,7 @@ type MediaInfo struct { /// The extension currently used to store this video file Extension string `json:"extension"` /// The whole mimetype (defined as the RFC 6381). ex: `video/mp4; codecs="avc1.640028, mp4a.40.2"` - MimeCodec *string `json:"mimeCodec" db:"mime_codec"` + MimeCodec *string `json:"mimeCodec"` /// The file size of the video file. Size int64 `json:"size"` /// The length of the media in seconds. @@ -59,7 +59,7 @@ type MediaInfo struct { type Video struct { /// The index of this track on the media. - Index uint32 `json:"index" db:"idx"` + Index uint32 `json:"index"` /// The title of the stream. Title *string `json:"title"` /// The language of this stream (as a ISO-639-2 language code) @@ -67,7 +67,7 @@ type Video struct { /// The human readable codec name. Codec string `json:"codec"` /// The codec of this stream (defined as the RFC 6381). - MimeCodec *string `json:"mimeCodec" db:"mime_codec"` + MimeCodec *string `json:"mimeCodec"` /// The max quality of this video track. Quality Quality `json:"quality"` /// The width of the video stream @@ -83,7 +83,7 @@ type Video struct { type Audio struct { /// The index of this track on the media. - Index uint32 `json:"index" db:"idx"` + Index uint32 `json:"index"` /// The title of the stream. Title *string `json:"title"` /// The language of this stream (as a IETF-BCP-47 language code) @@ -91,9 +91,9 @@ type Audio struct { /// The human readable codec name. Codec string `json:"codec"` /// The codec of this stream (defined as the RFC 6381). - MimeCodec *string `json:"mimeCodec" db:"mime_codec"` + MimeCodec *string `json:"mimeCodec"` /// Is this stream the default one of it's type? - IsDefault bool `json:"isDefault" db:"is_default"` + IsDefault bool `json:"isDefault"` /// Keyframes of this video Keyframes *Keyframe `json:"-"` @@ -101,7 +101,7 @@ type Audio struct { type Subtitle struct { /// The index of this track on the media. - Index *uint32 `json:"index" db:"idx"` + Index *uint32 `json:"index"` /// The title of the stream. Title *string `json:"title"` /// The language of this stream (as a IETF-BCP-47 language code) @@ -111,11 +111,11 @@ type Subtitle struct { /// The extension for the codec. Extension *string `json:"extension"` /// Is this stream the default one of it's type? - IsDefault bool `json:"isDefault" db:"is_default"` + IsDefault bool `json:"isDefault"` /// Is this stream tagged as forced? - IsForced bool `json:"isForced" db:"is_forced"` + IsForced bool `json:"isForced"` /// Is this an external subtitle (as in stored in a different file) - IsExternal bool `json:"isExternal" db:"is_external"` + IsExternal bool `json:"isExternal"` /// Where the subtitle is stored (either in library if IsExternal is true or in transcoder cache if false) /// Null if the subtitle can't be extracted (unsupported format) Path *string `json:"path"` @@ -125,9 +125,9 @@ type Subtitle struct { type Chapter struct { /// The start time of the chapter (in second from the start of the episode). - StartTime float32 `json:"startTime" db:"start_time"` + StartTime float32 `json:"startTime"` /// The end time of the chapter (in second from the start of the episode). - EndTime float32 `json:"endTime" db:"end_time"` + EndTime float32 `json:"endTime"` /// The name of this chapter. This should be a human-readable name that could be presented to the user. Name string `json:"name"` /// The type value is used to mark special chapters (openning/credits...) diff --git a/transcoder/src/keyframes.go b/transcoder/src/keyframes.go index 64db7222..6a442c5c 100644 --- a/transcoder/src/keyframes.go +++ b/transcoder/src/keyframes.go @@ -118,17 +118,15 @@ func (s *MetadataService) GetKeyframes(info *MediaInfo, isVideo bool, idx int32) return } - _, err = s.database.NamedExec( + _, err = s.database.Exec( fmt.Sprint( - `update %s set keyframes = :keyframes, ver_keyframes = :version where sha = :sha and idx = :idx`, + `update %s set keyframes = $3, ver_keyframes = $4 where sha = $1 and idx = $2`, table, ), - map[string]interface{}{ - "sha": info.Sha, - "idx": idx, - "keyframes": kf.Keyframes, - "version": KeyframeVersion, - }, + info.Sha, + idx, + kf.Keyframes, + KeyframeVersion, ) if err != nil { log.Printf("Couldn't store keyframes on database: %v", err) diff --git a/transcoder/src/metadata.go b/transcoder/src/metadata.go index 23ebf305..54c3d58a 100644 --- a/transcoder/src/metadata.go +++ b/transcoder/src/metadata.go @@ -1,6 +1,8 @@ package src import ( + "database/sql" + "encoding/base64" "fmt" "net/url" "os" @@ -8,12 +10,11 @@ import ( "github.com/golang-migrate/migrate/v4" "github.com/golang-migrate/migrate/v4/database/postgres" _ "github.com/golang-migrate/migrate/v4/source/file" - "github.com/jmoiron/sqlx" _ "github.com/lib/pq" ) type MetadataService struct { - database *sqlx.DB + database *sql.DB lock RunLock[string, *MediaInfo] thumbLock RunLock[string, interface{}] extractLock RunLock[string, interface{}] @@ -29,25 +30,25 @@ func NewMetadataService() (*MetadataService, error) { url.QueryEscape(os.Getenv("POSTGRES_PORT")), url.QueryEscape(os.Getenv("POSTGRES_DB")), ) - db, err := sqlx.Open("postgres", con) + db, err := sql.Open("postgres", con) if err != nil { return nil, err } - db.MustExec("create schema if not exists gocoder") + _, err = db.Exec("create schema if not exists gocoder") + if err != nil { + return nil, err + } - driver, err := postgres.WithInstance(db.DB, &postgres.Config{}) + driver, err := postgres.WithInstance(db, &postgres.Config{}) if err != nil { return nil, err } - m, err := migrate.NewWithDatabaseInstance("file://./migrations", "postgres", driver) - if err != nil { - return nil, err - } - err = m.Up() + m, err := migrate.NewWithDatabaseInstance("file://migrations", "postgres", driver) if err != nil { return nil, err } + m.Up() return &MetadataService{ database: db, @@ -77,15 +78,17 @@ func (s *MetadataService) GetMetadata(path string, sha string) (*MediaInfo, erro for _, audio := range ret.Audios { audio.Keyframes = nil } - s.database.NamedExec(` - update videos set keyframes = nil where sha = :sha; - update audios set keyframes = nil where sha = :sha; - update info set ver_keyframes = 0 where sha = :sha; - `, - map[string]interface{}{ - "sha": sha, - }, - ) + tx, err := s.database.Begin() + if err != nil { + return nil, err + } + tx.Exec(`update videos set keyframes = nil where sha = $1`, sha) + tx.Exec(`update audios set keyframes = nil where sha = $1`, sha) + tx.Exec(`update info set ver_keyframes = 0 where sha = $1`, sha) + err = tx.Commit() + if err != nil { + fmt.Printf("error deleteing old keyframes from database: %v", err) + } } return ret, nil @@ -93,58 +96,100 @@ func (s *MetadataService) GetMetadata(path string, sha string) (*MediaInfo, erro func (s *MetadataService) getMetadata(path string, sha string) (*MediaInfo, error) { var ret MediaInfo - rows, err := s.database.Queryx(` - select * from info as i where i.sha=$1; - select * from videos as v where v.sha=$1; - select * from audios as a where a.sha=$1; - select * from subtitles as s where s.sha=$1; - select * from chapters as c where c.sha=$1; - `, + err := s.database.QueryRow( + `select i.sha, i.path, i.extension, i.mime_codec, i.size, i.duration, i.container, + i.fonts, i.ver_info, i.ver_extract, i.ver_thumbs, i.ver_keyframes + from info as i where i.sha=$1`, + sha, + ).Scan( + ret.Sha, ret.Path, ret.Extension, ret.MimeCodec, ret.Size, ret.Duration, ret.Container, + ret.Fonts, ret.Versions.Info, ret.Versions.Extract, ret.Versions.Thumbs, ret.Versions.Keyframes, + ) + + if err == sql.ErrNoRows || (ret.Versions.Info < InfoVersion && ret.Versions.Info != 0) { + return s.storeFreshMetadata(path, sha) + } + if err != nil { + return nil, err + } + + rows, err := s.database.Query( + `select v.idx, v.title, v.language, v.codec, v.mime_codec, v.width, v.height, v.bitrate, v.keyframes + from videos as v where v.sha=$1`, sha, ) if err != nil { return nil, err } - defer rows.Close() - - if !rows.Next() { - if err = rows.Err(); err != nil { + for rows.Next() { + var v Video + err := rows.Scan(v.Index, v.Title, v.Language, v.Codec, v.MimeCodec, v.Width, v.Height, v.Bitrate, v.Keyframes) + if err != nil { return nil, err } - return s.storeFreshMetadata(path, sha) - } - rows.StructScan(ret) - - if ret.Versions.Info != InfoVersion { - return s.storeFreshMetadata(path, sha) + v.Quality = QualityFromHeight(v.Height) + ret.Videos = append(ret.Videos, v) } - rows.NextResultSet() + rows, err = s.database.Query( + `select a.idx, a.title, a.language, a.codec, a.mime_codec, a.is_default, a.keyframes + from audios as a where a.sha=$1`, + sha, + ) + if err != nil { + return nil, err + } for rows.Next() { - var video Video - rows.StructScan(video) - ret.Videos = append(ret.Videos, video) + var a Audio + err := rows.Scan(a.Index, a.Title, a.Language, a.Codec, a.MimeCodec, a.IsDefault, a.Keyframes) + if err != nil { + return nil, err + } + ret.Audios = append(ret.Audios, a) } - rows.NextResultSet() + rows, err = s.database.Query( + `select s.idx, s.title, s.language, s.codec, s.extension, s.is_default, s.is_forced, s.is_external, s.path + from subtitles as s where s.sha=$1`, + sha, + ) + if err != nil { + return nil, err + } for rows.Next() { - var audio Audio - rows.StructScan(audio) - ret.Audios = append(ret.Audios, audio) + var s Subtitle + err := rows.Scan(s.Index, s.Title, s.Language, s.Codec, s.Extension, s.IsDefault, s.IsForced, s.IsExternal, s.Path) + if err != nil { + return nil, err + } + if s.Extension != nil { + link := fmt.Sprintf( + "%s/%s/subtitle/%d.%s", + Settings.RoutePrefix, + base64.StdEncoding.EncodeToString([]byte(ret.Path)), + s.Index, + *s.Extension, + ) + s.Link = &link + } + ret.Subtitles = append(ret.Subtitles, s) } - rows.NextResultSet() - for rows.Next() { - var subtitle Subtitle - rows.StructScan(subtitle) - ret.Subtitles = append(ret.Subtitles, subtitle) + rows, err = s.database.Query( + `select c.start_time, c.end_time, c.name, c.type + from chapters as c where c.sha=$1`, + sha, + ) + if err != nil { + return nil, err } - - rows.NextResultSet() for rows.Next() { - var chapter Chapter - rows.StructScan(chapter) - ret.Chapters = append(ret.Chapters, chapter) + var c Chapter + err := rows.Scan(c.StartTime, c.EndTime, c.Name, c.Type) + if err != nil { + return nil, err + } + ret.Chapters = append(ret.Chapters, c) } return &ret, nil @@ -161,38 +206,38 @@ func (s *MetadataService) storeFreshMetadata(path string, sha string) (*MediaInf return set(nil, err) } - tx := s.database.MustBegin() - tx.NamedExec( + tx, err := s.database.Begin() + tx.Exec( `insert into info(sha, path, extension, mime_codec, size, duration, container, fonts, ver_info) - values (:sha, :path, :extension, :mime_codec, :size, :duration, :container, :fonts, :ver_info)`, - ret, + values ($1, $2, $3, $4, $5, $6, $7, $8, $9)`, + ret.Sha, ret.Path, ret.Extension, ret.MimeCodec, ret.Size, ret.Duration, ret.Container, ret.Fonts, ret.Versions.Info, ) - for _, video := range ret.Videos { - tx.NamedExec( + for _, v := range ret.Videos { + tx.Exec( `insert into videos(sha, idx, title, language, codec, mime_codec, width, height, bitrate) - values (:sha, :idx, :title, :language, :codec, :mime_codec, :width, :height, :bitrate)`, - video, + values ($1, $2, $3, $4, $5, $6, $7, $8, $9)`, + ret.Sha, v.Index, v.Title, v.Language, v.Codec, v.MimeCodec, v.Width, v.Height, v.Bitrate, ) } - for _, audio := range ret.Audios { - tx.NamedExec( + for _, a := range ret.Audios { + tx.Exec( `insert into audios(sha, idx, title, language, codec, mime_codec, is_default) - values (:sha, :idx, :title, :language, :codec, :mime_codec, :is_default)`, - audio, + values ($1, $2, $3, $4, $5, $6, $7)`, + ret.Sha, a.Index, a.Title, a.Language, a.Codec, a.MimeCodec, a.IsDefault, ) } - for _, subtitle := range ret.Subtitles { - tx.NamedExec( + for _, s := range ret.Subtitles { + tx.Exec( `insert into subtitles(sha, idx, title, language, codec, extension, is_default, is_forced, is_external, path) - values (:sha, :idx, :title, :language, :codec, :extension, :is_default, :is_forced, :is_external, :path)`, - subtitle, + values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)`, + ret.Sha, s.Index, s.Title, s.Language, s.Codec, s.Extension, s.IsDefault, s.IsForced, s.IsExternal, s.Path, ) } - for _, chapter := range ret.Chapters { - tx.NamedExec( + for _, c := range ret.Chapters { + tx.Exec( `insert into chapters(sha, start_time, end_time, name, type) - values (:sha, :start_time, :end_time, :name, :type)`, - chapter, + values ($1, $2, $3, $4, $5)`, + ret.Sha, c.StartTime, c.EndTime, c.Name, c.Type, ) } err = tx.Commit() diff --git a/transcoder/src/thumbnails.go b/transcoder/src/thumbnails.go index 5a3da6bc..b20367e8 100644 --- a/transcoder/src/thumbnails.go +++ b/transcoder/src/thumbnails.go @@ -68,13 +68,7 @@ func (s *MetadataService) ExtractThumbs(path string, sha string) (interface{}, e if err != nil { return set(nil, err) } - _, err = s.database.NamedExec( - `update info set ver_thumbs = :version where sha = :sha`, - map[string]interface{}{ - "sha": sha, - "version": ThumbsVersion, - }, - ) + _, err = s.database.Exec(`update info set ver_thumbs = $2 where sha = $1`, sha, ThumbsVersion) return set(nil, err) }