Remove sqlx

This commit is contained in:
Zoe Roux 2024-08-05 01:15:29 +02:00
parent cc3b6f268a
commit a112ecb8e9
8 changed files with 145 additions and 124 deletions

View File

@ -1,4 +1,4 @@
# FROM golang:1.21 as build
# FROM golang:1.22 as build
FROM debian:trixie-slim as build
# those were copied from https://github.com/docker-library/golang/blob/master/Dockerfile-linux.template
ENV GOTOOLCHAIN=local
@ -53,6 +53,7 @@ RUN set -x && apt-get update \
WORKDIR /app
COPY --from=build /app/transcoder /app/transcoder
COPY ./migrations /app/migrations
# flags for nvidia acceleration on docker < 25.0
ENV NVIDIA_VISIBLE_DEVICES="all"

View File

@ -4,7 +4,6 @@ go 1.22
require (
github.com/golang-migrate/migrate/v4 v4.17.1
github.com/jmoiron/sqlx v1.4.0
github.com/labstack/echo/v4 v4.12.0
github.com/lib/pq v1.10.9
gopkg.in/vansante/go-ffprobe.v2 v2.2.0
@ -29,5 +28,3 @@ require (
golang.org/x/text v0.14.0
golang.org/x/time v0.5.0 // indirect
)
replace github.com/jmoiron/sqlx v1.4.0 => github.com/kmpm/sqlx v1.3.5-0.20220614102404-845a9a7f1301

View File

@ -17,8 +17,6 @@ github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKoh
github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE=
github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY=
@ -30,13 +28,10 @@ github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY
github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
github.com/kmpm/sqlx v1.3.5-0.20220614102404-845a9a7f1301 h1:BX43DPpHzm4o6tYs3W+92d0F3iCD5/n4zKjv1FV8Ews=
github.com/kmpm/sqlx v1.3.5-0.20220614102404-845a9a7f1301/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ=
github.com/labstack/echo/v4 v4.12.0 h1:IKpw49IMryVB2p1a4dzwlhP1O2Tf2E0Ir/450lH+kI0=
github.com/labstack/echo/v4 v4.12.0/go.mod h1:UP9Cr2DJXbOK3Kr9ONYzNowSh7HP0aG0ShAyycHSJvM=
github.com/labstack/gommon v0.4.2 h1:F8qTUNXgG1+6WQmqoUWnz8WiEU60mXVVw0P4ht1WRA0=
github.com/labstack/gommon v0.4.2/go.mod h1:QlUFxVM+SNXhDL/Z7YhocGIBYOiwB0mXm1+1bAPHPyU=
github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
@ -44,9 +39,6 @@ github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovk
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwpU1Y=
github.com/mattn/go-sqlite3 v1.14.16/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0=
github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y=
github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=

View File

@ -21,13 +21,7 @@ func (s *MetadataService) ExtractSubs(info *MediaInfo) (interface{}, error) {
if err != nil {
return set(nil, err)
}
_, err = s.database.NamedExec(
`update info set ver_extract = :version where sha = :sha`,
map[string]interface{}{
"sha": info.Sha,
"version": ExtractVersion,
},
)
_, err = s.database.Exec(`update info set ver_extract = $2 where sha = $1`, info.Sha, ExtractVersion)
return set(nil, err)
}

View File

@ -18,10 +18,10 @@ import (
const InfoVersion = 1
type Versions struct {
Info int32 `db:"ver_info"`
Extract int32 `db:"ver_extract"`
Thumbs int32 `db:"ver_thumbs"`
Keyframes int32 `db:"ver_keyframes"`
Info int32
Extract int32
Thumbs int32
Keyframes int32
}
type MediaInfo struct {
@ -32,7 +32,7 @@ type MediaInfo struct {
/// The extension currently used to store this video file
Extension string `json:"extension"`
/// The whole mimetype (defined as the RFC 6381). ex: `video/mp4; codecs="avc1.640028, mp4a.40.2"`
MimeCodec *string `json:"mimeCodec" db:"mime_codec"`
MimeCodec *string `json:"mimeCodec"`
/// The file size of the video file.
Size int64 `json:"size"`
/// The length of the media in seconds.
@ -59,7 +59,7 @@ type MediaInfo struct {
type Video struct {
/// The index of this track on the media.
Index uint32 `json:"index" db:"idx"`
Index uint32 `json:"index"`
/// The title of the stream.
Title *string `json:"title"`
/// The language of this stream (as a ISO-639-2 language code)
@ -67,7 +67,7 @@ type Video struct {
/// The human readable codec name.
Codec string `json:"codec"`
/// The codec of this stream (defined as the RFC 6381).
MimeCodec *string `json:"mimeCodec" db:"mime_codec"`
MimeCodec *string `json:"mimeCodec"`
/// The max quality of this video track.
Quality Quality `json:"quality"`
/// The width of the video stream
@ -83,7 +83,7 @@ type Video struct {
type Audio struct {
/// The index of this track on the media.
Index uint32 `json:"index" db:"idx"`
Index uint32 `json:"index"`
/// The title of the stream.
Title *string `json:"title"`
/// The language of this stream (as a IETF-BCP-47 language code)
@ -91,9 +91,9 @@ type Audio struct {
/// The human readable codec name.
Codec string `json:"codec"`
/// The codec of this stream (defined as the RFC 6381).
MimeCodec *string `json:"mimeCodec" db:"mime_codec"`
MimeCodec *string `json:"mimeCodec"`
/// Is this stream the default one of it's type?
IsDefault bool `json:"isDefault" db:"is_default"`
IsDefault bool `json:"isDefault"`
/// Keyframes of this video
Keyframes *Keyframe `json:"-"`
@ -101,7 +101,7 @@ type Audio struct {
type Subtitle struct {
/// The index of this track on the media.
Index *uint32 `json:"index" db:"idx"`
Index *uint32 `json:"index"`
/// The title of the stream.
Title *string `json:"title"`
/// The language of this stream (as a IETF-BCP-47 language code)
@ -111,11 +111,11 @@ type Subtitle struct {
/// The extension for the codec.
Extension *string `json:"extension"`
/// Is this stream the default one of it's type?
IsDefault bool `json:"isDefault" db:"is_default"`
IsDefault bool `json:"isDefault"`
/// Is this stream tagged as forced?
IsForced bool `json:"isForced" db:"is_forced"`
IsForced bool `json:"isForced"`
/// Is this an external subtitle (as in stored in a different file)
IsExternal bool `json:"isExternal" db:"is_external"`
IsExternal bool `json:"isExternal"`
/// Where the subtitle is stored (either in library if IsExternal is true or in transcoder cache if false)
/// Null if the subtitle can't be extracted (unsupported format)
Path *string `json:"path"`
@ -125,9 +125,9 @@ type Subtitle struct {
type Chapter struct {
/// The start time of the chapter (in second from the start of the episode).
StartTime float32 `json:"startTime" db:"start_time"`
StartTime float32 `json:"startTime"`
/// The end time of the chapter (in second from the start of the episode).
EndTime float32 `json:"endTime" db:"end_time"`
EndTime float32 `json:"endTime"`
/// The name of this chapter. This should be a human-readable name that could be presented to the user.
Name string `json:"name"`
/// The type value is used to mark special chapters (openning/credits...)

View File

@ -118,17 +118,15 @@ func (s *MetadataService) GetKeyframes(info *MediaInfo, isVideo bool, idx int32)
return
}
_, err = s.database.NamedExec(
_, err = s.database.Exec(
fmt.Sprint(
`update %s set keyframes = :keyframes, ver_keyframes = :version where sha = :sha and idx = :idx`,
`update %s set keyframes = $3, ver_keyframes = $4 where sha = $1 and idx = $2`,
table,
),
map[string]interface{}{
"sha": info.Sha,
"idx": idx,
"keyframes": kf.Keyframes,
"version": KeyframeVersion,
},
info.Sha,
idx,
kf.Keyframes,
KeyframeVersion,
)
if err != nil {
log.Printf("Couldn't store keyframes on database: %v", err)

View File

@ -1,6 +1,8 @@
package src
import (
"database/sql"
"encoding/base64"
"fmt"
"net/url"
"os"
@ -8,12 +10,11 @@ import (
"github.com/golang-migrate/migrate/v4"
"github.com/golang-migrate/migrate/v4/database/postgres"
_ "github.com/golang-migrate/migrate/v4/source/file"
"github.com/jmoiron/sqlx"
_ "github.com/lib/pq"
)
type MetadataService struct {
database *sqlx.DB
database *sql.DB
lock RunLock[string, *MediaInfo]
thumbLock RunLock[string, interface{}]
extractLock RunLock[string, interface{}]
@ -29,25 +30,25 @@ func NewMetadataService() (*MetadataService, error) {
url.QueryEscape(os.Getenv("POSTGRES_PORT")),
url.QueryEscape(os.Getenv("POSTGRES_DB")),
)
db, err := sqlx.Open("postgres", con)
db, err := sql.Open("postgres", con)
if err != nil {
return nil, err
}
db.MustExec("create schema if not exists gocoder")
_, err = db.Exec("create schema if not exists gocoder")
if err != nil {
return nil, err
}
driver, err := postgres.WithInstance(db.DB, &postgres.Config{})
driver, err := postgres.WithInstance(db, &postgres.Config{})
if err != nil {
return nil, err
}
m, err := migrate.NewWithDatabaseInstance("file://./migrations", "postgres", driver)
if err != nil {
return nil, err
}
err = m.Up()
m, err := migrate.NewWithDatabaseInstance("file://migrations", "postgres", driver)
if err != nil {
return nil, err
}
m.Up()
return &MetadataService{
database: db,
@ -77,15 +78,17 @@ func (s *MetadataService) GetMetadata(path string, sha string) (*MediaInfo, erro
for _, audio := range ret.Audios {
audio.Keyframes = nil
}
s.database.NamedExec(`
update videos set keyframes = nil where sha = :sha;
update audios set keyframes = nil where sha = :sha;
update info set ver_keyframes = 0 where sha = :sha;
`,
map[string]interface{}{
"sha": sha,
},
)
tx, err := s.database.Begin()
if err != nil {
return nil, err
}
tx.Exec(`update videos set keyframes = nil where sha = $1`, sha)
tx.Exec(`update audios set keyframes = nil where sha = $1`, sha)
tx.Exec(`update info set ver_keyframes = 0 where sha = $1`, sha)
err = tx.Commit()
if err != nil {
fmt.Printf("error deleteing old keyframes from database: %v", err)
}
}
return ret, nil
@ -93,58 +96,100 @@ func (s *MetadataService) GetMetadata(path string, sha string) (*MediaInfo, erro
func (s *MetadataService) getMetadata(path string, sha string) (*MediaInfo, error) {
var ret MediaInfo
rows, err := s.database.Queryx(`
select * from info as i where i.sha=$1;
select * from videos as v where v.sha=$1;
select * from audios as a where a.sha=$1;
select * from subtitles as s where s.sha=$1;
select * from chapters as c where c.sha=$1;
`,
err := s.database.QueryRow(
`select i.sha, i.path, i.extension, i.mime_codec, i.size, i.duration, i.container,
i.fonts, i.ver_info, i.ver_extract, i.ver_thumbs, i.ver_keyframes
from info as i where i.sha=$1`,
sha,
).Scan(
ret.Sha, ret.Path, ret.Extension, ret.MimeCodec, ret.Size, ret.Duration, ret.Container,
ret.Fonts, ret.Versions.Info, ret.Versions.Extract, ret.Versions.Thumbs, ret.Versions.Keyframes,
)
if err == sql.ErrNoRows || (ret.Versions.Info < InfoVersion && ret.Versions.Info != 0) {
return s.storeFreshMetadata(path, sha)
}
if err != nil {
return nil, err
}
rows, err := s.database.Query(
`select v.idx, v.title, v.language, v.codec, v.mime_codec, v.width, v.height, v.bitrate, v.keyframes
from videos as v where v.sha=$1`,
sha,
)
if err != nil {
return nil, err
}
defer rows.Close()
if !rows.Next() {
if err = rows.Err(); err != nil {
for rows.Next() {
var v Video
err := rows.Scan(v.Index, v.Title, v.Language, v.Codec, v.MimeCodec, v.Width, v.Height, v.Bitrate, v.Keyframes)
if err != nil {
return nil, err
}
return s.storeFreshMetadata(path, sha)
}
rows.StructScan(ret)
if ret.Versions.Info != InfoVersion {
return s.storeFreshMetadata(path, sha)
v.Quality = QualityFromHeight(v.Height)
ret.Videos = append(ret.Videos, v)
}
rows.NextResultSet()
rows, err = s.database.Query(
`select a.idx, a.title, a.language, a.codec, a.mime_codec, a.is_default, a.keyframes
from audios as a where a.sha=$1`,
sha,
)
if err != nil {
return nil, err
}
for rows.Next() {
var video Video
rows.StructScan(video)
ret.Videos = append(ret.Videos, video)
var a Audio
err := rows.Scan(a.Index, a.Title, a.Language, a.Codec, a.MimeCodec, a.IsDefault, a.Keyframes)
if err != nil {
return nil, err
}
ret.Audios = append(ret.Audios, a)
}
rows.NextResultSet()
rows, err = s.database.Query(
`select s.idx, s.title, s.language, s.codec, s.extension, s.is_default, s.is_forced, s.is_external, s.path
from subtitles as s where s.sha=$1`,
sha,
)
if err != nil {
return nil, err
}
for rows.Next() {
var audio Audio
rows.StructScan(audio)
ret.Audios = append(ret.Audios, audio)
var s Subtitle
err := rows.Scan(s.Index, s.Title, s.Language, s.Codec, s.Extension, s.IsDefault, s.IsForced, s.IsExternal, s.Path)
if err != nil {
return nil, err
}
if s.Extension != nil {
link := fmt.Sprintf(
"%s/%s/subtitle/%d.%s",
Settings.RoutePrefix,
base64.StdEncoding.EncodeToString([]byte(ret.Path)),
s.Index,
*s.Extension,
)
s.Link = &link
}
ret.Subtitles = append(ret.Subtitles, s)
}
rows.NextResultSet()
for rows.Next() {
var subtitle Subtitle
rows.StructScan(subtitle)
ret.Subtitles = append(ret.Subtitles, subtitle)
rows, err = s.database.Query(
`select c.start_time, c.end_time, c.name, c.type
from chapters as c where c.sha=$1`,
sha,
)
if err != nil {
return nil, err
}
rows.NextResultSet()
for rows.Next() {
var chapter Chapter
rows.StructScan(chapter)
ret.Chapters = append(ret.Chapters, chapter)
var c Chapter
err := rows.Scan(c.StartTime, c.EndTime, c.Name, c.Type)
if err != nil {
return nil, err
}
ret.Chapters = append(ret.Chapters, c)
}
return &ret, nil
@ -161,38 +206,38 @@ func (s *MetadataService) storeFreshMetadata(path string, sha string) (*MediaInf
return set(nil, err)
}
tx := s.database.MustBegin()
tx.NamedExec(
tx, err := s.database.Begin()
tx.Exec(
`insert into info(sha, path, extension, mime_codec, size, duration, container, fonts, ver_info)
values (:sha, :path, :extension, :mime_codec, :size, :duration, :container, :fonts, :ver_info)`,
ret,
values ($1, $2, $3, $4, $5, $6, $7, $8, $9)`,
ret.Sha, ret.Path, ret.Extension, ret.MimeCodec, ret.Size, ret.Duration, ret.Container, ret.Fonts, ret.Versions.Info,
)
for _, video := range ret.Videos {
tx.NamedExec(
for _, v := range ret.Videos {
tx.Exec(
`insert into videos(sha, idx, title, language, codec, mime_codec, width, height, bitrate)
values (:sha, :idx, :title, :language, :codec, :mime_codec, :width, :height, :bitrate)`,
video,
values ($1, $2, $3, $4, $5, $6, $7, $8, $9)`,
ret.Sha, v.Index, v.Title, v.Language, v.Codec, v.MimeCodec, v.Width, v.Height, v.Bitrate,
)
}
for _, audio := range ret.Audios {
tx.NamedExec(
for _, a := range ret.Audios {
tx.Exec(
`insert into audios(sha, idx, title, language, codec, mime_codec, is_default)
values (:sha, :idx, :title, :language, :codec, :mime_codec, :is_default)`,
audio,
values ($1, $2, $3, $4, $5, $6, $7)`,
ret.Sha, a.Index, a.Title, a.Language, a.Codec, a.MimeCodec, a.IsDefault,
)
}
for _, subtitle := range ret.Subtitles {
tx.NamedExec(
for _, s := range ret.Subtitles {
tx.Exec(
`insert into subtitles(sha, idx, title, language, codec, extension, is_default, is_forced, is_external, path)
values (:sha, :idx, :title, :language, :codec, :extension, :is_default, :is_forced, :is_external, :path)`,
subtitle,
values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)`,
ret.Sha, s.Index, s.Title, s.Language, s.Codec, s.Extension, s.IsDefault, s.IsForced, s.IsExternal, s.Path,
)
}
for _, chapter := range ret.Chapters {
tx.NamedExec(
for _, c := range ret.Chapters {
tx.Exec(
`insert into chapters(sha, start_time, end_time, name, type)
values (:sha, :start_time, :end_time, :name, :type)`,
chapter,
values ($1, $2, $3, $4, $5)`,
ret.Sha, c.StartTime, c.EndTime, c.Name, c.Type,
)
}
err = tx.Commit()

View File

@ -68,13 +68,7 @@ func (s *MetadataService) ExtractThumbs(path string, sha string) (interface{}, e
if err != nil {
return set(nil, err)
}
_, err = s.database.NamedExec(
`update info set ver_thumbs = :version where sha = :sha`,
map[string]interface{}{
"sha": sha,
"version": ThumbsVersion,
},
)
_, err = s.database.Exec(`update info set ver_thumbs = $2 where sha = $1`, sha, ThumbsVersion)
return set(nil, err)
}