From b147ee8850711e51e07ecdf99d3246e00fb7c3e5 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Sun, 21 Jan 2024 23:20:40 +0100 Subject: [PATCH] Create a thumbnails sprite api --- transcoder/go.mod | 4 ++ transcoder/go.sum | 7 +++ transcoder/main.go | 47 ++++++++++++++++++ transcoder/src/extract.go | 4 +- transcoder/src/thumbnails.go | 92 ++++++++++++++++++++++++++++++++++++ 5 files changed, 152 insertions(+), 2 deletions(-) create mode 100644 transcoder/src/thumbnails.go diff --git a/transcoder/go.mod b/transcoder/go.mod index db2cc6e1..8a7cf42a 100644 --- a/transcoder/go.mod +++ b/transcoder/go.mod @@ -5,6 +5,7 @@ go 1.21 require github.com/labstack/echo/v4 v4.11.4 // direct require ( + github.com/disintegration/imaging v1.6.2 github.com/golang-jwt/jwt v3.2.2+incompatible // indirect github.com/labstack/gommon v0.4.2 // indirect github.com/mattn/go-colorable v0.1.13 // indirect @@ -12,9 +13,12 @@ require ( github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/valyala/fasttemplate v1.2.2 // indirect github.com/zoriya/go-mediainfo v0.0.0-20240113000440-36f500affcfd // indirect + gitlab.com/opennota/screengen v1.0.2 golang.org/x/crypto v0.17.0 // indirect golang.org/x/net v0.19.0 // indirect golang.org/x/sys v0.15.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.5.0 // indirect ) + +require golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8 // indirect diff --git a/transcoder/go.sum b/transcoder/go.sum index 54740a91..965b1c07 100644 --- a/transcoder/go.sum +++ b/transcoder/go.sum @@ -1,3 +1,5 @@ +github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1ei82L+c= +github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4= github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/labstack/echo/v4 v4.11.4 h1:vDZmA+qNeh1pd/cCkEicDMrjtrnMGQ1QFI9gWN1zGq8= @@ -17,14 +19,19 @@ github.com/zoriya/go-mediainfo v0.0.0-20240112235842-ce0c807be738 h1:FV9TIvf/T84 github.com/zoriya/go-mediainfo v0.0.0-20240112235842-ce0c807be738/go.mod h1:jzun1oQGoJSh65g1XKaolTmjd6HW/34WHH7VMdJdbvM= github.com/zoriya/go-mediainfo v0.0.0-20240113000440-36f500affcfd h1:AOdEpcmYJkmIW4I76TQim6LT4+9duYTdXNgkQsPHpuA= github.com/zoriya/go-mediainfo v0.0.0-20240113000440-36f500affcfd/go.mod h1:jzun1oQGoJSh65g1XKaolTmjd6HW/34WHH7VMdJdbvM= +gitlab.com/opennota/screengen v1.0.2 h1:GxYTJdAPEzmg5v5CV4dgn45JVW+EcXXAvCxhE7w6UDw= +gitlab.com/opennota/screengen v1.0.2/go.mod h1:4kED4yriw2zslwYmXFCa5qCvEKwleBA7l5OE+d94NTU= golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= +golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8 h1:hVwzHzIUGRjiF7EcUjqNxk3NCfkPxbDKRdnNE1Rpg0U= +golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= diff --git a/transcoder/main.go b/transcoder/main.go index 47d7cd84..8346da96 100644 --- a/transcoder/main.go +++ b/transcoder/main.go @@ -262,6 +262,51 @@ func (h *Handler) GetSubtitle(c echo.Context) error { return c.File(path) } +// Get thumbnail sprite +// +// Get a sprite file containing all the thumbnails of the show +// +// Path: /:resource/:slug/thumbnails.png +func (h *Handler) GetThumbnails(c echo.Context) error { + resource := c.Param("resource") + slug := c.Param("slug") + + path, err := GetPath(resource, slug) + if err != nil { + return err + } + + out, err := src.ExtractThumbnail(path) + if err != nil { + return err + } + + return c.File(fmt.Sprintf("%s/sprite.png", out)) +} + +// Get thumbnail vtt +// +// Get a vtt file containing timing/position of thumbnails inside the sprite file. +// https://developer.bitmovin.com/playback/docs/webvtt-based-thumbnails for more info. +// +// Path: /:resource/:slug/thumbnails.vtt +func (h *Handler) GetThumbnailsVtt(c echo.Context) error { + resource := c.Param("resource") + slug := c.Param("slug") + + path, err := GetPath(resource, slug) + if err != nil { + return err + } + + out, err := src.ExtractThumbnail(path) + if err != nil { + return err + } + + return c.File(fmt.Sprintf("%s/sprite.vtt", out)) +} + type Handler struct { transcoder *src.Transcoder extractor *src.Extractor @@ -286,6 +331,8 @@ func main() { e.GET("/:resource/:slug/:quality/:chunk", h.GetVideoSegment) e.GET("/:resource/:slug/audio/:audio/:chunk", h.GetAudioSegment) e.GET("/:resource/:slug/info", h.GetInfo) + e.GET("/:resource/:slug/thumbnails.png", h.GetThumbnails) + e.GET("/:resource/:slug/thumbnails.vtt", h.GetThumbnailsVtt) e.GET("/:sha/attachment/:name", h.GetAttachment) e.GET("/:sha/subtitle/:name", h.GetSubtitle) diff --git a/transcoder/src/extract.go b/transcoder/src/extract.go index 5d1ddcae..1430d89b 100644 --- a/transcoder/src/extract.go +++ b/transcoder/src/extract.go @@ -49,8 +49,8 @@ func (e *Extractor) RunExtractor(path string, sha string, subs *[]Subtitle) <-ch e.lock.Unlock() go func() { - attachment_path := fmt.Sprintf("%s/%s/att/", GetMetadataPath(), sha) - subs_path := fmt.Sprintf("%s/%s/sub/", GetMetadataPath(), sha) + attachment_path := fmt.Sprintf("%s/%s/att", GetMetadataPath(), sha) + subs_path := fmt.Sprintf("%s/%s/sub", GetMetadataPath(), sha) os.MkdirAll(attachment_path, 0o644) os.MkdirAll(subs_path, 0o755) diff --git a/transcoder/src/thumbnails.go b/transcoder/src/thumbnails.go new file mode 100644 index 00000000..025216fa --- /dev/null +++ b/transcoder/src/thumbnails.go @@ -0,0 +1,92 @@ +package src + +import ( + "fmt" + "image" + "image/color" + "log" + "math" + "os" + + "github.com/disintegration/imaging" + "gitlab.com/opennota/screengen" +) + +// We want to have a thumbnail every ${interval} seconds. +var default_interval = 10 + +func ExtractThumbnail(path string) (string, error) { + ret, err := GetInfo(path) + if err != nil { + return "", err + } + out := fmt.Sprintf("%s/%s", GetMetadataPath(), ret.Sha) + sprite_path := fmt.Sprintf("%s/sprite.png", out) + vtt_path := fmt.Sprintf("%s/sprite.vtt", out) + + gen, err := screengen.NewGenerator(path) + if err != nil { + log.Printf("Error reading video file: %v", err) + return "", err + } + defer gen.Close() + + gen.Fast = true + + // truncate duration to full seconds + // this prevents empty/black images when the movie is some milliseconds longer + // ffmpeg then sometimes takes a black screenshot AFTER the movie finished for some reason + duration := 1000 * (int(gen.Duration) / 1000) + var numcaps, interval int + if default_interval < duration { + numcaps = duration / default_interval * 1000 + interval = default_interval + } else { + numcaps = duration / 10 + interval = duration / numcaps + } + columns := int(math.Sqrt(float64(numcaps))) + rows := int(math.Ceil(float64(numcaps) / float64(columns))) + + width := gen.Width() + height := gen.Height() + + sprite := imaging.New(width*columns, height*rows, color.Black) + vtt := "WEBVTT\n\n" + + ts := 0 + for i := 0; i < numcaps; i++ { + img, err := gen.Image(int64(ts)) + if err != nil { + log.Printf("Could not generate screenshot %s", err) + return "", err + } + + // TODO: resize image + + x := i % width + y := i / width + sprite = imaging.Paste(sprite, img, image.Pt(x, y)) + + timestamps := ts + ts += interval + vtt += fmt.Sprintf( + "%s --> %s\n%s#xywh=%d,%d,%d,%d\n\n", + tsToVttTime(timestamps), + tsToVttTime(ts), + name, + x, + y, + width, + height, + ) + } + + os.WriteFile(vtt_path, []byte(vtt), 0o644) + imaging.Save(sprite, sprite_path) + return out, nil +} + +func tsToVttTime(ts int) string { + return fmt.Sprintf("%02d:%02d:%02d.000", ts/3600, ts/60, ts%60) +}