Create a thumbnails sprite api

transcoder/go.mod

@@ -5,6 +5,7 @@ go 1.21
 require github.com/labstack/echo/v4 v4.11.4 // direct
 
 require (
+	github.com/disintegration/imaging v1.6.2
 	github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
 	github.com/labstack/gommon v0.4.2 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
@@ -12,9 +13,12 @@ require (
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	github.com/valyala/fasttemplate v1.2.2 // indirect
 	github.com/zoriya/go-mediainfo v0.0.0-20240113000440-36f500affcfd // indirect
+	gitlab.com/opennota/screengen v1.0.2
 	golang.org/x/crypto v0.17.0 // indirect
 	golang.org/x/net v0.19.0 // indirect
 	golang.org/x/sys v0.15.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	golang.org/x/time v0.5.0 // indirect
 )
+
+require golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8 // indirect

transcoder/go.sum

@@ -1,3 +1,5 @@
+github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1ei82L+c=
+github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4=
 github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY=
 github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
 github.com/labstack/echo/v4 v4.11.4 h1:vDZmA+qNeh1pd/cCkEicDMrjtrnMGQ1QFI9gWN1zGq8=
@@ -17,14 +19,19 @@ github.com/zoriya/go-mediainfo v0.0.0-20240112235842-ce0c807be738 h1:FV9TIvf/T84
 github.com/zoriya/go-mediainfo v0.0.0-20240112235842-ce0c807be738/go.mod h1:jzun1oQGoJSh65g1XKaolTmjd6HW/34WHH7VMdJdbvM=
 github.com/zoriya/go-mediainfo v0.0.0-20240113000440-36f500affcfd h1:AOdEpcmYJkmIW4I76TQim6LT4+9duYTdXNgkQsPHpuA=
 github.com/zoriya/go-mediainfo v0.0.0-20240113000440-36f500affcfd/go.mod h1:jzun1oQGoJSh65g1XKaolTmjd6HW/34WHH7VMdJdbvM=
+gitlab.com/opennota/screengen v1.0.2 h1:GxYTJdAPEzmg5v5CV4dgn45JVW+EcXXAvCxhE7w6UDw=
+gitlab.com/opennota/screengen v1.0.2/go.mod h1:4kED4yriw2zslwYmXFCa5qCvEKwleBA7l5OE+d94NTU=
 golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k=
 golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
+golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8 h1:hVwzHzIUGRjiF7EcUjqNxk3NCfkPxbDKRdnNE1Rpg0U=
+golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c=
 golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
 golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
 golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=

transcoder/main.go

@@ -262,6 +262,51 @@ func (h *Handler) GetSubtitle(c echo.Context) error {
 	return c.File(path)
 }
 
+// Get thumbnail sprite
+//
+// Get a sprite file containing all the thumbnails of the show
+//
+// Path: /:resource/:slug/thumbnails.png
+func (h *Handler) GetThumbnails(c echo.Context) error {
+	resource := c.Param("resource")
+	slug := c.Param("slug")
+
+	path, err := GetPath(resource, slug)
+	if err != nil {
+		return err
+	}
+
+	out, err := src.ExtractThumbnail(path)
+	if err != nil {
+		return err
+	}
+
+	return c.File(fmt.Sprintf("%s/sprite.png", out))
+}
+
+// Get thumbnail vtt
+//
+// Get a vtt file containing timing/position of thumbnails inside the sprite file.
+// https://developer.bitmovin.com/playback/docs/webvtt-based-thumbnails for more info.
+//
+// Path: /:resource/:slug/thumbnails.vtt
+func (h *Handler) GetThumbnailsVtt(c echo.Context) error {
+	resource := c.Param("resource")
+	slug := c.Param("slug")
+
+	path, err := GetPath(resource, slug)
+	if err != nil {
+		return err
+	}
+
+	out, err := src.ExtractThumbnail(path)
+	if err != nil {
+		return err
+	}
+
+	return c.File(fmt.Sprintf("%s/sprite.vtt", out))
+}
+
 type Handler struct {
 	transcoder *src.Transcoder
 	extractor  *src.Extractor
@@ -286,6 +331,8 @@ func main() {
 	e.GET("/:resource/:slug/:quality/:chunk", h.GetVideoSegment)
 	e.GET("/:resource/:slug/audio/:audio/:chunk", h.GetAudioSegment)
 	e.GET("/:resource/:slug/info", h.GetInfo)
+	e.GET("/:resource/:slug/thumbnails.png", h.GetThumbnails)
+	e.GET("/:resource/:slug/thumbnails.vtt", h.GetThumbnailsVtt)
 	e.GET("/:sha/attachment/:name", h.GetAttachment)
 	e.GET("/:sha/subtitle/:name", h.GetSubtitle)
 

transcoder/src/extract.go

@@ -49,8 +49,8 @@ func (e *Extractor) RunExtractor(path string, sha string, subs *[]Subtitle) <-ch
 	e.lock.Unlock()
 
 	go func() {
-		attachment_path := fmt.Sprintf("%s/%s/att/", GetMetadataPath(), sha)
+		attachment_path := fmt.Sprintf("%s/%s/att", GetMetadataPath(), sha)
-		subs_path := fmt.Sprintf("%s/%s/sub/", GetMetadataPath(), sha)
+		subs_path := fmt.Sprintf("%s/%s/sub", GetMetadataPath(), sha)
 		os.MkdirAll(attachment_path, 0o644)
 		os.MkdirAll(subs_path, 0o755)
 

transcoder/src/thumbnails.go

@@ -0,0 +1,92 @@
+package src
+
+import (
+	"fmt"
+	"image"
+	"image/color"
+	"log"
+	"math"
+	"os"
+
+	"github.com/disintegration/imaging"
+	"gitlab.com/opennota/screengen"
+)
+
+// We want to have a thumbnail every ${interval} seconds.
+var default_interval = 10
+
+func ExtractThumbnail(path string) (string, error) {
+	ret, err := GetInfo(path)
+	if err != nil {
+		return "", err
+	}
+	out := fmt.Sprintf("%s/%s", GetMetadataPath(), ret.Sha)
+	sprite_path := fmt.Sprintf("%s/sprite.png", out)
+	vtt_path := fmt.Sprintf("%s/sprite.vtt", out)
+
+	gen, err := screengen.NewGenerator(path)
+	if err != nil {
+		log.Printf("Error reading video file: %v", err)
+		return "", err
+	}
+	defer gen.Close()
+
+	gen.Fast = true
+
+	// truncate duration to full seconds
+	// this prevents empty/black images when the movie is some milliseconds longer
+	// ffmpeg then sometimes takes a black screenshot AFTER the movie finished for some reason
+	duration := 1000 * (int(gen.Duration) / 1000)
+	var numcaps, interval int
+	if default_interval < duration {
+		numcaps = duration / default_interval * 1000
+		interval = default_interval
+	} else {
+		numcaps = duration / 10
+		interval = duration / numcaps
+	}
+	columns := int(math.Sqrt(float64(numcaps)))
+	rows := int(math.Ceil(float64(numcaps) / float64(columns)))
+
+	width := gen.Width()
+	height := gen.Height()
+
+	sprite := imaging.New(width*columns, height*rows, color.Black)
+	vtt := "WEBVTT\n\n"
+
+	ts := 0
+	for i := 0; i < numcaps; i++ {
+		img, err := gen.Image(int64(ts))
+		if err != nil {
+			log.Printf("Could not generate screenshot %s", err)
+			return "", err
+		}
+
+		// TODO: resize image
+
+		x := i % width
+		y := i / width
+		sprite = imaging.Paste(sprite, img, image.Pt(x, y))
+
+		timestamps := ts
+		ts += interval
+		vtt += fmt.Sprintf(
+			"%s --> %s\n%s#xywh=%d,%d,%d,%d\n\n",
+			tsToVttTime(timestamps),
+			tsToVttTime(ts),
+			name,
+			x,
+			y,
+			width,
+			height,
+		)
+	}
+
+	os.WriteFile(vtt_path, []byte(vtt), 0o644)
+	imaging.Save(sprite, sprite_path)
+	return out, nil
+}
+
+func tsToVttTime(ts int) string {
+	return fmt.Sprintf("%02d:%02d:%02d.000", ts/3600, ts/60, ts%60)
+}