From 425de0b3154d32a44a08b34bc356b46f9745c265 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Thu, 18 Jan 2024 05:50:52 +0100 Subject: [PATCH] Add extraction cache and wait for extraction to end to return subtitles/attachments --- transcoder/main.go | 32 ++++++++++---- transcoder/src/extract.go | 91 ++++++++++++++++++++++++++++----------- transcoder/src/info.go | 2 - 3 files changed, 90 insertions(+), 35 deletions(-) diff --git a/transcoder/main.go b/transcoder/main.go index 7ff1b6d8..d9e6af03 100644 --- a/transcoder/main.go +++ b/transcoder/main.go @@ -190,10 +190,10 @@ func (h *Handler) GetAudioSegment(c echo.Context) error { // Identify // -// # Identify metadata about a file +// Identify metadata about a file // // Path: /:resource/:slug/info -func GetInfo(c echo.Context) error { +func (h *Handler) GetInfo(c echo.Context) error { resource := c.Param("resource") slug := c.Param("slug") @@ -206,6 +206,7 @@ func GetInfo(c echo.Context) error { if err != nil { return err } + h.extractor.RunExtractor(ret.Path, ret.Sha, &ret.Subtitles) return c.JSON(http.StatusOK, ret) } @@ -214,7 +215,7 @@ func GetInfo(c echo.Context) error { // Get a specific attachment // // Path: /:sha/attachment/:name -func GetAttachment(c echo.Context) error { +func (h *Handler) GetAttachment(c echo.Context) error { sha := c.Param("sha") name := c.Param("name") @@ -225,6 +226,12 @@ func GetAttachment(c echo.Context) error { return err } + wait, ok := h.extractor.Extract(sha) + if !ok { + return echo.NewHTTPError(http.StatusBadRequest, "Not extracted yet. Call /info to extract.") + } + <-wait + path := fmt.Sprintf("%s/%s/att/%s", src.GetMetadataPath(), sha, name) return c.File(path) } @@ -233,8 +240,8 @@ func GetAttachment(c echo.Context) error { // // Get a specific subtitle // -// Path: /:sha/sub/:name -func GetSubtitle(c echo.Context) error { +// Path: /:sha/subtitle/:name +func (h *Handler) GetSubtitle(c echo.Context) error { sha := c.Param("sha") name := c.Param("name") @@ -245,12 +252,19 @@ func GetSubtitle(c echo.Context) error { return err } + wait, ok := h.extractor.Extract(sha) + if !ok { + return echo.NewHTTPError(http.StatusBadRequest, "Not extracted yet. Call /info to extract.") + } + <-wait + path := fmt.Sprintf("%s/%s/sub/%s", src.GetMetadataPath(), sha, name) return c.File(path) } type Handler struct { transcoder *src.Transcoder + extractor *src.Extractor } func main() { @@ -263,7 +277,7 @@ func main() { e.Logger.Fatal(err) return } - h := Handler{transcoder: transcoder} + h := Handler{transcoder: transcoder, extractor: src.NewExtractor()} e.GET("/:resource/:slug/direct", DirectStream) e.GET("/:resource/:slug/master.m3u8", h.GetMaster) @@ -271,9 +285,9 @@ func main() { e.GET("/:resource/:slug/audio/:audio/index.m3u8", h.GetAudioIndex) e.GET("/:resource/:slug/:quality/:chunk", h.GetVideoSegment) e.GET("/:resource/:slug/audio/:audio/:chunk", h.GetAudioSegment) - e.GET("/:resource/:slug/info", GetInfo) - e.GET("/:sha/attachment/:name", GetAttachment) - e.GET("/:sha/sub/:name", GetSubtitle) + e.GET("/:resource/:slug/info", h.GetInfo) + e.GET("/:sha/attachment/:name", h.GetAttachment) + e.GET("/:sha/subtitle/:name", h.GetSubtitle) e.Logger.Fatal(e.Start(":7666")) } diff --git a/transcoder/src/extract.go b/transcoder/src/extract.go index 0ec4d24e..41878069 100644 --- a/transcoder/src/extract.go +++ b/transcoder/src/extract.go @@ -4,8 +4,14 @@ import ( "fmt" "os" "os/exec" + "sync" ) +type Extractor struct { + extracted map[string]<-chan struct{} + lock sync.RWMutex +} + func GetMetadataPath() string { out := os.Getenv("GOCODER_METADATA_ROOT") if out == "" { @@ -14,29 +20,66 @@ func GetMetadataPath() string { return out } -func extract(path string, sha string, subs *[]Subtitle) { - fmt.Printf("Extract subs and fonts for %s", path) - cmd := exec.Command( - "ffmpeg", - "-dump_attachment:t", "", - "-i", path, - ) - cmd.Dir = fmt.Sprintf("/%s/%s/att/", GetMetadataPath(), sha) - - for _, sub := range *subs { - if ext := sub.Extension; ext != nil { - cmd.Args = append( - cmd.Args, - "-map", fmt.Sprintf("0:s:%d", sub.Index), - "-c:s", "copy", - fmt.Sprintf("/%s/%s/sub/%d.%s", GetMetadataPath(), sha, sub.Index, *ext), - ) - } - } - fmt.Printf("Starting extraction with the command: %s", cmd) - cmd.Stdout = nil - err := cmd.Run() - if err != nil { - fmt.Println("Error starting ffmpeg extract:", err) +func NewExtractor() *Extractor { + return &Extractor{ + extracted: make(map[string]<-chan struct{}), } } + +func (e *Extractor) Extract(sha string) (<-chan struct{}, bool) { + e.lock.RLock() + existing, ok := e.extracted[sha] + e.lock.RUnlock() + + if ok { + return existing, true + } + return nil, false +} + +func (e *Extractor) RunExtractor(path string, sha string, subs *[]Subtitle) <-chan struct{} { + existing, ok := e.Extract(sha) + if ok { + return existing + } + + ret := make(chan struct{}) + e.lock.Lock() + e.extracted[sha] = ret + e.lock.Unlock() + + go func() { + attachment_path := fmt.Sprintf("%s/%s/att/", GetMetadataPath(), sha) + subs_path := fmt.Sprintf("%s/%s/sub/", GetMetadataPath(), sha) + os.MkdirAll(attachment_path, 0o644) + os.MkdirAll(subs_path, 0o644) + + fmt.Printf("Extract subs and fonts for %s", path) + cmd := exec.Command( + "ffmpeg", + "-dump_attachment:t", "", + "-i", path, + ) + cmd.Dir = attachment_path + + for _, sub := range *subs { + if ext := sub.Extension; ext != nil { + cmd.Args = append( + cmd.Args, + "-map", fmt.Sprintf("0:s:%d", sub.Index), + "-c:s", "copy", + fmt.Sprintf("%s/%d.%s", subs_path, sub.Index, *ext), + ) + } + } + fmt.Printf("Starting extraction with the command: %s", cmd) + cmd.Stdout = nil + err := cmd.Run() + if err != nil { + fmt.Println("Error starting ffmpeg extract:", err) + } + close(ret) + }() + + return ret +} diff --git a/transcoder/src/info.go b/transcoder/src/info.go index 7afb0591..6673247b 100644 --- a/transcoder/src/info.go +++ b/transcoder/src/info.go @@ -169,8 +169,6 @@ func GetInfo(path string) (*MediaInfo, error) { } defer mi.Close() - // TODO: extract - sha := mi.Parameter(mediainfo.StreamGeneral, 0, "UniqueID") // Remove dummy values that some tools use. if len(sha) <= 5 {