Use cmap for transcode streams

This commit is contained in:
Zoe Roux 2024-02-19 01:33:09 +01:00
parent f54a876636
commit 5389e1b783
4 changed files with 67 additions and 779 deletions

View File

@ -42,7 +42,7 @@ func (h *Handler) GetMaster(c echo.Context) error {
return err
}
ret, err := h.transcoder.GetMaster(path, client)
ret, err := h.transcoder.GetMaster(path, client, GetRoute(c))
if err != nil {
return err
}
@ -70,7 +70,7 @@ func (h *Handler) GetVideoIndex(c echo.Context) error {
return err
}
ret, err := h.transcoder.GetVideoIndex(path, quality, client)
ret, err := h.transcoder.GetVideoIndex(path, quality, client, GetRoute(c))
if err != nil {
return err
}
@ -98,7 +98,7 @@ func (h *Handler) GetAudioIndex(c echo.Context) error {
return err
}
ret, err := h.transcoder.GetAudioIndex(path, int32(audio), client)
ret, err := h.transcoder.GetAudioIndex(path, int32(audio), client, GetRoute(c))
if err != nil {
return err
}
@ -128,7 +128,7 @@ func (h *Handler) GetVideoSegment(c echo.Context) error {
return err
}
ret, err := h.transcoder.GetVideoSegment(path, quality, segment, client)
ret, err := h.transcoder.GetVideoSegment(path, quality, segment, client, GetRoute(c))
if err != nil {
return err
}
@ -158,7 +158,7 @@ func (h *Handler) GetAudioSegment(c echo.Context) error {
return err
}
ret, err := h.transcoder.GetAudioSegment(path, int32(audio), segment, client)
ret, err := h.transcoder.GetAudioSegment(path, int32(audio), segment, client, GetRoute(c))
if err != nil {
return err
}

View File

@ -1,697 +0,0 @@
{
"openapi": "3.0.3",
"info": {
"title": "transcoder",
"description": "Transcoder's open api.",
"license": {
"name": ""
},
"version": "0.1.0"
},
"paths": {
"/{resource}/{slug}/audio/{audio}/index.m3u8": {
"get": {
"tags": [
"crate"
],
"summary": "Transcode audio",
"description": "Transcode audio\n\nGet the selected audio\nThis route can take a few seconds to respond since it will way for at least one segment to be\navailable.",
"operationId": "get_audio_transcoded",
"parameters": [
{
"name": "resource",
"in": "path",
"description": "Episode or movie",
"required": true,
"schema": {
"type": "string"
}
},
{
"name": "slug",
"in": "path",
"description": "The slug of the movie/episode.",
"required": true,
"schema": {
"type": "string"
}
},
{
"name": "audio",
"in": "path",
"description": "Specify the audio stream you want. For mappings, refer to the audios fields of the /watch response.",
"required": true,
"schema": {
"type": "integer",
"format": "int32",
"minimum": 0
}
}
],
"responses": {
"200": {
"description": "Get the m3u8 playlist."
},
"404": {
"description": "Invalid slug."
}
}
}
},
"/{resource}/{slug}/audio/{audio}/segments-{chunk}.ts": {
"get": {
"tags": [
"crate"
],
"summary": "Get audio chunk",
"description": "Get audio chunk\n\nRetrieve a chunk of a transcoded audio.",
"operationId": "get_audio_chunk",
"parameters": [
{
"name": "resource",
"in": "path",
"description": "Episode or movie",
"required": true,
"schema": {
"type": "string"
}
},
{
"name": "slug",
"in": "path",
"description": "The slug of the movie/episode.",
"required": true,
"schema": {
"type": "string"
}
},
{
"name": "audio",
"in": "path",
"description": "Specify the audio you want",
"required": true,
"schema": {
"type": "integer",
"format": "int32",
"minimum": 0
}
},
{
"name": "chunk",
"in": "path",
"description": "The number of the chunk",
"required": true,
"schema": {
"type": "integer",
"format": "int32",
"minimum": 0
}
}
],
"responses": {
"200": {
"description": "Get a hls chunk."
},
"404": {
"description": "Invalid slug."
}
}
}
},
"/{resource}/{slug}/direct": {
"get": {
"tags": [
"crate"
],
"summary": "Direct video",
"description": "Direct video\n\nRetrieve the raw video stream, in the same container as the one on the server. No transcoding or\ntransmuxing is done.",
"operationId": "get_direct",
"parameters": [
{
"name": "resource",
"in": "path",
"description": "Episode or movie",
"required": true,
"schema": {
"type": "string"
}
},
{
"name": "slug",
"in": "path",
"description": "The slug of the movie/episode.",
"required": true,
"schema": {
"type": "string"
}
}
],
"responses": {
"200": {
"description": "The item is returned"
},
"404": {
"description": "Invalid slug."
}
}
}
},
"/{resource}/{slug}/info": {
"get": {
"tags": [
"crate"
],
"summary": "Identify",
"description": "Identify\n\nIdentify metadata about a file",
"operationId": "identify_resource",
"parameters": [
{
"name": "resource",
"in": "path",
"description": "Episode or movie",
"required": true,
"schema": {
"type": "string"
}
},
{
"name": "slug",
"in": "path",
"description": "The slug of the movie/episode.",
"required": true,
"schema": {
"type": "string"
}
}
],
"responses": {
"200": {
"description": "Ok",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/MediaInfo"
}
}
}
},
"404": {
"description": "Invalid slug."
}
}
}
},
"/{resource}/{slug}/master.m3u8": {
"get": {
"tags": [
"crate"
],
"summary": "Get master playlist",
"description": "Get master playlist\n\nGet a master playlist containing all possible video qualities and audios available for this resource.\nNote that the direct stream is missing (since the direct is not an hls stream) and\nsubtitles/fonts are not included to support more codecs than just webvtt.",
"operationId": "get_master",
"parameters": [
{
"name": "resource",
"in": "path",
"description": "Episode or movie",
"required": true,
"schema": {
"type": "string"
}
},
{
"name": "slug",
"in": "path",
"description": "The slug of the movie/episode.",
"required": true,
"schema": {
"type": "string"
}
}
],
"responses": {
"200": {
"description": "Get the m3u8 master playlist."
},
"404": {
"description": "Invalid slug."
}
}
}
},
"/{resource}/{slug}/{quality}/index.m3u8": {
"get": {
"tags": [
"crate"
],
"summary": "Transcode video",
"description": "Transcode video\n\nTranscode the video to the selected quality.\nThis route can take a few seconds to respond since it will way for at least one segment to be\navailable.",
"operationId": "get_transcoded",
"parameters": [
{
"name": "resource",
"in": "path",
"description": "Episode or movie",
"required": true,
"schema": {
"type": "string"
}
},
{
"name": "slug",
"in": "path",
"description": "The slug of the movie/episode.",
"required": true,
"schema": {
"type": "string"
}
},
{
"name": "quality",
"in": "path",
"description": "Specify the quality you want",
"required": true,
"schema": {
"type": "string"
}
},
{
"name": "x-client-id",
"in": "header",
"description": "A unique identify for a player's instance. Used to cancel unused transcode",
"required": true,
"schema": {
"type": "string"
}
}
],
"responses": {
"200": {
"description": "Get the m3u8 playlist."
},
"404": {
"description": "Invalid slug."
}
}
}
},
"/{resource}/{slug}/{quality}/segments-{chunk}.ts": {
"get": {
"tags": [
"crate"
],
"summary": "Get transmuxed chunk",
"description": "Get transmuxed chunk\n\nRetrieve a chunk of a transmuxed video.",
"operationId": "get_chunk",
"parameters": [
{
"name": "resource",
"in": "path",
"description": "Episode or movie",
"required": true,
"schema": {
"type": "string"
}
},
{
"name": "slug",
"in": "path",
"description": "The slug of the movie/episode.",
"required": true,
"schema": {
"type": "string"
}
},
{
"name": "quality",
"in": "path",
"description": "Specify the quality you want",
"required": true,
"schema": {
"type": "string"
}
},
{
"name": "chunk",
"in": "path",
"description": "The number of the chunk",
"required": true,
"schema": {
"type": "integer",
"format": "int32",
"minimum": 0
}
},
{
"name": "x-client-id",
"in": "header",
"description": "A unique identify for a player's instance. Used to cancel unused transcode",
"required": true,
"schema": {
"type": "string"
}
}
],
"responses": {
"200": {
"description": "Get a hls chunk."
},
"404": {
"description": "Invalid slug."
}
}
}
},
"/{sha}/attachment/{name}": {
"get": {
"tags": [
"crate"
],
"summary": "Get attachments",
"description": "Get attachments\n\nGet a specific attachment",
"operationId": "get_attachment",
"parameters": [
{
"name": "sha",
"in": "path",
"description": "The sha1 of the file",
"required": true,
"schema": {
"type": "string"
}
},
{
"name": "name",
"in": "path",
"description": "The name of the attachment.",
"required": true,
"schema": {
"type": "string"
}
}
],
"responses": {
"200": {
"description": "Ok",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/MediaInfo"
}
}
}
},
"404": {
"description": "Invalid slug."
}
}
}
},
"/{sha}/subtitle/{name}": {
"get": {
"tags": [
"crate"
],
"summary": "Get subtitle",
"description": "Get subtitle\n\nGet a specific subtitle",
"operationId": "get_subtitle",
"parameters": [
{
"name": "sha",
"in": "path",
"description": "The sha1 of the file",
"required": true,
"schema": {
"type": "string"
}
},
{
"name": "name",
"in": "path",
"description": "The name of the subtitle.",
"required": true,
"schema": {
"type": "string"
}
}
],
"responses": {
"200": {
"description": "Ok",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/MediaInfo"
}
}
}
},
"404": {
"description": "Invalid slug."
}
}
}
}
},
"components": {
"schemas": {
"Audio": {
"type": "object",
"required": [
"index",
"codec",
"isDefault",
"isForced"
],
"properties": {
"codec": {
"type": "string",
"description": "The codec of this stream."
},
"index": {
"type": "integer",
"format": "int32",
"description": "The index of this track on the media.",
"minimum": 0
},
"isDefault": {
"type": "boolean",
"description": "Is this stream the default one of it's type?"
},
"isForced": {
"type": "boolean",
"description": "Is this stream tagged as forced? (useful only for subtitles)"
},
"language": {
"type": "string",
"description": "The language of this stream (as a ISO-639-2 language code)",
"nullable": true
},
"title": {
"type": "string",
"description": "The title of the stream.",
"nullable": true
}
}
},
"Chapter": {
"type": "object",
"required": [
"startTime",
"endTime",
"name"
],
"properties": {
"endTime": {
"type": "number",
"format": "float",
"description": "The end time of the chapter (in second from the start of the episode)."
},
"name": {
"type": "string",
"description": "The name of this chapter. This should be a human-readable name that could be presented to the user."
},
"startTime": {
"type": "number",
"format": "float",
"description": "The start time of the chapter (in second from the start of the episode)."
}
}
},
"MediaInfo": {
"type": "object",
"required": [
"sha",
"path",
"extension",
"length",
"container",
"video",
"audios",
"subtitles",
"fonts",
"chapters"
],
"properties": {
"audios": {
"type": "array",
"items": {
"$ref": "#/components/schemas/Audio"
},
"description": "The list of audio tracks."
},
"chapters": {
"type": "array",
"items": {
"$ref": "#/components/schemas/Chapter"
},
"description": "The list of chapters. See Chapter for more information."
},
"container": {
"type": "string",
"description": "The container of the video file of this episode."
},
"extension": {
"type": "string",
"description": "The extension currently used to store this video file"
},
"fonts": {
"type": "array",
"items": {
"type": "string"
},
"description": "The list of fonts that can be used to display subtitles."
},
"length": {
"type": "number",
"format": "float",
"description": "The length of the media in seconds."
},
"path": {
"type": "string",
"description": "The internal path of the video file."
},
"sha": {
"type": "string",
"description": "The sha1 of the video file."
},
"subtitles": {
"type": "array",
"items": {
"$ref": "#/components/schemas/Subtitle"
},
"description": "The list of subtitles tracks."
},
"video": {
"$ref": "#/components/schemas/Video"
}
}
},
"Quality": {
"type": "string",
"enum": [
"P240",
"P360",
"P480",
"P720",
"P1080",
"P1440",
"P4k",
"P8k",
"Original"
]
},
"Subtitle": {
"type": "object",
"required": [
"index",
"codec",
"isDefault",
"isForced"
],
"properties": {
"codec": {
"type": "string",
"description": "The codec of this stream."
},
"extension": {
"type": "string",
"description": "The extension for the codec.",
"nullable": true
},
"index": {
"type": "integer",
"format": "int32",
"description": "The index of this track on the media.",
"minimum": 0
},
"isDefault": {
"type": "boolean",
"description": "Is this stream the default one of it's type?"
},
"isForced": {
"type": "boolean",
"description": "Is this stream tagged as forced? (useful only for subtitles)"
},
"language": {
"type": "string",
"description": "The language of this stream (as a ISO-639-2 language code)",
"nullable": true
},
"link": {
"type": "string",
"description": "The link to access this subtitle.",
"nullable": true
},
"title": {
"type": "string",
"description": "The title of the stream.",
"nullable": true
}
}
},
"Video": {
"type": "object",
"required": [
"codec",
"quality",
"width",
"height",
"bitrate"
],
"properties": {
"bitrate": {
"type": "integer",
"format": "int32",
"description": "The average bitrate of the video in bytes/s",
"minimum": 0
},
"codec": {
"type": "string",
"description": "The codec of this stream (defined as the RFC 6381)."
},
"height": {
"type": "integer",
"format": "int32",
"description": "The height of the video stream",
"minimum": 0
},
"language": {
"type": "string",
"description": "The language of this stream (as a ISO-639-2 language code)",
"nullable": true
},
"quality": {
"$ref": "#/components/schemas/Quality"
},
"width": {
"type": "integer",
"format": "int32",
"description": "The width of the video stream",
"minimum": 0
}
}
}
}
}
}

View File

@ -12,6 +12,8 @@ import (
)
type FileStream struct {
ready sync.WaitGroup
err error
Path string
Out string
Keyframes []float64
@ -23,37 +25,36 @@ type FileStream struct {
alock sync.Mutex
}
func NewFileStream(path string) (*FileStream, error) {
info_chan := make(chan struct {
info *MediaInfo
err error
})
func NewFileStream(path string, sha string, route string) *FileStream {
ret := &FileStream{
Path: path,
Out: fmt.Sprintf("%s/%s", Settings.Outpath, sha),
streams: make(map[Quality]*VideoStream),
audios: make(map[int32]*AudioStream),
}
ret.ready.Add(1)
go func() {
ret, err := GetInfo(path)
info_chan <- struct {
info *MediaInfo
err error
}{ret, err}
defer ret.ready.Done()
info, err := GetInfo(path, sha, route)
ret.Info = info
if err != nil {
ret.err = err
}
}()
keyframes, can_transmux, err := GetKeyframes(path)
if err != nil {
return nil, err
}
info := <-info_chan
if info.err != nil {
return nil, err
}
ret.ready.Add(1)
go func() {
defer ret.ready.Done()
keyframes, can_transmux, err := GetKeyframes(path)
ret.Keyframes = keyframes
ret.CanTransmux = can_transmux
if err != nil {
ret.err = err
}
}()
return &FileStream{
Path: path,
Out: fmt.Sprintf("%s/%s", Settings.Outpath, info.info.Sha),
Keyframes: keyframes,
CanTransmux: can_transmux,
Info: info.info,
streams: make(map[Quality]*VideoStream),
audios: make(map[int32]*AudioStream),
}, nil
return ret
}
func GetKeyframes(path string) ([]float64, bool, error) {

View File

@ -1,19 +1,13 @@
package src
import (
"errors"
"log"
"os"
"path"
"sync"
)
type Transcoder struct {
// All file streams currently running, index is file path
streams map[string]*FileStream
// Streams that are staring up
preparing map[string]chan *FileStream
mutex sync.Mutex
streams CMap[string, *FileStream]
clientChan chan ClientInfo
tracker *Tracker
}
@ -32,54 +26,32 @@ func NewTranscoder() (*Transcoder, error) {
}
ret := &Transcoder{
streams: make(map[string]*FileStream),
preparing: make(map[string]chan *FileStream),
streams: NewCMap[string, *FileStream](),
clientChan: make(chan ClientInfo, 10),
}
ret.tracker = NewTracker(ret)
return ret, nil
}
func (t *Transcoder) getFileStream(path string) (*FileStream, error) {
t.mutex.Lock()
stream, ok := t.streams[path]
channel, preparing := t.preparing[path]
if !preparing && !ok {
channel = make(chan *FileStream, 1)
t.preparing[path] = channel
}
t.mutex.Unlock()
if preparing {
stream = <-channel
if stream == nil {
return nil, errors.New("could not transcode file. Try again later")
}
} else if !ok {
var err error
stream, err = NewFileStream(path)
log.Printf("Stream created for %s", path)
func (t *Transcoder) getFileStream(path string, route string) (*FileStream, error) {
var err error
ret, _ := t.streams.GetOrCreate(path, func() *FileStream {
sha, err := GetHash(path)
if err != nil {
t.mutex.Lock()
delete(t.preparing, path)
t.mutex.Unlock()
channel <- nil
return nil, err
return nil
}
t.mutex.Lock()
t.streams[path] = stream
delete(t.preparing, path)
t.mutex.Unlock()
channel <- stream
return NewFileStream(path, sha, route)
})
ret.ready.Wait()
if err != nil || ret.err != nil {
t.streams.Remove(path)
return nil, ret.err
}
return stream, nil
return ret, nil
}
func (t *Transcoder) GetMaster(path string, client string) (string, error) {
stream, err := t.getFileStream(path)
func (t *Transcoder) GetMaster(path string, client string, route string) (string, error) {
stream, err := t.getFileStream(path, route)
if err != nil {
return "", err
}
@ -93,8 +65,13 @@ func (t *Transcoder) GetMaster(path string, client string) (string, error) {
return stream.GetMaster(), nil
}
func (t *Transcoder) GetVideoIndex(path string, quality Quality, client string) (string, error) {
stream, err := t.getFileStream(path)
func (t *Transcoder) GetVideoIndex(
path string,
quality Quality,
client string,
route string,
) (string, error) {
stream, err := t.getFileStream(path, route)
if err != nil {
return "", err
}
@ -108,8 +85,13 @@ func (t *Transcoder) GetVideoIndex(path string, quality Quality, client string)
return stream.GetVideoIndex(quality)
}
func (t *Transcoder) GetAudioIndex(path string, audio int32, client string) (string, error) {
stream, err := t.getFileStream(path)
func (t *Transcoder) GetAudioIndex(
path string,
audio int32,
client string,
route string,
) (string, error) {
stream, err := t.getFileStream(path, route)
if err != nil {
return "", err
}
@ -127,8 +109,9 @@ func (t *Transcoder) GetVideoSegment(
quality Quality,
segment int32,
client string,
route string,
) (string, error) {
stream, err := t.getFileStream(path)
stream, err := t.getFileStream(path, route)
if err != nil {
return "", err
}
@ -147,8 +130,9 @@ func (t *Transcoder) GetAudioSegment(
audio int32,
segment int32,
client string,
route string,
) (string, error) {
stream, err := t.getFileStream(path)
stream, err := t.getFileStream(path, route)
if err != nil {
return "", err
}