diff --git a/scanner/scanner/scanner.py b/scanner/scanner/scanner.py index 705a606f..984cde7f 100644 --- a/scanner/scanner/scanner.py +++ b/scanner/scanner/scanner.py @@ -7,6 +7,7 @@ import re from aiohttp import ClientSession from pathlib import Path from typing import List, Literal, Any +from urllib.parse import quote from providers.provider import Provider, ProviderError from providers.types.collection import Collection from providers.types.show import Show @@ -273,7 +274,7 @@ class Scanner: if type is None or type == "movie": async with self._client.delete( - f'{self._url}/movies?filter=path eq "{path}"', + f'{self._url}/movies?filter=path eq "{quote(path)}"', headers={"X-API-Key": self._api_key}, ) as r: if not r.ok: @@ -282,7 +283,7 @@ class Scanner: if type is None or type == "episode": async with self._client.delete( - f'{self._url}/episodes?filter=path eq "{path}"', + f'{self._url}/episodes?filter=path eq "{quote(path)}"', headers={"X-API-Key": self._api_key}, ) as r: if not r.ok: @@ -292,6 +293,6 @@ class Scanner: if path in self.issues: self.issues = filter(lambda x: x != path, self.issues) await self._client.delete( - f'{self._url}/issues?filter=domain eq scanner and cause eq "{path}"', + f'{self._url}/issues?filter=domain eq scanner and cause eq "{quote(path)}"', headers={"X-API-Key": self._api_key}, ) diff --git a/transcoder/src/codec.go b/transcoder/src/codec.go new file mode 100644 index 00000000..6aefe265 --- /dev/null +++ b/transcoder/src/codec.go @@ -0,0 +1,130 @@ +package src + +import ( + "fmt" + "log" + "strings" + + "github.com/zoriya/go-mediainfo" +) + +// convert mediainfo to RFC 6381, waiting for either of those tickets to be resolved: +// +// https://sourceforge.net/p/mediainfo/feature-requests/499 +// https://trac.ffmpeg.org/ticket/6617 +// +// this code is addapted from https://github.com/jellyfin/jellyfin/blob/master/Jellyfin.Api/Helpers/HlsCodecStringHelpers.cs +func GetMimeCodec(mi *mediainfo.File, kind mediainfo.StreamKind, i int) *string { + codec := Or( + mi.Parameter(kind, i, "InternetMediaType"), + mi.Parameter(kind, i, "Format"), + ) + + log.Printf("codec: %s", codec) + switch codec { + case "video/H264", "AVC": + ret := "avc1" + info := strings.Split(strings.ToLower(mi.Parameter(kind, i, "Format_Profile")), "@") + + format := info[0] + switch format { + case "high": + ret += ".6400" + case "main": + ret += ".4D40" + case "baseline": + ret += ".42E0" + default: + // Default to constrained baseline if profile is invalid + ret += ".4240" + } + + // level format is l3.1 for level 31 + level := ParseFloat(info[1][1:]) + log.Printf("%s %s %f", format, info[1], level) + ret += fmt.Sprintf("%02x", int(level*10)) + return &ret + + case "video/H265", "HEVC": + // The h265 syntax is a bit of a mystery at the time this comment was written. + // This is what I've found through various sources: + // FORMAT: [codecTag].[profile].[constraint?].L[level * 30].[UNKNOWN] + ret := "hvc1" + info := strings.Split(strings.ToLower(mi.Parameter(kind, i, "Format_Profile")), "@") + + profile := info[0] + if profile == "main 10" { + ret += ".2.4" + } else { + ret += ".1.4" + } + + level := ParseFloat(info[1][:1]) + ret += fmt.Sprintf(".L%02X.BO", int(level*30)) + return &ret + + case "AV1": + // https://aomedia.org/av1/specification/annex-a/ + // FORMAT: [codecTag].[profile].[level][tier].[bitDepth] + ret := "av01" + info := strings.Split(strings.ToLower(mi.Parameter(kind, i, "Format_Profile")), "@") + + profile := info[0] + switch profile { + case "main": + ret += ".0" + case "high": + ret += ".1" + case "professional": + ret += ".2" + default: + // Default to Main + ret += ".0" + } + + // level is not defined in mediainfo. using a default + // Default to the maximum defined level 6.3 + level := 19 + + bitdepth := ParseUint(mi.Parameter(kind, i, "BitDepth")) + if bitdepth != 8 && bitdepth != 10 && bitdepth != 12 { + // Default to 8 bits + bitdepth = 8 + } + + tierflag := 'M' + ret += fmt.Sprintf(".%02X%c.%02d", level, tierflag, bitdepth) + + return &ret + + case "AAC": + ret := "mp4a" + + profile := strings.ToLower(mi.Parameter(kind, i, "Format_AdditionalFeatures")) + switch profile { + case "he": + ret += ".40.5" + case "lc": + ret += ".40.2" + default: + ret += ".40.2" + } + + return &ret + + case "audio/opus", "Opus": + ret := "Opus" + return &ret + + case "AC-3": + ret := "mp4a.a5" + return &ret + + case "audio/x-flac", "FLAC": + ret := "fLaC" + return &ret + + default: + return nil + } +} diff --git a/transcoder/src/filestream.go b/transcoder/src/filestream.go index 8793a2f8..dd8c2a03 100644 --- a/transcoder/src/filestream.go +++ b/transcoder/src/filestream.go @@ -74,25 +74,35 @@ func (fs *FileStream) GetMaster() string { break } } - // TODO: also check if the codec is valid in a hls before putting transmux - if true { + // original stream + { bitrate := float64(fs.Info.Video.Bitrate) master += "#EXT-X-STREAM-INF:" master += fmt.Sprintf("AVERAGE-BANDWIDTH=%d,", int(math.Min(bitrate*0.8, float64(transmux_quality.AverageBitrate())))) master += fmt.Sprintf("BANDWIDTH=%d,", int(math.Min(bitrate, float64(transmux_quality.MaxBitrate())))) master += fmt.Sprintf("RESOLUTION=%dx%d,", fs.Info.Video.Width, fs.Info.Video.Height) + if fs.Info.Video.MimeCodec != nil { + master += fmt.Sprintf("CODECS=\"%s\",", *fs.Info.Video.MimeCodec) + } master += "AUDIO=\"audio\"," master += "CLOSED-CAPTIONS=NONE\n" master += fmt.Sprintf("./%s/index.m3u8\n", Original) } + aspectRatio := float32(fs.Info.Video.Width) / float32(fs.Info.Video.Height) + transmux_codec := "avc1.640028" + for _, quality := range Qualities { - if quality.Height() < fs.Info.Video.Quality.Height() && quality.AverageBitrate() < fs.Info.Video.Bitrate { + same_codec := fs.Info.Video.MimeCodec != nil && *fs.Info.Video.MimeCodec == transmux_codec + inc_lvl := quality.Height() < fs.Info.Video.Quality.Height() || + (quality.Height() == fs.Info.Video.Quality.Height() && !same_codec) + + if inc_lvl { master += "#EXT-X-STREAM-INF:" master += fmt.Sprintf("AVERAGE-BANDWIDTH=%d,", quality.AverageBitrate()) master += fmt.Sprintf("BANDWIDTH=%d,", quality.MaxBitrate()) master += fmt.Sprintf("RESOLUTION=%dx%d,", int(aspectRatio*float32(quality.Height())+0.5), quality.Height()) - master += "CODECS=\"avc1.640028\"," + master += fmt.Sprintf("CODECS=\"%s\",", transmux_codec) master += "AUDIO=\"audio\"," master += "CLOSED-CAPTIONS=NONE\n" master += fmt.Sprintf("./%s/index.m3u8\n", quality) diff --git a/transcoder/src/hwaccel.go b/transcoder/src/hwaccel.go index 683c5574..b5056a83 100644 --- a/transcoder/src/hwaccel.go +++ b/transcoder/src/hwaccel.go @@ -29,6 +29,8 @@ func DetectHardwareAccel() HwAccelT { // this is on by default and inserts keyframes where we don't want to (it also breaks force_key_frames) // we disable it to prevents whole scenes from behing removed due to the -f segment failing to find the corresonding keyframe "-sc_threshold", "0", + // force 8bits output (by default it keeps the same as the source but 10bits is not playable on some devices) + "-pix_fmt", "yuv420p", }, // we could put :force_original_aspect_ratio=decrease:force_divisible_by=2 here but we already calculate a correct width and // aspect ratio in our code so there is no need. @@ -48,6 +50,8 @@ func DetectHardwareAccel() HwAccelT { "-preset", preset, // the exivalent of -sc_threshold on nvidia. "-no-scenecut", "1", + // force 8bits output (by default it keeps the same as the source but 10bits is not playable on some devices) + "-pix_fmt", "yuv420p", }, // if the decode goes into system memory, we need to prepend the filters with "hwupload_cuda". // since we use hwaccel_output_format, decoded data stays in gpu memory so we must not specify it (it errors) @@ -64,12 +68,22 @@ func DetectHardwareAccel() HwAccelT { EncodeFlags: []string{ // h264_vaapi does not have any preset or scenecut flags. "-c:v", "h264_vaapi", - // if the hardware decoder could not work and fallbacked to soft decode, we need to instruct ffmpeg to - // upload back frames to gpu space (after converting them) - // see https://trac.ffmpeg.org/wiki/Hardware/VAAPI#Encoding for more info - // "-vf", "format=nv12|vaapi,hwupload", }, - ScaleFilter: "scale_vaapi=%d:%d", + // if the hardware decoder could not work and fallbacked to soft decode, we need to instruct ffmpeg to + // upload back frames to gpu space (after converting them) + // see https://trac.ffmpeg.org/wiki/Hardware/VAAPI#Encoding for more info + // we also need to force the format to be nv12 since 10bits is not supported via hwaccel. + // this filter is equivalent to this pseudocode: + // if (vaapi) { + // hwupload, passthrough, keep vaapi as is + // convert whatever to nv12 on GPU + // } else { + // convert whatever to nv12 on CPU + // hwupload to vaapi(nv12) + // convert whatever to nv12 on GPU // scale_vaapi doesn't support passthrough option, so it has to make a copy + // } + // See https://www.reddit.com/r/ffmpeg/comments/1bqn60w/hardware_accelerated_decoding_without_hwdownload/ for more info + ScaleFilter: "format=nv12|vaapi,hwupload,scale_vaapi=%d:%d:format=nv12", } case "qsv", "intel": return HwAccelT{ diff --git a/transcoder/src/info.go b/transcoder/src/info.go index de8413a5..06643e02 100644 --- a/transcoder/src/info.go +++ b/transcoder/src/info.go @@ -42,8 +42,10 @@ type MediaInfo struct { } type Video struct { - /// The codec of this stream (defined as the RFC 6381). + /// The human readable codec name. Codec string `json:"codec"` + /// The codec of this stream (defined as the RFC 6381). + MimeCodec *string `json:"mimeCodec"` /// The language of this stream (as a ISO-639-2 language code) Language *string `json:"language"` /// The max quality of this video track. @@ -63,8 +65,10 @@ type Audio struct { Title *string `json:"title"` /// The language of this stream (as a ISO-639-2 language code) Language *string `json:"language"` - /// The codec of this stream. + /// The human readable codec name. Codec string `json:"codec"` + /// The codec of this stream (defined as the RFC 6381). + MimeCodec *string `json:"mimeCodec"` /// Is this stream the default one of it's type? IsDefault bool `json:"isDefault"` /// Is this stream tagged as forced? (useful only for subtitles) @@ -263,28 +267,28 @@ func getInfo(path string, route string) (*MediaInfo, error) { Container: OrNull(mi.Parameter(mediainfo.StreamGeneral, 0, "Format")), Videos: Map(make([]Video, ParseUint(mi.Parameter(mediainfo.StreamVideo, 0, "StreamCount"))), func(_ Video, i int) Video { return Video{ - // This codec is not in the right format (does not include bitdepth...). - Codec: mi.Parameter(mediainfo.StreamVideo, 0, "Format"), - Language: OrNull(mi.Parameter(mediainfo.StreamVideo, 0, "Language")), - Quality: QualityFromHeight(ParseUint(mi.Parameter(mediainfo.StreamVideo, 0, "Height"))), - Width: ParseUint(mi.Parameter(mediainfo.StreamVideo, 0, "Width")), - Height: ParseUint(mi.Parameter(mediainfo.StreamVideo, 0, "Height")), + Codec: mi.Parameter(mediainfo.StreamVideo, i, "Format"), + MimeCodec: GetMimeCodec(mi, mediainfo.StreamVideo, i), + Language: OrNull(mi.Parameter(mediainfo.StreamVideo, i, "Language")), + Quality: QualityFromHeight(ParseUint(mi.Parameter(mediainfo.StreamVideo, i, "Height"))), + Width: ParseUint(mi.Parameter(mediainfo.StreamVideo, i, "Width")), + Height: ParseUint(mi.Parameter(mediainfo.StreamVideo, i, "Height")), Bitrate: ParseUint( Or( - mi.Parameter(mediainfo.StreamVideo, 0, "BitRate"), - mi.Parameter(mediainfo.StreamVideo, 0, "OverallBitRate"), - mi.Parameter(mediainfo.StreamVideo, 0, "BitRate_Nominal"), + mi.Parameter(mediainfo.StreamVideo, i, "BitRate"), + mi.Parameter(mediainfo.StreamVideo, i, "OverallBitRate"), + mi.Parameter(mediainfo.StreamVideo, i, "BitRate_Nominal"), ), ), } }), Audios: Map(make([]Audio, ParseUint(mi.Parameter(mediainfo.StreamAudio, 0, "StreamCount"))), func(_ Audio, i int) Audio { return Audio{ - Index: uint32(i), - Title: OrNull(mi.Parameter(mediainfo.StreamAudio, i, "Title")), - Language: OrNull(mi.Parameter(mediainfo.StreamAudio, i, "Language")), - // TODO: format is invalid. Channels count missing... + Index: uint32(i), + Title: OrNull(mi.Parameter(mediainfo.StreamAudio, i, "Title")), + Language: OrNull(mi.Parameter(mediainfo.StreamAudio, i, "Language")), Codec: mi.Parameter(mediainfo.StreamAudio, i, "Format"), + MimeCodec: GetMimeCodec(mi, mediainfo.StreamAudio, i), IsDefault: mi.Parameter(mediainfo.StreamAudio, i, "Default") == "Yes", IsForced: mi.Parameter(mediainfo.StreamAudio, i, "Forced") == "Yes", }