From 7e95370ce0510c2f409034a138b651924f6a0904 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Sun, 12 May 2024 02:02:05 +0200 Subject: [PATCH 1/9] Add whole file mimetype in /info --- transcoder/src/info.go | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/transcoder/src/info.go b/transcoder/src/info.go index 5a78a1f2..3a75c621 100644 --- a/transcoder/src/info.go +++ b/transcoder/src/info.go @@ -7,6 +7,7 @@ import ( "fmt" "io" "log" + "mime" "os" "path/filepath" "strconv" @@ -23,6 +24,8 @@ type MediaInfo struct { Path string `json:"path"` /// The extension currently used to store this video file Extension string `json:"extension"` + /// The whole mimetype (defined as the RFC 6381). ex: `video/mp4; codecs="avc1.640028, mp4a.40.2"` + MimeCodec *string `json:"mimeCodec"` /// The file size of the video file. Size uint64 `json:"size"` /// The length of the media in seconds. @@ -311,6 +314,24 @@ func getInfo(path string) (*MediaInfo, error) { return fmt.Sprintf("%s/%s/attachment/%s", Settings.RoutePrefix, base64.StdEncoding.EncodeToString([]byte(path)), font) }), } + var codecs []string + if len(ret.Videos) > 0 && ret.Videos[0].MimeCodec != nil { + codecs = append(codecs, *ret.Videos[0].MimeCodec) + } + if len(ret.Audios) > 0 && ret.Audios[0].MimeCodec != nil { + codecs = append(codecs, *ret.Audios[0].MimeCodec) + } + container := mime.TypeByExtension(fmt.Sprintf(".%s", ret.Extension)) + if container != "" { + if len(codecs) > 0 { + codecs_str := strings.Join(codecs, ", ") + mime := fmt.Sprintf("%s; codecs=\"%s\"", container, codecs_str) + ret.MimeCodec = &mime + } else { + ret.MimeCodec = &container + } + } + if len(ret.Videos) > 0 { ret.Video = &ret.Videos[0] } From 55f55db0306b536633fb3e92723f47f9127f640a Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Sun, 12 May 2024 02:53:20 +0200 Subject: [PATCH 2/9] Reset hls when leaving play page --- front/packages/ui/src/player/video.web.tsx | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/front/packages/ui/src/player/video.web.tsx b/front/packages/ui/src/player/video.web.tsx index 1646ec4c..faee33f3 100644 --- a/front/packages/ui/src/player/video.web.tsx +++ b/front/packages/ui/src/player/video.web.tsx @@ -173,6 +173,10 @@ const Video = forwardRef<{ seek: (value: number) => void }, VideoProps>(function }); }); } + return () => { + if (hls) hls.destroy(); + hls = null; + }; }, [source.uri, source.hls]); const mode = useAtomValue(playModeAtom); @@ -348,7 +352,7 @@ export const AudiosMenu = ({ useEffect(() => { if (!hls) return; hls.on(Hls.Events.AUDIO_TRACK_LOADED, rerender); - return () => hls!.off(Hls.Events.AUDIO_TRACK_LOADED, rerender); + return () => hls?.off(Hls.Events.AUDIO_TRACK_LOADED, rerender); }); if (!hls) return ; @@ -376,7 +380,7 @@ export const QualitiesMenu = (props: ComponentProps) => { useEffect(() => { if (!hls) return; hls.on(Hls.Events.LEVEL_SWITCHED, rerender); - return () => hls!.off(Hls.Events.LEVEL_SWITCHED, rerender); + return () => hls?.off(Hls.Events.LEVEL_SWITCHED, rerender); }); const levelName = (label: Level, auto?: boolean): string => { From 69a2821477c49a4e4dd628759ff85f960672db85 Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Sun, 12 May 2024 02:54:42 +0200 Subject: [PATCH 3/9] wip: CanPlay check --- .../models/src/resources/watch-info.ts | 5 ++++ front/packages/ui/src/player/index.tsx | 1 + front/packages/ui/src/player/state.tsx | 27 ++++++++++++++++--- front/packages/ui/src/player/video.tsx | 4 +++ front/packages/ui/src/player/video.web.tsx | 14 +++++++--- front/packages/ui/src/settings/playback.tsx | 5 +--- 6 files changed, 46 insertions(+), 10 deletions(-) diff --git a/front/packages/models/src/resources/watch-info.ts b/front/packages/models/src/resources/watch-info.ts index 1fd8ce09..82e624bd 100644 --- a/front/packages/models/src/resources/watch-info.ts +++ b/front/packages/models/src/resources/watch-info.ts @@ -149,6 +149,11 @@ export const WatchInfoP = z * The extension used to store this video file. */ extension: z.string(), + /** + * The whole mimetype (defined as the RFC 6381). + * ex: `video/mp4; codecs="avc1.640028, mp4a.40.2"` + */ + mimeCodec: z.string(), /** * The file size of the video file. */ diff --git a/front/packages/ui/src/player/index.tsx b/front/packages/ui/src/player/index.tsx index 54a1eb42..bd496e29 100644 --- a/front/packages/ui/src/player/index.tsx +++ b/front/packages/ui/src/player/index.tsx @@ -153,6 +153,7 @@ export const Player = ({ links={data?.links} audios={info?.audios} subtitles={info?.subtitles} + codec={info?.mimeCodec} setError={setPlaybackError} fonts={info?.fonts} startTime={startTime} diff --git a/front/packages/ui/src/player/state.tsx b/front/packages/ui/src/player/state.tsx index 59394dd2..9285fa34 100644 --- a/front/packages/ui/src/player/state.tsx +++ b/front/packages/ui/src/player/state.tsx @@ -100,6 +100,7 @@ export const Video = memo(function Video({ links, subtitles, audios, + codec, setError, fonts, startTime: startTimeP, @@ -108,6 +109,7 @@ export const Video = memo(function Video({ links?: Episode["links"]; subtitles?: Subtitle[]; audios?: Audio[]; + codec?: string; setError: (error: string | undefined) => void; fonts?: string[]; startTime?: number | null; @@ -133,9 +135,19 @@ export const Video = memo(function Video({ const getProgress = useAtomCallback(useCallback((get) => get(progressAtom), [])); const oldLinks = useRef(null); - useLayoutEffect(() => { + useEffect(() => { // Reset the state when a new video is loaded. - setSource((mode === PlayMode.Direct ? links?.direct : links?.hls) ?? null); + + let newMode = getLocalSetting("playmode", "direct") !== "auto" ? PlayMode.Direct : PlayMode.Hls; + // Only allow direct play if the device supports it + console.log(codec, ref.current, ref.current?.canPlay?.(codec!)) + if (newMode === PlayMode.Direct && codec && ref.current?.canPlay?.(codec) === false) { + console.log(`Browser can't natively play ${codec}, switching to hls stream.`); + newMode = PlayMode.Hls; + } + setPlayMode(newMode); + + setSource((newMode === PlayMode.Direct ? links?.direct : links?.hls) ?? null); setLoad(true); if (oldLinks.current !== links) { setPrivateProgress(startTime.current ?? 0); @@ -146,7 +158,16 @@ export const Video = memo(function Video({ } oldLinks.current = links; setPlay(true); - }, [mode, links, setLoad, setPrivateProgress, setPublicProgress, setPlay, getProgress]); + }, [ + links, + codec, + setLoad, + setPrivateProgress, + setPublicProgress, + setPlay, + getProgress, + setPlayMode, + ]); const account = useAccount(); const defaultSubLanguage = account?.settings.subtitleLanguage; diff --git a/front/packages/ui/src/player/video.tsx b/front/packages/ui/src/player/video.tsx index 3f4d5dbc..ed8e7567 100644 --- a/front/packages/ui/src/player/video.tsx +++ b/front/packages/ui/src/player/video.tsx @@ -29,6 +29,10 @@ declare module "react-native-video" { export type VideoProps = Omit & { source: { uri: string; hls: string | null; startPosition?: number }; }; + + interface VideoRef { + canPlay?: (codec: string) => boolean; + } } export * from "react-native-video"; diff --git a/front/packages/ui/src/player/video.web.tsx b/front/packages/ui/src/player/video.web.tsx index faee33f3..9f7106dd 100644 --- a/front/packages/ui/src/player/video.web.tsx +++ b/front/packages/ui/src/player/video.web.tsx @@ -104,7 +104,10 @@ const initHls = (): Hls => { return hls; }; -const Video = forwardRef<{ seek: (value: number) => void }, VideoProps>(function Video( +const Video = forwardRef< + { seek: (value: number) => void; canPlay: (codec: string) => boolean }, + VideoProps +>(function Video( { source, paused, @@ -124,6 +127,8 @@ const Video = forwardRef<{ seek: (value: number) => void }, VideoProps>(function const ref = useRef(null); const oldHls = useRef(null); const { css } = useYoshiki(); + const errorHandler = useRef(onError); + errorHandler.current = onError; useImperativeHandle( forwaredRef, @@ -131,6 +136,10 @@ const Video = forwardRef<{ seek: (value: number) => void }, VideoProps>(function seek: (value: number) => { if (ref.current) ref.current.currentTime = value; }, + canPlay: (codec: string) => { + if (!ref.current) return false; + return !!ref.current.canPlayType(codec); + }, }), [], ); @@ -148,7 +157,6 @@ const Video = forwardRef<{ seek: (value: number) => void }, VideoProps>(function const subtitle = useAtomValue(subtitleAtom); useSubtitle(ref, subtitle, fonts); - // biome-ignore lint/correctness/useExhaustiveDependencies: onError changes should not restart the playback. useLayoutEffect(() => { if (!ref?.current || !source.uri) return; if (!hls || oldHls.current !== source.hls) { @@ -168,7 +176,7 @@ const Video = forwardRef<{ seek: (value: number) => void }, VideoProps>(function hls.on(Hls.Events.ERROR, (_, d) => { if (!d.fatal || !hls?.media) return; console.warn("Hls error", d); - onError?.call(null, { + errorHandler.current?.({ error: { errorString: d.reason ?? d.error?.message ?? "Unknown hls error" }, }); }); diff --git a/front/packages/ui/src/settings/playback.tsx b/front/packages/ui/src/settings/playback.tsx index 1ceb5b74..55450864 100644 --- a/front/packages/ui/src/settings/playback.tsx +++ b/front/packages/ui/src/settings/playback.tsx @@ -55,10 +55,7 @@ export const PlaybackSettings = () => {