From de7aa58195de02733c0521880e3b4b3c69f7d8ca Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Thu, 20 Jul 2023 12:32:47 +0900 Subject: [PATCH] Fix subtitles on android with transcode --- front/packages/ui/src/player/index.tsx | 6 ++-- front/packages/ui/src/player/state.tsx | 18 +--------- front/packages/ui/src/player/video.tsx | 38 +++++++++++++++++----- front/packages/ui/src/player/video.web.tsx | 1 - transcoder/src/state.rs | 27 +++++---------- 5 files changed, 42 insertions(+), 48 deletions(-) diff --git a/front/packages/ui/src/player/index.tsx b/front/packages/ui/src/player/index.tsx index d894fd01..22a80522 100644 --- a/front/packages/ui/src/player/index.tsx +++ b/front/packages/ui/src/player/index.tsx @@ -220,8 +220,10 @@ export const Player: QueryPage<{ slug: string }> = ({ slug }) => { if (e.nativeEvent.pointerType === "mouse") setHover(false); }} onPointerDown={(e) => { - if (!displayControls) onPointerDown(e); - if (Platform.OS === "web") e.preventDefault(); + if (!displayControls) { + onPointerDown(e); + if (Platform.OS === "web") e.preventDefault(); + } }} onMenuOpen={() => setMenuOpen(true)} onMenuClose={() => { diff --git a/front/packages/ui/src/player/state.tsx b/front/packages/ui/src/player/state.tsx index 2450f62f..b4cfb1da 100644 --- a/front/packages/ui/src/player/state.tsx +++ b/front/packages/ui/src/player/state.tsx @@ -123,8 +123,6 @@ export const Video = memo(function _Video({ return () => document.removeEventListener("fullscreenchange", handler); }); - const subtitle = useAtomValue(subtitleAtom); - if (!source || !links) return null; return ( ({ - type: "text/x-ssa" as any, //MimeTypes[x.codec], - uri: x.link!, - title: x.title!, - language: x.language!, - }))} - selectedTextTrack={ - subtitle - ? { - type: "index", - value: subtitle.trackIndex, - } - : { type: "disabled" } - } fonts={fonts} + subtitles={subtitles} onMediaUnsupported={() => { if (mode == PlayMode.Direct) setPlayMode(PlayMode.Hls); }} - // TODO: textTracks: external subtitles /> ); }); diff --git a/front/packages/ui/src/player/video.tsx b/front/packages/ui/src/player/video.tsx index a70e0e1d..1d369dac 100644 --- a/front/packages/ui/src/player/video.tsx +++ b/front/packages/ui/src/player/video.tsx @@ -26,22 +26,29 @@ declare module "react-native-video" { } export type VideoProps = Omit & { source: { uri: string; hls: string }; + subtitles?: WatchItem["subtitles"]; }; } export * from "react-native-video"; -import { Font, getToken } from "@kyoo/models"; +import { Font, getToken, WatchItem } from "@kyoo/models"; import { IconButton, Menu } from "@kyoo/primitives"; import { ComponentProps, forwardRef, useEffect, useRef } from "react"; import { atom, useAtom, useAtomValue, useSetAtom } from "jotai"; import NativeVideo, { OnLoadData, VideoProps } from "react-native-video"; import { useTranslation } from "react-i18next"; -import { PlayMode, playModeAtom } from "./state"; +import { PlayMode, playModeAtom, subtitleAtom } from "./state"; import uuid from "react-native-uuid"; import { Pressable } from "react-native"; import { useYoshiki } from "yoshiki/native"; +const MimeTypes: Map = new Map([ + ["subrip", "application/x-subrip"], + ["ass", "text/x-ssa"], + ["vtt", "text/vtt"], +]); + const infoAtom = atom(null); const videoAtom = atom(0); const audioAtom = atom(0); @@ -49,7 +56,7 @@ const audioAtom = atom(0); const clientId = uuid.v4() as string; const Video = forwardRef(function _NativeVideo( - { onLoad, source, onPointerDown, ...props }, + { onLoad, source, onPointerDown, subtitles, ...props }, ref, ) { const { css } = useYoshiki(); @@ -57,9 +64,7 @@ const Video = forwardRef(function _NativeVideo( const setInfo = useSetAtom(infoAtom); const video = useAtomValue(videoAtom); const audio = useAtomValue(audioAtom); - - const info = useAtomValue(infoAtom); - console.log(info); + const subtitle = useAtomValue(subtitleAtom); useEffect(() => { async function run() { @@ -73,7 +78,7 @@ const Video = forwardRef(function _NativeVideo( focusable={false} onPress={() => onPointerDown?.({ nativeEvent: { pointerType: "pointer" } } as any)} {...css({ flexGrow: 1, flexShrink: 1 })} - > + > (function _NativeVideo( headers: { Authorization: `Bearer: ${token.current}`, "X-CLIENT-ID": clientId, - } + }, }} onLoad={(info) => { setInfo(info); @@ -89,8 +94,23 @@ const Video = forwardRef(function _NativeVideo( }} selectedVideoTrack={video === -1 ? { type: "auto" } : { type: "resolution", value: video }} selectedAudioTrack={{ type: "index", value: audio }} + textTracks={subtitles?.map((x) => ({ + type: MimeTypes.get(x.codec) as any, + uri: x.link!, + title: x.title ?? "Unknown", + language: x.language ?? "Unknown", + }))} + selectedTextTrack={ + subtitle + ? { + type: "index", + value: subtitles?.indexOf(subtitle), + } + : { type: "disabled" } + } {...props} - /> + /> + ); }); diff --git a/front/packages/ui/src/player/video.web.tsx b/front/packages/ui/src/player/video.web.tsx index 02bdb82a..ffd21b07 100644 --- a/front/packages/ui/src/player/video.web.tsx +++ b/front/packages/ui/src/player/video.web.tsx @@ -107,7 +107,6 @@ const Video = forwardRef<{ seek: (value: number) => void }, VideoProps>(function ref.current.volume = Math.max(0, Math.min(volume, 100)) / 100; }, [volume]); - // This should use the selectedTextTrack prop instead of the atom but this is so much simpler const subtitle = useAtomValue(subtitleAtom); useSubtitle(ref, subtitle, fonts); diff --git a/transcoder/src/state.rs b/transcoder/src/state.rs index 2ecc70c5..a91091fc 100644 --- a/transcoder/src/state.rs +++ b/transcoder/src/state.rs @@ -39,7 +39,10 @@ impl Transcoder { // TODO: Find codecs in the RFC 6381 format. // master.push_str("CODECS=\"avc1.640028\","); // TODO: With multiple audio qualities, maybe switch qualities depending on the video quality. - master.push_str("AUDIO=\"audio\"\n"); + master.push_str("AUDIO=\"audio\","); + // Not adding this attribute result in some players (like android's exoplayer) to + // assume a non existant closed caption exist + master.push_str("CLOSED-CAPTIONS=NONE\n"); master.push_str(format!("./{}/index.m3u8\n", Quality::Original).as_str()); } @@ -61,7 +64,10 @@ impl Transcoder { ); master.push_str("CODECS=\"avc1.640028\","); // TODO: With multiple audio qualities, maybe switch qualities depending on the video quality. - master.push_str("AUDIO=\"audio\"\n"); + master.push_str("AUDIO=\"audio\","); + // Not adding this attribute result in some players (like android's exoplayer) to + // assume a non existant closed caption exist + master.push_str("CLOSED-CAPTIONS=NONE\n"); master.push_str(format!("./{}/index.m3u8\n", quality).as_str()); } for audio in info.audios { @@ -87,23 +93,6 @@ impl Transcoder { master.push_str(format!("URI=\"./audio/{}/index.m3u8\"\n", audio.index).as_str()); } - for subtitle in info.subtitles { - master.push_str("#EXT-X-MEDIA:TYPE=SUBTITLES,"); - master.push_str("GROUP-ID=\"subtitles\","); - if let Some(language) = subtitle.language.clone() { - master.push_str(format!("LANGUAGE=\"{}\",", language).as_str()); - } - if let Some(title) = subtitle.title { - master.push_str(format!("NAME=\"{}\",", title).as_str()); - } else if let Some(language) = subtitle.language { - master.push_str(format!("NAME=\"{}\",", language).as_str()); - } else { - master.push_str(format!("NAME=\"Subtitle {}\",", subtitle.index).as_str()); - } - master.push_str("URI=\"https://kyoo.sdg.moe/api/subtitle/akudama-drive-s1e1.eng.subtitle\"\n"); - // master.push_str(format!("URI=\"./subtitles/{}/index.m3u8\"\n", subtitle.index).as_str()); - } - Some(master) }