Fix subtitles on android with transcode

front/packages/ui/src/player/index.tsx

@@ -220,8 +220,10 @@ export const Player: QueryPage<{ slug: string }> = ({ slug }) => {
 						if (e.nativeEvent.pointerType === "mouse") setHover(false);
 					}}
 					onPointerDown={(e) => {
-						if (!displayControls) onPointerDown(e);
+						if (!displayControls) {
-						if (Platform.OS === "web") e.preventDefault();
+							onPointerDown(e);
+							if (Platform.OS === "web") e.preventDefault();
+						}
 					}}
 					onMenuOpen={() => setMenuOpen(true)}
 					onMenuClose={() => {

front/packages/ui/src/player/state.tsx

@@ -123,8 +123,6 @@ export const Video = memo(function _Video({
 		return () => document.removeEventListener("fullscreenchange", handler);
 	});
 
-	const subtitle = useAtomValue(subtitleAtom);
-
 	if (!source || !links) return null;
 	return (
 		<NativeVideo
@@ -151,25 +149,11 @@ export const Video = memo(function _Video({
 				setDuration(info.duration);
 			}}
 			onPlayPause={setPlay}
-			textTracks={subtitles?.map(x => ({
-				type: "text/x-ssa" as any, //MimeTypes[x.codec],
-				uri: x.link!,
-				title: x.title!,
-				language: x.language!,
-			}))}
-			selectedTextTrack={
-				subtitle
-					? {
-							type: "index",
-							value: subtitle.trackIndex,
-					  }
-					: { type: "disabled" }
-			}
 			fonts={fonts}
+			subtitles={subtitles}
 			onMediaUnsupported={() => {
 				if (mode == PlayMode.Direct) setPlayMode(PlayMode.Hls);
 			}}
-			// TODO: textTracks: external subtitles
 		/>
 	);
 });

front/packages/ui/src/player/video.tsx

@@ -26,22 +26,29 @@ declare module "react-native-video" {
 	}
 	export type VideoProps = Omit<VideoProperties, "source"> & {
 		source: { uri: string; hls: string };
+		subtitles?: WatchItem["subtitles"];
 	};
 }
 
 export * from "react-native-video";
 
-import { Font, getToken } from "@kyoo/models";
+import { Font, getToken, WatchItem } from "@kyoo/models";
 import { IconButton, Menu } from "@kyoo/primitives";
 import { ComponentProps, forwardRef, useEffect, useRef } from "react";
 import { atom, useAtom, useAtomValue, useSetAtom } from "jotai";
 import NativeVideo, { OnLoadData, VideoProps } from "react-native-video";
 import { useTranslation } from "react-i18next";
-import { PlayMode, playModeAtom } from "./state";
+import { PlayMode, playModeAtom, subtitleAtom } from "./state";
 import uuid from "react-native-uuid";
 import { Pressable } from "react-native";
 import { useYoshiki } from "yoshiki/native";
 
+const MimeTypes: Map<string, string> = new Map([
+	["subrip", "application/x-subrip"],
+	["ass", "text/x-ssa"],
+	["vtt", "text/vtt"],
+]);
+
 const infoAtom = atom<OnLoadData | null>(null);
 const videoAtom = atom(0);
 const audioAtom = atom(0);
@@ -49,7 +56,7 @@ const audioAtom = atom(0);
 const clientId = uuid.v4() as string;
 
 const Video = forwardRef<NativeVideo, VideoProps>(function _NativeVideo(
-	{ onLoad, source, onPointerDown, ...props },
+	{ onLoad, source, onPointerDown, subtitles, ...props },
 	ref,
 ) {
 	const { css } = useYoshiki();
@@ -57,9 +64,7 @@ const Video = forwardRef<NativeVideo, VideoProps>(function _NativeVideo(
 	const setInfo = useSetAtom(infoAtom);
 	const video = useAtomValue(videoAtom);
 	const audio = useAtomValue(audioAtom);
-
+	const subtitle = useAtomValue(subtitleAtom);
-	const info = useAtomValue(infoAtom);
-	console.log(info);
 
 	useEffect(() => {
 		async function run() {
@@ -73,7 +78,7 @@ const Video = forwardRef<NativeVideo, VideoProps>(function _NativeVideo(
 			focusable={false}
 			onPress={() => onPointerDown?.({ nativeEvent: { pointerType: "pointer" } } as any)}
 			{...css({ flexGrow: 1, flexShrink: 1 })}
-			>
+		>
 			<NativeVideo
 				ref={ref}
 				source={{
@@ -81,7 +86,7 @@ const Video = forwardRef<NativeVideo, VideoProps>(function _NativeVideo(
 					headers: {
 						Authorization: `Bearer: ${token.current}`,
 						"X-CLIENT-ID": clientId,
-					}
+					},
 				}}
 				onLoad={(info) => {
 					setInfo(info);
@@ -89,8 +94,23 @@ const Video = forwardRef<NativeVideo, VideoProps>(function _NativeVideo(
 				}}
 				selectedVideoTrack={video === -1 ? { type: "auto" } : { type: "resolution", value: video }}
 				selectedAudioTrack={{ type: "index", value: audio }}
+				textTracks={subtitles?.map((x) => ({
+					type: MimeTypes.get(x.codec) as any,
+					uri: x.link!,
+					title: x.title ?? "Unknown",
+					language: x.language ?? "Unknown",
+				}))}
+				selectedTextTrack={
+					subtitle
+						? {
+								type: "index",
+								value: subtitles?.indexOf(subtitle),
+						  }
+						: { type: "disabled" }
+				}
 				{...props}
-			/></Pressable>
+			/>
+		</Pressable>
 	);
 });
 

front/packages/ui/src/player/video.web.tsx

@@ -107,7 +107,6 @@ const Video = forwardRef<{ seek: (value: number) => void }, VideoProps>(function
 		ref.current.volume = Math.max(0, Math.min(volume, 100)) / 100;
 	}, [volume]);
 
-	// This should use the selectedTextTrack prop instead of the atom but this is so much simpler
 	const subtitle = useAtomValue(subtitleAtom);
 	useSubtitle(ref, subtitle, fonts);
 

transcoder/src/state.rs

@@ -39,7 +39,10 @@ impl Transcoder {
 			// TODO: Find codecs in the RFC 6381 format.
 			// master.push_str("CODECS=\"avc1.640028\",");
 			// TODO: With multiple audio qualities, maybe switch qualities depending on the video quality.
-			master.push_str("AUDIO=\"audio\"\n");
+			master.push_str("AUDIO=\"audio\",");
+			// Not adding this attribute result in some players (like android's exoplayer) to
+			// assume a non existant closed caption exist
+			master.push_str("CLOSED-CAPTIONS=NONE\n");
 			master.push_str(format!("./{}/index.m3u8\n", Quality::Original).as_str());
 		}
 
@@ -61,7 +64,10 @@ impl Transcoder {
 			);
 			master.push_str("CODECS=\"avc1.640028\",");
 			// TODO: With multiple audio qualities, maybe switch qualities depending on the video quality.
-			master.push_str("AUDIO=\"audio\"\n");
+			master.push_str("AUDIO=\"audio\",");
+			// Not adding this attribute result in some players (like android's exoplayer) to
+			// assume a non existant closed caption exist
+			master.push_str("CLOSED-CAPTIONS=NONE\n");
 			master.push_str(format!("./{}/index.m3u8\n", quality).as_str());
 		}
 		for audio in info.audios {
@@ -87,23 +93,6 @@ impl Transcoder {
 			master.push_str(format!("URI=\"./audio/{}/index.m3u8\"\n", audio.index).as_str());
 		}
 
-		for subtitle in info.subtitles {
-			master.push_str("#EXT-X-MEDIA:TYPE=SUBTITLES,");
-			master.push_str("GROUP-ID=\"subtitles\",");
-			if let Some(language) = subtitle.language.clone() {
-				master.push_str(format!("LANGUAGE=\"{}\",", language).as_str());
-			}
-			if let Some(title) = subtitle.title {
-				master.push_str(format!("NAME=\"{}\",", title).as_str());
-			} else if let Some(language) = subtitle.language {
-				master.push_str(format!("NAME=\"{}\",", language).as_str());
-			} else {
-				master.push_str(format!("NAME=\"Subtitle {}\",", subtitle.index).as_str());
-			}
-			master.push_str("URI=\"https://kyoo.sdg.moe/api/subtitle/akudama-drive-s1e1.eng.subtitle\"\n");
-			// master.push_str(format!("URI=\"./subtitles/{}/index.m3u8\"\n", subtitle.index).as_str());
-		}
-
 		Some(master)
 	}