Add subtitles support for the web

front/packages/ui/src/player/components/right-buttons.tsx

@@ -21,7 +21,7 @@
 import { Font, Track } from "@kyoo/models";
 import { IconButton, tooltip, Menu, ts } from "@kyoo/primitives";
 import { useAtom, useSetAtom } from "jotai";
-import { useEffect } from "react";
+import { useEffect, useState } from "react";
 import { Platform, View } from "react-native";
 import { useTranslation } from "react-i18next";
 import ClosedCaption from "@material-symbols/svg-400/rounded/closed_caption-fill.svg";
@@ -49,7 +49,7 @@ export const RightButtons = ({
 	const { t } = useTranslation();
 	const [isFullscreen, setFullscreen] = useAtom(fullscreenAtom);
 	const setSubAtom = useSetAtom(subtitleAtom);
-	const [selectedSubtitle, setSubtitle] = useParam("subtitle");
+	const [selectedSubtitle, setSubtitle] = useState<string | undefined>(undefined);
 
 	useEffect(() => {
 		const sub =

front/packages/ui/src/player/index.tsx

@@ -188,6 +188,7 @@ export const Player: QueryPage<{ slug: string }> = ({ slug }) => {
 					<Video
 						links={data?.link}
 						setError={setPlaybackError}
+						fonts={data?.fonts}
 						{...css(StyleSheet.absoluteFillObject)}
 						// onEnded={() => {
 						// 	if (!data) return;

front/packages/ui/src/player/state.tsx

@@ -18,7 +18,7 @@
  * along with Kyoo. If not, see <https://www.gnu.org/licenses/>.
  */
 
-import { Track, WatchItem } from "@kyoo/models";
+import { Font, Track, WatchItem } from "@kyoo/models";
 import { atom, useAtomValue, useSetAtom } from "jotai";
 import { useEffect, useLayoutEffect, useRef } from "react";
 import NativeVideo, { VideoProperties as VideoProps } from "./video";
@@ -73,8 +73,12 @@ export const subtitleAtom = atom<Track | null>(null);
 export const Video = ({
 	links,
 	setError,
+	fonts,
 	...props
-}: { links?: WatchItem["link"]; setError: (error: string | undefined) => void } & VideoProps) => {
+}: {
+	links?: WatchItem["link"];
+	setError: (error: string | undefined) => void;
+} & VideoProps) => {
 	const ref = useRef<NativeVideo | null>(null);
 	const isPlaying = useAtomValue(playAtom);
 	const setLoad = useSetAtom(loadAtom);
@@ -158,6 +162,7 @@ export const Video = ({
 					  }
 					: { type: "disabled" }
 			}
+			fonts={fonts}
 			// TODO: textTracks: external subtitles
 			// onError: () => {
 			// 	if (player?.current?.error?.code === MediaError.MEDIA_ERR_SRC_NOT_SUPPORTED)
@@ -166,81 +171,3 @@ export const Video = ({
 		/>
 	);
 };
-
-// const htmlTrackAtom = atom<HTMLTrackElement | null>(null);
-// const suboctoAtom = atom<SubtitleOctopus | null>(null);
-// export const [_subtitleAtom, subtitleAtom] = bakedAtom<
-// 	Track | null,
-// 	{ track: Track; fonts: Font[] } | null
-// >(null, (get, set, value, baked) => {
-// 	const removeHtmlSubtitle = () => {
-// 		const htmlTrack = get(htmlTrackAtom);
-// 		if (htmlTrack) htmlTrack.remove();
-// 		set(htmlTrackAtom, null);
-// 	};
-// 	const removeOctoSub = () => {
-// 		const subocto = get(suboctoAtom);
-// 		if (subocto) {
-// 			subocto.freeTrack();
-// 			subocto.dispose();
-// 		}
-// 		set(suboctoAtom, null);
-// 	};
-
-// 	const player = get(playerAtom);
-// 	if (!player?.current) return;
-
-// 	if (get(baked)?.id === value?.track.id) return;
-
-// 	set(baked, value?.track ?? null);
-// 	if (!value) {
-// 		removeHtmlSubtitle();
-// 		removeOctoSub();
-// 	} else if (value.track.codec === "vtt" || value.track.codec === "subrip") {
-// 		removeOctoSub();
-// 		if (player.current.textTracks.length > 0) player.current.textTracks[0].mode = "hidden";
-// 		const track: HTMLTrackElement = get(htmlTrackAtom) ?? document.createElement("track");
-// 		track.kind = "subtitles";
-// 		track.label = value.track.displayName;
-// 		if (value.track.language) track.srclang = value.track.language;
-// 		track.src = value.track.link! + ".vtt";
-// 		track.className = "subtitle_container";
-// 		track.default = true;
-// 		track.onload = () => {
-// 			if (player.current) player.current.textTracks[0].mode = "showing";
-// 		};
-// 		if (!get(htmlTrackAtom)) player.current.appendChild(track);
-// 		set(htmlTrackAtom, track);
-// 	} else if (value.track.codec === "ass") {
-// 		removeHtmlSubtitle();
-// 		removeOctoSub();
-// 		set(
-// 			suboctoAtom,
-// 			new SubtitleOctopus({
-// 				video: player.current,
-// 				subUrl: value.track.link!,
-// 				workerUrl: "/_next/static/chunks/subtitles-octopus-worker.js",
-// 				legacyWorkerUrl: "/_next/static/chunks/subtitles-octopus-worker-legacy.js",
-// 				fonts: value.fonts?.map((x) => x.link),
-// 				renderMode: "wasm-blend",
-// 			}),
-// 		);
-// 	}
-// });
-
-// const { useParam } = createParam<{ subtitle: string }>();
-
-// export const useSubtitleController = (
-// 	player: RefObject<HTMLVideoElement>,
-// 	subtitles?: Track[],
-// 	fonts?: Font[],
-// ) => {
-// 	const [subtitle] = useParam("subtitle");
-// 	const selectSubtitle = useSetAtom(subtitleAtom);
-
-// 	const newSub = subtitles?.find((x) => x.language === subtitle);
-// 	useEffect(() => {
-// 		if (newSub === undefined) return;
-// 		selectSubtitle({ track: newSub, fonts: fonts ?? [] });
-// 	}, [player.current?.src, newSub, fonts, selectSubtitle]);
-// };

front/packages/ui/src/player/subtitle-octopus.d.ts

@@ -26,8 +26,8 @@ declare module "libass-wasm" {
 		video?: HTMLVideoElement;
 
 		/**
-		 * The canvas to render the subtitles to. If none is given it will create a new canvas and insert
+		 * The canvas to render the subtitles to. If none is given it will create a new canvas and
-		 * it as a sibling of the video element (only if the video element exists)
+		 * insert it as a sibling of the video element (only if the video element exists)
 		 */
 		canvas?: HTMLCanvasElement;
 
@@ -71,6 +71,19 @@ declare module "libass-wasm" {
 		 */
 		debug?: boolean;
 
+		/**
+		* The default font.
+		*/
+		fallbackFont?: string;
+
+		/**
+		 * A boolean, whether to load files in a lazy way via FS.createLazyFile(). Requires
+		 * Access-Control-Expose-Headers for Accept-Ranges, Content-Length, and Content-Encoding. If
+		 * encoding is compressed or length is not set, file will be fully fetched instead of just a
+		 * HEAD request.
+		 */
+		lazyFileLoading?: boolean;
+
 		/**
 		 * Function that's called when SubtitlesOctopus is ready
 		 */
@@ -86,7 +99,7 @@ declare module "libass-wasm" {
 		/**
 		 * Change the render mode
 		 *
-		 * @default wasm-blend
+		 * @default wasm
 		 */
 		renderMode?: "js-blend" | "wasm-blend" | "lossy";
 	}
@@ -127,8 +140,8 @@ declare module "libass-wasm" {
 		setTrack(content: string): void;
 
 		/**
-		 * This simply removes the subtitles. You can use {@link setTrackByUrl} or {@link setTrack} methods
+		 * This simply removes the subtitles. You can use {@link setTrackByUrl} or {@link setTrack}
-		 * to set a new subtitle file to be displayed.
+		 * methods to set a new subtitle file to be displayed.
 		 */
 		freeTrack(): void;
 

front/packages/ui/src/player/video.web.tsx

@@ -18,19 +18,27 @@
  * along with Kyoo. If not, see <https://www.gnu.org/licenses/>.
  */
 
-import { forwardRef, useEffect, useImperativeHandle, useRef } from "react";
+import { Font, Track } from "@kyoo/models";
+import { forwardRef, RefObject, useEffect, useImperativeHandle, useRef } from "react";
 import { VideoProperties } from "react-native-video";
+import { useAtomValue } from "jotai";
 import { useYoshiki } from "yoshiki";
-// import SubtitleOctopus from "libass-wasm";
+import SubtitleOctopus from "libass-wasm";
+import { subtitleAtom } from "./state";
 // import Hls from "hls.js";
 
 // let hls: Hls | null = null;
 
 // TODO fallback via links and hls.
-// TODO: Subtitle (vtt, srt and ass)
+
+declare module "react-native-video" {
+	interface VideoProperties {
+		fonts?: Font[];
+	}
+}
 
 const Video = forwardRef<{ seek: (value: number) => void }, VideoProperties>(function _Video(
-	{ source, paused, muted, volume, onBuffer, onLoad, onProgress, onError },
+	{ source, paused, muted, volume, onBuffer, onLoad, onProgress, onError, fonts },
 	forwaredRef,
 ) {
 	const ref = useRef<HTMLVideoElement>(null);
@@ -55,6 +63,10 @@ const Video = forwardRef<{ seek: (value: number) => void }, VideoProperties>(fun
 		ref.current.volume = Math.max(0, Math.min(volume, 100)) / 100;
 	}, [volume]);
 
+	// This should use the selectedTextTrack prop instead of the atom but this is so much simpler
+	const subtitle = useAtomValue(subtitleAtom);
+	useSubtitle(ref, subtitle, fonts);
+
 	return (
 		<video
 			ref={ref}
@@ -88,3 +100,60 @@ const Video = forwardRef<{ seek: (value: number) => void }, VideoProperties>(fun
 });
 
 export default Video;
+
+let htmlTrack: HTMLTrackElement | null;
+let subOcto: SubtitleOctopus | null;
+const useSubtitle = (player: RefObject<HTMLVideoElement>, value: Track | null, fonts?: Font[]) => {
+	useEffect(() => {
+		if (!player.current) return;
+
+		const removeHtmlSubtitle = () => {
+			if (htmlTrack) htmlTrack.remove();
+			htmlTrack = null;
+		};
+
+		const removeOctoSub = () => {
+			if (subOcto) {
+				subOcto.freeTrack();
+				subOcto.dispose();
+			}
+			subOcto = null;
+		};
+
+		if (!value) {
+			removeHtmlSubtitle();
+			removeOctoSub();
+		} else if (value.codec === "vtt" || value.codec === "subrip") {
+			removeOctoSub();
+			if (player.current.textTracks.length > 0) player.current.textTracks[0].mode = "hidden";
+			const track: HTMLTrackElement = htmlTrack ?? document.createElement("track");
+			track.kind = "subtitles";
+			track.label = value.displayName;
+			if (value.language) track.srclang = value.language;
+			track.src = value.link! + ".vtt";
+			track.className = "subtitle_container";
+			track.default = true;
+			track.onload = () => {
+				if (player.current) player.current.textTracks[0].mode = "showing";
+			};
+			if (!htmlTrack) {
+				player.current.appendChild(track);
+				htmlTrack = track;
+			}
+		} else if (value.codec === "ass") {
+			removeHtmlSubtitle();
+			removeOctoSub();
+			subOcto = new SubtitleOctopus({
+				video: player.current,
+				subUrl: value.link!,
+				workerUrl: "/_next/static/chunks/subtitles-octopus-worker.js",
+				legacyWorkerUrl: "/_next/static/chunks/subtitles-octopus-worker-legacy.js",
+				fallbackFont: "/default.woff2",
+				fonts: fonts?.map((x) => x.link),
+				// availableFonts: fonts ? Object.fromEntries(fonts.map((x) => [x.slug, x.link])) : undefined,
+				// lazyFileLoading: true,
+				renderMode: "wasm-blend",
+			});
+		}
+	}, [player, value, fonts]);
+};