first version of a really buggy and ugly caption selector and renderer

apps/expo/package.json

@@ -45,12 +45,14 @@
     "react-native-context-menu-view": "^1.14.1",
     "react-native-css-interop": "~0.0.22",
     "react-native-gesture-handler": "~2.14.1",
+    "react-native-modal": "^13.0.1",
     "react-native-quick-base64": "^2.0.8",
     "react-native-quick-crypto": "^0.6.1",
     "react-native-reanimated": "~3.6.2",
     "react-native-safe-area-context": "~4.8.2",
     "react-native-screens": "~3.29.0",
     "react-native-web": "^0.19.10",
+    "subsrt-ts": "^2.1.2",
     "tailwind-merge": "^2.2.1",
     "zustand": "^4.4.7"
   },

apps/expo/src/app/videoPlayer/index.tsx

@@ -22,6 +22,7 @@ import {
 
 import type { ItemData } from "~/components/item/item";
 import type { HeaderData } from "~/components/player/Header";
+import { CaptionRenderer } from "~/components/player/CaptionRenderer";
 import { ControlsOverlay } from "~/components/player/ControlsOverlay";
 import { Text } from "~/components/ui/Text";
 import { useBrightness } from "~/hooks/player/useBrightness";
@@ -69,9 +70,10 @@ const VideoPlayer: React.FC<VideoPlayerProps> = ({ data }) => {
   const router = useRouter();
   const scale = useSharedValue(1);
 
+  const isIdle = usePlayerStore((state) => state.interface.isIdle);
+  const setStream = usePlayerStore((state) => state.setStream);
   const setVideoRef = usePlayerStore((state) => state.setVideoRef);
   const setStatus = usePlayerStore((state) => state.setStatus);
-  const isIdle = usePlayerStore((state) => state.interface.isIdle);
   const setIsIdle = usePlayerStore((state) => state.setIsIdle);
   const presentFullscreenPlayer = usePlayerStore(
     (state) => state.presentFullscreenPlayer,
@@ -160,6 +162,8 @@ const VideoPlayer: React.FC<VideoPlayerProps> = ({ data }) => {
 
       const { item, stream, media } = data;
 
+      setStream(stream);
+
       setHeaderData({
         title: item.title,
         year: item.year,
@@ -252,6 +256,7 @@ const VideoPlayer: React.FC<VideoPlayerProps> = ({ data }) => {
             <Text className="font-bold">Brightness: {debouncedBrightness}</Text>
           </View>
         )}
+        <CaptionRenderer />
       </View>
     </GestureDetector>
   );

apps/expo/src/components/player/BottomControls.tsx

@@ -3,15 +3,18 @@ import { TouchableOpacity, View } from "react-native";
 
 import { usePlayerStore } from "~/stores/player/store";
 import { Text } from "../ui/Text";
+import { CaptionsSelector } from "./CaptionsSelector";
 import { Controls } from "./Controls";
 import { ProgressBar } from "./ProgressBar";
 import { mapMillisecondsToTime } from "./utils";
 
 export const BottomControls = () => {
   const status = usePlayerStore((state) => state.status);
+  const setIsIdle = usePlayerStore((state) => state.setIsIdle);
   const [showRemaining, setShowRemaining] = useState(false);
 
   const toggleTimeDisplay = () => {
+    setIsIdle(false);
     setShowRemaining(!showRemaining);
   };
 
@@ -32,9 +35,9 @@ export const BottomControls = () => {
   if (status?.isLoaded) {
     return (
       <Controls>
-        <View className="flex h-16 w-full flex-col items-center justify-center">
-          <View className="w-full px-4">
-            <View className="ml-10 flex flex-row items-center">
+        <View className="flex h-40 w-full flex-col items-center justify-center p-6">
+          <View className="w-full">
+            <View className="flex flex-row items-center">
               <Text className="font-bold">{getCurrentTime()}</Text>
               <Text className="mx-1 font-bold">/</Text>
               <TouchableOpacity onPress={toggleTimeDisplay}>
@@ -46,9 +49,12 @@ export const BottomControls = () => {
               </TouchableOpacity>
             </View>
 
-            <View className="py-2">
+            <View>
               <ProgressBar />
             </View>
+            <View className="flex w-full flex-row items-center justify-between">
+              <CaptionsSelector />
+            </View>
           </View>
         </View>
       </Controls>

apps/expo/src/components/player/CaptionRenderer.tsx

@@ -0,0 +1,101 @@
+import { useMemo } from "react";
+import { View } from "react-native";
+import Animated, {
+  useAnimatedReaction,
+  useAnimatedStyle,
+  useDerivedValue,
+  useSharedValue,
+  withSpring,
+} from "react-native-reanimated";
+
+import { Text } from "~/components/ui/Text";
+import { convertMilliSecondsToSeconds } from "~/lib/number";
+import { useCaptionsStore } from "~/stores/captions";
+import { usePlayerStore } from "~/stores/player/store";
+
+export const captionIsVisible = (
+  start: number,
+  end: number,
+  delay: number,
+  currentTime: number,
+) => {
+  const delayedStart = start / 1000 + delay;
+  const delayedEnd = end / 1000 + delay;
+  return (
+    Math.max(0, delayedStart) <= currentTime &&
+    Math.max(0, delayedEnd) >= currentTime
+  );
+};
+
+export const CaptionRenderer = () => {
+  const isIdle = usePlayerStore((state) => state.interface.isIdle);
+  const selectedCaption = useCaptionsStore((state) => state.selectedCaption);
+  const delay = useCaptionsStore((state) => state.delay);
+  const status = usePlayerStore((state) => state.status);
+
+  const translateY = useSharedValue(0);
+
+  const animatedStyles = useAnimatedStyle(() => {
+    return {
+      transform: [{ translateY: translateY.value }],
+    };
+  });
+
+  const transitionValue = useDerivedValue(() => {
+    return isIdle ? 50 : 0;
+  }, [isIdle]);
+
+  useAnimatedReaction(
+    () => {
+      return transitionValue.value;
+    },
+    (newValue) => {
+      translateY.value = withSpring(newValue);
+    },
+  );
+
+  const visibleCaptions = useMemo(
+    () =>
+      selectedCaption?.data.filter(({ start, end }) =>
+        captionIsVisible(
+          start,
+          end,
+          delay,
+          status?.isLoaded
+            ? convertMilliSecondsToSeconds(status.positionMillis)
+            : 0,
+        ),
+      ),
+    [selectedCaption, delay, status],
+  );
+
+  console.log(visibleCaptions);
+
+  if (!status?.isLoaded || !selectedCaption || !visibleCaptions?.length)
+    return null;
+
+  return (
+    // https://github.com/marklawlor/nativewind/issues/790
+    <Animated.View
+      // className="rounded px-4 py-1 text-center leading-normal [text-shadow:0_2px_4px_rgba(0,0,0,0.5)]"
+      style={[
+        {
+          backgroundColor: "rgba(0, 0, 0, 0.5)",
+          paddingLeft: 16,
+          paddingRight: 16,
+          paddingTop: 4,
+          paddingBottom: 4,
+          borderRadius: 10,
+          marginTop: 32,
+        },
+        animatedStyles,
+      ]}
+    >
+      {visibleCaptions?.map((caption) => (
+        <View key={caption.index}>
+          <Text>{caption.text}</Text>
+        </View>
+      ))}
+    </Animated.View>
+  );
+};

apps/expo/src/components/player/CaptionsSelector.tsx

@@ -0,0 +1,84 @@
+import type { ContentCaption } from "subsrt-ts/dist/types/handler";
+import { useCallback } from "react";
+import { ScrollView, View } from "react-native";
+import Modal from "react-native-modal";
+import { MaterialCommunityIcons } from "@expo/vector-icons";
+import { parse } from "subsrt-ts";
+
+import type { Stream } from "@movie-web/provider-utils";
+import colors from "@movie-web/tailwind-config/colors";
+
+import { useBoolean } from "~/hooks/useBoolean";
+import { useCaptionsStore } from "~/stores/captions";
+import { usePlayerStore } from "~/stores/player/store";
+import { Button } from "../ui/Button";
+import { Text } from "../ui/Text";
+
+const parseCaption = async (
+  caption: Stream["captions"][0],
+): Promise<ContentCaption[]> => {
+  const response = await fetch(caption.url);
+  const data = await response.text();
+  return parse(data).filter(
+    (cue) => cue.type === "caption",
+  ) as ContentCaption[];
+};
+
+export const CaptionsSelector = () => {
+  const captions = usePlayerStore((state) => state.interface.stream?.captions);
+  const setSelectedCaption = useCaptionsStore(
+    (state) => state.setSelectedCaption,
+  );
+  const { isTrue, on, off } = useBoolean();
+
+  const downloadAndSetCaption = useCallback(
+    (caption: Stream["captions"][0]) => {
+      parseCaption(caption)
+        .then((data) => {
+          setSelectedCaption({ ...caption, data });
+        })
+        .catch(console.error);
+    },
+    [setSelectedCaption],
+  );
+
+  if (!captions?.length) return null;
+
+  return (
+    <View className="max-w-36 flex-1">
+      <Button
+        title="Subtitles"
+        variant="outline"
+        onPress={on}
+        iconLeft={
+          <MaterialCommunityIcons
+            name="subtitles"
+            size={24}
+            color={colors.primary[300]}
+          />
+        }
+      />
+
+      <Modal
+        isVisible={isTrue}
+        onBackdropPress={off}
+        supportedOrientations={["portrait", "landscape"]}
+      >
+        <ScrollView className="flex-1 bg-gray-900">
+          <Text className="text-center font-bold">Select subtitle</Text>
+          {captions?.map((caption) => (
+            <Button
+              key={caption.id}
+              title={caption.language}
+              onPress={() => {
+                downloadAndSetCaption(caption);
+                off();
+              }}
+              className="max-w-16"
+            />
+          ))}
+        </ScrollView>
+      </Modal>
+    </View>
+  );
+};

apps/expo/src/components/player/ControlsOverlay.tsx

@@ -11,7 +11,7 @@ interface ControlsOverlayProps {
 
 export const ControlsOverlay = ({ headerData }: ControlsOverlayProps) => {
   return (
-    <View className="absolute left-0 top-0 flex h-full w-full flex-1">
+    <View className="absolute left-0 top-0 flex h-full w-full flex-1 flex-col justify-between">
       <Header data={headerData} />
       <MiddleControls />
       <BottomControls />

apps/expo/src/components/player/Header.tsx

@@ -22,7 +22,7 @@ export const Header = ({ data }: HeaderProps) => {
 
   if (!isIdle) {
     return (
-      <View className="flex h-16 w-full flex-row items-center justify-between px-6 pt-6">
+      <View className="flex h-16 w-full flex-row justify-between px-6 pt-6">
         <Controls>
           <BackButton className="w-36" />
         </Controls>
@@ -31,7 +31,7 @@ export const Header = ({ data }: HeaderProps) => {
             ? `${data.title} (${data.year}) S${data.season.toString().padStart(2, "0")}E${data.episode.toString().padStart(2, "0")}`
             : `${data.title} (${data.year})`}
         </Text>
-        <View className="flex w-36 flex-row items-center justify-center gap-2 space-x-2 rounded-full bg-secondary-300 px-4 py-2 opacity-80">
+        <View className="flex h-12 w-36 flex-row items-center justify-center gap-2 space-x-2 rounded-full bg-secondary-300 px-4 py-2 opacity-80">
           <Image source={Icon} className="h-6 w-6" />
           <Text className="font-bold">movie-web</Text>
         </View>

apps/expo/src/components/player/MiddleControls.tsx

@@ -32,6 +32,9 @@ export const MiddleControls = () => {
 
 const styles = StyleSheet.create({
   container: {
+    position: "absolute",
+    height: "100%",
+    width: "100%",
     flex: 1,
     flexDirection: "row",
     alignItems: "center",

apps/expo/src/components/player/ProgressBar.tsx

@@ -21,7 +21,7 @@ export const ProgressBar = () => {
   if (status?.isLoaded) {
     return (
       <TouchableOpacity
-        className="flex h-10 flex-1 items-center justify-center p-8"
+        className="flex flex-1 items-center justify-center pb-12 pt-6"
         onPress={() => setIsIdle(false)}
       >
         <VideoSlider onSlidingComplete={updateProgress} />

apps/expo/src/components/player/VideoSlider.tsx

@@ -36,7 +36,7 @@ const VideoSlider = ({ onSlidingComplete }: VideoSliderProps) => {
   const status = usePlayerStore((state) => state.status);
   const setIsIdle = usePlayerStore((state) => state.setIsIdle);
 
-  const width = Dimensions.get("screen").width - 100;
+  const width = Dimensions.get("screen").width - 40;
   const knobSize_ = 20;
   const trackSize_ = 8;
   const minimumValue = 0;

apps/expo/src/components/ui/Button.tsx

@@ -0,0 +1,60 @@
+import type { VariantProps } from "class-variance-authority";
+import type { ReactNode } from "react";
+import type { PressableProps } from "react-native";
+import { Pressable } from "react-native";
+import { cva } from "class-variance-authority";
+
+import { cn } from "~/lib/utils";
+import { Text } from "./Text";
+
+const buttonVariants = cva(
+  "flex flex-row items-center justify-center gap-4 rounded-md disabled:opacity-50",
+  {
+    variants: {
+      variant: {
+        default: "bg-primary-300",
+        outline: "border border-primary-400 bg-transparent",
+        secondary: "bg-secondary-300",
+      },
+      size: {
+        default: "h-10 px-4 py-2",
+        sm: "h-9 rounded-md px-3",
+        lg: "h-11 rounded-md px-8",
+        icon: "h-10 w-10",
+      },
+    },
+    defaultVariants: {
+      variant: "default",
+      size: "default",
+    },
+  },
+);
+
+export interface ButtonProps
+  extends PressableProps,
+    VariantProps<typeof buttonVariants> {
+  iconLeft?: ReactNode;
+  iconRight?: ReactNode;
+  title: string;
+}
+
+export function Button({
+  onPress,
+  variant,
+  size,
+  className,
+  iconLeft,
+  iconRight,
+  title,
+}: ButtonProps) {
+  return (
+    <Pressable
+      onPress={onPress}
+      className={cn(buttonVariants({ variant, size, className }))}
+    >
+      {iconLeft}
+      <Text className="font-bold">{title}</Text>
+      {iconRight}
+    </Pressable>
+  );
+}

apps/expo/src/hooks/useBoolean.ts

@@ -0,0 +1,19 @@
+import { useMemo, useState } from "react";
+
+type InitialState = boolean | (() => boolean);
+
+export const useBoolean = (initialState: InitialState = false) => {
+  const [value, setValue] = useState(initialState);
+  const callbacks = useMemo(
+    () => ({
+      on: () => setValue(true),
+      off: () => setValue(false),
+      toggle: () => setValue((prev) => !prev),
+    }),
+    [],
+  );
+  return {
+    isTrue: value,
+    ...callbacks,
+  };
+};

apps/expo/src/lib/number.ts

@@ -0,0 +1,3 @@
+export const convertMilliSecondsToSeconds = (milliSeconds: number) => {
+  return milliSeconds / 1000;
+};

apps/expo/src/stores/captions/index.ts

@@ -0,0 +1,33 @@
+import type { ContentCaption } from "subsrt-ts/dist/types/handler";
+import { create } from "zustand";
+import { immer } from "zustand/middleware/immer";
+
+import type { Stream } from "@movie-web/provider-utils";
+
+type CaptionWithData = Stream["captions"][0] & {
+  data: ContentCaption[];
+};
+
+export interface CaptionsStore {
+  selectedCaption: CaptionWithData | null;
+  delay: number;
+  setSelectedCaption(caption: CaptionWithData | null): void;
+  setDelay(delay: number): void;
+}
+
+export const useCaptionsStore = create(
+  immer<CaptionsStore>((set) => ({
+    selectedCaption: null,
+    delay: 0,
+    setSelectedCaption: (caption) => {
+      set((s) => {
+        s.selectedCaption = caption;
+      });
+    },
+    setDelay: (delay) => {
+      set((s) => {
+        s.delay = delay;
+      });
+    },
+  })),
+);

apps/expo/src/stores/player/slices/interface.ts

@@ -1,13 +1,18 @@
 import * as ScreenOrientation from "expo-screen-orientation";
 
+import type { Stream } from "@movie-web/provider-utils";
+
 import type { MakeSlice } from "./types";
 
 export interface InterfaceSlice {
   interface: {
     isIdle: boolean;
     idleTimeout: NodeJS.Timeout | null;
+    stream: Stream | null;
+    selectedCaption: Stream["captions"][0] | null;
   };
   setIsIdle(state: boolean): void;
+  setStream(stream: Stream): void;
   lockOrientation: () => Promise<void>;
   unlockOrientation: () => Promise<void>;
   presentFullscreenPlayer: () => Promise<void>;
@@ -18,6 +23,8 @@ export const createInterfaceSlice: MakeSlice<InterfaceSlice> = (set, get) => ({
   interface: {
     isIdle: true,
     idleTimeout: null,
+    stream: null,
+    selectedCaption: null,
   },
   setIsIdle: (state) => {
     set((s) => {
@@ -34,6 +41,11 @@ export const createInterfaceSlice: MakeSlice<InterfaceSlice> = (set, get) => ({
       s.interface.isIdle = state;
     });
   },
+  setStream: (stream) => {
+    set((s) => {
+      s.interface.stream = stream;
+    });
+  },
   lockOrientation: async () => {
     await ScreenOrientation.lockAsync(
       ScreenOrientation.OrientationLock.LANDSCAPE,

pnpm-lock.yaml

@@ -112,6 +112,9 @@ importers:
       react-native-gesture-handler:
         specifier: ~2.14.1
         version: 2.14.1(react-native@0.73.2)(react@18.2.0)
+      react-native-modal:
+        specifier: ^13.0.1
+        version: 13.0.1(react-native@0.73.2)(react@18.2.0)
       react-native-quick-base64:
         specifier: ^2.0.8
         version: 2.0.8(react-native@0.73.2)(react@18.2.0)
@@ -130,6 +133,9 @@ importers:
       react-native-web:
         specifier: ^0.19.10
         version: 0.19.10(react-dom@18.2.0)(react@18.2.0)
+      subsrt-ts:
+        specifier: ^2.1.2
+        version: 2.1.2
       tailwind-merge:
         specifier: ^2.2.1
         version: 2.2.1
@@ -8952,6 +8958,12 @@ packages:
     resolution: {integrity: sha512-xWGDIW6x921xtzPkhiULtthJHoJvBbF3q26fzloPCK0hsvxtPVelvftw3zjbHWSkR2km9Z+4uxbDDK/6Zw9B8w==}
     dev: false
 
+  /react-native-animatable@1.3.3:
+    resolution: {integrity: sha512-2ckIxZQAsvWn25Ho+DK3d1mXIgj7tITkrS4pYDvx96WyOttSvzzFeQnM2od0+FUMzILbdHDsDEqZvnz1DYNQ1w==}
+    dependencies:
+      prop-types: 15.8.1
+    dev: false
+
   /react-native-context-menu-view@1.14.1(react-native@0.73.2)(react@18.2.0):
     resolution: {integrity: sha512-rPtC6RCbEVismTQ6M7WSt1HisNvgbS9bWqWX4RQXNXHKOKsVvXpI+bWRypFAjeBN/P+winn6Dxn1+meLBMrjmQ==}
     peerDependencies:
@@ -9008,6 +9020,18 @@ packages:
       react-native: 0.73.2(@babel/core@7.23.9)(@babel/preset-env@7.23.9)(react@18.2.0)
     dev: false
 
+  /react-native-modal@13.0.1(react-native@0.73.2)(react@18.2.0):
+    resolution: {integrity: sha512-UB+mjmUtf+miaG/sDhOikRfBOv0gJdBU2ZE1HtFWp6UixW9jCk/bhGdHUgmZljbPpp0RaO/6YiMmQSSK3kkMaw==}
+    peerDependencies:
+      react: '*'
+      react-native: '>=0.65.0'
+    dependencies:
+      prop-types: 15.8.1
+      react: 18.2.0
+      react-native: 0.73.2(@babel/core@7.23.9)(@babel/preset-env@7.23.9)(react@18.2.0)
+      react-native-animatable: 1.3.3
+    dev: false
+
   /react-native-quick-base64@2.0.8(react-native@0.73.2)(react@18.2.0):
     resolution: {integrity: sha512-2kMlnLSy0qz4NA0KXMGugd3qNB5EAizxZ6ghEVNGIxAOlc9CGvC8miv35wgpFbSKeiaBRfcPfkdTM/5Erb/6SQ==}
     peerDependencies:
@@ -9985,6 +10009,11 @@ packages:
     resolution: {integrity: sha512-3ZUifmCDCQanjeej1f6kyl/BeP/Vae5EYkQ9iJfUm/QwZvlgnZzyflqAsAWYURdtea8Vkvswu2GrC57h3qffcA==}
     dev: false
 
+  /subsrt-ts@2.1.2:
+    resolution: {integrity: sha512-45yNlK42Z0pz4lAaNYbR5P60M2jmHl+gfAaiJxDIXsXXqoE7TkDCzl/00HgWyZXKkdIU6s8FiNtRvrlOZb+5Qg==}
+    hasBin: true
+    dev: false
+
   /sucrase@3.34.0:
     resolution: {integrity: sha512-70/LQEZ07TEcxiU2dz51FKaE6hCTWC6vr7FOk3Gr0U60C3shtAN+H+BFr9XlYe5xqf3RA8nrc+VIwzCfnxuXJw==}
     engines: {node: '>=8'}

tooling/tailwind/colors.ts

@@ -10,5 +10,8 @@ export default {
     300: "#32324F",
     700: "#131322",
   },
+  playerSettings: {
+    captionBackground: "#161b23",
+  },
   background: "#0a0a12",
 };