add scraper screen

apps/expo/src/app/(tabs)/_layout.tsx

@@ -9,6 +9,7 @@ import { defaultTheme } from "@movie-web/tailwind-config/themes";
 import { MovieWebSvg } from "~/components/Icon";
 import SvgTabBarIcon from "~/components/SvgTabBarIcon";
 import TabBarIcon from "~/components/TabBarIcon";
+import { cn } from "~/lib/utils";
 import SearchTabContext from "../../components/ui/SearchTabContext";
 
 export default function TabLayout() {
@@ -83,7 +84,13 @@ export default function TabLayout() {
             tabBarLabel: "",
             tabBarIcon: ({ focused }) => (
               <View
-                className={`android:top-2 ios:top-2 flex h-14 w-14 items-center justify-center overflow-hidden rounded-full ${focused ? "bg-primary-300" : "bg-primary-400"} text-center align-middle text-2xl text-white`}
+                className={cn(
+                  `top-2 flex h-14 w-14 items-center justify-center overflow-hidden rounded-full text-center align-middle text-2xl text-white ${focused ? "bg-tabBar-active" : "bg-tabBar-inactive"}`,
+                  {
+                    "bg-tabBar-active": focused,
+                    "bg-tabBar-inactive": !focused,
+                  },
+                )}
               >
                 <TabBarIcon name="search" color="#FFF" />
               </View>

apps/expo/src/components/player/Header.tsx

@@ -16,7 +16,7 @@ export const Header = () => {
 
   if (!isIdle && meta) {
     return (
-      <View className="z-50 flex h-16 w-full flex-row justify-between px-6 pt-6">
+      <View className="z-50 flex h-16 w-full flex-row items-center justify-between px-6 pt-6">
         <Controls>
           <BackButton className="w-36" />
         </Controls>
@@ -29,7 +29,7 @@ export const Header = () => {
               )
             : ""}
         </Text>
-        <View className="bg-secondary-300 flex h-12 w-36 flex-row items-center justify-center gap-2 space-x-2 rounded-full px-4 py-2 opacity-80">
+        <View className="flex h-12 w-36 flex-row items-center justify-center gap-2 space-x-2 rounded-full bg-pill-background px-4 py-2 opacity-80">
           <Image source={Icon} className="h-6 w-6" />
           <Text className="font-bold">movie-web</Text>
         </View>

apps/expo/src/components/player/ScrapeCard.tsx

@@ -0,0 +1,119 @@
+import type { ReactNode } from "react";
+import React from "react";
+import { View } from "react-native";
+import { MaterialCommunityIcons, MaterialIcons } from "@expo/vector-icons";
+
+import { defaultTheme } from "@movie-web/tailwind-config/themes";
+
+import { cn } from "~/lib/utils";
+import { Text } from "../ui/Text";
+
+export interface ScrapeItemProps {
+  status: "failure" | "pending" | "notfound" | "success" | "waiting";
+  name: string;
+  id?: string;
+  percentage?: number;
+  children?: ReactNode;
+}
+
+export interface ScrapeCardProps extends ScrapeItemProps {
+  hasChildren?: boolean;
+}
+
+const statusTextMap: Partial<Record<ScrapeCardProps["status"], string>> = {
+  notfound: "Doesn't have the video",
+  failure: "Failed to scrape",
+  pending: "Checking for videos...",
+};
+
+const mapPercentageToIcon = (percentage: number) => {
+  const slice = Math.floor(percentage / 12.5);
+  return `circle-slice-${slice === 0 ? 1 : slice}`;
+};
+
+export function StatusCircle({
+  type,
+  percentage,
+}: {
+  type: ScrapeItemProps["status"];
+  percentage: number;
+}) {
+  return (
+    <>
+      {type === "waiting" && (
+        <MaterialCommunityIcons
+          name="circle-outline"
+          size={40}
+          color={defaultTheme.extend.colors.video.scraping.noresult}
+        />
+      )}
+      {type === "pending" && (
+        <MaterialCommunityIcons
+          name={mapPercentageToIcon(percentage) as "circle-slice-1"}
+          size={40}
+          color={defaultTheme.extend.colors.video.scraping.loading}
+        />
+      )}
+      {type === "failure" && (
+        <MaterialCommunityIcons
+          name="close-circle"
+          size={40}
+          color={defaultTheme.extend.colors.video.scraping.error}
+        />
+      )}
+      {type === "notfound" && (
+        <MaterialIcons
+          name="remove-circle"
+          size={40}
+          color={defaultTheme.extend.colors.video.scraping.noresult}
+        />
+      )}
+      {type === "success" && (
+        <MaterialIcons
+          name="check-circle"
+          size={40}
+          color={defaultTheme.extend.colors.video.scraping.success}
+        />
+      )}
+    </>
+  );
+}
+
+export function ScrapeItem(props: ScrapeItemProps) {
+  const text = statusTextMap[props.status];
+
+  return (
+    <View className="flex flex-col">
+      <View className="flex flex-row items-center gap-4">
+        <StatusCircle type={props.status} percentage={props.percentage ?? 0} />
+        <Text
+          className={cn("text-lg", {
+            "text-white": props.status === "pending",
+            "text-type-secondary": props.status !== "pending",
+          })}
+        >
+          {props.name}
+        </Text>
+      </View>
+      <View className="flex flex-row gap-4">
+        <View style={{ width: 40 }} />
+        <View>{text && <Text className="mt-1 text-lg">{text}</Text>}</View>
+      </View>
+      <View className="ml-12">{props.children}</View>
+    </View>
+  );
+}
+
+export function ScrapeCard(props: ScrapeCardProps) {
+  return (
+    <View className="w-96">
+      <View
+        className={cn("w-96 rounded-xl px-6 py-3", {
+          "bg-video-scraping-card": props.hasChildren,
+        })}
+      >
+        <ScrapeItem {...props} />
+      </View>
+    </View>
+  );
+}

apps/expo/src/components/player/ScraperProcess.tsx

@@ -1,145 +1,47 @@
-import { useCallback, useEffect, useState } from "react";
+import { useEffect, useRef } from "react";
-import { ActivityIndicator, View } from "react-native";
+import { SafeAreaView, View } from "react-native";
+import { ScrollView } from "react-native-gesture-handler";
 import { useRouter } from "expo-router";
 
-import type {
+import type { HlsBasedStream } from "@movie-web/provider-utils";
-  DiscoverEmbedsEvent,
+import { extractTracksFromHLS } from "@movie-web/provider-utils";
-  HlsBasedStream,
-  InitEvent,
-  RunnerEvent,
-  UpdateEvent,
-} from "@movie-web/provider-utils";
-import {
-  extractTracksFromHLS,
-  getVideoStream,
-  transformSearchResultToScrapeMedia,
-} from "@movie-web/provider-utils";
-import { fetchMediaDetails, fetchSeasonDetails } from "@movie-web/tmdb";
 
 import type { ItemData } from "../item/item";
 import type { AudioTrack } from "./AudioTrackSelector";
+import { useMeta } from "~/hooks/player/useMeta";
+import { useScrape } from "~/hooks/player/useSourceScrape";
 import { constructFullUrl } from "~/lib/url";
+import { cn } from "~/lib/utils";
 import { PlayerStatus } from "~/stores/player/slices/interface";
+import { convertMetaToScrapeMedia } from "~/stores/player/slices/video";
 import { usePlayerStore } from "~/stores/player/store";
-import { Text } from "../ui/Text";
+import { ScrapeCard, ScrapeItem } from "./ScrapeCard";
 
 interface ScraperProcessProps {
   data: ItemData;
 }
 
-enum ScrapeStatus {
-  LOADING = "loading",
-  SUCCESS = "success",
-  ERROR = "error",
-}
-
 export const ScraperProcess = ({ data }: ScraperProcessProps) => {
   const router = useRouter();
 
-  const meta = usePlayerStore((state) => state.meta);
+  const scrollViewRef = useRef<ScrollView>(null);
+
+  const { convertMovieIdToMeta } = useMeta();
+  const { startScraping, sourceOrder, sources, currentSource } = useScrape();
+
   const setStream = usePlayerStore((state) => state.setCurrentStream);
-  const setSeasonData = usePlayerStore((state) => state.setSeasonData);
   const setHlsTracks = usePlayerStore((state) => state.setHlsTracks);
   const setAudioTracks = usePlayerStore((state) => state.setAudioTracks);
   const setPlayerStatus = usePlayerStore((state) => state.setPlayerStatus);
   const setSourceId = usePlayerStore((state) => state.setSourceId);
-  const setMeta = usePlayerStore((state) => state.setMeta);
-  const [checkedSource, setCheckedSource] = useState("");
-
-  function isInitEvent(event: RunnerEvent): event is InitEvent {
-    return (event as InitEvent).sourceIds !== undefined;
-  }
-
-  function isUpdateEvent(event: RunnerEvent): event is UpdateEvent {
-    return (event as UpdateEvent).percentage !== undefined;
-  }
-
-  function isDiscoverEmbedsEvent(
-    event: RunnerEvent,
-  ): event is DiscoverEmbedsEvent {
-    return (event as DiscoverEmbedsEvent).sourceId !== undefined;
-  }
-
-  const handleEvent = useCallback((event: RunnerEvent) => {
-    if (typeof event === "string") {
-      setCheckedSource(event);
-      setScrapeStatus({ status: ScrapeStatus.LOADING, progress: 10 });
-    } else if (isUpdateEvent(event)) {
-      console.log(event.status);
-      switch (event.status) {
-        case "success":
-          setScrapeStatus({ status: ScrapeStatus.SUCCESS, progress: 100 });
-          break;
-        case "failure":
-          setScrapeStatus({ status: ScrapeStatus.ERROR, progress: 0 });
-          break;
-        case "pending":
-        case "notfound":
-      }
-      setCheckedSource(event.id);
-    } else if (isInitEvent(event) || isDiscoverEmbedsEvent(event)) {
-      setScrapeStatus((prevStatus) => ({
-        status: ScrapeStatus.LOADING,
-        progress: Math.min(prevStatus.progress + 20, 95),
-      }));
-    }
-  }, []);
-
-  const [_scrapeStatus, setScrapeStatus] = useState({
-    status: ScrapeStatus.LOADING,
-    progress: 0,
-  });
 
   useEffect(() => {
     const fetchData = async () => {
       if (!data) return router.back();
-      const media = await fetchMediaDetails(data.id, data.type);
+      const meta = await convertMovieIdToMeta(data.id, data.type);
-      if (!media) return router.back();
+      if (!meta) return;
-      const scrapeMedia = transformSearchResultToScrapeMedia(
+      const streamResult = await startScraping(convertMetaToScrapeMedia(meta));
-        media.type,
-        media.result,
-        meta?.season?.number,
-        meta?.episode?.number,
-      );
-      let seasonData = null;
-      if (scrapeMedia.type === "show") {
-        seasonData = await fetchSeasonDetails(
-          scrapeMedia.tmdbId,
-          scrapeMedia.season.number,
-        );
-      }
 
-      setMeta({
-        ...scrapeMedia,
-        poster: media.result.poster_path,
-        ...("season" in scrapeMedia
-          ? {
-              season: {
-                number: scrapeMedia.season.number,
-                tmdbId: scrapeMedia.tmdbId,
-              },
-              episode: {
-                number: scrapeMedia.episode.number,
-                tmdbId: scrapeMedia.episode.tmdbId,
-              },
-              episodes:
-                seasonData?.episodes.map((e) => ({
-                  tmdbId: e.id.toString(),
-                  number: e.episode_number,
-                  name: e.name,
-                })) ?? [],
-            }
-          : {}),
-      });
-      const streamResult = await getVideoStream({
-        media: scrapeMedia,
-        events: {
-          init: handleEvent,
-          update: handleEvent,
-          discoverEmbeds: handleEvent,
-          start: handleEvent,
-        },
-      });
       if (!streamResult) return router.back();
       setStream(streamResult.stream);
 
@@ -184,31 +86,82 @@ export const ScraperProcess = ({ data }: ScraperProcessProps) => {
     };
     void fetchData();
   }, [
+    convertMovieIdToMeta,
     data,
     router,
+    setAudioTracks,
     setHlsTracks,
-    setSeasonData,
-    setStream,
     setPlayerStatus,
     setSourceId,
-    setMeta,
+    setStream,
-    meta?.season?.number,
+    startScraping,
-    meta?.episode?.number,
-    setAudioTracks,
-    handleEvent,
   ]);
 
+  let currentProviderIndex = sourceOrder.findIndex(
+    (s) => s.id === currentSource || s.children.includes(currentSource ?? ""),
+  );
+  if (currentProviderIndex === -1) {
+    currentProviderIndex = sourceOrder.length - 1;
+  }
+
+  useEffect(() => {
+    scrollViewRef.current?.scrollTo({
+      y: currentProviderIndex * 80,
+      animated: true,
+    });
+  }, [currentProviderIndex]);
+
   return (
-    <View className="flex-1">
+    <SafeAreaView className="flex h-full flex-1 flex-col">
-      <View className="flex-1 items-center justify-center bg-black">
+      <View className="flex-1 items-center justify-center bg-background-main">
-        <View className="flex flex-col items-center">
+        <ScrollView
-          <Text className="mb-4 text-2xl text-white">
+          ref={scrollViewRef}
-            Checking {checkedSource}
+          contentContainerClassName="items-center flex flex-col py-16"
-          </Text>
+        >
-          <ActivityIndicator size="large" color="#0000ff" />
+          {sourceOrder.map((order) => {
-          {/* <StatusCircle type={scrapeStatus.status} percentage={scrapeStatus.progress} /> */}
+            const source = sources[order.id];
-        </View>
+            if (!source) return null;
+            const distance = Math.abs(
+              sourceOrder.findIndex((o) => o.id === order.id) -
+                currentProviderIndex,
+            );
+            return (
+              <View
+                key={order.id}
+                style={{ opacity: Math.max(0, 1 - distance * 0.3) }}
+              >
+                <ScrapeCard
+                  id={order.id}
+                  name={source.name}
+                  status={source.status}
+                  hasChildren={order.children.length > 0}
+                  percentage={source.percentage}
+                >
+                  <View
+                    className={cn({
+                      "mt-8 space-y-6": order.children.length > 0,
+                    })}
+                  >
+                    {order.children.map((embedId) => {
+                      const embed = sources[embedId];
+                      if (!embed) return null;
+                      return (
+                        <ScrapeItem
+                          id={embedId}
+                          name={embed.name}
+                          status={embed.status}
+                          percentage={embed.percentage}
+                          key={embedId}
+                        />
+                      );
+                    })}
+                  </View>
+                </ScrapeCard>
+              </View>
+            );
+          })}
+        </ScrollView>
       </View>
-    </View>
+    </SafeAreaView>
   );
 };

apps/expo/src/components/ui/Button.tsx

@@ -12,9 +12,9 @@ const buttonVariants = cva(
   {
     variants: {
       variant: {
-        default: "bg-primary-300",
+        default: "bg-buttons-purple",
-        outline: "border-primary-400 border bg-transparent",
+        outline: "border border-buttons-purple bg-transparent",
-        secondary: "bg-secondary-300",
+        secondary: "bg-buttons-secondary",
       },
       size: {
         default: "h-10 px-4 py-2",

apps/expo/src/hooks/player/useMeta.ts

@@ -0,0 +1,58 @@
+import { useCallback } from "react";
+
+import { transformSearchResultToScrapeMedia } from "@movie-web/provider-utils";
+import { fetchMediaDetails, fetchSeasonDetails } from "@movie-web/tmdb";
+
+import { usePlayerStore } from "~/stores/player/store";
+
+export const useMeta = () => {
+  const meta = usePlayerStore((state) => state.meta);
+  const setMeta = usePlayerStore((state) => state.setMeta);
+
+  const convertMovieIdToMeta = useCallback(
+    async (id: string, type: "movie" | "tv") => {
+      const media = await fetchMediaDetails(id, type);
+      if (!media) return;
+      const scrapeMedia = transformSearchResultToScrapeMedia(
+        media.type,
+        media.result,
+        meta?.season?.number,
+        meta?.episode?.number,
+      );
+      let seasonData = null;
+      if (scrapeMedia.type === "show") {
+        seasonData = await fetchSeasonDetails(
+          scrapeMedia.tmdbId,
+          scrapeMedia.season.number,
+        );
+      }
+      const m = {
+        ...scrapeMedia,
+        poster: media.result.poster_path,
+        ...("season" in scrapeMedia
+          ? {
+              season: {
+                number: scrapeMedia.season.number,
+                tmdbId: scrapeMedia.tmdbId,
+              },
+              episode: {
+                number: scrapeMedia.episode.number,
+                tmdbId: scrapeMedia.episode.tmdbId,
+              },
+              episodes:
+                seasonData?.episodes.map((e) => ({
+                  tmdbId: e.id.toString(),
+                  number: e.episode_number,
+                  name: e.name,
+                })) ?? [],
+            }
+          : {}),
+      };
+      setMeta(m);
+      return m;
+    },
+    [meta?.episode?.number, meta?.season?.number, setMeta],
+  );
+
+  return { convertMovieIdToMeta };
+};

apps/expo/src/hooks/player/useSourceScrape.ts

@@ -1,13 +1,194 @@
+import { useCallback, useRef, useState } from "react";
 import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
 
+import type {
+  FullScraperEvents,
+  RunOutput,
+  ScrapeMedia,
+} from "@movie-web/provider-utils";
 import {
+  getMetaData,
   getVideoStreamFromEmbed,
   getVideoStreamFromSource,
+  providers,
 } from "@movie-web/provider-utils";
 
 import { convertMetaToScrapeMedia } from "~/stores/player/slices/video";
 import { usePlayerStore } from "~/stores/player/store";
 
+export interface ScrapingItems {
+  id: string;
+  children: string[];
+}
+
+export interface ScrapingSegment {
+  name: string;
+  id: string;
+  embedId?: string;
+  status: "failure" | "pending" | "notfound" | "success" | "waiting";
+  reason?: string;
+  error?: any;
+  percentage: number;
+}
+
+type ScraperEvent<Event extends keyof FullScraperEvents> = Parameters<
+  NonNullable<FullScraperEvents[Event]>
+>[0];
+
+export const useBaseScrape = () => {
+  const [sources, setSources] = useState<Record<string, ScrapingSegment>>({});
+  const [sourceOrder, setSourceOrder] = useState<ScrapingItems[]>([]);
+  const [currentSource, setCurrentSource] = useState<string>();
+  const lastId = useRef<string | null>(null);
+
+  const initEvent = useCallback((evt: ScraperEvent<"init">) => {
+    setSources(
+      evt.sourceIds
+        .map((v) => {
+          const source = getMetaData().find((s) => s.id === v);
+          if (!source) throw new Error("invalid source id");
+          const out: ScrapingSegment = {
+            name: source.name,
+            id: source.id,
+            status: "waiting",
+            percentage: 0,
+          };
+          return out;
+        })
+        .reduce<Record<string, ScrapingSegment>>((a, v) => {
+          a[v.id] = v;
+          return a;
+        }, {}),
+    );
+    setSourceOrder(evt.sourceIds.map((v) => ({ id: v, children: [] })));
+  }, []);
+
+  const startEvent = useCallback((id: ScraperEvent<"start">) => {
+    const lastIdTmp = lastId.current;
+    setSources((s) => {
+      if (s[id]) s[id]!.status = "pending";
+      if (lastIdTmp && s[lastIdTmp] && s[lastIdTmp]!.status === "pending")
+        s[lastIdTmp]!.status = "success";
+      return { ...s };
+    });
+    setCurrentSource(id);
+    lastId.current = id;
+  }, []);
+
+  const updateEvent = useCallback((evt: ScraperEvent<"update">) => {
+    setSources((s) => {
+      if (s[evt.id]) {
+        s[evt.id]!.status = evt.status;
+        s[evt.id]!.reason = evt.reason;
+        s[evt.id]!.error = evt.error;
+        s[evt.id]!.percentage = evt.percentage;
+      }
+      return { ...s };
+    });
+  }, []);
+
+  const discoverEmbedsEvent = useCallback(
+    (evt: ScraperEvent<"discoverEmbeds">) => {
+      setSources((s) => {
+        evt.embeds.forEach((v) => {
+          const source = getMetaData().find(
+            (src) => src.id === v.embedScraperId,
+          );
+          if (!source) throw new Error("invalid source id");
+          const out: ScrapingSegment = {
+            embedId: v.embedScraperId,
+            name: source.name,
+            id: v.id,
+            status: "waiting",
+            percentage: 0,
+          };
+          s[v.id] = out;
+        });
+        return { ...s };
+      });
+      setSourceOrder((s) => {
+        const source = s.find((v) => v.id === evt.sourceId);
+        if (!source) throw new Error("invalid source id");
+        source.children = evt.embeds.map((v) => v.id);
+        return [...s];
+      });
+    },
+    [],
+  );
+
+  const startScrape = useCallback(() => {
+    lastId.current = null;
+  }, []);
+
+  const getResult = useCallback((output: RunOutput | null) => {
+    if (output && lastId.current) {
+      setSources((s) => {
+        if (!lastId.current) return s;
+        if (s[lastId.current]) s[lastId.current]!.status = "success";
+        return { ...s };
+      });
+    }
+    return output;
+  }, []);
+
+  return {
+    initEvent,
+    startEvent,
+    updateEvent,
+    discoverEmbedsEvent,
+    startScrape,
+    getResult,
+    sources,
+    sourceOrder,
+    currentSource,
+  };
+};
+
+export function useScrape() {
+  const {
+    sources,
+    sourceOrder,
+    currentSource,
+    updateEvent,
+    discoverEmbedsEvent,
+    initEvent,
+    getResult,
+    startEvent,
+    startScrape,
+  } = useBaseScrape();
+
+  const startScraping = useCallback(
+    async (media: ScrapeMedia) => {
+      startScrape();
+      const output = await providers.runAll({
+        media,
+        events: {
+          init: initEvent,
+          start: startEvent,
+          update: updateEvent,
+          discoverEmbeds: discoverEmbedsEvent,
+        },
+      });
+      return getResult(output);
+    },
+    [
+      initEvent,
+      startEvent,
+      updateEvent,
+      discoverEmbedsEvent,
+      getResult,
+      startScrape,
+    ],
+  );
+
+  return {
+    startScraping,
+    sourceOrder,
+    sources,
+    currentSource,
+  };
+}
+
 export const useEmbedScrape = (closeModal?: () => void) => {
   const setCurrentStream = usePlayerStore((state) => state.setCurrentStream);
 

packages/provider-utils/src/util.ts

@@ -2,6 +2,12 @@ import type { AppendToResponse, MovieDetails, TvShowDetails } from "tmdb-ts";
 
 import type { ScrapeMedia } from "@movie-web/providers";
 
+import { providers } from "./video";
+
+export function getMetaData() {
+  return [...providers.listSources(), ...providers.listEmbeds()];
+}
+
 export function transformSearchResultToScrapeMedia<T extends "tv" | "movie">(
   type: T,
   result: T extends "tv"

tooling/eslint/base.js

@@ -29,6 +29,7 @@ const config = {
       { checksVoidReturn: { attributes: false } },
     ],
     "import/consistent-type-specifier-style": ["error", "prefer-top-level"],
+    "@typescript-eslint/no-explicit-any": "off",
   },
   ignorePatterns: [
     "**/*.config.js",