feat: hls downloads

apps/expo/app.config.ts

@@ -27,6 +27,9 @@ const defineConfig = (): ExpoConfig => ({
       CFBundleName: "movie-web",
       NSPhotoLibraryUsageDescription:
         "This app saves videos to the photo library.",
+      NSAppTransportSecurity: {
+        NSAllowsArbitraryLoads: true,
+      },
     },
   },
   android: {

apps/expo/package.json

@@ -25,6 +25,7 @@
     "@octokit/rest": "^20.0.2",
     "@react-native-anywhere/polyfill-base64": "0.0.1-alpha.0",
     "@react-navigation/native": "^6.1.9",
+    "@salihgun/react-native-video-processor": "^0.3.1",
     "@tamagui/animations-moti": "^1.91.4",
     "@tamagui/babel-plugin": "^1.91.4",
     "@tamagui/config": "^1.91.4",

apps/expo/src/app/(tabs)/downloads.tsx

@@ -3,7 +3,7 @@ import React from "react";
 import { ScrollView } from "react-native-gesture-handler";
 import { useRouter } from "expo-router";
 import { MaterialCommunityIcons } from "@expo/vector-icons";
-import { useTheme } from "tamagui";
+import { useTheme, YStack } from "tamagui";
 
 import { DownloadItem } from "~/components/DownloadItem";
 import ScreenLayout from "~/components/layout/ScreenLayout";
@@ -29,28 +29,46 @@ const DownloadsScreen: React.FC = () => {
 
   return (
     <ScreenLayout>
-      <MWButton
+      <YStack space={2} style={{ padding: 10 }}>
-        type="secondary"
+        <MWButton
-        backgroundColor="$sheetItemBackground"
+          type="secondary"
-        icon={
+          backgroundColor="$sheetItemBackground"
-          <MaterialCommunityIcons
+          icon={
-            name="download"
+            <MaterialCommunityIcons
-            size={24}
+              name="download"
-            color={theme.buttonSecondaryText.val}
+              size={24}
-          />
+              color={theme.buttonSecondaryText.val}
-        }
+            />
-        onPress={async () => {
-          const asset = await startDownload(
-            "https://samplelib.com/lib/preview/mp4/sample-5s.mp4",
-            "mp4",
-          ).catch(console.error);
-          if (asset) {
-            handlePress(asset);
           }
-        }}
+          onPress={async () => {
-      >
+            await startDownload(
-        test download (mp4)
+              "https://samplelib.com/lib/preview/mp4/sample-5s.mp4",
-      </MWButton>
+              "mp4",
+            ).catch(console.error);
+          }}
+        >
+          test download (mp4)
+        </MWButton>
+        <MWButton
+          type="secondary"
+          backgroundColor="$sheetItemBackground"
+          icon={
+            <MaterialCommunityIcons
+              name="download"
+              size={24}
+              color={theme.buttonSecondaryText.val}
+            />
+          }
+          onPress={async () => {
+            await startDownload(
+              "http://sample.vodobox.com/skate_phantom_flex_4k/skate_phantom_flex_4k.m3u8",
+              "hls",
+            ).catch(console.error);
+          }}
+        >
+          test download (hls)
+        </MWButton>
+      </YStack>
       <ScrollView>
         {downloads.map((item) => (
           <DownloadItem

apps/expo/src/components/DownloadItem.tsx

@@ -15,6 +15,7 @@ export interface DownloadItemProps {
   statusText?: string;
   asset?: Asset;
   onPress: (asset?: Asset) => void;
+  isHLS?: boolean;
 }
 
 const formatBytes = (bytes: number, decimals = 2) => {
@@ -38,6 +39,7 @@ export const DownloadItem: React.FC<DownloadItemProps> = ({
   statusText,
   asset,
   onPress,
+  isHLS,
 }) => {
   const percentage = progress * 100;
   const formattedFileSize = formatBytes(fileSize);
@@ -60,6 +62,7 @@ export const DownloadItem: React.FC<DownloadItemProps> = ({
         </Text>
       );
     } else {
+      if (isHLS) return null;
       return (
         <Text fontSize={12} color="gray">
           {speed.toFixed(2)} MB/s
@@ -95,8 +98,9 @@ export const DownloadItem: React.FC<DownloadItemProps> = ({
           justifyContent="space-between"
         >
           <Text fontSize={12} color="gray">
-            {percentage.toFixed()}% - {formattedDownloaded} of{" "}
+            {isHLS
-            {formattedFileSize}
+              ? `${percentage.toFixed()}% - ${downloaded} of ${fileSize} segments`
+              : `${percentage.toFixed()}% - ${formattedDownloaded} of ${formattedFileSize}`}
           </Text>
           {renderStatus()}
         </View>

apps/expo/src/components/player/DownloadButton.tsx

@@ -12,10 +12,15 @@ export const DownloadButton = () => {
   const theme = useTheme();
   const stream = usePlayerStore((state) => state.interface.currentStream);
   const { startDownload } = useDownloadManager();
-  if (stream?.type !== "file") return null;
+  let url: string | undefined | null = null;
+
+  if (stream?.type === "file") {
+    const highestQuality = findHighestQuality(stream);
+    url = highestQuality ? stream.qualities[highestQuality]?.url : null;
+  } else if (stream?.type === "hls") {
+    url = stream.playlist;
+  }
 
-  const highestQuality = findHighestQuality(stream);
-  const url = highestQuality ? stream.qualities[highestQuality]?.url : null;
   if (!url) return null;
 
   return (
@@ -30,7 +35,9 @@ export const DownloadButton = () => {
               color={theme.buttonSecondaryText.val}
             />
           }
-          onPress={() => startDownload(url, "mp4")}
+          onPress={() =>
+            url && startDownload(url, stream?.type === "hls" ? "hls" : "mp4")
+          }
         >
           Download
         </MWButton>

apps/expo/src/components/player/QualitySelector.tsx

@@ -2,7 +2,8 @@ import { useState } from "react";
 import { MaterialCommunityIcons, MaterialIcons } from "@expo/vector-icons";
 import { useTheme } from "tamagui";
 
-import { constructFullUrl } from "~/lib/url";
+import { constructFullUrl } from "@movie-web/provider-utils";
+
 import { usePlayerStore } from "~/stores/player/store";
 import { MWButton } from "../ui/Button";
 import { Controls } from "./Controls";

apps/expo/src/components/player/ScraperProcess.tsx

@@ -10,6 +10,7 @@ import type {
   ScrapeMedia,
 } from "@movie-web/provider-utils";
 import {
+  constructFullUrl,
   extractTracksFromHLS,
   findHighestQuality,
 } from "@movie-web/provider-utils";
@@ -21,7 +22,6 @@ import { useDownloadManager } from "~/hooks/DownloadManagerContext";
 import { useMeta } from "~/hooks/player/useMeta";
 import { useScrape } from "~/hooks/player/useSourceScrape";
 import { convertMetaToScrapeMedia } from "~/lib/meta";
-import { constructFullUrl } from "~/lib/url";
 import { PlayerStatus } from "~/stores/player/slices/interface";
 import { usePlayerStore } from "~/stores/player/store";
 import { ScrapeCard, ScrapeItem } from "./ScrapeCard";
@@ -76,6 +76,10 @@ export const ScraperProcess = ({
             : null;
           if (!url) return;
           startDownload(url, "mp4").catch(console.error);
+        } else if (streamResult.stream.type === "hls") {
+          startDownload(streamResult.stream.playlist, "hls").catch(
+            console.error,
+          );
         }
         return router.back();
       }

apps/expo/src/hooks/DownloadManagerContext.tsx

@@ -3,8 +3,11 @@ import type { ReactNode } from "react";
 import React, { createContext, useContext, useEffect, useState } from "react";
 import * as FileSystem from "expo-file-system";
 import * as MediaLibrary from "expo-media-library";
+import VideoManager from "@salihgun/react-native-video-processor";
 import { useToastController } from "@tamagui/toast";
 
+import { extractSegmentsFromHLS } from "@movie-web/provider-utils";
+
 import { useDownloadHistoryStore } from "~/stores/settings";
 
 export interface DownloadItem {
@@ -19,6 +22,7 @@ export interface DownloadItem {
   isFinished: boolean;
   statusText?: string;
   asset?: Asset;
+  isHLS?: boolean;
 }
 
 interface DownloadManagerContextType {
@@ -83,6 +87,7 @@ export const DownloadManagerProvider: React.FC<{ children: ReactNode }> = ({
       type,
       url,
       isFinished: false,
+      isHLS: type === "hls",
     };
 
     setDownloads((currentDownloads) => [newDownload, ...currentDownloads]);
@@ -91,7 +96,8 @@ export const DownloadManagerProvider: React.FC<{ children: ReactNode }> = ({
       const asset = await downloadMP4(url, newDownload.id, headers ?? {});
       return asset;
     } else if (type === "hls") {
-      // HLS stuff later
+      const asset = await downloadHLS(url, newDownload.id, headers ?? {});
+      return asset;
     }
   };
 
@@ -134,11 +140,11 @@ export const DownloadManagerProvider: React.FC<{ children: ReactNode }> = ({
       lastTimestamp = currentTime;
     };
 
-    const fileUri = FileSystem.documentDirectory
+    const fileUri = FileSystem.cacheDirectory
-      ? FileSystem.documentDirectory + url.split("/").pop()
+      ? FileSystem.cacheDirectory + url.split("/").pop()
       : null;
     if (!fileUri) {
-      console.error("Document directory is unavailable");
+      console.error("Cache directory is unavailable");
       return;
     }
 
@@ -164,6 +170,69 @@ export const DownloadManagerProvider: React.FC<{ children: ReactNode }> = ({
     }
   };
 
+  const downloadHLS = async (
+    url: string,
+    downloadId: string,
+    headers: Record<string, string>,
+  ) => {
+    const segments = await extractSegmentsFromHLS(url, headers);
+
+    if (!segments || segments.length === 0) {
+      return removeDownload(downloadId);
+    }
+
+    const totalSegments = segments.length;
+    let segmentsDownloaded = 0;
+
+    const segmentDir = FileSystem.cacheDirectory + "segments/";
+    await ensureDirExists(segmentDir);
+
+    const updateProgress = () => {
+      const progress = segmentsDownloaded / totalSegments;
+      updateDownloadItem(downloadId, {
+        progress,
+        downloaded: segmentsDownloaded,
+        fileSize: totalSegments,
+      });
+    };
+
+    const localSegmentPaths = [];
+
+    for (const [index, segment] of segments.entries()) {
+      const segmentFile = `${segmentDir}${index}.ts`;
+      localSegmentPaths.push(segmentFile);
+      const downloadResumable = FileSystem.createDownloadResumable(
+        segment,
+        segmentFile,
+        { headers },
+      );
+
+      try {
+        await downloadResumable.downloadAsync();
+        segmentsDownloaded++;
+        updateProgress();
+      } catch (e) {
+        console.error(e);
+      }
+    }
+
+    updateDownloadItem(downloadId, { statusText: "Merging" });
+    const uri = await VideoManager.mergeVideos(
+      localSegmentPaths,
+      `${FileSystem.cacheDirectory}output.mp4`,
+    );
+    const asset = await saveFileToMediaLibraryAndDeleteOriginal(
+      uri,
+      downloadId,
+    );
+    return asset;
+  };
+
+  async function ensureDirExists(dir: string) {
+    await FileSystem.deleteAsync(dir, { idempotent: true });
+    await FileSystem.makeDirectoryAsync(dir, { intermediates: true });
+  }
+
   const saveFileToMediaLibraryAndDeleteOriginal = async (
     fileUri: string,
     downloadId: string,

apps/expo/src/lib/url.ts

@@ -1,6 +0,0 @@
-export const constructFullUrl = (playlistUrl: string, uri: string) => {
-  const baseUrl = playlistUrl.substring(0, playlistUrl.lastIndexOf("/") + 1);
-  return uri.startsWith("http://") || uri.startsWith("https://")
-    ? uri
-    : baseUrl + uri;
-};

packages/provider-utils/src/video.ts

@@ -193,6 +193,50 @@ export async function extractTracksFromHLS(
   }
 }
 
+export async function extractSegmentsFromHLS(
+  playlistUrl: string,
+  headers: Record<string, string>,
+) {
+  try {
+    const response = await fetch(playlistUrl, { headers }).then((res) =>
+      res.text(),
+    );
+    const playlist = hls.parse(response);
+
+    const sortedStreams = playlist.streamRenditions
+      .filter((stream) => stream.properties[0]?.attributes.resolution)
+      .sort((a, b) => {
+        const [widthA, heightA] = (
+          a.properties[0]?.attributes.resolution as string | undefined
+        )
+          ?.split("x")
+          .map(Number) ?? [0, 0];
+        const [widthB, heightB] = (
+          b.properties[0]?.attributes.resolution as string | undefined
+        )
+          ?.split("x")
+          .map(Number) ?? [0, 0];
+        if (!widthA || !heightA || !widthB || !heightB) return 0;
+        return widthB * heightB - widthA * heightA;
+      });
+
+    const highestQuality = sortedStreams[0];
+    if (!highestQuality) return null;
+
+    const highestQualityUri = constructFullUrl(playlistUrl, highestQuality.uri);
+    const highestQualityResponse = await fetch(highestQualityUri, {
+      headers,
+    }).then((res) => res.text());
+    const highestQualityPlaylist = hls.parse(highestQualityResponse);
+
+    return highestQualityPlaylist.segments.map((segment) =>
+      constructFullUrl(highestQualityUri, segment.uri),
+    );
+  } catch (e) {
+    return null;
+  }
+}
+
 export async function convertStreamCaptionsToWebVTT(
   stream: Stream,
 ): Promise<Stream> {
@@ -207,3 +251,10 @@ export async function convertStreamCaptionsToWebVTT(
   }
   return stream;
 }
+
+export const constructFullUrl = (playlistUrl: string, uri: string) => {
+  const baseUrl = playlistUrl.substring(0, playlistUrl.lastIndexOf("/") + 1);
+  return uri.startsWith("http://") || uri.startsWith("https://")
+    ? uri
+    : baseUrl + uri;
+};

pnpm-lock.yaml

@@ -47,6 +47,9 @@ importers:
       '@react-navigation/native':
         specifier: ^6.1.9
         version: 6.1.9(react-native@0.73.6)(react@18.2.0)
+      '@salihgun/react-native-video-processor':
+        specifier: ^0.3.1
+        version: 0.3.1(ffmpeg-kit-react-native@6.0.2)(react-native-video@5.2.1)(react-native@0.73.6)(react@18.2.0)
       '@tamagui/animations-moti':
         specifier: ^1.91.4
         version: 1.91.4(react-dom@18.2.0)(react-native-reanimated@3.6.2)
@@ -3548,6 +3551,20 @@ packages:
       web-streams-polyfill: 3.3.2
     dev: false
 
+  /@salihgun/react-native-video-processor@0.3.1(ffmpeg-kit-react-native@6.0.2)(react-native-video@5.2.1)(react-native@0.73.6)(react@18.2.0):
+    resolution: {integrity: sha512-LBHmH7dp+gxaXZFaVc+OXwLxhHI/zrqyPO7Y7e0NL0k7/hG3ern/y7T4jlSl8lLjz20nqi1BU3sIE+QBEqNJxg==}
+    peerDependencies:
+      ffmpeg-kit-react-native: ^5.1.0
+      react: '*'
+      react-native: '*'
+      react-native-video: ^5.2.1
+    dependencies:
+      ffmpeg-kit-react-native: 6.0.2(react-native@0.73.6)(react@18.2.0)
+      react: 18.2.0
+      react-native: 0.73.6(@babel/core@7.23.9)(@babel/preset-env@7.23.9)(react@18.2.0)
+      react-native-video: 5.2.1
+    dev: false
+
   /@segment/loosely-validate-event@2.0.0:
     resolution: {integrity: sha512-ZMCSfztDBqwotkl848ODgVcAmN4OItEWDCkshcKz0/W6gGSQayuuCtWV/MlodFivAZD793d6UgANd6wCXUfrIw==}
     dependencies:
@@ -7008,6 +7025,14 @@ packages:
     engines: {node: '>= 0.8'}
     dev: false
 
+  /deprecated-react-native-prop-types@2.3.0:
+    resolution: {integrity: sha512-pWD0voFtNYxrVqvBMYf5gq3NA2GCpfodS1yNynTPc93AYA/KEMGeWDqqeUB6R2Z9ZofVhks2aeJXiuQqKNpesA==}
+    dependencies:
+      '@react-native/normalize-color': 2.1.0
+      invariant: 2.2.4
+      prop-types: 15.8.1
+    dev: false
+
   /deprecated-react-native-prop-types@5.0.0:
     resolution: {integrity: sha512-cIK8KYiiGVOFsKdPMmm1L3tA/Gl+JopXL6F5+C7x39MyPsQYnP57Im/D6bNUzcborD7fcMwiwZqcBdBXXZucYQ==}
     engines: {node: '>=18'}
@@ -7164,6 +7189,10 @@ packages:
       minimalistic-crypto-utils: 1.0.1
     dev: false
 
+  /eme-encryption-scheme-polyfill@2.1.1:
+    resolution: {integrity: sha512-njD17wcUrbqCj0ArpLu5zWXtaiupHb/2fIUQGdInf83GlI+Q6mmqaPGLdrke4savKAu15J/z1Tg/ivDgl14g0g==}
+    dev: false
+
   /emoji-regex@8.0.0:
     resolution: {integrity: sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==}
 
@@ -9486,6 +9515,10 @@ packages:
       object.values: 1.1.7
     dev: false
 
+  /keymirror@0.1.1:
+    resolution: {integrity: sha512-vIkZAFWoDijgQT/Nvl2AHCMmnegN2ehgTPYuyy2hWQkQSntI0S7ESYqdLkoSe1HyEBFHHkCgSIvVdSEiWwKvCg==}
+    dev: false
+
   /keyv@4.5.4:
     resolution: {integrity: sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==}
     dependencies:
@@ -11697,6 +11730,15 @@ packages:
       react-native: 0.73.6(@babel/core@7.23.9)(@babel/preset-env@7.23.9)(react@18.2.0)
     dev: false
 
+  /react-native-video@5.2.1:
+    resolution: {integrity: sha512-aJlr9MeTuQ0LpZ4n+EC9RvhoKeiPbLtI2Rxy8u7zo/wzGevbRpWHSBj9xZ5YDBXnAVXzuqyNIkGhdw7bfdIBZw==}
+    dependencies:
+      deprecated-react-native-prop-types: 2.3.0
+      keymirror: 0.1.1
+      prop-types: 15.8.1
+      shaka-player: 2.5.23
+    dev: false
+
   /react-native-web-internals@1.91.4:
     resolution: {integrity: sha512-9mBQxUgdsVUdLHRE42skzDmfCSTDMzL0vN5SGNJzSxq1wIzfOp8S1t/wEilKZVvTQMnuyTNzqv5Nc6VtyIuPpQ==}
     dependencies:
@@ -12352,6 +12394,13 @@ packages:
       safe-buffer: 5.2.1
     dev: false
 
+  /shaka-player@2.5.23:
+    resolution: {integrity: sha512-3MC9k0OXJGw8AZ4n/ZNCZS2yDxx+3as5KgH6Tx4Q5TRboTBBCu6dYPI5vp1DxKeyU12MBN1Zcbs7AKzXv2EnCg==}
+    deprecated: Shaka Player < v4.2 is no longer supported.
+    dependencies:
+      eme-encryption-scheme-polyfill: 2.1.1
+    dev: false
+
   /shallow-clone@3.0.1:
     resolution: {integrity: sha512-/6KqX+GVUdqPuPPd2LxDDxzX6CAbjJehAAOKlNpqqUpAqPM6HeL8f+o3a+JsyGjn2lv0WY8UsTgUJjU9Ok55NA==}
     engines: {node: '>=8'}