Merge branch 'dev' into fix227

.docs/content/2.essentials/0.usage-on-x.md

@@ -39,6 +39,23 @@ const providers = makeProviders({
 ```
 
 ## React native
+To use the library in a react native app, you would also need a couple of polyfills to polyfill crypto and base64.
+
+1. First install the polyfills:
+```bash
+npm install @react-native-anywhere/polyfill-base64 react-native-quick-crypto
+```
+
+2. Add the polyfills to your app:
+```ts
+// Import in your entry file
+import '@react-native-anywhere/polyfill-base64';
+```
+
+And follow the [react-native-quick-crypto documentation](https://github.com/margelo/react-native-quick-crypto) to set up the crypto polyfill.
+
+3. Then you can use the library like this:
+
 ```ts
 import { makeProviders, makeStandardFetcher, targets } from '@movie-web/providers';
 

src/providers/all.ts

@@ -27,6 +27,7 @@ import { vidCloudScraper } from './embeds/vidcloud';
 import { vidplayScraper } from './embeds/vidplay';
 import { wootlyScraper } from './embeds/wootly';
 import { goojaraScraper } from './sources/goojara';
+import { hdRezkaScraper } from './sources/hdrezka';
 import { nepuScraper } from './sources/nepu';
 import { ridooMoviesScraper } from './sources/ridomovies';
 import { smashyStreamScraper } from './sources/smashystream';
@@ -48,6 +49,7 @@ export function gatherAllSources(): Array<Sourcerer> {
     vidSrcToScraper,
     nepuScraper,
     goojaraScraper,
+    hdRezkaScraper,
   ];
 }
 

src/providers/embeds/dood.ts

@@ -30,6 +30,8 @@ export const doodScraper = makeEmbed({
     });
     const downloadURL = `${doodPage}${nanoid()}?token=${dataForLater}&expiry=${Date.now()}`;
 
+    if (!downloadURL.startsWith('http')) throw new Error('Invalid URL');
+
     return {
       stream: [
         {

src/providers/embeds/filemoon/index.ts

@@ -1,5 +1,7 @@
 import { unpack } from 'unpacker';
 
+import { flags } from '@/entrypoint/utils/targets';
+
 import { SubtitleResult } from './types';
 import { makeEmbed } from '../../base';
 import { Caption, getCaptionTypeFromUrl, labelToLanguageCode } from '../../captions';
@@ -49,7 +51,7 @@ export const fileMoonScraper = makeEmbed({
           id: 'primary',
           type: 'hls',
           playlist: file[1],
-          flags: [],
+          flags: [flags.CORS_ALLOWED],
           captions,
         },
       ],

src/providers/embeds/vidplay/index.ts

@@ -1,7 +1,8 @@
+import { flags } from '@/entrypoint/utils/targets';
 import { makeEmbed } from '@/providers/base';
 import { Caption, getCaptionTypeFromUrl, labelToLanguageCode } from '@/providers/captions';
 
-import { getFileUrl, referer } from './common';
+import { getFileUrl } from './common';
 import { SubtitleResult, VidplaySourceResponse } from './types';
 
 export const vidplayScraper = makeEmbed({
@@ -44,12 +45,8 @@ export const vidplayScraper = makeEmbed({
           id: 'primary',
           type: 'hls',
           playlist: source,
-          flags: [],
+          flags: [flags.CORS_ALLOWED],
           captions,
-          preferredHeaders: {
-            Referer: referer,
-            Origin: referer,
-          },
         },
       ],
     };

src/providers/sources/hdrezka/index.ts

@@ -0,0 +1,127 @@
+import { flags } from '@/entrypoint/utils/targets';
+import { SourcererOutput, makeSourcerer } from '@/providers/base';
+import { MovieScrapeContext, ShowScrapeContext } from '@/utils/context';
+import { NotFoundError } from '@/utils/errors';
+
+import { MovieData, VideoLinks } from './types';
+import { extractTitleAndYear, generateRandomFavs, parseSubtitleLinks, parseVideoLinks } from './utils';
+
+const rezkaBase = 'https://hdrzk.org';
+const baseHeaders = {
+  'X-Hdrezka-Android-App': '1',
+  'X-Hdrezka-Android-App-Version': '2.2.0',
+};
+
+async function searchAndFindMediaId(ctx: ShowScrapeContext | MovieScrapeContext): Promise<MovieData | null> {
+  const itemRegexPattern = /<a href="([^"]+)"><span class="enty">([^<]+)<\/span> \(([^)]+)\)/g;
+  const idRegexPattern = /\/(\d+)-[^/]+\.html$/;
+
+  const searchData = await ctx.proxiedFetcher<string>(`/engine/ajax/search.php`, {
+    baseUrl: rezkaBase,
+    headers: baseHeaders,
+    query: { q: ctx.media.title },
+  });
+
+  const movieData: MovieData[] = [];
+
+  for (const match of searchData.matchAll(itemRegexPattern)) {
+    const url = match[1];
+    const titleAndYear = match[3];
+
+    const result = extractTitleAndYear(titleAndYear);
+    if (result !== null) {
+      const id = url.match(idRegexPattern)?.[1] || null;
+
+      movieData.push({ id: id ?? '', year: result.year ?? 0, type: ctx.media.type, url });
+    }
+  }
+
+  const filteredItems = movieData.filter((item) => item.type === ctx.media.type && item.year === ctx.media.releaseYear);
+
+  return filteredItems[0] || null;
+}
+
+async function getStream(
+  id: string,
+  translatorId: string,
+  ctx: ShowScrapeContext | MovieScrapeContext,
+): Promise<VideoLinks> {
+  const searchParams = new URLSearchParams();
+  searchParams.append('id', id);
+  searchParams.append('translator_id', translatorId);
+  if (ctx.media.type === 'show') {
+    searchParams.append('season', ctx.media.season.number.toString());
+    searchParams.append('episode', ctx.media.episode.number.toString());
+  }
+  if (ctx.media.type === 'movie') {
+    searchParams.append('is_camprip', '0');
+    searchParams.append('is_ads', '0');
+    searchParams.append('is_director', '0');
+  }
+  searchParams.append('favs', generateRandomFavs());
+  searchParams.append('action', ctx.media.type === 'show' ? 'get_stream' : 'get_movie');
+
+  const response = await ctx.proxiedFetcher<string>('/ajax/get_cdn_series/', {
+    baseUrl: rezkaBase,
+    method: 'POST',
+    body: searchParams,
+    headers: baseHeaders,
+  });
+
+  // Response content-type is text/html, but it's actually JSON
+  return JSON.parse(response);
+}
+
+async function getTranslatorId(
+  url: string,
+  id: string,
+  ctx: ShowScrapeContext | MovieScrapeContext,
+): Promise<string | null> {
+  const response = await ctx.proxiedFetcher<string>(url, {
+    headers: baseHeaders,
+  });
+
+  // Translator ID 238 represents the Original + subtitles player.
+  if (response.includes(`data-translator_id="238"`)) return '238';
+
+  const functionName = ctx.media.type === 'movie' ? 'initCDNMoviesEvents' : 'initCDNSeriesEvents';
+  const regexPattern = new RegExp(`sof\\.tv\\.${functionName}\\(${id}, ([^,]+)`, 'i');
+  const match = response.match(regexPattern);
+  const translatorId = match ? match[1] : null;
+
+  return translatorId;
+}
+
+const universalScraper = async (ctx: ShowScrapeContext | MovieScrapeContext): Promise<SourcererOutput> => {
+  const result = await searchAndFindMediaId(ctx);
+  if (!result || !result.id) throw new NotFoundError('No result found');
+
+  const translatorId = await getTranslatorId(result.url, result.id, ctx);
+  if (!translatorId) throw new NotFoundError('No translator id found');
+
+  const { url: streamUrl, subtitle: streamSubtitle } = await getStream(result.id, translatorId, ctx);
+  const parsedVideos = parseVideoLinks(streamUrl);
+  const parsedSubtitles = parseSubtitleLinks(streamSubtitle);
+
+  return {
+    embeds: [],
+    stream: [
+      {
+        id: 'primary',
+        type: 'file',
+        flags: [flags.CORS_ALLOWED, flags.IP_LOCKED],
+        captions: parsedSubtitles,
+        qualities: parsedVideos,
+      },
+    ],
+  };
+};
+
+export const hdRezkaScraper = makeSourcerer({
+  id: 'hdrezka',
+  name: 'HDRezka',
+  rank: 195,
+  flags: [flags.CORS_ALLOWED, flags.IP_LOCKED],
+  scrapeShow: universalScraper,
+  scrapeMovie: universalScraper,
+});

src/providers/sources/hdrezka/types.ts

@@ -0,0 +1,20 @@
+import { ScrapeMedia } from '@/index';
+
+export type VideoLinks = {
+  success: boolean;
+  message: string;
+  premium_content: number;
+  url: string;
+  quality: string;
+  subtitle: boolean | string;
+  subtitle_lns: boolean;
+  subtitle_def: boolean;
+  thumbnails: string;
+};
+
+export interface MovieData {
+  id: string | null;
+  year: number;
+  type: ScrapeMedia['type'];
+  url: string;
+}

src/providers/sources/hdrezka/utils.ts

@@ -0,0 +1,76 @@
+import { getCaptionTypeFromUrl, labelToLanguageCode } from '@/providers/captions';
+import { FileBasedStream } from '@/providers/streams';
+import { NotFoundError } from '@/utils/errors';
+import { getValidQualityFromString } from '@/utils/quality';
+
+function generateRandomFavs(): string {
+  const randomHex = () => Math.floor(Math.random() * 16).toString(16);
+  const generateSegment = (length: number) => Array.from({ length }, randomHex).join('');
+
+  return `${generateSegment(8)}-${generateSegment(4)}-${generateSegment(4)}-${generateSegment(4)}-${generateSegment(
+    12,
+  )}`;
+}
+
+function parseSubtitleLinks(inputString?: string | boolean): FileBasedStream['captions'] {
+  if (!inputString || typeof inputString === 'boolean') return [];
+  const linksArray = inputString.split(',');
+  const captions: FileBasedStream['captions'] = [];
+
+  linksArray.forEach((link) => {
+    const match = link.match(/\[([^\]]+)\](https?:\/\/\S+?)(?=,\[|$)/);
+
+    if (match) {
+      const type = getCaptionTypeFromUrl(match[2]);
+      const language = labelToLanguageCode(match[1]);
+      if (!type || !language) return;
+
+      captions.push({
+        id: match[2],
+        language,
+        hasCorsRestrictions: false,
+        type,
+        url: match[2],
+      });
+    }
+  });
+
+  return captions;
+}
+
+function parseVideoLinks(inputString?: string): FileBasedStream['qualities'] {
+  if (!inputString) throw new NotFoundError('No video links found');
+  const linksArray = inputString.split(',');
+  const result: FileBasedStream['qualities'] = {};
+
+  linksArray.forEach((link) => {
+    const match = link.match(/\[([^]+)](https?:\/\/[^\s,]+\.mp4)/);
+    if (match) {
+      const qualityText = match[1];
+      const mp4Url = match[2];
+
+      const numericQualityMatch = qualityText.match(/(\d+p)/);
+      const quality = numericQualityMatch ? numericQualityMatch[1] : 'Unknown';
+
+      console.log(quality, mp4Url);
+      const validQuality = getValidQualityFromString(quality);
+      result[validQuality] = { type: 'mp4', url: mp4Url };
+    }
+  });
+
+  return result;
+}
+
+function extractTitleAndYear(input: string) {
+  const regex = /^(.*?),.*?(\d{4})/;
+  const match = input.match(regex);
+
+  if (match) {
+    const title = match[1];
+    const year = match[2];
+    return { title: title.trim(), year: year ? parseInt(year, 10) : null };
+  }
+  return null;
+}
+
+export { extractTitleAndYear, parseSubtitleLinks, parseVideoLinks, generateRandomFavs };

src/providers/sources/vidsrcto/index.ts

@@ -1,5 +1,6 @@
 import { load } from 'cheerio';
 
+import { flags } from '@/entrypoint/utils/targets';
 import { SourcererEmbed, SourcererOutput, makeSourcerer } from '@/providers/base';
 import { MovieScrapeContext, ShowScrapeContext } from '@/utils/context';
 
@@ -33,7 +34,7 @@ const universalScraper = async (ctx: ShowScrapeContext | MovieScrapeContext): Pr
   if (sources.status !== 200) throw new Error('No sources found');
 
   const embeds: SourcererEmbed[] = [];
-  const embedUrls = [];
+  const embedArr = [];
   for (const source of sources.result) {
     const sourceRes = await ctx.proxiedFetcher<SourceResult>(`/ajax/embed/source/${source.id}`, {
       baseUrl: vidSrcToBase,
@@ -42,28 +43,23 @@ const universalScraper = async (ctx: ShowScrapeContext | MovieScrapeContext): Pr
       },
     });
     const decryptedUrl = decryptSourceUrl(sourceRes.result.url);
-    embedUrls.push(decryptedUrl);
+    embedArr.push({ source: source.title, url: decryptedUrl });
   }
 
-  // Originally Filemoon does not have subtitles. But we can use the ones from Vidplay.
-  const urlWithSubtitles = embedUrls.find((v) => v.includes('sub.info'));
-  let subtitleUrl: string | null = null;
-  if (urlWithSubtitles) subtitleUrl = new URL(urlWithSubtitles).searchParams.get('sub.info');
-
-  for (const source of sources.result) {
-    if (source.title === 'Vidplay') {
-      const embedUrl = embedUrls.find((v) => v.includes('vidplay'));
-      if (!embedUrl) continue;
+  for (const embedObj of embedArr) {
+    if (embedObj.source === 'Vidplay') {
+      const fullUrl = new URL(embedObj.url);
       embeds.push({
         embedId: 'vidplay',
-        url: embedUrl,
+        url: fullUrl.toString(),
       });
     }
 
-    if (source.title === 'Filemoon') {
-      const embedUrl = embedUrls.find((v) => v.includes('filemoon'));
-      if (!embedUrl) continue;
-      const fullUrl = new URL(embedUrl);
+    if (embedObj.source === 'Filemoon') {
+      const fullUrl = new URL(embedObj.url);
+      // Originally Filemoon does not have subtitles. But we can use the ones from Vidplay.
+      const urlWithSubtitles = embedArr.find((v) => v.source === 'Vidplay' && v.url.includes('sub.info'))?.url;
+      const subtitleUrl = urlWithSubtitles ? new URL(urlWithSubtitles).searchParams.get('sub.info') : null;
       if (subtitleUrl) fullUrl.searchParams.set('sub.info', subtitleUrl);
       embeds.push({
         embedId: 'filemoon',
@@ -82,6 +78,6 @@ export const vidSrcToScraper = makeSourcerer({
   name: 'VidSrcTo',
   scrapeMovie: universalScraper,
   scrapeShow: universalScraper,
-  flags: [],
+  flags: [flags.CORS_ALLOWED],
   rank: 300,
 });

src/runners/runner.ts

@@ -1,4 +1,4 @@
-import { FullScraperEvents } from '@/entrypoint/utils/events';
+import { FullScraperEvents, UpdateEvent } from '@/entrypoint/utils/events';
 import { ScrapeMedia } from '@/entrypoint/utils/media';
 import { FeatureMap, flagsAllowedInFeatures } from '@/entrypoint/utils/targets';
 import { UseableFetcher } from '@/fetchers/types';
@@ -38,13 +38,13 @@ export type ProviderRunnerOptions = {
 };
 
 export async function runAllProviders(list: ProviderList, ops: ProviderRunnerOptions): Promise<RunOutput | null> {
-  const sources = reorderOnIdList(ops.sourceOrder ?? [], list.sources).filter((v) => {
-    if (ops.media.type === 'movie') return !!v.scrapeMovie;
-    if (ops.media.type === 'show') return !!v.scrapeShow;
+  const sources = reorderOnIdList(ops.sourceOrder ?? [], list.sources).filter((source) => {
+    if (ops.media.type === 'movie') return !!source.scrapeMovie;
+    if (ops.media.type === 'show') return !!source.scrapeShow;
     return false;
   });
   const embeds = reorderOnIdList(ops.embedOrder ?? [], list.embeds);
-  const embedIds = embeds.map((v) => v.id);
+  const embedIds = embeds.map((embed) => embed.id);
   let lastId = '';
 
   const contextBase: ScrapeContext = {
@@ -63,47 +63,41 @@ export async function runAllProviders(list: ProviderList, ops: ProviderRunnerOpt
     sourceIds: sources.map((v) => v.id),
   });
 
-  for (const s of sources) {
-    ops.events?.start?.(s.id);
-    lastId = s.id;
+  for (const source of sources) {
+    ops.events?.start?.(source.id);
+    lastId = source.id;
 
     // run source scrapers
     let output: SourcererOutput | null = null;
     try {
-      if (ops.media.type === 'movie' && s.scrapeMovie)
-        output = await s.scrapeMovie({
+      if (ops.media.type === 'movie' && source.scrapeMovie)
+        output = await source.scrapeMovie({
           ...contextBase,
           media: ops.media,
         });
-      else if (ops.media.type === 'show' && s.scrapeShow)
-        output = await s.scrapeShow({
+      else if (ops.media.type === 'show' && source.scrapeShow)
+        output = await source.scrapeShow({
           ...contextBase,
           media: ops.media,
         });
       if (output) {
         output.stream = (output.stream ?? [])
-          .filter((stream) => isValidStream(stream))
+          .filter(isValidStream)
           .filter((stream) => flagsAllowedInFeatures(ops.features, stream.flags));
       }
-      if (!output) throw Error('No output');
-      if ((!output.stream || output.stream.length === 0) && output.embeds.length === 0)
+      if (!output || (!output.stream?.length && !output.embeds.length)) {
         throw new NotFoundError('No streams found');
-    } catch (err) {
-      if (err instanceof NotFoundError) {
-        ops.events?.update?.({
-          id: s.id,
-          percentage: 100,
-          status: 'notfound',
-          reason: err.message,
-        });
-        continue;
       }
-      ops.events?.update?.({
-        id: s.id,
+    } catch (error) {
+      const updateParams: UpdateEvent = {
+        id: source.id,
         percentage: 100,
-        status: 'failure',
-        error: err,
-      });
+        status: error instanceof NotFoundError ? 'notfound' : 'failure',
+        reason: error instanceof NotFoundError ? error.message : undefined,
+        error: error instanceof NotFoundError ? undefined : error,
+      };
+
+      ops.events?.update?.(updateParams);
       continue;
     }
     if (!output) throw new Error('Invalid media type');
@@ -111,7 +105,7 @@ export async function runAllProviders(list: ProviderList, ops: ProviderRunnerOpt
     // return stream is there are any
     if (output.stream?.[0]) {
       return {
-        sourceId: s.id,
+        sourceId: source.id,
         stream: output.stream[0],
       };
     }
@@ -120,62 +114,56 @@ export async function runAllProviders(list: ProviderList, ops: ProviderRunnerOpt
     const sortedEmbeds = output.embeds
       .filter((embed) => {
         const e = list.embeds.find((v) => v.id === embed.embedId);
-        if (!e || e.disabled) return false;
-        return true;
+        return e && !e.disabled;
       })
       .sort((a, b) => embedIds.indexOf(a.embedId) - embedIds.indexOf(b.embedId));
 
     if (sortedEmbeds.length > 0) {
       ops.events?.discoverEmbeds?.({
-        embeds: sortedEmbeds.map((v, i) => ({
-          id: [s.id, i].join('-'),
-          embedScraperId: v.embedId,
+        embeds: sortedEmbeds.map((embed, i) => ({
+          id: [source.id, i].join('-'),
+          embedScraperId: embed.embedId,
         })),
-        sourceId: s.id,
+        sourceId: source.id,
       });
     }
 
-    for (const ind in sortedEmbeds) {
-      if (!Object.prototype.hasOwnProperty.call(sortedEmbeds, ind)) continue;
-      const e = sortedEmbeds[ind];
-      const scraper = embeds.find((v) => v.id === e.embedId);
+    for (const [ind, embed] of sortedEmbeds.entries()) {
+      const scraper = embeds.find((v) => v.id === embed.embedId);
       if (!scraper) throw new Error('Invalid embed returned');
 
       // run embed scraper
-      const id = [s.id, ind].join('-');
+      const id = [source.id, ind].join('-');
       ops.events?.start?.(id);
       lastId = id;
+
       let embedOutput: EmbedOutput;
       try {
         embedOutput = await scraper.scrape({
           ...contextBase,
-          url: e.url,
+          url: embed.url,
         });
         embedOutput.stream = embedOutput.stream
-          .filter((stream) => isValidStream(stream))
+          .filter(isValidStream)
           .filter((stream) => flagsAllowedInFeatures(ops.features, stream.flags));
-        if (embedOutput.stream.length === 0) throw new NotFoundError('No streams found');
-      } catch (err) {
-        if (err instanceof NotFoundError) {
-          ops.events?.update?.({
-            id,
-            percentage: 100,
-            status: 'notfound',
-            reason: err.message,
-          });
-          continue;
+        if (embedOutput.stream.length === 0) {
+          throw new NotFoundError('No streams found');
         }
-        ops.events?.update?.({
-          id,
+      } catch (error) {
+        const updateParams: UpdateEvent = {
+          id: source.id,
           percentage: 100,
-          status: 'failure',
-          error: err,
-        });
+          status: error instanceof NotFoundError ? 'notfound' : 'failure',
+          reason: error instanceof NotFoundError ? error.message : undefined,
+          error: error instanceof NotFoundError ? undefined : error,
+        };
+
+        ops.events?.update?.(updateParams);
         continue;
       }
 
       return {
-        sourceId: s.id,
+        sourceId: source.id,
         embedId: scraper.id,
         stream: embedOutput.stream[0],
       };

src/utils/quality.ts

@@ -0,0 +1,20 @@
+import { Qualities } from '@/providers/streams';
+
+export function getValidQualityFromString(quality: string): Qualities {
+  switch (quality.toLowerCase().replace('p', '')) {
+    case '360':
+      return '360';
+    case '480':
+      return '480';
+    case '720':
+      return '720';
+    case '1080':
+      return '1080';
+    case '2160':
+      return '4k';
+    case '4k':
+      return '4k';
+    default:
+      return 'unknown';
+  }
+}