Merge 3f68b00ffa into 047b69c309

src/providers/all.ts

@@ -14,7 +14,6 @@ import { vidsrcembedScraper } from '@/providers/embeds/vidsrc';
 import { vTubeScraper } from '@/providers/embeds/vtube';
 import { flixhqScraper } from '@/providers/sources/flixhq/index';
 import { goMoviesScraper } from '@/providers/sources/gomovies/index';
-import { insertunitScraper } from '@/providers/sources/insertunit';
 import { kissAsianScraper } from '@/providers/sources/kissasian/index';
 import { lookmovieScraper } from '@/providers/sources/lookmovie';
 import { remotestreamScraper } from '@/providers/sources/remotestream';
@@ -41,7 +40,6 @@ import { nepuScraper } from './sources/nepu';
 import { primewireScraper } from './sources/primewire';
 import { ridooMoviesScraper } from './sources/ridomovies';
 import { smashyStreamScraper } from './sources/smashystream';
-import { soaperTvScraper } from './sources/soapertv';
 import { vidSrcToScraper } from './sources/vidsrcto';
 import { warezcdnScraper } from './sources/warezcdn';
 
@@ -64,8 +62,6 @@ export function gatherAllSources(): Array<Sourcerer> {
     hdRezkaScraper,
     primewireScraper,
     warezcdnScraper,
-    insertunitScraper,
-    soaperTvScraper,
   ];
 }
 

src/providers/embeds/warezcdn/common.ts

@@ -23,7 +23,7 @@ function decrypt(input: string) {
 }
 
 export async function getDecryptedId(ctx: EmbedScrapeContext) {
-  const page = await ctx.proxiedFetcher<string>(`/player.php`, {
+  const page = await ctx.proxiedFetcher<string>(`/player.php?${new URLSearchParams({ id: ctx.url })}`, {
     baseUrl: warezcdnPlayerBase,
     headers: {
       Referer: `${warezcdnPlayerBase}/getEmbed.php?${new URLSearchParams({
@@ -31,9 +31,6 @@ export async function getDecryptedId(ctx: EmbedScrapeContext) {
         sv: 'warezcdn',
       })}`,
     },
-    query: {
-      id: ctx.url,
-    },
   });
   const allowanceKey = page.match(/let allowanceKey = "(.*?)";/)?.[1];
   if (!allowanceKey) throw new NotFoundError('Failed to get allowanceKey');
@@ -56,3 +53,5 @@ export async function getDecryptedId(ctx: EmbedScrapeContext) {
 
   return decryptedId;
 }
+
+export const warezcdnWorkerProxy = 'https://workerproxy.warezcdn.workers.dev';

src/providers/embeds/warezcdn/hls.ts

@@ -21,7 +21,7 @@ async function getVideowlUrlStream(ctx: EmbedScrapeContext, decryptedId: string)
 export const warezcdnembedHlsScraper = makeEmbed({
   id: 'warezcdnembedhls', // WarezCDN is both a source and an embed host
   name: 'WarezCDN HLS',
-  rank: 83,
+  rank: 82,
   async scrape(ctx) {
     const decryptedId = await getDecryptedId(ctx);
 

src/providers/embeds/warezcdn/mp4.ts

@@ -1,10 +1,8 @@
-import { flags } from '@/entrypoint/utils/targets';
 import { makeEmbed } from '@/providers/base';
-import { warezcdnWorkerProxy } from '@/providers/sources/warezcdn/common';
 import { EmbedScrapeContext } from '@/utils/context';
 import { NotFoundError } from '@/utils/errors';
 
-import { getDecryptedId } from './common';
+import { getDecryptedId, warezcdnWorkerProxy } from './common';
 
 const cdnListing = [50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64];
 
@@ -25,7 +23,7 @@ async function checkUrls(ctx: EmbedScrapeContext, fileId: string) {
 export const warezcdnembedMp4Scraper = makeEmbed({
   id: 'warezcdnembedmp4', // WarezCDN is both a source and an embed host
   name: 'WarezCDN MP4',
-  rank: 82,
+  rank: 83,
   disabled: false,
   async scrape(ctx) {
     const decryptedId = await getDecryptedId(ctx);
@@ -50,7 +48,7 @@ export const warezcdnembedMp4Scraper = makeEmbed({
             },
           },
           type: 'file',
-          flags: [flags.CORS_ALLOWED],
+          flags: [],
         },
       ],
     };

src/providers/sources/insertunit/captions.ts

@@ -1,30 +0,0 @@
-import { Caption, removeDuplicatedLanguages } from '@/providers/captions';
-
-import { Subtitle } from './types';
-
-export async function getCaptions(data: Subtitle[]) {
-  let captions: Caption[] = [];
-  for (const subtitle of data) {
-    let language = '';
-
-    if (subtitle.name.includes('Рус')) {
-      language = 'ru';
-    } else if (subtitle.name.includes('Укр')) {
-      language = 'uk';
-    } else if (subtitle.name.includes('Eng')) {
-      language = 'en';
-    } else {
-      continue;
-    }
-
-    captions.push({
-      id: subtitle.url,
-      url: subtitle.url,
-      language,
-      type: 'vtt',
-      hasCorsRestrictions: false,
-    });
-  }
-  captions = removeDuplicatedLanguages(captions);
-  return captions;
-}

src/providers/sources/insertunit/index.ts

@@ -1,103 +0,0 @@
-import { flags } from '@/entrypoint/utils/targets';
-import { makeSourcerer } from '@/providers/base';
-import { Caption } from '@/providers/captions';
-import { NotFoundError } from '@/utils/errors';
-
-import { getCaptions } from './captions';
-import { Season } from './types';
-
-const insertUnitBase = 'https://api.insertunit.ws/';
-
-export const insertunitScraper = makeSourcerer({
-  id: 'insertunit',
-  name: 'Insertunit',
-  disabled: false,
-  rank: 60,
-  flags: [flags.CORS_ALLOWED],
-  async scrapeShow(ctx) {
-    const playerData = await ctx.fetcher<string>(`/embed/imdb/${ctx.media.imdbId}`, {
-      baseUrl: insertUnitBase,
-    });
-    ctx.progress(30);
-
-    const seasonDataJSONregex = /seasons:(.*)/;
-    const seasonData = seasonDataJSONregex.exec(playerData);
-
-    if (seasonData === null || seasonData[1] === null) {
-      throw new NotFoundError('No result found');
-    }
-    ctx.progress(60);
-
-    const seasonTable: Season[] = JSON.parse(seasonData[1]) as Season[];
-
-    const currentSeason = seasonTable.find(
-      (seasonElement) => seasonElement.season === ctx.media.season.number && !seasonElement.blocked,
-    );
-
-    const currentEpisode = currentSeason?.episodes.find((episodeElement) =>
-      episodeElement.episode.includes(ctx.media.episode.number.toString()),
-    );
-
-    if (!currentEpisode?.hls) throw new NotFoundError('No result found');
-
-    let captions: Caption[] = [];
-
-    if (currentEpisode.cc != null) {
-      captions = await getCaptions(currentEpisode.cc);
-    }
-
-    ctx.progress(95);
-
-    return {
-      embeds: [],
-      stream: [
-        {
-          id: 'primary',
-          playlist: currentEpisode.hls,
-          type: 'hls',
-          flags: [flags.CORS_ALLOWED],
-          captions,
-        },
-      ],
-    };
-  },
-  async scrapeMovie(ctx) {
-    const playerData = await ctx.fetcher<string>(`/embed/imdb/${ctx.media.imdbId}`, {
-      baseUrl: insertUnitBase,
-    });
-    ctx.progress(35);
-
-    const streamRegex = /hls: "([^"]*)/;
-    const streamData = streamRegex.exec(playerData);
-
-    if (streamData === null || streamData[1] === null) {
-      throw new NotFoundError('No result found');
-    }
-    ctx.progress(75);
-
-    const subtitleRegex = /cc: (.*)/;
-    const subtitleJSONData = subtitleRegex.exec(playerData);
-
-    let captions: Caption[] = [];
-
-    if (subtitleJSONData != null && subtitleJSONData[1] != null) {
-      const subtitleData = JSON.parse(subtitleJSONData[1]);
-      captions = await getCaptions(subtitleData);
-    }
-
-    ctx.progress(90);
-
-    return {
-      embeds: [],
-      stream: [
-        {
-          id: 'primary',
-          type: 'hls',
-          playlist: streamData[1],
-          flags: [flags.CORS_ALLOWED],
-          captions,
-        },
-      ],
-    };
-  },
-});

src/providers/sources/insertunit/types.ts

@@ -1,30 +0,0 @@
-export interface Subtitle {
-  url: string;
-  name: string;
-}
-
-export interface Episode {
-  episode: string;
-  id: number;
-  videoKey: string;
-  hls: string;
-  audio: {
-    names: string[];
-    order: number[];
-  };
-  cc: Subtitle[];
-  duration: number;
-  title: string;
-  download: string;
-  sections: string[];
-  poster: string;
-  preview: {
-    src: string;
-  };
-}
-
-export interface Season {
-  season: number;
-  blocked: boolean;
-  episodes: Episode[];
-}

src/providers/sources/soapertv/index.ts

@@ -1,120 +0,0 @@
-import { load } from 'cheerio';
-
-import { flags } from '@/entrypoint/utils/targets';
-import { Caption, labelToLanguageCode } from '@/providers/captions';
-import { MovieScrapeContext, ShowScrapeContext } from '@/utils/context';
-import { NotFoundError } from '@/utils/errors';
-
-import { InfoResponse } from './types';
-import { SourcererOutput, makeSourcerer } from '../../base';
-
-const baseUrl = 'https://soaper.tv';
-
-const universalScraper = async (ctx: MovieScrapeContext | ShowScrapeContext): Promise<SourcererOutput> => {
-  const searchResult = await ctx.proxiedFetcher('/search.html', {
-    baseUrl,
-    query: {
-      keyword: ctx.media.title,
-    },
-  });
-  const searchResult$ = load(searchResult);
-  let showLink = searchResult$('a')
-    .filter((_, el) => searchResult$(el).text() === ctx.media.title)
-    .attr('href');
-  if (!showLink) throw new NotFoundError('Content not found');
-
-  if (ctx.media.type === 'show') {
-    const seasonNumber = ctx.media.season.number;
-    const episodeNumber = ctx.media.episode.number;
-    const showPage = await ctx.proxiedFetcher(showLink, { baseUrl });
-    const showPage$ = load(showPage);
-    const seasonBlock = showPage$('h4')
-      .filter((_, el) => showPage$(el).text().trim().split(':')[0].trim() === `Season${seasonNumber}`)
-      .parent();
-    const episodes = seasonBlock.find('a').toArray();
-    showLink = showPage$(
-      episodes.find((el) => parseInt(showPage$(el).text().split('.')[0], 10) === episodeNumber),
-    ).attr('href');
-  }
-  if (!showLink) throw new NotFoundError('Content not found');
-  const contentPage = await ctx.proxiedFetcher(showLink, { baseUrl });
-  const contentPage$ = load(contentPage);
-
-  const pass = contentPage$('#hId').attr('value');
-  const param = contentPage$('#divU').text();
-
-  if (!pass || !param) throw new NotFoundError('Content not found');
-
-  const formData = new URLSearchParams();
-  formData.append('pass', pass);
-  formData.append('param', param);
-  formData.append('e2', '0');
-  formData.append('server', '0');
-
-  const infoEndpoint = ctx.media.type === 'show' ? '/home/index/getEInfoAjax' : '/home/index/getMInfoAjax';
-  const streamRes = await ctx.proxiedFetcher<string>(infoEndpoint, {
-    baseUrl,
-    method: 'POST',
-    body: formData,
-    headers: {
-      referer: `${baseUrl}${showLink}`,
-    },
-  });
-
-  const streamResJson: InfoResponse = JSON.parse(streamRes);
-
-  const captions: Caption[] = [];
-  for (const sub of streamResJson.subs) {
-    // Some subtitles are named <Language>.srt, some are named <LanguageCode>:hi, or just <LanguageCode>
-    let language: string | null = '';
-    if (sub.name.includes('.srt')) {
-      language = labelToLanguageCode(sub.name.split('.srt')[0]);
-    } else if (sub.name.includes(':')) {
-      language = sub.name.split(':')[0];
-    } else {
-      language = sub.name;
-    }
-    if (!language) continue;
-
-    captions.push({
-      id: sub.path,
-      url: sub.path,
-      type: 'srt',
-      hasCorsRestrictions: false,
-      language,
-    });
-  }
-
-  return {
-    embeds: [],
-    stream: [
-      {
-        id: 'primary',
-        playlist: streamResJson.val,
-        type: 'hls',
-        flags: [flags.IP_LOCKED],
-        captions,
-      },
-      ...(streamResJson.val_bak
-        ? [
-            {
-              id: 'backup',
-              playlist: streamResJson.val_bak,
-              type: 'hls' as const,
-              flags: [flags.IP_LOCKED],
-              captions,
-            },
-          ]
-        : []),
-    ],
-  };
-};
-
-export const soaperTvScraper = makeSourcerer({
-  id: 'soapertv',
-  name: 'SoaperTV',
-  rank: 115,
-  flags: [flags.IP_LOCKED],
-  scrapeMovie: universalScraper,
-  scrapeShow: universalScraper,
-});

src/providers/sources/soapertv/types.ts

@@ -1,15 +0,0 @@
-export interface Subtitle {
-  path: string;
-  name: string;
-}
-
-export interface InfoResponse {
-  key: boolean;
-  val: string;
-  vtt: string;
-  val_bak: string;
-  pos: number;
-  type: string;
-  subs: Subtitle[];
-  ip: string;
-}

src/providers/sources/warezcdn/common.ts

@@ -1,4 +1,3 @@
 export const warezcdnBase = 'https://embed.warezcdn.com';
 export const warezcdnApiBase = 'https://warezcdn.com/embed';
 export const warezcdnPlayerBase = 'https://warezcdn.com/player';
-export const warezcdnWorkerProxy = 'https://workerproxy.warezcdn.workers.dev';

src/providers/sources/warezcdn/index.ts

@@ -1,7 +1,6 @@
 import { load } from 'cheerio';
 
-import { flags } from '@/entrypoint/utils/targets';
+import { SourcererEmbed, SourcererOutput, makeSourcerer } from '@/providers/base';
-import { SourcererEmbed, makeSourcerer } from '@/providers/base';
 import { mixdropScraper } from '@/providers/embeds/mixdrop';
 import { warezcdnembedHlsScraper } from '@/providers/embeds/warezcdn/hls';
 import { warezcdnembedMp4Scraper } from '@/providers/embeds/warezcdn/mp4';
@@ -31,16 +30,15 @@ const universalScraper = async (ctx: MovieScrapeContext | ShowScrapeContext) =>
     const embedUrl = $(element).attr('data-load-embed')!;
 
     if (embedHost === 'mixdrop') {
-      const params = {
+      const params = new URLSearchParams({
         id: embedUrl,
         sv: 'mixdrop',
-      };
+      });
-      const realUrl = await ctx.proxiedFetcher<string>(`/getPlay.php`, {
+      const realUrl = await ctx.proxiedFetcher<string>(`/getPlay.php?${params}`, {
         baseUrl: warezcdnApiBase,
         headers: {
-          Referer: `${warezcdnApiBase}/getEmbed.php?${new URLSearchParams(params)}`,
+          Referer: `${warezcdnApiBase}/getEmbed.php?${params}`,
         },
-        query: params,
       });
 
       const realEmbedUrl = realUrl.match(/window\.location\.href="([^"]*)";/);
@@ -65,14 +63,14 @@ const universalScraper = async (ctx: MovieScrapeContext | ShowScrapeContext) =>
 
   return {
     embeds,
-  };
+  } as SourcererOutput;
 };
 
 export const warezcdnScraper = makeSourcerer({
   id: 'warezcdn',
   name: 'WarezCDN',
   rank: 81,
-  flags: [flags.CORS_ALLOWED],
+  flags: [],
   scrapeMovie: universalScraper,
   scrapeShow: universalScraper,
 });

src/runners/individualRunner.ts

@@ -71,7 +71,7 @@ export async function scrapeInvidualSource(
 
   // only check for playable streams if there are streams, and if there are no embeds
   if (output.stream && output.stream.length > 0 && output.embeds.length === 0) {
-    const playableStreams = await validatePlayableStreams(output.stream, ops, sourceScraper.id);
+    const playableStreams = await validatePlayableStreams(output.stream, ops);
     if (playableStreams.length === 0) throw new NotFoundError('No playable streams found');
     output.stream = playableStreams;
   }
@@ -112,7 +112,7 @@ export async function scrapeIndividualEmbed(
     .filter((stream) => flagsAllowedInFeatures(ops.features, stream.flags));
   if (output.stream.length === 0) throw new NotFoundError('No streams found');
 
-  const playableStreams = await validatePlayableStreams(output.stream, ops, embedScraper.id);
+  const playableStreams = await validatePlayableStreams(output.stream, ops);
   if (playableStreams.length === 0) throw new NotFoundError('No playable streams found');
   output.stream = playableStreams;
 

src/runners/runner.ts

@@ -104,7 +104,7 @@ export async function runAllProviders(list: ProviderList, ops: ProviderRunnerOpt
 
     // return stream is there are any
     if (output.stream?.[0]) {
-      const playableStream = await validatePlayableStream(output.stream[0], ops, source.id);
+      const playableStream = await validatePlayableStream(output.stream[0], ops);
       if (!playableStream) throw new NotFoundError('No streams found');
       return {
         sourceId: source.id,
@@ -151,7 +151,7 @@ export async function runAllProviders(list: ProviderList, ops: ProviderRunnerOpt
         if (embedOutput.stream.length === 0) {
           throw new NotFoundError('No streams found');
         }
-        const playableStream = await validatePlayableStream(embedOutput.stream[0], ops, embed.embedId);
+        const playableStream = await validatePlayableStream(embedOutput.stream[0], ops);
         if (!playableStream) throw new NotFoundError('No streams found');
         embedOutput.stream = [playableStream];
       } catch (error) {

src/utils/valid.ts

@@ -1,10 +1,7 @@
-import { warezcdnembedMp4Scraper } from '@/providers/embeds/warezcdn/mp4';
 import { Stream } from '@/providers/streams';
 import { IndividualEmbedRunnerOptions } from '@/runners/individualRunner';
 import { ProviderRunnerOptions } from '@/runners/runner';
 
-const SKIP_VALIDATION_CHECK_IDS = [warezcdnembedMp4Scraper.id];
-
 export function isValidStream(stream: Stream | undefined): boolean {
   if (!stream) return false;
   if (stream.type === 'hls') {
@@ -24,10 +21,7 @@ export function isValidStream(stream: Stream | undefined): boolean {
 export async function validatePlayableStream(
   stream: Stream,
   ops: ProviderRunnerOptions | IndividualEmbedRunnerOptions,
-  sourcererId: string,
 ): Promise<Stream | null> {
-  if (SKIP_VALIDATION_CHECK_IDS.includes(sourcererId)) return stream;
-
   if (stream.type === 'hls') {
     const result = await ops.proxiedFetcher.full(stream.playlist, {
       method: 'GET',
@@ -69,11 +63,8 @@ export async function validatePlayableStream(
 export async function validatePlayableStreams(
   streams: Stream[],
   ops: ProviderRunnerOptions | IndividualEmbedRunnerOptions,
-  sourcererId: string,
 ): Promise<Stream[]> {
-  if (SKIP_VALIDATION_CHECK_IDS.includes(sourcererId)) return streams;
+  return (await Promise.all(streams.map((stream) => validatePlayableStream(stream, ops)))).filter(
-
-  return (await Promise.all(streams.map((stream) => validatePlayableStream(stream, ops, sourcererId)))).filter(
     (v) => v !== null,
   ) as Stream[];
 }