Merge branch 'dev' into feature/filter-duplicate-embeds

.docs/content/1.get-started/4.changelog.md

@@ -2,12 +2,6 @@
 title: 'Changelog'
 ---
 
-# Version 2.2.9
-- Fixed VidSrcTo (both Vidplay and Filemoon embeds) 
-- Added dropload, filelions and vtube embeds to Primewire
-- Fixed and enabled Smashystream
-- Improved RidoMovies search results
-
 # Version 2.2.8
 - Fix package exports for CJS and ESM
 - Fixed Mixdrop embed

.docs/pnpm-lock.yaml

@@ -7370,7 +7370,7 @@ packages:
       fast-glob: 3.3.2
       js-yaml: 4.1.0
       supports-color: 9.4.0
-      undici: 5.28.4
+      undici: 5.28.3
       yargs-parser: 21.1.1
     dev: true
 
@@ -9117,8 +9117,8 @@ packages:
     resolution: {integrity: sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==}
     dev: true
 
-  /undici@5.28.4:
-    resolution: {integrity: sha512-72RFADWFqKmUb2hmmvNODKL3p9hcB6Gt2DOQMis1SEBaV6a4MH8soBvzg+95CYhCKPFedut2JY9bMfrDl9D23g==}
+  /undici@5.28.3:
+    resolution: {integrity: sha512-3ItfzbrhDlINjaP0duwnNsKpDQk3acHI3gVJ1z4fmwMK31k5G9OVIAMLSIaP6w4FaGkaAkN6zaQO9LUvZ1t7VA==}
     engines: {node: '>=14.0'}
     dependencies:
       '@fastify/busboy': 2.1.1

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@movie-web/providers",
-  "version": "2.2.9",
+  "version": "2.2.8",
   "description": "Package that contains all the providers of movie-web",
   "type": "module",
   "main": "./lib/index.js",

src/providers/all.ts

@@ -14,7 +14,6 @@ import { vidsrcembedScraper } from '@/providers/embeds/vidsrc';
 import { vTubeScraper } from '@/providers/embeds/vtube';
 import { flixhqScraper } from '@/providers/sources/flixhq/index';
 import { goMoviesScraper } from '@/providers/sources/gomovies/index';
-import { insertunitScraper } from '@/providers/sources/insertunit';
 import { kissAsianScraper } from '@/providers/sources/kissasian/index';
 import { lookmovieScraper } from '@/providers/sources/lookmovie';
 import { remotestreamScraper } from '@/providers/sources/remotestream';
@@ -25,15 +24,13 @@ import { zoechipScraper } from '@/providers/sources/zoechip';
 import { closeLoadScraper } from './embeds/closeload';
 import { fileMoonScraper } from './embeds/filemoon';
 import { ridooScraper } from './embeds/ridoo';
-import { smashyStreamOScraper } from './embeds/smashystream/opstream';
+import { smashyStreamDScraper } from './embeds/smashystream/dued';
 import { smashyStreamFScraper } from './embeds/smashystream/video1';
 import { streamtapeScraper } from './embeds/streamtape';
 import { streamvidScraper } from './embeds/streamvid';
 import { vidCloudScraper } from './embeds/vidcloud';
 import { vidplayScraper } from './embeds/vidplay';
 import { voeScraper } from './embeds/voe';
-import { warezcdnembedHlsScraper } from './embeds/warezcdn/hls';
-import { warezcdnembedMp4Scraper } from './embeds/warezcdn/mp4';
 import { wootlyScraper } from './embeds/wootly';
 import { goojaraScraper } from './sources/goojara';
 import { hdRezkaScraper } from './sources/hdrezka';
@@ -41,9 +38,7 @@ import { nepuScraper } from './sources/nepu';
 import { primewireScraper } from './sources/primewire';
 import { ridooMoviesScraper } from './sources/ridomovies';
 import { smashyStreamScraper } from './sources/smashystream';
-import { soaperTvScraper } from './sources/soapertv';
 import { vidSrcToScraper } from './sources/vidsrcto';
-import { warezcdnScraper } from './sources/warezcdn';
 
 export function gatherAllSources(): Array<Sourcerer> {
   // all sources are gathered here
@@ -63,9 +58,6 @@ export function gatherAllSources(): Array<Sourcerer> {
     goojaraScraper,
     hdRezkaScraper,
     primewireScraper,
-    warezcdnScraper,
-    insertunitScraper,
-    soaperTvScraper,
   ];
 }
 
@@ -83,7 +75,7 @@ export function gatherAllEmbeds(): Array<Embed> {
     vidsrcembedScraper,
     streambucketScraper,
     smashyStreamFScraper,
-    smashyStreamOScraper,
+    smashyStreamDScraper,
     ridooScraper,
     closeLoadScraper,
     fileMoonScraper,
@@ -96,7 +88,5 @@ export function gatherAllEmbeds(): Array<Embed> {
     droploadScraper,
     filelionsScraper,
     vTubeScraper,
-    warezcdnembedHlsScraper,
-    warezcdnembedMp4Scraper,
   ];
 }

src/providers/embeds/smashystream/dued.ts

@@ -0,0 +1,71 @@
+import { load } from 'cheerio';
+
+import { flags } from '@/entrypoint/utils/targets';
+import { makeEmbed } from '@/providers/base';
+
+type DPlayerSourcesResponse = {
+  title: string;
+  id: string;
+  file: string;
+}[];
+
+export const smashyStreamDScraper = makeEmbed({
+  id: 'smashystream-d',
+  name: 'SmashyStream (D)',
+  rank: 71,
+  async scrape(ctx) {
+    const mainPageRes = await ctx.proxiedFetcher<string>(ctx.url, {
+      headers: {
+        Referer: ctx.url,
+      },
+    });
+    const mainPageRes$ = load(mainPageRes);
+    const iframeUrl = mainPageRes$('iframe').attr('src');
+    if (!iframeUrl) throw new Error(`[${this.name}] failed to find iframe url`);
+    const mainUrl = new URL(iframeUrl);
+    const iframeRes = await ctx.proxiedFetcher<string>(iframeUrl, {
+      headers: {
+        Referer: ctx.url,
+      },
+    });
+    const textFilePath = iframeRes.match(/"file":"([^"]+)"/)?.[1];
+    const csrfToken = iframeRes.match(/"key":"([^"]+)"/)?.[1];
+    if (!textFilePath || !csrfToken) throw new Error(`[${this.name}] failed to find text file url or token`);
+    const textFileUrl = `${mainUrl.origin}${textFilePath}`;
+    const textFileRes = await ctx.proxiedFetcher<DPlayerSourcesResponse>(textFileUrl, {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/x-www-form-urlencoded',
+        'X-CSRF-TOKEN': csrfToken,
+        Referer: iframeUrl,
+      },
+    });
+    // Playlists in Hindi, English, Tamil and Telugu are available. We only get the english one.
+    const textFilePlaylist = textFileRes.find((x) => x.title === 'English')?.file;
+    if (!textFilePlaylist) throw new Error(`[${this.name}] failed to find an english playlist`);
+
+    const playlistRes = await ctx.proxiedFetcher<string>(
+      `${mainUrl.origin}/playlist/${textFilePlaylist.slice(1)}.txt`,
+      {
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/x-www-form-urlencoded',
+          'X-CSRF-TOKEN': csrfToken,
+          Referer: iframeUrl,
+        },
+      },
+    );
+
+    return {
+      stream: [
+        {
+          id: 'primary',
+          playlist: playlistRes,
+          type: 'hls',
+          flags: [flags.CORS_ALLOWED],
+          captions: [],
+        },
+      ],
+    };
+  },
+});

src/providers/embeds/smashystream/opstream.ts

@@ -1,17 +0,0 @@
-import { makeEmbed } from '@/providers/base';
-
-import { smashyStreamFScraper } from './video1';
-
-export const smashyStreamOScraper = makeEmbed({
-  // the scraping logic for all smashystream embeds is the same
-  // all the embeds can be added in the same way
-  id: 'smashystream-o',
-  name: 'SmashyStream (O)',
-  rank: 70,
-  async scrape(ctx) {
-    const result = await smashyStreamFScraper.scrape(ctx);
-    return {
-      stream: result.stream,
-    };
-  },
-});

src/providers/embeds/smashystream/video1.ts

@@ -1,29 +1,16 @@
 import { flags } from '@/entrypoint/utils/targets';
 import { makeEmbed } from '@/providers/base';
 import { Caption, getCaptionTypeFromUrl, labelToLanguageCode } from '@/providers/captions';
-import { NotFoundError } from '@/utils/errors';
 
 type FPlayerResponse = {
   sourceUrls: string[];
-  subtitles: string | null;
+  subtitleUrls: string;
 };
 
-// if you don't understand how this is reversed
-// check https://discord.com/channels/871713465100816424/1186646348137775164/1225644477188935770
-// feel free to reach out atpn or ciaran_ds on discord if you've any problems
-function decode(str: string): string {
-  const b = ['U0ZML2RVN0IvRGx4', 'MGNhL0JWb0kvTlM5', 'Ym94LzJTSS9aU0Zj', 'SGJ0L1dGakIvN0dX', 'eE52L1QwOC96N0Yz'];
-  let formatedB64 = str.slice(2);
-  for (let i = 4; i > -1; i--) {
-    formatedB64 = formatedB64.replace(`//${b[i]}`, '');
-  }
-  return atob(formatedB64);
-}
-
 export const smashyStreamFScraper = makeEmbed({
   id: 'smashystream-f',
   name: 'SmashyStream (F)',
-  rank: 71,
+  rank: 70,
   async scrape(ctx) {
     const res = await ctx.proxiedFetcher<FPlayerResponse>(ctx.url, {
       headers: {
@@ -31,20 +18,15 @@ export const smashyStreamFScraper = makeEmbed({
       },
     });
 
-    if (!res.sourceUrls[0]) throw new NotFoundError('No watchable item found');
-
-    const playlist = decode(res.sourceUrls[0]);
-    if (!playlist.includes('.m3u8')) throw new Error('Failed to decode');
-
     const captions: Caption[] =
-      res.subtitles
-        ?.match(/\[([^\]]+)\](https?:\/\/\S+?)(?=,\[|$)/g)
+      res.subtitleUrls
+        .match(/\[([^\]]+)\](https?:\/\/\S+?)(?=,\[|$)/g)
         ?.map<Caption | null>((entry: string) => {
           const match = entry.match(/\[([^\]]+)\](https?:\/\/\S+?)(?=,\[|$)/);
           if (match) {
             const [, language, url] = match;
             if (language && url) {
-              const languageCode = labelToLanguageCode(language.replace(/ - .*/, ''));
+              const languageCode = labelToLanguageCode(language);
               const captionType = getCaptionTypeFromUrl(url);
               if (!languageCode || !captionType) return null;
               return {
@@ -64,7 +46,7 @@ export const smashyStreamFScraper = makeEmbed({
       stream: [
         {
           id: 'primary',
-          playlist,
+          playlist: res.sourceUrls[0],
           type: 'hls',
           flags: [flags.CORS_ALLOWED],
           captions,

src/providers/embeds/vidsrc.ts

@@ -1,5 +1,5 @@
+import { flags } from '@/entrypoint/utils/targets';
 import { makeEmbed } from '@/providers/base';
-import { vidsrcRCPBase } from '@/providers/sources/vidsrc/common';
 
 const hlsURLRegex = /file:"(.*?)"/;
 const setPassRegex = /var pass_path = "(.*set_pass\.php.*)";/;
@@ -32,35 +32,28 @@ export const vidsrcembedScraper = makeEmbed({
     if (!finalUrl.includes('.m3u8')) throw new Error('Unable to find HLS playlist');
 
     let setPassLink = html.match(setPassRegex)?.[1];
+    if (!setPassLink) throw new Error('Unable to find set_pass.php link');
 
-    // isn't neeeded, the stream works without it anyway
-    // shouldn't fail if the setpass link is not found
-    if (setPassLink) {
-      if (setPassLink.startsWith('//')) {
-        setPassLink = `https:${setPassLink}`;
-      }
-
-      // VidSrc uses a password endpoint to temporarily whitelist the user's IP. This is called in an interval by the player.
-      // It currently has no effect on the player itself, the content plays fine without it.
-      // In the future we might have to introduce hooks for the frontend to call this endpoint.
-      await ctx.proxiedFetcher(setPassLink, {
-        headers: {
-          referer: ctx.url,
-        },
-      });
+    if (setPassLink.startsWith('//')) {
+      setPassLink = `https:${setPassLink}`;
     }
 
+    // VidSrc uses a password endpoint to temporarily whitelist the user's IP. This is called in an interval by the player.
+    // It currently has no effect on the player itself, the content plays fine without it.
+    // In the future we might have to introduce hooks for the frontend to call this endpoint.
+    await ctx.proxiedFetcher(setPassLink, {
+      headers: {
+        referer: ctx.url,
+      },
+    });
+
     return {
       stream: [
         {
           id: 'primary',
           type: 'hls',
           playlist: finalUrl,
-          headers: {
-            Referer: vidsrcRCPBase,
-            Origin: vidsrcRCPBase,
-          },
-          flags: [],
+          flags: [flags.CORS_ALLOWED],
           captions: [],
         },
       ],

src/providers/embeds/warezcdn/common.ts

@@ -1,58 +0,0 @@
-import { warezcdnPlayerBase } from '@/providers/sources/warezcdn/common';
-import { EmbedScrapeContext } from '@/utils/context';
-import { NotFoundError } from '@/utils/errors';
-
-function decrypt(input: string) {
-  let output = atob(input);
-
-  // Remove leading and trailing whitespaces
-  output = output.trim();
-
-  // Reverse the string
-  output = output.split('').reverse().join('');
-
-  // Get the last 5 characters and reverse them
-  let last = output.slice(-5);
-  last = last.split('').reverse().join('');
-
-  // Remove the last 5 characters from the original string
-  output = output.slice(0, -5);
-
-  // Return the original string concatenated with the reversed last 5 characters
-  return `${output}${last}`;
-}
-
-export async function getDecryptedId(ctx: EmbedScrapeContext) {
-  const page = await ctx.proxiedFetcher<string>(`/player.php`, {
-    baseUrl: warezcdnPlayerBase,
-    headers: {
-      Referer: `${warezcdnPlayerBase}/getEmbed.php?${new URLSearchParams({
-        id: ctx.url,
-        sv: 'warezcdn',
-      })}`,
-    },
-    query: {
-      id: ctx.url,
-    },
-  });
-  const allowanceKey = page.match(/let allowanceKey = "(.*?)";/)?.[1];
-  if (!allowanceKey) throw new NotFoundError('Failed to get allowanceKey');
-
-  const streamData = await ctx.proxiedFetcher<string>('/functions.php', {
-    baseUrl: warezcdnPlayerBase,
-    method: 'POST',
-    body: new URLSearchParams({
-      getVideo: ctx.url,
-      key: allowanceKey,
-    }),
-  });
-  const stream = JSON.parse(streamData);
-
-  if (!stream.id) throw new NotFoundError("can't get stream id");
-
-  const decryptedId = decrypt(stream.id);
-
-  if (!decryptedId) throw new NotFoundError("can't get file id");
-
-  return decryptedId;
-}

src/providers/embeds/warezcdn/hls.ts

@@ -1,44 +0,0 @@
-import { flags } from '@/entrypoint/utils/targets';
-import { makeEmbed } from '@/providers/base';
-import { EmbedScrapeContext } from '@/utils/context';
-import { NotFoundError } from '@/utils/errors';
-
-import { getDecryptedId } from './common';
-
-// Method found by atpn
-async function getVideowlUrlStream(ctx: EmbedScrapeContext, decryptedId: string) {
-  const sharePage = await ctx.proxiedFetcher<string>('https://cloud.mail.ru/public/uaRH/2PYWcJRpH');
-  const regex = /"videowl_view":\{"count":"(\d+)","url":"([^"]+)"\}/g;
-  const videowlUrl = regex.exec(sharePage)?.[2];
-
-  if (!videowlUrl) throw new NotFoundError('Failed to get videoOwlUrl');
-
-  return `${videowlUrl}/0p/${btoa(decryptedId)}.m3u8?${new URLSearchParams({
-    double_encode: '1',
-  })}`;
-}
-
-export const warezcdnembedHlsScraper = makeEmbed({
-  id: 'warezcdnembedhls', // WarezCDN is both a source and an embed host
-  name: 'WarezCDN HLS',
-  rank: 83,
-  async scrape(ctx) {
-    const decryptedId = await getDecryptedId(ctx);
-
-    if (!decryptedId) throw new NotFoundError("can't get file id");
-
-    const streamUrl = await getVideowlUrlStream(ctx, decryptedId);
-
-    return {
-      stream: [
-        {
-          id: 'primary',
-          type: 'hls',
-          flags: [flags.IP_LOCKED],
-          captions: [],
-          playlist: streamUrl,
-        },
-      ],
-    };
-  },
-});

src/providers/embeds/warezcdn/mp4.ts

@@ -1,58 +0,0 @@
-import { flags } from '@/entrypoint/utils/targets';
-import { makeEmbed } from '@/providers/base';
-import { warezcdnWorkerProxy } from '@/providers/sources/warezcdn/common';
-import { EmbedScrapeContext } from '@/utils/context';
-import { NotFoundError } from '@/utils/errors';
-
-import { getDecryptedId } from './common';
-
-const cdnListing = [50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64];
-
-async function checkUrls(ctx: EmbedScrapeContext, fileId: string) {
-  for (const id of cdnListing) {
-    const url = `https://cloclo${id}.cloud.mail.ru/weblink/view/${fileId}`;
-    const response = await ctx.proxiedFetcher.full(url, {
-      method: 'GET',
-      headers: {
-        Range: 'bytes=0-1',
-      },
-    });
-    if (response.statusCode === 206) return url;
-  }
-  return null;
-}
-
-export const warezcdnembedMp4Scraper = makeEmbed({
-  id: 'warezcdnembedmp4', // WarezCDN is both a source and an embed host
-  name: 'WarezCDN MP4',
-  rank: 82,
-  disabled: false,
-  async scrape(ctx) {
-    const decryptedId = await getDecryptedId(ctx);
-
-    if (!decryptedId) throw new NotFoundError("can't get file id");
-
-    const streamUrl = await checkUrls(ctx, decryptedId);
-
-    if (!streamUrl) throw new NotFoundError("can't get stream id");
-
-    return {
-      stream: [
-        {
-          id: 'primary',
-          captions: [],
-          qualities: {
-            unknown: {
-              type: 'mp4',
-              url: `${warezcdnWorkerProxy}/?${new URLSearchParams({
-                url: streamUrl,
-              })}`,
-            },
-          },
-          type: 'file',
-          flags: [flags.CORS_ALLOWED],
-        },
-      ],
-    };
-  },
-});

src/providers/sources/insertunit/captions.ts

@@ -1,30 +0,0 @@
-import { Caption, removeDuplicatedLanguages } from '@/providers/captions';
-
-import { Subtitle } from './types';
-
-export async function getCaptions(data: Subtitle[]) {
-  let captions: Caption[] = [];
-  for (const subtitle of data) {
-    let language = '';
-
-    if (subtitle.name.includes('Рус')) {
-      language = 'ru';
-    } else if (subtitle.name.includes('Укр')) {
-      language = 'uk';
-    } else if (subtitle.name.includes('Eng')) {
-      language = 'en';
-    } else {
-      continue;
-    }
-
-    captions.push({
-      id: subtitle.url,
-      url: subtitle.url,
-      language,
-      type: 'vtt',
-      hasCorsRestrictions: false,
-    });
-  }
-  captions = removeDuplicatedLanguages(captions);
-  return captions;
-}

src/providers/sources/insertunit/index.ts

@@ -1,103 +0,0 @@
-import { flags } from '@/entrypoint/utils/targets';
-import { makeSourcerer } from '@/providers/base';
-import { Caption } from '@/providers/captions';
-import { NotFoundError } from '@/utils/errors';
-
-import { getCaptions } from './captions';
-import { Season } from './types';
-
-const insertUnitBase = 'https://api.insertunit.ws/';
-
-export const insertunitScraper = makeSourcerer({
-  id: 'insertunit',
-  name: 'Insertunit',
-  disabled: false,
-  rank: 60,
-  flags: [flags.CORS_ALLOWED],
-  async scrapeShow(ctx) {
-    const playerData = await ctx.fetcher<string>(`/embed/imdb/${ctx.media.imdbId}`, {
-      baseUrl: insertUnitBase,
-    });
-    ctx.progress(30);
-
-    const seasonDataJSONregex = /seasons:(.*)/;
-    const seasonData = seasonDataJSONregex.exec(playerData);
-
-    if (seasonData === null || seasonData[1] === null) {
-      throw new NotFoundError('No result found');
-    }
-    ctx.progress(60);
-
-    const seasonTable: Season[] = JSON.parse(seasonData[1]) as Season[];
-
-    const currentSeason = seasonTable.find(
-      (seasonElement) => seasonElement.season === ctx.media.season.number && !seasonElement.blocked,
-    );
-
-    const currentEpisode = currentSeason?.episodes.find((episodeElement) =>
-      episodeElement.episode.includes(ctx.media.episode.number.toString()),
-    );
-
-    if (!currentEpisode?.hls) throw new NotFoundError('No result found');
-
-    let captions: Caption[] = [];
-
-    if (currentEpisode.cc != null) {
-      captions = await getCaptions(currentEpisode.cc);
-    }
-
-    ctx.progress(95);
-
-    return {
-      embeds: [],
-      stream: [
-        {
-          id: 'primary',
-          playlist: currentEpisode.hls,
-          type: 'hls',
-          flags: [flags.CORS_ALLOWED],
-          captions,
-        },
-      ],
-    };
-  },
-  async scrapeMovie(ctx) {
-    const playerData = await ctx.fetcher<string>(`/embed/imdb/${ctx.media.imdbId}`, {
-      baseUrl: insertUnitBase,
-    });
-    ctx.progress(35);
-
-    const streamRegex = /hls: "([^"]*)/;
-    const streamData = streamRegex.exec(playerData);
-
-    if (streamData === null || streamData[1] === null) {
-      throw new NotFoundError('No result found');
-    }
-    ctx.progress(75);
-
-    const subtitleRegex = /cc: (.*)/;
-    const subtitleJSONData = subtitleRegex.exec(playerData);
-
-    let captions: Caption[] = [];
-
-    if (subtitleJSONData != null && subtitleJSONData[1] != null) {
-      const subtitleData = JSON.parse(subtitleJSONData[1]);
-      captions = await getCaptions(subtitleData);
-    }
-
-    ctx.progress(90);
-
-    return {
-      embeds: [],
-      stream: [
-        {
-          id: 'primary',
-          type: 'hls',
-          playlist: streamData[1],
-          flags: [flags.CORS_ALLOWED],
-          captions,
-        },
-      ],
-    };
-  },
-});

src/providers/sources/insertunit/types.ts

@@ -1,30 +0,0 @@
-export interface Subtitle {
-  url: string;
-  name: string;
-}
-
-export interface Episode {
-  episode: string;
-  id: number;
-  videoKey: string;
-  hls: string;
-  audio: {
-    names: string[];
-    order: number[];
-  };
-  cc: Subtitle[];
-  duration: number;
-  title: string;
-  download: string;
-  sections: string[];
-  poster: string;
-  preview: {
-    src: string;
-  };
-}
-
-export interface Season {
-  season: number;
-  blocked: boolean;
-  episodes: Episode[];
-}

src/providers/sources/ridomovies/index.ts

@@ -19,19 +19,13 @@ const universalScraper = async (ctx: MovieScrapeContext | ShowScrapeContext) =>
       q: ctx.media.title,
     },
   });
-  const mediaData = searchResult.data.items.map((movieEl) => {
-    const name = movieEl.title;
-    const year = movieEl.contentable.releaseYear;
-    const fullSlug = movieEl.fullSlug;
-    return { name, year, fullSlug };
-  });
-  const targetMedia = mediaData.find((m) => m.name === ctx.media.title && m.year === ctx.media.releaseYear.toString());
-  if (!targetMedia?.fullSlug) throw new NotFoundError('No watchable item found');
+  const show = searchResult.data.items[0];
+  if (!show) throw new NotFoundError('No watchable item found');
 
-  let iframeSourceUrl = `/${targetMedia.fullSlug}/videos`;
+  let iframeSourceUrl = `/${show.fullSlug}/videos`;
 
   if (ctx.media.type === 'show') {
-    const showPageResult = await ctx.proxiedFetcher<string>(`/${targetMedia.fullSlug}`, {
+    const showPageResult = await ctx.proxiedFetcher<string>(`/${show.fullSlug}`, {
       baseUrl: ridoMoviesBase,
     });
     const fullEpisodeSlug = `season-${ctx.media.season.number}/episode-${ctx.media.episode.number}`;

src/providers/sources/showbox/index.ts

@@ -42,7 +42,6 @@ export const showboxScraper = makeSourcerer({
   id: 'showbox',
   name: 'Showbox',
   rank: 150,
-  disabled: true,
   flags: [flags.CORS_ALLOWED, flags.CF_BLOCKED],
   scrapeShow: comboScraper,
   scrapeMovie: comboScraper,

src/providers/sources/smashystream/index.ts

@@ -1,29 +1,56 @@
+import { load } from 'cheerio';
+
 import { flags } from '@/entrypoint/utils/targets';
-import { SourcererOutput, makeSourcerer } from '@/providers/base';
-import { smashyStreamOScraper } from '@/providers/embeds/smashystream/opstream';
+import { SourcererEmbed, SourcererOutput, makeSourcerer } from '@/providers/base';
+import { smashyStreamDScraper } from '@/providers/embeds/smashystream/dued';
 import { smashyStreamFScraper } from '@/providers/embeds/smashystream/video1';
 import { MovieScrapeContext, ShowScrapeContext } from '@/utils/context';
 
+const smashyStreamBase = 'https://embed.smashystream.com';
+const referer = 'https://smashystream.com/';
+
 const universalScraper = async (ctx: ShowScrapeContext | MovieScrapeContext): Promise<SourcererOutput> => {
-  // theres no point in fetching the player page
-  // because it too just calls the api with the tmdb id
-  // thats the only way to find out if the embed has any streams
-  const query =
-    ctx.media.type === 'movie'
-      ? `?tmdb=${ctx.media.tmdbId}`
-      : `?tmdbId=${ctx.media.tmdbId}&season=${ctx.media.season.number}&episode=${ctx.media.episode.number}`;
+  const mainPage = await ctx.proxiedFetcher<string>('/playere.php', {
+    query: {
+      tmdb: ctx.media.tmdbId,
+      ...(ctx.media.type === 'show' && {
+        season: ctx.media.season.number.toString(),
+        episode: ctx.media.episode.number.toString(),
+      }),
+    },
+    headers: {
+      Referer: referer,
+    },
+    baseUrl: smashyStreamBase,
+  });
+
+  ctx.progress(30);
+
+  const mainPage$ = load(mainPage);
+  const sourceUrls = mainPage$('.dropdown-menu a[data-url]')
+    .map((_, el) => mainPage$(el).attr('data-url'))
+    .get();
+
+  const embeds: SourcererEmbed[] = [];
+  for (const sourceUrl of sourceUrls) {
+    if (sourceUrl.includes('video1d.php')) {
+      embeds.push({
+        embedId: smashyStreamFScraper.id,
+        url: sourceUrl,
+      });
+    }
+    if (sourceUrl.includes('dued.php')) {
+      embeds.push({
+        embedId: smashyStreamDScraper.id,
+        url: sourceUrl,
+      });
+    }
+  }
+
+  ctx.progress(60);
 
   return {
-    embeds: [
-      {
-        embedId: smashyStreamFScraper.id,
-        url: `https://embed.smashystream.com/video1dn.php${query}`,
-      },
-      {
-        embedId: smashyStreamOScraper.id,
-        url: `https://embed.smashystream.com/videoop.php${query}`,
-      },
-    ],
+    embeds,
   };
 };
 
@@ -32,6 +59,7 @@ export const smashyStreamScraper = makeSourcerer({
   name: 'SmashyStream',
   rank: 30,
   flags: [flags.CORS_ALLOWED],
+  disabled: true,
   scrapeMovie: universalScraper,
   scrapeShow: universalScraper,
 });

src/providers/sources/soapertv/index.ts

@@ -1,120 +0,0 @@
-import { load } from 'cheerio';
-
-import { flags } from '@/entrypoint/utils/targets';
-import { Caption, labelToLanguageCode } from '@/providers/captions';
-import { MovieScrapeContext, ShowScrapeContext } from '@/utils/context';
-import { NotFoundError } from '@/utils/errors';
-
-import { InfoResponse } from './types';
-import { SourcererOutput, makeSourcerer } from '../../base';
-
-const baseUrl = 'https://soaper.tv';
-
-const universalScraper = async (ctx: MovieScrapeContext | ShowScrapeContext): Promise<SourcererOutput> => {
-  const searchResult = await ctx.proxiedFetcher('/search.html', {
-    baseUrl,
-    query: {
-      keyword: ctx.media.title,
-    },
-  });
-  const searchResult$ = load(searchResult);
-  let showLink = searchResult$('a')
-    .filter((_, el) => searchResult$(el).text() === ctx.media.title)
-    .attr('href');
-  if (!showLink) throw new NotFoundError('Content not found');
-
-  if (ctx.media.type === 'show') {
-    const seasonNumber = ctx.media.season.number;
-    const episodeNumber = ctx.media.episode.number;
-    const showPage = await ctx.proxiedFetcher(showLink, { baseUrl });
-    const showPage$ = load(showPage);
-    const seasonBlock = showPage$('h4')
-      .filter((_, el) => showPage$(el).text().trim().split(':')[0].trim() === `Season${seasonNumber}`)
-      .parent();
-    const episodes = seasonBlock.find('a').toArray();
-    showLink = showPage$(
-      episodes.find((el) => parseInt(showPage$(el).text().split('.')[0], 10) === episodeNumber),
-    ).attr('href');
-  }
-  if (!showLink) throw new NotFoundError('Content not found');
-  const contentPage = await ctx.proxiedFetcher(showLink, { baseUrl });
-  const contentPage$ = load(contentPage);
-
-  const pass = contentPage$('#hId').attr('value');
-  const param = contentPage$('#divU').text();
-
-  if (!pass || !param) throw new NotFoundError('Content not found');
-
-  const formData = new URLSearchParams();
-  formData.append('pass', pass);
-  formData.append('param', param);
-  formData.append('e2', '0');
-  formData.append('server', '0');
-
-  const infoEndpoint = ctx.media.type === 'show' ? '/home/index/getEInfoAjax' : '/home/index/getMInfoAjax';
-  const streamRes = await ctx.proxiedFetcher<string>(infoEndpoint, {
-    baseUrl,
-    method: 'POST',
-    body: formData,
-    headers: {
-      referer: `${baseUrl}${showLink}`,
-    },
-  });
-
-  const streamResJson: InfoResponse = JSON.parse(streamRes);
-
-  const captions: Caption[] = [];
-  for (const sub of streamResJson.subs) {
-    // Some subtitles are named <Language>.srt, some are named <LanguageCode>:hi, or just <LanguageCode>
-    let language: string | null = '';
-    if (sub.name.includes('.srt')) {
-      language = labelToLanguageCode(sub.name.split('.srt')[0]);
-    } else if (sub.name.includes(':')) {
-      language = sub.name.split(':')[0];
-    } else {
-      language = sub.name;
-    }
-    if (!language) continue;
-
-    captions.push({
-      id: sub.path,
-      url: sub.path,
-      type: 'srt',
-      hasCorsRestrictions: false,
-      language,
-    });
-  }
-
-  return {
-    embeds: [],
-    stream: [
-      {
-        id: 'primary',
-        playlist: streamResJson.val,
-        type: 'hls',
-        flags: [flags.IP_LOCKED],
-        captions,
-      },
-      ...(streamResJson.val_bak
-        ? [
-            {
-              id: 'backup',
-              playlist: streamResJson.val_bak,
-              type: 'hls' as const,
-              flags: [flags.IP_LOCKED],
-              captions,
-            },
-          ]
-        : []),
-    ],
-  };
-};
-
-export const soaperTvScraper = makeSourcerer({
-  id: 'soapertv',
-  name: 'SoaperTV',
-  rank: 115,
-  flags: [flags.IP_LOCKED],
-  scrapeMovie: universalScraper,
-  scrapeShow: universalScraper,
-});

src/providers/sources/soapertv/types.ts

@@ -1,15 +0,0 @@
-export interface Subtitle {
-  path: string;
-  name: string;
-}
-
-export interface InfoResponse {
-  key: boolean;
-  val: string;
-  vtt: string;
-  val_bak: string;
-  pos: number;
-  type: string;
-  subs: Subtitle[];
-  ip: string;
-}

src/providers/sources/vidsrc/common.ts

@@ -1,2 +1,2 @@
 export const vidsrcBase = 'https://vidsrc.me';
-export const vidsrcRCPBase = 'https://vidsrc.stream';
+export const vidsrcRCPBase = 'https://rcp.vidsrc.me';

src/providers/sources/vidsrc/index.ts

@@ -1,3 +1,4 @@
+import { flags } from '@/entrypoint/utils/targets';
 import { makeSourcerer } from '@/providers/base';
 import { scrapeMovie } from '@/providers/sources/vidsrc/scrape-movie';
 import { scrapeShow } from '@/providers/sources/vidsrc/scrape-show';
@@ -6,7 +7,7 @@ export const vidsrcScraper = makeSourcerer({
   id: 'vidsrc',
   name: 'VidSrc',
   rank: 90,
-  flags: [],
+  flags: [flags.CORS_ALLOWED],
   scrapeMovie,
   scrapeShow,
 });

src/providers/sources/warezcdn/common.ts

@@ -1,4 +0,0 @@
-export const warezcdnBase = 'https://embed.warezcdn.com';
-export const warezcdnApiBase = 'https://warezcdn.com/embed';
-export const warezcdnPlayerBase = 'https://warezcdn.com/player';
-export const warezcdnWorkerProxy = 'https://workerproxy.warezcdn.workers.dev';

src/providers/sources/warezcdn/index.ts

@@ -1,78 +0,0 @@
-import { load } from 'cheerio';
-
-import { flags } from '@/entrypoint/utils/targets';
-import { SourcererEmbed, makeSourcerer } from '@/providers/base';
-import { mixdropScraper } from '@/providers/embeds/mixdrop';
-import { warezcdnembedHlsScraper } from '@/providers/embeds/warezcdn/hls';
-import { warezcdnembedMp4Scraper } from '@/providers/embeds/warezcdn/mp4';
-import { MovieScrapeContext, ShowScrapeContext } from '@/utils/context';
-import { NotFoundError } from '@/utils/errors';
-
-import { warezcdnApiBase, warezcdnBase } from './common';
-
-const universalScraper = async (ctx: MovieScrapeContext | ShowScrapeContext) => {
-  if (!ctx.media.imdbId) throw new NotFoundError('This source requires IMDB id.');
-
-  let id = `filme/${ctx.media.imdbId}`;
-  if (ctx.media.type === 'show')
-    id = `serie/${ctx.media.imdbId}/${ctx.media.season.number}/${ctx.media.episode.number}`;
-
-  const serversPage = await ctx.proxiedFetcher<string>(`/${id}`, {
-    baseUrl: warezcdnBase,
-  });
-  const $ = load(serversPage);
-
-  const embedsHost = $('.hostList.active [data-load-embed]').get();
-
-  const embeds: SourcererEmbed[] = [];
-
-  embedsHost.forEach(async (element) => {
-    const embedHost = $(element).attr('data-load-embed-host')!;
-    const embedUrl = $(element).attr('data-load-embed')!;
-
-    if (embedHost === 'mixdrop') {
-      const params = {
-        id: embedUrl,
-        sv: 'mixdrop',
-      };
-      const realUrl = await ctx.proxiedFetcher<string>(`/getPlay.php`, {
-        baseUrl: warezcdnApiBase,
-        headers: {
-          Referer: `${warezcdnApiBase}/getEmbed.php?${new URLSearchParams(params)}`,
-        },
-        query: params,
-      });
-
-      const realEmbedUrl = realUrl.match(/window\.location\.href="([^"]*)";/);
-      if (!realEmbedUrl) throw new Error('Could not find embed url');
-      embeds.push({
-        embedId: mixdropScraper.id,
-        url: realEmbedUrl[1],
-      });
-    } else if (embedHost === 'warezcdn') {
-      embeds.push(
-        {
-          embedId: warezcdnembedHlsScraper.id,
-          url: embedUrl,
-        },
-        {
-          embedId: warezcdnembedMp4Scraper.id,
-          url: embedUrl,
-        },
-      );
-    }
-  });
-
-  return {
-    embeds,
-  };
-};
-
-export const warezcdnScraper = makeSourcerer({
-  id: 'warezcdn',
-  name: 'WarezCDN',
-  rank: 81,
-  flags: [flags.CORS_ALLOWED],
-  scrapeMovie: universalScraper,
-  scrapeShow: universalScraper,
-});

src/runners/individualRunner.ts

@@ -59,19 +59,21 @@ export async function scrapeInvidualSource(
 
   if (!output) throw new Error('output is null');
 
-  // filter output with only valid embeds that are not disabled
-  output.embeds = output.embeds.filter((embed) => {
-    const e = list.embeds.find((v) => v.id === embed.embedId);
-    if (!e || e.disabled) return false;
-    return true;
-  });
+  // filter output with only valid embeds that are not disabled, and remove duplicates
+  output.embeds = output.embeds
+    .filter((embed) => {
+      const e = list.embeds.find((v) => v.id === embed.embedId);
+      if (!e || e.disabled) return false;
+      return true;
+    })
+    .filter((v, i, a) => a.findIndex((t) => t.embedId === v.embedId) === i);
 
   if ((!output.stream || output.stream.length === 0) && output.embeds.length === 0)
     throw new NotFoundError('No streams found');
 
   // only check for playable streams if there are streams, and if there are no embeds
   if (output.stream && output.stream.length > 0 && output.embeds.length === 0) {
-    const playableStreams = await validatePlayableStreams(output.stream, ops, sourceScraper.id);
+    const playableStreams = await validatePlayableStreams(output.stream, ops);
     if (playableStreams.length === 0) throw new NotFoundError('No playable streams found');
     output.stream = playableStreams;
   }
@@ -112,7 +114,7 @@ export async function scrapeIndividualEmbed(
     .filter((stream) => flagsAllowedInFeatures(ops.features, stream.flags));
   if (output.stream.length === 0) throw new NotFoundError('No streams found');
 
-  const playableStreams = await validatePlayableStreams(output.stream, ops, embedScraper.id);
+  const playableStreams = await validatePlayableStreams(output.stream, ops);
   if (playableStreams.length === 0) throw new NotFoundError('No playable streams found');
   output.stream = playableStreams;
 

src/runners/runner.ts

@@ -104,7 +104,7 @@ export async function runAllProviders(list: ProviderList, ops: ProviderRunnerOpt
 
     // return stream is there are any
     if (output.stream?.[0]) {
-      const playableStream = await validatePlayableStream(output.stream[0], ops, source.id);
+      const playableStream = await validatePlayableStream(output.stream[0], ops);
       if (!playableStream) throw new NotFoundError('No streams found');
       return {
         sourceId: source.id,
@@ -112,12 +112,13 @@ export async function runAllProviders(list: ProviderList, ops: ProviderRunnerOpt
       };
     }
 
-    // filter disabled and run embed scrapers on listed embeds
+    // filter disabled, filter out duplicates, run embed scrapers on listed embeds
     const sortedEmbeds = output.embeds
       .filter((embed) => {
         const e = list.embeds.find((v) => v.id === embed.embedId);
         return e && !e.disabled;
       })
+      .filter((v, i, a) => a.findIndex((t) => t.embedId === v.embedId) === i)
       .sort((a, b) => embedIds.indexOf(a.embedId) - embedIds.indexOf(b.embedId));
 
     if (sortedEmbeds.length > 0) {
@@ -151,7 +152,7 @@ export async function runAllProviders(list: ProviderList, ops: ProviderRunnerOpt
         if (embedOutput.stream.length === 0) {
           throw new NotFoundError('No streams found');
         }
-        const playableStream = await validatePlayableStream(embedOutput.stream[0], ops, embed.embedId);
+        const playableStream = await validatePlayableStream(embedOutput.stream[0], ops);
         if (!playableStream) throw new NotFoundError('No streams found');
         embedOutput.stream = [playableStream];
       } catch (error) {

src/utils/valid.ts

@@ -1,10 +1,7 @@
-import { warezcdnembedMp4Scraper } from '@/providers/embeds/warezcdn/mp4';
 import { Stream } from '@/providers/streams';
 import { IndividualEmbedRunnerOptions } from '@/runners/individualRunner';
 import { ProviderRunnerOptions } from '@/runners/runner';
 
-const SKIP_VALIDATION_CHECK_IDS = [warezcdnembedMp4Scraper.id];
-
 export function isValidStream(stream: Stream | undefined): boolean {
   if (!stream) return false;
   if (stream.type === 'hls') {
@@ -24,10 +21,7 @@ export function isValidStream(stream: Stream | undefined): boolean {
 export async function validatePlayableStream(
   stream: Stream,
   ops: ProviderRunnerOptions | IndividualEmbedRunnerOptions,
-  sourcererId: string,
 ): Promise<Stream | null> {
-  if (SKIP_VALIDATION_CHECK_IDS.includes(sourcererId)) return stream;
-
   if (stream.type === 'hls') {
     const result = await ops.proxiedFetcher.full(stream.playlist, {
       method: 'GET',
@@ -69,11 +63,8 @@ export async function validatePlayableStream(
 export async function validatePlayableStreams(
   streams: Stream[],
   ops: ProviderRunnerOptions | IndividualEmbedRunnerOptions,
-  sourcererId: string,
 ): Promise<Stream[]> {
-  if (SKIP_VALIDATION_CHECK_IDS.includes(sourcererId)) return streams;
-
-  return (await Promise.all(streams.map((stream) => validatePlayableStream(stream, ops, sourcererId)))).filter(
+  return (await Promise.all(streams.map((stream) => validatePlayableStream(stream, ops)))).filter(
     (v) => v !== null,
   ) as Stream[];
 }