From 849347afbe830b6a9d693c9ef6181f47dc4ecef7 Mon Sep 17 00:00:00 2001 From: Jonathan Barrow Date: Sat, 30 Sep 2023 17:44:32 -0400 Subject: [PATCH 01/35] add vidsrc source+embed and StreamBucket embed --- .eslintrc.js | 2 + README.md | 1 + src/dev-cli.ts | 9 +- src/fetchers/common.ts | 1 + src/fetchers/fetch.ts | 1 + src/fetchers/standardFetch.ts | 4 + src/fetchers/types.ts | 6 +- src/main/builder.ts | 3 + src/main/individualRunner.ts | 2 + src/providers/all.ts | 23 ++- src/providers/base.ts | 1 + src/providers/embeds/streambucket.ts | 97 ++++++++++++ src/providers/embeds/vidsrc.ts | 50 +++++++ src/providers/sources/vidsrc/common.ts | 13 ++ src/providers/sources/vidsrc/index.ts | 13 ++ src/providers/sources/vidsrc/scrape-movie.ts | 8 + src/providers/sources/vidsrc/scrape-show.ts | 8 + src/providers/sources/vidsrc/scrape.ts | 147 +++++++++++++++++++ src/utils/context.ts | 1 + 19 files changed, 385 insertions(+), 5 deletions(-) create mode 100644 src/providers/embeds/streambucket.ts create mode 100644 src/providers/embeds/vidsrc.ts create mode 100644 src/providers/sources/vidsrc/common.ts create mode 100644 src/providers/sources/vidsrc/index.ts create mode 100644 src/providers/sources/vidsrc/scrape-movie.ts create mode 100644 src/providers/sources/vidsrc/scrape-show.ts create mode 100644 src/providers/sources/vidsrc/scrape.ts diff --git a/.eslintrc.js b/.eslintrc.js index 7939452..d62f0d6 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -18,6 +18,8 @@ module.exports = { }, plugins: ['@typescript-eslint', 'import', 'prettier'], rules: { + 'no-plusplus': 'off', + 'no-bitwise': 'off', 'no-underscore-dangle': 'off', '@typescript-eslint/no-explicit-any': 'off', 'no-console': 'off', diff --git a/README.md b/README.md index d30d021..0042142 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,7 @@ The following CLI Mode arguments are available | `--season` | `-s` | Season number. Only used if type is `show` | `0` | | `--episode` | `-e` | Episode number. Only used if type is `show` | `0` | | `--url` | `-u` | URL to a video embed. Only used if source is an embed | | +| `--headers` | `-h` | Optional headers to send while scraping | | | `--help` | `-h` | Shows help for the command arguments | | Example testing the FlixHQ source on the movie "Spirited Away" diff --git a/src/dev-cli.ts b/src/dev-cli.ts index 4989354..4f72256 100644 --- a/src/dev-cli.ts +++ b/src/dev-cli.ts @@ -37,6 +37,7 @@ type CommandLineArguments = { season: string; episode: string; url: string; + headers?: Record; }; const TMDB_API_KEY = process.env.MOVIE_WEB_TMDB_API_KEY ?? ''; @@ -179,6 +180,7 @@ async function runScraper(providers: ProviderControls, source: MetaOutput, optio const result = await providers.runEmbedScraper({ url: options.url, id: source.id, + headers: options.headers, }); spinnies.succeed('scrape', { text: 'Done!' }); console.log(result); @@ -273,6 +275,10 @@ async function processOptions(options: CommandLineArguments) { } } + if (typeof options.headers === 'string') { + options.headers = JSON.parse(options.headers); + } + let fetcher; if (options.fetcher === 'native') { @@ -403,7 +409,8 @@ async function runCommandLine() { .option('-t, --type ', "Media type. Either 'movie' or 'show'. Only used if source is a provider", 'movie') .option('-s, --season ', "Season number. Only used if type is 'show'", '0') .option('-e, --episode ', "Episode number. Only used if type is 'show'", '0') - .option('-u, --url ', 'URL to a video embed. Only used if source is an embed', ''); + .option('-u, --url ', 'URL to a video embed. Only used if source is an embed', '') + .option('-h, --headers ', 'Optional headers to pass to scrapers. JSON encoded'); program.parse(); diff --git a/src/fetchers/common.ts b/src/fetchers/common.ts index e31b6d1..dff6978 100644 --- a/src/fetchers/common.ts +++ b/src/fetchers/common.ts @@ -34,6 +34,7 @@ export function makeFullFetcher(fetcher: Fetcher): UseableFetcher { query: ops?.query ?? {}, baseUrl: ops?.baseUrl ?? '', body: ops?.body, + returnRaw: ops?.returnRaw, }); }; } diff --git a/src/fetchers/fetch.ts b/src/fetchers/fetch.ts index 1d419f0..8311fb2 100644 --- a/src/fetchers/fetch.ts +++ b/src/fetchers/fetch.ts @@ -17,6 +17,7 @@ export type FetchReply = { text(): Promise; json(): Promise; headers: FetchHeaders; + url: string; }; export type FetchLike = (url: string, ops?: FetchOps | undefined) => Promise; diff --git a/src/fetchers/standardFetch.ts b/src/fetchers/standardFetch.ts index dd84893..bdf14d1 100644 --- a/src/fetchers/standardFetch.ts +++ b/src/fetchers/standardFetch.ts @@ -17,6 +17,10 @@ export function makeStandardFetcher(f: FetchLike): Fetcher { body: seralizedBody.body, }); + if (ops.returnRaw) { + return res; + } + const isJson = res.headers.get('content-type')?.includes('application/json'); if (isJson) return res.json(); return res.text(); diff --git a/src/fetchers/types.ts b/src/fetchers/types.ts index 2d14748..1ad3092 100644 --- a/src/fetchers/types.ts +++ b/src/fetchers/types.ts @@ -4,8 +4,9 @@ export type FetcherOptions = { baseUrl?: string; headers?: Record; query?: Record; - method?: 'GET' | 'POST'; + method?: 'HEAD' | 'GET' | 'POST'; body?: Record | string | FormData | URLSearchParams; + returnRaw?: boolean; }; export type DefaultedFetcherOptions = { @@ -13,7 +14,8 @@ export type DefaultedFetcherOptions = { body?: Record | string | FormData; headers: Record; query: Record; - method: 'GET' | 'POST'; + method: 'HEAD' | 'GET' | 'POST'; + returnRaw?: boolean; }; export type Fetcher = { diff --git a/src/main/builder.ts b/src/main/builder.ts index 0322dbd..a02f298 100644 --- a/src/main/builder.ts +++ b/src/main/builder.ts @@ -57,6 +57,9 @@ export interface EmbedRunnerOptions { // id of the embed scraper you want to scrape from id: string; + + // optional headers for the embed scraper to use + headers?: Record; } export interface ProviderControls { diff --git a/src/main/individualRunner.ts b/src/main/individualRunner.ts index 957edd0..bb64587 100644 --- a/src/main/individualRunner.ts +++ b/src/main/individualRunner.ts @@ -65,6 +65,7 @@ export type IndividualEmbedRunnerOptions = { url: string; id: string; events?: IndividualScraperEvents; + headers?: Record; }; export async function scrapeIndividualEmbed( @@ -78,6 +79,7 @@ export async function scrapeIndividualEmbed( fetcher: ops.fetcher, proxiedFetcher: ops.proxiedFetcher, url: ops.url, + headers: ops.headers, progress(val) { ops.events?.update?.({ id: embedScraper.id, diff --git a/src/providers/all.ts b/src/providers/all.ts index ef26094..00c139f 100644 --- a/src/providers/all.ts +++ b/src/providers/all.ts @@ -1,22 +1,41 @@ import { Embed, Sourcerer } from '@/providers/base'; import { mixdropScraper } from '@/providers/embeds/mixdrop'; import { mp4uploadScraper } from '@/providers/embeds/mp4upload'; +import { streambucketScraper } from '@/providers/embeds/streambucket'; import { streamsbScraper } from '@/providers/embeds/streamsb'; import { upcloudScraper } from '@/providers/embeds/upcloud'; import { upstreamScraper } from '@/providers/embeds/upstream'; +import { vidsrcembedScraper } from '@/providers/embeds/vidsrc'; import { flixhqScraper } from '@/providers/sources/flixhq/index'; import { goMoviesScraper } from '@/providers/sources/gomovies/index'; import { kissAsianScraper } from '@/providers/sources/kissasian/index'; import { remotestreamScraper } from '@/providers/sources/remotestream'; import { superStreamScraper } from '@/providers/sources/superstream/index'; +import { vidsrcScraper } from '@/providers/sources/vidsrc'; import { zoechipScraper } from '@/providers/sources/zoechip'; export function gatherAllSources(): Array { // all sources are gathered here - return [flixhqScraper, remotestreamScraper, kissAsianScraper, superStreamScraper, goMoviesScraper, zoechipScraper]; + return [ + flixhqScraper, + remotestreamScraper, + kissAsianScraper, + superStreamScraper, + goMoviesScraper, + zoechipScraper, + vidsrcScraper, + ]; } export function gatherAllEmbeds(): Array { // all embeds are gathered here - return [upcloudScraper, mp4uploadScraper, streamsbScraper, upstreamScraper, mixdropScraper]; + return [ + upcloudScraper, + mp4uploadScraper, + streamsbScraper, + upstreamScraper, + mixdropScraper, + vidsrcembedScraper, + streambucketScraper, + ]; } diff --git a/src/providers/base.ts b/src/providers/base.ts index 902a5a6..f7bc982 100644 --- a/src/providers/base.ts +++ b/src/providers/base.ts @@ -7,6 +7,7 @@ export type SourcererOutput = { embeds: { embedId: string; url: string; + headers?: Record; }[]; stream?: Stream; }; diff --git a/src/providers/embeds/streambucket.ts b/src/providers/embeds/streambucket.ts new file mode 100644 index 0000000..a3d2689 --- /dev/null +++ b/src/providers/embeds/streambucket.ts @@ -0,0 +1,97 @@ +import { flags } from '@/main/targets'; +import { makeEmbed } from '@/providers/base'; + +// StreamBucket makes use of https://github.com/nicxlau/hunter-php-javascript-obfuscator + +const hunterRegex = /eval\(function\(h,u,n,t,e,r\).*?\("(.*?)",\d*?,"(.*?)",(\d*?),(\d*?),\d*?\)\)/; +const linkRegex = /file:"(.*?)"/; + +// This is a much more simple and optimized version of the "h,u,n,t,e,r" +// obfuscation algorithm. It's just basic chunked+mask encoding. +// I have seen this same encoding used on some sites under the name +// "p,l,a,y,e,r" as well +function decodeHunter(encoded: string, mask: string, charCodeOffset: number, delimiterOffset: number) { + // The encoded string is made up of 'n' number of chunks. + // Each chunk is separated by a delimiter inside the mask. + // This offset is also used as the exponentiation base in + // the charCode calculations + const delimiter = mask[delimiterOffset]; + + // Split the 'encoded' string into chunks using the delimiter, + // and filter out any empty chunks. + const chunks = encoded.split(delimiter).filter((chunk) => chunk); + + // Decode each chunk and concatenate the results to form the final 'decoded' string. + const decoded = chunks + .map((chunk) => { + // Chunks are in reverse order. 'reduceRight' removes the + // need to 'reverse' the array first + const charCode = chunk.split('').reduceRight((c, value, index) => { + // Calculate the character code for each character in the chunk. + // This involves finding the index of 'value' in the 'mask' and + // multiplying it by (delimiterOffset^position). + return c + mask.indexOf(value) * delimiterOffset ** (chunk.length - 1 - index); + }, 0); + + // The actual character code is offset by the given amount + return String.fromCharCode(charCode - charCodeOffset); + }) + .join(''); + + return decoded; +} + +export const streambucketScraper = makeEmbed({ + id: 'streambucket', + name: 'StreamBucket', + rank: 196, + // TODO - Disabled until ctx.fetcher and ctx.proxiedFetcher don't trigger bot detection + disabled: true, + async scrape(ctx) { + // Using the context fetchers make the site return just the string "No bots please!"? + // TODO - Fix this. Native fetch does not trigger this. No idea why right now + const response = await fetch(ctx.url); + const html = await response.text(); + + // This is different than the above mentioned bot detection + if (html.includes('captcha-checkbox')) { + // TODO - This doesn't use recaptcha, just really basic "image match". Maybe could automate? + throw new Error('StreamBucket got captchaed'); + } + + let regexResult = html.match(hunterRegex); + + if (!regexResult) { + throw new Error('Failed to find StreamBucket hunter JavaScript'); + } + + const encoded = regexResult[1]; + const mask = regexResult[2]; + const charCodeOffset = Number(regexResult[3]); + const delimiterOffset = Number(regexResult[4]); + + if (Number.isNaN(charCodeOffset)) { + throw new Error('StreamBucket hunter JavaScript charCodeOffset is not a valid number'); + } + + if (Number.isNaN(delimiterOffset)) { + throw new Error('StreamBucket hunter JavaScript delimiterOffset is not a valid number'); + } + + const decoded = decodeHunter(encoded, mask, charCodeOffset, delimiterOffset); + + regexResult = decoded.match(linkRegex); + + if (!regexResult) { + throw new Error('Failed to find StreamBucket HLS link'); + } + + return { + stream: { + type: 'hls', + playlist: regexResult[1], + flags: [flags.NO_CORS], + }, + }; + }, +}); diff --git a/src/providers/embeds/vidsrc.ts b/src/providers/embeds/vidsrc.ts new file mode 100644 index 0000000..3e025b6 --- /dev/null +++ b/src/providers/embeds/vidsrc.ts @@ -0,0 +1,50 @@ +import { flags } from '@/main/targets'; +import { makeEmbed } from '@/providers/base'; + +const hlsURLRegex = /var hls_url = "(.*?)";/; +const setPassRegex = /var path = "(.*set_pass\.php.*)";/; + +export const vidsrcembedScraper = makeEmbed({ + id: 'vidsrcembed', // VidSrc is both a source and an embed host + name: 'VidSrc', + rank: 197, + async scrape(ctx) { + if (!ctx.headers || (!ctx.headers.referer && !ctx.headers.Referer)) { + throw new Error('VidSrc embeds require the referer header to be set'); + } + const html = await ctx.proxiedFetcher(ctx.url, { + headers: ctx.headers, + }); + + let regexResult = html.match(setPassRegex); + if (!regexResult) { + throw new Error('Unable to find VidSrc set_pass.php link'); + } + + let setPassLink = regexResult[1]; + + if (setPassLink.startsWith('//')) { + setPassLink = `https:${setPassLink}`; + } + + regexResult = html.match(hlsURLRegex); + if (!regexResult) { + throw new Error('Unable to find VidSrc HLS stream'); + } + + // Must call set_pass.php BEFORE using the stream + await fetch(setPassLink, { + headers: { + referer: ctx.url, + }, + }); + + return { + stream: { + type: 'hls', + playlist: regexResult[1], + flags: [flags.NO_CORS], + }, + }; + }, +}); diff --git a/src/providers/sources/vidsrc/common.ts b/src/providers/sources/vidsrc/common.ts new file mode 100644 index 0000000..6f8d773 --- /dev/null +++ b/src/providers/sources/vidsrc/common.ts @@ -0,0 +1,13 @@ +import { MovieMedia, ShowMedia } from '@/main/media'; +import { ScrapeContext } from '@/utils/context'; + +export const vidsrcBase = 'https://vidsrc.me'; +export const vidsrcRCPBase = 'https://rcp.vidsrc.me'; + +export type MovieContext = ScrapeContext & { + media: MovieMedia; +}; + +export type ShowContext = ScrapeContext & { + media: ShowMedia; +}; diff --git a/src/providers/sources/vidsrc/index.ts b/src/providers/sources/vidsrc/index.ts new file mode 100644 index 0000000..f7ad792 --- /dev/null +++ b/src/providers/sources/vidsrc/index.ts @@ -0,0 +1,13 @@ +import { flags } from '@/main/targets'; +import { makeSourcerer } from '@/providers/base'; +import { scrapeMovie } from '@/providers/sources/vidsrc/scrape-movie'; +import { scrapeShow } from '@/providers/sources/vidsrc/scrape-show'; + +export const vidsrcScraper = makeSourcerer({ + id: 'vidsrc', + name: 'VidSrc', + rank: 120, + flags: [flags.NO_CORS], + scrapeMovie, + scrapeShow, +}); diff --git a/src/providers/sources/vidsrc/scrape-movie.ts b/src/providers/sources/vidsrc/scrape-movie.ts new file mode 100644 index 0000000..7e0a796 --- /dev/null +++ b/src/providers/sources/vidsrc/scrape-movie.ts @@ -0,0 +1,8 @@ +import { MovieContext } from '@/providers/sources/vidsrc/common'; +import { getVidSrcMovieSources } from '@/providers/sources/vidsrc/scrape'; + +export async function scrapeMovie(ctx: MovieContext) { + return { + embeds: await getVidSrcMovieSources(ctx), + }; +} diff --git a/src/providers/sources/vidsrc/scrape-show.ts b/src/providers/sources/vidsrc/scrape-show.ts new file mode 100644 index 0000000..7214332 --- /dev/null +++ b/src/providers/sources/vidsrc/scrape-show.ts @@ -0,0 +1,8 @@ +import { ShowContext } from '@/providers/sources/vidsrc/common'; +import { getVidSrcShowSources } from '@/providers/sources/vidsrc/scrape'; + +export async function scrapeShow(ctx: ShowContext) { + return { + embeds: await getVidSrcShowSources(ctx), + }; +} diff --git a/src/providers/sources/vidsrc/scrape.ts b/src/providers/sources/vidsrc/scrape.ts new file mode 100644 index 0000000..a8aa4fc --- /dev/null +++ b/src/providers/sources/vidsrc/scrape.ts @@ -0,0 +1,147 @@ +import { load } from 'cheerio'; + +import { FetchReply } from '@/fetchers/fetch'; +import { streambucketScraper } from '@/providers/embeds/streambucket'; +import { vidsrcembedScraper } from '@/providers/embeds/vidsrc'; +import { MovieContext, ShowContext, vidsrcBase, vidsrcRCPBase } from '@/providers/sources/vidsrc/common'; + +function decodeSrc(encoded: string, seed: string) { + const encodedBuffer = Buffer.from(encoded, 'hex'); + let decoded = ''; + + for (let i = 0; i < encodedBuffer.length; i++) { + decoded += String.fromCharCode(encodedBuffer[i] ^ seed.charCodeAt(i % seed.length)); + } + + return decoded; +} + +async function getVidSrcEmbeds(ctx: MovieContext | ShowContext, startingURL: string) { + // VidSrc works by using hashes and a redirect system. + // The hashes are stored in the html, and VidSrc will + // make requests to their servers with the hash. This + // will trigger a 302 response with a Location header + // sending the user to the correct embed. To get the + // real embed links, we must do the same. Slow, but + // required + + const embeds: { + embedId: string; + url: string; + headers?: Record; + }[] = []; + + let html = await ctx.proxiedFetcher(startingURL, { + baseUrl: vidsrcBase, + }); + + let $ = load(html); + + const sourceHashes = $('.source[data-hash]') + .toArray() + .map((el) => $(el).attr('data-hash')) + .filter((hash) => hash !== undefined); + + for (const hash of sourceHashes) { + html = await ctx.proxiedFetcher(`/rcp/${hash}`, { + baseUrl: vidsrcRCPBase, + headers: { + referer: `${vidsrcBase}${startingURL}`, + }, + }); + + $ = load(html); + const encoded = $('#hidden').attr('data-h'); + const seed = $('body').attr('data-i'); + + if (!encoded || !seed) { + throw new Error('Failed to find encoded iframe src'); + } + + let redirectURL = decodeSrc(encoded, seed); + if (redirectURL.startsWith('//')) { + redirectURL = `https:${redirectURL}`; + } + + // Return the raw fetch response here. + // When a Location header is sent, fetch + // will silently follow it. The "url" inside + // the Response is the final requested URL, + // which is the real embeds URL + const { url: embedURL } = await ctx.proxiedFetcher(redirectURL, { + returnRaw: true, + method: 'HEAD', // We don't care about the actual response body here + headers: { + referer: `${vidsrcRCPBase}/rcp/${hash}`, + }, + }); + + const embed: { + embedId: string; + url: string; + headers?: Record; + } = { + embedId: '', + url: embedURL, + }; + + const parsedUrl = new URL(embedURL); + + switch (parsedUrl.host) { + case 'vidsrc.stream': + embed.embedId = vidsrcembedScraper.id; + embed.headers = { + referer: `${vidsrcRCPBase}/rcp/${hash}`, + }; + break; + case 'streambucket.net': + embed.embedId = streambucketScraper.id; + break; + case '2embed.cc': + case 'www.2embed.cc': + // Just ignore this. This embed just sources from other embeds we can scrape as a 'source' + break; + case 'player-cdn.com': + // Just ignore this. This embed streams video over a custom WebSocket connection + break; + default: + throw new Error(`Failed to find VidSrc embed source for ${embedURL}`); + } + + // Since some embeds are ignored on purpose, check if a valid one was found + if (embed.embedId !== '') { + embeds.push(embed); + } + } + + return embeds; +} + +export async function getVidSrcMovieSources(ctx: MovieContext) { + return getVidSrcEmbeds(ctx, `/embed/${ctx.media.tmdbId}`); +} + +export async function getVidSrcShowSources(ctx: ShowContext) { + // VidSrc will always default to season 1 episode 1 + // no matter what embed URL is used. It sends back + // a list of ALL the shows episodes, in order, for + // all seasons. To get the real embed URL, have to + // parse this from the response + const html = await ctx.proxiedFetcher(`/embed/${ctx.media.tmdbId}`, { + baseUrl: vidsrcBase, + }); + + const $ = load(html); + + const episodeElement = $(`.ep[data-s="${ctx.media.season.number}"][data-e="${ctx.media.episode.number}"]`).first(); + if (episodeElement.length === 0) { + throw new Error('failed to find episode element'); + } + + const startingURL = episodeElement.attr('data-iframe'); + if (!startingURL) { + throw new Error('failed to find episode starting URL'); + } + + return getVidSrcEmbeds(ctx, startingURL); +} diff --git a/src/utils/context.ts b/src/utils/context.ts index 5a9664a..83c645d 100644 --- a/src/utils/context.ts +++ b/src/utils/context.ts @@ -8,6 +8,7 @@ export type ScrapeContext = { export type EmbedInput = { url: string; + headers?: Record; }; export type EmbedScrapeContext = EmbedInput & ScrapeContext; From 64050df3501c18ca302765e61bb8f0c95d9e2d0e Mon Sep 17 00:00:00 2001 From: Jorrin Date: Mon, 25 Dec 2023 22:51:55 +0100 Subject: [PATCH 02/35] fix vidsrc --- src/fetchers/common.ts | 2 +- src/fetchers/types.ts | 2 +- src/providers/embeds/streambucket.ts | 1 + src/providers/embeds/vidsrc.ts | 37 ++++++-------------- src/providers/sources/vidsrc/common.ts | 11 ------ src/providers/sources/vidsrc/scrape-movie.ts | 4 +-- src/providers/sources/vidsrc/scrape-show.ts | 4 +-- src/providers/sources/vidsrc/scrape.ts | 24 +++++-------- 8 files changed, 27 insertions(+), 58 deletions(-) diff --git a/src/fetchers/common.ts b/src/fetchers/common.ts index dff6978..a2b77aa 100644 --- a/src/fetchers/common.ts +++ b/src/fetchers/common.ts @@ -34,7 +34,7 @@ export function makeFullFetcher(fetcher: Fetcher): UseableFetcher { query: ops?.query ?? {}, baseUrl: ops?.baseUrl ?? '', body: ops?.body, - returnRaw: ops?.returnRaw, + returnRaw: ops?.returnRaw ?? false, }); }; } diff --git a/src/fetchers/types.ts b/src/fetchers/types.ts index 1ad3092..8e581a6 100644 --- a/src/fetchers/types.ts +++ b/src/fetchers/types.ts @@ -15,7 +15,7 @@ export type DefaultedFetcherOptions = { headers: Record; query: Record; method: 'HEAD' | 'GET' | 'POST'; - returnRaw?: boolean; + returnRaw: boolean; }; export type Fetcher = { diff --git a/src/providers/embeds/streambucket.ts b/src/providers/embeds/streambucket.ts index a3d2689..f137ceb 100644 --- a/src/providers/embeds/streambucket.ts +++ b/src/providers/embeds/streambucket.ts @@ -91,6 +91,7 @@ export const streambucketScraper = makeEmbed({ type: 'hls', playlist: regexResult[1], flags: [flags.NO_CORS], + captions: [], }, }; }, diff --git a/src/providers/embeds/vidsrc.ts b/src/providers/embeds/vidsrc.ts index 3e025b6..da3f2aa 100644 --- a/src/providers/embeds/vidsrc.ts +++ b/src/providers/embeds/vidsrc.ts @@ -1,8 +1,6 @@ -import { flags } from '@/main/targets'; import { makeEmbed } from '@/providers/base'; -const hlsURLRegex = /var hls_url = "(.*?)";/; -const setPassRegex = /var path = "(.*set_pass\.php.*)";/; +const hlsURLRegex = /file:"(.*?)"/; export const vidsrcembedScraper = makeEmbed({ id: 'vidsrcembed', // VidSrc is both a source and an embed host @@ -16,34 +14,21 @@ export const vidsrcembedScraper = makeEmbed({ headers: ctx.headers, }); - let regexResult = html.match(setPassRegex); - if (!regexResult) { - throw new Error('Unable to find VidSrc set_pass.php link'); - } + const match = html + .match(hlsURLRegex)?.[1] + ?.replace(/(\/\/\S+?=)/g, '') + .replace('#2', ''); + if (!match) throw new Error('Unable to find HLS playlist'); + const finalUrl = atob(match); - let setPassLink = regexResult[1]; - - if (setPassLink.startsWith('//')) { - setPassLink = `https:${setPassLink}`; - } - - regexResult = html.match(hlsURLRegex); - if (!regexResult) { - throw new Error('Unable to find VidSrc HLS stream'); - } - - // Must call set_pass.php BEFORE using the stream - await fetch(setPassLink, { - headers: { - referer: ctx.url, - }, - }); + if (!finalUrl.includes('.m3u8')) throw new Error('Unable to find HLS playlist'); return { stream: { type: 'hls', - playlist: regexResult[1], - flags: [flags.NO_CORS], + playlist: finalUrl, + flags: [], + captions: [], }, }; }, diff --git a/src/providers/sources/vidsrc/common.ts b/src/providers/sources/vidsrc/common.ts index 6f8d773..4ccc93c 100644 --- a/src/providers/sources/vidsrc/common.ts +++ b/src/providers/sources/vidsrc/common.ts @@ -1,13 +1,2 @@ -import { MovieMedia, ShowMedia } from '@/main/media'; -import { ScrapeContext } from '@/utils/context'; - export const vidsrcBase = 'https://vidsrc.me'; export const vidsrcRCPBase = 'https://rcp.vidsrc.me'; - -export type MovieContext = ScrapeContext & { - media: MovieMedia; -}; - -export type ShowContext = ScrapeContext & { - media: ShowMedia; -}; diff --git a/src/providers/sources/vidsrc/scrape-movie.ts b/src/providers/sources/vidsrc/scrape-movie.ts index 7e0a796..585eb31 100644 --- a/src/providers/sources/vidsrc/scrape-movie.ts +++ b/src/providers/sources/vidsrc/scrape-movie.ts @@ -1,7 +1,7 @@ -import { MovieContext } from '@/providers/sources/vidsrc/common'; import { getVidSrcMovieSources } from '@/providers/sources/vidsrc/scrape'; +import { MovieScrapeContext } from '@/utils/context'; -export async function scrapeMovie(ctx: MovieContext) { +export async function scrapeMovie(ctx: MovieScrapeContext) { return { embeds: await getVidSrcMovieSources(ctx), }; diff --git a/src/providers/sources/vidsrc/scrape-show.ts b/src/providers/sources/vidsrc/scrape-show.ts index 7214332..ff5d2a4 100644 --- a/src/providers/sources/vidsrc/scrape-show.ts +++ b/src/providers/sources/vidsrc/scrape-show.ts @@ -1,7 +1,7 @@ -import { ShowContext } from '@/providers/sources/vidsrc/common'; import { getVidSrcShowSources } from '@/providers/sources/vidsrc/scrape'; +import { ShowScrapeContext } from '@/utils/context'; -export async function scrapeShow(ctx: ShowContext) { +export async function scrapeShow(ctx: ShowScrapeContext) { return { embeds: await getVidSrcShowSources(ctx), }; diff --git a/src/providers/sources/vidsrc/scrape.ts b/src/providers/sources/vidsrc/scrape.ts index a8aa4fc..6ea5256 100644 --- a/src/providers/sources/vidsrc/scrape.ts +++ b/src/providers/sources/vidsrc/scrape.ts @@ -1,9 +1,11 @@ import { load } from 'cheerio'; import { FetchReply } from '@/fetchers/fetch'; +import { SourcererEmbed } from '@/providers/base'; import { streambucketScraper } from '@/providers/embeds/streambucket'; import { vidsrcembedScraper } from '@/providers/embeds/vidsrc'; -import { MovieContext, ShowContext, vidsrcBase, vidsrcRCPBase } from '@/providers/sources/vidsrc/common'; +import { vidsrcBase, vidsrcRCPBase } from '@/providers/sources/vidsrc/common'; +import { MovieScrapeContext, ShowScrapeContext } from '@/utils/context'; function decodeSrc(encoded: string, seed: string) { const encodedBuffer = Buffer.from(encoded, 'hex'); @@ -16,7 +18,7 @@ function decodeSrc(encoded: string, seed: string) { return decoded; } -async function getVidSrcEmbeds(ctx: MovieContext | ShowContext, startingURL: string) { +async function getVidSrcEmbeds(ctx: MovieScrapeContext | ShowScrapeContext, startingURL: string) { // VidSrc works by using hashes and a redirect system. // The hashes are stored in the html, and VidSrc will // make requests to their servers with the hash. This @@ -25,11 +27,7 @@ async function getVidSrcEmbeds(ctx: MovieContext | ShowContext, startingURL: str // real embed links, we must do the same. Slow, but // required - const embeds: { - embedId: string; - url: string; - headers?: Record; - }[] = []; + const embeds: SourcererEmbed[] = []; let html = await ctx.proxiedFetcher(startingURL, { baseUrl: vidsrcBase, @@ -37,7 +35,7 @@ async function getVidSrcEmbeds(ctx: MovieContext | ShowContext, startingURL: str let $ = load(html); - const sourceHashes = $('.source[data-hash]') + const sourceHashes = $('.server[data-hash]') .toArray() .map((el) => $(el).attr('data-hash')) .filter((hash) => hash !== undefined); @@ -76,11 +74,7 @@ async function getVidSrcEmbeds(ctx: MovieContext | ShowContext, startingURL: str }, }); - const embed: { - embedId: string; - url: string; - headers?: Record; - } = { + const embed: SourcererEmbed = { embedId: '', url: embedURL, }; @@ -117,11 +111,11 @@ async function getVidSrcEmbeds(ctx: MovieContext | ShowContext, startingURL: str return embeds; } -export async function getVidSrcMovieSources(ctx: MovieContext) { +export async function getVidSrcMovieSources(ctx: MovieScrapeContext) { return getVidSrcEmbeds(ctx, `/embed/${ctx.media.tmdbId}`); } -export async function getVidSrcShowSources(ctx: ShowContext) { +export async function getVidSrcShowSources(ctx: ShowScrapeContext) { // VidSrc will always default to season 1 episode 1 // no matter what embed URL is used. It sends back // a list of ALL the shows episodes, in order, for From 65bbf28442d040474b7c77cf7c3dc00fa383c0a3 Mon Sep 17 00:00:00 2001 From: Jorrin Date: Tue, 26 Dec 2023 23:16:26 +0100 Subject: [PATCH 03/35] merge with dev --- README.md | 1 - src/fetchers/common.ts | 1 - src/fetchers/standardFetch.ts | 4 ---- src/fetchers/types.ts | 2 -- src/providers/embeds/streambucket.ts | 17 ++++++++++------- src/providers/embeds/vidsrc.ts | 23 +++++++++++++---------- src/providers/sources/vidsrc/index.ts | 4 ++-- src/providers/sources/vidsrc/scrape.ts | 24 +++++++----------------- src/utils/context.ts | 10 ++++++++-- 9 files changed, 40 insertions(+), 46 deletions(-) diff --git a/README.md b/README.md index 2b40e6e..8b7633c 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,6 @@ The following CLI Mode arguments are available | `--season` | `-s` | Season number. Only used if type is `show` | `0` | | `--episode` | `-e` | Episode number. Only used if type is `show` | `0` | | `--url` | `-u` | URL to a video embed. Only used if source is an embed | | -| `--headers` | `-h` | Optional headers to send while scraping | | | `--help` | `-h` | Shows help for the command arguments | | Example testing the FlixHQ source on the movie "Spirited Away" diff --git a/src/fetchers/common.ts b/src/fetchers/common.ts index 90e87ce..71956ba 100644 --- a/src/fetchers/common.ts +++ b/src/fetchers/common.ts @@ -35,7 +35,6 @@ export function makeFetcher(fetcher: Fetcher): UseableFetcher { baseUrl: ops?.baseUrl ?? '', readHeaders: ops?.readHeaders ?? [], body: ops?.body, - returnRaw: ops?.returnRaw ?? false, }); }; const output: UseableFetcher = async (url, ops) => (await newFetcher(url, ops)).body; diff --git a/src/fetchers/standardFetch.ts b/src/fetchers/standardFetch.ts index 4eba0d0..9fb6afa 100644 --- a/src/fetchers/standardFetch.ts +++ b/src/fetchers/standardFetch.ts @@ -29,10 +29,6 @@ export function makeStandardFetcher(f: FetchLike): Fetcher { body: seralizedBody.body, }); - if (ops.returnRaw) { - return res; - } - let body: any; const isJson = res.headers.get('content-type')?.includes('application/json'); if (isJson) body = await res.json(); diff --git a/src/fetchers/types.ts b/src/fetchers/types.ts index 09640aa..8904173 100644 --- a/src/fetchers/types.ts +++ b/src/fetchers/types.ts @@ -7,7 +7,6 @@ export type FetcherOptions = { method?: 'HEAD' | 'GET' | 'POST'; readHeaders?: string[]; body?: Record | string | FormData | URLSearchParams; - returnRaw?: boolean; }; // Version of the options that always has the defaults set @@ -18,7 +17,6 @@ export type DefaultedFetcherOptions = { headers: Record; query: Record; method: 'HEAD' | 'GET' | 'POST'; - returnRaw: boolean; readHeaders: string[]; }; diff --git a/src/providers/embeds/streambucket.ts b/src/providers/embeds/streambucket.ts index f137ceb..f22ad42 100644 --- a/src/providers/embeds/streambucket.ts +++ b/src/providers/embeds/streambucket.ts @@ -1,4 +1,4 @@ -import { flags } from '@/main/targets'; +import { flags } from '@/index'; import { makeEmbed } from '@/providers/base'; // StreamBucket makes use of https://github.com/nicxlau/hunter-php-javascript-obfuscator @@ -87,12 +87,15 @@ export const streambucketScraper = makeEmbed({ } return { - stream: { - type: 'hls', - playlist: regexResult[1], - flags: [flags.NO_CORS], - captions: [], - }, + stream: [ + { + id: 'primary', + type: 'hls', + playlist: regexResult[1], + flags: [flags.CORS_ALLOWED], + captions: [], + }, + ], }; }, }); diff --git a/src/providers/embeds/vidsrc.ts b/src/providers/embeds/vidsrc.ts index da3f2aa..efd3a18 100644 --- a/src/providers/embeds/vidsrc.ts +++ b/src/providers/embeds/vidsrc.ts @@ -1,3 +1,4 @@ +import { flags } from '@/../lib'; import { makeEmbed } from '@/providers/base'; const hlsURLRegex = /file:"(.*?)"/; @@ -7,11 +8,10 @@ export const vidsrcembedScraper = makeEmbed({ name: 'VidSrc', rank: 197, async scrape(ctx) { - if (!ctx.headers || (!ctx.headers.referer && !ctx.headers.Referer)) { - throw new Error('VidSrc embeds require the referer header to be set'); - } const html = await ctx.proxiedFetcher(ctx.url, { - headers: ctx.headers, + headers: { + referer: ctx.url, + }, }); const match = html @@ -24,12 +24,15 @@ export const vidsrcembedScraper = makeEmbed({ if (!finalUrl.includes('.m3u8')) throw new Error('Unable to find HLS playlist'); return { - stream: { - type: 'hls', - playlist: finalUrl, - flags: [], - captions: [], - }, + stream: [ + { + id: 'primary', + type: 'hls', + playlist: finalUrl, + flags: [flags.CORS_ALLOWED], + captions: [], + }, + ], }; }, }); diff --git a/src/providers/sources/vidsrc/index.ts b/src/providers/sources/vidsrc/index.ts index f7ad792..6331a05 100644 --- a/src/providers/sources/vidsrc/index.ts +++ b/src/providers/sources/vidsrc/index.ts @@ -1,4 +1,4 @@ -import { flags } from '@/main/targets'; +import { flags } from '@/entrypoint/utils/targets'; import { makeSourcerer } from '@/providers/base'; import { scrapeMovie } from '@/providers/sources/vidsrc/scrape-movie'; import { scrapeShow } from '@/providers/sources/vidsrc/scrape-show'; @@ -7,7 +7,7 @@ export const vidsrcScraper = makeSourcerer({ id: 'vidsrc', name: 'VidSrc', rank: 120, - flags: [flags.NO_CORS], + flags: [flags.CORS_ALLOWED], scrapeMovie, scrapeShow, }); diff --git a/src/providers/sources/vidsrc/scrape.ts b/src/providers/sources/vidsrc/scrape.ts index 6ea5256..65a709c 100644 --- a/src/providers/sources/vidsrc/scrape.ts +++ b/src/providers/sources/vidsrc/scrape.ts @@ -1,6 +1,5 @@ import { load } from 'cheerio'; -import { FetchReply } from '@/fetchers/fetch'; import { SourcererEmbed } from '@/providers/base'; import { streambucketScraper } from '@/providers/embeds/streambucket'; import { vidsrcembedScraper } from '@/providers/embeds/vidsrc'; @@ -44,7 +43,7 @@ async function getVidSrcEmbeds(ctx: MovieScrapeContext | ShowScrapeContext, star html = await ctx.proxiedFetcher(`/rcp/${hash}`, { baseUrl: vidsrcRCPBase, headers: { - referer: `${vidsrcBase}${startingURL}`, + referer: vidsrcBase, }, }); @@ -61,32 +60,23 @@ async function getVidSrcEmbeds(ctx: MovieScrapeContext | ShowScrapeContext, star redirectURL = `https:${redirectURL}`; } - // Return the raw fetch response here. - // When a Location header is sent, fetch - // will silently follow it. The "url" inside - // the Response is the final requested URL, - // which is the real embeds URL - const { url: embedURL } = await ctx.proxiedFetcher(redirectURL, { - returnRaw: true, - method: 'HEAD', // We don't care about the actual response body here + const { finalUrl } = await ctx.proxiedFetcher.full(redirectURL, { + method: 'HEAD', headers: { - referer: `${vidsrcRCPBase}/rcp/${hash}`, + referer: vidsrcBase, }, }); const embed: SourcererEmbed = { embedId: '', - url: embedURL, + url: finalUrl, }; - const parsedUrl = new URL(embedURL); + const parsedUrl = new URL(finalUrl); switch (parsedUrl.host) { case 'vidsrc.stream': embed.embedId = vidsrcembedScraper.id; - embed.headers = { - referer: `${vidsrcRCPBase}/rcp/${hash}`, - }; break; case 'streambucket.net': embed.embedId = streambucketScraper.id; @@ -99,7 +89,7 @@ async function getVidSrcEmbeds(ctx: MovieScrapeContext | ShowScrapeContext, star // Just ignore this. This embed streams video over a custom WebSocket connection break; default: - throw new Error(`Failed to find VidSrc embed source for ${embedURL}`); + throw new Error(`Failed to find VidSrc embed source for ${finalUrl}`); } // Since some embeds are ignored on purpose, check if a valid one was found diff --git a/src/utils/context.ts b/src/utils/context.ts index cd11f21..f7c005e 100644 --- a/src/utils/context.ts +++ b/src/utils/context.ts @@ -2,8 +2,14 @@ import { MovieMedia, ShowMedia } from '@/entrypoint/utils/media'; import { UseableFetcher } from '@/fetchers/types'; export type ScrapeContext = { - proxiedFetcher: (...params: Parameters>) => ReturnType>; - fetcher: (...params: Parameters>) => ReturnType>; + proxiedFetcher: { + (...params: Parameters>): ReturnType>; + full(...params: Parameters['full']>): ReturnType['full']>; + }; + fetcher: { + (...params: Parameters>): ReturnType>; + full(...params: Parameters['full']>): ReturnType['full']>; + }; progress(val: number): void; }; From 2117b417f4b225885cec796de55e5b97934eca6d Mon Sep 17 00:00:00 2001 From: Jorrin Date: Tue, 26 Dec 2023 23:23:20 +0100 Subject: [PATCH 04/35] fix ci --- src/providers/embeds/streambucket.ts | 2 +- src/providers/embeds/vidsrc.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/providers/embeds/streambucket.ts b/src/providers/embeds/streambucket.ts index f22ad42..9e21a93 100644 --- a/src/providers/embeds/streambucket.ts +++ b/src/providers/embeds/streambucket.ts @@ -1,4 +1,4 @@ -import { flags } from '@/index'; +import { flags } from '@/entrypoint/utils/targets'; import { makeEmbed } from '@/providers/base'; // StreamBucket makes use of https://github.com/nicxlau/hunter-php-javascript-obfuscator diff --git a/src/providers/embeds/vidsrc.ts b/src/providers/embeds/vidsrc.ts index efd3a18..d9eafb3 100644 --- a/src/providers/embeds/vidsrc.ts +++ b/src/providers/embeds/vidsrc.ts @@ -1,4 +1,4 @@ -import { flags } from '@/../lib'; +import { flags } from '@/entrypoint/utils/targets'; import { makeEmbed } from '@/providers/base'; const hlsURLRegex = /file:"(.*?)"/; From 06acec4675fca367a8496a79632850caada4beab Mon Sep 17 00:00:00 2001 From: Jorrin Date: Wed, 27 Dec 2023 00:21:13 +0100 Subject: [PATCH 05/35] remove references of embed headers --- src/entrypoint/controls.ts | 3 --- src/fetchers/types.ts | 2 +- src/providers/base.ts | 1 - src/runners/individualRunner.ts | 2 -- src/utils/context.ts | 1 - 5 files changed, 1 insertion(+), 8 deletions(-) diff --git a/src/entrypoint/controls.ts b/src/entrypoint/controls.ts index 2ac7898..5ff400b 100644 --- a/src/entrypoint/controls.ts +++ b/src/entrypoint/controls.ts @@ -52,9 +52,6 @@ export interface EmbedRunnerOptions { // id of the embed scraper you want to scrape from id: string; - - // optional headers for the embed scraper to use - headers?: Record; } export interface ProviderControls { diff --git a/src/fetchers/types.ts b/src/fetchers/types.ts index 8904173..d542298 100644 --- a/src/fetchers/types.ts +++ b/src/fetchers/types.ts @@ -16,8 +16,8 @@ export type DefaultedFetcherOptions = { body?: Record | string | FormData; headers: Record; query: Record; - method: 'HEAD' | 'GET' | 'POST'; readHeaders: string[]; + method: 'HEAD' | 'GET' | 'POST'; }; export type FetcherResponse = { diff --git a/src/providers/base.ts b/src/providers/base.ts index 1625b50..0d43895 100644 --- a/src/providers/base.ts +++ b/src/providers/base.ts @@ -7,7 +7,6 @@ export type MediaScraperTypes = 'show' | 'movie'; export type SourcererEmbed = { embedId: string; url: string; - headers?: Record; }; export type SourcererOutput = { diff --git a/src/runners/individualRunner.ts b/src/runners/individualRunner.ts index 43730c7..2befd84 100644 --- a/src/runners/individualRunner.ts +++ b/src/runners/individualRunner.ts @@ -70,7 +70,6 @@ export type IndividualEmbedRunnerOptions = { url: string; id: string; events?: IndividualScraperEvents; - headers?: Record; }; export async function scrapeIndividualEmbed( @@ -84,7 +83,6 @@ export async function scrapeIndividualEmbed( fetcher: ops.fetcher, proxiedFetcher: ops.proxiedFetcher, url: ops.url, - headers: ops.headers, progress(val) { ops.events?.update?.({ id: embedScraper.id, diff --git a/src/utils/context.ts b/src/utils/context.ts index f7c005e..cf7acc7 100644 --- a/src/utils/context.ts +++ b/src/utils/context.ts @@ -15,7 +15,6 @@ export type ScrapeContext = { export type EmbedInput = { url: string; - headers?: Record; }; export type EmbedScrapeContext = EmbedInput & ScrapeContext; From 8b7e840814a5322eb90470ae49432036f30fb566 Mon Sep 17 00:00:00 2001 From: Jorrin Date: Wed, 27 Dec 2023 02:03:53 +0100 Subject: [PATCH 06/35] vidsrcto draft --- .eslintrc.js | 1 + src/dev-cli/tmdb.ts | 2 + src/providers/all.ts | 6 +++ src/providers/embeds/filemoon.ts | 34 ++++++++++++++ src/providers/embeds/vidplay/common.ts | 53 ++++++++++++++++++++++ src/providers/embeds/vidplay/index.ts | 32 +++++++++++++ src/providers/embeds/vidplay/types.ts | 11 +++++ src/providers/sources/vidsrcto/common.ts | 51 +++++++++++++++++++++ src/providers/sources/vidsrcto/index.ts | 58 ++++++++++++++++++++++++ src/providers/sources/vidsrcto/types.ts | 15 ++++++ 10 files changed, 263 insertions(+) create mode 100644 src/providers/embeds/filemoon.ts create mode 100644 src/providers/embeds/vidplay/common.ts create mode 100644 src/providers/embeds/vidplay/index.ts create mode 100644 src/providers/embeds/vidplay/types.ts create mode 100644 src/providers/sources/vidsrcto/common.ts create mode 100644 src/providers/sources/vidsrcto/index.ts create mode 100644 src/providers/sources/vidsrcto/types.ts diff --git a/.eslintrc.js b/.eslintrc.js index 0e7322b..2bb81f9 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -18,6 +18,7 @@ module.exports = { }, plugins: ['@typescript-eslint', 'import', 'prettier'], rules: { + 'no-bitwise': 'off', 'no-underscore-dangle': 'off', '@typescript-eslint/no-explicit-any': 'off', 'no-console': 'off', diff --git a/src/dev-cli/tmdb.ts b/src/dev-cli/tmdb.ts index 7490336..90e87f8 100644 --- a/src/dev-cli/tmdb.ts +++ b/src/dev-cli/tmdb.ts @@ -45,6 +45,7 @@ export async function getMovieMediaDetails(id: string): Promise { title: movie.title, releaseYear: Number(movie.release_date.split('-')[0]), tmdbId: id, + imdbId: movie.imdb_id, }; } @@ -91,5 +92,6 @@ export async function getShowMediaDetails(id: string, seasonNumber: string, epis number: season.season_number, tmdbId: season.id, }, + imdbId: series.imdb_id, }; } diff --git a/src/providers/all.ts b/src/providers/all.ts index d1e7885..07458b9 100644 --- a/src/providers/all.ts +++ b/src/providers/all.ts @@ -14,9 +14,12 @@ import { remotestreamScraper } from '@/providers/sources/remotestream'; import { showboxScraper } from '@/providers/sources/showbox/index'; import { zoechipScraper } from '@/providers/sources/zoechip'; +import { fileMoonScraper } from './embeds/filemoon'; import { smashyStreamDScraper } from './embeds/smashystream/dued'; import { smashyStreamFScraper } from './embeds/smashystream/video1'; +import { vidplayScraper } from './embeds/vidplay'; import { smashyStreamScraper } from './sources/smashystream'; +import { vidSrcToScraper } from './sources/vidsrcto'; export function gatherAllSources(): Array { // all sources are gathered here @@ -29,6 +32,7 @@ export function gatherAllSources(): Array { zoechipScraper, lookmovieScraper, smashyStreamScraper, + vidSrcToScraper, ]; } @@ -44,5 +48,7 @@ export function gatherAllEmbeds(): Array { mixdropScraper, smashyStreamFScraper, smashyStreamDScraper, + fileMoonScraper, + vidplayScraper, ]; } diff --git a/src/providers/embeds/filemoon.ts b/src/providers/embeds/filemoon.ts new file mode 100644 index 0000000..9b092a2 --- /dev/null +++ b/src/providers/embeds/filemoon.ts @@ -0,0 +1,34 @@ +import { unpack } from 'unpacker'; + +import { flags } from '@/entrypoint/utils/targets'; + +import { makeEmbed } from '../base'; + +const evalCodeRegex = /eval\((.*)\)/g; +const fileRegex = /file:"(.*?)"/g; + +export const fileMoonScraper = makeEmbed({ + id: 'filemoon', + name: 'FileMoon', + rank: 501, + scrape: async (ctx) => { + const embedRes = await ctx.fetcher(ctx.url); + const evalCode = evalCodeRegex.exec(embedRes); + if (!evalCode) throw new Error('Failed to find eval code'); + const unpacked = unpack(evalCode[1]); + const file = fileRegex.exec(unpacked); + if (!file?.[1]) throw new Error('Failed to find file'); + + return { + stream: [ + { + id: 'primary', + type: 'hls', + playlist: file[1], + flags: [flags.CORS_ALLOWED], + captions: [], + }, + ], + }; + }, +}); diff --git a/src/providers/embeds/vidplay/common.ts b/src/providers/embeds/vidplay/common.ts new file mode 100644 index 0000000..9fbc8d2 --- /dev/null +++ b/src/providers/embeds/vidplay/common.ts @@ -0,0 +1,53 @@ +import { createCipheriv } from 'crypto'; +import { Buffer } from 'node:buffer'; + +import { EmbedScrapeContext } from '@/utils/context'; + +export const vidplayBase = 'https://vidplay.site'; + +export const getDecryptionKeys = async (ctx: EmbedScrapeContext): Promise => { + const res = await ctx.fetcher( + 'https://raw.githubusercontent.com/Claudemirovsky/worstsource-keys/keys/keys.json', + ); + return JSON.parse(res); +}; + +// TODO: Fix this so its cross platform compatible +export const getEncodedId = async (ctx: EmbedScrapeContext) => { + const id = ctx.url.split('/e/')[1].split('?')[0]; + const keys = await getDecryptionKeys(ctx); + const c1 = createCipheriv('rc4', Buffer.from(keys[0]), ''); + const c2 = createCipheriv('rc4', Buffer.from(keys[1]), ''); + + let input = Buffer.from(id); + input = Buffer.concat([c1.update(input), c1.final()]); + input = Buffer.concat([c2.update(input), c2.final()]); + + return input.toString('base64').replace('/', '_'); +}; + +export const getFuTokenKey = async (ctx: EmbedScrapeContext) => { + const id = await getEncodedId(ctx); + console.log(`ENCODED ID: ${id}`); + const fuTokenRes = await ctx.proxiedFetcher('/futoken', { + baseUrl: vidplayBase, + headers: { + referer: ctx.url, + }, + }); + const fuKey = fuTokenRes.match(/var\s+k\s*=\s*'([^']+)'/)?.[1]; + console.log(`FU KEY: ${fuKey}`); + if (!fuKey) throw new Error('No fuKey found'); + const tokens = []; + for (let i = 0; i < id.length; i += 1) { + tokens.push(fuKey.charCodeAt(i % fuKey.length) + id.charCodeAt(i)); + } + console.log(`${fuKey},${tokens.join(',')}`); + return `${fuKey},${tokens.join(',')}`; +}; + +export const getFileUrl = async (ctx: EmbedScrapeContext) => { + console.log(ctx.url); + const fuToken = await getFuTokenKey(ctx); + return `${vidplayBase}/mediainfo/${fuToken}?${ctx.url.split('?')[1]}`; +}; diff --git a/src/providers/embeds/vidplay/index.ts b/src/providers/embeds/vidplay/index.ts new file mode 100644 index 0000000..93243e5 --- /dev/null +++ b/src/providers/embeds/vidplay/index.ts @@ -0,0 +1,32 @@ +import { makeEmbed } from '@/providers/base'; + +import { getFileUrl } from './common'; +import { VidplaySourceResponse } from './types'; + +export const vidplayScraper = makeEmbed({ + id: 'vidplay', + name: 'VidPlay', + rank: 499, + scrape: async (ctx) => { + const fileUrl = await getFileUrl(ctx); + console.log(fileUrl); + const fileUrlRes = await ctx.proxiedFetcher(`${fileUrl}&autostart=true`, { + headers: { + referer: ctx.url, + }, + }); + const source = fileUrlRes.result.sources[0].file; + + return { + stream: [ + { + id: 'primary', + type: 'hls', + playlist: source, + flags: [], + captions: [], + }, + ], + }; + }, +}); diff --git a/src/providers/embeds/vidplay/types.ts b/src/providers/embeds/vidplay/types.ts new file mode 100644 index 0000000..14c33a8 --- /dev/null +++ b/src/providers/embeds/vidplay/types.ts @@ -0,0 +1,11 @@ +export type VidplaySourceResponse = { + result: { + sources: { + file: string; + tracks: { + file: string; + kind: string; + }[]; + }[]; + }; +}; diff --git a/src/providers/sources/vidsrcto/common.ts b/src/providers/sources/vidsrcto/common.ts new file mode 100644 index 0000000..e31b58d --- /dev/null +++ b/src/providers/sources/vidsrcto/common.ts @@ -0,0 +1,51 @@ +const DECRYPTION_KEY = '8z5Ag5wgagfsOuhz'; + +export const decodeBase64UrlSafe = (str: string) => { + const standardizedInput = str.replace(/_/g, '/').replace(/-/g, '+'); + + const binaryData = Buffer.from(standardizedInput, 'base64').toString('binary'); + + const bytes = new Uint8Array(binaryData.length); + for (let i = 0; i < bytes.length; i += 1) { + bytes[i] = binaryData.charCodeAt(i); + } + + return bytes; +}; + +export const decode = (str: Uint8Array) => { + const keyBytes = new TextEncoder().encode(DECRYPTION_KEY); + + let j = 0; + const s = new Uint8Array(256); + for (let i = 0; i < 256; i += 1) { + s[i] = i; + } + + for (let i = 0, k = 0; i < 256; i += 1) { + j = (j + s[i] + keyBytes[k % keyBytes.length]) & 0xff; + [s[i], s[j]] = [s[j], s[i]]; + k += 1; + } + + const decoded = new Uint8Array(str.length); + let i = 0; + let k = 0; + for (let index = 0; index < str.length; index += 1) { + i = (i + 1) & 0xff; + k = (k + s[i]) & 0xff; + [s[i], s[k]] = [s[k], s[i]]; + const t = (s[i] + s[k]) & 0xff; + decoded[index] = str[index] ^ s[t]; + } + + return decoded; +}; + +export const decryptSourceUrl = (sourceUrl: string) => { + const encoded = decodeBase64UrlSafe(sourceUrl); + const decoded = decode(encoded); + const decodedText = new TextDecoder().decode(decoded); + + return decodeURIComponent(decodeURIComponent(decodedText)); +}; diff --git a/src/providers/sources/vidsrcto/index.ts b/src/providers/sources/vidsrcto/index.ts new file mode 100644 index 0000000..eec22d4 --- /dev/null +++ b/src/providers/sources/vidsrcto/index.ts @@ -0,0 +1,58 @@ +import { load } from 'cheerio'; + +import { SourcererEmbed, SourcererOutput, makeSourcerer } from '@/providers/base'; +import { MovieScrapeContext, ShowScrapeContext } from '@/utils/context'; + +import { decryptSourceUrl } from './common'; +import { SourceResult, SourcesResult } from './types'; + +const vidSrcToBase = 'https://vidsrc.to'; + +const universalScraper = async (ctx: ShowScrapeContext | MovieScrapeContext): Promise => { + const imdbId = ctx.media.imdbId; + const url = + ctx.media.type === 'movie' + ? `${vidSrcToBase}/embed/movie/${imdbId}` + : `${vidSrcToBase}}/embed/tv/${imdbId}/${ctx.media.season.number}/${ctx.media.episode.number}`; + + const mainPage = await ctx.fetcher(url); + const mainPage$ = load(mainPage); + const dataId = mainPage$('a[data-id]').attr('data-id'); + if (!dataId) throw new Error('No data-id found'); + const sources = await ctx.fetcher(`/ajax/embed/episode/${dataId}/sources`, { + baseUrl: vidSrcToBase, + }); + if (sources.status !== 200) throw new Error('No sources found'); + + const embeds: SourcererEmbed[] = []; + for (const source of sources.result) { + const sourceRes = await ctx.fetcher(`/ajax/embed/source/${source.id}`, { + baseUrl: vidSrcToBase, + }); + const decryptedUrl = decryptSourceUrl(sourceRes.result.url); + if (source.title === 'Filemoon') { + embeds.push({ + embedId: 'filemoon', + url: decryptedUrl, + }); + } + if (source.title === 'Vidplay') { + embeds.push({ + embedId: 'vidplay', + url: decryptedUrl, + }); + } + } + return { + embeds, + }; +}; + +export const vidSrcToScraper = makeSourcerer({ + id: 'vidsrcto', + name: 'VidSrcTo', + scrapeMovie: universalScraper, + scrapeShow: universalScraper, + flags: [], + rank: 500, +}); diff --git a/src/providers/sources/vidsrcto/types.ts b/src/providers/sources/vidsrcto/types.ts new file mode 100644 index 0000000..0694b15 --- /dev/null +++ b/src/providers/sources/vidsrcto/types.ts @@ -0,0 +1,15 @@ +export type VidSrcToResponse = { + status: number; + result: T; +}; + +export type SourcesResult = VidSrcToResponse< + { + id: string; + title: 'Filemoon' | 'Vidplay'; + }[] +>; + +export type SourceResult = VidSrcToResponse<{ + url: string; +}>; From f39aaca3e32fd11a3fca7f56711675f331592473 Mon Sep 17 00:00:00 2001 From: Jorrin Date: Wed, 27 Dec 2023 20:35:08 +0100 Subject: [PATCH 07/35] fix vidplay, add captions to filemoon --- src/providers/embeds/filemoon.ts | 34 --------------- src/providers/embeds/filemoon/index.ts | 56 +++++++++++++++++++++++++ src/providers/embeds/filemoon/types.ts | 5 +++ src/providers/embeds/vidplay/common.ts | 53 +++++++++++++++-------- src/providers/embeds/vidplay/index.ts | 29 +++++++++++-- src/providers/embeds/vidplay/types.ts | 26 ++++++++---- src/providers/sources/vidsrcto/index.ts | 29 +++++++++---- 7 files changed, 160 insertions(+), 72 deletions(-) delete mode 100644 src/providers/embeds/filemoon.ts create mode 100644 src/providers/embeds/filemoon/index.ts create mode 100644 src/providers/embeds/filemoon/types.ts diff --git a/src/providers/embeds/filemoon.ts b/src/providers/embeds/filemoon.ts deleted file mode 100644 index 9b092a2..0000000 --- a/src/providers/embeds/filemoon.ts +++ /dev/null @@ -1,34 +0,0 @@ -import { unpack } from 'unpacker'; - -import { flags } from '@/entrypoint/utils/targets'; - -import { makeEmbed } from '../base'; - -const evalCodeRegex = /eval\((.*)\)/g; -const fileRegex = /file:"(.*?)"/g; - -export const fileMoonScraper = makeEmbed({ - id: 'filemoon', - name: 'FileMoon', - rank: 501, - scrape: async (ctx) => { - const embedRes = await ctx.fetcher(ctx.url); - const evalCode = evalCodeRegex.exec(embedRes); - if (!evalCode) throw new Error('Failed to find eval code'); - const unpacked = unpack(evalCode[1]); - const file = fileRegex.exec(unpacked); - if (!file?.[1]) throw new Error('Failed to find file'); - - return { - stream: [ - { - id: 'primary', - type: 'hls', - playlist: file[1], - flags: [flags.CORS_ALLOWED], - captions: [], - }, - ], - }; - }, -}); diff --git a/src/providers/embeds/filemoon/index.ts b/src/providers/embeds/filemoon/index.ts new file mode 100644 index 0000000..584be7e --- /dev/null +++ b/src/providers/embeds/filemoon/index.ts @@ -0,0 +1,56 @@ +import { unpack } from 'unpacker'; + +import { flags } from '@/entrypoint/utils/targets'; + +import { SubtitleResult } from './types'; +import { makeEmbed } from '../../base'; +import { Caption, getCaptionTypeFromUrl, labelToLanguageCode } from '../../captions'; + +const evalCodeRegex = /eval\((.*)\)/g; +const fileRegex = /file:"(.*?)"/g; + +export const fileMoonScraper = makeEmbed({ + id: 'filemoon', + name: 'Filemoon', + rank: 501, + scrape: async (ctx) => { + const embedRes = await ctx.fetcher(ctx.url); + const evalCode = evalCodeRegex.exec(embedRes); + if (!evalCode) throw new Error('Failed to find eval code'); + const unpacked = unpack(evalCode[1]); + const file = fileRegex.exec(unpacked); + if (!file?.[1]) throw new Error('Failed to find file'); + + const url = new URL(ctx.url); + const subtitlesLink = url.searchParams.get('sub.info'); + const captions: Caption[] = []; + if (subtitlesLink) { + const captionsResult = await ctx.fetcher(subtitlesLink); + + for (const caption of captionsResult) { + const language = labelToLanguageCode(caption.label); + const captionType = getCaptionTypeFromUrl(caption.file); + if (!language || !captionType) continue; + captions.push({ + id: caption.file, + url: caption.file, + type: captionType, + language, + hasCorsRestrictions: false, + }); + } + } + + return { + stream: [ + { + id: 'primary', + type: 'hls', + playlist: file[1], + flags: [flags.CORS_ALLOWED], + captions, + }, + ], + }; + }, +}); diff --git a/src/providers/embeds/filemoon/types.ts b/src/providers/embeds/filemoon/types.ts new file mode 100644 index 0000000..caa27af --- /dev/null +++ b/src/providers/embeds/filemoon/types.ts @@ -0,0 +1,5 @@ +export type SubtitleResult = { + file: string; + label: string; + kind: string; +}[]; diff --git a/src/providers/embeds/vidplay/common.ts b/src/providers/embeds/vidplay/common.ts index 9fbc8d2..cf2eb0f 100644 --- a/src/providers/embeds/vidplay/common.ts +++ b/src/providers/embeds/vidplay/common.ts @@ -1,10 +1,34 @@ -import { createCipheriv } from 'crypto'; -import { Buffer } from 'node:buffer'; - import { EmbedScrapeContext } from '@/utils/context'; export const vidplayBase = 'https://vidplay.site'; +export function keyPermutation(key: string, data: any) { + const state = Array.from(Array(256).keys()); + let index1 = 0; + for (let i = 0; i < 256; i += 1) { + index1 = (index1 + state[i] + key.charCodeAt(i % key.length)) % 256; + const temp = state[i]; + state[i] = state[index1]; + state[index1] = temp; + } + index1 = 0; + let index2 = 0; + let finalKey = ''; + for (let char = 0; char < data.length; char += 1) { + index1 = (index1 + 1) % 256; + index2 = (index2 + state[index1]) % 256; + const temp = state[index1]; + state[index1] = state[index2]; + state[index2] = temp; + if (typeof data[char] === 'string') { + finalKey += String.fromCharCode(data[char].charCodeAt(0) ^ state[(state[index1] + state[index2]) % 256]); + } else if (typeof data[char] === 'number') { + finalKey += String.fromCharCode(data[char] ^ state[(state[index1] + state[index2]) % 256]); + } + } + return finalKey; +} + export const getDecryptionKeys = async (ctx: EmbedScrapeContext): Promise => { const res = await ctx.fetcher( 'https://raw.githubusercontent.com/Claudemirovsky/worstsource-keys/keys/keys.json', @@ -12,23 +36,19 @@ export const getDecryptionKeys = async (ctx: EmbedScrapeContext): Promise { - const id = ctx.url.split('/e/')[1].split('?')[0]; - const keys = await getDecryptionKeys(ctx); - const c1 = createCipheriv('rc4', Buffer.from(keys[0]), ''); - const c2 = createCipheriv('rc4', Buffer.from(keys[1]), ''); + const url = new URL(ctx.url); + const id = url.pathname.replace('/e/', ''); + const keyList = await getDecryptionKeys(ctx); - let input = Buffer.from(id); - input = Buffer.concat([c1.update(input), c1.final()]); - input = Buffer.concat([c2.update(input), c2.final()]); - - return input.toString('base64').replace('/', '_'); + const decodedId = keyPermutation(keyList[0], id); + const encodedResult = keyPermutation(keyList[1], decodedId); + const base64 = btoa(encodedResult); + return base64.replace('/', '_'); }; export const getFuTokenKey = async (ctx: EmbedScrapeContext) => { const id = await getEncodedId(ctx); - console.log(`ENCODED ID: ${id}`); const fuTokenRes = await ctx.proxiedFetcher('/futoken', { baseUrl: vidplayBase, headers: { @@ -36,18 +56,15 @@ export const getFuTokenKey = async (ctx: EmbedScrapeContext) => { }, }); const fuKey = fuTokenRes.match(/var\s+k\s*=\s*'([^']+)'/)?.[1]; - console.log(`FU KEY: ${fuKey}`); if (!fuKey) throw new Error('No fuKey found'); const tokens = []; for (let i = 0; i < id.length; i += 1) { tokens.push(fuKey.charCodeAt(i % fuKey.length) + id.charCodeAt(i)); } - console.log(`${fuKey},${tokens.join(',')}`); return `${fuKey},${tokens.join(',')}`; }; export const getFileUrl = async (ctx: EmbedScrapeContext) => { - console.log(ctx.url); const fuToken = await getFuTokenKey(ctx); - return `${vidplayBase}/mediainfo/${fuToken}?${ctx.url.split('?')[1]}`; + return `${vidplayBase}/mediainfo/${fuToken}${new URL(ctx.url).search}&autostart=true`; }; diff --git a/src/providers/embeds/vidplay/index.ts b/src/providers/embeds/vidplay/index.ts index 93243e5..425d583 100644 --- a/src/providers/embeds/vidplay/index.ts +++ b/src/providers/embeds/vidplay/index.ts @@ -1,7 +1,8 @@ import { makeEmbed } from '@/providers/base'; +import { Caption, getCaptionTypeFromUrl, labelToLanguageCode } from '@/providers/captions'; import { getFileUrl } from './common'; -import { VidplaySourceResponse } from './types'; +import { SubtitleResult, VidplaySourceResponse } from './types'; export const vidplayScraper = makeEmbed({ id: 'vidplay', @@ -9,14 +10,34 @@ export const vidplayScraper = makeEmbed({ rank: 499, scrape: async (ctx) => { const fileUrl = await getFileUrl(ctx); - console.log(fileUrl); - const fileUrlRes = await ctx.proxiedFetcher(`${fileUrl}&autostart=true`, { + const fileUrlRes = await ctx.proxiedFetcher(fileUrl, { headers: { referer: ctx.url, }, }); + if (typeof fileUrlRes.result === 'number') throw new Error('File not found'); const source = fileUrlRes.result.sources[0].file; + const url = new URL(ctx.url); + const subtitlesLink = url.searchParams.get('sub.info'); + const captions: Caption[] = []; + if (subtitlesLink) { + const captionsResult = await ctx.fetcher(subtitlesLink); + + for (const caption of captionsResult) { + const language = labelToLanguageCode(caption.label); + const captionType = getCaptionTypeFromUrl(caption.file); + if (!language || !captionType) continue; + captions.push({ + id: caption.file, + url: caption.file, + type: captionType, + language, + hasCorsRestrictions: false, + }); + } + } + return { stream: [ { @@ -24,7 +45,7 @@ export const vidplayScraper = makeEmbed({ type: 'hls', playlist: source, flags: [], - captions: [], + captions, }, ], }; diff --git a/src/providers/embeds/vidplay/types.ts b/src/providers/embeds/vidplay/types.ts index 14c33a8..29cde1d 100644 --- a/src/providers/embeds/vidplay/types.ts +++ b/src/providers/embeds/vidplay/types.ts @@ -1,11 +1,19 @@ export type VidplaySourceResponse = { - result: { - sources: { - file: string; - tracks: { - file: string; - kind: string; - }[]; - }[]; - }; + result: + | { + sources: { + file: string; + tracks: { + file: string; + kind: string; + }[]; + }[]; + } + | number; }; + +export type SubtitleResult = { + file: string; + label: string; + kind: string; +}[]; diff --git a/src/providers/sources/vidsrcto/index.ts b/src/providers/sources/vidsrcto/index.ts index eec22d4..8b43e47 100644 --- a/src/providers/sources/vidsrcto/index.ts +++ b/src/providers/sources/vidsrcto/index.ts @@ -25,24 +25,39 @@ const universalScraper = async (ctx: ShowScrapeContext | MovieScrapeContext): Pr if (sources.status !== 200) throw new Error('No sources found'); const embeds: SourcererEmbed[] = []; + const embedUrls = []; for (const source of sources.result) { const sourceRes = await ctx.fetcher(`/ajax/embed/source/${source.id}`, { baseUrl: vidSrcToBase, }); const decryptedUrl = decryptSourceUrl(sourceRes.result.url); - if (source.title === 'Filemoon') { - embeds.push({ - embedId: 'filemoon', - url: decryptedUrl, - }); - } + embedUrls.push(decryptedUrl); + } + + // Originally Filemoon does not have subtitles. But we can use the ones from Vidplay. + const subtitleUrl = new URL(embedUrls.find((v) => v.includes('sub.info')) ?? '').searchParams.get('sub.info'); + for (const source of sources.result) { if (source.title === 'Vidplay') { + const embedUrl = embedUrls.find((v) => v.includes('vidplay')); + if (!embedUrl) continue; embeds.push({ embedId: 'vidplay', - url: decryptedUrl, + url: embedUrl, + }); + } + + if (source.title === 'Filemoon') { + const embedUrl = embedUrls.find((v) => v.includes('filemoon')); + if (!embedUrl) continue; + const fullUrl = new URL(embedUrl); + if (subtitleUrl) fullUrl.searchParams.set('sub.info', subtitleUrl); + embeds.push({ + embedId: 'filemoon', + url: fullUrl.toString(), }); } } + return { embeds, }; From 30e6067a3fa46fe5f75decca38554e7693eabc73 Mon Sep 17 00:00:00 2001 From: Jorrin Date: Wed, 27 Dec 2023 20:36:50 +0100 Subject: [PATCH 08/35] ranks --- src/providers/embeds/filemoon/index.ts | 2 +- src/providers/embeds/vidplay/index.ts | 2 +- src/providers/sources/vidsrcto/index.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/providers/embeds/filemoon/index.ts b/src/providers/embeds/filemoon/index.ts index 584be7e..f1eda29 100644 --- a/src/providers/embeds/filemoon/index.ts +++ b/src/providers/embeds/filemoon/index.ts @@ -12,7 +12,7 @@ const fileRegex = /file:"(.*?)"/g; export const fileMoonScraper = makeEmbed({ id: 'filemoon', name: 'Filemoon', - rank: 501, + rank: 301, scrape: async (ctx) => { const embedRes = await ctx.fetcher(ctx.url); const evalCode = evalCodeRegex.exec(embedRes); diff --git a/src/providers/embeds/vidplay/index.ts b/src/providers/embeds/vidplay/index.ts index 425d583..9225eff 100644 --- a/src/providers/embeds/vidplay/index.ts +++ b/src/providers/embeds/vidplay/index.ts @@ -7,7 +7,7 @@ import { SubtitleResult, VidplaySourceResponse } from './types'; export const vidplayScraper = makeEmbed({ id: 'vidplay', name: 'VidPlay', - rank: 499, + rank: 300, scrape: async (ctx) => { const fileUrl = await getFileUrl(ctx); const fileUrlRes = await ctx.proxiedFetcher(fileUrl, { diff --git a/src/providers/sources/vidsrcto/index.ts b/src/providers/sources/vidsrcto/index.ts index 8b43e47..3b22bbb 100644 --- a/src/providers/sources/vidsrcto/index.ts +++ b/src/providers/sources/vidsrcto/index.ts @@ -69,5 +69,5 @@ export const vidSrcToScraper = makeSourcerer({ scrapeMovie: universalScraper, scrapeShow: universalScraper, flags: [], - rank: 500, + rank: 300, }); From c44d13f0bd9be6307c4d0a43d20726c83f101992 Mon Sep 17 00:00:00 2001 From: Jorrin Date: Wed, 27 Dec 2023 20:42:22 +0100 Subject: [PATCH 09/35] fix ranks --- src/providers/embeds/filemoon/index.ts | 2 +- src/providers/embeds/vidplay/index.ts | 2 +- src/providers/sources/vidsrcto/index.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/providers/embeds/filemoon/index.ts b/src/providers/embeds/filemoon/index.ts index f1eda29..2729b5c 100644 --- a/src/providers/embeds/filemoon/index.ts +++ b/src/providers/embeds/filemoon/index.ts @@ -12,7 +12,7 @@ const fileRegex = /file:"(.*?)"/g; export const fileMoonScraper = makeEmbed({ id: 'filemoon', name: 'Filemoon', - rank: 301, + rank: 400, scrape: async (ctx) => { const embedRes = await ctx.fetcher(ctx.url); const evalCode = evalCodeRegex.exec(embedRes); diff --git a/src/providers/embeds/vidplay/index.ts b/src/providers/embeds/vidplay/index.ts index 9225eff..f7adca2 100644 --- a/src/providers/embeds/vidplay/index.ts +++ b/src/providers/embeds/vidplay/index.ts @@ -7,7 +7,7 @@ import { SubtitleResult, VidplaySourceResponse } from './types'; export const vidplayScraper = makeEmbed({ id: 'vidplay', name: 'VidPlay', - rank: 300, + rank: 401, scrape: async (ctx) => { const fileUrl = await getFileUrl(ctx); const fileUrlRes = await ctx.proxiedFetcher(fileUrl, { diff --git a/src/providers/sources/vidsrcto/index.ts b/src/providers/sources/vidsrcto/index.ts index 3b22bbb..09a0320 100644 --- a/src/providers/sources/vidsrcto/index.ts +++ b/src/providers/sources/vidsrcto/index.ts @@ -69,5 +69,5 @@ export const vidSrcToScraper = makeSourcerer({ scrapeMovie: universalScraper, scrapeShow: universalScraper, flags: [], - rank: 300, + rank: 400, }); From a208aef364d60ea7967f4490f88b309046feb336 Mon Sep 17 00:00:00 2001 From: Jorrin Date: Wed, 27 Dec 2023 21:14:18 +0100 Subject: [PATCH 10/35] remove Buffer --- src/providers/sources/vidsrcto/common.ts | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/providers/sources/vidsrcto/common.ts b/src/providers/sources/vidsrcto/common.ts index e31b58d..fb9890d 100644 --- a/src/providers/sources/vidsrcto/common.ts +++ b/src/providers/sources/vidsrcto/common.ts @@ -2,12 +2,11 @@ const DECRYPTION_KEY = '8z5Ag5wgagfsOuhz'; export const decodeBase64UrlSafe = (str: string) => { const standardizedInput = str.replace(/_/g, '/').replace(/-/g, '+'); + const decodedData = atob(standardizedInput); - const binaryData = Buffer.from(standardizedInput, 'base64').toString('binary'); - - const bytes = new Uint8Array(binaryData.length); + const bytes = new Uint8Array(decodedData.length); for (let i = 0; i < bytes.length; i += 1) { - bytes[i] = binaryData.charCodeAt(i); + bytes[i] = decodedData.charCodeAt(i); } return bytes; From c9a611d6b765478ec81dadf956a452fb4aed7ca4 Mon Sep 17 00:00:00 2001 From: mrjvs Date: Wed, 27 Dec 2023 22:27:37 +0100 Subject: [PATCH 11/35] Add console logs to browser dev cli --- src/dev-cli/scraper.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/dev-cli/scraper.ts b/src/dev-cli/scraper.ts index 882d321..39f75f6 100644 --- a/src/dev-cli/scraper.ts +++ b/src/dev-cli/scraper.ts @@ -41,6 +41,7 @@ async function runBrowserScraping( args: ['--no-sandbox', '--disable-setuid-sandbox'], }); const page = await browser.newPage(); + page.on('console', (message) => console.log(`${message.type().slice(0, 3).toUpperCase()} ${message.text()}`)); await page.goto(server.resolvedUrls.local[0]); await page.waitForFunction('!!window.scrape', { timeout: 5000 }); From fb3f2378087f8e414f4458c8f10f1eac8f1d52f8 Mon Sep 17 00:00:00 2001 From: mrjvs Date: Wed, 27 Dec 2023 22:33:13 +0100 Subject: [PATCH 12/35] Better types for fetchers --- src/fetchers/types.ts | 10 +++++----- src/utils/context.ts | 10 ++-------- 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/src/fetchers/types.ts b/src/fetchers/types.ts index d542298..f5dbe06 100644 --- a/src/fetchers/types.ts +++ b/src/fetchers/types.ts @@ -28,12 +28,12 @@ export type FetcherResponse = { }; // This is the version that will be inputted by library users -export type Fetcher = { - (url: string, ops: DefaultedFetcherOptions): Promise>; +export type Fetcher = { + (url: string, ops: DefaultedFetcherOptions): Promise>; }; // This is the version that scrapers will be interacting with -export type UseableFetcher = { - (url: string, ops?: FetcherOptions): Promise; - full: (url: string, ops?: FetcherOptions) => Promise>; +export type UseableFetcher = { + (url: string, ops?: FetcherOptions): Promise; + full: (url: string, ops?: FetcherOptions) => Promise>; }; diff --git a/src/utils/context.ts b/src/utils/context.ts index cf7acc7..6f16bca 100644 --- a/src/utils/context.ts +++ b/src/utils/context.ts @@ -2,14 +2,8 @@ import { MovieMedia, ShowMedia } from '@/entrypoint/utils/media'; import { UseableFetcher } from '@/fetchers/types'; export type ScrapeContext = { - proxiedFetcher: { - (...params: Parameters>): ReturnType>; - full(...params: Parameters['full']>): ReturnType['full']>; - }; - fetcher: { - (...params: Parameters>): ReturnType>; - full(...params: Parameters['full']>): ReturnType['full']>; - }; + proxiedFetcher: UseableFetcher; + fetcher: UseableFetcher; progress(val: number): void; }; From 9b338b6f3bd2eb2d76dffbf2a3a181b9d499efa7 Mon Sep 17 00:00:00 2001 From: Jorrin Date: Thu, 28 Dec 2023 18:39:49 +0100 Subject: [PATCH 13/35] fixed feedback, added external_ids --- src/dev-cli/tmdb.ts | 16 ++++++++++------ src/providers/embeds/vidplay/common.ts | 9 ++++++++- src/providers/sources/vidsrcto/index.ts | 12 +++++++----- 3 files changed, 25 insertions(+), 12 deletions(-) diff --git a/src/dev-cli/tmdb.ts b/src/dev-cli/tmdb.ts index 90e87f8..c03307d 100644 --- a/src/dev-cli/tmdb.ts +++ b/src/dev-cli/tmdb.ts @@ -2,7 +2,7 @@ import { getConfig } from '@/dev-cli/config'; import { MovieMedia, ShowMedia } from '..'; -export async function makeTMDBRequest(url: string): Promise { +export async function makeTMDBRequest(url: string, appendToResponse?: string): Promise { const headers: { accept: 'application/json'; authorization?: string; @@ -10,7 +10,7 @@ export async function makeTMDBRequest(url: string): Promise { accept: 'application/json', }; - let requestURL = url; + const requestURL = new URL(url); const key = getConfig().tmdbApiKey; // * JWT keys always start with ey and are ONLY valid as a header. @@ -19,7 +19,11 @@ export async function makeTMDBRequest(url: string): Promise { if (key.startsWith('ey')) { headers.authorization = `Bearer ${key}`; } else { - requestURL += `?api_key=${key}`; + requestURL.searchParams.append('api_key', key); + } + + if (appendToResponse) { + requestURL.searchParams.append('append_to_response', appendToResponse); } return fetch(requestURL, { @@ -29,7 +33,7 @@ export async function makeTMDBRequest(url: string): Promise { } export async function getMovieMediaDetails(id: string): Promise { - const response = await makeTMDBRequest(`https://api.themoviedb.org/3/movie/${id}`); + const response = await makeTMDBRequest(`https://api.themoviedb.org/3/movie/${id}`, 'external_ids'); const movie = await response.json(); if (movie.success === false) { @@ -52,7 +56,7 @@ export async function getMovieMediaDetails(id: string): Promise { export async function getShowMediaDetails(id: string, seasonNumber: string, episodeNumber: string): Promise { // * TV shows require the TMDB ID for the series, season, and episode // * and the name of the series. Needs multiple requests - let response = await makeTMDBRequest(`https://api.themoviedb.org/3/tv/${id}`); + let response = await makeTMDBRequest(`https://api.themoviedb.org/3/tv/${id}`, 'external_ids'); const series = await response.json(); if (series.success === false) { @@ -92,6 +96,6 @@ export async function getShowMediaDetails(id: string, seasonNumber: string, epis number: season.season_number, tmdbId: season.id, }, - imdbId: series.imdb_id, + imdbId: series.external_ids.imdb_id, }; } diff --git a/src/providers/embeds/vidplay/common.ts b/src/providers/embeds/vidplay/common.ts index cf2eb0f..765edc7 100644 --- a/src/providers/embeds/vidplay/common.ts +++ b/src/providers/embeds/vidplay/common.ts @@ -1,3 +1,4 @@ +import { makeFullUrl } from '@/fetchers/common'; import { EmbedScrapeContext } from '@/utils/context'; export const vidplayBase = 'https://vidplay.site'; @@ -66,5 +67,11 @@ export const getFuTokenKey = async (ctx: EmbedScrapeContext) => { export const getFileUrl = async (ctx: EmbedScrapeContext) => { const fuToken = await getFuTokenKey(ctx); - return `${vidplayBase}/mediainfo/${fuToken}${new URL(ctx.url).search}&autostart=true`; + return makeFullUrl(`/mediainfo/${fuToken}`, { + baseUrl: vidplayBase, + query: { + ...Object.fromEntries(new URL(ctx.url).searchParams.entries()), + autostart: 'true', + }, + }); }; diff --git a/src/providers/sources/vidsrcto/index.ts b/src/providers/sources/vidsrcto/index.ts index 09a0320..5e73bc1 100644 --- a/src/providers/sources/vidsrcto/index.ts +++ b/src/providers/sources/vidsrcto/index.ts @@ -1,5 +1,6 @@ import { load } from 'cheerio'; +import { flags } from '@/entrypoint/utils/targets'; import { SourcererEmbed, SourcererOutput, makeSourcerer } from '@/providers/base'; import { MovieScrapeContext, ShowScrapeContext } from '@/utils/context'; @@ -12,10 +13,11 @@ const universalScraper = async (ctx: ShowScrapeContext | MovieScrapeContext): Pr const imdbId = ctx.media.imdbId; const url = ctx.media.type === 'movie' - ? `${vidSrcToBase}/embed/movie/${imdbId}` - : `${vidSrcToBase}}/embed/tv/${imdbId}/${ctx.media.season.number}/${ctx.media.episode.number}`; - - const mainPage = await ctx.fetcher(url); + ? `/embed/movie/${imdbId}` + : `/embed/tv/${imdbId}/${ctx.media.season.number}/${ctx.media.episode.number}`; + const mainPage = await ctx.fetcher(url, { + baseUrl: vidSrcToBase, + }); const mainPage$ = load(mainPage); const dataId = mainPage$('a[data-id]').attr('data-id'); if (!dataId) throw new Error('No data-id found'); @@ -68,6 +70,6 @@ export const vidSrcToScraper = makeSourcerer({ name: 'VidSrcTo', scrapeMovie: universalScraper, scrapeShow: universalScraper, - flags: [], + flags: [flags.CORS_ALLOWED], rank: 400, }); From a07f54e0cfdae270d82f11c911abf64f6fa91ebd Mon Sep 17 00:00:00 2001 From: Jorrin <43169049+JorrinKievit@users.noreply.github.com> Date: Fri, 29 Dec 2023 22:03:54 +0100 Subject: [PATCH 14/35] add credits to decryption code --- src/providers/embeds/vidplay/common.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/providers/embeds/vidplay/common.ts b/src/providers/embeds/vidplay/common.ts index 765edc7..57b94bb 100644 --- a/src/providers/embeds/vidplay/common.ts +++ b/src/providers/embeds/vidplay/common.ts @@ -3,6 +3,8 @@ import { EmbedScrapeContext } from '@/utils/context'; export const vidplayBase = 'https://vidplay.site'; +// This file is based on https://github.com/Ciarands/vidsrc-to-resolver/blob/960afb11c30aa6497804b4691fb1c401e539cfe7/vidsrc.py#L10 +// Full credits to @Ciarands! export function keyPermutation(key: string, data: any) { const state = Array.from(Array(256).keys()); let index1 = 0; From 67d936f58c121092ad85061d4cf772dd2aa3a610 Mon Sep 17 00:00:00 2001 From: Mahir <128459141+iMahir@users.noreply.github.com> Date: Sun, 31 Dec 2023 21:26:27 +0530 Subject: [PATCH 15/35] feat(lookmovie): add captions support --- src/providers/sources/lookmovie/index.ts | 8 ++++---- src/providers/sources/lookmovie/type.ts | 9 +++++++++ src/providers/sources/lookmovie/util.ts | 14 ++++++++------ src/providers/sources/lookmovie/video.ts | 23 ++++++++++++++++++----- 4 files changed, 39 insertions(+), 15 deletions(-) diff --git a/src/providers/sources/lookmovie/index.ts b/src/providers/sources/lookmovie/index.ts index 5cd82e9..8611373 100644 --- a/src/providers/sources/lookmovie/index.ts +++ b/src/providers/sources/lookmovie/index.ts @@ -10,8 +10,8 @@ async function universalScraper(ctx: MovieScrapeContext | ShowScrapeContext): Pr if (!lookmovieData) throw new NotFoundError('Media not found'); ctx.progress(30); - const videoUrl = await scrape(ctx, ctx.media, lookmovieData); - if (!videoUrl) throw new NotFoundError('No video found'); + const video = await scrape(ctx, ctx.media, lookmovieData); + if (!video.playlist) throw new NotFoundError('No video found'); ctx.progress(60); @@ -20,10 +20,10 @@ async function universalScraper(ctx: MovieScrapeContext | ShowScrapeContext): Pr stream: [ { id: 'primary', - playlist: videoUrl, + playlist: video.playlist, type: 'hls', flags: [flags.IP_LOCKED], - captions: [], + captions: video.captions, }, ], }; diff --git a/src/providers/sources/lookmovie/type.ts b/src/providers/sources/lookmovie/type.ts index 8335c96..47ccc55 100644 --- a/src/providers/sources/lookmovie/type.ts +++ b/src/providers/sources/lookmovie/type.ts @@ -39,8 +39,17 @@ interface VideoSources { [key: string]: string; } +interface VideoSubtitles { + id?: number; + id_movie?: number; + url: string; + language: string; + shard?: string; +} + export interface StreamsDataResult { streams: VideoSources; + subtitles: VideoSubtitles[]; } export interface ResultItem { diff --git a/src/providers/sources/lookmovie/util.ts b/src/providers/sources/lookmovie/util.ts index 7c8f202..6057f76 100644 --- a/src/providers/sources/lookmovie/util.ts +++ b/src/providers/sources/lookmovie/util.ts @@ -4,7 +4,9 @@ import { ScrapeContext } from '@/utils/context'; import { NotFoundError } from '@/utils/errors'; import { Result, ResultItem, ShowDataResult, episodeObj } from './type'; -import { getVideoUrl } from './video'; +import { getVideo } from './video'; + +export const baseUrl = 'https://lmscript.xyz'; export async function searchAndFindMedia( ctx: ScrapeContext, @@ -12,7 +14,7 @@ export async function searchAndFindMedia( ): Promise { if (media.type === 'show') { const searchRes = await ctx.fetcher(`/v1/shows`, { - baseUrl: 'https://lmscript.xyz', + baseUrl, query: { 'filters[q]': media.title }, }); @@ -23,7 +25,7 @@ export async function searchAndFindMedia( } if (media.type === 'movie') { const searchRes = await ctx.fetcher(`/v1/movies`, { - baseUrl: 'https://lmscript.xyz', + baseUrl, query: { 'filters[q]': media.title }, }); @@ -40,7 +42,7 @@ export async function scrape(ctx: ScrapeContext, media: MovieMedia | ShowMedia, id = result.id_movie; } else if (media.type === 'show') { const data = await ctx.fetcher(`/v1/shows`, { - baseUrl: 'https://lmscript.xyz', + baseUrl, query: { expand: 'episodes', id: result.id_show }, }); @@ -54,6 +56,6 @@ export async function scrape(ctx: ScrapeContext, media: MovieMedia | ShowMedia, // Check ID if (id === null) throw new NotFoundError('Not found'); - const videoUrl = await getVideoUrl(ctx, id, media); - return videoUrl; + const video = await getVideo(ctx, id, media); + return video; } diff --git a/src/providers/sources/lookmovie/video.ts b/src/providers/sources/lookmovie/video.ts index f439229..48bcafc 100644 --- a/src/providers/sources/lookmovie/video.ts +++ b/src/providers/sources/lookmovie/video.ts @@ -1,7 +1,9 @@ import { MovieMedia, ShowMedia } from '@/entrypoint/utils/media'; +import { Caption } from '@/providers/captions'; import { ScrapeContext } from '@/utils/context'; import { StreamsDataResult } from './type'; +import { baseUrl } from './util'; export async function getVideoSources( ctx: ScrapeContext, @@ -17,17 +19,17 @@ export async function getVideoSources( path = `/v1/movies/view`; } const data = await ctx.fetcher(path, { - baseUrl: 'https://lmscript.xyz', - query: { expand: 'streams', id }, + baseUrl, + query: { expand: 'streams,subtitles', id }, }); return data; } -export async function getVideoUrl( +export async function getVideo( ctx: ScrapeContext, id: string, media: MovieMedia | ShowMedia, -): Promise { +): Promise<{ playlist: string | null; captions: Caption[] }> { // Get sources const data = await getVideoSources(ctx, id, media); const videoSources = data.streams; @@ -42,5 +44,16 @@ export async function getVideoUrl( } } - return videoUrl; + const captions: Caption[] = data.subtitles.map((sub) => ({ + id: sub.url, + type: 'vtt', + url: `${baseUrl}${sub.url}`, + hasCorsRestrictions: false, + language: sub.language, + })); + + return { + playlist: videoUrl, + captions, + }; } From 79181ff87d4f79f71f26017dfabf914b0fe37f8e Mon Sep 17 00:00:00 2001 From: lonelil <51315646+lonelil@users.noreply.github.com> Date: Mon, 1 Jan 2024 01:48:19 +0900 Subject: [PATCH 16/35] Remove CORS flag for febbox hls --- src/providers/embeds/febbox/hls.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/providers/embeds/febbox/hls.ts b/src/providers/embeds/febbox/hls.ts index d9fa54f..2a3a03f 100644 --- a/src/providers/embeds/febbox/hls.ts +++ b/src/providers/embeds/febbox/hls.ts @@ -1,5 +1,4 @@ import { MediaTypes } from '@/entrypoint/utils/media'; -import { flags } from '@/entrypoint/utils/targets'; import { makeEmbed } from '@/providers/base'; import { parseInputUrl } from '@/providers/embeds/febbox/common'; import { getStreams } from '@/providers/embeds/febbox/fileList'; @@ -40,7 +39,7 @@ export const febboxHlsScraper = makeEmbed({ { id: 'primary', type: 'hls', - flags: [flags.CORS_ALLOWED], + flags: [], captions: await getSubtitles(ctx, id, firstStream.fid, type as MediaTypes, season, episode), playlist: `https://www.febbox.com/hls/main/${firstStream.oss_fid}.m3u8`, }, From 2e9c1b4baaac25c69869aff2a4fb2c26e357c532 Mon Sep 17 00:00:00 2001 From: Jip Fr Date: Mon, 1 Jan 2024 18:20:50 +0100 Subject: [PATCH 17/35] Fix language showing up more than once --- src/providers/embeds/febbox/subtitles.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/providers/embeds/febbox/subtitles.ts b/src/providers/embeds/febbox/subtitles.ts index b1b3064..70a94cb 100644 --- a/src/providers/embeds/febbox/subtitles.ts +++ b/src/providers/embeds/febbox/subtitles.ts @@ -37,22 +37,28 @@ export async function getSubtitles( const subResult = (await sendRequest(ctx, subtitleApiQuery)) as CaptionApiResponse; const subtitleList = subResult.data.list; const output: Caption[] = []; + const languagesAdded: Record = {}; subtitleList.forEach((sub) => { const subtitle = sub.subtitles.sort((a, b) => b.order - a.order)[0]; if (!subtitle) return; + const subtitleFilePath = subtitle.file_path .replace(captionsDomains[0], captionsDomains[1]) .replace(/\s/g, '+') .replace(/[()]/g, (c) => { return `%${c.charCodeAt(0).toString(16)}`; }); + const subtitleType = getCaptionTypeFromUrl(subtitleFilePath); if (!subtitleType) return; const validCode = isValidLanguageCode(subtitle.lang); if (!validCode) return; + if (languagesAdded[subtitle.lang]) return; + languagesAdded[subtitle.lang] = true; + output.push({ id: subtitleFilePath, language: subtitle.lang, From 34a2dd2ec5591a9303f904be3889d44137338ced Mon Sep 17 00:00:00 2001 From: Jip Frijlink Date: Mon, 1 Jan 2024 22:06:00 +0100 Subject: [PATCH 18/35] Increase package version --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 4b91441..890aa50 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@movie-web/providers", - "version": "2.0.1", + "version": "2.0.2", "description": "Package that contains all the providers of movie-web", "main": "./lib/index.umd.js", "types": "./lib/index.d.ts", From 27e1ae7e0b406964f2c775dcb812349ee89c5165 Mon Sep 17 00:00:00 2001 From: erynith <135774005+erynith@users.noreply.github.com> Date: Tue, 2 Jan 2024 13:54:13 -0500 Subject: [PATCH 19/35] Fix remotestream base --- src/providers/sources/remotestream.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/providers/sources/remotestream.ts b/src/providers/sources/remotestream.ts index 6d0a44c..d358899 100644 --- a/src/providers/sources/remotestream.ts +++ b/src/providers/sources/remotestream.ts @@ -2,7 +2,7 @@ import { flags } from '@/entrypoint/utils/targets'; import { makeSourcerer } from '@/providers/base'; import { NotFoundError } from '@/utils/errors'; -const remotestreamBase = `https://fsa.remotestre.am`; +const remotestreamBase = atob('aHR0cHM6Ly9mc2IuOG1ldDNkdGpmcmNxY2hjb25xcGtsd3hzeGIyb2N1bWMuc3RyZWFt'); export const remotestreamScraper = makeSourcerer({ id: 'remotestream', From 98230470f14ed6fa4fcb3797e484cfb6607a93f6 Mon Sep 17 00:00:00 2001 From: mrjvs Date: Tue, 2 Jan 2024 20:12:28 +0100 Subject: [PATCH 20/35] remove febbox HLS, it doesnt work --- src/entrypoint/providers.ts | 4 ++-- src/providers/embeds/febbox/hls.ts | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/entrypoint/providers.ts b/src/entrypoint/providers.ts index b306417..e456eb0 100644 --- a/src/entrypoint/providers.ts +++ b/src/entrypoint/providers.ts @@ -2,9 +2,9 @@ import { gatherAllEmbeds, gatherAllSources } from '@/providers/all'; import { Embed, Sourcerer } from '@/providers/base'; export function getBuiltinSources(): Sourcerer[] { - return gatherAllSources(); + return gatherAllSources().filter((v) => !v.disabled); } export function getBuiltinEmbeds(): Embed[] { - return gatherAllEmbeds(); + return gatherAllEmbeds().filter((v) => !v.disabled); } diff --git a/src/providers/embeds/febbox/hls.ts b/src/providers/embeds/febbox/hls.ts index 2a3a03f..792c112 100644 --- a/src/providers/embeds/febbox/hls.ts +++ b/src/providers/embeds/febbox/hls.ts @@ -15,6 +15,7 @@ export const febboxHlsScraper = makeEmbed({ id: 'febbox-hls', name: 'Febbox (HLS)', rank: 160, + disabled: true, async scrape(ctx) { const { type, id, season, episode } = parseInputUrl(ctx.url); const sharelinkResult = await ctx.proxiedFetcher<{ From 10eb0cc8f6e319366b56ebe63930ecfb24040def Mon Sep 17 00:00:00 2001 From: mrjvs Date: Tue, 2 Jan 2024 20:21:55 +0100 Subject: [PATCH 21/35] Update codeowner file --- .github/CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index d0f0ca6..0ccc218 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,3 +1,3 @@ -* @movie-web/core +* @movie-web/project-leads .github @binaryoverload From 7f4e412b9d28493abee46d737a040df505a172c3 Mon Sep 17 00:00:00 2001 From: mrjvs Date: Tue, 2 Jan 2024 20:22:06 +0100 Subject: [PATCH 22/35] Remove binary --- .github/CODEOWNERS | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 0ccc218..7458772 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,3 +1 @@ * @movie-web/project-leads - -.github @binaryoverload From 7dfeeb270092672d6a1ad99acb6e1e8a37c0058b Mon Sep 17 00:00:00 2001 From: mrjvs Date: Tue, 2 Jan 2024 20:33:43 +0100 Subject: [PATCH 23/35] Fix CLI not working for ip locked sources + disable lookmovie due to bug --- src/dev-cli/validate.ts | 1 + src/providers/sources/lookmovie/index.ts | 1 + 2 files changed, 2 insertions(+) diff --git a/src/dev-cli/validate.ts b/src/dev-cli/validate.ts index b600454..dd1f638 100644 --- a/src/dev-cli/validate.ts +++ b/src/dev-cli/validate.ts @@ -81,6 +81,7 @@ export async function processOptions(sources: Array, options: const providerOptions: ProviderMakerOptions = { fetcher, target: targets.ANY, + consistentIpForRequests: true, }; return { diff --git a/src/providers/sources/lookmovie/index.ts b/src/providers/sources/lookmovie/index.ts index 8611373..73226dc 100644 --- a/src/providers/sources/lookmovie/index.ts +++ b/src/providers/sources/lookmovie/index.ts @@ -33,6 +33,7 @@ export const lookmovieScraper = makeSourcerer({ id: 'lookmovie', name: 'LookMovie', rank: 1, + disabled: true, flags: [flags.IP_LOCKED], scrapeShow: universalScraper, scrapeMovie: universalScraper, From 7ea4ac0c48dfaee99ddace4ac1537e943dc44541 Mon Sep 17 00:00:00 2001 From: Jip Fr Date: Tue, 2 Jan 2024 20:54:57 +0100 Subject: [PATCH 24/35] Fix LookMovie caption codes, create removeDuplicateLanguages util --- src/providers/captions.ts | 10 ++++++++++ src/providers/embeds/febbox/subtitles.ts | 15 +++++++++------ src/providers/sources/lookmovie/index.ts | 1 - src/providers/sources/lookmovie/video.ts | 24 ++++++++++++++++-------- 4 files changed, 35 insertions(+), 15 deletions(-) diff --git a/src/providers/captions.ts b/src/providers/captions.ts index ce3f398..92e5db3 100644 --- a/src/providers/captions.ts +++ b/src/providers/captions.ts @@ -31,3 +31,13 @@ export function isValidLanguageCode(code: string | null): boolean { if (!code) return false; return ISO6391.validate(code); } + +export function removeDuplicatedLanguages(list: Caption[]) { + const beenSeen: Record = {}; + + return list.filter((sub) => { + if (beenSeen[sub.language]) return false; + beenSeen[sub.language] = true; + return true; + }); +} diff --git a/src/providers/embeds/febbox/subtitles.ts b/src/providers/embeds/febbox/subtitles.ts index 70a94cb..fbb2e85 100644 --- a/src/providers/embeds/febbox/subtitles.ts +++ b/src/providers/embeds/febbox/subtitles.ts @@ -1,4 +1,9 @@ -import { Caption, getCaptionTypeFromUrl, isValidLanguageCode } from '@/providers/captions'; +import { + Caption, + getCaptionTypeFromUrl, + isValidLanguageCode, + removeDuplicatedLanguages as removeDuplicateLanguages, +} from '@/providers/captions'; import { captionsDomains } from '@/providers/sources/showbox/common'; import { sendRequest } from '@/providers/sources/showbox/sendRequest'; import { ScrapeContext } from '@/utils/context'; @@ -36,8 +41,7 @@ export async function getSubtitles( const subResult = (await sendRequest(ctx, subtitleApiQuery)) as CaptionApiResponse; const subtitleList = subResult.data.list; - const output: Caption[] = []; - const languagesAdded: Record = {}; + let output: Caption[] = []; subtitleList.forEach((sub) => { const subtitle = sub.subtitles.sort((a, b) => b.order - a.order)[0]; @@ -56,9 +60,6 @@ export async function getSubtitles( const validCode = isValidLanguageCode(subtitle.lang); if (!validCode) return; - if (languagesAdded[subtitle.lang]) return; - languagesAdded[subtitle.lang] = true; - output.push({ id: subtitleFilePath, language: subtitle.lang, @@ -68,5 +69,7 @@ export async function getSubtitles( }); }); + output = removeDuplicateLanguages(output); + return output; } diff --git a/src/providers/sources/lookmovie/index.ts b/src/providers/sources/lookmovie/index.ts index 73226dc..8611373 100644 --- a/src/providers/sources/lookmovie/index.ts +++ b/src/providers/sources/lookmovie/index.ts @@ -33,7 +33,6 @@ export const lookmovieScraper = makeSourcerer({ id: 'lookmovie', name: 'LookMovie', rank: 1, - disabled: true, flags: [flags.IP_LOCKED], scrapeShow: universalScraper, scrapeMovie: universalScraper, diff --git a/src/providers/sources/lookmovie/video.ts b/src/providers/sources/lookmovie/video.ts index 48bcafc..8e8e3c4 100644 --- a/src/providers/sources/lookmovie/video.ts +++ b/src/providers/sources/lookmovie/video.ts @@ -1,5 +1,5 @@ import { MovieMedia, ShowMedia } from '@/entrypoint/utils/media'; -import { Caption } from '@/providers/captions'; +import { Caption, labelToLanguageCode, removeDuplicatedLanguages } from '@/providers/captions'; import { ScrapeContext } from '@/utils/context'; import { StreamsDataResult } from './type'; @@ -44,13 +44,21 @@ export async function getVideo( } } - const captions: Caption[] = data.subtitles.map((sub) => ({ - id: sub.url, - type: 'vtt', - url: `${baseUrl}${sub.url}`, - hasCorsRestrictions: false, - language: sub.language, - })); + let captions: Caption[] = []; + + for (const sub of data.subtitles) { + const language = labelToLanguageCode(sub.language); + if (!language) continue; + captions.push({ + id: sub.url, + type: 'vtt', + url: `${baseUrl}${sub.url}`, + hasCorsRestrictions: false, + language, + }); + } + + captions = removeDuplicatedLanguages(captions); return { playlist: videoUrl, From cdb59c604693bb05f997f27806560fdd9d3c93e7 Mon Sep 17 00:00:00 2001 From: mrjvs Date: Tue, 2 Jan 2024 22:29:14 +0100 Subject: [PATCH 25/35] Fix sujtasdfASDFG --- src/providers/sources/showbox/index.ts | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/providers/sources/showbox/index.ts b/src/providers/sources/showbox/index.ts index 267a6ef..d6c4887 100644 --- a/src/providers/sources/showbox/index.ts +++ b/src/providers/sources/showbox/index.ts @@ -1,6 +1,5 @@ import { flags } from '@/entrypoint/utils/targets'; import { SourcererOutput, makeSourcerer } from '@/providers/base'; -import { febboxHlsScraper } from '@/providers/embeds/febbox/hls'; import { febboxMp4Scraper } from '@/providers/embeds/febbox/mp4'; import { compareTitle } from '@/utils/compare'; import { MovieScrapeContext, ShowScrapeContext } from '@/utils/context'; @@ -31,10 +30,6 @@ async function comboScraper(ctx: ShowScrapeContext | MovieScrapeContext): Promis return { embeds: [ - { - embedId: febboxHlsScraper.id, - url: `/${ctx.media.type}/${id}/${season}/${episode}`, - }, { embedId: febboxMp4Scraper.id, url: `/${ctx.media.type}/${id}/${season}/${episode}`, From 0d36d51dd4a173525f68fc0257304ad6e659d0bc Mon Sep 17 00:00:00 2001 From: mrjvs Date: Tue, 2 Jan 2024 22:31:08 +0100 Subject: [PATCH 26/35] asdfua --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 890aa50..f7d1e18 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@movie-web/providers", - "version": "2.0.2", + "version": "2.0.3", "description": "Package that contains all the providers of movie-web", "main": "./lib/index.umd.js", "types": "./lib/index.d.ts", From d7134d4daf289570c55af5857cb8bb6fe44833ad Mon Sep 17 00:00:00 2001 From: Jorrin Date: Thu, 4 Jan 2024 20:18:25 +0100 Subject: [PATCH 27/35] proxied fetcher --- src/providers/embeds/filemoon/index.ts | 8 ++++++-- src/providers/embeds/vidplay/index.ts | 2 +- src/providers/sources/vidsrcto/index.ts | 16 +++++++++++++--- 3 files changed, 20 insertions(+), 6 deletions(-) diff --git a/src/providers/embeds/filemoon/index.ts b/src/providers/embeds/filemoon/index.ts index 2729b5c..3f8a2f3 100644 --- a/src/providers/embeds/filemoon/index.ts +++ b/src/providers/embeds/filemoon/index.ts @@ -14,7 +14,11 @@ export const fileMoonScraper = makeEmbed({ name: 'Filemoon', rank: 400, scrape: async (ctx) => { - const embedRes = await ctx.fetcher(ctx.url); + const embedRes = await ctx.proxiedFetcher(ctx.url, { + headers: { + referer: ctx.url, + }, + }); const evalCode = evalCodeRegex.exec(embedRes); if (!evalCode) throw new Error('Failed to find eval code'); const unpacked = unpack(evalCode[1]); @@ -25,7 +29,7 @@ export const fileMoonScraper = makeEmbed({ const subtitlesLink = url.searchParams.get('sub.info'); const captions: Caption[] = []; if (subtitlesLink) { - const captionsResult = await ctx.fetcher(subtitlesLink); + const captionsResult = await ctx.proxiedFetcher(subtitlesLink); for (const caption of captionsResult) { const language = labelToLanguageCode(caption.label); diff --git a/src/providers/embeds/vidplay/index.ts b/src/providers/embeds/vidplay/index.ts index f7adca2..3c1f6a2 100644 --- a/src/providers/embeds/vidplay/index.ts +++ b/src/providers/embeds/vidplay/index.ts @@ -22,7 +22,7 @@ export const vidplayScraper = makeEmbed({ const subtitlesLink = url.searchParams.get('sub.info'); const captions: Caption[] = []; if (subtitlesLink) { - const captionsResult = await ctx.fetcher(subtitlesLink); + const captionsResult = await ctx.proxiedFetcher(subtitlesLink); for (const caption of captionsResult) { const language = labelToLanguageCode(caption.label); diff --git a/src/providers/sources/vidsrcto/index.ts b/src/providers/sources/vidsrcto/index.ts index 5e73bc1..983fd12 100644 --- a/src/providers/sources/vidsrcto/index.ts +++ b/src/providers/sources/vidsrcto/index.ts @@ -8,6 +8,7 @@ import { decryptSourceUrl } from './common'; import { SourceResult, SourcesResult } from './types'; const vidSrcToBase = 'https://vidsrc.to'; +const referer = `${vidSrcToBase}/`; const universalScraper = async (ctx: ShowScrapeContext | MovieScrapeContext): Promise => { const imdbId = ctx.media.imdbId; @@ -15,22 +16,31 @@ const universalScraper = async (ctx: ShowScrapeContext | MovieScrapeContext): Pr ctx.media.type === 'movie' ? `/embed/movie/${imdbId}` : `/embed/tv/${imdbId}/${ctx.media.season.number}/${ctx.media.episode.number}`; - const mainPage = await ctx.fetcher(url, { + const mainPage = await ctx.proxiedFetcher(url, { baseUrl: vidSrcToBase, + headers: { + referer, + }, }); const mainPage$ = load(mainPage); const dataId = mainPage$('a[data-id]').attr('data-id'); if (!dataId) throw new Error('No data-id found'); - const sources = await ctx.fetcher(`/ajax/embed/episode/${dataId}/sources`, { + const sources = await ctx.proxiedFetcher(`/ajax/embed/episode/${dataId}/sources`, { baseUrl: vidSrcToBase, + headers: { + referer, + }, }); if (sources.status !== 200) throw new Error('No sources found'); const embeds: SourcererEmbed[] = []; const embedUrls = []; for (const source of sources.result) { - const sourceRes = await ctx.fetcher(`/ajax/embed/source/${source.id}`, { + const sourceRes = await ctx.proxiedFetcher(`/ajax/embed/source/${source.id}`, { baseUrl: vidSrcToBase, + headers: { + referer, + }, }); const decryptedUrl = decryptSourceUrl(sourceRes.result.url); embedUrls.push(decryptedUrl); From f1c29aab9003bd85ce36ec9a12d0938e5b0757ae Mon Sep 17 00:00:00 2001 From: mrjvs Date: Thu, 4 Jan 2024 21:11:24 +0100 Subject: [PATCH 28/35] Fix showbox + add support for new simple-proxy version --- src/fetchers/simpleProxy.ts | 1 + src/providers/sources/showbox/sendRequest.ts | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/fetchers/simpleProxy.ts b/src/fetchers/simpleProxy.ts index 21ed5ca..b9c6c5b 100644 --- a/src/fetchers/simpleProxy.ts +++ b/src/fetchers/simpleProxy.ts @@ -7,6 +7,7 @@ const headerMap: Record = { cookie: 'X-Cookie', referer: 'X-Referer', origin: 'X-Origin', + 'user-agent': 'X-User-Agent', }; const responseHeaderMap: Record = { diff --git a/src/providers/sources/showbox/sendRequest.ts b/src/providers/sources/showbox/sendRequest.ts index 2f77767..7ea9024 100644 --- a/src/providers/sources/showbox/sendRequest.ts +++ b/src/providers/sources/showbox/sendRequest.ts @@ -49,9 +49,9 @@ export const sendRequest = async (ctx: ScrapeContext, data: object, altApi = fal headers: { Platform: 'android', 'Content-Type': 'application/x-www-form-urlencoded', + 'User-Agent': 'okhttp/3.2.0', }, body: formatted, }); - return JSON.parse(response); }; From a694230bee1fc5e17e820cda1203bf9da498e184 Mon Sep 17 00:00:00 2001 From: mrjvs Date: Thu, 4 Jan 2024 21:20:14 +0100 Subject: [PATCH 29/35] Add another simple-proxy header --- src/fetchers/simpleProxy.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/fetchers/simpleProxy.ts b/src/fetchers/simpleProxy.ts index b9c6c5b..360a149 100644 --- a/src/fetchers/simpleProxy.ts +++ b/src/fetchers/simpleProxy.ts @@ -8,6 +8,7 @@ const headerMap: Record = { referer: 'X-Referer', origin: 'X-Origin', 'user-agent': 'X-User-Agent', + 'x-real-ip': 'X-X-Real-Ip', }; const responseHeaderMap: Record = { From 7b5c53e6bd051e09aad9d1666fb631d9527eb45a Mon Sep 17 00:00:00 2001 From: Jorrin Date: Thu, 4 Jan 2024 22:35:02 +0100 Subject: [PATCH 30/35] browser compatible decodeSrc function --- src/providers/sources/vidsrc/scrape.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/providers/sources/vidsrc/scrape.ts b/src/providers/sources/vidsrc/scrape.ts index 65a709c..81dceff 100644 --- a/src/providers/sources/vidsrc/scrape.ts +++ b/src/providers/sources/vidsrc/scrape.ts @@ -7,11 +7,13 @@ import { vidsrcBase, vidsrcRCPBase } from '@/providers/sources/vidsrc/common'; import { MovieScrapeContext, ShowScrapeContext } from '@/utils/context'; function decodeSrc(encoded: string, seed: string) { - const encodedBuffer = Buffer.from(encoded, 'hex'); let decoded = ''; + const seedLength = seed.length; - for (let i = 0; i < encodedBuffer.length; i++) { - decoded += String.fromCharCode(encodedBuffer[i] ^ seed.charCodeAt(i % seed.length)); + for (let i = 0; i < encoded.length; i += 2) { + const byte = parseInt(encoded.substr(i, 2), 16); + const seedChar = seed.charCodeAt((i / 2) % seedLength); + decoded += String.fromCharCode(byte ^ seedChar); } return decoded; From fe1f8d364b3e162e73e41cdaaf451ee326f72e97 Mon Sep 17 00:00:00 2001 From: Jorrin Date: Fri, 5 Jan 2024 14:51:54 +0100 Subject: [PATCH 31/35] reintroduce the password endpoint --- src/providers/embeds/vidsrc.ts | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/providers/embeds/vidsrc.ts b/src/providers/embeds/vidsrc.ts index d9eafb3..cd47e21 100644 --- a/src/providers/embeds/vidsrc.ts +++ b/src/providers/embeds/vidsrc.ts @@ -2,6 +2,7 @@ import { flags } from '@/entrypoint/utils/targets'; import { makeEmbed } from '@/providers/base'; const hlsURLRegex = /file:"(.*?)"/; +const setPassRegex = /var pass_path = "(.*set_pass\.php.*)";/; export const vidsrcembedScraper = makeEmbed({ id: 'vidsrcembed', // VidSrc is both a source and an embed host @@ -23,6 +24,22 @@ export const vidsrcembedScraper = makeEmbed({ if (!finalUrl.includes('.m3u8')) throw new Error('Unable to find HLS playlist'); + let setPassLink = html.match(setPassRegex)?.[1]; + if (!setPassLink) throw new Error('Unable to find set_pass.php link'); + + if (setPassLink.startsWith('//')) { + setPassLink = `https:${setPassLink}`; + } + + // VidSrc uses a password endpoint to temporarily whitelist the user's IP. This is called in an interval by the player. + // It currently has no effect on the player itself, the content plays fine without it. + // In the future we might have to introduce hooks for the frontend to call this endpoint. + await ctx.proxiedFetcher(setPassLink, { + headers: { + referer: ctx.url, + }, + }); + return { stream: [ { From 605b9d78d1b8cdf82c6ad4092005bda4eac70577 Mon Sep 17 00:00:00 2001 From: Jorrin Date: Fri, 5 Jan 2024 16:12:19 +0100 Subject: [PATCH 32/35] filemoon added CORS --- src/providers/embeds/filemoon/index.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/providers/embeds/filemoon/index.ts b/src/providers/embeds/filemoon/index.ts index 3f8a2f3..9f96a07 100644 --- a/src/providers/embeds/filemoon/index.ts +++ b/src/providers/embeds/filemoon/index.ts @@ -1,7 +1,5 @@ import { unpack } from 'unpacker'; -import { flags } from '@/entrypoint/utils/targets'; - import { SubtitleResult } from './types'; import { makeEmbed } from '../../base'; import { Caption, getCaptionTypeFromUrl, labelToLanguageCode } from '../../captions'; @@ -51,7 +49,7 @@ export const fileMoonScraper = makeEmbed({ id: 'primary', type: 'hls', playlist: file[1], - flags: [flags.CORS_ALLOWED], + flags: [], captions, }, ], From 1970e11443cfdb07652ad1ccc0092a3985952102 Mon Sep 17 00:00:00 2001 From: Jorrin Date: Fri, 5 Jan 2024 17:01:54 +0100 Subject: [PATCH 33/35] applied feedback from ciarands --- src/providers/embeds/vidplay/common.ts | 37 +++------------- src/providers/sources/vidsrcto/common.ts | 55 ++++++++++++------------ 2 files changed, 33 insertions(+), 59 deletions(-) diff --git a/src/providers/embeds/vidplay/common.ts b/src/providers/embeds/vidplay/common.ts index 57b94bb..224e3dc 100644 --- a/src/providers/embeds/vidplay/common.ts +++ b/src/providers/embeds/vidplay/common.ts @@ -1,36 +1,11 @@ import { makeFullUrl } from '@/fetchers/common'; +import { decodeData } from '@/providers/sources/vidsrcto/common'; import { EmbedScrapeContext } from '@/utils/context'; export const vidplayBase = 'https://vidplay.site'; -// This file is based on https://github.com/Ciarands/vidsrc-to-resolver/blob/960afb11c30aa6497804b4691fb1c401e539cfe7/vidsrc.py#L10 +// This file is based on https://github.com/Ciarands/vidsrc-to-resolver/blob/dffa45e726a4b944cb9af0c9e7630476c93c0213/vidsrc.py#L16 // Full credits to @Ciarands! -export function keyPermutation(key: string, data: any) { - const state = Array.from(Array(256).keys()); - let index1 = 0; - for (let i = 0; i < 256; i += 1) { - index1 = (index1 + state[i] + key.charCodeAt(i % key.length)) % 256; - const temp = state[i]; - state[i] = state[index1]; - state[index1] = temp; - } - index1 = 0; - let index2 = 0; - let finalKey = ''; - for (let char = 0; char < data.length; char += 1) { - index1 = (index1 + 1) % 256; - index2 = (index2 + state[index1]) % 256; - const temp = state[index1]; - state[index1] = state[index2]; - state[index2] = temp; - if (typeof data[char] === 'string') { - finalKey += String.fromCharCode(data[char].charCodeAt(0) ^ state[(state[index1] + state[index2]) % 256]); - } else if (typeof data[char] === 'number') { - finalKey += String.fromCharCode(data[char] ^ state[(state[index1] + state[index2]) % 256]); - } - } - return finalKey; -} export const getDecryptionKeys = async (ctx: EmbedScrapeContext): Promise => { const res = await ctx.fetcher( @@ -44,10 +19,10 @@ export const getEncodedId = async (ctx: EmbedScrapeContext) => { const id = url.pathname.replace('/e/', ''); const keyList = await getDecryptionKeys(ctx); - const decodedId = keyPermutation(keyList[0], id); - const encodedResult = keyPermutation(keyList[1], decodedId); - const base64 = btoa(encodedResult); - return base64.replace('/', '_'); + const decodedId = decodeData(keyList[0], id); + const encodedResult = decodeData(keyList[1], decodedId); + const b64encoded = btoa(encodedResult); + return b64encoded.replace('/', '_'); }; export const getFuTokenKey = async (ctx: EmbedScrapeContext) => { diff --git a/src/providers/sources/vidsrcto/common.ts b/src/providers/sources/vidsrcto/common.ts index fb9890d..2c7272f 100644 --- a/src/providers/sources/vidsrcto/common.ts +++ b/src/providers/sources/vidsrcto/common.ts @@ -1,3 +1,6 @@ +// This file is based on https://github.com/Ciarands/vidsrc-to-resolver/blob/dffa45e726a4b944cb9af0c9e7630476c93c0213/vidsrc.py#L16 +// Full credits to @Ciarands! + const DECRYPTION_KEY = '8z5Ag5wgagfsOuhz'; export const decodeBase64UrlSafe = (str: string) => { @@ -12,39 +15,35 @@ export const decodeBase64UrlSafe = (str: string) => { return bytes; }; -export const decode = (str: Uint8Array) => { - const keyBytes = new TextEncoder().encode(DECRYPTION_KEY); - - let j = 0; - const s = new Uint8Array(256); +export const decodeData = (key: string, data: any) => { + const state = Array.from(Array(256).keys()); + let index1 = 0; for (let i = 0; i < 256; i += 1) { - s[i] = i; + index1 = (index1 + state[i] + key.charCodeAt(i % key.length)) % 256; + const temp = state[i]; + state[i] = state[index1]; + state[index1] = temp; } - - for (let i = 0, k = 0; i < 256; i += 1) { - j = (j + s[i] + keyBytes[k % keyBytes.length]) & 0xff; - [s[i], s[j]] = [s[j], s[i]]; - k += 1; + index1 = 0; + let index2 = 0; + let finalKey = ''; + for (let char = 0; char < data.length; char += 1) { + index1 = (index1 + 1) % 256; + index2 = (index2 + state[index1]) % 256; + const temp = state[index1]; + state[index1] = state[index2]; + state[index2] = temp; + if (typeof data[char] === 'string') { + finalKey += String.fromCharCode(data[char].charCodeAt(0) ^ state[(state[index1] + state[index2]) % 256]); + } else if (typeof data[char] === 'number') { + finalKey += String.fromCharCode(data[char] ^ state[(state[index1] + state[index2]) % 256]); + } } - - const decoded = new Uint8Array(str.length); - let i = 0; - let k = 0; - for (let index = 0; index < str.length; index += 1) { - i = (i + 1) & 0xff; - k = (k + s[i]) & 0xff; - [s[i], s[k]] = [s[k], s[i]]; - const t = (s[i] + s[k]) & 0xff; - decoded[index] = str[index] ^ s[t]; - } - - return decoded; + return finalKey; }; export const decryptSourceUrl = (sourceUrl: string) => { const encoded = decodeBase64UrlSafe(sourceUrl); - const decoded = decode(encoded); - const decodedText = new TextDecoder().decode(decoded); - - return decodeURIComponent(decodeURIComponent(decodedText)); + const decoded = decodeData(DECRYPTION_KEY, encoded); + return decodeURIComponent(decodeURIComponent(decoded)); }; From d49c4081975f9dd0d5b6d86c9ac77bcc97966523 Mon Sep 17 00:00:00 2001 From: mrjvs Date: Fri, 5 Jan 2024 19:36:26 +0100 Subject: [PATCH 34/35] Fix remotestream for real --- src/providers/sources/remotestream.ts | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/providers/sources/remotestream.ts b/src/providers/sources/remotestream.ts index d358899..8a3090b 100644 --- a/src/providers/sources/remotestream.ts +++ b/src/providers/sources/remotestream.ts @@ -16,8 +16,12 @@ export const remotestreamScraper = makeSourcerer({ const playlistLink = `${remotestreamBase}/Shows/${ctx.media.tmdbId}/${seasonNumber}/${episodeNumber}/${episodeNumber}.m3u8`; ctx.progress(30); - const streamRes = await ctx.fetcher(playlistLink); // TODO support blobs in fetchers - if (streamRes.type !== 'application/x-mpegurl') throw new NotFoundError('No watchable item found'); + const streamRes = await ctx.fetcher.full(playlistLink, { + method: 'HEAD', + readHeaders: ['content-type'], + }); + if (!streamRes.headers.get('content-type')?.toLowerCase().includes('application/x-mpegurl')) + throw new NotFoundError('No watchable item found'); ctx.progress(90); return { @@ -37,8 +41,12 @@ export const remotestreamScraper = makeSourcerer({ const playlistLink = `${remotestreamBase}/Movies/${ctx.media.tmdbId}/${ctx.media.tmdbId}.m3u8`; ctx.progress(30); - const streamRes = await ctx.fetcher(playlistLink); - if (streamRes.type !== 'application/x-mpegurl') throw new NotFoundError('No watchable item found'); + const streamRes = await ctx.fetcher.full(playlistLink, { + method: 'HEAD', + readHeaders: ['content-type'], + }); + if (!streamRes.headers.get('content-type')?.toLowerCase().includes('application/x-mpegurl')) + throw new NotFoundError('No watchable item found'); ctx.progress(90); return { From 9276694ead47d46f8a2d8f8c693f75fb17bcfdbb Mon Sep 17 00:00:00 2001 From: Jorrin Date: Fri, 5 Jan 2024 19:52:28 +0100 Subject: [PATCH 35/35] remove flag --- src/providers/sources/vidsrcto/index.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/providers/sources/vidsrcto/index.ts b/src/providers/sources/vidsrcto/index.ts index 983fd12..b85b068 100644 --- a/src/providers/sources/vidsrcto/index.ts +++ b/src/providers/sources/vidsrcto/index.ts @@ -1,6 +1,5 @@ import { load } from 'cheerio'; -import { flags } from '@/entrypoint/utils/targets'; import { SourcererEmbed, SourcererOutput, makeSourcerer } from '@/providers/base'; import { MovieScrapeContext, ShowScrapeContext } from '@/utils/context'; @@ -80,6 +79,6 @@ export const vidSrcToScraper = makeSourcerer({ name: 'VidSrcTo', scrapeMovie: universalScraper, scrapeShow: universalScraper, - flags: [flags.CORS_ALLOWED], + flags: [], rank: 400, });