diff --git a/.eslintrc.js b/.eslintrc.js index 0e7322b..927283f 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -18,6 +18,8 @@ module.exports = { }, plugins: ['@typescript-eslint', 'import', 'prettier'], rules: { + 'no-plusplus': 'off', + 'no-bitwise': 'off', 'no-underscore-dangle': 'off', '@typescript-eslint/no-explicit-any': 'off', 'no-console': 'off', diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index d0f0ca6..7458772 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,3 +1 @@ -* @movie-web/core - -.github @binaryoverload +* @movie-web/project-leads diff --git a/package.json b/package.json index 4b91441..f7d1e18 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@movie-web/providers", - "version": "2.0.1", + "version": "2.0.3", "description": "Package that contains all the providers of movie-web", "main": "./lib/index.umd.js", "types": "./lib/index.d.ts", diff --git a/src/dev-cli/scraper.ts b/src/dev-cli/scraper.ts index 882d321..39f75f6 100644 --- a/src/dev-cli/scraper.ts +++ b/src/dev-cli/scraper.ts @@ -41,6 +41,7 @@ async function runBrowserScraping( args: ['--no-sandbox', '--disable-setuid-sandbox'], }); const page = await browser.newPage(); + page.on('console', (message) => console.log(`${message.type().slice(0, 3).toUpperCase()} ${message.text()}`)); await page.goto(server.resolvedUrls.local[0]); await page.waitForFunction('!!window.scrape', { timeout: 5000 }); diff --git a/src/dev-cli/tmdb.ts b/src/dev-cli/tmdb.ts index 7490336..c03307d 100644 --- a/src/dev-cli/tmdb.ts +++ b/src/dev-cli/tmdb.ts @@ -2,7 +2,7 @@ import { getConfig } from '@/dev-cli/config'; import { MovieMedia, ShowMedia } from '..'; -export async function makeTMDBRequest(url: string): Promise { +export async function makeTMDBRequest(url: string, appendToResponse?: string): Promise { const headers: { accept: 'application/json'; authorization?: string; @@ -10,7 +10,7 @@ export async function makeTMDBRequest(url: string): Promise { accept: 'application/json', }; - let requestURL = url; + const requestURL = new URL(url); const key = getConfig().tmdbApiKey; // * JWT keys always start with ey and are ONLY valid as a header. @@ -19,7 +19,11 @@ export async function makeTMDBRequest(url: string): Promise { if (key.startsWith('ey')) { headers.authorization = `Bearer ${key}`; } else { - requestURL += `?api_key=${key}`; + requestURL.searchParams.append('api_key', key); + } + + if (appendToResponse) { + requestURL.searchParams.append('append_to_response', appendToResponse); } return fetch(requestURL, { @@ -29,7 +33,7 @@ export async function makeTMDBRequest(url: string): Promise { } export async function getMovieMediaDetails(id: string): Promise { - const response = await makeTMDBRequest(`https://api.themoviedb.org/3/movie/${id}`); + const response = await makeTMDBRequest(`https://api.themoviedb.org/3/movie/${id}`, 'external_ids'); const movie = await response.json(); if (movie.success === false) { @@ -45,13 +49,14 @@ export async function getMovieMediaDetails(id: string): Promise { title: movie.title, releaseYear: Number(movie.release_date.split('-')[0]), tmdbId: id, + imdbId: movie.imdb_id, }; } export async function getShowMediaDetails(id: string, seasonNumber: string, episodeNumber: string): Promise { // * TV shows require the TMDB ID for the series, season, and episode // * and the name of the series. Needs multiple requests - let response = await makeTMDBRequest(`https://api.themoviedb.org/3/tv/${id}`); + let response = await makeTMDBRequest(`https://api.themoviedb.org/3/tv/${id}`, 'external_ids'); const series = await response.json(); if (series.success === false) { @@ -91,5 +96,6 @@ export async function getShowMediaDetails(id: string, seasonNumber: string, epis number: season.season_number, tmdbId: season.id, }, + imdbId: series.external_ids.imdb_id, }; } diff --git a/src/dev-cli/validate.ts b/src/dev-cli/validate.ts index b600454..dd1f638 100644 --- a/src/dev-cli/validate.ts +++ b/src/dev-cli/validate.ts @@ -81,6 +81,7 @@ export async function processOptions(sources: Array, options: const providerOptions: ProviderMakerOptions = { fetcher, target: targets.ANY, + consistentIpForRequests: true, }; return { diff --git a/src/entrypoint/providers.ts b/src/entrypoint/providers.ts index b306417..e456eb0 100644 --- a/src/entrypoint/providers.ts +++ b/src/entrypoint/providers.ts @@ -2,9 +2,9 @@ import { gatherAllEmbeds, gatherAllSources } from '@/providers/all'; import { Embed, Sourcerer } from '@/providers/base'; export function getBuiltinSources(): Sourcerer[] { - return gatherAllSources(); + return gatherAllSources().filter((v) => !v.disabled); } export function getBuiltinEmbeds(): Embed[] { - return gatherAllEmbeds(); + return gatherAllEmbeds().filter((v) => !v.disabled); } diff --git a/src/fetchers/simpleProxy.ts b/src/fetchers/simpleProxy.ts index 21ed5ca..360a149 100644 --- a/src/fetchers/simpleProxy.ts +++ b/src/fetchers/simpleProxy.ts @@ -7,6 +7,8 @@ const headerMap: Record = { cookie: 'X-Cookie', referer: 'X-Referer', origin: 'X-Origin', + 'user-agent': 'X-User-Agent', + 'x-real-ip': 'X-X-Real-Ip', }; const responseHeaderMap: Record = { diff --git a/src/fetchers/types.ts b/src/fetchers/types.ts index 7daa5df..f5dbe06 100644 --- a/src/fetchers/types.ts +++ b/src/fetchers/types.ts @@ -4,7 +4,7 @@ export type FetcherOptions = { baseUrl?: string; headers?: Record; query?: Record; - method?: 'GET' | 'POST'; + method?: 'HEAD' | 'GET' | 'POST'; readHeaders?: string[]; body?: Record | string | FormData | URLSearchParams; }; @@ -17,7 +17,7 @@ export type DefaultedFetcherOptions = { headers: Record; query: Record; readHeaders: string[]; - method: 'GET' | 'POST'; + method: 'HEAD' | 'GET' | 'POST'; }; export type FetcherResponse = { diff --git a/src/providers/all.ts b/src/providers/all.ts index d1e7885..de22b4c 100644 --- a/src/providers/all.ts +++ b/src/providers/all.ts @@ -3,20 +3,26 @@ import { febboxHlsScraper } from '@/providers/embeds/febbox/hls'; import { febboxMp4Scraper } from '@/providers/embeds/febbox/mp4'; import { mixdropScraper } from '@/providers/embeds/mixdrop'; import { mp4uploadScraper } from '@/providers/embeds/mp4upload'; +import { streambucketScraper } from '@/providers/embeds/streambucket'; import { streamsbScraper } from '@/providers/embeds/streamsb'; import { upcloudScraper } from '@/providers/embeds/upcloud'; import { upstreamScraper } from '@/providers/embeds/upstream'; +import { vidsrcembedScraper } from '@/providers/embeds/vidsrc'; import { flixhqScraper } from '@/providers/sources/flixhq/index'; import { goMoviesScraper } from '@/providers/sources/gomovies/index'; import { kissAsianScraper } from '@/providers/sources/kissasian/index'; import { lookmovieScraper } from '@/providers/sources/lookmovie'; import { remotestreamScraper } from '@/providers/sources/remotestream'; import { showboxScraper } from '@/providers/sources/showbox/index'; +import { vidsrcScraper } from '@/providers/sources/vidsrc/index'; import { zoechipScraper } from '@/providers/sources/zoechip'; +import { fileMoonScraper } from './embeds/filemoon'; import { smashyStreamDScraper } from './embeds/smashystream/dued'; import { smashyStreamFScraper } from './embeds/smashystream/video1'; +import { vidplayScraper } from './embeds/vidplay'; import { smashyStreamScraper } from './sources/smashystream'; +import { vidSrcToScraper } from './sources/vidsrcto'; export function gatherAllSources(): Array { // all sources are gathered here @@ -27,8 +33,10 @@ export function gatherAllSources(): Array { showboxScraper, goMoviesScraper, zoechipScraper, + vidsrcScraper, lookmovieScraper, smashyStreamScraper, + vidSrcToScraper, ]; } @@ -42,7 +50,11 @@ export function gatherAllEmbeds(): Array { febboxMp4Scraper, febboxHlsScraper, mixdropScraper, + vidsrcembedScraper, + streambucketScraper, smashyStreamFScraper, smashyStreamDScraper, + fileMoonScraper, + vidplayScraper, ]; } diff --git a/src/providers/captions.ts b/src/providers/captions.ts index ce3f398..92e5db3 100644 --- a/src/providers/captions.ts +++ b/src/providers/captions.ts @@ -31,3 +31,13 @@ export function isValidLanguageCode(code: string | null): boolean { if (!code) return false; return ISO6391.validate(code); } + +export function removeDuplicatedLanguages(list: Caption[]) { + const beenSeen: Record = {}; + + return list.filter((sub) => { + if (beenSeen[sub.language]) return false; + beenSeen[sub.language] = true; + return true; + }); +} diff --git a/src/providers/embeds/febbox/hls.ts b/src/providers/embeds/febbox/hls.ts index d9fa54f..792c112 100644 --- a/src/providers/embeds/febbox/hls.ts +++ b/src/providers/embeds/febbox/hls.ts @@ -1,5 +1,4 @@ import { MediaTypes } from '@/entrypoint/utils/media'; -import { flags } from '@/entrypoint/utils/targets'; import { makeEmbed } from '@/providers/base'; import { parseInputUrl } from '@/providers/embeds/febbox/common'; import { getStreams } from '@/providers/embeds/febbox/fileList'; @@ -16,6 +15,7 @@ export const febboxHlsScraper = makeEmbed({ id: 'febbox-hls', name: 'Febbox (HLS)', rank: 160, + disabled: true, async scrape(ctx) { const { type, id, season, episode } = parseInputUrl(ctx.url); const sharelinkResult = await ctx.proxiedFetcher<{ @@ -40,7 +40,7 @@ export const febboxHlsScraper = makeEmbed({ { id: 'primary', type: 'hls', - flags: [flags.CORS_ALLOWED], + flags: [], captions: await getSubtitles(ctx, id, firstStream.fid, type as MediaTypes, season, episode), playlist: `https://www.febbox.com/hls/main/${firstStream.oss_fid}.m3u8`, }, diff --git a/src/providers/embeds/febbox/subtitles.ts b/src/providers/embeds/febbox/subtitles.ts index b1b3064..fbb2e85 100644 --- a/src/providers/embeds/febbox/subtitles.ts +++ b/src/providers/embeds/febbox/subtitles.ts @@ -1,4 +1,9 @@ -import { Caption, getCaptionTypeFromUrl, isValidLanguageCode } from '@/providers/captions'; +import { + Caption, + getCaptionTypeFromUrl, + isValidLanguageCode, + removeDuplicatedLanguages as removeDuplicateLanguages, +} from '@/providers/captions'; import { captionsDomains } from '@/providers/sources/showbox/common'; import { sendRequest } from '@/providers/sources/showbox/sendRequest'; import { ScrapeContext } from '@/utils/context'; @@ -36,17 +41,19 @@ export async function getSubtitles( const subResult = (await sendRequest(ctx, subtitleApiQuery)) as CaptionApiResponse; const subtitleList = subResult.data.list; - const output: Caption[] = []; + let output: Caption[] = []; subtitleList.forEach((sub) => { const subtitle = sub.subtitles.sort((a, b) => b.order - a.order)[0]; if (!subtitle) return; + const subtitleFilePath = subtitle.file_path .replace(captionsDomains[0], captionsDomains[1]) .replace(/\s/g, '+') .replace(/[()]/g, (c) => { return `%${c.charCodeAt(0).toString(16)}`; }); + const subtitleType = getCaptionTypeFromUrl(subtitleFilePath); if (!subtitleType) return; @@ -62,5 +69,7 @@ export async function getSubtitles( }); }); + output = removeDuplicateLanguages(output); + return output; } diff --git a/src/providers/embeds/filemoon/index.ts b/src/providers/embeds/filemoon/index.ts new file mode 100644 index 0000000..9f96a07 --- /dev/null +++ b/src/providers/embeds/filemoon/index.ts @@ -0,0 +1,58 @@ +import { unpack } from 'unpacker'; + +import { SubtitleResult } from './types'; +import { makeEmbed } from '../../base'; +import { Caption, getCaptionTypeFromUrl, labelToLanguageCode } from '../../captions'; + +const evalCodeRegex = /eval\((.*)\)/g; +const fileRegex = /file:"(.*?)"/g; + +export const fileMoonScraper = makeEmbed({ + id: 'filemoon', + name: 'Filemoon', + rank: 400, + scrape: async (ctx) => { + const embedRes = await ctx.proxiedFetcher(ctx.url, { + headers: { + referer: ctx.url, + }, + }); + const evalCode = evalCodeRegex.exec(embedRes); + if (!evalCode) throw new Error('Failed to find eval code'); + const unpacked = unpack(evalCode[1]); + const file = fileRegex.exec(unpacked); + if (!file?.[1]) throw new Error('Failed to find file'); + + const url = new URL(ctx.url); + const subtitlesLink = url.searchParams.get('sub.info'); + const captions: Caption[] = []; + if (subtitlesLink) { + const captionsResult = await ctx.proxiedFetcher(subtitlesLink); + + for (const caption of captionsResult) { + const language = labelToLanguageCode(caption.label); + const captionType = getCaptionTypeFromUrl(caption.file); + if (!language || !captionType) continue; + captions.push({ + id: caption.file, + url: caption.file, + type: captionType, + language, + hasCorsRestrictions: false, + }); + } + } + + return { + stream: [ + { + id: 'primary', + type: 'hls', + playlist: file[1], + flags: [], + captions, + }, + ], + }; + }, +}); diff --git a/src/providers/embeds/filemoon/types.ts b/src/providers/embeds/filemoon/types.ts new file mode 100644 index 0000000..caa27af --- /dev/null +++ b/src/providers/embeds/filemoon/types.ts @@ -0,0 +1,5 @@ +export type SubtitleResult = { + file: string; + label: string; + kind: string; +}[]; diff --git a/src/providers/embeds/streambucket.ts b/src/providers/embeds/streambucket.ts new file mode 100644 index 0000000..9e21a93 --- /dev/null +++ b/src/providers/embeds/streambucket.ts @@ -0,0 +1,101 @@ +import { flags } from '@/entrypoint/utils/targets'; +import { makeEmbed } from '@/providers/base'; + +// StreamBucket makes use of https://github.com/nicxlau/hunter-php-javascript-obfuscator + +const hunterRegex = /eval\(function\(h,u,n,t,e,r\).*?\("(.*?)",\d*?,"(.*?)",(\d*?),(\d*?),\d*?\)\)/; +const linkRegex = /file:"(.*?)"/; + +// This is a much more simple and optimized version of the "h,u,n,t,e,r" +// obfuscation algorithm. It's just basic chunked+mask encoding. +// I have seen this same encoding used on some sites under the name +// "p,l,a,y,e,r" as well +function decodeHunter(encoded: string, mask: string, charCodeOffset: number, delimiterOffset: number) { + // The encoded string is made up of 'n' number of chunks. + // Each chunk is separated by a delimiter inside the mask. + // This offset is also used as the exponentiation base in + // the charCode calculations + const delimiter = mask[delimiterOffset]; + + // Split the 'encoded' string into chunks using the delimiter, + // and filter out any empty chunks. + const chunks = encoded.split(delimiter).filter((chunk) => chunk); + + // Decode each chunk and concatenate the results to form the final 'decoded' string. + const decoded = chunks + .map((chunk) => { + // Chunks are in reverse order. 'reduceRight' removes the + // need to 'reverse' the array first + const charCode = chunk.split('').reduceRight((c, value, index) => { + // Calculate the character code for each character in the chunk. + // This involves finding the index of 'value' in the 'mask' and + // multiplying it by (delimiterOffset^position). + return c + mask.indexOf(value) * delimiterOffset ** (chunk.length - 1 - index); + }, 0); + + // The actual character code is offset by the given amount + return String.fromCharCode(charCode - charCodeOffset); + }) + .join(''); + + return decoded; +} + +export const streambucketScraper = makeEmbed({ + id: 'streambucket', + name: 'StreamBucket', + rank: 196, + // TODO - Disabled until ctx.fetcher and ctx.proxiedFetcher don't trigger bot detection + disabled: true, + async scrape(ctx) { + // Using the context fetchers make the site return just the string "No bots please!"? + // TODO - Fix this. Native fetch does not trigger this. No idea why right now + const response = await fetch(ctx.url); + const html = await response.text(); + + // This is different than the above mentioned bot detection + if (html.includes('captcha-checkbox')) { + // TODO - This doesn't use recaptcha, just really basic "image match". Maybe could automate? + throw new Error('StreamBucket got captchaed'); + } + + let regexResult = html.match(hunterRegex); + + if (!regexResult) { + throw new Error('Failed to find StreamBucket hunter JavaScript'); + } + + const encoded = regexResult[1]; + const mask = regexResult[2]; + const charCodeOffset = Number(regexResult[3]); + const delimiterOffset = Number(regexResult[4]); + + if (Number.isNaN(charCodeOffset)) { + throw new Error('StreamBucket hunter JavaScript charCodeOffset is not a valid number'); + } + + if (Number.isNaN(delimiterOffset)) { + throw new Error('StreamBucket hunter JavaScript delimiterOffset is not a valid number'); + } + + const decoded = decodeHunter(encoded, mask, charCodeOffset, delimiterOffset); + + regexResult = decoded.match(linkRegex); + + if (!regexResult) { + throw new Error('Failed to find StreamBucket HLS link'); + } + + return { + stream: [ + { + id: 'primary', + type: 'hls', + playlist: regexResult[1], + flags: [flags.CORS_ALLOWED], + captions: [], + }, + ], + }; + }, +}); diff --git a/src/providers/embeds/vidplay/common.ts b/src/providers/embeds/vidplay/common.ts new file mode 100644 index 0000000..224e3dc --- /dev/null +++ b/src/providers/embeds/vidplay/common.ts @@ -0,0 +1,54 @@ +import { makeFullUrl } from '@/fetchers/common'; +import { decodeData } from '@/providers/sources/vidsrcto/common'; +import { EmbedScrapeContext } from '@/utils/context'; + +export const vidplayBase = 'https://vidplay.site'; + +// This file is based on https://github.com/Ciarands/vidsrc-to-resolver/blob/dffa45e726a4b944cb9af0c9e7630476c93c0213/vidsrc.py#L16 +// Full credits to @Ciarands! + +export const getDecryptionKeys = async (ctx: EmbedScrapeContext): Promise => { + const res = await ctx.fetcher( + 'https://raw.githubusercontent.com/Claudemirovsky/worstsource-keys/keys/keys.json', + ); + return JSON.parse(res); +}; + +export const getEncodedId = async (ctx: EmbedScrapeContext) => { + const url = new URL(ctx.url); + const id = url.pathname.replace('/e/', ''); + const keyList = await getDecryptionKeys(ctx); + + const decodedId = decodeData(keyList[0], id); + const encodedResult = decodeData(keyList[1], decodedId); + const b64encoded = btoa(encodedResult); + return b64encoded.replace('/', '_'); +}; + +export const getFuTokenKey = async (ctx: EmbedScrapeContext) => { + const id = await getEncodedId(ctx); + const fuTokenRes = await ctx.proxiedFetcher('/futoken', { + baseUrl: vidplayBase, + headers: { + referer: ctx.url, + }, + }); + const fuKey = fuTokenRes.match(/var\s+k\s*=\s*'([^']+)'/)?.[1]; + if (!fuKey) throw new Error('No fuKey found'); + const tokens = []; + for (let i = 0; i < id.length; i += 1) { + tokens.push(fuKey.charCodeAt(i % fuKey.length) + id.charCodeAt(i)); + } + return `${fuKey},${tokens.join(',')}`; +}; + +export const getFileUrl = async (ctx: EmbedScrapeContext) => { + const fuToken = await getFuTokenKey(ctx); + return makeFullUrl(`/mediainfo/${fuToken}`, { + baseUrl: vidplayBase, + query: { + ...Object.fromEntries(new URL(ctx.url).searchParams.entries()), + autostart: 'true', + }, + }); +}; diff --git a/src/providers/embeds/vidplay/index.ts b/src/providers/embeds/vidplay/index.ts new file mode 100644 index 0000000..3c1f6a2 --- /dev/null +++ b/src/providers/embeds/vidplay/index.ts @@ -0,0 +1,53 @@ +import { makeEmbed } from '@/providers/base'; +import { Caption, getCaptionTypeFromUrl, labelToLanguageCode } from '@/providers/captions'; + +import { getFileUrl } from './common'; +import { SubtitleResult, VidplaySourceResponse } from './types'; + +export const vidplayScraper = makeEmbed({ + id: 'vidplay', + name: 'VidPlay', + rank: 401, + scrape: async (ctx) => { + const fileUrl = await getFileUrl(ctx); + const fileUrlRes = await ctx.proxiedFetcher(fileUrl, { + headers: { + referer: ctx.url, + }, + }); + if (typeof fileUrlRes.result === 'number') throw new Error('File not found'); + const source = fileUrlRes.result.sources[0].file; + + const url = new URL(ctx.url); + const subtitlesLink = url.searchParams.get('sub.info'); + const captions: Caption[] = []; + if (subtitlesLink) { + const captionsResult = await ctx.proxiedFetcher(subtitlesLink); + + for (const caption of captionsResult) { + const language = labelToLanguageCode(caption.label); + const captionType = getCaptionTypeFromUrl(caption.file); + if (!language || !captionType) continue; + captions.push({ + id: caption.file, + url: caption.file, + type: captionType, + language, + hasCorsRestrictions: false, + }); + } + } + + return { + stream: [ + { + id: 'primary', + type: 'hls', + playlist: source, + flags: [], + captions, + }, + ], + }; + }, +}); diff --git a/src/providers/embeds/vidplay/types.ts b/src/providers/embeds/vidplay/types.ts new file mode 100644 index 0000000..29cde1d --- /dev/null +++ b/src/providers/embeds/vidplay/types.ts @@ -0,0 +1,19 @@ +export type VidplaySourceResponse = { + result: + | { + sources: { + file: string; + tracks: { + file: string; + kind: string; + }[]; + }[]; + } + | number; +}; + +export type SubtitleResult = { + file: string; + label: string; + kind: string; +}[]; diff --git a/src/providers/embeds/vidsrc.ts b/src/providers/embeds/vidsrc.ts new file mode 100644 index 0000000..cd47e21 --- /dev/null +++ b/src/providers/embeds/vidsrc.ts @@ -0,0 +1,55 @@ +import { flags } from '@/entrypoint/utils/targets'; +import { makeEmbed } from '@/providers/base'; + +const hlsURLRegex = /file:"(.*?)"/; +const setPassRegex = /var pass_path = "(.*set_pass\.php.*)";/; + +export const vidsrcembedScraper = makeEmbed({ + id: 'vidsrcembed', // VidSrc is both a source and an embed host + name: 'VidSrc', + rank: 197, + async scrape(ctx) { + const html = await ctx.proxiedFetcher(ctx.url, { + headers: { + referer: ctx.url, + }, + }); + + const match = html + .match(hlsURLRegex)?.[1] + ?.replace(/(\/\/\S+?=)/g, '') + .replace('#2', ''); + if (!match) throw new Error('Unable to find HLS playlist'); + const finalUrl = atob(match); + + if (!finalUrl.includes('.m3u8')) throw new Error('Unable to find HLS playlist'); + + let setPassLink = html.match(setPassRegex)?.[1]; + if (!setPassLink) throw new Error('Unable to find set_pass.php link'); + + if (setPassLink.startsWith('//')) { + setPassLink = `https:${setPassLink}`; + } + + // VidSrc uses a password endpoint to temporarily whitelist the user's IP. This is called in an interval by the player. + // It currently has no effect on the player itself, the content plays fine without it. + // In the future we might have to introduce hooks for the frontend to call this endpoint. + await ctx.proxiedFetcher(setPassLink, { + headers: { + referer: ctx.url, + }, + }); + + return { + stream: [ + { + id: 'primary', + type: 'hls', + playlist: finalUrl, + flags: [flags.CORS_ALLOWED], + captions: [], + }, + ], + }; + }, +}); diff --git a/src/providers/sources/lookmovie/index.ts b/src/providers/sources/lookmovie/index.ts index 5cd82e9..8611373 100644 --- a/src/providers/sources/lookmovie/index.ts +++ b/src/providers/sources/lookmovie/index.ts @@ -10,8 +10,8 @@ async function universalScraper(ctx: MovieScrapeContext | ShowScrapeContext): Pr if (!lookmovieData) throw new NotFoundError('Media not found'); ctx.progress(30); - const videoUrl = await scrape(ctx, ctx.media, lookmovieData); - if (!videoUrl) throw new NotFoundError('No video found'); + const video = await scrape(ctx, ctx.media, lookmovieData); + if (!video.playlist) throw new NotFoundError('No video found'); ctx.progress(60); @@ -20,10 +20,10 @@ async function universalScraper(ctx: MovieScrapeContext | ShowScrapeContext): Pr stream: [ { id: 'primary', - playlist: videoUrl, + playlist: video.playlist, type: 'hls', flags: [flags.IP_LOCKED], - captions: [], + captions: video.captions, }, ], }; diff --git a/src/providers/sources/lookmovie/type.ts b/src/providers/sources/lookmovie/type.ts index 8335c96..47ccc55 100644 --- a/src/providers/sources/lookmovie/type.ts +++ b/src/providers/sources/lookmovie/type.ts @@ -39,8 +39,17 @@ interface VideoSources { [key: string]: string; } +interface VideoSubtitles { + id?: number; + id_movie?: number; + url: string; + language: string; + shard?: string; +} + export interface StreamsDataResult { streams: VideoSources; + subtitles: VideoSubtitles[]; } export interface ResultItem { diff --git a/src/providers/sources/lookmovie/util.ts b/src/providers/sources/lookmovie/util.ts index 7c8f202..6057f76 100644 --- a/src/providers/sources/lookmovie/util.ts +++ b/src/providers/sources/lookmovie/util.ts @@ -4,7 +4,9 @@ import { ScrapeContext } from '@/utils/context'; import { NotFoundError } from '@/utils/errors'; import { Result, ResultItem, ShowDataResult, episodeObj } from './type'; -import { getVideoUrl } from './video'; +import { getVideo } from './video'; + +export const baseUrl = 'https://lmscript.xyz'; export async function searchAndFindMedia( ctx: ScrapeContext, @@ -12,7 +14,7 @@ export async function searchAndFindMedia( ): Promise { if (media.type === 'show') { const searchRes = await ctx.fetcher(`/v1/shows`, { - baseUrl: 'https://lmscript.xyz', + baseUrl, query: { 'filters[q]': media.title }, }); @@ -23,7 +25,7 @@ export async function searchAndFindMedia( } if (media.type === 'movie') { const searchRes = await ctx.fetcher(`/v1/movies`, { - baseUrl: 'https://lmscript.xyz', + baseUrl, query: { 'filters[q]': media.title }, }); @@ -40,7 +42,7 @@ export async function scrape(ctx: ScrapeContext, media: MovieMedia | ShowMedia, id = result.id_movie; } else if (media.type === 'show') { const data = await ctx.fetcher(`/v1/shows`, { - baseUrl: 'https://lmscript.xyz', + baseUrl, query: { expand: 'episodes', id: result.id_show }, }); @@ -54,6 +56,6 @@ export async function scrape(ctx: ScrapeContext, media: MovieMedia | ShowMedia, // Check ID if (id === null) throw new NotFoundError('Not found'); - const videoUrl = await getVideoUrl(ctx, id, media); - return videoUrl; + const video = await getVideo(ctx, id, media); + return video; } diff --git a/src/providers/sources/lookmovie/video.ts b/src/providers/sources/lookmovie/video.ts index f439229..8e8e3c4 100644 --- a/src/providers/sources/lookmovie/video.ts +++ b/src/providers/sources/lookmovie/video.ts @@ -1,7 +1,9 @@ import { MovieMedia, ShowMedia } from '@/entrypoint/utils/media'; +import { Caption, labelToLanguageCode, removeDuplicatedLanguages } from '@/providers/captions'; import { ScrapeContext } from '@/utils/context'; import { StreamsDataResult } from './type'; +import { baseUrl } from './util'; export async function getVideoSources( ctx: ScrapeContext, @@ -17,17 +19,17 @@ export async function getVideoSources( path = `/v1/movies/view`; } const data = await ctx.fetcher(path, { - baseUrl: 'https://lmscript.xyz', - query: { expand: 'streams', id }, + baseUrl, + query: { expand: 'streams,subtitles', id }, }); return data; } -export async function getVideoUrl( +export async function getVideo( ctx: ScrapeContext, id: string, media: MovieMedia | ShowMedia, -): Promise { +): Promise<{ playlist: string | null; captions: Caption[] }> { // Get sources const data = await getVideoSources(ctx, id, media); const videoSources = data.streams; @@ -42,5 +44,24 @@ export async function getVideoUrl( } } - return videoUrl; + let captions: Caption[] = []; + + for (const sub of data.subtitles) { + const language = labelToLanguageCode(sub.language); + if (!language) continue; + captions.push({ + id: sub.url, + type: 'vtt', + url: `${baseUrl}${sub.url}`, + hasCorsRestrictions: false, + language, + }); + } + + captions = removeDuplicatedLanguages(captions); + + return { + playlist: videoUrl, + captions, + }; } diff --git a/src/providers/sources/remotestream.ts b/src/providers/sources/remotestream.ts index 6d0a44c..8a3090b 100644 --- a/src/providers/sources/remotestream.ts +++ b/src/providers/sources/remotestream.ts @@ -2,7 +2,7 @@ import { flags } from '@/entrypoint/utils/targets'; import { makeSourcerer } from '@/providers/base'; import { NotFoundError } from '@/utils/errors'; -const remotestreamBase = `https://fsa.remotestre.am`; +const remotestreamBase = atob('aHR0cHM6Ly9mc2IuOG1ldDNkdGpmcmNxY2hjb25xcGtsd3hzeGIyb2N1bWMuc3RyZWFt'); export const remotestreamScraper = makeSourcerer({ id: 'remotestream', @@ -16,8 +16,12 @@ export const remotestreamScraper = makeSourcerer({ const playlistLink = `${remotestreamBase}/Shows/${ctx.media.tmdbId}/${seasonNumber}/${episodeNumber}/${episodeNumber}.m3u8`; ctx.progress(30); - const streamRes = await ctx.fetcher(playlistLink); // TODO support blobs in fetchers - if (streamRes.type !== 'application/x-mpegurl') throw new NotFoundError('No watchable item found'); + const streamRes = await ctx.fetcher.full(playlistLink, { + method: 'HEAD', + readHeaders: ['content-type'], + }); + if (!streamRes.headers.get('content-type')?.toLowerCase().includes('application/x-mpegurl')) + throw new NotFoundError('No watchable item found'); ctx.progress(90); return { @@ -37,8 +41,12 @@ export const remotestreamScraper = makeSourcerer({ const playlistLink = `${remotestreamBase}/Movies/${ctx.media.tmdbId}/${ctx.media.tmdbId}.m3u8`; ctx.progress(30); - const streamRes = await ctx.fetcher(playlistLink); - if (streamRes.type !== 'application/x-mpegurl') throw new NotFoundError('No watchable item found'); + const streamRes = await ctx.fetcher.full(playlistLink, { + method: 'HEAD', + readHeaders: ['content-type'], + }); + if (!streamRes.headers.get('content-type')?.toLowerCase().includes('application/x-mpegurl')) + throw new NotFoundError('No watchable item found'); ctx.progress(90); return { diff --git a/src/providers/sources/showbox/index.ts b/src/providers/sources/showbox/index.ts index 267a6ef..d6c4887 100644 --- a/src/providers/sources/showbox/index.ts +++ b/src/providers/sources/showbox/index.ts @@ -1,6 +1,5 @@ import { flags } from '@/entrypoint/utils/targets'; import { SourcererOutput, makeSourcerer } from '@/providers/base'; -import { febboxHlsScraper } from '@/providers/embeds/febbox/hls'; import { febboxMp4Scraper } from '@/providers/embeds/febbox/mp4'; import { compareTitle } from '@/utils/compare'; import { MovieScrapeContext, ShowScrapeContext } from '@/utils/context'; @@ -31,10 +30,6 @@ async function comboScraper(ctx: ShowScrapeContext | MovieScrapeContext): Promis return { embeds: [ - { - embedId: febboxHlsScraper.id, - url: `/${ctx.media.type}/${id}/${season}/${episode}`, - }, { embedId: febboxMp4Scraper.id, url: `/${ctx.media.type}/${id}/${season}/${episode}`, diff --git a/src/providers/sources/showbox/sendRequest.ts b/src/providers/sources/showbox/sendRequest.ts index 2f77767..7ea9024 100644 --- a/src/providers/sources/showbox/sendRequest.ts +++ b/src/providers/sources/showbox/sendRequest.ts @@ -49,9 +49,9 @@ export const sendRequest = async (ctx: ScrapeContext, data: object, altApi = fal headers: { Platform: 'android', 'Content-Type': 'application/x-www-form-urlencoded', + 'User-Agent': 'okhttp/3.2.0', }, body: formatted, }); - return JSON.parse(response); }; diff --git a/src/providers/sources/vidsrc/common.ts b/src/providers/sources/vidsrc/common.ts new file mode 100644 index 0000000..4ccc93c --- /dev/null +++ b/src/providers/sources/vidsrc/common.ts @@ -0,0 +1,2 @@ +export const vidsrcBase = 'https://vidsrc.me'; +export const vidsrcRCPBase = 'https://rcp.vidsrc.me'; diff --git a/src/providers/sources/vidsrc/index.ts b/src/providers/sources/vidsrc/index.ts new file mode 100644 index 0000000..6331a05 --- /dev/null +++ b/src/providers/sources/vidsrc/index.ts @@ -0,0 +1,13 @@ +import { flags } from '@/entrypoint/utils/targets'; +import { makeSourcerer } from '@/providers/base'; +import { scrapeMovie } from '@/providers/sources/vidsrc/scrape-movie'; +import { scrapeShow } from '@/providers/sources/vidsrc/scrape-show'; + +export const vidsrcScraper = makeSourcerer({ + id: 'vidsrc', + name: 'VidSrc', + rank: 120, + flags: [flags.CORS_ALLOWED], + scrapeMovie, + scrapeShow, +}); diff --git a/src/providers/sources/vidsrc/scrape-movie.ts b/src/providers/sources/vidsrc/scrape-movie.ts new file mode 100644 index 0000000..585eb31 --- /dev/null +++ b/src/providers/sources/vidsrc/scrape-movie.ts @@ -0,0 +1,8 @@ +import { getVidSrcMovieSources } from '@/providers/sources/vidsrc/scrape'; +import { MovieScrapeContext } from '@/utils/context'; + +export async function scrapeMovie(ctx: MovieScrapeContext) { + return { + embeds: await getVidSrcMovieSources(ctx), + }; +} diff --git a/src/providers/sources/vidsrc/scrape-show.ts b/src/providers/sources/vidsrc/scrape-show.ts new file mode 100644 index 0000000..ff5d2a4 --- /dev/null +++ b/src/providers/sources/vidsrc/scrape-show.ts @@ -0,0 +1,8 @@ +import { getVidSrcShowSources } from '@/providers/sources/vidsrc/scrape'; +import { ShowScrapeContext } from '@/utils/context'; + +export async function scrapeShow(ctx: ShowScrapeContext) { + return { + embeds: await getVidSrcShowSources(ctx), + }; +} diff --git a/src/providers/sources/vidsrc/scrape.ts b/src/providers/sources/vidsrc/scrape.ts new file mode 100644 index 0000000..81dceff --- /dev/null +++ b/src/providers/sources/vidsrc/scrape.ts @@ -0,0 +1,133 @@ +import { load } from 'cheerio'; + +import { SourcererEmbed } from '@/providers/base'; +import { streambucketScraper } from '@/providers/embeds/streambucket'; +import { vidsrcembedScraper } from '@/providers/embeds/vidsrc'; +import { vidsrcBase, vidsrcRCPBase } from '@/providers/sources/vidsrc/common'; +import { MovieScrapeContext, ShowScrapeContext } from '@/utils/context'; + +function decodeSrc(encoded: string, seed: string) { + let decoded = ''; + const seedLength = seed.length; + + for (let i = 0; i < encoded.length; i += 2) { + const byte = parseInt(encoded.substr(i, 2), 16); + const seedChar = seed.charCodeAt((i / 2) % seedLength); + decoded += String.fromCharCode(byte ^ seedChar); + } + + return decoded; +} + +async function getVidSrcEmbeds(ctx: MovieScrapeContext | ShowScrapeContext, startingURL: string) { + // VidSrc works by using hashes and a redirect system. + // The hashes are stored in the html, and VidSrc will + // make requests to their servers with the hash. This + // will trigger a 302 response with a Location header + // sending the user to the correct embed. To get the + // real embed links, we must do the same. Slow, but + // required + + const embeds: SourcererEmbed[] = []; + + let html = await ctx.proxiedFetcher(startingURL, { + baseUrl: vidsrcBase, + }); + + let $ = load(html); + + const sourceHashes = $('.server[data-hash]') + .toArray() + .map((el) => $(el).attr('data-hash')) + .filter((hash) => hash !== undefined); + + for (const hash of sourceHashes) { + html = await ctx.proxiedFetcher(`/rcp/${hash}`, { + baseUrl: vidsrcRCPBase, + headers: { + referer: vidsrcBase, + }, + }); + + $ = load(html); + const encoded = $('#hidden').attr('data-h'); + const seed = $('body').attr('data-i'); + + if (!encoded || !seed) { + throw new Error('Failed to find encoded iframe src'); + } + + let redirectURL = decodeSrc(encoded, seed); + if (redirectURL.startsWith('//')) { + redirectURL = `https:${redirectURL}`; + } + + const { finalUrl } = await ctx.proxiedFetcher.full(redirectURL, { + method: 'HEAD', + headers: { + referer: vidsrcBase, + }, + }); + + const embed: SourcererEmbed = { + embedId: '', + url: finalUrl, + }; + + const parsedUrl = new URL(finalUrl); + + switch (parsedUrl.host) { + case 'vidsrc.stream': + embed.embedId = vidsrcembedScraper.id; + break; + case 'streambucket.net': + embed.embedId = streambucketScraper.id; + break; + case '2embed.cc': + case 'www.2embed.cc': + // Just ignore this. This embed just sources from other embeds we can scrape as a 'source' + break; + case 'player-cdn.com': + // Just ignore this. This embed streams video over a custom WebSocket connection + break; + default: + throw new Error(`Failed to find VidSrc embed source for ${finalUrl}`); + } + + // Since some embeds are ignored on purpose, check if a valid one was found + if (embed.embedId !== '') { + embeds.push(embed); + } + } + + return embeds; +} + +export async function getVidSrcMovieSources(ctx: MovieScrapeContext) { + return getVidSrcEmbeds(ctx, `/embed/${ctx.media.tmdbId}`); +} + +export async function getVidSrcShowSources(ctx: ShowScrapeContext) { + // VidSrc will always default to season 1 episode 1 + // no matter what embed URL is used. It sends back + // a list of ALL the shows episodes, in order, for + // all seasons. To get the real embed URL, have to + // parse this from the response + const html = await ctx.proxiedFetcher(`/embed/${ctx.media.tmdbId}`, { + baseUrl: vidsrcBase, + }); + + const $ = load(html); + + const episodeElement = $(`.ep[data-s="${ctx.media.season.number}"][data-e="${ctx.media.episode.number}"]`).first(); + if (episodeElement.length === 0) { + throw new Error('failed to find episode element'); + } + + const startingURL = episodeElement.attr('data-iframe'); + if (!startingURL) { + throw new Error('failed to find episode starting URL'); + } + + return getVidSrcEmbeds(ctx, startingURL); +} diff --git a/src/providers/sources/vidsrcto/common.ts b/src/providers/sources/vidsrcto/common.ts new file mode 100644 index 0000000..2c7272f --- /dev/null +++ b/src/providers/sources/vidsrcto/common.ts @@ -0,0 +1,49 @@ +// This file is based on https://github.com/Ciarands/vidsrc-to-resolver/blob/dffa45e726a4b944cb9af0c9e7630476c93c0213/vidsrc.py#L16 +// Full credits to @Ciarands! + +const DECRYPTION_KEY = '8z5Ag5wgagfsOuhz'; + +export const decodeBase64UrlSafe = (str: string) => { + const standardizedInput = str.replace(/_/g, '/').replace(/-/g, '+'); + const decodedData = atob(standardizedInput); + + const bytes = new Uint8Array(decodedData.length); + for (let i = 0; i < bytes.length; i += 1) { + bytes[i] = decodedData.charCodeAt(i); + } + + return bytes; +}; + +export const decodeData = (key: string, data: any) => { + const state = Array.from(Array(256).keys()); + let index1 = 0; + for (let i = 0; i < 256; i += 1) { + index1 = (index1 + state[i] + key.charCodeAt(i % key.length)) % 256; + const temp = state[i]; + state[i] = state[index1]; + state[index1] = temp; + } + index1 = 0; + let index2 = 0; + let finalKey = ''; + for (let char = 0; char < data.length; char += 1) { + index1 = (index1 + 1) % 256; + index2 = (index2 + state[index1]) % 256; + const temp = state[index1]; + state[index1] = state[index2]; + state[index2] = temp; + if (typeof data[char] === 'string') { + finalKey += String.fromCharCode(data[char].charCodeAt(0) ^ state[(state[index1] + state[index2]) % 256]); + } else if (typeof data[char] === 'number') { + finalKey += String.fromCharCode(data[char] ^ state[(state[index1] + state[index2]) % 256]); + } + } + return finalKey; +}; + +export const decryptSourceUrl = (sourceUrl: string) => { + const encoded = decodeBase64UrlSafe(sourceUrl); + const decoded = decodeData(DECRYPTION_KEY, encoded); + return decodeURIComponent(decodeURIComponent(decoded)); +}; diff --git a/src/providers/sources/vidsrcto/index.ts b/src/providers/sources/vidsrcto/index.ts new file mode 100644 index 0000000..b85b068 --- /dev/null +++ b/src/providers/sources/vidsrcto/index.ts @@ -0,0 +1,84 @@ +import { load } from 'cheerio'; + +import { SourcererEmbed, SourcererOutput, makeSourcerer } from '@/providers/base'; +import { MovieScrapeContext, ShowScrapeContext } from '@/utils/context'; + +import { decryptSourceUrl } from './common'; +import { SourceResult, SourcesResult } from './types'; + +const vidSrcToBase = 'https://vidsrc.to'; +const referer = `${vidSrcToBase}/`; + +const universalScraper = async (ctx: ShowScrapeContext | MovieScrapeContext): Promise => { + const imdbId = ctx.media.imdbId; + const url = + ctx.media.type === 'movie' + ? `/embed/movie/${imdbId}` + : `/embed/tv/${imdbId}/${ctx.media.season.number}/${ctx.media.episode.number}`; + const mainPage = await ctx.proxiedFetcher(url, { + baseUrl: vidSrcToBase, + headers: { + referer, + }, + }); + const mainPage$ = load(mainPage); + const dataId = mainPage$('a[data-id]').attr('data-id'); + if (!dataId) throw new Error('No data-id found'); + const sources = await ctx.proxiedFetcher(`/ajax/embed/episode/${dataId}/sources`, { + baseUrl: vidSrcToBase, + headers: { + referer, + }, + }); + if (sources.status !== 200) throw new Error('No sources found'); + + const embeds: SourcererEmbed[] = []; + const embedUrls = []; + for (const source of sources.result) { + const sourceRes = await ctx.proxiedFetcher(`/ajax/embed/source/${source.id}`, { + baseUrl: vidSrcToBase, + headers: { + referer, + }, + }); + const decryptedUrl = decryptSourceUrl(sourceRes.result.url); + embedUrls.push(decryptedUrl); + } + + // Originally Filemoon does not have subtitles. But we can use the ones from Vidplay. + const subtitleUrl = new URL(embedUrls.find((v) => v.includes('sub.info')) ?? '').searchParams.get('sub.info'); + for (const source of sources.result) { + if (source.title === 'Vidplay') { + const embedUrl = embedUrls.find((v) => v.includes('vidplay')); + if (!embedUrl) continue; + embeds.push({ + embedId: 'vidplay', + url: embedUrl, + }); + } + + if (source.title === 'Filemoon') { + const embedUrl = embedUrls.find((v) => v.includes('filemoon')); + if (!embedUrl) continue; + const fullUrl = new URL(embedUrl); + if (subtitleUrl) fullUrl.searchParams.set('sub.info', subtitleUrl); + embeds.push({ + embedId: 'filemoon', + url: fullUrl.toString(), + }); + } + } + + return { + embeds, + }; +}; + +export const vidSrcToScraper = makeSourcerer({ + id: 'vidsrcto', + name: 'VidSrcTo', + scrapeMovie: universalScraper, + scrapeShow: universalScraper, + flags: [], + rank: 400, +}); diff --git a/src/providers/sources/vidsrcto/types.ts b/src/providers/sources/vidsrcto/types.ts new file mode 100644 index 0000000..0694b15 --- /dev/null +++ b/src/providers/sources/vidsrcto/types.ts @@ -0,0 +1,15 @@ +export type VidSrcToResponse = { + status: number; + result: T; +}; + +export type SourcesResult = VidSrcToResponse< + { + id: string; + title: 'Filemoon' | 'Vidplay'; + }[] +>; + +export type SourceResult = VidSrcToResponse<{ + url: string; +}>;