diff --git a/.docs/content/2.essentials/0.usage-on-x.md b/.docs/content/2.essentials/0.usage-on-x.md index 0c7443d..da53dc1 100644 --- a/.docs/content/2.essentials/0.usage-on-x.md +++ b/.docs/content/2.essentials/0.usage-on-x.md @@ -39,6 +39,23 @@ const providers = makeProviders({ ``` ## React native +To use the library in a react native app, you would also need a couple of polyfills to polyfill crypto and base64. + +1. First install the polyfills: +```bash +npm install @react-native-anywhere/polyfill-base64 react-native-quick-crypto +``` + +2. Add the polyfills to your app: +```ts +// Import in your entry file +import '@react-native-anywhere/polyfill-base64'; +``` + +And follow the [react-native-quick-crypto documentation](https://github.com/margelo/react-native-quick-crypto) to set up the crypto polyfill. + +3. Then you can use the library like this: + ```ts import { makeProviders, makeStandardFetcher, targets } from '@movie-web/providers'; diff --git a/src/providers/all.ts b/src/providers/all.ts index c93a669..7725558 100644 --- a/src/providers/all.ts +++ b/src/providers/all.ts @@ -27,6 +27,7 @@ import { vidCloudScraper } from './embeds/vidcloud'; import { vidplayScraper } from './embeds/vidplay'; import { wootlyScraper } from './embeds/wootly'; import { goojaraScraper } from './sources/goojara'; +import { hdRezkaScraper } from './sources/hdrezka'; import { nepuScraper } from './sources/nepu'; import { ridooMoviesScraper } from './sources/ridomovies'; import { smashyStreamScraper } from './sources/smashystream'; @@ -48,6 +49,7 @@ export function gatherAllSources(): Array { vidSrcToScraper, nepuScraper, goojaraScraper, + hdRezkaScraper, ]; } diff --git a/src/providers/embeds/dood.ts b/src/providers/embeds/dood.ts index 4eff019..3f0a371 100644 --- a/src/providers/embeds/dood.ts +++ b/src/providers/embeds/dood.ts @@ -30,6 +30,8 @@ export const doodScraper = makeEmbed({ }); const downloadURL = `${doodPage}${nanoid()}?token=${dataForLater}&expiry=${Date.now()}`; + if (!downloadURL.startsWith('http')) throw new Error('Invalid URL'); + return { stream: [ { diff --git a/src/providers/embeds/filemoon/index.ts b/src/providers/embeds/filemoon/index.ts index 9f96a07..3f8a2f3 100644 --- a/src/providers/embeds/filemoon/index.ts +++ b/src/providers/embeds/filemoon/index.ts @@ -1,5 +1,7 @@ import { unpack } from 'unpacker'; +import { flags } from '@/entrypoint/utils/targets'; + import { SubtitleResult } from './types'; import { makeEmbed } from '../../base'; import { Caption, getCaptionTypeFromUrl, labelToLanguageCode } from '../../captions'; @@ -49,7 +51,7 @@ export const fileMoonScraper = makeEmbed({ id: 'primary', type: 'hls', playlist: file[1], - flags: [], + flags: [flags.CORS_ALLOWED], captions, }, ], diff --git a/src/providers/embeds/vidplay/index.ts b/src/providers/embeds/vidplay/index.ts index 48af6c1..045e19e 100644 --- a/src/providers/embeds/vidplay/index.ts +++ b/src/providers/embeds/vidplay/index.ts @@ -1,7 +1,8 @@ +import { flags } from '@/entrypoint/utils/targets'; import { makeEmbed } from '@/providers/base'; import { Caption, getCaptionTypeFromUrl, labelToLanguageCode } from '@/providers/captions'; -import { getFileUrl, referer } from './common'; +import { getFileUrl } from './common'; import { SubtitleResult, VidplaySourceResponse } from './types'; export const vidplayScraper = makeEmbed({ @@ -44,12 +45,8 @@ export const vidplayScraper = makeEmbed({ id: 'primary', type: 'hls', playlist: source, - flags: [], + flags: [flags.CORS_ALLOWED], captions, - preferredHeaders: { - Referer: referer, - Origin: referer, - }, }, ], }; diff --git a/src/providers/sources/hdrezka/index.ts b/src/providers/sources/hdrezka/index.ts new file mode 100644 index 0000000..f3de725 --- /dev/null +++ b/src/providers/sources/hdrezka/index.ts @@ -0,0 +1,127 @@ +import { flags } from '@/entrypoint/utils/targets'; +import { SourcererOutput, makeSourcerer } from '@/providers/base'; +import { MovieScrapeContext, ShowScrapeContext } from '@/utils/context'; +import { NotFoundError } from '@/utils/errors'; + +import { MovieData, VideoLinks } from './types'; +import { extractTitleAndYear, generateRandomFavs, parseSubtitleLinks, parseVideoLinks } from './utils'; + +const rezkaBase = 'https://hdrzk.org'; +const baseHeaders = { + 'X-Hdrezka-Android-App': '1', + 'X-Hdrezka-Android-App-Version': '2.2.0', +}; + +async function searchAndFindMediaId(ctx: ShowScrapeContext | MovieScrapeContext): Promise { + const itemRegexPattern = /([^<]+)<\/span> \(([^)]+)\)/g; + const idRegexPattern = /\/(\d+)-[^/]+\.html$/; + + const searchData = await ctx.proxiedFetcher(`/engine/ajax/search.php`, { + baseUrl: rezkaBase, + headers: baseHeaders, + query: { q: ctx.media.title }, + }); + + const movieData: MovieData[] = []; + + for (const match of searchData.matchAll(itemRegexPattern)) { + const url = match[1]; + const titleAndYear = match[3]; + + const result = extractTitleAndYear(titleAndYear); + if (result !== null) { + const id = url.match(idRegexPattern)?.[1] || null; + + movieData.push({ id: id ?? '', year: result.year ?? 0, type: ctx.media.type, url }); + } + } + + const filteredItems = movieData.filter((item) => item.type === ctx.media.type && item.year === ctx.media.releaseYear); + + return filteredItems[0] || null; +} + +async function getStream( + id: string, + translatorId: string, + ctx: ShowScrapeContext | MovieScrapeContext, +): Promise { + const searchParams = new URLSearchParams(); + searchParams.append('id', id); + searchParams.append('translator_id', translatorId); + if (ctx.media.type === 'show') { + searchParams.append('season', ctx.media.season.number.toString()); + searchParams.append('episode', ctx.media.episode.number.toString()); + } + if (ctx.media.type === 'movie') { + searchParams.append('is_camprip', '0'); + searchParams.append('is_ads', '0'); + searchParams.append('is_director', '0'); + } + searchParams.append('favs', generateRandomFavs()); + searchParams.append('action', ctx.media.type === 'show' ? 'get_stream' : 'get_movie'); + + const response = await ctx.proxiedFetcher('/ajax/get_cdn_series/', { + baseUrl: rezkaBase, + method: 'POST', + body: searchParams, + headers: baseHeaders, + }); + + // Response content-type is text/html, but it's actually JSON + return JSON.parse(response); +} + +async function getTranslatorId( + url: string, + id: string, + ctx: ShowScrapeContext | MovieScrapeContext, +): Promise { + const response = await ctx.proxiedFetcher(url, { + headers: baseHeaders, + }); + + // Translator ID 238 represents the Original + subtitles player. + if (response.includes(`data-translator_id="238"`)) return '238'; + + const functionName = ctx.media.type === 'movie' ? 'initCDNMoviesEvents' : 'initCDNSeriesEvents'; + const regexPattern = new RegExp(`sof\\.tv\\.${functionName}\\(${id}, ([^,]+)`, 'i'); + const match = response.match(regexPattern); + const translatorId = match ? match[1] : null; + + return translatorId; +} + +const universalScraper = async (ctx: ShowScrapeContext | MovieScrapeContext): Promise => { + const result = await searchAndFindMediaId(ctx); + if (!result || !result.id) throw new NotFoundError('No result found'); + + const translatorId = await getTranslatorId(result.url, result.id, ctx); + if (!translatorId) throw new NotFoundError('No translator id found'); + + const { url: streamUrl, subtitle: streamSubtitle } = await getStream(result.id, translatorId, ctx); + const parsedVideos = parseVideoLinks(streamUrl); + const parsedSubtitles = parseSubtitleLinks(streamSubtitle); + + return { + embeds: [], + stream: [ + { + id: 'primary', + type: 'file', + flags: [flags.CORS_ALLOWED, flags.IP_LOCKED], + captions: parsedSubtitles, + qualities: parsedVideos, + }, + ], + }; +}; + +export const hdRezkaScraper = makeSourcerer({ + id: 'hdrezka', + name: 'HDRezka', + rank: 195, + flags: [flags.CORS_ALLOWED, flags.IP_LOCKED], + scrapeShow: universalScraper, + scrapeMovie: universalScraper, +}); diff --git a/src/providers/sources/hdrezka/types.ts b/src/providers/sources/hdrezka/types.ts new file mode 100644 index 0000000..d7ccdc2 --- /dev/null +++ b/src/providers/sources/hdrezka/types.ts @@ -0,0 +1,20 @@ +import { ScrapeMedia } from '@/index'; + +export type VideoLinks = { + success: boolean; + message: string; + premium_content: number; + url: string; + quality: string; + subtitle: boolean | string; + subtitle_lns: boolean; + subtitle_def: boolean; + thumbnails: string; +}; + +export interface MovieData { + id: string | null; + year: number; + type: ScrapeMedia['type']; + url: string; +} diff --git a/src/providers/sources/hdrezka/utils.ts b/src/providers/sources/hdrezka/utils.ts new file mode 100644 index 0000000..f8c7589 --- /dev/null +++ b/src/providers/sources/hdrezka/utils.ts @@ -0,0 +1,76 @@ +import { getCaptionTypeFromUrl, labelToLanguageCode } from '@/providers/captions'; +import { FileBasedStream } from '@/providers/streams'; +import { NotFoundError } from '@/utils/errors'; +import { getValidQualityFromString } from '@/utils/quality'; + +function generateRandomFavs(): string { + const randomHex = () => Math.floor(Math.random() * 16).toString(16); + const generateSegment = (length: number) => Array.from({ length }, randomHex).join(''); + + return `${generateSegment(8)}-${generateSegment(4)}-${generateSegment(4)}-${generateSegment(4)}-${generateSegment( + 12, + )}`; +} + +function parseSubtitleLinks(inputString?: string | boolean): FileBasedStream['captions'] { + if (!inputString || typeof inputString === 'boolean') return []; + const linksArray = inputString.split(','); + const captions: FileBasedStream['captions'] = []; + + linksArray.forEach((link) => { + const match = link.match(/\[([^\]]+)\](https?:\/\/\S+?)(?=,\[|$)/); + + if (match) { + const type = getCaptionTypeFromUrl(match[2]); + const language = labelToLanguageCode(match[1]); + if (!type || !language) return; + + captions.push({ + id: match[2], + language, + hasCorsRestrictions: false, + type, + url: match[2], + }); + } + }); + + return captions; +} + +function parseVideoLinks(inputString?: string): FileBasedStream['qualities'] { + if (!inputString) throw new NotFoundError('No video links found'); + const linksArray = inputString.split(','); + const result: FileBasedStream['qualities'] = {}; + + linksArray.forEach((link) => { + const match = link.match(/\[([^]+)](https?:\/\/[^\s,]+\.mp4)/); + if (match) { + const qualityText = match[1]; + const mp4Url = match[2]; + + const numericQualityMatch = qualityText.match(/(\d+p)/); + const quality = numericQualityMatch ? numericQualityMatch[1] : 'Unknown'; + + console.log(quality, mp4Url); + const validQuality = getValidQualityFromString(quality); + result[validQuality] = { type: 'mp4', url: mp4Url }; + } + }); + + return result; +} + +function extractTitleAndYear(input: string) { + const regex = /^(.*?),.*?(\d{4})/; + const match = input.match(regex); + + if (match) { + const title = match[1]; + const year = match[2]; + return { title: title.trim(), year: year ? parseInt(year, 10) : null }; + } + return null; +} + +export { extractTitleAndYear, parseSubtitleLinks, parseVideoLinks, generateRandomFavs }; diff --git a/src/providers/sources/vidsrcto/index.ts b/src/providers/sources/vidsrcto/index.ts index 94edc3d..c6c9fb5 100644 --- a/src/providers/sources/vidsrcto/index.ts +++ b/src/providers/sources/vidsrcto/index.ts @@ -1,5 +1,6 @@ import { load } from 'cheerio'; +import { flags } from '@/entrypoint/utils/targets'; import { SourcererEmbed, SourcererOutput, makeSourcerer } from '@/providers/base'; import { MovieScrapeContext, ShowScrapeContext } from '@/utils/context'; @@ -33,7 +34,7 @@ const universalScraper = async (ctx: ShowScrapeContext | MovieScrapeContext): Pr if (sources.status !== 200) throw new Error('No sources found'); const embeds: SourcererEmbed[] = []; - const embedUrls = []; + const embedArr = []; for (const source of sources.result) { const sourceRes = await ctx.proxiedFetcher(`/ajax/embed/source/${source.id}`, { baseUrl: vidSrcToBase, @@ -42,28 +43,23 @@ const universalScraper = async (ctx: ShowScrapeContext | MovieScrapeContext): Pr }, }); const decryptedUrl = decryptSourceUrl(sourceRes.result.url); - embedUrls.push(decryptedUrl); + embedArr.push({ source: source.title, url: decryptedUrl }); } - // Originally Filemoon does not have subtitles. But we can use the ones from Vidplay. - const urlWithSubtitles = embedUrls.find((v) => v.includes('sub.info')); - let subtitleUrl: string | null = null; - if (urlWithSubtitles) subtitleUrl = new URL(urlWithSubtitles).searchParams.get('sub.info'); - - for (const source of sources.result) { - if (source.title === 'Vidplay') { - const embedUrl = embedUrls.find((v) => v.includes('vidplay')); - if (!embedUrl) continue; + for (const embedObj of embedArr) { + if (embedObj.source === 'Vidplay') { + const fullUrl = new URL(embedObj.url); embeds.push({ embedId: 'vidplay', - url: embedUrl, + url: fullUrl.toString(), }); } - if (source.title === 'Filemoon') { - const embedUrl = embedUrls.find((v) => v.includes('filemoon')); - if (!embedUrl) continue; - const fullUrl = new URL(embedUrl); + if (embedObj.source === 'Filemoon') { + const fullUrl = new URL(embedObj.url); + // Originally Filemoon does not have subtitles. But we can use the ones from Vidplay. + const urlWithSubtitles = embedArr.find((v) => v.source === 'Vidplay' && v.url.includes('sub.info'))?.url; + const subtitleUrl = urlWithSubtitles ? new URL(urlWithSubtitles).searchParams.get('sub.info') : null; if (subtitleUrl) fullUrl.searchParams.set('sub.info', subtitleUrl); embeds.push({ embedId: 'filemoon', @@ -82,6 +78,6 @@ export const vidSrcToScraper = makeSourcerer({ name: 'VidSrcTo', scrapeMovie: universalScraper, scrapeShow: universalScraper, - flags: [], + flags: [flags.CORS_ALLOWED], rank: 300, }); diff --git a/src/runners/runner.ts b/src/runners/runner.ts index e3c0976..c351174 100644 --- a/src/runners/runner.ts +++ b/src/runners/runner.ts @@ -1,4 +1,4 @@ -import { FullScraperEvents } from '@/entrypoint/utils/events'; +import { FullScraperEvents, UpdateEvent } from '@/entrypoint/utils/events'; import { ScrapeMedia } from '@/entrypoint/utils/media'; import { FeatureMap, flagsAllowedInFeatures } from '@/entrypoint/utils/targets'; import { UseableFetcher } from '@/fetchers/types'; @@ -38,13 +38,13 @@ export type ProviderRunnerOptions = { }; export async function runAllProviders(list: ProviderList, ops: ProviderRunnerOptions): Promise { - const sources = reorderOnIdList(ops.sourceOrder ?? [], list.sources).filter((v) => { - if (ops.media.type === 'movie') return !!v.scrapeMovie; - if (ops.media.type === 'show') return !!v.scrapeShow; + const sources = reorderOnIdList(ops.sourceOrder ?? [], list.sources).filter((source) => { + if (ops.media.type === 'movie') return !!source.scrapeMovie; + if (ops.media.type === 'show') return !!source.scrapeShow; return false; }); const embeds = reorderOnIdList(ops.embedOrder ?? [], list.embeds); - const embedIds = embeds.map((v) => v.id); + const embedIds = embeds.map((embed) => embed.id); let lastId = ''; const contextBase: ScrapeContext = { @@ -63,47 +63,41 @@ export async function runAllProviders(list: ProviderList, ops: ProviderRunnerOpt sourceIds: sources.map((v) => v.id), }); - for (const s of sources) { - ops.events?.start?.(s.id); - lastId = s.id; + for (const source of sources) { + ops.events?.start?.(source.id); + lastId = source.id; // run source scrapers let output: SourcererOutput | null = null; try { - if (ops.media.type === 'movie' && s.scrapeMovie) - output = await s.scrapeMovie({ + if (ops.media.type === 'movie' && source.scrapeMovie) + output = await source.scrapeMovie({ ...contextBase, media: ops.media, }); - else if (ops.media.type === 'show' && s.scrapeShow) - output = await s.scrapeShow({ + else if (ops.media.type === 'show' && source.scrapeShow) + output = await source.scrapeShow({ ...contextBase, media: ops.media, }); if (output) { output.stream = (output.stream ?? []) - .filter((stream) => isValidStream(stream)) + .filter(isValidStream) .filter((stream) => flagsAllowedInFeatures(ops.features, stream.flags)); } - if (!output) throw Error('No output'); - if ((!output.stream || output.stream.length === 0) && output.embeds.length === 0) + if (!output || (!output.stream?.length && !output.embeds.length)) { throw new NotFoundError('No streams found'); - } catch (err) { - if (err instanceof NotFoundError) { - ops.events?.update?.({ - id: s.id, - percentage: 100, - status: 'notfound', - reason: err.message, - }); - continue; } - ops.events?.update?.({ - id: s.id, + } catch (error) { + const updateParams: UpdateEvent = { + id: source.id, percentage: 100, - status: 'failure', - error: err, - }); + status: error instanceof NotFoundError ? 'notfound' : 'failure', + reason: error instanceof NotFoundError ? error.message : undefined, + error: error instanceof NotFoundError ? undefined : error, + }; + + ops.events?.update?.(updateParams); continue; } if (!output) throw new Error('Invalid media type'); @@ -111,7 +105,7 @@ export async function runAllProviders(list: ProviderList, ops: ProviderRunnerOpt // return stream is there are any if (output.stream?.[0]) { return { - sourceId: s.id, + sourceId: source.id, stream: output.stream[0], }; } @@ -120,62 +114,56 @@ export async function runAllProviders(list: ProviderList, ops: ProviderRunnerOpt const sortedEmbeds = output.embeds .filter((embed) => { const e = list.embeds.find((v) => v.id === embed.embedId); - if (!e || e.disabled) return false; - return true; + return e && !e.disabled; }) .sort((a, b) => embedIds.indexOf(a.embedId) - embedIds.indexOf(b.embedId)); if (sortedEmbeds.length > 0) { ops.events?.discoverEmbeds?.({ - embeds: sortedEmbeds.map((v, i) => ({ - id: [s.id, i].join('-'), - embedScraperId: v.embedId, + embeds: sortedEmbeds.map((embed, i) => ({ + id: [source.id, i].join('-'), + embedScraperId: embed.embedId, })), - sourceId: s.id, + sourceId: source.id, }); } - for (const ind in sortedEmbeds) { - if (!Object.prototype.hasOwnProperty.call(sortedEmbeds, ind)) continue; - const e = sortedEmbeds[ind]; - const scraper = embeds.find((v) => v.id === e.embedId); + for (const [ind, embed] of sortedEmbeds.entries()) { + const scraper = embeds.find((v) => v.id === embed.embedId); if (!scraper) throw new Error('Invalid embed returned'); // run embed scraper - const id = [s.id, ind].join('-'); + const id = [source.id, ind].join('-'); ops.events?.start?.(id); lastId = id; + let embedOutput: EmbedOutput; try { embedOutput = await scraper.scrape({ ...contextBase, - url: e.url, + url: embed.url, }); embedOutput.stream = embedOutput.stream - .filter((stream) => isValidStream(stream)) + .filter(isValidStream) .filter((stream) => flagsAllowedInFeatures(ops.features, stream.flags)); - if (embedOutput.stream.length === 0) throw new NotFoundError('No streams found'); - } catch (err) { - if (err instanceof NotFoundError) { - ops.events?.update?.({ - id, - percentage: 100, - status: 'notfound', - reason: err.message, - }); - continue; + if (embedOutput.stream.length === 0) { + throw new NotFoundError('No streams found'); } - ops.events?.update?.({ - id, + } catch (error) { + const updateParams: UpdateEvent = { + id: source.id, percentage: 100, - status: 'failure', - error: err, - }); + status: error instanceof NotFoundError ? 'notfound' : 'failure', + reason: error instanceof NotFoundError ? error.message : undefined, + error: error instanceof NotFoundError ? undefined : error, + }; + + ops.events?.update?.(updateParams); continue; } return { - sourceId: s.id, + sourceId: source.id, embedId: scraper.id, stream: embedOutput.stream[0], }; diff --git a/src/utils/quality.ts b/src/utils/quality.ts new file mode 100644 index 0000000..8854ca5 --- /dev/null +++ b/src/utils/quality.ts @@ -0,0 +1,20 @@ +import { Qualities } from '@/providers/streams'; + +export function getValidQualityFromString(quality: string): Qualities { + switch (quality.toLowerCase().replace('p', '')) { + case '360': + return '360'; + case '480': + return '480'; + case '720': + return '720'; + case '1080': + return '1080'; + case '2160': + return '4k'; + case '4k': + return '4k'; + default: + return 'unknown'; + } +}