From da7c83774cd2cc785e44773952b172ee4c8147b9 Mon Sep 17 00:00:00 2001 From: Vijay <74645268+vijaysingh2219@users.noreply.github.com> Date: Wed, 6 Mar 2024 15:40:53 +0530 Subject: [PATCH 01/15] Refactor for readability and simplicity. --- src/runners/runner.ts | 95 +++++++++++++++++++++---------------------- 1 file changed, 47 insertions(+), 48 deletions(-) diff --git a/src/runners/runner.ts b/src/runners/runner.ts index e3c0976..eb5b57f 100644 --- a/src/runners/runner.ts +++ b/src/runners/runner.ts @@ -1,4 +1,4 @@ -import { FullScraperEvents } from '@/entrypoint/utils/events'; +import { FullScraperEvents, UpdateEvent } from '@/entrypoint/utils/events'; import { ScrapeMedia } from '@/entrypoint/utils/media'; import { FeatureMap, flagsAllowedInFeatures } from '@/entrypoint/utils/targets'; import { UseableFetcher } from '@/fetchers/types'; @@ -38,13 +38,13 @@ export type ProviderRunnerOptions = { }; export async function runAllProviders(list: ProviderList, ops: ProviderRunnerOptions): Promise { - const sources = reorderOnIdList(ops.sourceOrder ?? [], list.sources).filter((v) => { - if (ops.media.type === 'movie') return !!v.scrapeMovie; - if (ops.media.type === 'show') return !!v.scrapeShow; + const sources = reorderOnIdList(ops.sourceOrder ?? [], list.sources).filter((source) => { + if (ops.media.type === 'movie') return !!source.scrapeMovie; + if (ops.media.type === 'show') return !!source.scrapeShow; return false; }); const embeds = reorderOnIdList(ops.embedOrder ?? [], list.embeds); - const embedIds = embeds.map((v) => v.id); + const embedIds = embeds.map((embed) => embed.id); let lastId = ''; const contextBase: ScrapeContext = { @@ -63,35 +63,35 @@ export async function runAllProviders(list: ProviderList, ops: ProviderRunnerOpt sourceIds: sources.map((v) => v.id), }); - for (const s of sources) { - ops.events?.start?.(s.id); - lastId = s.id; + for (const source of sources) { + ops.events?.start?.(source.id); + lastId = source.id; // run source scrapers let output: SourcererOutput | null = null; try { - if (ops.media.type === 'movie' && s.scrapeMovie) - output = await s.scrapeMovie({ + if (ops.media.type === 'movie' && source.scrapeMovie) + output = await source.scrapeMovie({ ...contextBase, media: ops.media, }); - else if (ops.media.type === 'show' && s.scrapeShow) - output = await s.scrapeShow({ + else if (ops.media.type === 'show' && source.scrapeShow) + output = await source.scrapeShow({ ...contextBase, media: ops.media, }); if (output) { output.stream = (output.stream ?? []) - .filter((stream) => isValidStream(stream)) + .filter(isValidStream) .filter((stream) => flagsAllowedInFeatures(ops.features, stream.flags)); } - if (!output) throw Error('No output'); - if ((!output.stream || output.stream.length === 0) && output.embeds.length === 0) + if (!output || (!output.stream?.length && !output.embeds.length)) { throw new NotFoundError('No streams found'); + } } catch (err) { if (err instanceof NotFoundError) { ops.events?.update?.({ - id: s.id, + id: source.id, percentage: 100, status: 'notfound', reason: err.message, @@ -99,7 +99,7 @@ export async function runAllProviders(list: ProviderList, ops: ProviderRunnerOpt continue; } ops.events?.update?.({ - id: s.id, + id: source.id, percentage: 100, status: 'failure', error: err, @@ -111,7 +111,7 @@ export async function runAllProviders(list: ProviderList, ops: ProviderRunnerOpt // return stream is there are any if (output.stream?.[0]) { return { - sourceId: s.id, + sourceId: source.id, stream: output.stream[0], }; } @@ -120,62 +120,61 @@ export async function runAllProviders(list: ProviderList, ops: ProviderRunnerOpt const sortedEmbeds = output.embeds .filter((embed) => { const e = list.embeds.find((v) => v.id === embed.embedId); - if (!e || e.disabled) return false; - return true; + return e && !e.disabled; }) .sort((a, b) => embedIds.indexOf(a.embedId) - embedIds.indexOf(b.embedId)); if (sortedEmbeds.length > 0) { ops.events?.discoverEmbeds?.({ - embeds: sortedEmbeds.map((v, i) => ({ - id: [s.id, i].join('-'), - embedScraperId: v.embedId, + embeds: sortedEmbeds.map((embed, i) => ({ + id: [source.id, i].join('-'), + embedScraperId: embed.embedId, })), - sourceId: s.id, + sourceId: source.id, }); } - for (const ind in sortedEmbeds) { - if (!Object.prototype.hasOwnProperty.call(sortedEmbeds, ind)) continue; - const e = sortedEmbeds[ind]; - const scraper = embeds.find((v) => v.id === e.embedId); + for (const [ind, embed] of sortedEmbeds.entries()) { + const scraper = embeds.find((v) => v.id === embed.embedId); if (!scraper) throw new Error('Invalid embed returned'); // run embed scraper - const id = [s.id, ind].join('-'); + const id = [source.id, ind].join('-'); ops.events?.start?.(id); lastId = id; + let embedOutput: EmbedOutput; try { embedOutput = await scraper.scrape({ ...contextBase, - url: e.url, + url: embed.url, }); embedOutput.stream = embedOutput.stream - .filter((stream) => isValidStream(stream)) + .filter(isValidStream) .filter((stream) => flagsAllowedInFeatures(ops.features, stream.flags)); - if (embedOutput.stream.length === 0) throw new NotFoundError('No streams found'); - } catch (err) { - if (err instanceof NotFoundError) { - ops.events?.update?.({ - id, - percentage: 100, - status: 'notfound', - reason: err.message, - }); + if (embedOutput.stream.length === 0) { + throw new NotFoundError('No streams found'); + } + } catch (error) { + const updateParams: UpdateEvent = { + id: source.id, + percentage: 100, + status: error instanceof NotFoundError ? 'notfound' : 'failure', + reason: error instanceof NotFoundError ? error.message : undefined, + error, + }; + + ops.events?.update?.(updateParams); + + if (error instanceof NotFoundError) { continue; } - ops.events?.update?.({ - id, - percentage: 100, - status: 'failure', - error: err, - }); - continue; + + throw error; } return { - sourceId: s.id, + sourceId: source.id, embedId: scraper.id, stream: embedOutput.stream[0], }; From 284226166ecf6415908f6b80b1b999ba9c1f01ce Mon Sep 17 00:00:00 2001 From: Vijay <74645268+vijaysingh2219@users.noreply.github.com> Date: Thu, 7 Mar 2024 21:32:46 +0530 Subject: [PATCH 02/15] Modified the try-catch block --- src/runners/runner.ts | 31 ++++++++++--------------------- 1 file changed, 10 insertions(+), 21 deletions(-) diff --git a/src/runners/runner.ts b/src/runners/runner.ts index eb5b57f..c351174 100644 --- a/src/runners/runner.ts +++ b/src/runners/runner.ts @@ -88,22 +88,16 @@ export async function runAllProviders(list: ProviderList, ops: ProviderRunnerOpt if (!output || (!output.stream?.length && !output.embeds.length)) { throw new NotFoundError('No streams found'); } - } catch (err) { - if (err instanceof NotFoundError) { - ops.events?.update?.({ - id: source.id, - percentage: 100, - status: 'notfound', - reason: err.message, - }); - continue; - } - ops.events?.update?.({ + } catch (error) { + const updateParams: UpdateEvent = { id: source.id, percentage: 100, - status: 'failure', - error: err, - }); + status: error instanceof NotFoundError ? 'notfound' : 'failure', + reason: error instanceof NotFoundError ? error.message : undefined, + error: error instanceof NotFoundError ? undefined : error, + }; + + ops.events?.update?.(updateParams); continue; } if (!output) throw new Error('Invalid media type'); @@ -161,16 +155,11 @@ export async function runAllProviders(list: ProviderList, ops: ProviderRunnerOpt percentage: 100, status: error instanceof NotFoundError ? 'notfound' : 'failure', reason: error instanceof NotFoundError ? error.message : undefined, - error, + error: error instanceof NotFoundError ? undefined : error, }; ops.events?.update?.(updateParams); - - if (error instanceof NotFoundError) { - continue; - } - - throw error; + continue; } return { From 853da85d4176b83737e2bfd2a8c2fd9c90faa8a2 Mon Sep 17 00:00:00 2001 From: Jorrin Date: Thu, 7 Mar 2024 23:40:40 +0100 Subject: [PATCH 03/15] Add setup steps to React native --- .docs/content/2.essentials/0.usage-on-x.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.docs/content/2.essentials/0.usage-on-x.md b/.docs/content/2.essentials/0.usage-on-x.md index 0c7443d..da53dc1 100644 --- a/.docs/content/2.essentials/0.usage-on-x.md +++ b/.docs/content/2.essentials/0.usage-on-x.md @@ -39,6 +39,23 @@ const providers = makeProviders({ ``` ## React native +To use the library in a react native app, you would also need a couple of polyfills to polyfill crypto and base64. + +1. First install the polyfills: +```bash +npm install @react-native-anywhere/polyfill-base64 react-native-quick-crypto +``` + +2. Add the polyfills to your app: +```ts +// Import in your entry file +import '@react-native-anywhere/polyfill-base64'; +``` + +And follow the [react-native-quick-crypto documentation](https://github.com/margelo/react-native-quick-crypto) to set up the crypto polyfill. + +3. Then you can use the library like this: + ```ts import { makeProviders, makeStandardFetcher, targets } from '@movie-web/providers'; From 284ad4d2222091e1b451f3a94b8a756530903226 Mon Sep 17 00:00:00 2001 From: Jorrin Date: Mon, 11 Mar 2024 14:49:21 +0100 Subject: [PATCH 04/15] check if dood returns a valid url --- src/providers/embeds/dood.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/providers/embeds/dood.ts b/src/providers/embeds/dood.ts index 4eff019..3f0a371 100644 --- a/src/providers/embeds/dood.ts +++ b/src/providers/embeds/dood.ts @@ -30,6 +30,8 @@ export const doodScraper = makeEmbed({ }); const downloadURL = `${doodPage}${nanoid()}?token=${dataForLater}&expiry=${Date.now()}`; + if (!downloadURL.startsWith('http')) throw new Error('Invalid URL'); + return { stream: [ { From af9d9c17cd7cfdbecb0ddfdcb394b01781b3569c Mon Sep 17 00:00:00 2001 From: Jorrin Date: Wed, 13 Mar 2024 00:12:18 +0100 Subject: [PATCH 05/15] add hdrezka --- src/providers/all.ts | 2 + src/providers/sources/hdrezka/index.ts | 120 +++++++++++++++++++++++++ src/providers/sources/hdrezka/types.ts | 12 +++ src/providers/sources/hdrezka/utils.ts | 76 ++++++++++++++++ src/utils/quality.ts | 20 +++++ 5 files changed, 230 insertions(+) create mode 100644 src/providers/sources/hdrezka/index.ts create mode 100644 src/providers/sources/hdrezka/types.ts create mode 100644 src/providers/sources/hdrezka/utils.ts create mode 100644 src/utils/quality.ts diff --git a/src/providers/all.ts b/src/providers/all.ts index c93a669..7725558 100644 --- a/src/providers/all.ts +++ b/src/providers/all.ts @@ -27,6 +27,7 @@ import { vidCloudScraper } from './embeds/vidcloud'; import { vidplayScraper } from './embeds/vidplay'; import { wootlyScraper } from './embeds/wootly'; import { goojaraScraper } from './sources/goojara'; +import { hdRezkaScraper } from './sources/hdrezka'; import { nepuScraper } from './sources/nepu'; import { ridooMoviesScraper } from './sources/ridomovies'; import { smashyStreamScraper } from './sources/smashystream'; @@ -48,6 +49,7 @@ export function gatherAllSources(): Array { vidSrcToScraper, nepuScraper, goojaraScraper, + hdRezkaScraper, ]; } diff --git a/src/providers/sources/hdrezka/index.ts b/src/providers/sources/hdrezka/index.ts new file mode 100644 index 0000000..f93f285 --- /dev/null +++ b/src/providers/sources/hdrezka/index.ts @@ -0,0 +1,120 @@ +import { flags } from '@/entrypoint/utils/targets'; +import { SourcererOutput, makeSourcerer } from '@/providers/base'; +import { MovieScrapeContext, ScrapeContext, ShowScrapeContext } from '@/utils/context'; +import { NotFoundError } from '@/utils/errors'; + +import { VideoLinks } from './types'; +import { extractTitleAndYear, generateRandomFavs, parseSubtitleLinks, parseVideoLinks } from './utils'; + +const rezkaBase = 'https://hdrzk.org'; +const baseHeaders = { + 'X-Hdrezka-Android-App': '1', + 'X-Hdrezka-Android-App-Version': '2.2.0', +}; + +async function searchAndFindMediaId({ + type, + title, + releaseYear, + ctx, +}: { + type: 'show' | 'movie'; + title: string; + releaseYear: number; + ctx: ScrapeContext; +}): Promise { + const itemRegexPattern = /([^<]+)<\/span> \(([^)]+)\)/g; + const idRegexPattern = /\/(\d+)-[^/]+\.html$/; + + const searchData = await ctx.proxiedFetcher(`/engine/ajax/search.php`, { + baseUrl: rezkaBase, + headers: baseHeaders, + query: { q: title }, + }); + + const movieData: { + id: string | null; + year: number | null; + type: 'show' | 'movie'; + }[] = []; + + for (const match of searchData.matchAll(itemRegexPattern)) { + const url = match[1]; + const titleAndYear = match[3]; + + const result = extractTitleAndYear(titleAndYear); + if (result !== null) { + const id = url.match(idRegexPattern)?.[1] || null; + + movieData.push({ id, year: result.year, type }); + } + } + + const filteredItems = movieData.filter((item) => item.type === type && item.year === releaseYear); + + return filteredItems[0]?.id || null; +} + +async function getStream(id: string, ctx: ShowScrapeContext | MovieScrapeContext): Promise { + const searchParams = new URLSearchParams(); + searchParams.append('id', id); + // Translator ID 238 represents the Original + subtitles player. + searchParams.append('translator_id', '238'); + if (ctx.media.type === 'show') { + searchParams.append('season', ctx.media.season.number.toString()); + searchParams.append('episode', ctx.media.episode.number.toString()); + } + if (ctx.media.type === 'movie') { + searchParams.append('is_camprip', '0'); + searchParams.append('is_ads', '0'); + searchParams.append('is_director', '0'); + } + searchParams.append('favs', generateRandomFavs()); + searchParams.append('action', ctx.media.type === 'show' ? 'get_stream' : 'get_movie'); + + const response = await ctx.proxiedFetcher('/ajax/get_cdn_series/', { + baseUrl: rezkaBase, + method: 'POST', + body: searchParams, + headers: baseHeaders, + }); + + // Response content-type is text/html, but it's actually JSON + return JSON.parse(response); +} + +const universalScraper = async (ctx: ShowScrapeContext | MovieScrapeContext): Promise => { + const id = await searchAndFindMediaId({ + type: ctx.media.type === 'show' ? 'show' : 'movie', + title: ctx.media.title, + releaseYear: ctx.media.releaseYear, + ctx, + }); + if (!id) throw new NotFoundError('No result found'); + + const { url: streamUrl, subtitle: streamSubtitle } = await getStream(id, ctx); + const parsedVideos = parseVideoLinks(streamUrl); + const parsedSubtitles = parseSubtitleLinks(streamSubtitle); + + return { + embeds: [], + stream: [ + { + id: 'primary', + type: 'file', + flags: [flags.CORS_ALLOWED], + captions: parsedSubtitles, + qualities: parsedVideos, + }, + ], + }; +}; + +export const hdRezkaScraper = makeSourcerer({ + id: 'hdrezka', + name: 'HDRezka', + rank: 195, + flags: [flags.CORS_ALLOWED, flags.IP_LOCKED], + scrapeShow: universalScraper, + scrapeMovie: universalScraper, +}); diff --git a/src/providers/sources/hdrezka/types.ts b/src/providers/sources/hdrezka/types.ts new file mode 100644 index 0000000..a72d85d --- /dev/null +++ b/src/providers/sources/hdrezka/types.ts @@ -0,0 +1,12 @@ +export type VideoLinks = { + success: boolean; + message: string; + premium_content: number; + url: string; + parseVideoLinks: string; + quality: string; + subtitle: boolean | string; + subtitle_lns: boolean; + subtitle_def: boolean; + thumbnails: string; +}; diff --git a/src/providers/sources/hdrezka/utils.ts b/src/providers/sources/hdrezka/utils.ts new file mode 100644 index 0000000..f8c7589 --- /dev/null +++ b/src/providers/sources/hdrezka/utils.ts @@ -0,0 +1,76 @@ +import { getCaptionTypeFromUrl, labelToLanguageCode } from '@/providers/captions'; +import { FileBasedStream } from '@/providers/streams'; +import { NotFoundError } from '@/utils/errors'; +import { getValidQualityFromString } from '@/utils/quality'; + +function generateRandomFavs(): string { + const randomHex = () => Math.floor(Math.random() * 16).toString(16); + const generateSegment = (length: number) => Array.from({ length }, randomHex).join(''); + + return `${generateSegment(8)}-${generateSegment(4)}-${generateSegment(4)}-${generateSegment(4)}-${generateSegment( + 12, + )}`; +} + +function parseSubtitleLinks(inputString?: string | boolean): FileBasedStream['captions'] { + if (!inputString || typeof inputString === 'boolean') return []; + const linksArray = inputString.split(','); + const captions: FileBasedStream['captions'] = []; + + linksArray.forEach((link) => { + const match = link.match(/\[([^\]]+)\](https?:\/\/\S+?)(?=,\[|$)/); + + if (match) { + const type = getCaptionTypeFromUrl(match[2]); + const language = labelToLanguageCode(match[1]); + if (!type || !language) return; + + captions.push({ + id: match[2], + language, + hasCorsRestrictions: false, + type, + url: match[2], + }); + } + }); + + return captions; +} + +function parseVideoLinks(inputString?: string): FileBasedStream['qualities'] { + if (!inputString) throw new NotFoundError('No video links found'); + const linksArray = inputString.split(','); + const result: FileBasedStream['qualities'] = {}; + + linksArray.forEach((link) => { + const match = link.match(/\[([^]+)](https?:\/\/[^\s,]+\.mp4)/); + if (match) { + const qualityText = match[1]; + const mp4Url = match[2]; + + const numericQualityMatch = qualityText.match(/(\d+p)/); + const quality = numericQualityMatch ? numericQualityMatch[1] : 'Unknown'; + + console.log(quality, mp4Url); + const validQuality = getValidQualityFromString(quality); + result[validQuality] = { type: 'mp4', url: mp4Url }; + } + }); + + return result; +} + +function extractTitleAndYear(input: string) { + const regex = /^(.*?),.*?(\d{4})/; + const match = input.match(regex); + + if (match) { + const title = match[1]; + const year = match[2]; + return { title: title.trim(), year: year ? parseInt(year, 10) : null }; + } + return null; +} + +export { extractTitleAndYear, parseSubtitleLinks, parseVideoLinks, generateRandomFavs }; diff --git a/src/utils/quality.ts b/src/utils/quality.ts new file mode 100644 index 0000000..609f2e8 --- /dev/null +++ b/src/utils/quality.ts @@ -0,0 +1,20 @@ +import { Qualities } from '../../lib'; + +export function getValidQualityFromString(quality: string): Qualities { + switch (quality.toLowerCase().replace('p', '')) { + case '360': + return '360'; + case '480': + return '480'; + case '720': + return '720'; + case '1080': + return '1080'; + case '2160': + return '4k'; + case '4k': + return '4k'; + default: + return 'unknown'; + } +} From dd3942488c539bb246c237f81e4bf448b1534502 Mon Sep 17 00:00:00 2001 From: Jorrin Date: Wed, 13 Mar 2024 00:19:21 +0100 Subject: [PATCH 06/15] cleanup --- src/providers/sources/hdrezka/index.ts | 30 +++++++------------------- src/utils/quality.ts | 2 +- 2 files changed, 9 insertions(+), 23 deletions(-) diff --git a/src/providers/sources/hdrezka/index.ts b/src/providers/sources/hdrezka/index.ts index f93f285..353bafc 100644 --- a/src/providers/sources/hdrezka/index.ts +++ b/src/providers/sources/hdrezka/index.ts @@ -1,6 +1,7 @@ import { flags } from '@/entrypoint/utils/targets'; +import { ScrapeMedia } from '@/index'; import { SourcererOutput, makeSourcerer } from '@/providers/base'; -import { MovieScrapeContext, ScrapeContext, ShowScrapeContext } from '@/utils/context'; +import { MovieScrapeContext, ShowScrapeContext } from '@/utils/context'; import { NotFoundError } from '@/utils/errors'; import { VideoLinks } from './types'; @@ -12,30 +13,20 @@ const baseHeaders = { 'X-Hdrezka-Android-App-Version': '2.2.0', }; -async function searchAndFindMediaId({ - type, - title, - releaseYear, - ctx, -}: { - type: 'show' | 'movie'; - title: string; - releaseYear: number; - ctx: ScrapeContext; -}): Promise { +async function searchAndFindMediaId(ctx: ShowScrapeContext | MovieScrapeContext): Promise { const itemRegexPattern = /([^<]+)<\/span> \(([^)]+)\)/g; const idRegexPattern = /\/(\d+)-[^/]+\.html$/; const searchData = await ctx.proxiedFetcher(`/engine/ajax/search.php`, { baseUrl: rezkaBase, headers: baseHeaders, - query: { q: title }, + query: { q: ctx.media.title }, }); const movieData: { id: string | null; year: number | null; - type: 'show' | 'movie'; + type: ScrapeMedia['type']; }[] = []; for (const match of searchData.matchAll(itemRegexPattern)) { @@ -46,11 +37,11 @@ async function searchAndFindMediaId({ if (result !== null) { const id = url.match(idRegexPattern)?.[1] || null; - movieData.push({ id, year: result.year, type }); + movieData.push({ id, year: result.year, type: ctx.media.type }); } } - const filteredItems = movieData.filter((item) => item.type === type && item.year === releaseYear); + const filteredItems = movieData.filter((item) => item.type === ctx.media.type && item.year === ctx.media.releaseYear); return filteredItems[0]?.id || null; } @@ -84,12 +75,7 @@ async function getStream(id: string, ctx: ShowScrapeContext | MovieScrapeContext } const universalScraper = async (ctx: ShowScrapeContext | MovieScrapeContext): Promise => { - const id = await searchAndFindMediaId({ - type: ctx.media.type === 'show' ? 'show' : 'movie', - title: ctx.media.title, - releaseYear: ctx.media.releaseYear, - ctx, - }); + const id = await searchAndFindMediaId(ctx); if (!id) throw new NotFoundError('No result found'); const { url: streamUrl, subtitle: streamSubtitle } = await getStream(id, ctx); diff --git a/src/utils/quality.ts b/src/utils/quality.ts index 609f2e8..8854ca5 100644 --- a/src/utils/quality.ts +++ b/src/utils/quality.ts @@ -1,4 +1,4 @@ -import { Qualities } from '../../lib'; +import { Qualities } from '@/providers/streams'; export function getValidQualityFromString(quality: string): Qualities { switch (quality.toLowerCase().replace('p', '')) { From 845bf7c3dbb7b2a3088d0e1912ecca273a0e2f10 Mon Sep 17 00:00:00 2001 From: Jorrin Date: Wed, 13 Mar 2024 00:27:49 +0100 Subject: [PATCH 07/15] Update types.ts --- src/providers/sources/hdrezka/types.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/providers/sources/hdrezka/types.ts b/src/providers/sources/hdrezka/types.ts index a72d85d..b578a4c 100644 --- a/src/providers/sources/hdrezka/types.ts +++ b/src/providers/sources/hdrezka/types.ts @@ -3,7 +3,6 @@ export type VideoLinks = { message: string; premium_content: number; url: string; - parseVideoLinks: string; quality: string; subtitle: boolean | string; subtitle_lns: boolean; From e1a0ce72cc997ca0214356b524e5c27ad20c6311 Mon Sep 17 00:00:00 2001 From: Jorrin Date: Wed, 13 Mar 2024 00:30:02 +0100 Subject: [PATCH 08/15] add IP_LOCKED flag --- src/providers/sources/hdrezka/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/providers/sources/hdrezka/index.ts b/src/providers/sources/hdrezka/index.ts index 353bafc..92f64fe 100644 --- a/src/providers/sources/hdrezka/index.ts +++ b/src/providers/sources/hdrezka/index.ts @@ -88,7 +88,7 @@ const universalScraper = async (ctx: ShowScrapeContext | MovieScrapeContext): Pr { id: 'primary', type: 'file', - flags: [flags.CORS_ALLOWED], + flags: [flags.CORS_ALLOWED, flags.IP_LOCKED], captions: parsedSubtitles, qualities: parsedVideos, }, From 59e1700ea1688447165fc163cd349bd7ab117150 Mon Sep 17 00:00:00 2001 From: lonelil <51315646+lonelil@users.noreply.github.com> Date: Wed, 13 Mar 2024 08:17:36 +0800 Subject: [PATCH 09/15] rezka: check for translator --- src/providers/sources/hdrezka/index.ts | 55 ++++++++++++++++++-------- src/providers/sources/hdrezka/types.ts | 9 +++++ 2 files changed, 47 insertions(+), 17 deletions(-) diff --git a/src/providers/sources/hdrezka/index.ts b/src/providers/sources/hdrezka/index.ts index 92f64fe..c98869f 100644 --- a/src/providers/sources/hdrezka/index.ts +++ b/src/providers/sources/hdrezka/index.ts @@ -1,10 +1,9 @@ import { flags } from '@/entrypoint/utils/targets'; -import { ScrapeMedia } from '@/index'; import { SourcererOutput, makeSourcerer } from '@/providers/base'; import { MovieScrapeContext, ShowScrapeContext } from '@/utils/context'; import { NotFoundError } from '@/utils/errors'; -import { VideoLinks } from './types'; +import { MovieData, VideoLinks } from './types'; import { extractTitleAndYear, generateRandomFavs, parseSubtitleLinks, parseVideoLinks } from './utils'; const rezkaBase = 'https://hdrzk.org'; @@ -13,7 +12,7 @@ const baseHeaders = { 'X-Hdrezka-Android-App-Version': '2.2.0', }; -async function searchAndFindMediaId(ctx: ShowScrapeContext | MovieScrapeContext): Promise { +async function searchAndFindMediaId(ctx: ShowScrapeContext | MovieScrapeContext): Promise { const itemRegexPattern = /([^<]+)<\/span> \(([^)]+)\)/g; const idRegexPattern = /\/(\d+)-[^/]+\.html$/; @@ -23,11 +22,7 @@ async function searchAndFindMediaId(ctx: ShowScrapeContext | MovieScrapeContext) query: { q: ctx.media.title }, }); - const movieData: { - id: string | null; - year: number | null; - type: ScrapeMedia['type']; - }[] = []; + const movieData: MovieData[] = []; for (const match of searchData.matchAll(itemRegexPattern)) { const url = match[1]; @@ -37,20 +32,23 @@ async function searchAndFindMediaId(ctx: ShowScrapeContext | MovieScrapeContext) if (result !== null) { const id = url.match(idRegexPattern)?.[1] || null; - movieData.push({ id, year: result.year, type: ctx.media.type }); + movieData.push({ id: id ?? '', year: result.year ?? 0, type: ctx.media.type, url }); } } const filteredItems = movieData.filter((item) => item.type === ctx.media.type && item.year === ctx.media.releaseYear); - return filteredItems[0]?.id || null; + return filteredItems[0] || null; } -async function getStream(id: string, ctx: ShowScrapeContext | MovieScrapeContext): Promise { +async function getStream( + id: string, + translatorId: string, + ctx: ShowScrapeContext | MovieScrapeContext, +): Promise { const searchParams = new URLSearchParams(); searchParams.append('id', id); - // Translator ID 238 represents the Original + subtitles player. - searchParams.append('translator_id', '238'); + searchParams.append('translator_id', translatorId); if (ctx.media.type === 'show') { searchParams.append('season', ctx.media.season.number.toString()); searchParams.append('episode', ctx.media.episode.number.toString()); @@ -74,11 +72,34 @@ async function getStream(id: string, ctx: ShowScrapeContext | MovieScrapeContext return JSON.parse(response); } -const universalScraper = async (ctx: ShowScrapeContext | MovieScrapeContext): Promise => { - const id = await searchAndFindMediaId(ctx); - if (!id) throw new NotFoundError('No result found'); +async function getTranslator( + url: string, + id: string, + ctx: ShowScrapeContext | MovieScrapeContext, +): Promise { + const response = await ctx.proxiedFetcher(url, { + headers: baseHeaders, + }); - const { url: streamUrl, subtitle: streamSubtitle } = await getStream(id, ctx); + // Translator ID 238 represents the Original + subtitles player. + if (response.includes(`data-translator_id="238"`)) return '238'; + + const functionName = ctx.media.type === 'movie' ? 'initCDNMoviesEvents' : 'initCDNSeriesEvents'; + const regexPattern = new RegExp(`sof\\.tv\\.${functionName}\\(${id}, ([^,]+)`, 'i'); + const match = response.match(regexPattern); + const translatorId = match ? match[1] : null; + + return translatorId; +} + +const universalScraper = async (ctx: ShowScrapeContext | MovieScrapeContext): Promise => { + const result = await searchAndFindMediaId(ctx); + if (!result) throw new NotFoundError('No result found'); + + const translatorId = await getTranslator(result.url, result.id, ctx); + if (!translatorId) throw new NotFoundError('No translator id found'); + + const { url: streamUrl, subtitle: streamSubtitle } = await getStream(result.id, translatorId, ctx); const parsedVideos = parseVideoLinks(streamUrl); const parsedSubtitles = parseSubtitleLinks(streamSubtitle); diff --git a/src/providers/sources/hdrezka/types.ts b/src/providers/sources/hdrezka/types.ts index b578a4c..66b2b9c 100644 --- a/src/providers/sources/hdrezka/types.ts +++ b/src/providers/sources/hdrezka/types.ts @@ -1,3 +1,5 @@ +import { ScrapeMedia } from '@/index'; + export type VideoLinks = { success: boolean; message: string; @@ -9,3 +11,10 @@ export type VideoLinks = { subtitle_def: boolean; thumbnails: string; }; + +export interface MovieData { + id: string; + year: number; + type: ScrapeMedia['type']; + url: string; +} From afb261717d67b29120c329f48c8d070549040017 Mon Sep 17 00:00:00 2001 From: lonelil <51315646+lonelil@users.noreply.github.com> Date: Wed, 13 Mar 2024 08:34:23 +0800 Subject: [PATCH 10/15] complete requested changes --- src/providers/sources/hdrezka/index.ts | 6 +++--- src/providers/sources/hdrezka/types.ts | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/providers/sources/hdrezka/index.ts b/src/providers/sources/hdrezka/index.ts index c98869f..f3de725 100644 --- a/src/providers/sources/hdrezka/index.ts +++ b/src/providers/sources/hdrezka/index.ts @@ -72,7 +72,7 @@ async function getStream( return JSON.parse(response); } -async function getTranslator( +async function getTranslatorId( url: string, id: string, ctx: ShowScrapeContext | MovieScrapeContext, @@ -94,9 +94,9 @@ async function getTranslator( const universalScraper = async (ctx: ShowScrapeContext | MovieScrapeContext): Promise => { const result = await searchAndFindMediaId(ctx); - if (!result) throw new NotFoundError('No result found'); + if (!result || !result.id) throw new NotFoundError('No result found'); - const translatorId = await getTranslator(result.url, result.id, ctx); + const translatorId = await getTranslatorId(result.url, result.id, ctx); if (!translatorId) throw new NotFoundError('No translator id found'); const { url: streamUrl, subtitle: streamSubtitle } = await getStream(result.id, translatorId, ctx); diff --git a/src/providers/sources/hdrezka/types.ts b/src/providers/sources/hdrezka/types.ts index 66b2b9c..d7ccdc2 100644 --- a/src/providers/sources/hdrezka/types.ts +++ b/src/providers/sources/hdrezka/types.ts @@ -13,7 +13,7 @@ export type VideoLinks = { }; export interface MovieData { - id: string; + id: string | null; year: number; type: ScrapeMedia['type']; url: string; From fed8678ef1d8b05e187e149b1f339885c0661750 Mon Sep 17 00:00:00 2001 From: Paradox-77 Date: Thu, 14 Mar 2024 22:45:19 +0530 Subject: [PATCH 11/15] Fix embedUrl matching for Vidplay and Filemoon sources --- src/providers/sources/vidsrcto/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/providers/sources/vidsrcto/index.ts b/src/providers/sources/vidsrcto/index.ts index 94edc3d..3739ab5 100644 --- a/src/providers/sources/vidsrcto/index.ts +++ b/src/providers/sources/vidsrcto/index.ts @@ -52,7 +52,7 @@ const universalScraper = async (ctx: ShowScrapeContext | MovieScrapeContext): Pr for (const source of sources.result) { if (source.title === 'Vidplay') { - const embedUrl = embedUrls.find((v) => v.includes('vidplay')); + const embedUrl = embedUrls.find((v) => v.match(/https:\/\/(?:[a-zA-Z0-9]{10})\./)); if (!embedUrl) continue; embeds.push({ embedId: 'vidplay', @@ -61,7 +61,7 @@ const universalScraper = async (ctx: ShowScrapeContext | MovieScrapeContext): Pr } if (source.title === 'Filemoon') { - const embedUrl = embedUrls.find((v) => v.includes('filemoon')); + const embedUrl = embedUrls.find((v) => v.includes('kerapoxy')); if (!embedUrl) continue; const fullUrl = new URL(embedUrl); if (subtitleUrl) fullUrl.searchParams.set('sub.info', subtitleUrl); From bd0f1e2ddc9d0e14bae0c5f5a77cdf79a4a5b544 Mon Sep 17 00:00:00 2001 From: Paradox-77 Date: Fri, 15 Mar 2024 00:18:30 +0530 Subject: [PATCH 12/15] Refactored Source to Url association, no longer breaks if domain changes --- src/providers/sources/vidsrcto/index.ts | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/src/providers/sources/vidsrcto/index.ts b/src/providers/sources/vidsrcto/index.ts index 3739ab5..fcc7197 100644 --- a/src/providers/sources/vidsrcto/index.ts +++ b/src/providers/sources/vidsrcto/index.ts @@ -33,7 +33,7 @@ const universalScraper = async (ctx: ShowScrapeContext | MovieScrapeContext): Pr if (sources.status !== 200) throw new Error('No sources found'); const embeds: SourcererEmbed[] = []; - const embedUrls = []; + const embedArr = []; for (const source of sources.result) { const sourceRes = await ctx.proxiedFetcher(`/ajax/embed/source/${source.id}`, { baseUrl: vidSrcToBase, @@ -42,28 +42,26 @@ const universalScraper = async (ctx: ShowScrapeContext | MovieScrapeContext): Pr }, }); const decryptedUrl = decryptSourceUrl(sourceRes.result.url); - embedUrls.push(decryptedUrl); + embedArr.push({ source: source.title, url: decryptedUrl }); } // Originally Filemoon does not have subtitles. But we can use the ones from Vidplay. - const urlWithSubtitles = embedUrls.find((v) => v.includes('sub.info')); + const urlWithSubtitles = embedArr.find((v) => v.url.includes('sub.info'))?.url; let subtitleUrl: string | null = null; if (urlWithSubtitles) subtitleUrl = new URL(urlWithSubtitles).searchParams.get('sub.info'); - for (const source of sources.result) { - if (source.title === 'Vidplay') { - const embedUrl = embedUrls.find((v) => v.match(/https:\/\/(?:[a-zA-Z0-9]{10})\./)); - if (!embedUrl) continue; + for (const embedObj of embedArr) { + if (embedObj.source === 'Vidplay') { + const fullUrl = new URL(embedObj.url); + if (subtitleUrl) fullUrl.searchParams.set('sub.info', subtitleUrl); embeds.push({ embedId: 'vidplay', - url: embedUrl, + url: fullUrl.toString(), }); } - if (source.title === 'Filemoon') { - const embedUrl = embedUrls.find((v) => v.includes('kerapoxy')); - if (!embedUrl) continue; - const fullUrl = new URL(embedUrl); + if (embedObj.source === 'Filemoon') { + const fullUrl = new URL(embedObj.url); if (subtitleUrl) fullUrl.searchParams.set('sub.info', subtitleUrl); embeds.push({ embedId: 'filemoon', From 44686f76deb15aa5cd4915abe6b23e5322c98316 Mon Sep 17 00:00:00 2001 From: Paradox-77 Date: Fri, 15 Mar 2024 00:41:32 +0530 Subject: [PATCH 13/15] Fixed vidplay subtitle oversight and refactored filemoon url to use vidplay's subtitles --- src/providers/sources/vidsrcto/index.ts | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/providers/sources/vidsrcto/index.ts b/src/providers/sources/vidsrcto/index.ts index fcc7197..b0b95f9 100644 --- a/src/providers/sources/vidsrcto/index.ts +++ b/src/providers/sources/vidsrcto/index.ts @@ -45,15 +45,9 @@ const universalScraper = async (ctx: ShowScrapeContext | MovieScrapeContext): Pr embedArr.push({ source: source.title, url: decryptedUrl }); } - // Originally Filemoon does not have subtitles. But we can use the ones from Vidplay. - const urlWithSubtitles = embedArr.find((v) => v.url.includes('sub.info'))?.url; - let subtitleUrl: string | null = null; - if (urlWithSubtitles) subtitleUrl = new URL(urlWithSubtitles).searchParams.get('sub.info'); - for (const embedObj of embedArr) { if (embedObj.source === 'Vidplay') { const fullUrl = new URL(embedObj.url); - if (subtitleUrl) fullUrl.searchParams.set('sub.info', subtitleUrl); embeds.push({ embedId: 'vidplay', url: fullUrl.toString(), @@ -62,6 +56,9 @@ const universalScraper = async (ctx: ShowScrapeContext | MovieScrapeContext): Pr if (embedObj.source === 'Filemoon') { const fullUrl = new URL(embedObj.url); + // Originally Filemoon does not have subtitles. But we can use the ones from Vidplay. + const urlWithSubtitles = embedArr.find((v) => v.source === 'Vidplay' && v.url.includes('sub.info'))?.url; + const subtitleUrl = urlWithSubtitles ? new URL(urlWithSubtitles).searchParams.get('sub.info') : null; if (subtitleUrl) fullUrl.searchParams.set('sub.info', subtitleUrl); embeds.push({ embedId: 'filemoon', From db4a27e51a908076651df3713c3e18f52080fe12 Mon Sep 17 00:00:00 2001 From: Jorrin Date: Thu, 14 Mar 2024 20:45:40 +0100 Subject: [PATCH 14/15] add cors allowed flags --- src/providers/embeds/filemoon/index.ts | 3 ++- src/providers/embeds/vidplay/index.ts | 9 +++------ src/providers/sources/vidsrcto/index.ts | 3 ++- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/providers/embeds/filemoon/index.ts b/src/providers/embeds/filemoon/index.ts index 9f96a07..4622959 100644 --- a/src/providers/embeds/filemoon/index.ts +++ b/src/providers/embeds/filemoon/index.ts @@ -1,6 +1,7 @@ import { unpack } from 'unpacker'; import { SubtitleResult } from './types'; +import { flags } from '../../../../lib'; import { makeEmbed } from '../../base'; import { Caption, getCaptionTypeFromUrl, labelToLanguageCode } from '../../captions'; @@ -49,7 +50,7 @@ export const fileMoonScraper = makeEmbed({ id: 'primary', type: 'hls', playlist: file[1], - flags: [], + flags: [flags.CORS_ALLOWED], captions, }, ], diff --git a/src/providers/embeds/vidplay/index.ts b/src/providers/embeds/vidplay/index.ts index 48af6c1..045e19e 100644 --- a/src/providers/embeds/vidplay/index.ts +++ b/src/providers/embeds/vidplay/index.ts @@ -1,7 +1,8 @@ +import { flags } from '@/entrypoint/utils/targets'; import { makeEmbed } from '@/providers/base'; import { Caption, getCaptionTypeFromUrl, labelToLanguageCode } from '@/providers/captions'; -import { getFileUrl, referer } from './common'; +import { getFileUrl } from './common'; import { SubtitleResult, VidplaySourceResponse } from './types'; export const vidplayScraper = makeEmbed({ @@ -44,12 +45,8 @@ export const vidplayScraper = makeEmbed({ id: 'primary', type: 'hls', playlist: source, - flags: [], + flags: [flags.CORS_ALLOWED], captions, - preferredHeaders: { - Referer: referer, - Origin: referer, - }, }, ], }; diff --git a/src/providers/sources/vidsrcto/index.ts b/src/providers/sources/vidsrcto/index.ts index b0b95f9..c6c9fb5 100644 --- a/src/providers/sources/vidsrcto/index.ts +++ b/src/providers/sources/vidsrcto/index.ts @@ -1,5 +1,6 @@ import { load } from 'cheerio'; +import { flags } from '@/entrypoint/utils/targets'; import { SourcererEmbed, SourcererOutput, makeSourcerer } from '@/providers/base'; import { MovieScrapeContext, ShowScrapeContext } from '@/utils/context'; @@ -77,6 +78,6 @@ export const vidSrcToScraper = makeSourcerer({ name: 'VidSrcTo', scrapeMovie: universalScraper, scrapeShow: universalScraper, - flags: [], + flags: [flags.CORS_ALLOWED], rank: 300, }); From d776069613451a7eac88296142a0c677e4bd6276 Mon Sep 17 00:00:00 2001 From: Jorrin Date: Thu, 14 Mar 2024 20:51:49 +0100 Subject: [PATCH 15/15] fix build --- src/providers/embeds/filemoon/index.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/providers/embeds/filemoon/index.ts b/src/providers/embeds/filemoon/index.ts index 4622959..3f8a2f3 100644 --- a/src/providers/embeds/filemoon/index.ts +++ b/src/providers/embeds/filemoon/index.ts @@ -1,7 +1,8 @@ import { unpack } from 'unpacker'; +import { flags } from '@/entrypoint/utils/targets'; + import { SubtitleResult } from './types'; -import { flags } from '../../../../lib'; import { makeEmbed } from '../../base'; import { Caption, getCaptionTypeFromUrl, labelToLanguageCode } from '../../captions';