From f88021fff692f08c78d3dfb9755f4997c172124c Mon Sep 17 00:00:00 2001 From: Jorrin Date: Fri, 29 Dec 2023 19:24:34 +0100 Subject: [PATCH 01/37] added ridomovies --- src/providers/all.ts | 6 ++ src/providers/embeds/closeload.ts | 67 +++++++++++++++++++ src/providers/embeds/ridoo.ts | 46 +++++++++++++ src/providers/sources/ridomovies/index.ts | 74 +++++++++++++++++++++ src/providers/sources/ridomovies/types.ts | 78 +++++++++++++++++++++++ 5 files changed, 271 insertions(+) create mode 100644 src/providers/embeds/closeload.ts create mode 100644 src/providers/embeds/ridoo.ts create mode 100644 src/providers/sources/ridomovies/index.ts create mode 100644 src/providers/sources/ridomovies/types.ts diff --git a/src/providers/all.ts b/src/providers/all.ts index d1e7885..e1a4ef0 100644 --- a/src/providers/all.ts +++ b/src/providers/all.ts @@ -14,8 +14,11 @@ import { remotestreamScraper } from '@/providers/sources/remotestream'; import { showboxScraper } from '@/providers/sources/showbox/index'; import { zoechipScraper } from '@/providers/sources/zoechip'; +import { closeLoadScraper } from './embeds/closeload'; +import { ridooScraper } from './embeds/ridoo'; import { smashyStreamDScraper } from './embeds/smashystream/dued'; import { smashyStreamFScraper } from './embeds/smashystream/video1'; +import { ridooMoviesScraper } from './sources/ridomovies'; import { smashyStreamScraper } from './sources/smashystream'; export function gatherAllSources(): Array { @@ -29,6 +32,7 @@ export function gatherAllSources(): Array { zoechipScraper, lookmovieScraper, smashyStreamScraper, + ridooMoviesScraper, ]; } @@ -44,5 +48,7 @@ export function gatherAllEmbeds(): Array { mixdropScraper, smashyStreamFScraper, smashyStreamDScraper, + ridooScraper, + closeLoadScraper, ]; } diff --git a/src/providers/embeds/closeload.ts b/src/providers/embeds/closeload.ts new file mode 100644 index 0000000..e8d47dd --- /dev/null +++ b/src/providers/embeds/closeload.ts @@ -0,0 +1,67 @@ +import { load } from 'cheerio'; +import { unpack } from 'unpacker'; + +import { flags } from '@/entrypoint/utils/targets'; +import { NotFoundError } from '@/utils/errors'; + +import { makeEmbed } from '../base'; +import { Caption, getCaptionTypeFromUrl, labelToLanguageCode } from '../captions'; + +const referer = 'https://ridomovies.tv/'; + +export const closeLoadScraper = makeEmbed({ + id: 'closeload', + name: 'CloseLoad', + rank: 501, + async scrape(ctx) { + const baseUrl = new URL(ctx.url).origin; + + const iframeRes = await ctx.proxiedFetcher(ctx.url, { + headers: { referer }, + }); + const iframeRes$ = load(iframeRes); + const captions: Caption[] = iframeRes$('track') + .map((_, el) => { + const track = iframeRes$(el); + const url = `${baseUrl}${track.attr('src')}`; + const label = track.attr('label') ?? ''; + const language = labelToLanguageCode(label); + const captionType = getCaptionTypeFromUrl(url); + + if (!language || !captionType) return null; + return { + id: url, + language, + hasCorsRestrictions: false, + type: captionType, + url, + }; + }) + .get() + .filter((x) => x !== null); + + const evalCode = iframeRes$('script') + .filter((_, el) => { + const script = iframeRes$(el); + return (script.attr('type') === 'text/javascript' && script.html()?.includes('eval')) ?? false; + }) + .html(); + if (!evalCode) throw new Error("Couldn't find eval code"); + const decoded = unpack(evalCode); + const regexPattern = /var\s+(\w+)\s*=\s*"([^"]+)";/g; + const base64EncodedUrl = regexPattern.exec(decoded)?.[2]; + if (!base64EncodedUrl) throw new NotFoundError('Unable to find source url'); + const url = atob(base64EncodedUrl); + return { + stream: [ + { + id: 'primary', + type: 'hls', + playlist: url, + captions, + flags: [flags.IP_LOCKED], + }, + ], + }; + }, +}); diff --git a/src/providers/embeds/ridoo.ts b/src/providers/embeds/ridoo.ts new file mode 100644 index 0000000..0473e9c --- /dev/null +++ b/src/providers/embeds/ridoo.ts @@ -0,0 +1,46 @@ +import { load } from 'cheerio'; +import { unpack } from 'unpacker'; + +import { flags } from '@/entrypoint/utils/targets'; +import { NotFoundError } from '@/utils/errors'; + +import { makeEmbed } from '../base'; + +const referer = 'https://ridomovies.tv/'; + +export const ridooScraper = makeEmbed({ + id: 'ridoo', + name: 'Ridoo', + rank: 500, + async scrape(ctx) { + const res = await ctx.proxiedFetcher(ctx.url, { + headers: { + referer, + }, + }); + const $ = load(res); + const evalCode = $('script') + .filter((_, el) => { + const script = $(el); + return (script.attr('type') === 'text/javascript' && script.html()?.includes('eval')) ?? false; + }) + .html(); + if (!evalCode) throw new Error(res); + const decoded = unpack(evalCode); + const regexPattern = /file:"([^"]+)"/g; + const url = regexPattern.exec(decoded)?.[1]; + if (!url) throw new NotFoundError('Unable to find source url'); + + return { + stream: [ + { + id: 'primary', + type: 'hls', + playlist: url, + captions: [], + flags: [flags.CORS_ALLOWED], + }, + ], + }; + }, +}); diff --git a/src/providers/sources/ridomovies/index.ts b/src/providers/sources/ridomovies/index.ts new file mode 100644 index 0000000..a7938f4 --- /dev/null +++ b/src/providers/sources/ridomovies/index.ts @@ -0,0 +1,74 @@ +import { load } from 'cheerio'; + +import { flags } from '@/entrypoint/utils/targets'; +import { SourcererEmbed, makeSourcerer } from '@/providers/base'; +import { closeLoadScraper } from '@/providers/embeds/closeload'; +import { ridooScraper } from '@/providers/embeds/ridoo'; +import { MovieScrapeContext, ShowScrapeContext } from '@/utils/context'; +import { NotFoundError } from '@/utils/errors'; + +import { IframeSourceResult, SearchResult } from './types'; + +const ridoMoviesBase = `https://ridomovies.tv`; +const ridoMoviesApiBase = `${ridoMoviesBase}/core/api`; + +const universalScraper = async (ctx: MovieScrapeContext | ShowScrapeContext) => { + const searchResult = await ctx.fetcher('/search', { + baseUrl: ridoMoviesApiBase, + query: { + q: encodeURIComponent(ctx.media.title), + }, + }); + const show = searchResult.data.items[0]; + + let iframeSourceUrl = `/${show.fullSlug}/videos`; + + if (ctx.media.type === 'show') { + const showPageResult = await ctx.fetcher(`/${show.fullSlug}`, { + baseUrl: ridoMoviesBase, + }); + const fullEpisodeSlug = `${show.fullSlug}/season-${ctx.media.season.number}/episode-${ctx.media.episode.number}`; + const regexPattern = new RegExp( + `\\\\"id\\\\":\\\\"(\\d+)\\\\"(?=.*?\\\\\\"fullSlug\\\\\\":\\\\\\"${fullEpisodeSlug}\\\\\\")`, + 'g', + ); + const matches = [...showPageResult.matchAll(regexPattern)]; + const episodeIds = matches.map((match) => match[1]); + if (episodeIds.length === 0) throw new NotFoundError('No watchable item found'); + const episodeId = episodeIds.at(-1); + iframeSourceUrl = `/episodes/${episodeId}/videos`; + } + + const iframeSource = await ctx.fetcher(iframeSourceUrl, { + baseUrl: ridoMoviesApiBase, + }); + const iframeSource$ = load(iframeSource.data[0].url); + const iframeUrl = iframeSource$('iframe').attr('data-src'); + if (!iframeUrl) throw new NotFoundError('No watchable item found'); + + const embeds: SourcererEmbed[] = []; + if (iframeUrl.includes('closeload')) { + embeds.push({ + embedId: closeLoadScraper.id, + url: iframeUrl, + }); + } + if (iframeUrl.includes('ridoo')) { + embeds.push({ + embedId: ridooScraper.id, + url: iframeUrl, + }); + } + return { + embeds, + }; +}; + +export const ridooMoviesScraper = makeSourcerer({ + id: 'ridomovies', + name: 'RidoMovies', + rank: 500, + flags: [flags.CORS_ALLOWED], + scrapeMovie: universalScraper, + scrapeShow: universalScraper, +}); diff --git a/src/providers/sources/ridomovies/types.ts b/src/providers/sources/ridomovies/types.ts new file mode 100644 index 0000000..a030738 --- /dev/null +++ b/src/providers/sources/ridomovies/types.ts @@ -0,0 +1,78 @@ +export interface Content { + id: string; + type: string; + slug: string; + title: string; + metaTitle: any; + metaDescription: any; + usersOnly: boolean; + userLevel: number; + vipOnly: boolean; + copyrighted: boolean; + status: string; + publishedAt: string; + createdAt: string; + updatedAt: string; + fullSlug: string; +} + +export interface Contentable { + id: string; + contentId: string; + revisionId: any; + originalTitle: string; + overview: string; + releaseDate: string; + releaseYear: string; + videoNote: any; + posterNote: any; + userRating: number; + imdbRating: number; + imdbVotes: number; + imdbId: string; + duration: number; + countryCode: string; + posterPath: string; + backdropPath: string; + apiPosterPath: string; + apiBackdropPath: string; + trailerUrl: string; + mpaaRating: string; + tmdbId: number; + manual: number; + directorId: number; + createdAt: string; + updatedAt: string; + content: Content; +} + +export interface SearchResultItem { + id: string; + type: string; + slug: string; + title: string; + metaTitle: any; + metaDescription: any; + usersOnly: boolean; + userLevel: number; + vipOnly: boolean; + copyrighted: boolean; + status: string; + publishedAt: string; + createdAt: string; + updatedAt: string; + fullSlug: string; + contentable: Contentable; +} + +export type SearchResult = { + data: { + items: SearchResultItem[]; + }; +}; + +export type IframeSourceResult = { + data: { + url: string; + }[]; +}; From 69e66ab855b9d445b1165668c4af802f4f3c275a Mon Sep 17 00:00:00 2001 From: Jorrin Date: Wed, 3 Jan 2024 13:55:11 +0100 Subject: [PATCH 02/37] proxied fetcher --- src/providers/embeds/ridoo.ts | 2 +- src/providers/sources/ridomovies/index.ts | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/providers/embeds/ridoo.ts b/src/providers/embeds/ridoo.ts index 0473e9c..326fc34 100644 --- a/src/providers/embeds/ridoo.ts +++ b/src/providers/embeds/ridoo.ts @@ -25,7 +25,7 @@ export const ridooScraper = makeEmbed({ return (script.attr('type') === 'text/javascript' && script.html()?.includes('eval')) ?? false; }) .html(); - if (!evalCode) throw new Error(res); + if (!evalCode) throw new Error("Couldn't find eval code"); const decoded = unpack(evalCode); const regexPattern = /file:"([^"]+)"/g; const url = regexPattern.exec(decoded)?.[1]; diff --git a/src/providers/sources/ridomovies/index.ts b/src/providers/sources/ridomovies/index.ts index a7938f4..2335149 100644 --- a/src/providers/sources/ridomovies/index.ts +++ b/src/providers/sources/ridomovies/index.ts @@ -13,7 +13,7 @@ const ridoMoviesBase = `https://ridomovies.tv`; const ridoMoviesApiBase = `${ridoMoviesBase}/core/api`; const universalScraper = async (ctx: MovieScrapeContext | ShowScrapeContext) => { - const searchResult = await ctx.fetcher('/search', { + const searchResult = await ctx.proxiedFetcher('/search', { baseUrl: ridoMoviesApiBase, query: { q: encodeURIComponent(ctx.media.title), @@ -24,7 +24,7 @@ const universalScraper = async (ctx: MovieScrapeContext | ShowScrapeContext) => let iframeSourceUrl = `/${show.fullSlug}/videos`; if (ctx.media.type === 'show') { - const showPageResult = await ctx.fetcher(`/${show.fullSlug}`, { + const showPageResult = await ctx.proxiedFetcher(`/${show.fullSlug}`, { baseUrl: ridoMoviesBase, }); const fullEpisodeSlug = `${show.fullSlug}/season-${ctx.media.season.number}/episode-${ctx.media.episode.number}`; @@ -39,7 +39,7 @@ const universalScraper = async (ctx: MovieScrapeContext | ShowScrapeContext) => iframeSourceUrl = `/episodes/${episodeId}/videos`; } - const iframeSource = await ctx.fetcher(iframeSourceUrl, { + const iframeSource = await ctx.proxiedFetcher(iframeSourceUrl, { baseUrl: ridoMoviesApiBase, }); const iframeSource$ = load(iframeSource.data[0].url); From cc400aebb716671769a73c34e0b2f359269009e1 Mon Sep 17 00:00:00 2001 From: Jorrin Date: Wed, 3 Jan 2024 14:02:08 +0100 Subject: [PATCH 03/37] ip locked ridoo --- src/providers/embeds/ridoo.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/providers/embeds/ridoo.ts b/src/providers/embeds/ridoo.ts index 326fc34..eeb770c 100644 --- a/src/providers/embeds/ridoo.ts +++ b/src/providers/embeds/ridoo.ts @@ -38,7 +38,7 @@ export const ridooScraper = makeEmbed({ type: 'hls', playlist: url, captions: [], - flags: [flags.CORS_ALLOWED], + flags: [flags.CORS_ALLOWED, flags.IP_LOCKED], }, ], }; From 2f892eec7b849831c4f88831a10ba7859a0fcf10 Mon Sep 17 00:00:00 2001 From: Jorrin Date: Thu, 4 Jan 2024 21:21:18 +0100 Subject: [PATCH 04/37] remove ip lock from ridoo --- src/providers/embeds/ridoo.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/providers/embeds/ridoo.ts b/src/providers/embeds/ridoo.ts index eeb770c..326fc34 100644 --- a/src/providers/embeds/ridoo.ts +++ b/src/providers/embeds/ridoo.ts @@ -38,7 +38,7 @@ export const ridooScraper = makeEmbed({ type: 'hls', playlist: url, captions: [], - flags: [flags.CORS_ALLOWED, flags.IP_LOCKED], + flags: [flags.CORS_ALLOWED], }, ], }; From 7e1c897e30995fb92eeef50dd5e9405352710b66 Mon Sep 17 00:00:00 2001 From: Jorrin Date: Fri, 5 Jan 2024 21:23:32 +0100 Subject: [PATCH 05/37] fix ranks and feedback --- src/providers/embeds/closeload.ts | 2 +- src/providers/embeds/ridoo.ts | 16 ++-------------- src/providers/sources/ridomovies/index.ts | 5 +++-- 3 files changed, 6 insertions(+), 17 deletions(-) diff --git a/src/providers/embeds/closeload.ts b/src/providers/embeds/closeload.ts index e8d47dd..ebe842c 100644 --- a/src/providers/embeds/closeload.ts +++ b/src/providers/embeds/closeload.ts @@ -12,7 +12,7 @@ const referer = 'https://ridomovies.tv/'; export const closeLoadScraper = makeEmbed({ id: 'closeload', name: 'CloseLoad', - rank: 501, + rank: 106, async scrape(ctx) { const baseUrl = new URL(ctx.url).origin; diff --git a/src/providers/embeds/ridoo.ts b/src/providers/embeds/ridoo.ts index 326fc34..d7f5df5 100644 --- a/src/providers/embeds/ridoo.ts +++ b/src/providers/embeds/ridoo.ts @@ -1,6 +1,3 @@ -import { load } from 'cheerio'; -import { unpack } from 'unpacker'; - import { flags } from '@/entrypoint/utils/targets'; import { NotFoundError } from '@/utils/errors'; @@ -11,24 +8,15 @@ const referer = 'https://ridomovies.tv/'; export const ridooScraper = makeEmbed({ id: 'ridoo', name: 'Ridoo', - rank: 500, + rank: 105, async scrape(ctx) { const res = await ctx.proxiedFetcher(ctx.url, { headers: { referer, }, }); - const $ = load(res); - const evalCode = $('script') - .filter((_, el) => { - const script = $(el); - return (script.attr('type') === 'text/javascript' && script.html()?.includes('eval')) ?? false; - }) - .html(); - if (!evalCode) throw new Error("Couldn't find eval code"); - const decoded = unpack(evalCode); const regexPattern = /file:"([^"]+)"/g; - const url = regexPattern.exec(decoded)?.[1]; + const url = regexPattern.exec(res)?.[1]; if (!url) throw new NotFoundError('Unable to find source url'); return { diff --git a/src/providers/sources/ridomovies/index.ts b/src/providers/sources/ridomovies/index.ts index 2335149..f028875 100644 --- a/src/providers/sources/ridomovies/index.ts +++ b/src/providers/sources/ridomovies/index.ts @@ -16,10 +16,11 @@ const universalScraper = async (ctx: MovieScrapeContext | ShowScrapeContext) => const searchResult = await ctx.proxiedFetcher('/search', { baseUrl: ridoMoviesApiBase, query: { - q: encodeURIComponent(ctx.media.title), + q: ctx.media.title, }, }); const show = searchResult.data.items[0]; + if (!show) throw new NotFoundError('No watchable item found'); let iframeSourceUrl = `/${show.fullSlug}/videos`; @@ -67,7 +68,7 @@ const universalScraper = async (ctx: MovieScrapeContext | ShowScrapeContext) => export const ridooMoviesScraper = makeSourcerer({ id: 'ridomovies', name: 'RidoMovies', - rank: 500, + rank: 105, flags: [flags.CORS_ALLOWED], scrapeMovie: universalScraper, scrapeShow: universalScraper, From 8819648023f912e9f37a46f0e64d56a0ac82962f Mon Sep 17 00:00:00 2001 From: Jorrin Date: Mon, 15 Jan 2024 19:27:36 +0100 Subject: [PATCH 06/37] add vidcloud to flixhq and zoechip --- src/providers/all.ts | 2 + src/providers/embeds/vidcloud.ts | 19 +++++++++ src/providers/sources/flixhq/index.ts | 52 +++++++++++++++++-------- src/providers/sources/zoechip/common.ts | 6 +++ 4 files changed, 62 insertions(+), 17 deletions(-) create mode 100644 src/providers/embeds/vidcloud.ts diff --git a/src/providers/all.ts b/src/providers/all.ts index de22b4c..8511673 100644 --- a/src/providers/all.ts +++ b/src/providers/all.ts @@ -20,6 +20,7 @@ import { zoechipScraper } from '@/providers/sources/zoechip'; import { fileMoonScraper } from './embeds/filemoon'; import { smashyStreamDScraper } from './embeds/smashystream/dued'; import { smashyStreamFScraper } from './embeds/smashystream/video1'; +import { vidCloudScraper } from './embeds/vidcloud'; import { vidplayScraper } from './embeds/vidplay'; import { smashyStreamScraper } from './sources/smashystream'; import { vidSrcToScraper } from './sources/vidsrcto'; @@ -44,6 +45,7 @@ export function gatherAllEmbeds(): Array { // all embeds are gathered here return [ upcloudScraper, + vidCloudScraper, mp4uploadScraper, streamsbScraper, upstreamScraper, diff --git a/src/providers/embeds/vidcloud.ts b/src/providers/embeds/vidcloud.ts new file mode 100644 index 0000000..2aca309 --- /dev/null +++ b/src/providers/embeds/vidcloud.ts @@ -0,0 +1,19 @@ +import { makeEmbed } from '@/providers/base'; + +import { upcloudScraper } from './upcloud'; + +export const vidCloudScraper = makeEmbed({ + id: 'vidcloud', + name: 'VidCloud', + rank: 201, + async scrape(ctx) { + // Both vidcloud and upcloud have the same embed domain (rabbitstream.net) + const result = await upcloudScraper.scrape(ctx); + return { + stream: result.stream.map((s) => ({ + ...s, + flags: [], + })), + }; + }, +}); diff --git a/src/providers/sources/flixhq/index.ts b/src/providers/sources/flixhq/index.ts index e9fbaef..45064ac 100644 --- a/src/providers/sources/flixhq/index.ts +++ b/src/providers/sources/flixhq/index.ts @@ -1,6 +1,7 @@ import { flags } from '@/entrypoint/utils/targets'; -import { makeSourcerer } from '@/providers/base'; +import { SourcererEmbed, makeSourcerer } from '@/providers/base'; import { upcloudScraper } from '@/providers/embeds/upcloud'; +import { vidCloudScraper } from '@/providers/embeds/vidcloud'; import { getFlixhqMovieSources, getFlixhqShowSources, getFlixhqSourceDetails } from '@/providers/sources/flixhq/scrape'; import { getFlixhqId } from '@/providers/sources/flixhq/search'; import { NotFoundError } from '@/utils/errors'; @@ -15,16 +16,25 @@ export const flixhqScraper = makeSourcerer({ if (!id) throw new NotFoundError('no search results match'); const sources = await getFlixhqMovieSources(ctx, ctx.media, id); - const upcloudStream = sources.find((v) => v.embed.toLowerCase() === 'upcloud'); - if (!upcloudStream) throw new NotFoundError('upcloud stream not found for flixhq'); + + const embeds: SourcererEmbed[] = []; + + for (const source of sources) { + if (source.embed.toLowerCase() === 'upcloud') { + embeds.push({ + embedId: upcloudScraper.id, + url: await getFlixhqSourceDetails(ctx, source.episodeId), + }); + } else if (source.embed.toLowerCase() === 'vidcloud') { + embeds.push({ + embedId: vidCloudScraper.id, + url: await getFlixhqSourceDetails(ctx, source.episodeId), + }); + } + } return { - embeds: [ - { - embedId: upcloudScraper.id, - url: await getFlixhqSourceDetails(ctx, upcloudStream.episodeId), - }, - ], + embeds, }; }, async scrapeShow(ctx) { @@ -32,16 +42,24 @@ export const flixhqScraper = makeSourcerer({ if (!id) throw new NotFoundError('no search results match'); const sources = await getFlixhqShowSources(ctx, ctx.media, id); - const upcloudStream = sources.find((v) => v.embed.toLowerCase() === 'server upcloud'); - if (!upcloudStream) throw new NotFoundError('upcloud stream not found for flixhq'); + + const embeds: SourcererEmbed[] = []; + for (const source of sources) { + if (source.embed.toLowerCase() === 'server upcloud') { + embeds.push({ + embedId: upcloudScraper.id, + url: await getFlixhqSourceDetails(ctx, source.episodeId), + }); + } else if (source.embed.toLowerCase() === 'server vidcloud') { + embeds.push({ + embedId: vidCloudScraper.id, + url: await getFlixhqSourceDetails(ctx, source.episodeId), + }); + } + } return { - embeds: [ - { - embedId: upcloudScraper.id, - url: await getFlixhqSourceDetails(ctx, upcloudStream.episodeId), - }, - ], + embeds, }; }, }); diff --git a/src/providers/sources/zoechip/common.ts b/src/providers/sources/zoechip/common.ts index a860125..55b37ac 100644 --- a/src/providers/sources/zoechip/common.ts +++ b/src/providers/sources/zoechip/common.ts @@ -1,6 +1,7 @@ import { mixdropScraper } from '@/providers/embeds/mixdrop'; import { upcloudScraper } from '@/providers/embeds/upcloud'; import { upstreamScraper } from '@/providers/embeds/upstream'; +import { vidCloudScraper } from '@/providers/embeds/vidcloud'; import { getZoeChipSourceURL, getZoeChipSources } from '@/providers/sources/zoechip/scrape'; import { MovieScrapeContext, ShowScrapeContext } from '@/utils/context'; @@ -55,6 +56,11 @@ export async function createZoeChipStreamData(ctx: MovieScrapeContext | ShowScra for (const source of sources) { const formatted = await formatSource(ctx, source); if (formatted) { + // Zoechip does not return titles for their sources, so we can not check if a source is upcloud or vidcloud because the domain is the same. + const upCloudAlreadyExists = embeds.find((e) => e.embedId === upcloudScraper.id); + if (formatted.embedId === upcloudScraper.id && upCloudAlreadyExists) { + formatted.embedId = vidCloudScraper.id; + } embeds.push(formatted); } } From 198dfdec872781107efe7b379d1dc6e9f1330ae0 Mon Sep 17 00:00:00 2001 From: MemeCornucopia <148788549+MemeCornucopia@users.noreply.github.com> Date: Fri, 19 Jan 2024 15:08:29 -0500 Subject: [PATCH 07/37] EPIC GOOJARA LESS FUCKING GO --- src/.env copy | 1 + src/providers/all.ts | 6 ++ src/providers/embeds/dood.ts | 75 ++++++++++++++ src/providers/embeds/wootly.ts | 77 ++++++++++++++ src/providers/sources/goojara/getEmbeds.ts | 52 ++++++++++ src/providers/sources/goojara/index.ts | 30 ++++++ src/providers/sources/goojara/type.ts | 35 +++++++ src/providers/sources/goojara/util.ts | 111 +++++++++++++++++++++ 8 files changed, 387 insertions(+) create mode 100644 src/.env copy create mode 100644 src/providers/embeds/dood.ts create mode 100644 src/providers/embeds/wootly.ts create mode 100644 src/providers/sources/goojara/getEmbeds.ts create mode 100644 src/providers/sources/goojara/index.ts create mode 100644 src/providers/sources/goojara/type.ts create mode 100644 src/providers/sources/goojara/util.ts diff --git a/src/.env copy b/src/.env copy new file mode 100644 index 0000000..daeee0b --- /dev/null +++ b/src/.env copy @@ -0,0 +1 @@ +MOVIE_WEB_TMDB_API_KEY="045e3dc79db2f0cbaa9ef67397afca39" \ No newline at end of file diff --git a/src/providers/all.ts b/src/providers/all.ts index de22b4c..b99777d 100644 --- a/src/providers/all.ts +++ b/src/providers/all.ts @@ -16,11 +16,14 @@ import { remotestreamScraper } from '@/providers/sources/remotestream'; import { showboxScraper } from '@/providers/sources/showbox/index'; import { vidsrcScraper } from '@/providers/sources/vidsrc/index'; import { zoechipScraper } from '@/providers/sources/zoechip'; +import { doodScraper } from '@/providers/embeds/dood'; import { fileMoonScraper } from './embeds/filemoon'; import { smashyStreamDScraper } from './embeds/smashystream/dued'; import { smashyStreamFScraper } from './embeds/smashystream/video1'; import { vidplayScraper } from './embeds/vidplay'; +import { wootlyScraper } from './embeds/wootly'; +import { goojaraScraper } from './sources/goojara'; import { smashyStreamScraper } from './sources/smashystream'; import { vidSrcToScraper } from './sources/vidsrcto'; @@ -37,6 +40,7 @@ export function gatherAllSources(): Array { lookmovieScraper, smashyStreamScraper, vidSrcToScraper, + goojaraScraper, ]; } @@ -56,5 +60,7 @@ export function gatherAllEmbeds(): Array { smashyStreamDScraper, fileMoonScraper, vidplayScraper, + wootlyScraper, + doodScraper ]; } diff --git a/src/providers/embeds/dood.ts b/src/providers/embeds/dood.ts new file mode 100644 index 0000000..82ab60e --- /dev/null +++ b/src/providers/embeds/dood.ts @@ -0,0 +1,75 @@ +import { load } from 'cheerio'; + +import { flags } from '@/entrypoint/utils/targets'; +import { makeEmbed } from '@/providers/base'; + +export const doodScraper = makeEmbed({ + id: 'dood', + name: 'dood', + rank: 173, + async scrape(ctx) { + function makeTheFunny() { + let result = ''; + const characters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789'; + const charactersLength = characters.length; + + for (let i = 0; i < 10; i++) { + result += characters.charAt(Math.floor(Math.random() * charactersLength)); + } + + return result; + } + + const baseUrl = 'https://do0od.com'; + + const id = ctx.url.split('/d/')[1] || ctx.url.split('/e/')[1]; + console.log(id); + + const doodData = await ctx.proxiedFetcher(`/e/${id}`, { + method: 'GET', + baseUrl, + }); + + // console.log(doodData); + + const dataForLater = doodData.split(`a+"?token=`)[1].split(`"`)[0]; + const path = doodData.split(`$.get('/pass_md5`)[1].split(`'`)[0]; + + const doodPage = await ctx.proxiedFetcher(`/pass_md5/${path}`, { + headers: { + referer: `${baseUrl}/e/${id}`, + }, + method: 'GET', + baseUrl, + }); + + console.log(`${baseUrl}/pass_md5/${path}`); + + console.log(doodPage); + + // const doodPage = await ctx.proxiedFetcher(`/download/${path}`, { method: 'GET', baseUrl }); + // console.log(doodPage); + const downloadURL = `${doodPage}${makeTheFunny()}?token=${dataForLater}${Date.now()}`; + + if (downloadURL) { + return { + stream: [ + { + id: 'primary', + type: 'hls', + playlist: downloadURL, + flags: [flags.CORS_ALLOWED], + captions: [], + preferredHeaders: { + referer: 'https://do0od.com/', + 'content-type': 'video/mp4', + range: 'bytes=0-', + }, + }, + ], + }; + } + + throw new Error('wootly source not found'); + }, +}); diff --git a/src/providers/embeds/wootly.ts b/src/providers/embeds/wootly.ts new file mode 100644 index 0000000..8b5199d --- /dev/null +++ b/src/providers/embeds/wootly.ts @@ -0,0 +1,77 @@ +import { load } from 'cheerio'; + +import { flags } from '@/entrypoint/utils/targets'; +import { makeEmbed } from '@/providers/base'; + +export const wootlyScraper = makeEmbed({ + id: 'wootly', + name: 'wootly', + rank: 172, + async scrape(ctx) { + const baseUrl = 'https://www.wootly.ch'; + + const wootlyData = await ctx.proxiedFetcher.full(ctx.url, { + method: 'GET', + readHeaders: ['Set-Cookie'], + }); + + const wootssesCookie = wootlyData.headers.get('Set-Cookie')?.split(';')[0].split('wootsses=')[1]; + + let $ = load(wootlyData.body); // load the html data + const iframeSrc = $('iframe').attr('src') ?? ''; + + const woozCookieRequest = await ctx.proxiedFetcher.full(iframeSrc, { + method: 'GET', + readHeaders: ['Set-Cookie'], + headers: { + cookie: `wootsses=${wootssesCookie};`, + }, + }); + + const woozCookie = woozCookieRequest.headers.get('Set-Cookie')?.split(';')[0].split('wooz=')[1]; + + const iframeData = await ctx.proxiedFetcher(iframeSrc, { + method: 'POST', + body: new URLSearchParams({ qdf: '1' }), + headers: { + cookie: `wooz=${woozCookie}`, + Referer: iframeSrc, + }, + }); + + $ = load(iframeData); + + const scriptText = $('script').html() ?? ''; + + // Regular expressions to match the variables + + const tk = scriptText.split('tk=')[1].split(';')[0].replaceAll('"', '').replaceAll(' ', ''); + const vd = scriptText.split('vd=')[1].split(',')[0].replaceAll('"', '').replaceAll(' ', ''); + const cv = scriptText.split('cv=')[1].split(',')[0].replaceAll('"', '').replaceAll(' ', ''); + + const url = await ctx.proxiedFetcher(`/grabd`, { + baseUrl, + query: { t: tk, id: vd }, + method: 'GET', + headers: { + cookie: `wooz=${woozCookie}; wootsses=${wootssesCookie};`, + }, + }); + + if (url) { + return { + stream: [ + { + id: 'primary', + type: 'hls', + playlist: url, + flags: [flags.CORS_ALLOWED, flags.IP_LOCKED], + captions: [], + }, + ], + }; + } + + throw new Error('wootly source not found'); + }, +}); diff --git a/src/providers/sources/goojara/getEmbeds.ts b/src/providers/sources/goojara/getEmbeds.ts new file mode 100644 index 0000000..d0a96a4 --- /dev/null +++ b/src/providers/sources/goojara/getEmbeds.ts @@ -0,0 +1,52 @@ +import { load } from 'cheerio'; + +import { ScrapeContext } from '@/utils/context'; + +import { EmbedsResult } from './type'; + +export async function getEmbeds(ctx: ScrapeContext, id: string): Promise { + const data = await ctx.fetcher.full(`/${id}`, { + baseUrl: 'https://ww1.goojara.to', + headers: { + accept: '*/*', + 'content-type': 'application/x-www-form-urlencoded', + Referer: 'https://www.goojara.to/', + 'Referrer-Policy': 'strict-origin-when-cross-origin', + }, + readHeaders: ['Set-Cookie'], + method: 'GET', + }); + + const aGoozCookie = data.headers.get('Set-Cookie')?.split(';')[0].split('aGooz=')[1]; + const $ = load(data.body); + const RandomCookieName = data.body.split(`_3chk('`)[1].split(`'`)[0]; + const RandomCookieValue = data.body.split(`_3chk('`)[1].split(`'`)[2]; + + const embedRedirectURLs = $('a') + .map((index, element) => $(element).attr('href')) + .get() + .filter((href) => href && href.includes('https://ww1.goojara.to/go.php')); + + const embeds = await Promise.all( + embedRedirectURLs.map((url) => + ctx.fetcher + .full(url, { + headers: { + cookie: `aGooz=${aGoozCookie}; ${RandomCookieName}=${RandomCookieValue};`, + Referer: 'https://ww1.goojara.to/eJwD5z', + }, + method: 'GET', + }) + .then((result) => { + if (result) { + const embedId = ['wootly', 'upstream', 'mixdrop', 'dood'].find((a) => result.finalUrl.includes(a)); + return embedId ? { embedId, url: result.finalUrl } : null; + } + return null; + }) + .catch(() => null), + ), + ).then((results) => results.filter((result) => result !== null) as EmbedsResult); + + return embeds; +} diff --git a/src/providers/sources/goojara/index.ts b/src/providers/sources/goojara/index.ts new file mode 100644 index 0000000..ce2b096 --- /dev/null +++ b/src/providers/sources/goojara/index.ts @@ -0,0 +1,30 @@ +import { SourcererOutput, makeSourcerer } from '@/providers/base'; +import { MovieScrapeContext, ShowScrapeContext } from '@/utils/context'; +import { NotFoundError } from '@/utils/errors'; + +import { scrapeIds, searchAndFindMedia } from './util'; + +async function universalScraper(ctx: ShowScrapeContext | MovieScrapeContext): Promise { + const goojaraData = await searchAndFindMedia(ctx, ctx.media); + if (!goojaraData) throw new NotFoundError('Media not found'); + + ctx.progress(30); + const embeds = await scrapeIds(ctx, ctx.media, goojaraData); + if (embeds?.length === 0) throw new NotFoundError('No embeds found'); + + + ctx.progress(60); + + return { + embeds + } +} + +export const goojaraScraper = makeSourcerer({ + id: 'goojara', + name: 'goojara', + rank: 69, + flags: [], + scrapeShow: universalScraper, + scrapeMovie: universalScraper, +}); diff --git a/src/providers/sources/goojara/type.ts b/src/providers/sources/goojara/type.ts new file mode 100644 index 0000000..7227ec8 --- /dev/null +++ b/src/providers/sources/goojara/type.ts @@ -0,0 +1,35 @@ +// ! Types +interface BaseConfig { + /** The website's slug. Formatted as `1839578-person-of-interest-2011` */ + slug: string; + /** Type of request */ + type: 'show' | 'movie'; + /** Hash */ + hash: string; + /** Hash expiry */ + expires: number; +} +interface TvConfig extends BaseConfig { + /** Type of request */ + type: 'show'; + /** The episode ID for a TV show. Given in search and URL */ + episodeId: string; +} +interface MovieConfig extends BaseConfig { + /** Type of request */ + type: 'movie'; + /** Movie's id */ + slug: string; +} +export type Config = MovieConfig | TvConfig; + +export type EmbedsResult = { embedId: string; url: string; }[] + +export interface Result { + title: string; + slug: any; + year: string; + type: string; + id_movie?: string; + id_show?: string; +} diff --git a/src/providers/sources/goojara/util.ts b/src/providers/sources/goojara/util.ts new file mode 100644 index 0000000..f2b452b --- /dev/null +++ b/src/providers/sources/goojara/util.ts @@ -0,0 +1,111 @@ +import { load } from 'cheerio'; + +import { MovieMedia, ShowMedia } from '@/entrypoint/utils/media'; +import { compareMedia } from '@/utils/compare'; +import { ScrapeContext } from '@/utils/context'; +import { NotFoundError } from '@/utils/errors'; + +import { getEmbeds } from './getEmbeds'; +import { Result, EmbedsResult } from './type'; + +const headersData = { + accept: '*/*', + 'accept-language': 'en-US,en;q=0.9', + 'content-type': 'application/x-www-form-urlencoded', + 'sec-ch-ua': '"Not_A Brand";v="8", "Chromium";v="120", "Google Chrome";v="120"', + 'sec-ch-ua-mobile': '?0', + 'sec-ch-ua-platform': '"Windows"', + 'sec-fetch-dest': 'empty', + 'sec-fetch-mode': 'cors', + 'sec-fetch-site': 'same-origin', + cookie: `aGooz=t9pmkdtef1b3lg3pmo1u2re816; bd9aa48e=0d7b89e8c79844e9df07a2; _b414=2151C6B12E2A88379AFF2C0DD65AC8298DEC2BF4; 9d287aaa=8f32ad589e1c4288fe152f`, + Referer: 'https://www.goojara.to/', + 'Referrer-Policy': 'strict-origin-when-cross-origin', +}; + +export async function searchAndFindMedia( + ctx: ScrapeContext, + media: MovieMedia | ShowMedia, +): Promise { + const data = await ctx.fetcher(`/xhrr.php`, { + baseUrl: 'https://www.goojara.to', + headers: headersData, + method: 'POST', + body: `q=${media.title}`, + }) + + const $ = load(data); + + const results: Result[] = []; + + $('.mfeed > li').each((index, element) => { + const title = $(element).find('strong').text(); + const yearMatch = $(element) + .text() + .match(/\((\d{4})\)/); + const typeDiv = $(element).find('div').attr('class'); + const type = typeDiv === 'it' ? 'show' : typeDiv === 'im' ? 'movie' : ''; + const year = yearMatch ? yearMatch[1] : ''; + const slug = $(element).find('a').attr('href')?.split('/')[3]; + + if (media.type === type) { + results.push({ title, year, slug, type }); + } + }); + + const result = results.find((res: Result) => compareMedia(media, res.title, Number(res.year))); + + return result; +} + +export async function scrapeIds(ctx: ScrapeContext, media: MovieMedia | ShowMedia, result: Result): Promise { + // Find the relevant id + let id = null; + if (media.type === 'movie') { + id = result.slug; + } else if (media.type === 'show') { + const data = await ctx.fetcher(`/${result.slug}`, { + baseUrl: 'https://www.goojara.to', + headers: headersData, + method: 'GET', + }); + + const $1 = load(data); + + const dataId = $1('#seon').data('id'); + + const data2 = await ctx.fetcher(`/xhrc.php`, { + baseUrl: 'https://ww1.goojara.to', + headers: headersData, + method: 'POST', + body: `s=${media.season.number}&t=${dataId}`, + }); + + let episodeId = ''; + + const $2 = load(data2); + + $2('.seho').each((index, element) => { + // Extracting the episode number as a string + const episodeNumber = $2(element).find('.seep .sea').text().trim(); + + // Comparing with the desired episode number as a string + if (parseInt(episodeNumber, 10) === media.episode.number) { + const href = $2(element).find('.snfo h1 a').attr('href'); + const idMatch = href?.match(/\/([a-zA-Z0-9]+)$/); + if (idMatch && idMatch[1]) { + episodeId = idMatch[1]; + return false; // Break out of the loop once the episode is found + } + } + }); + + id = episodeId; + } + + // Check ID + if (id === null) throw new NotFoundError('Not found'); + + const embeds = await getEmbeds(ctx, id); + return embeds; +} From bcb1d292187f1f7fb52c058bb85e97409f736f76 Mon Sep 17 00:00:00 2001 From: MemeCornucopia <148788549+MemeCornucopia@users.noreply.github.com> Date: Fri, 19 Jan 2024 15:19:38 -0500 Subject: [PATCH 08/37] OMG GOOJARA!?!?!?!?!??! EPIC SOURCE!?!?!?!? --- src/providers/embeds/dood.ts | 20 ++++++++++++-------- src/providers/embeds/wootly.ts | 11 ++++++++--- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/src/providers/embeds/dood.ts b/src/providers/embeds/dood.ts index 82ab60e..4b6d413 100644 --- a/src/providers/embeds/dood.ts +++ b/src/providers/embeds/dood.ts @@ -1,5 +1,3 @@ -import { load } from 'cheerio'; - import { flags } from '@/entrypoint/utils/targets'; import { makeEmbed } from '@/providers/base'; @@ -56,14 +54,20 @@ export const doodScraper = makeEmbed({ stream: [ { id: 'primary', - type: 'hls', - playlist: downloadURL, + type: 'file', + url: downloadURL, flags: [flags.CORS_ALLOWED], captions: [], - preferredHeaders: { - referer: 'https://do0od.com/', - 'content-type': 'video/mp4', - range: 'bytes=0-', + qualities: { + unknown: { + type: 'mp4', + url: downloadURL, + preferredHeaders: { + referer: 'https://do0od.com/', + 'content-type': 'video/mp4', + range: 'bytes=0-', + }, + }, }, }, ], diff --git a/src/providers/embeds/wootly.ts b/src/providers/embeds/wootly.ts index 8b5199d..d02962d 100644 --- a/src/providers/embeds/wootly.ts +++ b/src/providers/embeds/wootly.ts @@ -47,7 +47,6 @@ export const wootlyScraper = makeEmbed({ const tk = scriptText.split('tk=')[1].split(';')[0].replaceAll('"', '').replaceAll(' ', ''); const vd = scriptText.split('vd=')[1].split(',')[0].replaceAll('"', '').replaceAll(' ', ''); - const cv = scriptText.split('cv=')[1].split(',')[0].replaceAll('"', '').replaceAll(' ', ''); const url = await ctx.proxiedFetcher(`/grabd`, { baseUrl, @@ -63,10 +62,16 @@ export const wootlyScraper = makeEmbed({ stream: [ { id: 'primary', - type: 'hls', - playlist: url, + type: 'file', + url, flags: [flags.CORS_ALLOWED, flags.IP_LOCKED], captions: [], + qualities: { + unknown: { + type: 'mp4', + url, + }, + }, }, ], }; From e11885a1dfc8ebcf34819916372bba8fd07d988c Mon Sep 17 00:00:00 2001 From: MemeCornucopia <148788549+MemeCornucopia@users.noreply.github.com> Date: Fri, 19 Jan 2024 15:26:14 -0500 Subject: [PATCH 09/37] fixed imports --- src/providers/all.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/providers/all.ts b/src/providers/all.ts index b99777d..cb64896 100644 --- a/src/providers/all.ts +++ b/src/providers/all.ts @@ -1,4 +1,5 @@ import { Embed, Sourcerer } from '@/providers/base'; +import { doodScraper } from '@/providers/embeds/dood'; import { febboxHlsScraper } from '@/providers/embeds/febbox/hls'; import { febboxMp4Scraper } from '@/providers/embeds/febbox/mp4'; import { mixdropScraper } from '@/providers/embeds/mixdrop'; @@ -16,7 +17,6 @@ import { remotestreamScraper } from '@/providers/sources/remotestream'; import { showboxScraper } from '@/providers/sources/showbox/index'; import { vidsrcScraper } from '@/providers/sources/vidsrc/index'; import { zoechipScraper } from '@/providers/sources/zoechip'; -import { doodScraper } from '@/providers/embeds/dood'; import { fileMoonScraper } from './embeds/filemoon'; import { smashyStreamDScraper } from './embeds/smashystream/dued'; @@ -61,6 +61,6 @@ export function gatherAllEmbeds(): Array { fileMoonScraper, vidplayScraper, wootlyScraper, - doodScraper + doodScraper, ]; } From 9291bd6828b3f663e5c62c07d4040dd5c8e405d2 Mon Sep 17 00:00:00 2001 From: MemeCornucopia <148788549+MemeCornucopia@users.noreply.github.com> Date: Fri, 19 Jan 2024 15:27:37 -0500 Subject: [PATCH 10/37] more prettier crap --- src/providers/sources/goojara/index.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/providers/sources/goojara/index.ts b/src/providers/sources/goojara/index.ts index ce2b096..bd1e03f 100644 --- a/src/providers/sources/goojara/index.ts +++ b/src/providers/sources/goojara/index.ts @@ -12,12 +12,11 @@ async function universalScraper(ctx: ShowScrapeContext | MovieScrapeContext): Pr const embeds = await scrapeIds(ctx, ctx.media, goojaraData); if (embeds?.length === 0) throw new NotFoundError('No embeds found'); - ctx.progress(60); return { - embeds - } + embeds, + }; } export const goojaraScraper = makeSourcerer({ From 0396272dd926ab360e2b52d30dfffdb74bc9970b Mon Sep 17 00:00:00 2001 From: MemeCornucopia <148788549+MemeCornucopia@users.noreply.github.com> Date: Fri, 19 Jan 2024 15:29:11 -0500 Subject: [PATCH 11/37] more prettier stuff --- src/providers/sources/goojara/type.ts | 2 +- src/providers/sources/goojara/util.ts | 10 +++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/providers/sources/goojara/type.ts b/src/providers/sources/goojara/type.ts index 7227ec8..732ba4e 100644 --- a/src/providers/sources/goojara/type.ts +++ b/src/providers/sources/goojara/type.ts @@ -23,7 +23,7 @@ interface MovieConfig extends BaseConfig { } export type Config = MovieConfig | TvConfig; -export type EmbedsResult = { embedId: string; url: string; }[] +export type EmbedsResult = { embedId: string; url: string }[]; export interface Result { title: string; diff --git a/src/providers/sources/goojara/util.ts b/src/providers/sources/goojara/util.ts index f2b452b..0fcba4b 100644 --- a/src/providers/sources/goojara/util.ts +++ b/src/providers/sources/goojara/util.ts @@ -6,7 +6,7 @@ import { ScrapeContext } from '@/utils/context'; import { NotFoundError } from '@/utils/errors'; import { getEmbeds } from './getEmbeds'; -import { Result, EmbedsResult } from './type'; +import { EmbedsResult, Result } from './type'; const headersData = { accept: '*/*', @@ -32,7 +32,7 @@ export async function searchAndFindMedia( headers: headersData, method: 'POST', body: `q=${media.title}`, - }) + }); const $ = load(data); @@ -58,7 +58,11 @@ export async function searchAndFindMedia( return result; } -export async function scrapeIds(ctx: ScrapeContext, media: MovieMedia | ShowMedia, result: Result): Promise { +export async function scrapeIds( + ctx: ScrapeContext, + media: MovieMedia | ShowMedia, + result: Result, +): Promise { // Find the relevant id let id = null; if (media.type === 'movie') { From 5eb3ade82e8ad81d1cd28ad210d9fda06f4997d3 Mon Sep 17 00:00:00 2001 From: MemeCornucopia <148788549+MemeCornucopia@users.noreply.github.com> Date: Fri, 19 Jan 2024 15:48:35 -0500 Subject: [PATCH 12/37] deleted env copy xd --- src/.env copy | 1 - 1 file changed, 1 deletion(-) delete mode 100644 src/.env copy diff --git a/src/.env copy b/src/.env copy deleted file mode 100644 index daeee0b..0000000 --- a/src/.env copy +++ /dev/null @@ -1 +0,0 @@ -MOVIE_WEB_TMDB_API_KEY="045e3dc79db2f0cbaa9ef67397afca39" \ No newline at end of file From 4591bcbc2e26470f65f9c3d25cd0f1d882fe139c Mon Sep 17 00:00:00 2001 From: Ciarands Date: Fri, 26 Jan 2024 23:16:37 +0000 Subject: [PATCH 13/37] Fix vidsrc.me base64 decoding --- src/providers/embeds/vidsrc.ts | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/src/providers/embeds/vidsrc.ts b/src/providers/embeds/vidsrc.ts index 93d55c5..069c2f6 100644 --- a/src/providers/embeds/vidsrc.ts +++ b/src/providers/embeds/vidsrc.ts @@ -4,6 +4,14 @@ import { makeEmbed } from '@/providers/base'; const hlsURLRegex = /file:"(.*?)"/; const setPassRegex = /var pass_path = "(.*set_pass\.php.*)";/; +function formatHlsB64(data: string): string { + const encodedB64 = data.replace(/\/@#@\/[^=/]+==/g, ''); + if (encodedB64.match(/\/@#@\/[^=/]+==/)) { + return formatHlsB64(encodedB64); + } + return encodedB64; +} + export const vidsrcembedScraper = makeEmbed({ id: 'vidsrcembed', // VidSrc is both a source and an embed host name: 'VidSrc', @@ -15,10 +23,12 @@ export const vidsrcembedScraper = makeEmbed({ }, }); - const match = html.match(hlsURLRegex)?.[1]?.replace(/(\/\/\S+?=)|#2|=/g, ''); - if (!match) throw new Error('Unable to find HLS playlist'); - const finalUrl = atob(match); - + // When this eventually breaks see the player js @ pjs_main.js + // If you know what youre doing and are slightly confused about how to reverse this feel free to reach out to ciaran_ds on discord with any queries + let hlsMatch = html.match(hlsURLRegex)?.[1]?.slice(2); + if (!hlsMatch) throw new Error('Unable to find HLS playlist'); + hlsMatch = formatHlsB64(hlsMatch); + const finalUrl = atob(hlsMatch); if (!finalUrl.includes('.m3u8')) throw new Error('Unable to find HLS playlist'); let setPassLink = html.match(setPassRegex)?.[1]; From c6c921f80b3cac136158b72b0be20f48761b785b Mon Sep 17 00:00:00 2001 From: Jorrin Date: Sun, 28 Jan 2024 00:29:52 +0100 Subject: [PATCH 14/37] add cors restriction to caption, add preferredHeaders --- src/providers/embeds/closeload.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/providers/embeds/closeload.ts b/src/providers/embeds/closeload.ts index ebe842c..7afa86a 100644 --- a/src/providers/embeds/closeload.ts +++ b/src/providers/embeds/closeload.ts @@ -32,7 +32,7 @@ export const closeLoadScraper = makeEmbed({ return { id: url, language, - hasCorsRestrictions: false, + hasCorsRestrictions: true, type: captionType, url, }; @@ -60,6 +60,10 @@ export const closeLoadScraper = makeEmbed({ playlist: url, captions, flags: [flags.IP_LOCKED], + preferredHeaders: { + Referer: 'https://closeload.top/', + Origin: 'https://closeload.top', + }, }, ], }; From 25dbadd9090251066bb84f49d625bb27634d3dd9 Mon Sep 17 00:00:00 2001 From: Jorrin Date: Sun, 28 Jan 2024 00:33:33 +0100 Subject: [PATCH 15/37] rename --- src/providers/embeds/closeload.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/providers/embeds/closeload.ts b/src/providers/embeds/closeload.ts index 7afa86a..0235a49 100644 --- a/src/providers/embeds/closeload.ts +++ b/src/providers/embeds/closeload.ts @@ -60,7 +60,7 @@ export const closeLoadScraper = makeEmbed({ playlist: url, captions, flags: [flags.IP_LOCKED], - preferredHeaders: { + headers: { Referer: 'https://closeload.top/', Origin: 'https://closeload.top', }, From fb12c6bfa9dcf1248eb0bbac414f1f362e5d3acf Mon Sep 17 00:00:00 2001 From: MemeCornucopia <148788549+MemeCornucopia@users.noreply.github.com> Date: Sat, 27 Jan 2024 19:30:18 -0500 Subject: [PATCH 16/37] Changes requests by jvs --- src/providers/embeds/dood.ts | 70 ++++++++-------------- src/providers/embeds/wootly.ts | 46 +++++++------- src/providers/sources/goojara/getEmbeds.ts | 56 +++++++++-------- src/providers/sources/goojara/index.ts | 2 +- src/providers/sources/goojara/type.ts | 27 +-------- src/providers/sources/goojara/util.ts | 31 +++++----- 6 files changed, 96 insertions(+), 136 deletions(-) diff --git a/src/providers/embeds/dood.ts b/src/providers/embeds/dood.ts index 4b6d413..d977fdc 100644 --- a/src/providers/embeds/dood.ts +++ b/src/providers/embeds/dood.ts @@ -1,37 +1,26 @@ +import { customAlphabet } from 'nanoid'; + import { flags } from '@/entrypoint/utils/targets'; import { makeEmbed } from '@/providers/base'; +const nanoid = customAlphabet('ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789', 10); + export const doodScraper = makeEmbed({ id: 'dood', name: 'dood', rank: 173, async scrape(ctx) { - function makeTheFunny() { - let result = ''; - const characters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789'; - const charactersLength = characters.length; - - for (let i = 0; i < 10; i++) { - result += characters.charAt(Math.floor(Math.random() * charactersLength)); - } - - return result; - } - const baseUrl = 'https://do0od.com'; const id = ctx.url.split('/d/')[1] || ctx.url.split('/e/')[1]; - console.log(id); const doodData = await ctx.proxiedFetcher(`/e/${id}`, { method: 'GET', baseUrl, }); - // console.log(doodData); - - const dataForLater = doodData.split(`a+"?token=`)[1].split(`"`)[0]; - const path = doodData.split(`$.get('/pass_md5`)[1].split(`'`)[0]; + const dataForLater = doodData.match(/a\+"\?token=([^"]+)/)?.[1]; + const path = doodData.match(/\$\.get\('\/pass_md5([^']+)/)?.[1]; const doodPage = await ctx.proxiedFetcher(`/pass_md5/${path}`, { headers: { @@ -41,39 +30,28 @@ export const doodScraper = makeEmbed({ baseUrl, }); - console.log(`${baseUrl}/pass_md5/${path}`); + const downloadURL = `${doodPage}${nanoid()}?token=${dataForLater}${Date.now()}`; - console.log(doodPage); + if (!downloadURL) throw new Error('dood source not found'); - // const doodPage = await ctx.proxiedFetcher(`/download/${path}`, { method: 'GET', baseUrl }); - // console.log(doodPage); - const downloadURL = `${doodPage}${makeTheFunny()}?token=${dataForLater}${Date.now()}`; - - if (downloadURL) { - return { - stream: [ - { - id: 'primary', - type: 'file', - url: downloadURL, - flags: [flags.CORS_ALLOWED], - captions: [], - qualities: { - unknown: { - type: 'mp4', - url: downloadURL, - preferredHeaders: { - referer: 'https://do0od.com/', - 'content-type': 'video/mp4', - range: 'bytes=0-', - }, + return { + stream: [ + { + id: 'primary', + type: 'file', + flags: [], + captions: [], + qualities: { + unknown: { + type: 'mp4', + url: downloadURL, + headers: { + referer: 'https://do0od.com/', }, }, }, - ], - }; - } - - throw new Error('wootly source not found'); + }, + ], + }; }, }); diff --git a/src/providers/embeds/wootly.ts b/src/providers/embeds/wootly.ts index d02962d..701fae9 100644 --- a/src/providers/embeds/wootly.ts +++ b/src/providers/embeds/wootly.ts @@ -15,7 +15,7 @@ export const wootlyScraper = makeEmbed({ readHeaders: ['Set-Cookie'], }); - const wootssesCookie = wootlyData.headers.get('Set-Cookie')?.split(';')[0].split('wootsses=')[1]; + const wootssesCookie = wootlyData.headers.get('Set-Cookie')?.match(/wootsses=([^;]+)/)?.[1]; let $ = load(wootlyData.body); // load the html data const iframeSrc = $('iframe').attr('src') ?? ''; @@ -28,7 +28,7 @@ export const wootlyScraper = makeEmbed({ }, }); - const woozCookie = woozCookieRequest.headers.get('Set-Cookie')?.split(';')[0].split('wooz=')[1]; + const woozCookie = woozCookieRequest.headers.get('Set-Cookie')?.match(/wooz=([^;]+)/)?.[1]; const iframeData = await ctx.proxiedFetcher(iframeSrc, { method: 'POST', @@ -45,8 +45,10 @@ export const wootlyScraper = makeEmbed({ // Regular expressions to match the variables - const tk = scriptText.split('tk=')[1].split(';')[0].replaceAll('"', '').replaceAll(' ', ''); - const vd = scriptText.split('vd=')[1].split(',')[0].replaceAll('"', '').replaceAll(' ', ''); + const tk = scriptText.match(/tk=([^;]+)/)?.[0].replace(/tk=|["\s]/g, ''); + const vd = scriptText.match(/vd=([^,]+)/)?.[0].replace(/vd=|["\s]/g, ''); + + if (!tk || !vd) throw new Error('wootly source not found'); const url = await ctx.proxiedFetcher(`/grabd`, { baseUrl, @@ -57,26 +59,24 @@ export const wootlyScraper = makeEmbed({ }, }); - if (url) { - return { - stream: [ - { - id: 'primary', - type: 'file', - url, - flags: [flags.CORS_ALLOWED, flags.IP_LOCKED], - captions: [], - qualities: { - unknown: { - type: 'mp4', - url, - }, + if (!url) throw new Error('wootly source not found'); + + return { + stream: [ + { + id: 'primary', + type: 'file', + url, + flags: [flags.IP_LOCKED], + captions: [], + qualities: { + unknown: { + type: 'mp4', + url, }, }, - ], - }; - } - - throw new Error('wootly source not found'); + }, + ], + }; }, }); diff --git a/src/providers/sources/goojara/getEmbeds.ts b/src/providers/sources/goojara/getEmbeds.ts index d0a96a4..de2148a 100644 --- a/src/providers/sources/goojara/getEmbeds.ts +++ b/src/providers/sources/goojara/getEmbeds.ts @@ -8,16 +8,14 @@ export async function getEmbeds(ctx: ScrapeContext, id: string): Promise href && href.includes('https://ww1.goojara.to/go.php')); - const embeds = await Promise.all( - embedRedirectURLs.map((url) => - ctx.fetcher - .full(url, { - headers: { - cookie: `aGooz=${aGoozCookie}; ${RandomCookieName}=${RandomCookieValue};`, - Referer: 'https://ww1.goojara.to/eJwD5z', - }, - method: 'GET', - }) - .then((result) => { - if (result) { - const embedId = ['wootly', 'upstream', 'mixdrop', 'dood'].find((a) => result.finalUrl.includes(a)); - return embedId ? { embedId, url: result.finalUrl } : null; - } - return null; - }) - .catch(() => null), + const embedPages = await Promise.all( + embedRedirectURLs.map( + (url) => + ctx.fetcher + .full(url, { + headers: { + cookie: `aGooz=${aGoozCookie}; ${RandomCookieName}=${RandomCookieValue};`, + Referer: 'https://ww1.goojara.to/eJwD5z', + }, + method: 'GET', + }) + .catch(() => null), // Handle errors gracefully ), - ).then((results) => results.filter((result) => result !== null) as EmbedsResult); + ); - return embeds; + // Initialize an array to hold the results + const results = []; + + // Process each page result + for (const result of embedPages) { + // Ensure there's a result to process + if (result) { + // Attempt to find a matching embed ID + const embedId = ['wootly', 'upstream', 'mixdrop', 'dood'].find((a) => result.finalUrl.includes(a)); + // If an embed ID is found, push the result to the results array + if (embedId) { + results.push({ embedId, url: result.finalUrl }); + } + } + } + + return results; } diff --git a/src/providers/sources/goojara/index.ts b/src/providers/sources/goojara/index.ts index bd1e03f..ea85d3d 100644 --- a/src/providers/sources/goojara/index.ts +++ b/src/providers/sources/goojara/index.ts @@ -22,7 +22,7 @@ async function universalScraper(ctx: ShowScrapeContext | MovieScrapeContext): Pr export const goojaraScraper = makeSourcerer({ id: 'goojara', name: 'goojara', - rank: 69, + rank: 225, flags: [], scrapeShow: universalScraper, scrapeMovie: universalScraper, diff --git a/src/providers/sources/goojara/type.ts b/src/providers/sources/goojara/type.ts index 732ba4e..b4b4751 100644 --- a/src/providers/sources/goojara/type.ts +++ b/src/providers/sources/goojara/type.ts @@ -1,33 +1,8 @@ -// ! Types -interface BaseConfig { - /** The website's slug. Formatted as `1839578-person-of-interest-2011` */ - slug: string; - /** Type of request */ - type: 'show' | 'movie'; - /** Hash */ - hash: string; - /** Hash expiry */ - expires: number; -} -interface TvConfig extends BaseConfig { - /** Type of request */ - type: 'show'; - /** The episode ID for a TV show. Given in search and URL */ - episodeId: string; -} -interface MovieConfig extends BaseConfig { - /** Type of request */ - type: 'movie'; - /** Movie's id */ - slug: string; -} -export type Config = MovieConfig | TvConfig; - export type EmbedsResult = { embedId: string; url: string }[]; export interface Result { title: string; - slug: any; + slug: string; year: string; type: string; id_movie?: string; diff --git a/src/providers/sources/goojara/util.ts b/src/providers/sources/goojara/util.ts index 0fcba4b..4922ffa 100644 --- a/src/providers/sources/goojara/util.ts +++ b/src/providers/sources/goojara/util.ts @@ -1,4 +1,5 @@ import { load } from 'cheerio'; +import { stringify } from 'crypto-js/enc-base64'; import { MovieMedia, ShowMedia } from '@/entrypoint/utils/media'; import { compareMedia } from '@/utils/compare'; @@ -8,30 +9,24 @@ import { NotFoundError } from '@/utils/errors'; import { getEmbeds } from './getEmbeds'; import { EmbedsResult, Result } from './type'; +let data; + +// The cookie for this headerData doesn't matter, Goojara just checks it's there. T const headersData = { - accept: '*/*', - 'accept-language': 'en-US,en;q=0.9', 'content-type': 'application/x-www-form-urlencoded', - 'sec-ch-ua': '"Not_A Brand";v="8", "Chromium";v="120", "Google Chrome";v="120"', - 'sec-ch-ua-mobile': '?0', - 'sec-ch-ua-platform': '"Windows"', - 'sec-fetch-dest': 'empty', - 'sec-fetch-mode': 'cors', - 'sec-fetch-site': 'same-origin', cookie: `aGooz=t9pmkdtef1b3lg3pmo1u2re816; bd9aa48e=0d7b89e8c79844e9df07a2; _b414=2151C6B12E2A88379AFF2C0DD65AC8298DEC2BF4; 9d287aaa=8f32ad589e1c4288fe152f`, Referer: 'https://www.goojara.to/', - 'Referrer-Policy': 'strict-origin-when-cross-origin', }; export async function searchAndFindMedia( ctx: ScrapeContext, media: MovieMedia | ShowMedia, ): Promise { - const data = await ctx.fetcher(`/xhrr.php`, { + data = await ctx.fetcher(`/xhrr.php`, { baseUrl: 'https://www.goojara.to', headers: headersData, method: 'POST', - body: `q=${media.title}`, + body: new URLSearchParams({ q: media.title }), }); const $ = load(data); @@ -48,6 +43,8 @@ export async function searchAndFindMedia( const year = yearMatch ? yearMatch[1] : ''; const slug = $(element).find('a').attr('href')?.split('/')[3]; + if (!slug) throw new NotFoundError('Not found'); + if (media.type === type) { results.push({ title, year, slug, type }); } @@ -68,7 +65,7 @@ export async function scrapeIds( if (media.type === 'movie') { id = result.slug; } else if (media.type === 'show') { - const data = await ctx.fetcher(`/${result.slug}`, { + data = await ctx.fetcher(`/${result.slug}`, { baseUrl: 'https://www.goojara.to', headers: headersData, method: 'GET', @@ -76,18 +73,20 @@ export async function scrapeIds( const $1 = load(data); - const dataId = $1('#seon').data('id'); + const dataId = $1('#seon').attr('data-id'); - const data2 = await ctx.fetcher(`/xhrc.php`, { + if (!dataId) throw NotFoundError; + + data = await ctx.fetcher(`/xhrc.php`, { baseUrl: 'https://ww1.goojara.to', headers: headersData, method: 'POST', - body: `s=${media.season.number}&t=${dataId}`, + body: new URLSearchParams({ s: media.season.number.toString(), t: dataId }), }); let episodeId = ''; - const $2 = load(data2); + const $2 = load(data); $2('.seho').each((index, element) => { // Extracting the episode number as a string From d6624efc5a42b77a0b1d3d53a1294ed704d079fa Mon Sep 17 00:00:00 2001 From: MemeCornucopia <148788549+MemeCornucopia@users.noreply.github.com> Date: Sun, 28 Jan 2024 14:27:07 -0500 Subject: [PATCH 17/37] Added Base URL Declaration --- src/providers/sources/goojara/getEmbeds.ts | 10 +++++----- src/providers/sources/goojara/type.ts | 4 ++++ src/providers/sources/goojara/util.ts | 8 ++++---- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/src/providers/sources/goojara/getEmbeds.ts b/src/providers/sources/goojara/getEmbeds.ts index de2148a..33b7e16 100644 --- a/src/providers/sources/goojara/getEmbeds.ts +++ b/src/providers/sources/goojara/getEmbeds.ts @@ -2,13 +2,13 @@ import { load } from 'cheerio'; import { ScrapeContext } from '@/utils/context'; -import { EmbedsResult } from './type'; +import { EmbedsResult, baseUrl, baseUrl2 } from './type'; export async function getEmbeds(ctx: ScrapeContext, id: string): Promise { const data = await ctx.fetcher.full(`/${id}`, { - baseUrl: 'https://ww1.goojara.to', + baseUrl: baseUrl2, headers: { - Referer: 'https://www.goojara.to/', + Referer: baseUrl, }, readHeaders: ['Set-Cookie'], method: 'GET', @@ -23,7 +23,7 @@ export async function getEmbeds(ctx: ScrapeContext, id: string): Promise $(element).attr('href')) .get() - .filter((href) => href && href.includes('https://ww1.goojara.to/go.php')); + .filter((href) => href && href.includes(`${baseUrl2}/go.php`)); const embedPages = await Promise.all( embedRedirectURLs.map( @@ -32,7 +32,7 @@ export async function getEmbeds(ctx: ScrapeContext, id: string): Promise { data = await ctx.fetcher(`/xhrr.php`, { - baseUrl: 'https://www.goojara.to', + baseUrl, headers: headersData, method: 'POST', body: new URLSearchParams({ q: media.title }), @@ -66,7 +66,7 @@ export async function scrapeIds( id = result.slug; } else if (media.type === 'show') { data = await ctx.fetcher(`/${result.slug}`, { - baseUrl: 'https://www.goojara.to', + baseUrl, headers: headersData, method: 'GET', }); @@ -78,7 +78,7 @@ export async function scrapeIds( if (!dataId) throw NotFoundError; data = await ctx.fetcher(`/xhrc.php`, { - baseUrl: 'https://ww1.goojara.to', + baseUrl, headers: headersData, method: 'POST', body: new URLSearchParams({ s: media.season.number.toString(), t: dataId }), From 63bbd8b858ddd77f0290c00c9d8cb5cbcdcd64df Mon Sep 17 00:00:00 2001 From: MemeCornucopia <148788549+MemeCornucopia@users.noreply.github.com> Date: Sun, 28 Jan 2024 15:41:40 -0500 Subject: [PATCH 18/37] All changes done --- src/providers/all.ts | 3 --- src/providers/embeds/dood.ts | 2 -- src/providers/embeds/wootly.ts | 1 - src/providers/sources/goojara/getEmbeds.ts | 2 +- src/providers/sources/goojara/util.ts | 5 ++--- 5 files changed, 3 insertions(+), 10 deletions(-) diff --git a/src/providers/all.ts b/src/providers/all.ts index fe1be15..0f3ce9e 100644 --- a/src/providers/all.ts +++ b/src/providers/all.ts @@ -24,12 +24,9 @@ import { ridooScraper } from './embeds/ridoo'; import { smashyStreamDScraper } from './embeds/smashystream/dued'; import { smashyStreamFScraper } from './embeds/smashystream/video1'; import { vidplayScraper } from './embeds/vidplay'; - import { wootlyScraper } from './embeds/wootly'; import { goojaraScraper } from './sources/goojara'; - import { ridooMoviesScraper } from './sources/ridomovies'; - import { smashyStreamScraper } from './sources/smashystream'; import { vidSrcToScraper } from './sources/vidsrcto'; diff --git a/src/providers/embeds/dood.ts b/src/providers/embeds/dood.ts index d977fdc..e2c8534 100644 --- a/src/providers/embeds/dood.ts +++ b/src/providers/embeds/dood.ts @@ -32,8 +32,6 @@ export const doodScraper = makeEmbed({ const downloadURL = `${doodPage}${nanoid()}?token=${dataForLater}${Date.now()}`; - if (!downloadURL) throw new Error('dood source not found'); - return { stream: [ { diff --git a/src/providers/embeds/wootly.ts b/src/providers/embeds/wootly.ts index 701fae9..72202e1 100644 --- a/src/providers/embeds/wootly.ts +++ b/src/providers/embeds/wootly.ts @@ -66,7 +66,6 @@ export const wootlyScraper = makeEmbed({ { id: 'primary', type: 'file', - url, flags: [flags.IP_LOCKED], captions: [], qualities: { diff --git a/src/providers/sources/goojara/getEmbeds.ts b/src/providers/sources/goojara/getEmbeds.ts index 33b7e16..d72a0ff 100644 --- a/src/providers/sources/goojara/getEmbeds.ts +++ b/src/providers/sources/goojara/getEmbeds.ts @@ -41,7 +41,7 @@ export async function getEmbeds(ctx: ScrapeContext, id: string): Promise(`/xhrc.php`, { baseUrl, From 53f4d6699a65e1de043f5f715d4ae29f1a2c2d7b Mon Sep 17 00:00:00 2001 From: MemeCornucopia <148788549+MemeCornucopia@users.noreply.github.com> Date: Sun, 28 Jan 2024 15:44:09 -0500 Subject: [PATCH 19/37] Added reasoning --- src/providers/sources/goojara/util.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/providers/sources/goojara/util.ts b/src/providers/sources/goojara/util.ts index 1b19b50..e04f3ae 100644 --- a/src/providers/sources/goojara/util.ts +++ b/src/providers/sources/goojara/util.ts @@ -74,7 +74,7 @@ export async function scrapeIds( const dataId = $1('#seon').attr('data-id'); - if (!dataId) throw new NotFoundError(); + if (!dataId) throw new NotFoundError('Not found'); data = await ctx.fetcher(`/xhrc.php`, { baseUrl, From 46253bad0c7650fe136599e90e7b3a061393e0d7 Mon Sep 17 00:00:00 2001 From: MemeCornucopia <148788549+MemeCornucopia@users.noreply.github.com> Date: Sun, 28 Jan 2024 15:49:10 -0500 Subject: [PATCH 20/37] Fixed linting errors --- src/providers/embeds/dood.ts | 1 - src/providers/sources/goojara/util.ts | 1 - 2 files changed, 2 deletions(-) diff --git a/src/providers/embeds/dood.ts b/src/providers/embeds/dood.ts index e2c8534..faa79d8 100644 --- a/src/providers/embeds/dood.ts +++ b/src/providers/embeds/dood.ts @@ -1,6 +1,5 @@ import { customAlphabet } from 'nanoid'; -import { flags } from '@/entrypoint/utils/targets'; import { makeEmbed } from '@/providers/base'; const nanoid = customAlphabet('ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789', 10); diff --git a/src/providers/sources/goojara/util.ts b/src/providers/sources/goojara/util.ts index e04f3ae..30d63a9 100644 --- a/src/providers/sources/goojara/util.ts +++ b/src/providers/sources/goojara/util.ts @@ -1,5 +1,4 @@ import { load } from 'cheerio'; -import { stringify } from 'crypto-js/enc-base64'; import { MovieMedia, ShowMedia } from '@/entrypoint/utils/media'; import { compareMedia } from '@/utils/compare'; From c423a51b4cb489b1408a2344a85d298c862e84eb Mon Sep 17 00:00:00 2001 From: lonelil <51315646+lonelil@users.noreply.github.com> Date: Mon, 29 Jan 2024 20:36:36 +0800 Subject: [PATCH 21/37] add nepu provider --- src/providers/all.ts | 2 + src/providers/sources/nepu/index.ts | 78 +++++++++++++++++++++++++++++ src/providers/sources/nepu/types.ts | 9 ++++ 3 files changed, 89 insertions(+) create mode 100644 src/providers/sources/nepu/index.ts create mode 100644 src/providers/sources/nepu/types.ts diff --git a/src/providers/all.ts b/src/providers/all.ts index 13e9039..edbe50b 100644 --- a/src/providers/all.ts +++ b/src/providers/all.ts @@ -23,6 +23,7 @@ import { ridooScraper } from './embeds/ridoo'; import { smashyStreamDScraper } from './embeds/smashystream/dued'; import { smashyStreamFScraper } from './embeds/smashystream/video1'; import { vidplayScraper } from './embeds/vidplay'; +import { nepuScraper } from './sources/nepu'; import { ridooMoviesScraper } from './sources/ridomovies'; import { smashyStreamScraper } from './sources/smashystream'; import { vidSrcToScraper } from './sources/vidsrcto'; @@ -41,6 +42,7 @@ export function gatherAllSources(): Array { smashyStreamScraper, ridooMoviesScraper, vidSrcToScraper, + nepuScraper, ]; } diff --git a/src/providers/sources/nepu/index.ts b/src/providers/sources/nepu/index.ts new file mode 100644 index 0000000..d6e1505 --- /dev/null +++ b/src/providers/sources/nepu/index.ts @@ -0,0 +1,78 @@ +import { load } from 'cheerio'; + +import { flags } from '@/entrypoint/utils/targets'; +import { SourcererOutput, makeSourcerer } from '@/providers/base'; +import { MovieScrapeContext, ShowScrapeContext } from '@/utils/context'; +import { NotFoundError } from '@/utils/errors'; + +import { SearchResults } from './types'; + +const nepuBase = 'https://nepu.to'; +const nepuReferer = `${nepuBase}/`; + +const universalScraper = async (ctx: MovieScrapeContext | ShowScrapeContext) => { + const searchResultRequest = await ctx.proxiedFetcher('/ajax/posts', { + baseUrl: nepuBase, + query: { + q: ctx.media.title, + }, + }); + // json isn't parsed by searchResultRequest for some reason. + const searchResult = JSON.parse(searchResultRequest) as SearchResults; + + const show = searchResult.data[0]; + if (!show) throw new NotFoundError('No watchable item found'); + + let videoUrl = show.url; + + if (ctx.media.type === 'show') { + videoUrl = `${show.url}/season/${ctx.media.season.number}/episode/${ctx.media.episode.number}`; + } + + const videoPage = await ctx.proxiedFetcher(videoUrl, { + baseUrl: nepuBase, + }); + const videoPage$ = load(videoPage); + const embedId = videoPage$('a.btn-service').attr('data-embed'); + + if (!embedId) throw new NotFoundError('No embed found.'); + + const playerPage = await ctx.proxiedFetcher('/ajax/embed', { + method: 'POST', + baseUrl: nepuBase, + headers: { + 'Content-Type': 'application/x-www-form-urlencoded', + }, + body: `id=${embedId}`, + }); + + const streamUrl = playerPage.match(/"file":"(http[^"]+)"/); + + if (!streamUrl) throw new NotFoundError('No stream found.'); + + return { + embeds: [], + stream: [ + { + id: 'primary', + captions: [], + playlist: streamUrl[1], + type: 'hls', + flags: [flags.CORS_ALLOWED], + preferredHeaders: { + Origin: nepuBase, + Referer: nepuReferer, + }, + }, + ], + } as SourcererOutput; +}; + +export const nepuScraper = makeSourcerer({ + id: 'nepu', + name: 'Nepu', + rank: 111, + flags: [flags.CORS_ALLOWED], + scrapeMovie: universalScraper, + scrapeShow: universalScraper, +}); diff --git a/src/providers/sources/nepu/types.ts b/src/providers/sources/nepu/types.ts new file mode 100644 index 0000000..7c8275b --- /dev/null +++ b/src/providers/sources/nepu/types.ts @@ -0,0 +1,9 @@ +export type SearchResults = { + data: { + id: number; + name: string; + second_name: string; + url: string; + type: 'Movie' | 'Serie'; + }[]; +}; From 43faeec1e7e7bee10391a589043e60a09e0eed18 Mon Sep 17 00:00:00 2001 From: lonelil <51315646+lonelil@users.noreply.github.com> Date: Mon, 29 Jan 2024 20:58:46 +0800 Subject: [PATCH 22/37] add searchResult filtering --- src/providers/sources/nepu/index.ts | 13 +++++++++++-- src/providers/sources/nepu/types.ts | 1 - 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/providers/sources/nepu/index.ts b/src/providers/sources/nepu/index.ts index d6e1505..f5fcd83 100644 --- a/src/providers/sources/nepu/index.ts +++ b/src/providers/sources/nepu/index.ts @@ -6,6 +6,7 @@ import { MovieScrapeContext, ShowScrapeContext } from '@/utils/context'; import { NotFoundError } from '@/utils/errors'; import { SearchResults } from './types'; +import { compareTitle } from '@/utils/compare'; const nepuBase = 'https://nepu.to'; const nepuReferer = `${nepuBase}/`; @@ -17,10 +18,18 @@ const universalScraper = async (ctx: MovieScrapeContext | ShowScrapeContext) => q: ctx.media.title, }, }); - // json isn't parsed by searchResultRequest for some reason. + + // json isn't parsed by proxiedFetcher due to content-type being text/html. const searchResult = JSON.parse(searchResultRequest) as SearchResults; - const show = searchResult.data[0]; + const show = searchResult.data.find((item) => { + if (!item) return false; + if (ctx.media.type === 'movie' && item.type !== "Movie") return false; + if (ctx.media.type === "show" && item.type !== "Serie") return false + + return compareTitle(ctx.media.title, item.name); + }); + if (!show) throw new NotFoundError('No watchable item found'); let videoUrl = show.url; diff --git a/src/providers/sources/nepu/types.ts b/src/providers/sources/nepu/types.ts index 7c8275b..200995a 100644 --- a/src/providers/sources/nepu/types.ts +++ b/src/providers/sources/nepu/types.ts @@ -2,7 +2,6 @@ export type SearchResults = { data: { id: number; name: string; - second_name: string; url: string; type: 'Movie' | 'Serie'; }[]; From bd1e3234d1a5bc322c6bb3bc50ccaf255ad450da Mon Sep 17 00:00:00 2001 From: lonelil <51315646+lonelil@users.noreply.github.com> Date: Mon, 29 Jan 2024 21:54:47 +0800 Subject: [PATCH 23/37] fix linting issues --- src/providers/sources/nepu/index.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/providers/sources/nepu/index.ts b/src/providers/sources/nepu/index.ts index f5fcd83..5a69615 100644 --- a/src/providers/sources/nepu/index.ts +++ b/src/providers/sources/nepu/index.ts @@ -2,11 +2,11 @@ import { load } from 'cheerio'; import { flags } from '@/entrypoint/utils/targets'; import { SourcererOutput, makeSourcerer } from '@/providers/base'; +import { compareTitle } from '@/utils/compare'; import { MovieScrapeContext, ShowScrapeContext } from '@/utils/context'; import { NotFoundError } from '@/utils/errors'; import { SearchResults } from './types'; -import { compareTitle } from '@/utils/compare'; const nepuBase = 'https://nepu.to'; const nepuReferer = `${nepuBase}/`; @@ -24,8 +24,8 @@ const universalScraper = async (ctx: MovieScrapeContext | ShowScrapeContext) => const show = searchResult.data.find((item) => { if (!item) return false; - if (ctx.media.type === 'movie' && item.type !== "Movie") return false; - if (ctx.media.type === "show" && item.type !== "Serie") return false + if (ctx.media.type === 'movie' && item.type !== 'Movie') return false; + if (ctx.media.type === 'show' && item.type !== 'Serie') return false; return compareTitle(ctx.media.title, item.name); }); From c8ad3387c53903eb9c6637e3464538d85011bc23 Mon Sep 17 00:00:00 2001 From: MemeCornucopia <148788549+MemeCornucopia@users.noreply.github.com> Date: Mon, 29 Jan 2024 13:32:21 -0500 Subject: [PATCH 24/37] Added Cookie.ts. Changed request changes. --- src/entrypoint/utils/cookie.ts | 22 ++++++++++++++++++++++ src/providers/embeds/wootly.ts | 14 ++++++++------ src/providers/sources/goojara/getEmbeds.ts | 12 +++++++----- 3 files changed, 37 insertions(+), 11 deletions(-) create mode 100644 src/entrypoint/utils/cookie.ts diff --git a/src/entrypoint/utils/cookie.ts b/src/entrypoint/utils/cookie.ts new file mode 100644 index 0000000..4f2fcbe --- /dev/null +++ b/src/entrypoint/utils/cookie.ts @@ -0,0 +1,22 @@ +export interface Cookie { + name: string; + value: string; +} + +export function makeCookieHeader(cookies: Record): string { + return Object.entries(cookies) + .map(([name, value]) => `${name}=${value}`) + .join('; '); +} + +export function parseSetCookie(headerValue: string): Record { + const cookies: Record = {}; + const parts = headerValue.split(/; */); + for (const part of parts) { + const [name, value] = part.split('='); + if (name && value) { + cookies[name] = { name, value }; + } + } + return cookies; +} diff --git a/src/providers/embeds/wootly.ts b/src/providers/embeds/wootly.ts index 72202e1..e748707 100644 --- a/src/providers/embeds/wootly.ts +++ b/src/providers/embeds/wootly.ts @@ -1,5 +1,6 @@ import { load } from 'cheerio'; +import { makeCookieHeader, parseSetCookie } from '@/entrypoint/utils/cookie'; import { flags } from '@/entrypoint/utils/targets'; import { makeEmbed } from '@/providers/base'; @@ -15,7 +16,8 @@ export const wootlyScraper = makeEmbed({ readHeaders: ['Set-Cookie'], }); - const wootssesCookie = wootlyData.headers.get('Set-Cookie')?.match(/wootsses=([^;]+)/)?.[1]; + const cookies = parseSetCookie(wootlyData.headers.get('Set-Cookie') || ''); + const wootssesCookie = cookies.wootsses.value; let $ = load(wootlyData.body); // load the html data const iframeSrc = $('iframe').attr('src') ?? ''; @@ -24,17 +26,18 @@ export const wootlyScraper = makeEmbed({ method: 'GET', readHeaders: ['Set-Cookie'], headers: { - cookie: `wootsses=${wootssesCookie};`, + cookie: makeCookieHeader({ wootsses: wootssesCookie }), }, }); - const woozCookie = woozCookieRequest.headers.get('Set-Cookie')?.match(/wooz=([^;]+)/)?.[1]; + const woozCookies = parseSetCookie(woozCookieRequest.headers.get('Set-Cookie') || ''); + const woozCookie = woozCookies.wooz.value; const iframeData = await ctx.proxiedFetcher(iframeSrc, { method: 'POST', body: new URLSearchParams({ qdf: '1' }), headers: { - cookie: `wooz=${woozCookie}`, + cookie: makeCookieHeader({ wooz: woozCookie }), Referer: iframeSrc, }, }); @@ -44,7 +47,6 @@ export const wootlyScraper = makeEmbed({ const scriptText = $('script').html() ?? ''; // Regular expressions to match the variables - const tk = scriptText.match(/tk=([^;]+)/)?.[0].replace(/tk=|["\s]/g, ''); const vd = scriptText.match(/vd=([^,]+)/)?.[0].replace(/vd=|["\s]/g, ''); @@ -55,7 +57,7 @@ export const wootlyScraper = makeEmbed({ query: { t: tk, id: vd }, method: 'GET', headers: { - cookie: `wooz=${woozCookie}; wootsses=${wootssesCookie};`, + cookie: makeCookieHeader({ wooz: woozCookie, wootsses: wootssesCookie }), }, }); diff --git a/src/providers/sources/goojara/getEmbeds.ts b/src/providers/sources/goojara/getEmbeds.ts index d72a0ff..2b05a2d 100644 --- a/src/providers/sources/goojara/getEmbeds.ts +++ b/src/providers/sources/goojara/getEmbeds.ts @@ -1,5 +1,6 @@ import { load } from 'cheerio'; +import { makeCookieHeader, parseSetCookie } from '@/entrypoint/utils/cookie'; import { ScrapeContext } from '@/utils/context'; import { EmbedsResult, baseUrl, baseUrl2 } from './type'; @@ -14,7 +15,8 @@ export async function getEmbeds(ctx: ScrapeContext, id: string): Promise result.finalUrl.includes(a)); - // If an embed ID is found, push the result to the results array if (embedId) { results.push({ embedId, url: result.finalUrl }); } From 8b149458e060c648539a677f05581680216ee860 Mon Sep 17 00:00:00 2001 From: MemeCornucopia <148788549+MemeCornucopia@users.noreply.github.com> Date: Mon, 29 Jan 2024 13:43:32 -0500 Subject: [PATCH 25/37] Use cookie and set-cookie-parser --- package-lock.json | 36 ++++++++++++++++++++++++++++++++-- package.json | 4 ++++ src/entrypoint/utils/cookie.ts | 18 ++++++++--------- 3 files changed, 46 insertions(+), 12 deletions(-) diff --git a/package-lock.json b/package-lock.json index 3facef2..f5536e8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,26 +1,30 @@ { "name": "@movie-web/providers", - "version": "2.1.0", + "version": "2.1.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@movie-web/providers", - "version": "2.1.0", + "version": "2.1.1", "license": "MIT", "dependencies": { "cheerio": "^1.0.0-rc.12", + "cookie": "^0.6.0", "crypto-js": "^4.1.1", "form-data": "^4.0.0", "iso-639-1": "^3.1.0", "nanoid": "^3.3.6", "node-fetch": "^2.7.0", + "set-cookie-parser": "^2.6.0", "unpacker": "^1.0.1" }, "devDependencies": { + "@types/cookie": "^0.6.0", "@types/crypto-js": "^4.1.1", "@types/node-fetch": "^2.6.6", "@types/randombytes": "^2.0.1", + "@types/set-cookie-parser": "^2.4.7", "@types/spinnies": "^0.5.1", "@typescript-eslint/eslint-plugin": "^5.60.0", "@typescript-eslint/parser": "^5.60.0", @@ -687,6 +691,12 @@ "@types/chai": "*" } }, + "node_modules/@types/cookie": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/@types/cookie/-/cookie-0.6.0.tgz", + "integrity": "sha512-4Kh9a6B2bQciAhf7FSuMRRkUWecJgJu9nPnx3yzpsfXX/c50REIqpHY4C82bXP90qrLtXtkDxTZosYO3UpOwlA==", + "dev": true + }, "node_modules/@types/crypto-js": { "version": "4.2.1", "dev": true, @@ -751,6 +761,15 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/set-cookie-parser": { + "version": "2.4.7", + "resolved": "https://registry.npmjs.org/@types/set-cookie-parser/-/set-cookie-parser-2.4.7.tgz", + "integrity": "sha512-+ge/loa0oTozxip6zmhRIk8Z/boU51wl9Q6QdLZcokIGMzY5lFXYy/x7Htj2HTC6/KZP1hUbZ1ekx8DYXICvWg==", + "dev": true, + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/spinnies": { "version": "0.5.3", "dev": true, @@ -1764,6 +1783,14 @@ "dev": true, "license": "MIT" }, + "node_modules/cookie": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.6.0.tgz", + "integrity": "sha512-U71cyTamuh1CRNCfpGY6to28lxvNwPG4Guz/EVjgf3Jmzv0vlDp1atT9eS5dDjMYHucpHbWns6Lwf3BKz6svdw==", + "engines": { + "node": ">= 0.6" + } + }, "node_modules/cosmiconfig": { "version": "8.3.6", "resolved": "https://registry.npmjs.org/cosmiconfig/-/cosmiconfig-8.3.6.tgz", @@ -4789,6 +4816,11 @@ "node": ">=10" } }, + "node_modules/set-cookie-parser": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/set-cookie-parser/-/set-cookie-parser-2.6.0.tgz", + "integrity": "sha512-RVnVQxTXuerk653XfuliOxBP81Sf0+qfQE73LIYKcyMYHG94AuH0kgrQpRDuTZnSmjpysHmzxJXKNfa6PjFhyQ==" + }, "node_modules/set-function-length": { "version": "1.1.1", "dev": true, diff --git a/package.json b/package.json index 25c4d07..f21eb89 100644 --- a/package.json +++ b/package.json @@ -48,9 +48,11 @@ "prepublishOnly": "npm test && npm run lint" }, "devDependencies": { + "@types/cookie": "^0.6.0", "@types/crypto-js": "^4.1.1", "@types/node-fetch": "^2.6.6", "@types/randombytes": "^2.0.1", + "@types/set-cookie-parser": "^2.4.7", "@types/spinnies": "^0.5.1", "@typescript-eslint/eslint-plugin": "^5.60.0", "@typescript-eslint/parser": "^5.60.0", @@ -80,11 +82,13 @@ }, "dependencies": { "cheerio": "^1.0.0-rc.12", + "cookie": "^0.6.0", "crypto-js": "^4.1.1", "form-data": "^4.0.0", "iso-639-1": "^3.1.0", "nanoid": "^3.3.6", "node-fetch": "^2.7.0", + "set-cookie-parser": "^2.6.0", "unpacker": "^1.0.1" } } diff --git a/src/entrypoint/utils/cookie.ts b/src/entrypoint/utils/cookie.ts index 4f2fcbe..20fd3e4 100644 --- a/src/entrypoint/utils/cookie.ts +++ b/src/entrypoint/utils/cookie.ts @@ -1,3 +1,6 @@ +import cookie from 'cookie'; +import setCookieParser from 'set-cookie-parser'; + export interface Cookie { name: string; value: string; @@ -5,18 +8,13 @@ export interface Cookie { export function makeCookieHeader(cookies: Record): string { return Object.entries(cookies) - .map(([name, value]) => `${name}=${value}`) + .map(([name, value]) => cookie.serialize(name, value)) .join('; '); } export function parseSetCookie(headerValue: string): Record { - const cookies: Record = {}; - const parts = headerValue.split(/; */); - for (const part of parts) { - const [name, value] = part.split('='); - if (name && value) { - cookies[name] = { name, value }; - } - } - return cookies; + const parsedCookies = setCookieParser.parse(headerValue, { + map: true, + }); + return parsedCookies; } From a30881cf5d2ecd03109b6711bd3af2a1814da987 Mon Sep 17 00:00:00 2001 From: lonelil <51315646+lonelil@users.noreply.github.com> Date: Tue, 30 Jan 2024 07:31:43 +0800 Subject: [PATCH 26/37] Add requested changes --- src/providers/sources/nepu/index.ts | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/src/providers/sources/nepu/index.ts b/src/providers/sources/nepu/index.ts index 5a69615..38128d9 100644 --- a/src/providers/sources/nepu/index.ts +++ b/src/providers/sources/nepu/index.ts @@ -9,10 +9,9 @@ import { NotFoundError } from '@/utils/errors'; import { SearchResults } from './types'; const nepuBase = 'https://nepu.to'; -const nepuReferer = `${nepuBase}/`; const universalScraper = async (ctx: MovieScrapeContext | ShowScrapeContext) => { - const searchResultRequest = await ctx.proxiedFetcher('/ajax/posts', { + const searchResultRequest = await ctx.proxiedFetcher('/ajax/posts', { baseUrl: nepuBase, query: { q: ctx.media.title, @@ -38,7 +37,7 @@ const universalScraper = async (ctx: MovieScrapeContext | ShowScrapeContext) => videoUrl = `${show.url}/season/${ctx.media.season.number}/episode/${ctx.media.episode.number}`; } - const videoPage = await ctx.proxiedFetcher(videoUrl, { + const videoPage = await ctx.proxiedFetcher(videoUrl, { baseUrl: nepuBase, }); const videoPage$ = load(videoPage); @@ -46,13 +45,10 @@ const universalScraper = async (ctx: MovieScrapeContext | ShowScrapeContext) => if (!embedId) throw new NotFoundError('No embed found.'); - const playerPage = await ctx.proxiedFetcher('/ajax/embed', { + const playerPage = await ctx.proxiedFetcher('/ajax/embed', { method: 'POST', baseUrl: nepuBase, - headers: { - 'Content-Type': 'application/x-www-form-urlencoded', - }, - body: `id=${embedId}`, + body: new URLSearchParams({ id: embedId }), }); const streamUrl = playerPage.match(/"file":"(http[^"]+)"/); @@ -68,10 +64,6 @@ const universalScraper = async (ctx: MovieScrapeContext | ShowScrapeContext) => playlist: streamUrl[1], type: 'hls', flags: [flags.CORS_ALLOWED], - preferredHeaders: { - Origin: nepuBase, - Referer: nepuReferer, - }, }, ], } as SourcererOutput; From 7061e808ba5b34e85c1532d1d948157e06a4f52e Mon Sep 17 00:00:00 2001 From: mrjvs Date: Tue, 30 Jan 2024 19:13:23 +0100 Subject: [PATCH 27/37] Move cookie file --- src/providers/embeds/wootly.ts | 2 +- src/providers/sources/goojara/getEmbeds.ts | 2 +- src/{entrypoint => }/utils/cookie.ts | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename src/{entrypoint => }/utils/cookie.ts (100%) diff --git a/src/providers/embeds/wootly.ts b/src/providers/embeds/wootly.ts index e748707..0119926 100644 --- a/src/providers/embeds/wootly.ts +++ b/src/providers/embeds/wootly.ts @@ -1,8 +1,8 @@ import { load } from 'cheerio'; -import { makeCookieHeader, parseSetCookie } from '@/entrypoint/utils/cookie'; import { flags } from '@/entrypoint/utils/targets'; import { makeEmbed } from '@/providers/base'; +import { makeCookieHeader, parseSetCookie } from '@/utils/cookie'; export const wootlyScraper = makeEmbed({ id: 'wootly', diff --git a/src/providers/sources/goojara/getEmbeds.ts b/src/providers/sources/goojara/getEmbeds.ts index 2b05a2d..50f574a 100644 --- a/src/providers/sources/goojara/getEmbeds.ts +++ b/src/providers/sources/goojara/getEmbeds.ts @@ -1,7 +1,7 @@ import { load } from 'cheerio'; -import { makeCookieHeader, parseSetCookie } from '@/entrypoint/utils/cookie'; import { ScrapeContext } from '@/utils/context'; +import { makeCookieHeader, parseSetCookie } from '@/utils/cookie'; import { EmbedsResult, baseUrl, baseUrl2 } from './type'; diff --git a/src/entrypoint/utils/cookie.ts b/src/utils/cookie.ts similarity index 100% rename from src/entrypoint/utils/cookie.ts rename to src/utils/cookie.ts From f683cbf8ac875cc53c92294804abc83a68c74a3b Mon Sep 17 00:00:00 2001 From: Jorrin Date: Tue, 30 Jan 2024 19:56:58 +0100 Subject: [PATCH 28/37] Fixed VidSrcTo crashing if no subtitles are found --- src/providers/sources/vidsrcto/index.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/providers/sources/vidsrcto/index.ts b/src/providers/sources/vidsrcto/index.ts index b85b068..6912ae1 100644 --- a/src/providers/sources/vidsrcto/index.ts +++ b/src/providers/sources/vidsrcto/index.ts @@ -46,7 +46,10 @@ const universalScraper = async (ctx: ShowScrapeContext | MovieScrapeContext): Pr } // Originally Filemoon does not have subtitles. But we can use the ones from Vidplay. - const subtitleUrl = new URL(embedUrls.find((v) => v.includes('sub.info')) ?? '').searchParams.get('sub.info'); + const urlWithSubtitles = embedUrls.find((v) => v.includes('sub.info')); + let subtitleUrl: string | null = null; + if (urlWithSubtitles) subtitleUrl = new URL(urlWithSubtitles).searchParams.get('sub.info'); + for (const source of sources.result) { if (source.title === 'Vidplay') { const embedUrl = embedUrls.find((v) => v.includes('vidplay')); From fc052a9f08ec4eddc3ad487c9fc6920d3c9a4c6b Mon Sep 17 00:00:00 2001 From: lonelil <51315646+lonelil@users.noreply.github.com> Date: Wed, 31 Jan 2024 03:38:30 +0800 Subject: [PATCH 29/37] re-add headers --- src/providers/sources/nepu/index.ts | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/providers/sources/nepu/index.ts b/src/providers/sources/nepu/index.ts index 38128d9..1b31429 100644 --- a/src/providers/sources/nepu/index.ts +++ b/src/providers/sources/nepu/index.ts @@ -1,6 +1,5 @@ import { load } from 'cheerio'; -import { flags } from '@/entrypoint/utils/targets'; import { SourcererOutput, makeSourcerer } from '@/providers/base'; import { compareTitle } from '@/utils/compare'; import { MovieScrapeContext, ShowScrapeContext } from '@/utils/context'; @@ -9,6 +8,7 @@ import { NotFoundError } from '@/utils/errors'; import { SearchResults } from './types'; const nepuBase = 'https://nepu.to'; +const nepuReferer = `${nepuBase}/`; const universalScraper = async (ctx: MovieScrapeContext | ShowScrapeContext) => { const searchResultRequest = await ctx.proxiedFetcher('/ajax/posts', { @@ -63,7 +63,11 @@ const universalScraper = async (ctx: MovieScrapeContext | ShowScrapeContext) => captions: [], playlist: streamUrl[1], type: 'hls', - flags: [flags.CORS_ALLOWED], + flags: [], + headers: { + Origin: nepuBase, + Referer: nepuReferer, + }, }, ], } as SourcererOutput; @@ -73,7 +77,7 @@ export const nepuScraper = makeSourcerer({ id: 'nepu', name: 'Nepu', rank: 111, - flags: [flags.CORS_ALLOWED], + flags: [], scrapeMovie: universalScraper, scrapeShow: universalScraper, }); From cb44f663cad8188d235cddb89af07df4458f48de Mon Sep 17 00:00:00 2001 From: lonelil <51315646+lonelil@users.noreply.github.com> Date: Wed, 31 Jan 2024 23:39:44 +0800 Subject: [PATCH 30/37] Change embed selector --- src/providers/sources/nepu/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/providers/sources/nepu/index.ts b/src/providers/sources/nepu/index.ts index 1b31429..576c789 100644 --- a/src/providers/sources/nepu/index.ts +++ b/src/providers/sources/nepu/index.ts @@ -41,7 +41,7 @@ const universalScraper = async (ctx: MovieScrapeContext | ShowScrapeContext) => baseUrl: nepuBase, }); const videoPage$ = load(videoPage); - const embedId = videoPage$('a.btn-service').attr('data-embed'); + const embedId = videoPage$('a[data-embed]').attr('data-embed'); if (!embedId) throw new NotFoundError('No embed found.'); From 96a455c4c260da33b97d03e866c077d693ab2cae Mon Sep 17 00:00:00 2001 From: mrjvs Date: Wed, 31 Jan 2024 16:50:49 +0100 Subject: [PATCH 31/37] fix linting --- src/providers/all.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/providers/all.ts b/src/providers/all.ts index 4f3defd..9a17e46 100644 --- a/src/providers/all.ts +++ b/src/providers/all.ts @@ -24,9 +24,9 @@ import { ridooScraper } from './embeds/ridoo'; import { smashyStreamDScraper } from './embeds/smashystream/dued'; import { smashyStreamFScraper } from './embeds/smashystream/video1'; import { vidplayScraper } from './embeds/vidplay'; -import { nepuScraper } from './sources/nepu'; import { wootlyScraper } from './embeds/wootly'; import { goojaraScraper } from './sources/goojara'; +import { nepuScraper } from './sources/nepu'; import { ridooMoviesScraper } from './sources/ridomovies'; import { smashyStreamScraper } from './sources/smashystream'; import { vidSrcToScraper } from './sources/vidsrcto'; From fe90bdaa9ae03693f9c17537b8d0a5359a13ca23 Mon Sep 17 00:00:00 2001 From: Jorrin Date: Sat, 3 Feb 2024 01:28:37 +0100 Subject: [PATCH 32/37] fix keys being cached for 5 minutes --- src/providers/embeds/vidplay/common.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/providers/embeds/vidplay/common.ts b/src/providers/embeds/vidplay/common.ts index 3eb19a9..d19cc02 100644 --- a/src/providers/embeds/vidplay/common.ts +++ b/src/providers/embeds/vidplay/common.ts @@ -9,8 +9,12 @@ export const referer = `${vidplayBase}/`; // Full credits to @Ciarands! export const getDecryptionKeys = async (ctx: EmbedScrapeContext): Promise => { - const res = await ctx.fetcher('https://raw.githubusercontent.com/Ciarands/vidsrc-keys/main/keys.json'); - return JSON.parse(res); + const res = await ctx.fetcher('https://github.com/Ciarands/vidsrc-keys/blob/main/keys.json'); + const regex = /"rawLines":\s*\[([\s\S]*?)\]/; + const rawLines = res.match(regex)?.[1]; + if (!rawLines) throw new Error('No keys found'); + const keys = JSON.parse(`${rawLines.substring(1).replace(/\\"/g, '"')}]`); + return keys; }; export const getEncodedId = async (ctx: EmbedScrapeContext) => { From 16dcca270c79de5cd9f0a657c56fe80f705c0ab4 Mon Sep 17 00:00:00 2001 From: Jorrin Date: Sat, 3 Feb 2024 02:29:09 +0100 Subject: [PATCH 33/37] that should be proxied obviously --- src/providers/embeds/vidplay/common.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/providers/embeds/vidplay/common.ts b/src/providers/embeds/vidplay/common.ts index d19cc02..f10681a 100644 --- a/src/providers/embeds/vidplay/common.ts +++ b/src/providers/embeds/vidplay/common.ts @@ -9,7 +9,7 @@ export const referer = `${vidplayBase}/`; // Full credits to @Ciarands! export const getDecryptionKeys = async (ctx: EmbedScrapeContext): Promise => { - const res = await ctx.fetcher('https://github.com/Ciarands/vidsrc-keys/blob/main/keys.json'); + const res = await ctx.proxiedFetcher('https://github.com/Ciarands/vidsrc-keys/blob/main/keys.json'); const regex = /"rawLines":\s*\[([\s\S]*?)\]/; const rawLines = res.match(regex)?.[1]; if (!rawLines) throw new Error('No keys found'); From f4bc3831e1126b4ef6edf1a6881ebff16d2566b5 Mon Sep 17 00:00:00 2001 From: mrjvs Date: Wed, 7 Feb 2024 19:24:41 +0100 Subject: [PATCH 34/37] Add thumbnail track option to response --- src/providers/streams.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/providers/streams.ts b/src/providers/streams.ts index f4674c6..17567a4 100644 --- a/src/providers/streams.ts +++ b/src/providers/streams.ts @@ -8,10 +8,16 @@ export type StreamFile = { export type Qualities = 'unknown' | '360' | '480' | '720' | '1080' | '4k'; +type ThumbnailTrack = { + type: 'vtt'; + url: string; +}; + type StreamCommon = { id: string; // only unique per output flags: Flags[]; captions: Caption[]; + thumbnailTrack?: ThumbnailTrack; headers?: Record; // these headers HAVE to be set to watch the stream preferredHeaders?: Record; // these headers are optional, would improve the stream }; From 8a5392f08b21a7834a59914e7f358dbdd7cc7379 Mon Sep 17 00:00:00 2001 From: William Oldham Date: Sat, 10 Feb 2024 17:48:51 +0000 Subject: [PATCH 35/37] Bump version to 2.2 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index f21eb89..3ea5af9 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@movie-web/providers", - "version": "2.1.1", + "version": "2.2.0", "description": "Package that contains all the providers of movie-web", "main": "./lib/index.umd.js", "types": "./lib/index.d.ts", From 311eefb74f8c8b650e7f01ca6d6fa26928786024 Mon Sep 17 00:00:00 2001 From: William Oldham Date: Sat, 10 Feb 2024 19:12:00 +0000 Subject: [PATCH 36/37] Update package-lock.json --- package-lock.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index f5536e8..033f07f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@movie-web/providers", - "version": "2.1.1", + "version": "2.2.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@movie-web/providers", - "version": "2.1.1", + "version": "2.2.0", "license": "MIT", "dependencies": { "cheerio": "^1.0.0-rc.12", From 805e520f824c9f7b3580837642fd5c88030ffd4b Mon Sep 17 00:00:00 2001 From: William Oldham Date: Sat, 10 Feb 2024 19:23:30 +0000 Subject: [PATCH 37/37] Add 2.2 changelog and re-order sources --- .docs/content/1.get-started/4.changelog.md | 10 ++++++++++ src/providers/sources/lookmovie/index.ts | 1 + src/providers/sources/showbox/index.ts | 2 +- src/providers/sources/vidsrcto/index.ts | 2 +- 4 files changed, 13 insertions(+), 2 deletions(-) diff --git a/.docs/content/1.get-started/4.changelog.md b/.docs/content/1.get-started/4.changelog.md index f03e407..1e18e51 100644 --- a/.docs/content/1.get-started/4.changelog.md +++ b/.docs/content/1.get-started/4.changelog.md @@ -2,6 +2,16 @@ title: 'Changelog' --- +# Version 2.2.0 +- Fixed vidsrc.me URL decoding. +- Added ridomovies with Ridoo and Closeload embed. +- Added Goojara.to source. +- Fixed VidSrcTo crashing if no subtitles are found. +- Added Nepu Provider. +- Added vidcloud to flixhq and zoechip. +- Add thumbnail track option to response (Not supported by any providers yet). +- Disabled Lookmovie and swapped Showbox and VidSrcTo in ranking. + # Version 2.1.1 - Fixed vidplay decryption keys being wrong and switched the domain to one that works diff --git a/src/providers/sources/lookmovie/index.ts b/src/providers/sources/lookmovie/index.ts index 6fbe6de..db24196 100644 --- a/src/providers/sources/lookmovie/index.ts +++ b/src/providers/sources/lookmovie/index.ts @@ -32,6 +32,7 @@ async function universalScraper(ctx: MovieScrapeContext | ShowScrapeContext): Pr export const lookmovieScraper = makeSourcerer({ id: 'lookmovie', name: 'LookMovie', + disabled: true, rank: 700, flags: [flags.IP_LOCKED], scrapeShow: universalScraper, diff --git a/src/providers/sources/showbox/index.ts b/src/providers/sources/showbox/index.ts index c8a834a..ce3eceb 100644 --- a/src/providers/sources/showbox/index.ts +++ b/src/providers/sources/showbox/index.ts @@ -41,7 +41,7 @@ async function comboScraper(ctx: ShowScrapeContext | MovieScrapeContext): Promis export const showboxScraper = makeSourcerer({ id: 'showbox', name: 'Showbox', - rank: 300, + rank: 400, flags: [flags.CORS_ALLOWED, flags.CF_BLOCKED], scrapeShow: comboScraper, scrapeMovie: comboScraper, diff --git a/src/providers/sources/vidsrcto/index.ts b/src/providers/sources/vidsrcto/index.ts index 6912ae1..94edc3d 100644 --- a/src/providers/sources/vidsrcto/index.ts +++ b/src/providers/sources/vidsrcto/index.ts @@ -83,5 +83,5 @@ export const vidSrcToScraper = makeSourcerer({ scrapeMovie: universalScraper, scrapeShow: universalScraper, flags: [], - rank: 400, + rank: 300, });