From 8b7e840814a5322eb90470ae49432036f30fb566 Mon Sep 17 00:00:00 2001 From: Jorrin Date: Wed, 27 Dec 2023 02:03:53 +0100 Subject: [PATCH 01/43] vidsrcto draft --- .eslintrc.js | 1 + src/dev-cli/tmdb.ts | 2 + src/providers/all.ts | 6 +++ src/providers/embeds/filemoon.ts | 34 ++++++++++++++ src/providers/embeds/vidplay/common.ts | 53 ++++++++++++++++++++++ src/providers/embeds/vidplay/index.ts | 32 +++++++++++++ src/providers/embeds/vidplay/types.ts | 11 +++++ src/providers/sources/vidsrcto/common.ts | 51 +++++++++++++++++++++ src/providers/sources/vidsrcto/index.ts | 58 ++++++++++++++++++++++++ src/providers/sources/vidsrcto/types.ts | 15 ++++++ 10 files changed, 263 insertions(+) create mode 100644 src/providers/embeds/filemoon.ts create mode 100644 src/providers/embeds/vidplay/common.ts create mode 100644 src/providers/embeds/vidplay/index.ts create mode 100644 src/providers/embeds/vidplay/types.ts create mode 100644 src/providers/sources/vidsrcto/common.ts create mode 100644 src/providers/sources/vidsrcto/index.ts create mode 100644 src/providers/sources/vidsrcto/types.ts diff --git a/.eslintrc.js b/.eslintrc.js index 0e7322b..2bb81f9 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -18,6 +18,7 @@ module.exports = { }, plugins: ['@typescript-eslint', 'import', 'prettier'], rules: { + 'no-bitwise': 'off', 'no-underscore-dangle': 'off', '@typescript-eslint/no-explicit-any': 'off', 'no-console': 'off', diff --git a/src/dev-cli/tmdb.ts b/src/dev-cli/tmdb.ts index 7490336..90e87f8 100644 --- a/src/dev-cli/tmdb.ts +++ b/src/dev-cli/tmdb.ts @@ -45,6 +45,7 @@ export async function getMovieMediaDetails(id: string): Promise { title: movie.title, releaseYear: Number(movie.release_date.split('-')[0]), tmdbId: id, + imdbId: movie.imdb_id, }; } @@ -91,5 +92,6 @@ export async function getShowMediaDetails(id: string, seasonNumber: string, epis number: season.season_number, tmdbId: season.id, }, + imdbId: series.imdb_id, }; } diff --git a/src/providers/all.ts b/src/providers/all.ts index d1e7885..07458b9 100644 --- a/src/providers/all.ts +++ b/src/providers/all.ts @@ -14,9 +14,12 @@ import { remotestreamScraper } from '@/providers/sources/remotestream'; import { showboxScraper } from '@/providers/sources/showbox/index'; import { zoechipScraper } from '@/providers/sources/zoechip'; +import { fileMoonScraper } from './embeds/filemoon'; import { smashyStreamDScraper } from './embeds/smashystream/dued'; import { smashyStreamFScraper } from './embeds/smashystream/video1'; +import { vidplayScraper } from './embeds/vidplay'; import { smashyStreamScraper } from './sources/smashystream'; +import { vidSrcToScraper } from './sources/vidsrcto'; export function gatherAllSources(): Array { // all sources are gathered here @@ -29,6 +32,7 @@ export function gatherAllSources(): Array { zoechipScraper, lookmovieScraper, smashyStreamScraper, + vidSrcToScraper, ]; } @@ -44,5 +48,7 @@ export function gatherAllEmbeds(): Array { mixdropScraper, smashyStreamFScraper, smashyStreamDScraper, + fileMoonScraper, + vidplayScraper, ]; } diff --git a/src/providers/embeds/filemoon.ts b/src/providers/embeds/filemoon.ts new file mode 100644 index 0000000..9b092a2 --- /dev/null +++ b/src/providers/embeds/filemoon.ts @@ -0,0 +1,34 @@ +import { unpack } from 'unpacker'; + +import { flags } from '@/entrypoint/utils/targets'; + +import { makeEmbed } from '../base'; + +const evalCodeRegex = /eval\((.*)\)/g; +const fileRegex = /file:"(.*?)"/g; + +export const fileMoonScraper = makeEmbed({ + id: 'filemoon', + name: 'FileMoon', + rank: 501, + scrape: async (ctx) => { + const embedRes = await ctx.fetcher(ctx.url); + const evalCode = evalCodeRegex.exec(embedRes); + if (!evalCode) throw new Error('Failed to find eval code'); + const unpacked = unpack(evalCode[1]); + const file = fileRegex.exec(unpacked); + if (!file?.[1]) throw new Error('Failed to find file'); + + return { + stream: [ + { + id: 'primary', + type: 'hls', + playlist: file[1], + flags: [flags.CORS_ALLOWED], + captions: [], + }, + ], + }; + }, +}); diff --git a/src/providers/embeds/vidplay/common.ts b/src/providers/embeds/vidplay/common.ts new file mode 100644 index 0000000..9fbc8d2 --- /dev/null +++ b/src/providers/embeds/vidplay/common.ts @@ -0,0 +1,53 @@ +import { createCipheriv } from 'crypto'; +import { Buffer } from 'node:buffer'; + +import { EmbedScrapeContext } from '@/utils/context'; + +export const vidplayBase = 'https://vidplay.site'; + +export const getDecryptionKeys = async (ctx: EmbedScrapeContext): Promise => { + const res = await ctx.fetcher( + 'https://raw.githubusercontent.com/Claudemirovsky/worstsource-keys/keys/keys.json', + ); + return JSON.parse(res); +}; + +// TODO: Fix this so its cross platform compatible +export const getEncodedId = async (ctx: EmbedScrapeContext) => { + const id = ctx.url.split('/e/')[1].split('?')[0]; + const keys = await getDecryptionKeys(ctx); + const c1 = createCipheriv('rc4', Buffer.from(keys[0]), ''); + const c2 = createCipheriv('rc4', Buffer.from(keys[1]), ''); + + let input = Buffer.from(id); + input = Buffer.concat([c1.update(input), c1.final()]); + input = Buffer.concat([c2.update(input), c2.final()]); + + return input.toString('base64').replace('/', '_'); +}; + +export const getFuTokenKey = async (ctx: EmbedScrapeContext) => { + const id = await getEncodedId(ctx); + console.log(`ENCODED ID: ${id}`); + const fuTokenRes = await ctx.proxiedFetcher('/futoken', { + baseUrl: vidplayBase, + headers: { + referer: ctx.url, + }, + }); + const fuKey = fuTokenRes.match(/var\s+k\s*=\s*'([^']+)'/)?.[1]; + console.log(`FU KEY: ${fuKey}`); + if (!fuKey) throw new Error('No fuKey found'); + const tokens = []; + for (let i = 0; i < id.length; i += 1) { + tokens.push(fuKey.charCodeAt(i % fuKey.length) + id.charCodeAt(i)); + } + console.log(`${fuKey},${tokens.join(',')}`); + return `${fuKey},${tokens.join(',')}`; +}; + +export const getFileUrl = async (ctx: EmbedScrapeContext) => { + console.log(ctx.url); + const fuToken = await getFuTokenKey(ctx); + return `${vidplayBase}/mediainfo/${fuToken}?${ctx.url.split('?')[1]}`; +}; diff --git a/src/providers/embeds/vidplay/index.ts b/src/providers/embeds/vidplay/index.ts new file mode 100644 index 0000000..93243e5 --- /dev/null +++ b/src/providers/embeds/vidplay/index.ts @@ -0,0 +1,32 @@ +import { makeEmbed } from '@/providers/base'; + +import { getFileUrl } from './common'; +import { VidplaySourceResponse } from './types'; + +export const vidplayScraper = makeEmbed({ + id: 'vidplay', + name: 'VidPlay', + rank: 499, + scrape: async (ctx) => { + const fileUrl = await getFileUrl(ctx); + console.log(fileUrl); + const fileUrlRes = await ctx.proxiedFetcher(`${fileUrl}&autostart=true`, { + headers: { + referer: ctx.url, + }, + }); + const source = fileUrlRes.result.sources[0].file; + + return { + stream: [ + { + id: 'primary', + type: 'hls', + playlist: source, + flags: [], + captions: [], + }, + ], + }; + }, +}); diff --git a/src/providers/embeds/vidplay/types.ts b/src/providers/embeds/vidplay/types.ts new file mode 100644 index 0000000..14c33a8 --- /dev/null +++ b/src/providers/embeds/vidplay/types.ts @@ -0,0 +1,11 @@ +export type VidplaySourceResponse = { + result: { + sources: { + file: string; + tracks: { + file: string; + kind: string; + }[]; + }[]; + }; +}; diff --git a/src/providers/sources/vidsrcto/common.ts b/src/providers/sources/vidsrcto/common.ts new file mode 100644 index 0000000..e31b58d --- /dev/null +++ b/src/providers/sources/vidsrcto/common.ts @@ -0,0 +1,51 @@ +const DECRYPTION_KEY = '8z5Ag5wgagfsOuhz'; + +export const decodeBase64UrlSafe = (str: string) => { + const standardizedInput = str.replace(/_/g, '/').replace(/-/g, '+'); + + const binaryData = Buffer.from(standardizedInput, 'base64').toString('binary'); + + const bytes = new Uint8Array(binaryData.length); + for (let i = 0; i < bytes.length; i += 1) { + bytes[i] = binaryData.charCodeAt(i); + } + + return bytes; +}; + +export const decode = (str: Uint8Array) => { + const keyBytes = new TextEncoder().encode(DECRYPTION_KEY); + + let j = 0; + const s = new Uint8Array(256); + for (let i = 0; i < 256; i += 1) { + s[i] = i; + } + + for (let i = 0, k = 0; i < 256; i += 1) { + j = (j + s[i] + keyBytes[k % keyBytes.length]) & 0xff; + [s[i], s[j]] = [s[j], s[i]]; + k += 1; + } + + const decoded = new Uint8Array(str.length); + let i = 0; + let k = 0; + for (let index = 0; index < str.length; index += 1) { + i = (i + 1) & 0xff; + k = (k + s[i]) & 0xff; + [s[i], s[k]] = [s[k], s[i]]; + const t = (s[i] + s[k]) & 0xff; + decoded[index] = str[index] ^ s[t]; + } + + return decoded; +}; + +export const decryptSourceUrl = (sourceUrl: string) => { + const encoded = decodeBase64UrlSafe(sourceUrl); + const decoded = decode(encoded); + const decodedText = new TextDecoder().decode(decoded); + + return decodeURIComponent(decodeURIComponent(decodedText)); +}; diff --git a/src/providers/sources/vidsrcto/index.ts b/src/providers/sources/vidsrcto/index.ts new file mode 100644 index 0000000..eec22d4 --- /dev/null +++ b/src/providers/sources/vidsrcto/index.ts @@ -0,0 +1,58 @@ +import { load } from 'cheerio'; + +import { SourcererEmbed, SourcererOutput, makeSourcerer } from '@/providers/base'; +import { MovieScrapeContext, ShowScrapeContext } from '@/utils/context'; + +import { decryptSourceUrl } from './common'; +import { SourceResult, SourcesResult } from './types'; + +const vidSrcToBase = 'https://vidsrc.to'; + +const universalScraper = async (ctx: ShowScrapeContext | MovieScrapeContext): Promise => { + const imdbId = ctx.media.imdbId; + const url = + ctx.media.type === 'movie' + ? `${vidSrcToBase}/embed/movie/${imdbId}` + : `${vidSrcToBase}}/embed/tv/${imdbId}/${ctx.media.season.number}/${ctx.media.episode.number}`; + + const mainPage = await ctx.fetcher(url); + const mainPage$ = load(mainPage); + const dataId = mainPage$('a[data-id]').attr('data-id'); + if (!dataId) throw new Error('No data-id found'); + const sources = await ctx.fetcher(`/ajax/embed/episode/${dataId}/sources`, { + baseUrl: vidSrcToBase, + }); + if (sources.status !== 200) throw new Error('No sources found'); + + const embeds: SourcererEmbed[] = []; + for (const source of sources.result) { + const sourceRes = await ctx.fetcher(`/ajax/embed/source/${source.id}`, { + baseUrl: vidSrcToBase, + }); + const decryptedUrl = decryptSourceUrl(sourceRes.result.url); + if (source.title === 'Filemoon') { + embeds.push({ + embedId: 'filemoon', + url: decryptedUrl, + }); + } + if (source.title === 'Vidplay') { + embeds.push({ + embedId: 'vidplay', + url: decryptedUrl, + }); + } + } + return { + embeds, + }; +}; + +export const vidSrcToScraper = makeSourcerer({ + id: 'vidsrcto', + name: 'VidSrcTo', + scrapeMovie: universalScraper, + scrapeShow: universalScraper, + flags: [], + rank: 500, +}); diff --git a/src/providers/sources/vidsrcto/types.ts b/src/providers/sources/vidsrcto/types.ts new file mode 100644 index 0000000..0694b15 --- /dev/null +++ b/src/providers/sources/vidsrcto/types.ts @@ -0,0 +1,15 @@ +export type VidSrcToResponse = { + status: number; + result: T; +}; + +export type SourcesResult = VidSrcToResponse< + { + id: string; + title: 'Filemoon' | 'Vidplay'; + }[] +>; + +export type SourceResult = VidSrcToResponse<{ + url: string; +}>; From f39aaca3e32fd11a3fca7f56711675f331592473 Mon Sep 17 00:00:00 2001 From: Jorrin Date: Wed, 27 Dec 2023 20:35:08 +0100 Subject: [PATCH 02/43] fix vidplay, add captions to filemoon --- src/providers/embeds/filemoon.ts | 34 --------------- src/providers/embeds/filemoon/index.ts | 56 +++++++++++++++++++++++++ src/providers/embeds/filemoon/types.ts | 5 +++ src/providers/embeds/vidplay/common.ts | 53 +++++++++++++++-------- src/providers/embeds/vidplay/index.ts | 29 +++++++++++-- src/providers/embeds/vidplay/types.ts | 26 ++++++++---- src/providers/sources/vidsrcto/index.ts | 29 +++++++++---- 7 files changed, 160 insertions(+), 72 deletions(-) delete mode 100644 src/providers/embeds/filemoon.ts create mode 100644 src/providers/embeds/filemoon/index.ts create mode 100644 src/providers/embeds/filemoon/types.ts diff --git a/src/providers/embeds/filemoon.ts b/src/providers/embeds/filemoon.ts deleted file mode 100644 index 9b092a2..0000000 --- a/src/providers/embeds/filemoon.ts +++ /dev/null @@ -1,34 +0,0 @@ -import { unpack } from 'unpacker'; - -import { flags } from '@/entrypoint/utils/targets'; - -import { makeEmbed } from '../base'; - -const evalCodeRegex = /eval\((.*)\)/g; -const fileRegex = /file:"(.*?)"/g; - -export const fileMoonScraper = makeEmbed({ - id: 'filemoon', - name: 'FileMoon', - rank: 501, - scrape: async (ctx) => { - const embedRes = await ctx.fetcher(ctx.url); - const evalCode = evalCodeRegex.exec(embedRes); - if (!evalCode) throw new Error('Failed to find eval code'); - const unpacked = unpack(evalCode[1]); - const file = fileRegex.exec(unpacked); - if (!file?.[1]) throw new Error('Failed to find file'); - - return { - stream: [ - { - id: 'primary', - type: 'hls', - playlist: file[1], - flags: [flags.CORS_ALLOWED], - captions: [], - }, - ], - }; - }, -}); diff --git a/src/providers/embeds/filemoon/index.ts b/src/providers/embeds/filemoon/index.ts new file mode 100644 index 0000000..584be7e --- /dev/null +++ b/src/providers/embeds/filemoon/index.ts @@ -0,0 +1,56 @@ +import { unpack } from 'unpacker'; + +import { flags } from '@/entrypoint/utils/targets'; + +import { SubtitleResult } from './types'; +import { makeEmbed } from '../../base'; +import { Caption, getCaptionTypeFromUrl, labelToLanguageCode } from '../../captions'; + +const evalCodeRegex = /eval\((.*)\)/g; +const fileRegex = /file:"(.*?)"/g; + +export const fileMoonScraper = makeEmbed({ + id: 'filemoon', + name: 'Filemoon', + rank: 501, + scrape: async (ctx) => { + const embedRes = await ctx.fetcher(ctx.url); + const evalCode = evalCodeRegex.exec(embedRes); + if (!evalCode) throw new Error('Failed to find eval code'); + const unpacked = unpack(evalCode[1]); + const file = fileRegex.exec(unpacked); + if (!file?.[1]) throw new Error('Failed to find file'); + + const url = new URL(ctx.url); + const subtitlesLink = url.searchParams.get('sub.info'); + const captions: Caption[] = []; + if (subtitlesLink) { + const captionsResult = await ctx.fetcher(subtitlesLink); + + for (const caption of captionsResult) { + const language = labelToLanguageCode(caption.label); + const captionType = getCaptionTypeFromUrl(caption.file); + if (!language || !captionType) continue; + captions.push({ + id: caption.file, + url: caption.file, + type: captionType, + language, + hasCorsRestrictions: false, + }); + } + } + + return { + stream: [ + { + id: 'primary', + type: 'hls', + playlist: file[1], + flags: [flags.CORS_ALLOWED], + captions, + }, + ], + }; + }, +}); diff --git a/src/providers/embeds/filemoon/types.ts b/src/providers/embeds/filemoon/types.ts new file mode 100644 index 0000000..caa27af --- /dev/null +++ b/src/providers/embeds/filemoon/types.ts @@ -0,0 +1,5 @@ +export type SubtitleResult = { + file: string; + label: string; + kind: string; +}[]; diff --git a/src/providers/embeds/vidplay/common.ts b/src/providers/embeds/vidplay/common.ts index 9fbc8d2..cf2eb0f 100644 --- a/src/providers/embeds/vidplay/common.ts +++ b/src/providers/embeds/vidplay/common.ts @@ -1,10 +1,34 @@ -import { createCipheriv } from 'crypto'; -import { Buffer } from 'node:buffer'; - import { EmbedScrapeContext } from '@/utils/context'; export const vidplayBase = 'https://vidplay.site'; +export function keyPermutation(key: string, data: any) { + const state = Array.from(Array(256).keys()); + let index1 = 0; + for (let i = 0; i < 256; i += 1) { + index1 = (index1 + state[i] + key.charCodeAt(i % key.length)) % 256; + const temp = state[i]; + state[i] = state[index1]; + state[index1] = temp; + } + index1 = 0; + let index2 = 0; + let finalKey = ''; + for (let char = 0; char < data.length; char += 1) { + index1 = (index1 + 1) % 256; + index2 = (index2 + state[index1]) % 256; + const temp = state[index1]; + state[index1] = state[index2]; + state[index2] = temp; + if (typeof data[char] === 'string') { + finalKey += String.fromCharCode(data[char].charCodeAt(0) ^ state[(state[index1] + state[index2]) % 256]); + } else if (typeof data[char] === 'number') { + finalKey += String.fromCharCode(data[char] ^ state[(state[index1] + state[index2]) % 256]); + } + } + return finalKey; +} + export const getDecryptionKeys = async (ctx: EmbedScrapeContext): Promise => { const res = await ctx.fetcher( 'https://raw.githubusercontent.com/Claudemirovsky/worstsource-keys/keys/keys.json', @@ -12,23 +36,19 @@ export const getDecryptionKeys = async (ctx: EmbedScrapeContext): Promise { - const id = ctx.url.split('/e/')[1].split('?')[0]; - const keys = await getDecryptionKeys(ctx); - const c1 = createCipheriv('rc4', Buffer.from(keys[0]), ''); - const c2 = createCipheriv('rc4', Buffer.from(keys[1]), ''); + const url = new URL(ctx.url); + const id = url.pathname.replace('/e/', ''); + const keyList = await getDecryptionKeys(ctx); - let input = Buffer.from(id); - input = Buffer.concat([c1.update(input), c1.final()]); - input = Buffer.concat([c2.update(input), c2.final()]); - - return input.toString('base64').replace('/', '_'); + const decodedId = keyPermutation(keyList[0], id); + const encodedResult = keyPermutation(keyList[1], decodedId); + const base64 = btoa(encodedResult); + return base64.replace('/', '_'); }; export const getFuTokenKey = async (ctx: EmbedScrapeContext) => { const id = await getEncodedId(ctx); - console.log(`ENCODED ID: ${id}`); const fuTokenRes = await ctx.proxiedFetcher('/futoken', { baseUrl: vidplayBase, headers: { @@ -36,18 +56,15 @@ export const getFuTokenKey = async (ctx: EmbedScrapeContext) => { }, }); const fuKey = fuTokenRes.match(/var\s+k\s*=\s*'([^']+)'/)?.[1]; - console.log(`FU KEY: ${fuKey}`); if (!fuKey) throw new Error('No fuKey found'); const tokens = []; for (let i = 0; i < id.length; i += 1) { tokens.push(fuKey.charCodeAt(i % fuKey.length) + id.charCodeAt(i)); } - console.log(`${fuKey},${tokens.join(',')}`); return `${fuKey},${tokens.join(',')}`; }; export const getFileUrl = async (ctx: EmbedScrapeContext) => { - console.log(ctx.url); const fuToken = await getFuTokenKey(ctx); - return `${vidplayBase}/mediainfo/${fuToken}?${ctx.url.split('?')[1]}`; + return `${vidplayBase}/mediainfo/${fuToken}${new URL(ctx.url).search}&autostart=true`; }; diff --git a/src/providers/embeds/vidplay/index.ts b/src/providers/embeds/vidplay/index.ts index 93243e5..425d583 100644 --- a/src/providers/embeds/vidplay/index.ts +++ b/src/providers/embeds/vidplay/index.ts @@ -1,7 +1,8 @@ import { makeEmbed } from '@/providers/base'; +import { Caption, getCaptionTypeFromUrl, labelToLanguageCode } from '@/providers/captions'; import { getFileUrl } from './common'; -import { VidplaySourceResponse } from './types'; +import { SubtitleResult, VidplaySourceResponse } from './types'; export const vidplayScraper = makeEmbed({ id: 'vidplay', @@ -9,14 +10,34 @@ export const vidplayScraper = makeEmbed({ rank: 499, scrape: async (ctx) => { const fileUrl = await getFileUrl(ctx); - console.log(fileUrl); - const fileUrlRes = await ctx.proxiedFetcher(`${fileUrl}&autostart=true`, { + const fileUrlRes = await ctx.proxiedFetcher(fileUrl, { headers: { referer: ctx.url, }, }); + if (typeof fileUrlRes.result === 'number') throw new Error('File not found'); const source = fileUrlRes.result.sources[0].file; + const url = new URL(ctx.url); + const subtitlesLink = url.searchParams.get('sub.info'); + const captions: Caption[] = []; + if (subtitlesLink) { + const captionsResult = await ctx.fetcher(subtitlesLink); + + for (const caption of captionsResult) { + const language = labelToLanguageCode(caption.label); + const captionType = getCaptionTypeFromUrl(caption.file); + if (!language || !captionType) continue; + captions.push({ + id: caption.file, + url: caption.file, + type: captionType, + language, + hasCorsRestrictions: false, + }); + } + } + return { stream: [ { @@ -24,7 +45,7 @@ export const vidplayScraper = makeEmbed({ type: 'hls', playlist: source, flags: [], - captions: [], + captions, }, ], }; diff --git a/src/providers/embeds/vidplay/types.ts b/src/providers/embeds/vidplay/types.ts index 14c33a8..29cde1d 100644 --- a/src/providers/embeds/vidplay/types.ts +++ b/src/providers/embeds/vidplay/types.ts @@ -1,11 +1,19 @@ export type VidplaySourceResponse = { - result: { - sources: { - file: string; - tracks: { - file: string; - kind: string; - }[]; - }[]; - }; + result: + | { + sources: { + file: string; + tracks: { + file: string; + kind: string; + }[]; + }[]; + } + | number; }; + +export type SubtitleResult = { + file: string; + label: string; + kind: string; +}[]; diff --git a/src/providers/sources/vidsrcto/index.ts b/src/providers/sources/vidsrcto/index.ts index eec22d4..8b43e47 100644 --- a/src/providers/sources/vidsrcto/index.ts +++ b/src/providers/sources/vidsrcto/index.ts @@ -25,24 +25,39 @@ const universalScraper = async (ctx: ShowScrapeContext | MovieScrapeContext): Pr if (sources.status !== 200) throw new Error('No sources found'); const embeds: SourcererEmbed[] = []; + const embedUrls = []; for (const source of sources.result) { const sourceRes = await ctx.fetcher(`/ajax/embed/source/${source.id}`, { baseUrl: vidSrcToBase, }); const decryptedUrl = decryptSourceUrl(sourceRes.result.url); - if (source.title === 'Filemoon') { - embeds.push({ - embedId: 'filemoon', - url: decryptedUrl, - }); - } + embedUrls.push(decryptedUrl); + } + + // Originally Filemoon does not have subtitles. But we can use the ones from Vidplay. + const subtitleUrl = new URL(embedUrls.find((v) => v.includes('sub.info')) ?? '').searchParams.get('sub.info'); + for (const source of sources.result) { if (source.title === 'Vidplay') { + const embedUrl = embedUrls.find((v) => v.includes('vidplay')); + if (!embedUrl) continue; embeds.push({ embedId: 'vidplay', - url: decryptedUrl, + url: embedUrl, + }); + } + + if (source.title === 'Filemoon') { + const embedUrl = embedUrls.find((v) => v.includes('filemoon')); + if (!embedUrl) continue; + const fullUrl = new URL(embedUrl); + if (subtitleUrl) fullUrl.searchParams.set('sub.info', subtitleUrl); + embeds.push({ + embedId: 'filemoon', + url: fullUrl.toString(), }); } } + return { embeds, }; From 30e6067a3fa46fe5f75decca38554e7693eabc73 Mon Sep 17 00:00:00 2001 From: Jorrin Date: Wed, 27 Dec 2023 20:36:50 +0100 Subject: [PATCH 03/43] ranks --- src/providers/embeds/filemoon/index.ts | 2 +- src/providers/embeds/vidplay/index.ts | 2 +- src/providers/sources/vidsrcto/index.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/providers/embeds/filemoon/index.ts b/src/providers/embeds/filemoon/index.ts index 584be7e..f1eda29 100644 --- a/src/providers/embeds/filemoon/index.ts +++ b/src/providers/embeds/filemoon/index.ts @@ -12,7 +12,7 @@ const fileRegex = /file:"(.*?)"/g; export const fileMoonScraper = makeEmbed({ id: 'filemoon', name: 'Filemoon', - rank: 501, + rank: 301, scrape: async (ctx) => { const embedRes = await ctx.fetcher(ctx.url); const evalCode = evalCodeRegex.exec(embedRes); diff --git a/src/providers/embeds/vidplay/index.ts b/src/providers/embeds/vidplay/index.ts index 425d583..9225eff 100644 --- a/src/providers/embeds/vidplay/index.ts +++ b/src/providers/embeds/vidplay/index.ts @@ -7,7 +7,7 @@ import { SubtitleResult, VidplaySourceResponse } from './types'; export const vidplayScraper = makeEmbed({ id: 'vidplay', name: 'VidPlay', - rank: 499, + rank: 300, scrape: async (ctx) => { const fileUrl = await getFileUrl(ctx); const fileUrlRes = await ctx.proxiedFetcher(fileUrl, { diff --git a/src/providers/sources/vidsrcto/index.ts b/src/providers/sources/vidsrcto/index.ts index 8b43e47..3b22bbb 100644 --- a/src/providers/sources/vidsrcto/index.ts +++ b/src/providers/sources/vidsrcto/index.ts @@ -69,5 +69,5 @@ export const vidSrcToScraper = makeSourcerer({ scrapeMovie: universalScraper, scrapeShow: universalScraper, flags: [], - rank: 500, + rank: 300, }); From c44d13f0bd9be6307c4d0a43d20726c83f101992 Mon Sep 17 00:00:00 2001 From: Jorrin Date: Wed, 27 Dec 2023 20:42:22 +0100 Subject: [PATCH 04/43] fix ranks --- src/providers/embeds/filemoon/index.ts | 2 +- src/providers/embeds/vidplay/index.ts | 2 +- src/providers/sources/vidsrcto/index.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/providers/embeds/filemoon/index.ts b/src/providers/embeds/filemoon/index.ts index f1eda29..2729b5c 100644 --- a/src/providers/embeds/filemoon/index.ts +++ b/src/providers/embeds/filemoon/index.ts @@ -12,7 +12,7 @@ const fileRegex = /file:"(.*?)"/g; export const fileMoonScraper = makeEmbed({ id: 'filemoon', name: 'Filemoon', - rank: 301, + rank: 400, scrape: async (ctx) => { const embedRes = await ctx.fetcher(ctx.url); const evalCode = evalCodeRegex.exec(embedRes); diff --git a/src/providers/embeds/vidplay/index.ts b/src/providers/embeds/vidplay/index.ts index 9225eff..f7adca2 100644 --- a/src/providers/embeds/vidplay/index.ts +++ b/src/providers/embeds/vidplay/index.ts @@ -7,7 +7,7 @@ import { SubtitleResult, VidplaySourceResponse } from './types'; export const vidplayScraper = makeEmbed({ id: 'vidplay', name: 'VidPlay', - rank: 300, + rank: 401, scrape: async (ctx) => { const fileUrl = await getFileUrl(ctx); const fileUrlRes = await ctx.proxiedFetcher(fileUrl, { diff --git a/src/providers/sources/vidsrcto/index.ts b/src/providers/sources/vidsrcto/index.ts index 3b22bbb..09a0320 100644 --- a/src/providers/sources/vidsrcto/index.ts +++ b/src/providers/sources/vidsrcto/index.ts @@ -69,5 +69,5 @@ export const vidSrcToScraper = makeSourcerer({ scrapeMovie: universalScraper, scrapeShow: universalScraper, flags: [], - rank: 300, + rank: 400, }); From a208aef364d60ea7967f4490f88b309046feb336 Mon Sep 17 00:00:00 2001 From: Jorrin Date: Wed, 27 Dec 2023 21:14:18 +0100 Subject: [PATCH 05/43] remove Buffer --- src/providers/sources/vidsrcto/common.ts | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/providers/sources/vidsrcto/common.ts b/src/providers/sources/vidsrcto/common.ts index e31b58d..fb9890d 100644 --- a/src/providers/sources/vidsrcto/common.ts +++ b/src/providers/sources/vidsrcto/common.ts @@ -2,12 +2,11 @@ const DECRYPTION_KEY = '8z5Ag5wgagfsOuhz'; export const decodeBase64UrlSafe = (str: string) => { const standardizedInput = str.replace(/_/g, '/').replace(/-/g, '+'); + const decodedData = atob(standardizedInput); - const binaryData = Buffer.from(standardizedInput, 'base64').toString('binary'); - - const bytes = new Uint8Array(binaryData.length); + const bytes = new Uint8Array(decodedData.length); for (let i = 0; i < bytes.length; i += 1) { - bytes[i] = binaryData.charCodeAt(i); + bytes[i] = decodedData.charCodeAt(i); } return bytes; From 9b338b6f3bd2eb2d76dffbf2a3a181b9d499efa7 Mon Sep 17 00:00:00 2001 From: Jorrin Date: Thu, 28 Dec 2023 18:39:49 +0100 Subject: [PATCH 06/43] fixed feedback, added external_ids --- src/dev-cli/tmdb.ts | 16 ++++++++++------ src/providers/embeds/vidplay/common.ts | 9 ++++++++- src/providers/sources/vidsrcto/index.ts | 12 +++++++----- 3 files changed, 25 insertions(+), 12 deletions(-) diff --git a/src/dev-cli/tmdb.ts b/src/dev-cli/tmdb.ts index 90e87f8..c03307d 100644 --- a/src/dev-cli/tmdb.ts +++ b/src/dev-cli/tmdb.ts @@ -2,7 +2,7 @@ import { getConfig } from '@/dev-cli/config'; import { MovieMedia, ShowMedia } from '..'; -export async function makeTMDBRequest(url: string): Promise { +export async function makeTMDBRequest(url: string, appendToResponse?: string): Promise { const headers: { accept: 'application/json'; authorization?: string; @@ -10,7 +10,7 @@ export async function makeTMDBRequest(url: string): Promise { accept: 'application/json', }; - let requestURL = url; + const requestURL = new URL(url); const key = getConfig().tmdbApiKey; // * JWT keys always start with ey and are ONLY valid as a header. @@ -19,7 +19,11 @@ export async function makeTMDBRequest(url: string): Promise { if (key.startsWith('ey')) { headers.authorization = `Bearer ${key}`; } else { - requestURL += `?api_key=${key}`; + requestURL.searchParams.append('api_key', key); + } + + if (appendToResponse) { + requestURL.searchParams.append('append_to_response', appendToResponse); } return fetch(requestURL, { @@ -29,7 +33,7 @@ export async function makeTMDBRequest(url: string): Promise { } export async function getMovieMediaDetails(id: string): Promise { - const response = await makeTMDBRequest(`https://api.themoviedb.org/3/movie/${id}`); + const response = await makeTMDBRequest(`https://api.themoviedb.org/3/movie/${id}`, 'external_ids'); const movie = await response.json(); if (movie.success === false) { @@ -52,7 +56,7 @@ export async function getMovieMediaDetails(id: string): Promise { export async function getShowMediaDetails(id: string, seasonNumber: string, episodeNumber: string): Promise { // * TV shows require the TMDB ID for the series, season, and episode // * and the name of the series. Needs multiple requests - let response = await makeTMDBRequest(`https://api.themoviedb.org/3/tv/${id}`); + let response = await makeTMDBRequest(`https://api.themoviedb.org/3/tv/${id}`, 'external_ids'); const series = await response.json(); if (series.success === false) { @@ -92,6 +96,6 @@ export async function getShowMediaDetails(id: string, seasonNumber: string, epis number: season.season_number, tmdbId: season.id, }, - imdbId: series.imdb_id, + imdbId: series.external_ids.imdb_id, }; } diff --git a/src/providers/embeds/vidplay/common.ts b/src/providers/embeds/vidplay/common.ts index cf2eb0f..765edc7 100644 --- a/src/providers/embeds/vidplay/common.ts +++ b/src/providers/embeds/vidplay/common.ts @@ -1,3 +1,4 @@ +import { makeFullUrl } from '@/fetchers/common'; import { EmbedScrapeContext } from '@/utils/context'; export const vidplayBase = 'https://vidplay.site'; @@ -66,5 +67,11 @@ export const getFuTokenKey = async (ctx: EmbedScrapeContext) => { export const getFileUrl = async (ctx: EmbedScrapeContext) => { const fuToken = await getFuTokenKey(ctx); - return `${vidplayBase}/mediainfo/${fuToken}${new URL(ctx.url).search}&autostart=true`; + return makeFullUrl(`/mediainfo/${fuToken}`, { + baseUrl: vidplayBase, + query: { + ...Object.fromEntries(new URL(ctx.url).searchParams.entries()), + autostart: 'true', + }, + }); }; diff --git a/src/providers/sources/vidsrcto/index.ts b/src/providers/sources/vidsrcto/index.ts index 09a0320..5e73bc1 100644 --- a/src/providers/sources/vidsrcto/index.ts +++ b/src/providers/sources/vidsrcto/index.ts @@ -1,5 +1,6 @@ import { load } from 'cheerio'; +import { flags } from '@/entrypoint/utils/targets'; import { SourcererEmbed, SourcererOutput, makeSourcerer } from '@/providers/base'; import { MovieScrapeContext, ShowScrapeContext } from '@/utils/context'; @@ -12,10 +13,11 @@ const universalScraper = async (ctx: ShowScrapeContext | MovieScrapeContext): Pr const imdbId = ctx.media.imdbId; const url = ctx.media.type === 'movie' - ? `${vidSrcToBase}/embed/movie/${imdbId}` - : `${vidSrcToBase}}/embed/tv/${imdbId}/${ctx.media.season.number}/${ctx.media.episode.number}`; - - const mainPage = await ctx.fetcher(url); + ? `/embed/movie/${imdbId}` + : `/embed/tv/${imdbId}/${ctx.media.season.number}/${ctx.media.episode.number}`; + const mainPage = await ctx.fetcher(url, { + baseUrl: vidSrcToBase, + }); const mainPage$ = load(mainPage); const dataId = mainPage$('a[data-id]').attr('data-id'); if (!dataId) throw new Error('No data-id found'); @@ -68,6 +70,6 @@ export const vidSrcToScraper = makeSourcerer({ name: 'VidSrcTo', scrapeMovie: universalScraper, scrapeShow: universalScraper, - flags: [], + flags: [flags.CORS_ALLOWED], rank: 400, }); From a07f54e0cfdae270d82f11c911abf64f6fa91ebd Mon Sep 17 00:00:00 2001 From: Jorrin <43169049+JorrinKievit@users.noreply.github.com> Date: Fri, 29 Dec 2023 22:03:54 +0100 Subject: [PATCH 07/43] add credits to decryption code --- src/providers/embeds/vidplay/common.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/providers/embeds/vidplay/common.ts b/src/providers/embeds/vidplay/common.ts index 765edc7..57b94bb 100644 --- a/src/providers/embeds/vidplay/common.ts +++ b/src/providers/embeds/vidplay/common.ts @@ -3,6 +3,8 @@ import { EmbedScrapeContext } from '@/utils/context'; export const vidplayBase = 'https://vidplay.site'; +// This file is based on https://github.com/Ciarands/vidsrc-to-resolver/blob/960afb11c30aa6497804b4691fb1c401e539cfe7/vidsrc.py#L10 +// Full credits to @Ciarands! export function keyPermutation(key: string, data: any) { const state = Array.from(Array(256).keys()); let index1 = 0; From 1296915a424f49e4a9f58e6637044394339663b8 Mon Sep 17 00:00:00 2001 From: Paul Dee Date: Sun, 31 Dec 2023 03:35:48 +0000 Subject: [PATCH 08/43] spelling/grammar --- .docs/content/1.get-started/1.quick-start.md | 6 +++--- .docs/content/1.get-started/4.changelog.md | 2 +- .docs/content/2.essentials/0.usage-on-x.md | 6 +++--- .docs/content/2.essentials/1.targets.md | 2 +- .docs/content/2.essentials/2.fetchers.md | 18 +++++++++--------- .../2.essentials/3.customize-providers.md | 6 +++--- .docs/content/2.essentials/4.using-streams.md | 18 +++++++++--------- .docs/content/3.in-depth/2.flags.md | 6 +++--- .docs/content/4.extra-topics/0.development.md | 8 ++++---- .../content/5.api-reference/0.makeProviders.md | 6 +++--- .../1.ProviderControlsRunAll.md | 18 +++++++++--------- .../2.ProviderControlsrunSourceScraper.md | 8 ++++---- .../3.ProviderControlsrunEmbedScraper.md | 6 +++--- .../4.ProviderControlslistSources.md | 6 +++--- .../5.ProviderControlslistEmbeds.md | 6 +++--- .../6.ProviderControlsgetMetadata.md | 2 +- .../5.api-reference/7.makeStandardFetcher.md | 2 +- 17 files changed, 63 insertions(+), 63 deletions(-) diff --git a/.docs/content/1.get-started/1.quick-start.md b/.docs/content/1.get-started/1.quick-start.md index 8804083..69a33ed 100644 --- a/.docs/content/1.get-started/1.quick-start.md +++ b/.docs/content/1.get-started/1.quick-start.md @@ -21,7 +21,7 @@ Let's get started with `@movie-web/providers`. First lets install the package. To get started with scraping on the **server**, first you have to make an instance of the providers. ::alert{type="warning"} -This snippet will only work on a **server**, for other environments, check out [Usage on X](../2.essentials/0.usage-on-x.md). +This snippet will only work on a **server**. For other environments, check out [Usage on X](../2.essentials/0.usage-on-x.md). :: ```ts [index.ts (server)] @@ -39,8 +39,8 @@ const providers = makeProviders({ }) ``` -Perfect, this instance of the providers you can reuse everywhere where you need to. -Now lets actually scrape an item: +Perfect. You now have an instance of the providers you can reuse everywhere. +Now let's scrape an item: ```ts [index.ts (server)] // fetch some data from TMDB diff --git a/.docs/content/1.get-started/4.changelog.md b/.docs/content/1.get-started/4.changelog.md index 8e65d9e..fd00c1c 100644 --- a/.docs/content/1.get-started/4.changelog.md +++ b/.docs/content/1.get-started/4.changelog.md @@ -22,7 +22,7 @@ There are breaking changes in this list, make sure to read them thoroughly if yo - Fetchers can now return a full response with headers and everything **New features:** -- Added system to allow scraping ip locked sources through the consistentIpforRequests option. +- Added system to allow scraping IP locked sources through the consistentIpforRequests option. - There is now a `buildProviders()` function that gives a builder for the `ProviderControls`. It's an alternative to `makeProviders()`. - Streams can now return a headers object and a `preferredHeaders` object. which is required and optional headers for when using the stream. diff --git a/.docs/content/2.essentials/0.usage-on-x.md b/.docs/content/2.essentials/0.usage-on-x.md index a8c3e80..5294be4 100644 --- a/.docs/content/2.essentials/0.usage-on-x.md +++ b/.docs/content/2.essentials/0.usage-on-x.md @@ -2,12 +2,12 @@ The library can run in many environments, so it can be tricky to figure out how to set it up. -So here is a checklist, for more specific environments, keep reading below: +Here is a checklist. For more specific environments, keep reading below: - When requests are very restricted (like browser client-side). Configure a proxied fetcher. - - When your requests come from the same device it will be streamed on (Not compatible with proxied fetcher). Set `consistentIpForRequests: true`. + - When your requests come from the same device on which it will be streamed (not compatible with proxied fetcher). Set `consistentIpForRequests: true`. - To set a target. Consult [Targets](./1.targets.md). -To make use of the examples below, You check check out the following pages: +To make use of the examples below, check out the following pages: - [Quick start](../1.get-started/1.quick-start.md) - [Using streams](../2.essentials/4.using-streams.md) diff --git a/.docs/content/2.essentials/1.targets.md b/.docs/content/2.essentials/1.targets.md index cf1858b..08f3c2e 100644 --- a/.docs/content/2.essentials/1.targets.md +++ b/.docs/content/2.essentials/1.targets.md @@ -3,7 +3,7 @@ When creating provider controls, you will immediately be required to choose a target. ::alert{type="warning"} -A target is the device where the stream will be played on. +A target is the device on which the stream will be played. **Where the scraping is run has nothing to do with the target**, only where the stream is finally played in the end is significant in choosing a target. :: diff --git a/.docs/content/2.essentials/2.fetchers.md b/.docs/content/2.essentials/2.fetchers.md index 3fe0738..5f12097 100644 --- a/.docs/content/2.essentials/2.fetchers.md +++ b/.docs/content/2.essentials/2.fetchers.md @@ -1,7 +1,7 @@ # Fetchers -When creating provider controls, it will need you to configure a fetcher. -This comes with some considerations depending on the environment youre running. +When creating provider controls, a fetcher will need to be configured. +Depending on your environment, this can come with some considerations: ## Using `fetch()` In most cases, you can use the `fetch()` API. This will work in newer versions of Node.js (18 and above) and on the browser. @@ -19,7 +19,7 @@ const fetcher = makeStandardFetcher(fetch); ``` ## Using fetchers on the browser -When using this library on a browser, you will need a proxy. Browsers come with many restrictions on when a web request can be made, and to bypass those restrictions, you will need a cors proxy. +When using this library on a browser, you will need a proxy. Browsers restrict when a web request can be made. To bypass those restrictions, you will need a CORS proxy. The movie-web team has a proxy pre-made and pre-configured for you to use. For more information, check out [movie-web/simple-proxy](https://github.com/movie-web/simple-proxy). After installing, you can use this proxy like so: @@ -31,13 +31,13 @@ If you aren't able to use this specific proxy and need to use a different one, y ## Making a derived fetcher -In some rare cases, a custom fetcher will need to be made. This can be quite difficult to do from scratch so it's recommended to base it off an existing fetcher and building your own functionality around it. +In some rare cases, a custom fetcher is necessary. This can be quite difficult to make from scratch so it's recommended to base it off of an existing fetcher and building your own functionality around it. ```ts export function makeCustomFetcher(): Fetcher { const fetcher = makeStandardFetcher(f); const customFetcher: Fetcher = (url, ops) => { - // Do something with the options and url here + // Do something with the options and URL here return fetcher(url, ops); }; @@ -45,19 +45,19 @@ export function makeCustomFetcher(): Fetcher { } ``` -If you need to make your own fetcher for a proxy. Make sure you make it compatible with the following headers: `Set-Cookie`, `Cookie`, `Referer`, `Origin`. Proxied fetchers need to be able to write/read those headers when making a request. +If you need to make your own fetcher for a proxy, ensure you make it compatible with the following headers: `Set-Cookie`, `Cookie`, `Referer`, `Origin`. Proxied fetchers need to be able to write/read those headers when making a request. ## Making a fetcher from scratch -In some even rare cases, you need to make one completely from scratch. +In some rare cases, you need to make a fetcher from scratch. This is the list of features it needs: - Send/read every header - Parse JSON, otherwise parse as text - Send JSON, Formdata or normal strings - - get final destination url + - get final destination URL -It's not recommended to do this at all, but if you have to. You can base your code on the original implementation of `makeStandardFetcher`. Check the out [source code for it here](https://github.com/movie-web/providers/blob/dev/src/fetchers/standardFetch.ts). +It's not recommended to do this at all. If you have to, you can base your code on the original implementation of `makeStandardFetcher`. Check out the [source code for it here](https://github.com/movie-web/providers/blob/dev/src/fetchers/standardFetch.ts). Here is a basic template on how to make your own custom fetcher: diff --git a/.docs/content/2.essentials/3.customize-providers.md b/.docs/content/2.essentials/3.customize-providers.md index a37c846..ecb20c4 100644 --- a/.docs/content/2.essentials/3.customize-providers.md +++ b/.docs/content/2.essentials/3.customize-providers.md @@ -1,6 +1,6 @@ # Customize providers -You make a provider controls in two ways. Either with `makeProviders()` (the simpler option) or with `buildProviders()` (more elaborate and extensive option). +You make the provider controls in two ways. Either with `makeProviders()` (the simpler option) or with `buildProviders()` (more elaborate and extensive option). ## `makeProviders()` (simple) @@ -41,7 +41,7 @@ const providers = buildProviders() ### Adding only select few providers -Not all providers are great quality, so you can make a instance of the controls with only the providers you want. +Not all providers are great quality, so you can make an instance of the controls with only the providers you want. ```ts const providers = buildProviders() @@ -55,7 +55,7 @@ const providers = buildProviders() ### Adding your own scrapers to the providers -If you have your own scraper and still want to use the nice utils of the provider library or just want to add on to the builtin providers. You can add your own custom source. +If you have your own scraper and still want to use the nice utilities of the provider library or just want to add on to the built-in providers, you can add your own custom source. ```ts const providers = buildProviders() diff --git a/.docs/content/2.essentials/4.using-streams.md b/.docs/content/2.essentials/4.using-streams.md index 64ec16f..1cf8ff2 100644 --- a/.docs/content/2.essentials/4.using-streams.md +++ b/.docs/content/2.essentials/4.using-streams.md @@ -12,11 +12,11 @@ All streams have the same common parameters: - `Stream.headers`: Either undefined or a key value object of headers you must set to use the stream. - `Stream.preferredHeaders`: Either undefined or a key value object of headers you may want to set if you want optimal playback - but not required. -Now let's delve deeper into how to actually watch these streams! +Now let's delve deeper into how to watch these streams! ## Streams with type `hls` -HLS streams can be tough to watch, it's not a normal file you can just use. +HLS streams can be tough to watch. They're not normal files you can just use. These streams have an extra property `Stream.playlist` which contains the m3u8 playlist. Here is a code sample of how to use HLS streams in web context using hls.js @@ -39,17 +39,17 @@ Here is a code sample of how to use HLS streams in web context using hls.js ## Streams with type `file` -File streams are quite easy to use, it just returns a new property: `Stream.qualities`. -This property is a map of quality and a stream file. So if you want to get 1080p quality you do `stream["1080"]` to get your stream file. It will return undefined if there is no quality like that. +File streams are quite easy to use, they just return a new property: `Stream.qualities`. +This property is a map of quality and a stream file. So if you want to get 1080p quality you do `stream["1080"]` to get your stream file. It will return undefined if that quality is absent. The possibly qualities are: `unknown`, `360`, `480`, `720`, `1080`, `4k`. -File based streams are garuanteed to always have one quality. +File based streams are always guaranteed to have one quality. Once you get a streamfile, you have the following parameters: - `StreamFile.type`: Right now it can only be `mp4`. - `StreamFile.url`: The URL linking to the video file. -Here is a code sample of how to watch a file based stream the video in a browser: +Here is a code sample of how to watch a file based stream in a browser: ```html @@ -66,9 +66,9 @@ Here is a code sample of how to watch a file based stream the video in a browser ## Streams with headers Streams have both a `Stream.headers` and a `Stream.preferredHeaders`. -The difference between the two is that `Stream.headers` **must** be set in other for the stream to work. While the other one is optional, and can only enhance the quality or performance. +The difference between the two is that `Stream.headers` **must** be set in order for the stream to work. While the other is optional, and enhances the quality or performance. -If your target is set to `BROWSER`. There will never be required headers, as it's not possible to do. +If your target is set to `BROWSER`, headers will never be required, as it's not possible to do. ## Using captions/subtitles @@ -77,7 +77,7 @@ All streams have a list of captions at `Stream.captions`. The structure looks li type Caption = { type: CaptionType; // Language type, either "srt" or "vtt" id: string; // Unique per stream - url: string; // The url pointing to the subtitle file + url: string; // The URL pointing to the subtitle file hasCorsRestrictions: boolean; // If true, you will need to proxy it if you're running in a browser language: string; // Language code of the caption }; diff --git a/.docs/content/3.in-depth/2.flags.md b/.docs/content/3.in-depth/2.flags.md index 6ec7ffa..e5b25e2 100644 --- a/.docs/content/3.in-depth/2.flags.md +++ b/.docs/content/3.in-depth/2.flags.md @@ -1,10 +1,10 @@ # Flags -Flags is the primary way the library seperates entities between different environments. -For example some sources only give back content that has the CORS headers set to allow anyone, so that source gets the flag `CORS_ALLOWED`. Now if you set your target to `BROWSER`, sources without that flag won't even get listed. +Flags is the primary way the library separates entities between different environments. +For example, some sources only give back content that has the CORS headers set to allow anyone, so that source gets the flag `CORS_ALLOWED`. Now if you set your target to `BROWSER`, sources without that flag won't even get listed. This concept is applied in multiple away across the library. ## Flag options - `CORS_ALLOWED`: Headers from the output streams are set to allow any origin. - - `IP_LOCKED`: The streams are locked by ip, requester and watcher must be the same. + - `IP_LOCKED`: The streams are locked by IP: requester and watcher must be the same. diff --git a/.docs/content/4.extra-topics/0.development.md b/.docs/content/4.extra-topics/0.development.md index 8bf65a0..84f6587 100644 --- a/.docs/content/4.extra-topics/0.development.md +++ b/.docs/content/4.extra-topics/0.development.md @@ -15,13 +15,13 @@ TODO ## Testing using the CLI Testing can be quite difficult for this library, unit tests can't really be made because of the unreliable nature of scrapers. -But manually testing by writing an entrypoint is also really annoying. +But manually testing by writing an entry-point is also really annoying. -Our solution is to make a CLI that you can use to run the scrapers, for everything else there are unit tests. +Our solution is to make a CLI that you can use to run the scrapers. For everything else there are unit tests. ### Setup -Make a `.env` file in the root of the repository and add a TMDB api key: `MOVIE_WEB_TMDB_API_KEY=KEY_HERE`. -Then make sure you've ran `npm i` to get all the dependencies. +Make a `.env` file in the root of the repository and add a TMDB API key: `MOVIE_WEB_TMDB_API_KEY=KEY_HERE`. +Then make sure you've run `npm i` to get all the dependencies. ### Mode 1 - interactive diff --git a/.docs/content/5.api-reference/0.makeProviders.md b/.docs/content/5.api-reference/0.makeProviders.md index c76794a..a2392c6 100644 --- a/.docs/content/5.api-reference/0.makeProviders.md +++ b/.docs/content/5.api-reference/0.makeProviders.md @@ -1,7 +1,7 @@ # `makeProviders` Make an instance of provider controls with configuration. -This is the main entrypoint of the library. It is recommended to make one instance globally and reuse it throughout your application. +This is the main entry-point of the library. It is recommended to make one instance globally and reuse it throughout your application. ## Example @@ -23,9 +23,9 @@ interface ProviderBuilderOptions { // instance of a fetcher, all webrequests are made with the fetcher. fetcher: Fetcher; - // instance of a fetcher, in case the request has cors restrictions. + // instance of a fetcher, in case the request has CORS restrictions. // this fetcher will be called instead of normal fetcher. - // if your environment doesnt have cors restrictions (like Node.JS), there is no need to set this. + // if your environment doesn't have CORS restrictions (like Node.JS), there is no need to set this. proxiedFetcher?: Fetcher; // target to get streams for diff --git a/.docs/content/5.api-reference/1.ProviderControlsRunAll.md b/.docs/content/5.api-reference/1.ProviderControlsRunAll.md index 21999ef..5270b64 100644 --- a/.docs/content/5.api-reference/1.ProviderControlsRunAll.md +++ b/.docs/content/5.api-reference/1.ProviderControlsRunAll.md @@ -1,7 +1,7 @@ # `ProviderControls.runAll` Run all providers one by one in order of their built-in ranking. -You can attach events if you need to know what is going on while its processing. +You can attach events if you need to know what is going on while it is processing. ## Example @@ -20,7 +20,7 @@ const stream = await providers.runAll({ }) // scrape a stream, but prioritize flixhq above all -// (other scrapers are stil ran if flixhq fails, it just has priority) +// (other scrapers are still run if flixhq fails, it just has priority) const flixhqStream = await providers.runAll({ media: media, sourceOrder: ['flixhq'] @@ -33,12 +33,12 @@ const flixhqStream = await providers.runAll({ function runAll(runnerOps: RunnerOptions): Promise; interface RunnerOptions { - // overwrite the order of sources to run. list of ids - // any omitted ids are in added to the end in order of rank (highest first) + // overwrite the order of sources to run. List of IDs + // any omitted IDs are added to the end in order of rank (highest first) sourceOrder?: string[]; - // overwrite the order of embeds to run. list of ids - // any omitted ids are in added to the end in order of rank (highest first) + // overwrite the order of embeds to run. List of IDs + // any omitted IDs are added to the end in order of rank (highest first) embedOrder?: string[]; // object of event functions @@ -49,13 +49,13 @@ interface RunnerOptions { } type RunOutput = { - // source scraper id + // source scraper ID sourceId: string; - // if from an embed, this is the embed scraper id + // if from an embed, this is the embed scraper ID embedId?: string; - // the outputed stream + // the emitted stream stream: Stream; }; ``` diff --git a/.docs/content/5.api-reference/2.ProviderControlsrunSourceScraper.md b/.docs/content/5.api-reference/2.ProviderControlsrunSourceScraper.md index 341d77b..e99396b 100644 --- a/.docs/content/5.api-reference/2.ProviderControlsrunSourceScraper.md +++ b/.docs/content/5.api-reference/2.ProviderControlsrunSourceScraper.md @@ -1,6 +1,6 @@ # `ProviderControls.runSourceScraper` -Run a specific source scraper and get its outputted streams. +Run a specific source scraper and get its emitted streams. ## Example @@ -24,7 +24,7 @@ try { }) } catch (err) { if (err instanceof NotFoundError) { - console.log('source doesnt have this media'); + console.log('source does not have this media'); } else { console.log('failed to scrape') } @@ -48,13 +48,13 @@ interface SourceRunnerOptions { // the media you want to see sources from media: ScrapeMedia; - // id of the source scraper you want to scrape from + // ID of the source scraper you want to scrape from id: string; } type SourcererOutput = { // list of embeds that the source scraper found. - // embed id is a reference to an embed scraper + // embed ID is a reference to an embed scraper embeds: { embedId: string; url: string; diff --git a/.docs/content/5.api-reference/3.ProviderControlsrunEmbedScraper.md b/.docs/content/5.api-reference/3.ProviderControlsrunEmbedScraper.md index ceb9870..992d4c5 100644 --- a/.docs/content/5.api-reference/3.ProviderControlsrunEmbedScraper.md +++ b/.docs/content/5.api-reference/3.ProviderControlsrunEmbedScraper.md @@ -1,6 +1,6 @@ # `ProviderControls.runEmbedScraper` -Run a specific embed scraper and get its outputted streams. +Run a specific embed scraper and get its emitted streams. ## Example @@ -31,10 +31,10 @@ interface EmbedRunnerOptions { // object of event functions events?: IndividualScraperEvents; - // the embed url + // the embed URL url: string; - // id of the embed scraper you want to scrape from + // ID of the embed scraper you want to scrape from id: string; } diff --git a/.docs/content/5.api-reference/4.ProviderControlslistSources.md b/.docs/content/5.api-reference/4.ProviderControlslistSources.md index 4489028..ba7bf76 100644 --- a/.docs/content/5.api-reference/4.ProviderControlslistSources.md +++ b/.docs/content/5.api-reference/4.ProviderControlslistSources.md @@ -1,13 +1,13 @@ # `ProviderControls.listSources` -List all source scrapers that applicable for the target. -They are sorted by rank, highest first +List all source scrapers that are applicable for the target. +They are sorted by rank; highest first ## Example ```ts const sourceScrapers = providers.listSources(); -// Guaranteed to only return type: 'source' +// Guaranteed to only return the type: 'source' ``` ## Type diff --git a/.docs/content/5.api-reference/5.ProviderControlslistEmbeds.md b/.docs/content/5.api-reference/5.ProviderControlslistEmbeds.md index 6c69772..8bb2d46 100644 --- a/.docs/content/5.api-reference/5.ProviderControlslistEmbeds.md +++ b/.docs/content/5.api-reference/5.ProviderControlslistEmbeds.md @@ -1,13 +1,13 @@ # `ProviderControls.listEmbeds` -List all embed scrapers that applicable for the target. -They are sorted by rank, highest first +List all embed scrapers that are applicable for the target. +They are sorted by rank; highest first ## Example ```ts const embedScrapers = providers.listEmbeds(); -// Guaranteed to only return type: 'embed' +// Guaranteed to only return the type: 'embed' ``` ## Type diff --git a/.docs/content/5.api-reference/6.ProviderControlsgetMetadata.md b/.docs/content/5.api-reference/6.ProviderControlsgetMetadata.md index 8a83f00..1a1e94b 100644 --- a/.docs/content/5.api-reference/6.ProviderControlsgetMetadata.md +++ b/.docs/content/5.api-reference/6.ProviderControlsgetMetadata.md @@ -1,7 +1,7 @@ # `ProviderControls.getMetadata` Get meta data for a scraper, can be either source or embed scraper. -Returns null if the `id` is not recognized. +Returns `null` if the `id` is not recognized. ## Example diff --git a/.docs/content/5.api-reference/7.makeStandardFetcher.md b/.docs/content/5.api-reference/7.makeStandardFetcher.md index a7e5e76..c0422fc 100644 --- a/.docs/content/5.api-reference/7.makeStandardFetcher.md +++ b/.docs/content/5.api-reference/7.makeStandardFetcher.md @@ -1,6 +1,6 @@ # `makeStandardFetcher` -Make a fetcher from a `fetch()` API. It is used for making a instance of provider controls. +Make a fetcher from a `fetch()` API. It is used for making an instance of provider controls. ## Example From d7134d4daf289570c55af5857cb8bb6fe44833ad Mon Sep 17 00:00:00 2001 From: Jorrin Date: Thu, 4 Jan 2024 20:18:25 +0100 Subject: [PATCH 09/43] proxied fetcher --- src/providers/embeds/filemoon/index.ts | 8 ++++++-- src/providers/embeds/vidplay/index.ts | 2 +- src/providers/sources/vidsrcto/index.ts | 16 +++++++++++++--- 3 files changed, 20 insertions(+), 6 deletions(-) diff --git a/src/providers/embeds/filemoon/index.ts b/src/providers/embeds/filemoon/index.ts index 2729b5c..3f8a2f3 100644 --- a/src/providers/embeds/filemoon/index.ts +++ b/src/providers/embeds/filemoon/index.ts @@ -14,7 +14,11 @@ export const fileMoonScraper = makeEmbed({ name: 'Filemoon', rank: 400, scrape: async (ctx) => { - const embedRes = await ctx.fetcher(ctx.url); + const embedRes = await ctx.proxiedFetcher(ctx.url, { + headers: { + referer: ctx.url, + }, + }); const evalCode = evalCodeRegex.exec(embedRes); if (!evalCode) throw new Error('Failed to find eval code'); const unpacked = unpack(evalCode[1]); @@ -25,7 +29,7 @@ export const fileMoonScraper = makeEmbed({ const subtitlesLink = url.searchParams.get('sub.info'); const captions: Caption[] = []; if (subtitlesLink) { - const captionsResult = await ctx.fetcher(subtitlesLink); + const captionsResult = await ctx.proxiedFetcher(subtitlesLink); for (const caption of captionsResult) { const language = labelToLanguageCode(caption.label); diff --git a/src/providers/embeds/vidplay/index.ts b/src/providers/embeds/vidplay/index.ts index f7adca2..3c1f6a2 100644 --- a/src/providers/embeds/vidplay/index.ts +++ b/src/providers/embeds/vidplay/index.ts @@ -22,7 +22,7 @@ export const vidplayScraper = makeEmbed({ const subtitlesLink = url.searchParams.get('sub.info'); const captions: Caption[] = []; if (subtitlesLink) { - const captionsResult = await ctx.fetcher(subtitlesLink); + const captionsResult = await ctx.proxiedFetcher(subtitlesLink); for (const caption of captionsResult) { const language = labelToLanguageCode(caption.label); diff --git a/src/providers/sources/vidsrcto/index.ts b/src/providers/sources/vidsrcto/index.ts index 5e73bc1..983fd12 100644 --- a/src/providers/sources/vidsrcto/index.ts +++ b/src/providers/sources/vidsrcto/index.ts @@ -8,6 +8,7 @@ import { decryptSourceUrl } from './common'; import { SourceResult, SourcesResult } from './types'; const vidSrcToBase = 'https://vidsrc.to'; +const referer = `${vidSrcToBase}/`; const universalScraper = async (ctx: ShowScrapeContext | MovieScrapeContext): Promise => { const imdbId = ctx.media.imdbId; @@ -15,22 +16,31 @@ const universalScraper = async (ctx: ShowScrapeContext | MovieScrapeContext): Pr ctx.media.type === 'movie' ? `/embed/movie/${imdbId}` : `/embed/tv/${imdbId}/${ctx.media.season.number}/${ctx.media.episode.number}`; - const mainPage = await ctx.fetcher(url, { + const mainPage = await ctx.proxiedFetcher(url, { baseUrl: vidSrcToBase, + headers: { + referer, + }, }); const mainPage$ = load(mainPage); const dataId = mainPage$('a[data-id]').attr('data-id'); if (!dataId) throw new Error('No data-id found'); - const sources = await ctx.fetcher(`/ajax/embed/episode/${dataId}/sources`, { + const sources = await ctx.proxiedFetcher(`/ajax/embed/episode/${dataId}/sources`, { baseUrl: vidSrcToBase, + headers: { + referer, + }, }); if (sources.status !== 200) throw new Error('No sources found'); const embeds: SourcererEmbed[] = []; const embedUrls = []; for (const source of sources.result) { - const sourceRes = await ctx.fetcher(`/ajax/embed/source/${source.id}`, { + const sourceRes = await ctx.proxiedFetcher(`/ajax/embed/source/${source.id}`, { baseUrl: vidSrcToBase, + headers: { + referer, + }, }); const decryptedUrl = decryptSourceUrl(sourceRes.result.url); embedUrls.push(decryptedUrl); From f1c29aab9003bd85ce36ec9a12d0938e5b0757ae Mon Sep 17 00:00:00 2001 From: mrjvs Date: Thu, 4 Jan 2024 21:11:24 +0100 Subject: [PATCH 10/43] Fix showbox + add support for new simple-proxy version --- src/fetchers/simpleProxy.ts | 1 + src/providers/sources/showbox/sendRequest.ts | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/fetchers/simpleProxy.ts b/src/fetchers/simpleProxy.ts index 21ed5ca..b9c6c5b 100644 --- a/src/fetchers/simpleProxy.ts +++ b/src/fetchers/simpleProxy.ts @@ -7,6 +7,7 @@ const headerMap: Record = { cookie: 'X-Cookie', referer: 'X-Referer', origin: 'X-Origin', + 'user-agent': 'X-User-Agent', }; const responseHeaderMap: Record = { diff --git a/src/providers/sources/showbox/sendRequest.ts b/src/providers/sources/showbox/sendRequest.ts index 2f77767..7ea9024 100644 --- a/src/providers/sources/showbox/sendRequest.ts +++ b/src/providers/sources/showbox/sendRequest.ts @@ -49,9 +49,9 @@ export const sendRequest = async (ctx: ScrapeContext, data: object, altApi = fal headers: { Platform: 'android', 'Content-Type': 'application/x-www-form-urlencoded', + 'User-Agent': 'okhttp/3.2.0', }, body: formatted, }); - return JSON.parse(response); }; From a694230bee1fc5e17e820cda1203bf9da498e184 Mon Sep 17 00:00:00 2001 From: mrjvs Date: Thu, 4 Jan 2024 21:20:14 +0100 Subject: [PATCH 11/43] Add another simple-proxy header --- src/fetchers/simpleProxy.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/fetchers/simpleProxy.ts b/src/fetchers/simpleProxy.ts index b9c6c5b..360a149 100644 --- a/src/fetchers/simpleProxy.ts +++ b/src/fetchers/simpleProxy.ts @@ -8,6 +8,7 @@ const headerMap: Record = { referer: 'X-Referer', origin: 'X-Origin', 'user-agent': 'X-User-Agent', + 'x-real-ip': 'X-X-Real-Ip', }; const responseHeaderMap: Record = { From 605b9d78d1b8cdf82c6ad4092005bda4eac70577 Mon Sep 17 00:00:00 2001 From: Jorrin Date: Fri, 5 Jan 2024 16:12:19 +0100 Subject: [PATCH 12/43] filemoon added CORS --- src/providers/embeds/filemoon/index.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/providers/embeds/filemoon/index.ts b/src/providers/embeds/filemoon/index.ts index 3f8a2f3..9f96a07 100644 --- a/src/providers/embeds/filemoon/index.ts +++ b/src/providers/embeds/filemoon/index.ts @@ -1,7 +1,5 @@ import { unpack } from 'unpacker'; -import { flags } from '@/entrypoint/utils/targets'; - import { SubtitleResult } from './types'; import { makeEmbed } from '../../base'; import { Caption, getCaptionTypeFromUrl, labelToLanguageCode } from '../../captions'; @@ -51,7 +49,7 @@ export const fileMoonScraper = makeEmbed({ id: 'primary', type: 'hls', playlist: file[1], - flags: [flags.CORS_ALLOWED], + flags: [], captions, }, ], From 1970e11443cfdb07652ad1ccc0092a3985952102 Mon Sep 17 00:00:00 2001 From: Jorrin Date: Fri, 5 Jan 2024 17:01:54 +0100 Subject: [PATCH 13/43] applied feedback from ciarands --- src/providers/embeds/vidplay/common.ts | 37 +++------------- src/providers/sources/vidsrcto/common.ts | 55 ++++++++++++------------ 2 files changed, 33 insertions(+), 59 deletions(-) diff --git a/src/providers/embeds/vidplay/common.ts b/src/providers/embeds/vidplay/common.ts index 57b94bb..224e3dc 100644 --- a/src/providers/embeds/vidplay/common.ts +++ b/src/providers/embeds/vidplay/common.ts @@ -1,36 +1,11 @@ import { makeFullUrl } from '@/fetchers/common'; +import { decodeData } from '@/providers/sources/vidsrcto/common'; import { EmbedScrapeContext } from '@/utils/context'; export const vidplayBase = 'https://vidplay.site'; -// This file is based on https://github.com/Ciarands/vidsrc-to-resolver/blob/960afb11c30aa6497804b4691fb1c401e539cfe7/vidsrc.py#L10 +// This file is based on https://github.com/Ciarands/vidsrc-to-resolver/blob/dffa45e726a4b944cb9af0c9e7630476c93c0213/vidsrc.py#L16 // Full credits to @Ciarands! -export function keyPermutation(key: string, data: any) { - const state = Array.from(Array(256).keys()); - let index1 = 0; - for (let i = 0; i < 256; i += 1) { - index1 = (index1 + state[i] + key.charCodeAt(i % key.length)) % 256; - const temp = state[i]; - state[i] = state[index1]; - state[index1] = temp; - } - index1 = 0; - let index2 = 0; - let finalKey = ''; - for (let char = 0; char < data.length; char += 1) { - index1 = (index1 + 1) % 256; - index2 = (index2 + state[index1]) % 256; - const temp = state[index1]; - state[index1] = state[index2]; - state[index2] = temp; - if (typeof data[char] === 'string') { - finalKey += String.fromCharCode(data[char].charCodeAt(0) ^ state[(state[index1] + state[index2]) % 256]); - } else if (typeof data[char] === 'number') { - finalKey += String.fromCharCode(data[char] ^ state[(state[index1] + state[index2]) % 256]); - } - } - return finalKey; -} export const getDecryptionKeys = async (ctx: EmbedScrapeContext): Promise => { const res = await ctx.fetcher( @@ -44,10 +19,10 @@ export const getEncodedId = async (ctx: EmbedScrapeContext) => { const id = url.pathname.replace('/e/', ''); const keyList = await getDecryptionKeys(ctx); - const decodedId = keyPermutation(keyList[0], id); - const encodedResult = keyPermutation(keyList[1], decodedId); - const base64 = btoa(encodedResult); - return base64.replace('/', '_'); + const decodedId = decodeData(keyList[0], id); + const encodedResult = decodeData(keyList[1], decodedId); + const b64encoded = btoa(encodedResult); + return b64encoded.replace('/', '_'); }; export const getFuTokenKey = async (ctx: EmbedScrapeContext) => { diff --git a/src/providers/sources/vidsrcto/common.ts b/src/providers/sources/vidsrcto/common.ts index fb9890d..2c7272f 100644 --- a/src/providers/sources/vidsrcto/common.ts +++ b/src/providers/sources/vidsrcto/common.ts @@ -1,3 +1,6 @@ +// This file is based on https://github.com/Ciarands/vidsrc-to-resolver/blob/dffa45e726a4b944cb9af0c9e7630476c93c0213/vidsrc.py#L16 +// Full credits to @Ciarands! + const DECRYPTION_KEY = '8z5Ag5wgagfsOuhz'; export const decodeBase64UrlSafe = (str: string) => { @@ -12,39 +15,35 @@ export const decodeBase64UrlSafe = (str: string) => { return bytes; }; -export const decode = (str: Uint8Array) => { - const keyBytes = new TextEncoder().encode(DECRYPTION_KEY); - - let j = 0; - const s = new Uint8Array(256); +export const decodeData = (key: string, data: any) => { + const state = Array.from(Array(256).keys()); + let index1 = 0; for (let i = 0; i < 256; i += 1) { - s[i] = i; + index1 = (index1 + state[i] + key.charCodeAt(i % key.length)) % 256; + const temp = state[i]; + state[i] = state[index1]; + state[index1] = temp; } - - for (let i = 0, k = 0; i < 256; i += 1) { - j = (j + s[i] + keyBytes[k % keyBytes.length]) & 0xff; - [s[i], s[j]] = [s[j], s[i]]; - k += 1; + index1 = 0; + let index2 = 0; + let finalKey = ''; + for (let char = 0; char < data.length; char += 1) { + index1 = (index1 + 1) % 256; + index2 = (index2 + state[index1]) % 256; + const temp = state[index1]; + state[index1] = state[index2]; + state[index2] = temp; + if (typeof data[char] === 'string') { + finalKey += String.fromCharCode(data[char].charCodeAt(0) ^ state[(state[index1] + state[index2]) % 256]); + } else if (typeof data[char] === 'number') { + finalKey += String.fromCharCode(data[char] ^ state[(state[index1] + state[index2]) % 256]); + } } - - const decoded = new Uint8Array(str.length); - let i = 0; - let k = 0; - for (let index = 0; index < str.length; index += 1) { - i = (i + 1) & 0xff; - k = (k + s[i]) & 0xff; - [s[i], s[k]] = [s[k], s[i]]; - const t = (s[i] + s[k]) & 0xff; - decoded[index] = str[index] ^ s[t]; - } - - return decoded; + return finalKey; }; export const decryptSourceUrl = (sourceUrl: string) => { const encoded = decodeBase64UrlSafe(sourceUrl); - const decoded = decode(encoded); - const decodedText = new TextDecoder().decode(decoded); - - return decodeURIComponent(decodeURIComponent(decodedText)); + const decoded = decodeData(DECRYPTION_KEY, encoded); + return decodeURIComponent(decodeURIComponent(decoded)); }; From 9276694ead47d46f8a2d8f8c693f75fb17bcfdbb Mon Sep 17 00:00:00 2001 From: Jorrin Date: Fri, 5 Jan 2024 19:52:28 +0100 Subject: [PATCH 14/43] remove flag --- src/providers/sources/vidsrcto/index.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/providers/sources/vidsrcto/index.ts b/src/providers/sources/vidsrcto/index.ts index 983fd12..b85b068 100644 --- a/src/providers/sources/vidsrcto/index.ts +++ b/src/providers/sources/vidsrcto/index.ts @@ -1,6 +1,5 @@ import { load } from 'cheerio'; -import { flags } from '@/entrypoint/utils/targets'; import { SourcererEmbed, SourcererOutput, makeSourcerer } from '@/providers/base'; import { MovieScrapeContext, ShowScrapeContext } from '@/utils/context'; @@ -80,6 +79,6 @@ export const vidSrcToScraper = makeSourcerer({ name: 'VidSrcTo', scrapeMovie: universalScraper, scrapeShow: universalScraper, - flags: [flags.CORS_ALLOWED], + flags: [], rank: 400, }); From 5ef048c28fbbd82fe538a7ac829ec1d628954706 Mon Sep 17 00:00:00 2001 From: William Oldham Date: Sat, 6 Jan 2024 14:49:09 +0000 Subject: [PATCH 15/43] Bump Version --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index f7d1e18..7d3090d 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@movie-web/providers", - "version": "2.0.3", + "version": "2.0.4", "description": "Package that contains all the providers of movie-web", "main": "./lib/index.umd.js", "types": "./lib/index.d.ts", From c7dff57a409f944612f119674aa7ea9c851a685e Mon Sep 17 00:00:00 2001 From: William Oldham Date: Sat, 6 Jan 2024 14:59:54 +0000 Subject: [PATCH 16/43] Update changelog to 2.0.4 --- .docs/content/1.get-started/4.changelog.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/.docs/content/1.get-started/4.changelog.md b/.docs/content/1.get-started/4.changelog.md index fd00c1c..cfdaa23 100644 --- a/.docs/content/1.get-started/4.changelog.md +++ b/.docs/content/1.get-started/4.changelog.md @@ -2,6 +2,27 @@ title: 'Changelog' --- +# Version 2.0.4 +- Added providers: + - Add VidSrcTo provider with Vidplay and Filemoon embeds + - Add VidSrc provider with StreamBucket embeds +- Fixed providers: + - RemoteStream + - LookMovie - Fixed captions + - ShowBox +- Updated documentation to fix spelling + grammar +- User-agent header fix +- Needs the latest simple-proxy update +- Added utility to not return multiple subs for the same language - Applies to Lookmovie and Showbox + +# Version 2.0.3 +- Actually remove Febbox HLS + +# Version 2.0.2 +- Added Lookmovie caption support +- Fix Febbox duplicate subtitle languages +- Remove Febbox HLS + # Version 2.0.1 - Fixed issue where febbox-mp4 would not show all qualities - Fixed issue where discoverEmbeds event would not show the embeds in the right order From 48b3fcdd17517eac3cff9c87e18c142e421a799e Mon Sep 17 00:00:00 2001 From: mrjvs Date: Sat, 6 Jan 2024 21:52:01 +0100 Subject: [PATCH 17/43] Bump version and update febbox mp4 to not have subs as endpoint broke --- package.json | 2 +- src/providers/embeds/febbox/mp4.ts | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index 7d3090d..d4e7626 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@movie-web/providers", - "version": "2.0.4", + "version": "2.0.5", "description": "Package that contains all the providers of movie-web", "main": "./lib/index.umd.js", "types": "./lib/index.d.ts", diff --git a/src/providers/embeds/febbox/mp4.ts b/src/providers/embeds/febbox/mp4.ts index 1122e53..b69812f 100644 --- a/src/providers/embeds/febbox/mp4.ts +++ b/src/providers/embeds/febbox/mp4.ts @@ -2,7 +2,6 @@ import { flags } from '@/entrypoint/utils/targets'; import { makeEmbed } from '@/providers/base'; import { parseInputUrl } from '@/providers/embeds/febbox/common'; import { getStreamQualities } from '@/providers/embeds/febbox/qualities'; -import { getSubtitles } from '@/providers/embeds/febbox/subtitles'; export const febboxMp4Scraper = makeEmbed({ id: 'febbox-mp4', @@ -42,7 +41,7 @@ export const febboxMp4Scraper = makeEmbed({ stream: [ { id: 'primary', - captions: await getSubtitles(ctx, id, fid, type, episode, season), + captions: [], // subtitles temporarily disabled, the endpoints are broken qualities, type: 'file', flags: [flags.CORS_ALLOWED], From c3bd57562117f26a88115dd0b1666f25a51023aa Mon Sep 17 00:00:00 2001 From: mrjvs Date: Sat, 6 Jan 2024 21:53:09 +0100 Subject: [PATCH 18/43] Update docs --- .docs/content/1.get-started/4.changelog.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.docs/content/1.get-started/4.changelog.md b/.docs/content/1.get-started/4.changelog.md index cfdaa23..239ed6b 100644 --- a/.docs/content/1.get-started/4.changelog.md +++ b/.docs/content/1.get-started/4.changelog.md @@ -2,6 +2,9 @@ title: 'Changelog' --- +# Version 2.0.5 +- Disable subtitles for febbox-mp4. As their endpoint doesn't work anymore. + # Version 2.0.4 - Added providers: - Add VidSrcTo provider with Vidplay and Filemoon embeds From b5b2d341f9b9c48a958815491f097b080650b408 Mon Sep 17 00:00:00 2001 From: Jorrin Date: Sun, 7 Jan 2024 00:42:32 +0100 Subject: [PATCH 19/43] add disabled embed filter --- package-lock.json | 4 ++-- src/runners/individualRunner.ts | 8 ++++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 6a4523d..f38bd59 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@movie-web/providers", - "version": "1.1.5", + "version": "2.0.5", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@movie-web/providers", - "version": "1.1.5", + "version": "2.0.5", "license": "MIT", "dependencies": { "cheerio": "^1.0.0-rc.12", diff --git a/src/runners/individualRunner.ts b/src/runners/individualRunner.ts index 2befd84..ad3004b 100644 --- a/src/runners/individualRunner.ts +++ b/src/runners/individualRunner.ts @@ -58,6 +58,14 @@ export async function scrapeInvidualSource( } if (!output) throw new Error('output is null'); + + // filter output with only valid embeds that are not disabled + output.embeds = output.embeds.filter((embed) => { + const e = list.embeds.find((v) => v.id === embed.embedId); + if (!e || e.disabled) return false; + return true; + }); + if ((!output.stream || output.stream.length === 0) && output.embeds.length === 0) throw new NotFoundError('No streams found'); return output; From 6212a014a0b189847dfc91c715e2f7ad290b9d92 Mon Sep 17 00:00:00 2001 From: Jorrin Date: Sun, 7 Jan 2024 01:12:17 +0100 Subject: [PATCH 20/43] filter out disabled embeds in runAll --- src/runners/runner.ts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/runners/runner.ts b/src/runners/runner.ts index eaf6285..e3c0976 100644 --- a/src/runners/runner.ts +++ b/src/runners/runner.ts @@ -116,8 +116,14 @@ export async function runAllProviders(list: ProviderList, ops: ProviderRunnerOpt }; } - // run embed scrapers on listed embeds - const sortedEmbeds = output.embeds.sort((a, b) => embedIds.indexOf(a.embedId) - embedIds.indexOf(b.embedId)); + // filter disabled and run embed scrapers on listed embeds + const sortedEmbeds = output.embeds + .filter((embed) => { + const e = list.embeds.find((v) => v.id === embed.embedId); + if (!e || e.disabled) return false; + return true; + }) + .sort((a, b) => embedIds.indexOf(a.embedId) - embedIds.indexOf(b.embedId)); if (sortedEmbeds.length > 0) { ops.events?.discoverEmbeds?.({ From 1dc7c879afd4d96e7513bf977bc55c2b8257f736 Mon Sep 17 00:00:00 2001 From: Jorrin Date: Mon, 8 Jan 2024 16:27:59 +0100 Subject: [PATCH 21/43] add preferredHeaders to vidplay --- src/providers/embeds/vidplay/common.ts | 1 + src/providers/embeds/vidplay/index.ts | 7 ++++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/providers/embeds/vidplay/common.ts b/src/providers/embeds/vidplay/common.ts index 224e3dc..e3172f7 100644 --- a/src/providers/embeds/vidplay/common.ts +++ b/src/providers/embeds/vidplay/common.ts @@ -3,6 +3,7 @@ import { decodeData } from '@/providers/sources/vidsrcto/common'; import { EmbedScrapeContext } from '@/utils/context'; export const vidplayBase = 'https://vidplay.site'; +export const referer = 'https://vidplay.online/'; // This file is based on https://github.com/Ciarands/vidsrc-to-resolver/blob/dffa45e726a4b944cb9af0c9e7630476c93c0213/vidsrc.py#L16 // Full credits to @Ciarands! diff --git a/src/providers/embeds/vidplay/index.ts b/src/providers/embeds/vidplay/index.ts index 3c1f6a2..9d205e1 100644 --- a/src/providers/embeds/vidplay/index.ts +++ b/src/providers/embeds/vidplay/index.ts @@ -1,7 +1,8 @@ +import { flags } from '@/entrypoint/utils/targets'; import { makeEmbed } from '@/providers/base'; import { Caption, getCaptionTypeFromUrl, labelToLanguageCode } from '@/providers/captions'; -import { getFileUrl } from './common'; +import { getFileUrl, referer } from './common'; import { SubtitleResult, VidplaySourceResponse } from './types'; export const vidplayScraper = makeEmbed({ @@ -46,6 +47,10 @@ export const vidplayScraper = makeEmbed({ playlist: source, flags: [], captions, + preferredHeaders: { + Referer: referer, + Origin: referer, + }, }, ], }; From 7697c767214921084e1ba6c1af3fd528e75f19a3 Mon Sep 17 00:00:00 2001 From: Jorrin Date: Wed, 10 Jan 2024 21:21:47 +0100 Subject: [PATCH 22/43] add headers --- src/providers/embeds/upcloud.ts | 7 +++++++ src/providers/sources/remotestream.ts | 17 +++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/src/providers/embeds/upcloud.ts b/src/providers/embeds/upcloud.ts index 7880cc4..0f86cd0 100644 --- a/src/providers/embeds/upcloud.ts +++ b/src/providers/embeds/upcloud.ts @@ -4,6 +4,9 @@ import { flags } from '@/entrypoint/utils/targets'; import { makeEmbed } from '@/providers/base'; import { Caption, getCaptionTypeFromUrl, labelToLanguageCode } from '@/providers/captions'; +const origin = 'https://rabbitstream.net'; +const referer = 'https://rabbitstream.net/'; + const { AES, enc } = crypto; interface StreamRes { @@ -126,6 +129,10 @@ export const upcloudScraper = makeEmbed({ playlist: sources.file, flags: [flags.CORS_ALLOWED], captions, + preferredHeaders: { + Referer: referer, + Origin: origin, + }, }, ], }; diff --git a/src/providers/sources/remotestream.ts b/src/providers/sources/remotestream.ts index 8a3090b..f0a0ae8 100644 --- a/src/providers/sources/remotestream.ts +++ b/src/providers/sources/remotestream.ts @@ -4,6 +4,9 @@ import { NotFoundError } from '@/utils/errors'; const remotestreamBase = atob('aHR0cHM6Ly9mc2IuOG1ldDNkdGpmcmNxY2hjb25xcGtsd3hzeGIyb2N1bWMuc3RyZWFt'); +const origin = 'https://remotestre.am'; +const referer = 'https://remotestre.am/'; + export const remotestreamScraper = makeSourcerer({ id: 'remotestream', name: 'Remote Stream', @@ -19,6 +22,9 @@ export const remotestreamScraper = makeSourcerer({ const streamRes = await ctx.fetcher.full(playlistLink, { method: 'HEAD', readHeaders: ['content-type'], + headers: { + Referer: referer, + }, }); if (!streamRes.headers.get('content-type')?.toLowerCase().includes('application/x-mpegurl')) throw new NotFoundError('No watchable item found'); @@ -33,6 +39,10 @@ export const remotestreamScraper = makeSourcerer({ playlist: playlistLink, type: 'hls', flags: [flags.CORS_ALLOWED], + preferredHeaders: { + Referer: referer, + Origin: origin, + }, }, ], }; @@ -44,6 +54,9 @@ export const remotestreamScraper = makeSourcerer({ const streamRes = await ctx.fetcher.full(playlistLink, { method: 'HEAD', readHeaders: ['content-type'], + headers: { + Referer: referer, + }, }); if (!streamRes.headers.get('content-type')?.toLowerCase().includes('application/x-mpegurl')) throw new NotFoundError('No watchable item found'); @@ -58,6 +71,10 @@ export const remotestreamScraper = makeSourcerer({ playlist: playlistLink, type: 'hls', flags: [flags.CORS_ALLOWED], + preferredHeaders: { + Referer: referer, + Origin: origin, + }, }, ], }; From 854dec111dad2028b02273e8ab61398c0b334958 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 11 Jan 2024 00:39:32 +0000 Subject: [PATCH 23/43] Bump follow-redirects from 1.15.2 to 1.15.4 in /.docs Bumps [follow-redirects](https://github.com/follow-redirects/follow-redirects) from 1.15.2 to 1.15.4. - [Release notes](https://github.com/follow-redirects/follow-redirects/releases) - [Commits](https://github.com/follow-redirects/follow-redirects/compare/v1.15.2...v1.15.4) --- updated-dependencies: - dependency-name: follow-redirects dependency-type: indirect ... Signed-off-by: dependabot[bot] --- .docs/package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.docs/package-lock.json b/.docs/package-lock.json index 810205e..3a9fa95 100644 --- a/.docs/package-lock.json +++ b/.docs/package-lock.json @@ -6474,9 +6474,9 @@ } }, "node_modules/follow-redirects": { - "version": "1.15.2", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.2.tgz", - "integrity": "sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==", + "version": "1.15.4", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.4.tgz", + "integrity": "sha512-Cr4D/5wlrb0z9dgERpUL3LrmPKVDsETIJhaCMeDfuFYcqa5bldGV6wBsAN6X/vxlXQtFBMrXdXxdL8CbDTGniw==", "dev": true, "funding": [ { From 7a5cbb3428f325ac86dfdc270b56f0c018110de9 Mon Sep 17 00:00:00 2001 From: mrjvs Date: Thu, 11 Jan 2024 19:14:21 +0100 Subject: [PATCH 24/43] Add CF_BLOCKED flags --- src/entrypoint/utils/targets.ts | 4 ++++ src/providers/sources/showbox/index.ts | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/entrypoint/utils/targets.ts b/src/entrypoint/utils/targets.ts index 16a02da..80988a4 100644 --- a/src/entrypoint/utils/targets.ts +++ b/src/entrypoint/utils/targets.ts @@ -5,6 +5,10 @@ export const flags = { // the stream is locked on IP, so only works if // request maker is same as player (not compatible with proxies) IP_LOCKED: 'ip-locked', + + // The source/embed is blocking cloudflare ip's + // This flag is not compatible with a proxy hosted on cloudflare + CF_BLOCKED: 'cf-blocked', } as const; export type Flags = (typeof flags)[keyof typeof flags]; diff --git a/src/providers/sources/showbox/index.ts b/src/providers/sources/showbox/index.ts index d6c4887..c8a834a 100644 --- a/src/providers/sources/showbox/index.ts +++ b/src/providers/sources/showbox/index.ts @@ -42,7 +42,7 @@ export const showboxScraper = makeSourcerer({ id: 'showbox', name: 'Showbox', rank: 300, - flags: [flags.CORS_ALLOWED], + flags: [flags.CORS_ALLOWED, flags.CF_BLOCKED], scrapeShow: comboScraper, scrapeMovie: comboScraper, }); From 9798659af817b5d036b16fa6214b24589a4478cc Mon Sep 17 00:00:00 2001 From: mrjvs Date: Wed, 17 Jan 2024 17:18:12 +0100 Subject: [PATCH 25/43] add basic categories to tests --- package-lock.json | 26 ++++++-- package.json | 2 + src/__test__/providers/abc.test.ts | 7 ++ .../{ => standard}/fetchers/body.test.ts | 0 .../{ => standard}/fetchers/common.test.ts | 0 .../fetchers/simpleProxy.test.ts | 0 .../{ => standard}/fetchers/standard.test.ts | 0 src/__test__/{ => standard}/providerTests.ts | 66 +++++++++---------- .../{ => standard}/providers/checks.test.ts | 2 +- .../{ => standard}/runner/list.test.ts | 2 +- .../{ => standard}/runner/meta.test.ts | 2 +- .../{ => standard}/utils/features.test.ts | 0 .../{ => standard}/utils/list.test.ts | 0 .../{ => standard}/utils/valid.test.ts | 0 vite.config.js | 9 ++- 15 files changed, 71 insertions(+), 45 deletions(-) create mode 100644 src/__test__/providers/abc.test.ts rename src/__test__/{ => standard}/fetchers/body.test.ts (100%) rename src/__test__/{ => standard}/fetchers/common.test.ts (100%) rename src/__test__/{ => standard}/fetchers/simpleProxy.test.ts (100%) rename src/__test__/{ => standard}/fetchers/standard.test.ts (100%) rename src/__test__/{ => standard}/providerTests.ts (69%) rename src/__test__/{ => standard}/providers/checks.test.ts (98%) rename src/__test__/{ => standard}/runner/list.test.ts (98%) rename src/__test__/{ => standard}/runner/meta.test.ts (96%) rename src/__test__/{ => standard}/utils/features.test.ts (100%) rename src/__test__/{ => standard}/utils/list.test.ts (100%) rename src/__test__/{ => standard}/utils/valid.test.ts (100%) diff --git a/package-lock.json b/package-lock.json index f38bd59..9eebc03 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,6 +10,7 @@ "license": "MIT", "dependencies": { "cheerio": "^1.0.0-rc.12", + "cross-env": "^7.0.3", "crypto-js": "^4.1.1", "form-data": "^4.0.0", "iso-639-1": "^3.1.0", @@ -1794,6 +1795,23 @@ "dev": true, "license": "MIT" }, + "node_modules/cross-env": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/cross-env/-/cross-env-7.0.3.tgz", + "integrity": "sha512-+/HKd6EgcQCJGh2PSjZuUitQBQynKor4wrFbRg4DtAgS1aWO+gU52xpH7M9ScGgXSYmAVS9bIJ8EzuaGw0oNAw==", + "dependencies": { + "cross-spawn": "^7.0.1" + }, + "bin": { + "cross-env": "src/bin/cross-env.js", + "cross-env-shell": "src/bin/cross-env-shell.js" + }, + "engines": { + "node": ">=10.14", + "npm": ">=6", + "yarn": ">=1" + } + }, "node_modules/cross-fetch": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-4.0.0.tgz", @@ -1805,7 +1823,6 @@ }, "node_modules/cross-spawn": { "version": "7.0.3", - "dev": true, "license": "MIT", "dependencies": { "path-key": "^3.1.0", @@ -3585,8 +3602,7 @@ "node_modules/isexe": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", - "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", - "dev": true + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==" }, "node_modules/iso-639-1": { "version": "3.1.0", @@ -4277,7 +4293,6 @@ }, "node_modules/path-key": { "version": "3.1.1", - "dev": true, "license": "MIT", "engines": { "node": ">=8" @@ -4785,7 +4800,6 @@ }, "node_modules/shebang-command": { "version": "2.0.0", - "dev": true, "license": "MIT", "dependencies": { "shebang-regex": "^3.0.0" @@ -4796,7 +4810,6 @@ }, "node_modules/shebang-regex": { "version": "3.0.0", - "dev": true, "license": "MIT", "engines": { "node": ">=8" @@ -5774,7 +5787,6 @@ }, "node_modules/which": { "version": "2.0.2", - "dev": true, "license": "ISC", "dependencies": { "isexe": "^2.0.0" diff --git a/package.json b/package.json index d4e7626..9bb36e9 100644 --- a/package.json +++ b/package.json @@ -38,6 +38,7 @@ "cli": "ts-node ./src/dev-cli/index.ts", "test": "vitest run", "test:watch": "vitest", + "test:providers": "cross-env MW_TEST_PROVIDERS=true vitest run", "test:integration": "node ./tests/cjs && node ./tests/esm && node ./tests/browser", "test:coverage": "vitest run --coverage", "lint": "eslint --ext .ts,.js src/", @@ -78,6 +79,7 @@ }, "dependencies": { "cheerio": "^1.0.0-rc.12", + "cross-env": "^7.0.3", "crypto-js": "^4.1.1", "form-data": "^4.0.0", "iso-639-1": "^3.1.0", diff --git a/src/__test__/providers/abc.test.ts b/src/__test__/providers/abc.test.ts new file mode 100644 index 0000000..3aed2ea --- /dev/null +++ b/src/__test__/providers/abc.test.ts @@ -0,0 +1,7 @@ +import { describe, expect, it } from "vitest"; + +describe('abc', () => { + it('should do thing', () => { + expect(true).toBe(false); + }) +}) diff --git a/src/__test__/fetchers/body.test.ts b/src/__test__/standard/fetchers/body.test.ts similarity index 100% rename from src/__test__/fetchers/body.test.ts rename to src/__test__/standard/fetchers/body.test.ts diff --git a/src/__test__/fetchers/common.test.ts b/src/__test__/standard/fetchers/common.test.ts similarity index 100% rename from src/__test__/fetchers/common.test.ts rename to src/__test__/standard/fetchers/common.test.ts diff --git a/src/__test__/fetchers/simpleProxy.test.ts b/src/__test__/standard/fetchers/simpleProxy.test.ts similarity index 100% rename from src/__test__/fetchers/simpleProxy.test.ts rename to src/__test__/standard/fetchers/simpleProxy.test.ts diff --git a/src/__test__/fetchers/standard.test.ts b/src/__test__/standard/fetchers/standard.test.ts similarity index 100% rename from src/__test__/fetchers/standard.test.ts rename to src/__test__/standard/fetchers/standard.test.ts diff --git a/src/__test__/providerTests.ts b/src/__test__/standard/providerTests.ts similarity index 69% rename from src/__test__/providerTests.ts rename to src/__test__/standard/providerTests.ts index 551a3ec..f5c87b7 100644 --- a/src/__test__/providerTests.ts +++ b/src/__test__/standard/providerTests.ts @@ -2,7 +2,7 @@ import { vi } from 'vitest'; import { gatherAllEmbeds, gatherAllSources } from '@/providers/all'; -import { Embed, Sourcerer } from '@/providers/base'; +import { makeEmbed, makeSourcerer } from '@/providers/base'; export function makeProviderMocks() { const embedsMock = vi.fn, ReturnType>(); @@ -13,104 +13,104 @@ export function makeProviderMocks() { }; } -const sourceA = { +const sourceA = makeSourcerer({ id: 'a', name: 'A', rank: 1, disabled: false, flags: [], -} as Sourcerer; -const sourceB = { +}); +const sourceB = makeSourcerer({ id: 'b', name: 'B', rank: 2, disabled: false, flags: [], -} as Sourcerer; -const sourceCDisabled = { +}); +const sourceCDisabled = makeSourcerer({ id: 'c', name: 'C', rank: 3, disabled: true, flags: [], -} as Sourcerer; -const sourceAHigherRank = { +}); +const sourceAHigherRank = makeSourcerer({ id: 'a', name: 'A', rank: 100, disabled: false, flags: [], -} as Sourcerer; -const sourceGSameRankAsA = { +}); +const sourceGSameRankAsA = makeSourcerer({ id: 'g', name: 'G', rank: 1, disabled: false, flags: [], -} as Sourcerer; -const fullSourceYMovie = { +}); +const fullSourceYMovie = makeSourcerer({ id: 'y', name: 'Y', rank: 105, scrapeMovie: vi.fn(), flags: [], -} as Sourcerer; -const fullSourceYShow = { +}); +const fullSourceYShow = makeSourcerer({ id: 'y', name: 'Y', rank: 105, scrapeShow: vi.fn(), flags: [], -} as Sourcerer; -const fullSourceZBoth = { +}); +const fullSourceZBoth = makeSourcerer({ id: 'z', name: 'Z', rank: 106, scrapeMovie: vi.fn(), scrapeShow: vi.fn(), flags: [], -} as Sourcerer; +}); -const embedD = { +const embedD = makeEmbed({ id: 'd', rank: 4, disabled: false, -} as Embed; -const embedA = { +} as any); +const embedA = makeEmbed({ id: 'a', rank: 5, disabled: false, -} as Embed; -const embedEDisabled = { +} as any); +const embedEDisabled = makeEmbed({ id: 'e', rank: 6, disabled: true, -} as Embed; -const embedDHigherRank = { +} as any); +const embedDHigherRank = makeEmbed({ id: 'd', rank: 4000, disabled: false, -} as Embed; -const embedFSameRankAsA = { +} as any); +const embedFSameRankAsA = makeEmbed({ id: 'f', rank: 5, disabled: false, -} as Embed; -const embedHSameRankAsSourceA = { +} as any); +const embedHSameRankAsSourceA = makeEmbed({ id: 'h', rank: 1, disabled: false, -} as Embed; -const fullEmbedX = { +} as any); +const fullEmbedX = makeEmbed({ id: 'x', name: 'X', rank: 104, -} as Embed; -const fullEmbedZ = { +} as any); +const fullEmbedZ = makeEmbed({ id: 'z', name: 'Z', rank: 109, -} as Embed; +} as any); export const mockSources = { sourceA, diff --git a/src/__test__/providers/checks.test.ts b/src/__test__/standard/providers/checks.test.ts similarity index 98% rename from src/__test__/providers/checks.test.ts rename to src/__test__/standard/providers/checks.test.ts index 404fb31..747c240 100644 --- a/src/__test__/providers/checks.test.ts +++ b/src/__test__/standard/providers/checks.test.ts @@ -1,4 +1,4 @@ -import { mockEmbeds, mockSources } from '@/__test__/providerTests'; +import { mockEmbeds, mockSources } from '../providerTests'; import { getBuiltinEmbeds, getBuiltinSources } from '@/entrypoint/providers'; import { FeatureMap } from '@/entrypoint/utils/targets'; import { getProviders } from '@/providers/get'; diff --git a/src/__test__/runner/list.test.ts b/src/__test__/standard/runner/list.test.ts similarity index 98% rename from src/__test__/runner/list.test.ts rename to src/__test__/standard/runner/list.test.ts index 336de44..b615457 100644 --- a/src/__test__/runner/list.test.ts +++ b/src/__test__/standard/runner/list.test.ts @@ -1,4 +1,4 @@ -import { mockEmbeds, mockSources } from '@/__test__/providerTests'; +import { mockEmbeds, mockSources } from '../providerTests.ts'; import { makeProviders } from '@/entrypoint/declare'; import { targets } from '@/entrypoint/utils/targets'; import { afterEach, describe, expect, it, vi } from 'vitest'; diff --git a/src/__test__/runner/meta.test.ts b/src/__test__/standard/runner/meta.test.ts similarity index 96% rename from src/__test__/runner/meta.test.ts rename to src/__test__/standard/runner/meta.test.ts index 423a8e6..ac2fab6 100644 --- a/src/__test__/runner/meta.test.ts +++ b/src/__test__/standard/runner/meta.test.ts @@ -1,4 +1,4 @@ -import { mockEmbeds, mockSources } from '@/__test__/providerTests'; +import { mockEmbeds, mockSources } from '../providerTests.ts'; import { makeProviders } from '@/entrypoint/declare'; import { targets } from '@/entrypoint/utils/targets'; import { afterEach, describe, expect, it, vi } from 'vitest'; diff --git a/src/__test__/utils/features.test.ts b/src/__test__/standard/utils/features.test.ts similarity index 100% rename from src/__test__/utils/features.test.ts rename to src/__test__/standard/utils/features.test.ts diff --git a/src/__test__/utils/list.test.ts b/src/__test__/standard/utils/list.test.ts similarity index 100% rename from src/__test__/utils/list.test.ts rename to src/__test__/standard/utils/list.test.ts diff --git a/src/__test__/utils/valid.test.ts b/src/__test__/standard/utils/valid.test.ts similarity index 100% rename from src/__test__/utils/valid.test.ts rename to src/__test__/standard/utils/valid.test.ts diff --git a/vite.config.js b/vite.config.js index e5d4f10..8f6597d 100644 --- a/vite.config.js +++ b/vite.config.js @@ -5,7 +5,9 @@ const dts = require('vite-plugin-dts'); const pkg = require('./package.json'); const fs = require('fs/promises'); -const main = path.resolve(__dirname, 'src/index.ts'); +const shouldTestProviders = process.env.MW_TEST_PROVIDERS === "true" +let tests = ['src/__test__/standard/**/*.test.ts']; +if (shouldTestProviders) tests = ['src/__test__/providers/**/*.test.ts'] module.exports = defineConfig({ plugins: [ @@ -34,10 +36,13 @@ module.exports = defineConfig({ outDir: 'lib', lib: { - entry: main, + entry: path.resolve(__dirname, 'src/index.ts'), name: 'index', fileName: 'index', formats: ['umd', 'es'], }, }, + test: { + include: tests + } }); From 528e2774b530fe095d6717837e912ba9d1cf224b Mon Sep 17 00:00:00 2001 From: mrjvs Date: Wed, 17 Jan 2024 18:03:10 +0100 Subject: [PATCH 26/43] Add basic provider unit testing --- src/__test__/providers/abc.test.ts | 7 -- src/__test__/providers/providerUtils.ts | 88 ++++++++++++++++++++++++ src/__test__/providers/providers.test.ts | 23 +++++++ src/__test__/providers/testMedia.ts | 30 ++++++++ 4 files changed, 141 insertions(+), 7 deletions(-) delete mode 100644 src/__test__/providers/abc.test.ts create mode 100644 src/__test__/providers/providerUtils.ts create mode 100644 src/__test__/providers/providers.test.ts create mode 100644 src/__test__/providers/testMedia.ts diff --git a/src/__test__/providers/abc.test.ts b/src/__test__/providers/abc.test.ts deleted file mode 100644 index 3aed2ea..0000000 --- a/src/__test__/providers/abc.test.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { describe, expect, it } from "vitest"; - -describe('abc', () => { - it('should do thing', () => { - expect(true).toBe(false); - }) -}) diff --git a/src/__test__/providers/providerUtils.ts b/src/__test__/providers/providerUtils.ts new file mode 100644 index 0000000..b9b2bc8 --- /dev/null +++ b/src/__test__/providers/providerUtils.ts @@ -0,0 +1,88 @@ +import { ScrapeMedia } from "@/entrypoint/utils/media"; +import { Sourcerer } from "@/providers/base"; +import { buildProviders } from "@/entrypoint/builder"; +import { describe, expect, it } from "vitest"; +import { makeStandardFetcher } from "@/fetchers/standardFetch"; +import { ProviderControls } from "@/entrypoint/controls"; +import { NotFoundError } from "@/utils/errors"; +import { targets } from "@/entrypoint/utils/targets"; +import { getAllEmbedMetaSorted } from "@/entrypoint/utils/meta"; +import { getBuiltinEmbeds } from "@/entrypoint/providers"; + +export type TestTypes = 'standard' | 'ip:standard'; + +export interface TestSourceOptions { + source: Sourcerer; + testSuite: ScrapeMedia[]; + types: TestTypes[]; + debug?: boolean; + expect: { + embeds?: number; + streams?: number; + error?: boolean; + notfound?: boolean; + } +} + +// TODO add proxy support + +function makeBaseProviders() { + const builder = buildProviders() + .setTarget(targets.ANY) + .setFetcher(makeStandardFetcher(fetch)); + const embeds = getBuiltinEmbeds(); + embeds.forEach(embed => builder.addEmbed(embed)); + return builder; +} + +export function testSource(ops: TestSourceOptions) { + if (ops.testSuite.length === 0) throw new Error("Test suite must have at least one test"); + describe(`source:${ops.source.id}`, () => { + ops.testSuite.forEach((test, i) => { + async function runTest(providers: ProviderControls) { + let hasNotFound = false; + let hasError = false; + let streamCount = 0; + let embedCount = 0; + try { + const result = await providers.runSourceScraper({ + id: ops.source.id, + media: test, + }) + if (ops.debug) console.log(result); + streamCount = (result.stream ?? []).length; + embedCount = result.embeds.length; + } catch (err) { + if (ops.debug) console.log(err); + if (err instanceof NotFoundError) + hasNotFound = true; + else + hasError = true; + } + expect(ops.expect.error ?? false).toBe(hasError); + expect(ops.expect.notfound ?? false).toBe(hasNotFound); + expect(ops.expect.streams ?? 0).toBe(streamCount); + expect(ops.expect.embeds ?? 0).toBe(embedCount); + } + + if (ops.types.includes('standard')) { + it(`Should pass test ${i} - standard`, async () => { + const providers = makeBaseProviders() + .addSource(ops.source) + .build(); + await runTest(providers); + }) + } + + if (ops.types.includes('ip:standard')) { + it(`Should pass test ${i} - standard:ip`, async () => { + const providers = makeBaseProviders() + .addSource(ops.source) + .enableConsistentIpForRequests() + .build(); + await runTest(providers); + }) + } + }) + }) +} diff --git a/src/__test__/providers/providers.test.ts b/src/__test__/providers/providers.test.ts new file mode 100644 index 0000000..8a2165e --- /dev/null +++ b/src/__test__/providers/providers.test.ts @@ -0,0 +1,23 @@ +import { describe, expect, it } from "vitest"; +import { testSource } from "./providerUtils"; +import { lookmovieScraper } from "@/providers/sources/lookmovie"; +import { testMedia } from "./testMedia"; +import { showboxScraper } from "@/providers/sources/showbox"; + +testSource({ + source: lookmovieScraper, + testSuite: [testMedia.arcane, testMedia.hamilton], + types: ['ip:standard'], + expect: { + streams: 1, + } +}) + +testSource({ + source: showboxScraper, + testSuite: [testMedia.arcane, testMedia.hamilton], + types: ['standard'], + expect: { + embeds: 1, + } +}) diff --git a/src/__test__/providers/testMedia.ts b/src/__test__/providers/testMedia.ts new file mode 100644 index 0000000..0e0d050 --- /dev/null +++ b/src/__test__/providers/testMedia.ts @@ -0,0 +1,30 @@ +import { ScrapeMedia } from "@/entrypoint/utils/media"; + +function makeMedia(media: ScrapeMedia): ScrapeMedia { + return media; +} + +export const testMedia = { + arcane: makeMedia({ + type: "show", + title: "Arcane", + tmdbId: "94605", + releaseYear: 2021, + episode: { + number: 1, + tmdbId: '1953812', + }, + season: { + number: 1, + tmdbId: '134187', + }, + imdbId: 'tt11126994' + }), + hamilton: makeMedia({ + type: 'movie', + tmdbId: '556574', + imdbId: 'tt8503618', + releaseYear: 2020, + title: 'Hamilton' + }) +} From 327ba025c2000625af7a474de682640473383dac Mon Sep 17 00:00:00 2001 From: mrjvs Date: Wed, 17 Jan 2024 18:18:08 +0100 Subject: [PATCH 27/43] Enabled febbox subtitles --- src/__test__/providers/providers.test.ts | 3 ++- src/providers/embeds/febbox/mp4.ts | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/__test__/providers/providers.test.ts b/src/__test__/providers/providers.test.ts index 8a2165e..6114f37 100644 --- a/src/__test__/providers/providers.test.ts +++ b/src/__test__/providers/providers.test.ts @@ -1,9 +1,10 @@ -import { describe, expect, it } from "vitest"; import { testSource } from "./providerUtils"; import { lookmovieScraper } from "@/providers/sources/lookmovie"; import { testMedia } from "./testMedia"; import { showboxScraper } from "@/providers/sources/showbox"; +// TODO add embed testing + testSource({ source: lookmovieScraper, testSuite: [testMedia.arcane, testMedia.hamilton], diff --git a/src/providers/embeds/febbox/mp4.ts b/src/providers/embeds/febbox/mp4.ts index b69812f..1122e53 100644 --- a/src/providers/embeds/febbox/mp4.ts +++ b/src/providers/embeds/febbox/mp4.ts @@ -2,6 +2,7 @@ import { flags } from '@/entrypoint/utils/targets'; import { makeEmbed } from '@/providers/base'; import { parseInputUrl } from '@/providers/embeds/febbox/common'; import { getStreamQualities } from '@/providers/embeds/febbox/qualities'; +import { getSubtitles } from '@/providers/embeds/febbox/subtitles'; export const febboxMp4Scraper = makeEmbed({ id: 'febbox-mp4', @@ -41,7 +42,7 @@ export const febboxMp4Scraper = makeEmbed({ stream: [ { id: 'primary', - captions: [], // subtitles temporarily disabled, the endpoints are broken + captions: await getSubtitles(ctx, id, fid, type, episode, season), qualities, type: 'file', flags: [flags.CORS_ALLOWED], From 616c8a1e1b9a6a0bb7d8d7ce209e3ef3ff1e022f Mon Sep 17 00:00:00 2001 From: mrjvs Date: Wed, 17 Jan 2024 18:34:32 +0100 Subject: [PATCH 28/43] Add embed testing --- src/__test__/providers/embeds.test.ts | 17 +++++ src/__test__/providers/providerUtils.ts | 94 ++++++++++++++++++++++-- src/__test__/providers/providers.test.ts | 5 +- 3 files changed, 108 insertions(+), 8 deletions(-) create mode 100644 src/__test__/providers/embeds.test.ts diff --git a/src/__test__/providers/embeds.test.ts b/src/__test__/providers/embeds.test.ts new file mode 100644 index 0000000..5775a91 --- /dev/null +++ b/src/__test__/providers/embeds.test.ts @@ -0,0 +1,17 @@ +import { febboxMp4Scraper } from "@/providers/embeds/febbox/mp4"; +import { testEmbed } from "./providerUtils"; +import dotenv from 'dotenv'; + +dotenv.config(); + +testEmbed({ + embed: febboxMp4Scraper, + testUrls: [ + '/show/16448/1/1', + '/movie/27769//' + ], + types: ['standard', 'proxied'], + expect: { + streams: 1, + } +}) diff --git a/src/__test__/providers/providerUtils.ts b/src/__test__/providers/providerUtils.ts index b9b2bc8..4675f86 100644 --- a/src/__test__/providers/providerUtils.ts +++ b/src/__test__/providers/providerUtils.ts @@ -1,15 +1,15 @@ import { ScrapeMedia } from "@/entrypoint/utils/media"; -import { Sourcerer } from "@/providers/base"; +import { Embed, Sourcerer } from "@/providers/base"; import { buildProviders } from "@/entrypoint/builder"; import { describe, expect, it } from "vitest"; import { makeStandardFetcher } from "@/fetchers/standardFetch"; import { ProviderControls } from "@/entrypoint/controls"; import { NotFoundError } from "@/utils/errors"; import { targets } from "@/entrypoint/utils/targets"; -import { getAllEmbedMetaSorted } from "@/entrypoint/utils/meta"; import { getBuiltinEmbeds } from "@/entrypoint/providers"; +import { makeSimpleProxyFetcher } from "@/fetchers/simpleProxy"; -export type TestTypes = 'standard' | 'ip:standard'; +export type TestTypes = 'standard' | 'ip:standard' | 'proxied'; export interface TestSourceOptions { source: Sourcerer; @@ -24,14 +24,28 @@ export interface TestSourceOptions { } } -// TODO add proxy support +export interface TestEmbedOptions { + embed: Embed; + testUrls: string[]; + types: TestTypes[]; + debug?: boolean; + expect: { + streams?: number; + error?: boolean; + } +} function makeBaseProviders() { + const builder = makeBaseEmbedProviders(); + const embeds = getBuiltinEmbeds(); + embeds.forEach(embed => builder.addEmbed(embed)); + return builder; +} + +function makeBaseEmbedProviders() { const builder = buildProviders() .setTarget(targets.ANY) .setFetcher(makeStandardFetcher(fetch)); - const embeds = getBuiltinEmbeds(); - embeds.forEach(embed => builder.addEmbed(embed)); return builder; } @@ -83,6 +97,74 @@ export function testSource(ops: TestSourceOptions) { await runTest(providers); }) } + + if (ops.types.includes('proxied')) { + it(`Should pass test ${i} - proxied`, async () => { + if (!process.env.MOVIE_WEB_PROXY_URL) + throw new Error("Cant use proxied test without setting MOVIE_WEB_PROXY_URL env"); + const providers = makeBaseProviders() + .addSource(ops.source) + .setProxiedFetcher(makeSimpleProxyFetcher(process.env.MOVIE_WEB_PROXY_URL, fetch)) + .build(); + await runTest(providers); + }) + } + }) + }) +} + +export function testEmbed(ops: TestEmbedOptions) { + if (ops.testUrls.length === 0) throw new Error("Test urls must have at least one url"); + describe(`embed:${ops.embed.id}`, () => { + ops.testUrls.forEach((test, i) => { + async function runTest(providers: ProviderControls) { + let hasError = false; + let streamCount = 0; + try { + const result = await providers.runEmbedScraper({ + id: ops.embed.id, + url: test, + }) + if (ops.debug) console.log(result); + streamCount = (result.stream ?? []).length; + } catch (err) { + if (ops.debug) console.log(err); + hasError = true; + } + expect(ops.expect.error ?? false).toBe(hasError); + expect(ops.expect.streams ?? 0).toBe(streamCount); + } + + if (ops.types.includes('standard')) { + it(`Should pass test ${i} - standard`, async () => { + const providers = makeBaseEmbedProviders() + .addEmbed(ops.embed) + .build(); + await runTest(providers); + }) + } + + if (ops.types.includes('ip:standard')) { + it(`Should pass test ${i} - standard:ip`, async () => { + const providers = makeBaseEmbedProviders() + .addEmbed(ops.embed) + .enableConsistentIpForRequests() + .build(); + await runTest(providers); + }) + } + + if (ops.types.includes('proxied')) { + it(`Should pass test ${i} - proxied`, async () => { + if (!process.env.MOVIE_WEB_PROXY_URL) + throw new Error("Cant use proxied test without setting MOVIE_WEB_PROXY_URL env"); + const providers = makeBaseEmbedProviders() + .addEmbed(ops.embed) + .setProxiedFetcher(makeSimpleProxyFetcher(process.env.MOVIE_WEB_PROXY_URL, fetch)) + .build(); + await runTest(providers); + }) + } }) }) } diff --git a/src/__test__/providers/providers.test.ts b/src/__test__/providers/providers.test.ts index 6114f37..0f29dc1 100644 --- a/src/__test__/providers/providers.test.ts +++ b/src/__test__/providers/providers.test.ts @@ -2,8 +2,9 @@ import { testSource } from "./providerUtils"; import { lookmovieScraper } from "@/providers/sources/lookmovie"; import { testMedia } from "./testMedia"; import { showboxScraper } from "@/providers/sources/showbox"; +import dotenv from 'dotenv'; -// TODO add embed testing +dotenv.config(); testSource({ source: lookmovieScraper, @@ -17,7 +18,7 @@ testSource({ testSource({ source: showboxScraper, testSuite: [testMedia.arcane, testMedia.hamilton], - types: ['standard'], + types: ['standard', 'proxied'], expect: { embeds: 1, } From 03a628ea34de115f4f9e3a2ac9a7601b8456e836 Mon Sep 17 00:00:00 2001 From: mrjvs Date: Wed, 17 Jan 2024 18:54:42 +0100 Subject: [PATCH 29/43] Improve logging of provider unit tests --- package.json | 2 +- src/__test__/providers/providerUtils.ts | 197 ++++++++++++------------ 2 files changed, 102 insertions(+), 97 deletions(-) diff --git a/package.json b/package.json index 9bb36e9..d8a8680 100644 --- a/package.json +++ b/package.json @@ -38,7 +38,7 @@ "cli": "ts-node ./src/dev-cli/index.ts", "test": "vitest run", "test:watch": "vitest", - "test:providers": "cross-env MW_TEST_PROVIDERS=true vitest run", + "test:providers": "cross-env MW_TEST_PROVIDERS=true vitest run --reporter verbose", "test:integration": "node ./tests/cjs && node ./tests/esm && node ./tests/browser", "test:coverage": "vitest run --coverage", "lint": "eslint --ext .ts,.js src/", diff --git a/src/__test__/providers/providerUtils.ts b/src/__test__/providers/providerUtils.ts index 4675f86..f50d53f 100644 --- a/src/__test__/providers/providerUtils.ts +++ b/src/__test__/providers/providerUtils.ts @@ -53,62 +53,65 @@ export function testSource(ops: TestSourceOptions) { if (ops.testSuite.length === 0) throw new Error("Test suite must have at least one test"); describe(`source:${ops.source.id}`, () => { ops.testSuite.forEach((test, i) => { - async function runTest(providers: ProviderControls) { - let hasNotFound = false; - let hasError = false; - let streamCount = 0; - let embedCount = 0; - try { - const result = await providers.runSourceScraper({ - id: ops.source.id, - media: test, - }) - if (ops.debug) console.log(result); - streamCount = (result.stream ?? []).length; - embedCount = result.embeds.length; - } catch (err) { - if (ops.debug) console.log(err); - if (err instanceof NotFoundError) - hasNotFound = true; - else - hasError = true; + describe(`test ${i}`, () => { + async function runTest(providers: ProviderControls) { + let hasNotFound = false; + let hasError = false; + let streamCount = 0; + let embedCount = 0; + try { + const result = await providers.runSourceScraper({ + id: ops.source.id, + media: test, + }) + if (ops.debug) console.log(result); + streamCount = (result.stream ?? []).length; + embedCount = result.embeds.length; + } catch (err) { + if (ops.debug) console.log(err); + if (err instanceof NotFoundError) + hasNotFound = true; + else + hasError = true; + } + expect(ops.expect.error ?? false).toBe(hasError); + expect(ops.expect.notfound ?? false).toBe(hasNotFound); + expect(ops.expect.streams ?? 0).toBe(streamCount); + expect(ops.expect.embeds ?? 0).toBe(embedCount); } - expect(ops.expect.error ?? false).toBe(hasError); - expect(ops.expect.notfound ?? false).toBe(hasNotFound); - expect(ops.expect.streams ?? 0).toBe(streamCount); - expect(ops.expect.embeds ?? 0).toBe(embedCount); - } - if (ops.types.includes('standard')) { - it(`Should pass test ${i} - standard`, async () => { - const providers = makeBaseProviders() - .addSource(ops.source) - .build(); - await runTest(providers); - }) - } + if (ops.types.includes('standard')) { + it(`Should pass test ${i} - standard`, async () => { + const providers = makeBaseProviders() + .addSource(ops.source) + .build(); + await runTest(providers); + }) + } - if (ops.types.includes('ip:standard')) { - it(`Should pass test ${i} - standard:ip`, async () => { - const providers = makeBaseProviders() - .addSource(ops.source) - .enableConsistentIpForRequests() - .build(); - await runTest(providers); - }) - } + if (ops.types.includes('ip:standard')) { + it(`Should pass test ${i} - standard:ip`, async () => { + const providers = makeBaseProviders() + .addSource(ops.source) + .enableConsistentIpForRequests() + .build(); + await runTest(providers); + }) + } - if (ops.types.includes('proxied')) { - it(`Should pass test ${i} - proxied`, async () => { - if (!process.env.MOVIE_WEB_PROXY_URL) - throw new Error("Cant use proxied test without setting MOVIE_WEB_PROXY_URL env"); - const providers = makeBaseProviders() - .addSource(ops.source) - .setProxiedFetcher(makeSimpleProxyFetcher(process.env.MOVIE_WEB_PROXY_URL, fetch)) - .build(); - await runTest(providers); - }) - } + if (ops.types.includes('proxied')) { + it(`Should pass test ${i} - proxied`, async () => { + if (!process.env.MOVIE_WEB_PROXY_URL) + throw new Error("Cant use proxied test without setting MOVIE_WEB_PROXY_URL env"); + const providers = makeBaseProviders() + .addSource(ops.source) + .setProxiedFetcher(makeSimpleProxyFetcher(process.env.MOVIE_WEB_PROXY_URL, fetch)) + .build(); + await runTest(providers); + }) + } + + }) }) }) } @@ -117,54 +120,56 @@ export function testEmbed(ops: TestEmbedOptions) { if (ops.testUrls.length === 0) throw new Error("Test urls must have at least one url"); describe(`embed:${ops.embed.id}`, () => { ops.testUrls.forEach((test, i) => { - async function runTest(providers: ProviderControls) { - let hasError = false; - let streamCount = 0; - try { - const result = await providers.runEmbedScraper({ - id: ops.embed.id, - url: test, - }) - if (ops.debug) console.log(result); - streamCount = (result.stream ?? []).length; - } catch (err) { - if (ops.debug) console.log(err); - hasError = true; + describe(`test ${i}`, () => { + async function runTest(providers: ProviderControls) { + let hasError = false; + let streamCount = 0; + try { + const result = await providers.runEmbedScraper({ + id: ops.embed.id, + url: test, + }) + if (ops.debug) console.log(result); + streamCount = (result.stream ?? []).length; + } catch (err) { + if (ops.debug) console.log(err); + hasError = true; + } + expect(ops.expect.error ?? false).toBe(hasError); + expect(ops.expect.streams ?? 0).toBe(streamCount); } - expect(ops.expect.error ?? false).toBe(hasError); - expect(ops.expect.streams ?? 0).toBe(streamCount); - } - if (ops.types.includes('standard')) { - it(`Should pass test ${i} - standard`, async () => { - const providers = makeBaseEmbedProviders() - .addEmbed(ops.embed) - .build(); - await runTest(providers); - }) - } + if (ops.types.includes('standard')) { + it(`Should pass test ${i} - standard`, async () => { + const providers = makeBaseEmbedProviders() + .addEmbed(ops.embed) + .build(); + await runTest(providers); + }) + } - if (ops.types.includes('ip:standard')) { - it(`Should pass test ${i} - standard:ip`, async () => { - const providers = makeBaseEmbedProviders() - .addEmbed(ops.embed) - .enableConsistentIpForRequests() - .build(); - await runTest(providers); - }) - } + if (ops.types.includes('ip:standard')) { + it(`Should pass test ${i} - standard:ip`, async () => { + const providers = makeBaseEmbedProviders() + .addEmbed(ops.embed) + .enableConsistentIpForRequests() + .build(); + await runTest(providers); + }) + } - if (ops.types.includes('proxied')) { - it(`Should pass test ${i} - proxied`, async () => { - if (!process.env.MOVIE_WEB_PROXY_URL) - throw new Error("Cant use proxied test without setting MOVIE_WEB_PROXY_URL env"); - const providers = makeBaseEmbedProviders() - .addEmbed(ops.embed) - .setProxiedFetcher(makeSimpleProxyFetcher(process.env.MOVIE_WEB_PROXY_URL, fetch)) - .build(); - await runTest(providers); - }) - } + if (ops.types.includes('proxied')) { + it(`Should pass test ${i} - proxied`, async () => { + if (!process.env.MOVIE_WEB_PROXY_URL) + throw new Error("Cant use proxied test without setting MOVIE_WEB_PROXY_URL env"); + const providers = makeBaseEmbedProviders() + .addEmbed(ops.embed) + .setProxiedFetcher(makeSimpleProxyFetcher(process.env.MOVIE_WEB_PROXY_URL, fetch)) + .build(); + await runTest(providers); + }) + } + }); }) }) } From 949fe487bc09faca006411963268f6d2eae4c690 Mon Sep 17 00:00:00 2001 From: mrjvs Date: Wed, 17 Jan 2024 19:22:37 +0100 Subject: [PATCH 30/43] fix some stuff --- package-lock.json | 11 +++++++++-- package.json | 2 +- src/__test__/providers/providerUtils.ts | 12 ++++++------ 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/package-lock.json b/package-lock.json index 9eebc03..9921a7b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,7 +10,6 @@ "license": "MIT", "dependencies": { "cheerio": "^1.0.0-rc.12", - "cross-env": "^7.0.3", "crypto-js": "^4.1.1", "form-data": "^4.0.0", "iso-639-1": "^3.1.0", @@ -27,6 +26,7 @@ "@typescript-eslint/parser": "^5.60.0", "@vitest/coverage-v8": "^0.34.3", "commander": "^11.0.0", + "cross-env": "^7.0.3", "dotenv": "^16.3.1", "enquirer": "^2.4.1", "eslint": "^8.30.0", @@ -1799,6 +1799,7 @@ "version": "7.0.3", "resolved": "https://registry.npmjs.org/cross-env/-/cross-env-7.0.3.tgz", "integrity": "sha512-+/HKd6EgcQCJGh2PSjZuUitQBQynKor4wrFbRg4DtAgS1aWO+gU52xpH7M9ScGgXSYmAVS9bIJ8EzuaGw0oNAw==", + "dev": true, "dependencies": { "cross-spawn": "^7.0.1" }, @@ -1823,6 +1824,7 @@ }, "node_modules/cross-spawn": { "version": "7.0.3", + "dev": true, "license": "MIT", "dependencies": { "path-key": "^3.1.0", @@ -3602,7 +3604,8 @@ "node_modules/isexe": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", - "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==" + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "dev": true }, "node_modules/iso-639-1": { "version": "3.1.0", @@ -4293,6 +4296,7 @@ }, "node_modules/path-key": { "version": "3.1.1", + "dev": true, "license": "MIT", "engines": { "node": ">=8" @@ -4800,6 +4804,7 @@ }, "node_modules/shebang-command": { "version": "2.0.0", + "dev": true, "license": "MIT", "dependencies": { "shebang-regex": "^3.0.0" @@ -4810,6 +4815,7 @@ }, "node_modules/shebang-regex": { "version": "3.0.0", + "dev": true, "license": "MIT", "engines": { "node": ">=8" @@ -5787,6 +5793,7 @@ }, "node_modules/which": { "version": "2.0.2", + "dev": true, "license": "ISC", "dependencies": { "isexe": "^2.0.0" diff --git a/package.json b/package.json index d8a8680..4047136 100644 --- a/package.json +++ b/package.json @@ -56,6 +56,7 @@ "@typescript-eslint/parser": "^5.60.0", "@vitest/coverage-v8": "^0.34.3", "commander": "^11.0.0", + "cross-env": "^7.0.3", "dotenv": "^16.3.1", "enquirer": "^2.4.1", "eslint": "^8.30.0", @@ -79,7 +80,6 @@ }, "dependencies": { "cheerio": "^1.0.0-rc.12", - "cross-env": "^7.0.3", "crypto-js": "^4.1.1", "form-data": "^4.0.0", "iso-639-1": "^3.1.0", diff --git a/src/__test__/providers/providerUtils.ts b/src/__test__/providers/providerUtils.ts index f50d53f..65fe710 100644 --- a/src/__test__/providers/providerUtils.ts +++ b/src/__test__/providers/providerUtils.ts @@ -81,7 +81,7 @@ export function testSource(ops: TestSourceOptions) { } if (ops.types.includes('standard')) { - it(`Should pass test ${i} - standard`, async () => { + it(`standard`, async () => { const providers = makeBaseProviders() .addSource(ops.source) .build(); @@ -90,7 +90,7 @@ export function testSource(ops: TestSourceOptions) { } if (ops.types.includes('ip:standard')) { - it(`Should pass test ${i} - standard:ip`, async () => { + it(`standard:ip`, async () => { const providers = makeBaseProviders() .addSource(ops.source) .enableConsistentIpForRequests() @@ -100,7 +100,7 @@ export function testSource(ops: TestSourceOptions) { } if (ops.types.includes('proxied')) { - it(`Should pass test ${i} - proxied`, async () => { + it(`proxied`, async () => { if (!process.env.MOVIE_WEB_PROXY_URL) throw new Error("Cant use proxied test without setting MOVIE_WEB_PROXY_URL env"); const providers = makeBaseProviders() @@ -140,7 +140,7 @@ export function testEmbed(ops: TestEmbedOptions) { } if (ops.types.includes('standard')) { - it(`Should pass test ${i} - standard`, async () => { + it(`standard`, async () => { const providers = makeBaseEmbedProviders() .addEmbed(ops.embed) .build(); @@ -149,7 +149,7 @@ export function testEmbed(ops: TestEmbedOptions) { } if (ops.types.includes('ip:standard')) { - it(`Should pass test ${i} - standard:ip`, async () => { + it(`standard:ip`, async () => { const providers = makeBaseEmbedProviders() .addEmbed(ops.embed) .enableConsistentIpForRequests() @@ -159,7 +159,7 @@ export function testEmbed(ops: TestEmbedOptions) { } if (ops.types.includes('proxied')) { - it(`Should pass test ${i} - proxied`, async () => { + it(`proxied`, async () => { if (!process.env.MOVIE_WEB_PROXY_URL) throw new Error("Cant use proxied test without setting MOVIE_WEB_PROXY_URL env"); const providers = makeBaseEmbedProviders() From e8a8909ecb4cce8bb3484ce6058d6e019e5125f7 Mon Sep 17 00:00:00 2001 From: mrjvs Date: Wed, 17 Jan 2024 21:15:56 +0100 Subject: [PATCH 31/43] Add better embed testing + add everything --- src/__test__/providers/embedUtils.ts | 90 ++++++++++++++++++ src/__test__/providers/embeds.test.ts | 113 +++++++++++++++++++++-- src/__test__/providers/providerUtils.ts | 85 ++--------------- src/__test__/providers/providers.test.ts | 70 ++++++++++++++ 4 files changed, 273 insertions(+), 85 deletions(-) create mode 100644 src/__test__/providers/embedUtils.ts diff --git a/src/__test__/providers/embedUtils.ts b/src/__test__/providers/embedUtils.ts new file mode 100644 index 0000000..3ba88a3 --- /dev/null +++ b/src/__test__/providers/embedUtils.ts @@ -0,0 +1,90 @@ +import { buildProviders } from "@/entrypoint/builder"; +import { ScrapeMedia } from "@/entrypoint/utils/media"; +import { targets } from "@/entrypoint/utils/targets"; +import { makeStandardFetcher } from "@/fetchers/standardFetch"; +import { Embed, Sourcerer, SourcererEmbed } from "@/providers/base"; +import { TestTypes } from "./providerUtils"; +import { describe, expect, it } from "vitest"; +import { ProviderControls } from "@/entrypoint/controls"; +import { makeSimpleProxyFetcher } from "@/fetchers/simpleProxy"; + +export interface TestEmbedOptions { + embed: Embed; + source: Sourcerer; + testSuite: ScrapeMedia[]; + types: TestTypes[]; + debug?: boolean; + expect: { + embeds: number; + streams?: number; + error?: boolean; + } +} + +function makeBaseEmbedProviders() { + const builder = buildProviders() + .setTarget(targets.ANY) + .setFetcher(makeStandardFetcher(fetch)); + return builder; +} + +export function testEmbed(ops: TestEmbedOptions) { + if (ops.testSuite.length === 0) throw new Error("Test suite must have at least one test"); + describe(`embed:${ops.source.id}:${ops.embed.id}`, () => { + ops.testSuite.forEach((test) => { + describe(`test ${test.title}`, async () => { + async function gatherEmbeds(providers: ProviderControls): Promise { + const results = await providers.runSourceScraper({ + id: ops.source.id, + media: test, + }) + if (results.embeds.length !== ops.expect.embeds) throw new Error(`Embeds don't match expected amount of embeds (${ops.source.id}, ${ops.embed.id}, got ${results.embeds.length} but expected ${ops.expect.embeds})`); + return results.embeds; + } + + async function runTest(providers: ProviderControls, embedUrl: string) { + let hasError = false; + let streamCount = 0; + try { + const result = await providers.runEmbedScraper({ + id: ops.embed.id, + url: embedUrl, + }) + if (ops.debug) console.log(result); + streamCount = (result.stream ?? []).length; + } catch (err) { + if (ops.debug) console.log(err); + hasError = true; + } + expect(ops.expect.error ?? false).toBe(hasError); + expect(ops.expect.streams ?? 0).toBe(streamCount); + } + + for (const t of ops.types) { + const builder = makeBaseEmbedProviders().addSource(ops.source).addEmbed(ops.embed); + if (t === 'standard') {} + else if (t === 'ip:standard') + builder.enableConsistentIpForRequests(); + else if (t === 'proxied') { + if (!process.env.MOVIE_WEB_PROXY_URL) + throw new Error("Cant use proxied test without setting MOVIE_WEB_PROXY_URL env"); + builder.setProxiedFetcher(makeSimpleProxyFetcher(process.env.MOVIE_WEB_PROXY_URL, fetch)); + } + const providers = builder.build(); + try { + const embeds = await gatherEmbeds(providers); + embeds.forEach((embed, i) => { + it(`${t} - embed ${i}`, async () => { + await runTest(providers, embed.url); + }) + }) + } catch (err) { + it(`${t} - embed ??`, () => { + throw new Error("Failed to get streams: " + err); + }) + } + } + }) + }) + }) +} diff --git a/src/__test__/providers/embeds.test.ts b/src/__test__/providers/embeds.test.ts index 5775a91..d9d1d23 100644 --- a/src/__test__/providers/embeds.test.ts +++ b/src/__test__/providers/embeds.test.ts @@ -1,17 +1,118 @@ -import { febboxMp4Scraper } from "@/providers/embeds/febbox/mp4"; -import { testEmbed } from "./providerUtils"; import dotenv from 'dotenv'; +import { febboxMp4Scraper } from "@/providers/embeds/febbox/mp4"; +import { testEmbed } from "./embedUtils"; +import { showboxScraper } from "@/providers/sources/showbox"; +import { testMedia } from "./testMedia"; +import { flixhqScraper } from "@/providers/sources/flixhq"; +import { upcloudScraper } from "@/providers/embeds/upcloud"; +import { goMoviesScraper } from "@/providers/sources/gomovies"; +import { smashyStreamScraper } from "@/providers/sources/smashystream"; +import { smashyStreamDScraper } from "@/providers/embeds/smashystream/dued"; +import { vidsrcembedScraper } from '@/providers/embeds/vidsrc'; +import { vidsrcScraper } from '@/providers/sources/vidsrc'; +import { vidSrcToScraper } from '@/providers/sources/vidsrcto'; +import { vidplayScraper } from '@/providers/embeds/vidplay'; +import { fileMoonScraper } from '@/providers/embeds/filemoon'; +import { zoechipScraper } from '@/providers/sources/zoechip'; +import { mixdropScraper } from '@/providers/embeds/mixdrop'; dotenv.config(); testEmbed({ embed: febboxMp4Scraper, - testUrls: [ - '/show/16448/1/1', - '/movie/27769//' - ], + source: showboxScraper, + testSuite: [testMedia.arcane, testMedia.hamilton], types: ['standard', 'proxied'], expect: { + embeds: 1, + streams: 1, + } +}) + +testEmbed({ + embed: upcloudScraper, + source: flixhqScraper, + testSuite: [testMedia.arcane, testMedia.hamilton], + types: ['standard', 'proxied'], + expect: { + embeds: 1, + streams: 1, + } +}) + +testEmbed({ + embed: upcloudScraper, + source: goMoviesScraper, + testSuite: [testMedia.arcane, testMedia.hamilton], + types: ['standard', 'proxied'], + expect: { + embeds: 1, + streams: 1, + } +}) + +testEmbed({ + embed: smashyStreamDScraper, + source: smashyStreamScraper, + testSuite: [testMedia.arcane, testMedia.hamilton], + types: ['standard', 'proxied'], + expect: { + embeds: 1, + streams: 1, + } +}) + +testEmbed({ + embed: vidsrcembedScraper, + source: vidsrcScraper, + testSuite: [testMedia.arcane, testMedia.hamilton], + types: ['standard', 'proxied'], + expect: { + embeds: 1, + streams: 1, + } +}) + +testEmbed({ + embed: vidplayScraper, + source: vidSrcToScraper, + testSuite: [testMedia.arcane, testMedia.hamilton], + types: ['standard', 'proxied'], + expect: { + embeds: 1, + streams: 1, + } +}) + +testEmbed({ + embed: fileMoonScraper, + source: vidSrcToScraper, + testSuite: [testMedia.arcane, testMedia.hamilton], + types: ['standard', 'proxied'], + expect: { + embeds: 1, + streams: 1, + } +}) + +testEmbed({ + embed: upcloudScraper, + source: zoechipScraper, + testSuite: [testMedia.arcane, testMedia.hamilton], + types: ['standard', 'proxied'], + expect: { + embeds: 2, + streams: 1, + } +}) + +testEmbed({ + embed: mixdropScraper, + source: zoechipScraper, + testSuite: [testMedia.arcane, testMedia.hamilton], + types: ['standard', 'proxied'], + expect: { + embeds: 2, streams: 1, } }) diff --git a/src/__test__/providers/providerUtils.ts b/src/__test__/providers/providerUtils.ts index 65fe710..f5d126c 100644 --- a/src/__test__/providers/providerUtils.ts +++ b/src/__test__/providers/providerUtils.ts @@ -1,5 +1,5 @@ import { ScrapeMedia } from "@/entrypoint/utils/media"; -import { Embed, Sourcerer } from "@/providers/base"; +import { Embed, Sourcerer, SourcererEmbed } from "@/providers/base"; import { buildProviders } from "@/entrypoint/builder"; import { describe, expect, it } from "vitest"; import { makeStandardFetcher } from "@/fetchers/standardFetch"; @@ -24,41 +24,26 @@ export interface TestSourceOptions { } } -export interface TestEmbedOptions { - embed: Embed; - testUrls: string[]; - types: TestTypes[]; - debug?: boolean; - expect: { - streams?: number; - error?: boolean; - } -} - function makeBaseProviders() { - const builder = makeBaseEmbedProviders(); - const embeds = getBuiltinEmbeds(); - embeds.forEach(embed => builder.addEmbed(embed)); - return builder; -} - -function makeBaseEmbedProviders() { const builder = buildProviders() .setTarget(targets.ANY) .setFetcher(makeStandardFetcher(fetch)); + const embeds = getBuiltinEmbeds(); + embeds.forEach(embed => builder.addEmbed(embed)); return builder; } export function testSource(ops: TestSourceOptions) { if (ops.testSuite.length === 0) throw new Error("Test suite must have at least one test"); describe(`source:${ops.source.id}`, () => { - ops.testSuite.forEach((test, i) => { - describe(`test ${i}`, () => { + ops.testSuite.forEach((test) => { + describe(`test ${test.title}`, () => { async function runTest(providers: ProviderControls) { let hasNotFound = false; let hasError = false; let streamCount = 0; let embedCount = 0; + let embeds = []; try { const result = await providers.runSourceScraper({ id: ops.source.id, @@ -115,61 +100,3 @@ export function testSource(ops: TestSourceOptions) { }) }) } - -export function testEmbed(ops: TestEmbedOptions) { - if (ops.testUrls.length === 0) throw new Error("Test urls must have at least one url"); - describe(`embed:${ops.embed.id}`, () => { - ops.testUrls.forEach((test, i) => { - describe(`test ${i}`, () => { - async function runTest(providers: ProviderControls) { - let hasError = false; - let streamCount = 0; - try { - const result = await providers.runEmbedScraper({ - id: ops.embed.id, - url: test, - }) - if (ops.debug) console.log(result); - streamCount = (result.stream ?? []).length; - } catch (err) { - if (ops.debug) console.log(err); - hasError = true; - } - expect(ops.expect.error ?? false).toBe(hasError); - expect(ops.expect.streams ?? 0).toBe(streamCount); - } - - if (ops.types.includes('standard')) { - it(`standard`, async () => { - const providers = makeBaseEmbedProviders() - .addEmbed(ops.embed) - .build(); - await runTest(providers); - }) - } - - if (ops.types.includes('ip:standard')) { - it(`standard:ip`, async () => { - const providers = makeBaseEmbedProviders() - .addEmbed(ops.embed) - .enableConsistentIpForRequests() - .build(); - await runTest(providers); - }) - } - - if (ops.types.includes('proxied')) { - it(`proxied`, async () => { - if (!process.env.MOVIE_WEB_PROXY_URL) - throw new Error("Cant use proxied test without setting MOVIE_WEB_PROXY_URL env"); - const providers = makeBaseEmbedProviders() - .addEmbed(ops.embed) - .setProxiedFetcher(makeSimpleProxyFetcher(process.env.MOVIE_WEB_PROXY_URL, fetch)) - .build(); - await runTest(providers); - }) - } - }); - }) - }) -} diff --git a/src/__test__/providers/providers.test.ts b/src/__test__/providers/providers.test.ts index 0f29dc1..ecac090 100644 --- a/src/__test__/providers/providers.test.ts +++ b/src/__test__/providers/providers.test.ts @@ -3,6 +3,13 @@ import { lookmovieScraper } from "@/providers/sources/lookmovie"; import { testMedia } from "./testMedia"; import { showboxScraper } from "@/providers/sources/showbox"; import dotenv from 'dotenv'; +import { flixhqScraper } from "@/providers/sources/flixhq"; +import { goMoviesScraper } from "@/providers/sources/gomovies"; +import { smashyStreamScraper } from "@/providers/sources/smashystream"; +import { vidsrcScraper } from "@/providers/sources/vidsrc"; +import { vidSrcToScraper } from "@/providers/sources/vidsrcto"; +import { zoechipScraper } from "@/providers/sources/zoechip"; +import { remotestreamScraper } from "@/providers/sources/remotestream"; dotenv.config(); @@ -23,3 +30,66 @@ testSource({ embeds: 1, } }) + +testSource({ + source: flixhqScraper, + testSuite: [testMedia.arcane, testMedia.hamilton], + types: ['standard', 'proxied'], + expect: { + embeds: 1, + } +}) + +testSource({ + source: goMoviesScraper, + testSuite: [testMedia.arcane, testMedia.hamilton], + types: ['standard', 'proxied'], + expect: { + embeds: 1, + } +}) + +testSource({ + source: smashyStreamScraper, + testSuite: [testMedia.arcane, testMedia.hamilton], + types: ['standard', 'proxied'], + expect: { + embeds: 1, + } +}) + +testSource({ + source: vidsrcScraper, + testSuite: [testMedia.arcane, testMedia.hamilton], + types: ['standard', 'proxied'], + expect: { + embeds: 1, + } +}) + +testSource({ + source: vidSrcToScraper, + testSuite: [testMedia.arcane, testMedia.hamilton], + types: ['standard', 'proxied'], + expect: { + embeds: 2, + } +}) + +testSource({ + source: zoechipScraper, + testSuite: [testMedia.arcane, testMedia.hamilton], + types: ['standard', 'proxied'], + expect: { + embeds: 3, + } +}) + +testSource({ + source: remotestreamScraper, + testSuite: [testMedia.arcane, testMedia.hamilton], + types: ['standard', 'proxied'], + expect: { + streams: 1, + } +}) From f3e4786b72ceca26a38752cc288c4a78ce44c485 Mon Sep 17 00:00:00 2001 From: Jorrin Date: Fri, 19 Jan 2024 20:04:50 +0100 Subject: [PATCH 32/43] Remove equal signs inside base64 encoded string --- src/providers/embeds/vidsrc.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/providers/embeds/vidsrc.ts b/src/providers/embeds/vidsrc.ts index cd47e21..9da5a9f 100644 --- a/src/providers/embeds/vidsrc.ts +++ b/src/providers/embeds/vidsrc.ts @@ -18,7 +18,8 @@ export const vidsrcembedScraper = makeEmbed({ const match = html .match(hlsURLRegex)?.[1] ?.replace(/(\/\/\S+?=)/g, '') - .replace('#2', ''); + .replace('#2', '') + .replace(/=/g, ''); if (!match) throw new Error('Unable to find HLS playlist'); const finalUrl = atob(match); From b13578cefadd5639c7e2b0a6a7737ae3f615458d Mon Sep 17 00:00:00 2001 From: Jorrin Date: Fri, 19 Jan 2024 20:13:44 +0100 Subject: [PATCH 33/43] improve regex --- src/providers/embeds/vidsrc.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/providers/embeds/vidsrc.ts b/src/providers/embeds/vidsrc.ts index 9da5a9f..93d55c5 100644 --- a/src/providers/embeds/vidsrc.ts +++ b/src/providers/embeds/vidsrc.ts @@ -15,11 +15,7 @@ export const vidsrcembedScraper = makeEmbed({ }, }); - const match = html - .match(hlsURLRegex)?.[1] - ?.replace(/(\/\/\S+?=)/g, '') - .replace('#2', '') - .replace(/=/g, ''); + const match = html.match(hlsURLRegex)?.[1]?.replace(/(\/\/\S+?=)|#2|=/g, ''); if (!match) throw new Error('Unable to find HLS playlist'); const finalUrl = atob(match); From ec4f2d02edf4bb7c1c5a95db1521598e12c2a58a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 20 Jan 2024 05:46:23 +0000 Subject: [PATCH 34/43] Bump vite from 4.5.1 to 4.5.2 in /.docs Bumps [vite](https://github.com/vitejs/vite/tree/HEAD/packages/vite) from 4.5.1 to 4.5.2. - [Release notes](https://github.com/vitejs/vite/releases) - [Changelog](https://github.com/vitejs/vite/blob/v4.5.2/packages/vite/CHANGELOG.md) - [Commits](https://github.com/vitejs/vite/commits/v4.5.2/packages/vite) --- updated-dependencies: - dependency-name: vite dependency-type: indirect ... Signed-off-by: dependabot[bot] --- .docs/package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.docs/package-lock.json b/.docs/package-lock.json index 3a9fa95..11e9e1a 100644 --- a/.docs/package-lock.json +++ b/.docs/package-lock.json @@ -17286,9 +17286,9 @@ "dev": true }, "node_modules/vite": { - "version": "4.5.1", - "resolved": "https://registry.npmjs.org/vite/-/vite-4.5.1.tgz", - "integrity": "sha512-AXXFaAJ8yebyqzoNB9fu2pHoo/nWX+xZlaRwoeYUxEqBO+Zj4msE5G+BhGBll9lYEKv9Hfks52PAF2X7qDYXQA==", + "version": "4.5.2", + "resolved": "https://registry.npmjs.org/vite/-/vite-4.5.2.tgz", + "integrity": "sha512-tBCZBNSBbHQkaGyhGCDUGqeo2ph8Fstyp6FMSvTtsXeZSPpSMGlviAOav2hxVTqFcx8Hj/twtWKsMJXNY0xI8w==", "dev": true, "dependencies": { "esbuild": "^0.18.10", From efa4857b20a8f86ab9a88d6c4a5055bbf029cbc5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 20 Jan 2024 05:53:00 +0000 Subject: [PATCH 35/43] Bump vite from 4.5.1 to 4.5.2 Bumps [vite](https://github.com/vitejs/vite/tree/HEAD/packages/vite) from 4.5.1 to 4.5.2. - [Release notes](https://github.com/vitejs/vite/releases) - [Changelog](https://github.com/vitejs/vite/blob/v4.5.2/packages/vite/CHANGELOG.md) - [Commits](https://github.com/vitejs/vite/commits/v4.5.2/packages/vite) --- updated-dependencies: - dependency-name: vite dependency-type: direct:development ... Signed-off-by: dependabot[bot] --- package-lock.json | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index f38bd59..9954f93 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2848,6 +2848,20 @@ "dev": true, "license": "ISC" }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, "node_modules/function-bind": { "version": "1.1.2", "dev": true, @@ -5517,9 +5531,10 @@ } }, "node_modules/vite": { - "version": "4.5.1", + "version": "4.5.2", + "resolved": "https://registry.npmjs.org/vite/-/vite-4.5.2.tgz", + "integrity": "sha512-tBCZBNSBbHQkaGyhGCDUGqeo2ph8Fstyp6FMSvTtsXeZSPpSMGlviAOav2hxVTqFcx8Hj/twtWKsMJXNY0xI8w==", "dev": true, - "license": "MIT", "dependencies": { "esbuild": "^0.18.10", "postcss": "^8.4.27", From f2073e89d95f5669b2312e4cf356c97107917bf5 Mon Sep 17 00:00:00 2001 From: mrjvs Date: Sat, 20 Jan 2024 14:18:44 +0100 Subject: [PATCH 36/43] Fix remotestream and linting error --- src/providers/embeds/vidplay/index.ts | 1 - src/providers/sources/remotestream.ts | 8 ++++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/providers/embeds/vidplay/index.ts b/src/providers/embeds/vidplay/index.ts index 9d205e1..48af6c1 100644 --- a/src/providers/embeds/vidplay/index.ts +++ b/src/providers/embeds/vidplay/index.ts @@ -1,4 +1,3 @@ -import { flags } from '@/entrypoint/utils/targets'; import { makeEmbed } from '@/providers/base'; import { Caption, getCaptionTypeFromUrl, labelToLanguageCode } from '@/providers/captions'; diff --git a/src/providers/sources/remotestream.ts b/src/providers/sources/remotestream.ts index f0a0ae8..fa2efc4 100644 --- a/src/providers/sources/remotestream.ts +++ b/src/providers/sources/remotestream.ts @@ -19,8 +19,8 @@ export const remotestreamScraper = makeSourcerer({ const playlistLink = `${remotestreamBase}/Shows/${ctx.media.tmdbId}/${seasonNumber}/${episodeNumber}/${episodeNumber}.m3u8`; ctx.progress(30); - const streamRes = await ctx.fetcher.full(playlistLink, { - method: 'HEAD', + const streamRes = await ctx.proxiedFetcher.full(playlistLink, { + method: 'GET', readHeaders: ['content-type'], headers: { Referer: referer, @@ -51,8 +51,8 @@ export const remotestreamScraper = makeSourcerer({ const playlistLink = `${remotestreamBase}/Movies/${ctx.media.tmdbId}/${ctx.media.tmdbId}.m3u8`; ctx.progress(30); - const streamRes = await ctx.fetcher.full(playlistLink, { - method: 'HEAD', + const streamRes = await ctx.proxiedFetcher.full(playlistLink, { + method: 'GET', readHeaders: ['content-type'], headers: { Referer: referer, From 2df4f10f642287a00a01bc7bd0246f066de7814d Mon Sep 17 00:00:00 2001 From: mrjvs Date: Sat, 20 Jan 2024 17:54:44 +0100 Subject: [PATCH 37/43] Version bump + lookmovie reorder --- package.json | 2 +- src/providers/sources/lookmovie/index.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 4047136..0cd0ced 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@movie-web/providers", - "version": "2.0.5", + "version": "2.1.0", "description": "Package that contains all the providers of movie-web", "main": "./lib/index.umd.js", "types": "./lib/index.d.ts", diff --git a/src/providers/sources/lookmovie/index.ts b/src/providers/sources/lookmovie/index.ts index 8611373..6fbe6de 100644 --- a/src/providers/sources/lookmovie/index.ts +++ b/src/providers/sources/lookmovie/index.ts @@ -32,7 +32,7 @@ async function universalScraper(ctx: MovieScrapeContext | ShowScrapeContext): Pr export const lookmovieScraper = makeSourcerer({ id: 'lookmovie', name: 'LookMovie', - rank: 1, + rank: 700, flags: [flags.IP_LOCKED], scrapeShow: universalScraper, scrapeMovie: universalScraper, From 09cfafcaa265a977936f8640b56163f9046fd992 Mon Sep 17 00:00:00 2001 From: mrjvs Date: Sat, 20 Jan 2024 18:02:49 +0100 Subject: [PATCH 38/43] update changelog --- .docs/content/1.get-started/4.changelog.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.docs/content/1.get-started/4.changelog.md b/.docs/content/1.get-started/4.changelog.md index 239ed6b..a19930a 100644 --- a/.docs/content/1.get-started/4.changelog.md +++ b/.docs/content/1.get-started/4.changelog.md @@ -2,6 +2,12 @@ title: 'Changelog' --- +# Version 2.1.0 + - Add preferedHeaders to most sources + - Add CF_BLOCKED flag to sources that have blocked cloudflare API's + - Fix vidsrc sometimes having an equal sign where it shouldnt + - Increase ranking of lookmovie + # Version 2.0.5 - Disable subtitles for febbox-mp4. As their endpoint doesn't work anymore. From e5d9a3b23cdf0215fe26e9635ba3b9a8cc5c1728 Mon Sep 17 00:00:00 2001 From: mrjvs Date: Sat, 20 Jan 2024 18:03:39 +0100 Subject: [PATCH 39/43] add a new line to changelog --- .docs/content/1.get-started/4.changelog.md | 1 + 1 file changed, 1 insertion(+) diff --git a/.docs/content/1.get-started/4.changelog.md b/.docs/content/1.get-started/4.changelog.md index a19930a..518a414 100644 --- a/.docs/content/1.get-started/4.changelog.md +++ b/.docs/content/1.get-started/4.changelog.md @@ -7,6 +7,7 @@ title: 'Changelog' - Add CF_BLOCKED flag to sources that have blocked cloudflare API's - Fix vidsrc sometimes having an equal sign where it shouldnt - Increase ranking of lookmovie + - Re-enabled subtitles for febbox-mp4 # Version 2.0.5 - Disable subtitles for febbox-mp4. As their endpoint doesn't work anymore. From 5adca068a8248a5db41f768ee636597f48a932bd Mon Sep 17 00:00:00 2001 From: Jorrin <43169049+JorrinKievit@users.noreply.github.com> Date: Wed, 24 Jan 2024 17:32:34 +0100 Subject: [PATCH 40/43] replace vidplay decryption keys and domain - Massive thanks to @Ciarands! --- package-lock.json | 4 ++-- src/providers/embeds/vidplay/common.ts | 8 +++----- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/package-lock.json b/package-lock.json index d8a8961..3facef2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@movie-web/providers", - "version": "2.0.5", + "version": "2.1.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@movie-web/providers", - "version": "2.0.5", + "version": "2.1.0", "license": "MIT", "dependencies": { "cheerio": "^1.0.0-rc.12", diff --git a/src/providers/embeds/vidplay/common.ts b/src/providers/embeds/vidplay/common.ts index e3172f7..3eb19a9 100644 --- a/src/providers/embeds/vidplay/common.ts +++ b/src/providers/embeds/vidplay/common.ts @@ -2,16 +2,14 @@ import { makeFullUrl } from '@/fetchers/common'; import { decodeData } from '@/providers/sources/vidsrcto/common'; import { EmbedScrapeContext } from '@/utils/context'; -export const vidplayBase = 'https://vidplay.site'; -export const referer = 'https://vidplay.online/'; +export const vidplayBase = 'https://vidplay.online'; +export const referer = `${vidplayBase}/`; // This file is based on https://github.com/Ciarands/vidsrc-to-resolver/blob/dffa45e726a4b944cb9af0c9e7630476c93c0213/vidsrc.py#L16 // Full credits to @Ciarands! export const getDecryptionKeys = async (ctx: EmbedScrapeContext): Promise => { - const res = await ctx.fetcher( - 'https://raw.githubusercontent.com/Claudemirovsky/worstsource-keys/keys/keys.json', - ); + const res = await ctx.fetcher('https://raw.githubusercontent.com/Ciarands/vidsrc-keys/main/keys.json'); return JSON.parse(res); }; From 89c7f49be2c140f8a659e4e078850b451e473088 Mon Sep 17 00:00:00 2001 From: mrjvs Date: Wed, 24 Jan 2024 20:06:08 +0100 Subject: [PATCH 41/43] Update package.json --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 0cd0ced..25c4d07 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@movie-web/providers", - "version": "2.1.0", + "version": "2.1.1", "description": "Package that contains all the providers of movie-web", "main": "./lib/index.umd.js", "types": "./lib/index.d.ts", From 3c2e3bc07631189d0a14cd50dc5d1a474304f7e3 Mon Sep 17 00:00:00 2001 From: mrjvs Date: Wed, 24 Jan 2024 20:08:43 +0100 Subject: [PATCH 42/43] Update 4.changelog.md --- .docs/content/1.get-started/4.changelog.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.docs/content/1.get-started/4.changelog.md b/.docs/content/1.get-started/4.changelog.md index 518a414..f03e407 100644 --- a/.docs/content/1.get-started/4.changelog.md +++ b/.docs/content/1.get-started/4.changelog.md @@ -2,6 +2,9 @@ title: 'Changelog' --- +# Version 2.1.1 + - Fixed vidplay decryption keys being wrong and switched the domain to one that works + # Version 2.1.0 - Add preferedHeaders to most sources - Add CF_BLOCKED flag to sources that have blocked cloudflare API's From 4591bcbc2e26470f65f9c3d25cd0f1d882fe139c Mon Sep 17 00:00:00 2001 From: Ciarands Date: Fri, 26 Jan 2024 23:16:37 +0000 Subject: [PATCH 43/43] Fix vidsrc.me base64 decoding --- src/providers/embeds/vidsrc.ts | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/src/providers/embeds/vidsrc.ts b/src/providers/embeds/vidsrc.ts index 93d55c5..069c2f6 100644 --- a/src/providers/embeds/vidsrc.ts +++ b/src/providers/embeds/vidsrc.ts @@ -4,6 +4,14 @@ import { makeEmbed } from '@/providers/base'; const hlsURLRegex = /file:"(.*?)"/; const setPassRegex = /var pass_path = "(.*set_pass\.php.*)";/; +function formatHlsB64(data: string): string { + const encodedB64 = data.replace(/\/@#@\/[^=/]+==/g, ''); + if (encodedB64.match(/\/@#@\/[^=/]+==/)) { + return formatHlsB64(encodedB64); + } + return encodedB64; +} + export const vidsrcembedScraper = makeEmbed({ id: 'vidsrcembed', // VidSrc is both a source and an embed host name: 'VidSrc', @@ -15,10 +23,12 @@ export const vidsrcembedScraper = makeEmbed({ }, }); - const match = html.match(hlsURLRegex)?.[1]?.replace(/(\/\/\S+?=)|#2|=/g, ''); - if (!match) throw new Error('Unable to find HLS playlist'); - const finalUrl = atob(match); - + // When this eventually breaks see the player js @ pjs_main.js + // If you know what youre doing and are slightly confused about how to reverse this feel free to reach out to ciaran_ds on discord with any queries + let hlsMatch = html.match(hlsURLRegex)?.[1]?.slice(2); + if (!hlsMatch) throw new Error('Unable to find HLS playlist'); + hlsMatch = formatHlsB64(hlsMatch); + const finalUrl = atob(hlsMatch); if (!finalUrl.includes('.m3u8')) throw new Error('Unable to find HLS playlist'); let setPassLink = html.match(setPassRegex)?.[1];