From 55f963611ce06fa29a67d9633b6def264e1f228b Mon Sep 17 00:00:00 2001 From: Jorrin Date: Sun, 31 Mar 2024 17:25:07 +0200 Subject: [PATCH 1/3] Check if a stream returns a 200 --- src/runners/individualRunner.ts | 13 +++++++- src/runners/runner.ts | 10 +++++-- src/utils/valid.ts | 53 +++++++++++++++++++++++++++++++++ 3 files changed, 72 insertions(+), 4 deletions(-) diff --git a/src/runners/individualRunner.ts b/src/runners/individualRunner.ts index ad3004b..6c7e452 100644 --- a/src/runners/individualRunner.ts +++ b/src/runners/individualRunner.ts @@ -6,7 +6,7 @@ import { EmbedOutput, SourcererOutput } from '@/providers/base'; import { ProviderList } from '@/providers/get'; import { ScrapeContext } from '@/utils/context'; import { NotFoundError } from '@/utils/errors'; -import { isValidStream } from '@/utils/valid'; +import { isValidStream, validatePlayableStreams } from '@/utils/valid'; export type IndividualSourceRunnerOptions = { features: FeatureMap; @@ -68,6 +68,13 @@ export async function scrapeInvidualSource( if ((!output.stream || output.stream.length === 0) && output.embeds.length === 0) throw new NotFoundError('No streams found'); + + // only check for playable streams if there are streams, and if there are no embeds + if (output.stream && output.stream.length > 0 && output.embeds.length === 0) { + const playableStreams = await validatePlayableStreams(output.stream, ops); + if (playableStreams.length === 0) throw new NotFoundError('No playable streams found'); + output.stream = playableStreams; + } return output; } @@ -105,5 +112,9 @@ export async function scrapeIndividualEmbed( .filter((stream) => flagsAllowedInFeatures(ops.features, stream.flags)); if (output.stream.length === 0) throw new NotFoundError('No streams found'); + const playableStreams = await validatePlayableStreams(output.stream, ops); + if (playableStreams.length === 0) throw new NotFoundError('No playable streams found'); + output.stream = playableStreams; + return output; } diff --git a/src/runners/runner.ts b/src/runners/runner.ts index c351174..8456ec1 100644 --- a/src/runners/runner.ts +++ b/src/runners/runner.ts @@ -8,7 +8,7 @@ import { Stream } from '@/providers/streams'; import { ScrapeContext } from '@/utils/context'; import { NotFoundError } from '@/utils/errors'; import { reorderOnIdList } from '@/utils/list'; -import { isValidStream } from '@/utils/valid'; +import { isValidStream, validatePlayableStream } from '@/utils/valid'; export type RunOutput = { sourceId: string; @@ -104,9 +104,11 @@ export async function runAllProviders(list: ProviderList, ops: ProviderRunnerOpt // return stream is there are any if (output.stream?.[0]) { + const playableStream = await validatePlayableStream(output.stream[0], ops); + if (!playableStream) throw new NotFoundError('No streams found'); return { sourceId: source.id, - stream: output.stream[0], + stream: playableStream, }; } @@ -161,11 +163,13 @@ export async function runAllProviders(list: ProviderList, ops: ProviderRunnerOpt ops.events?.update?.(updateParams); continue; } + const playableStream = await validatePlayableStream(embedOutput.stream[0], ops); + if (!playableStream) throw new NotFoundError('No streams found'); return { sourceId: source.id, embedId: scraper.id, - stream: embedOutput.stream[0], + stream: playableStream, }; } } diff --git a/src/utils/valid.ts b/src/utils/valid.ts index 62347c3..2f292a2 100644 --- a/src/utils/valid.ts +++ b/src/utils/valid.ts @@ -1,4 +1,6 @@ import { Stream } from '@/providers/streams'; +import { IndividualEmbedRunnerOptions } from '@/runners/individualRunner'; +import { ProviderRunnerOptions } from '@/runners/runner'; export function isValidStream(stream: Stream | undefined): boolean { if (!stream) return false; @@ -15,3 +17,54 @@ export function isValidStream(stream: Stream | undefined): boolean { // unknown file type return false; } + +export async function validatePlayableStream( + stream: Stream, + ops: ProviderRunnerOptions | IndividualEmbedRunnerOptions, +): Promise { + const fetcher = stream.flags.includes('cors-allowed') ? ops.fetcher : ops.proxiedFetcher; + if (stream.type === 'hls') { + const headResult = await fetcher.full(stream.playlist, { + method: 'HEAD', + headers: { + ...stream.preferredHeaders, + ...stream.headers, + }, + }); + if (headResult.statusCode !== 200) return null; + return stream; + } + if (stream.type === 'file') { + const validQualitiesResults = await Promise.all( + Object.values(stream.qualities).map((quality) => + fetcher.full(quality.url, { + method: 'HEAD', + headers: { + ...stream.preferredHeaders, + ...stream.headers, + }, + }), + ), + ); + // remove invalid qualities from the stream + const validQualities = stream.qualities; + Object.keys(stream.qualities).forEach((quality, index) => { + if (validQualitiesResults[index].statusCode !== 200) { + delete validQualities[quality as keyof typeof stream.qualities]; + } + }); + + if (Object.keys(validQualities).length === 0) return null; + return { ...stream, qualities: validQualities }; + } + return null; +} + +export async function validatePlayableStreams( + streams: Stream[], + ops: ProviderRunnerOptions | IndividualEmbedRunnerOptions, +): Promise { + return (await Promise.all(streams.map((stream) => validatePlayableStream(stream, ops)))).filter( + (v) => v !== null, + ) as Stream[]; +} From 9258c97de8fc830d41f5ea5ace71a65b3c5e468b Mon Sep 17 00:00:00 2001 From: Jorrin Date: Sun, 31 Mar 2024 19:11:09 +0200 Subject: [PATCH 2/3] only use normal fetcher if its cors-allowed --- src/utils/valid.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/valid.ts b/src/utils/valid.ts index 2f292a2..65db1ea 100644 --- a/src/utils/valid.ts +++ b/src/utils/valid.ts @@ -22,7 +22,7 @@ export async function validatePlayableStream( stream: Stream, ops: ProviderRunnerOptions | IndividualEmbedRunnerOptions, ): Promise { - const fetcher = stream.flags.includes('cors-allowed') ? ops.fetcher : ops.proxiedFetcher; + const fetcher = stream.flags.length === 1 && stream.flags.includes('cors-allowed') ? ops.fetcher : ops.proxiedFetcher; if (stream.type === 'hls') { const headResult = await fetcher.full(stream.playlist, { method: 'HEAD', From db4a7e82dd6e46c826286c5adbd10786744d1946 Mon Sep 17 00:00:00 2001 From: Jorrin Date: Sun, 31 Mar 2024 20:02:35 +0200 Subject: [PATCH 3/3] fix scraper crashing if an embed is not playable --- src/runners/runner.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/runners/runner.ts b/src/runners/runner.ts index 8456ec1..c3bc9eb 100644 --- a/src/runners/runner.ts +++ b/src/runners/runner.ts @@ -151,6 +151,9 @@ export async function runAllProviders(list: ProviderList, ops: ProviderRunnerOpt if (embedOutput.stream.length === 0) { throw new NotFoundError('No streams found'); } + const playableStream = await validatePlayableStream(embedOutput.stream[0], ops); + if (!playableStream) throw new NotFoundError('No streams found'); + embedOutput.stream = [playableStream]; } catch (error) { const updateParams: UpdateEvent = { id: source.id, @@ -163,13 +166,11 @@ export async function runAllProviders(list: ProviderList, ops: ProviderRunnerOpt ops.events?.update?.(updateParams); continue; } - const playableStream = await validatePlayableStream(embedOutput.stream[0], ops); - if (!playableStream) throw new NotFoundError('No streams found'); return { sourceId: source.id, embedId: scraper.id, - stream: playableStream, + stream: embedOutput.stream[0], }; } }