From b120b90b02d99b20a1a189ace9e2c97d2d237365 Mon Sep 17 00:00:00 2001 From: mrjvs Date: Sun, 10 Sep 2023 23:52:51 +0200 Subject: [PATCH] Add stream targets --- README.md | 9 ------- src/__test__/providers/checks.test.ts | 18 ++++++++------ src/__test__/runner/list.test.ts | 9 +++++++ src/__test__/runner/meta.test.ts | 4 +++ src/index.ts | 2 ++ src/main/builder.ts | 8 +++++- src/main/individualRunner.ts | 15 +++++++++++- src/main/runner.ts | 8 ++++++ src/main/targets.ts | 35 +++++++++++++++++++++++++++ src/providers/base.ts | 2 ++ src/providers/embeds/upcloud.ts | 2 ++ src/providers/get.ts | 5 ++-- src/providers/sources/flixhq/index.ts | 2 ++ src/providers/streams.ts | 4 +++ test/index.js | 3 ++- test/index.mjs | 1 + 16 files changed, 106 insertions(+), 21 deletions(-) create mode 100644 src/main/targets.ts diff --git a/README.md b/README.md index 081e7b4..1aa330b 100644 --- a/README.md +++ b/README.md @@ -10,13 +10,4 @@ features: > **This package is still WIP** Todos: - - add all real providers - make default fetcher maker thing work with both undici and node-fetch - -Future todos: - - docs: examples for nodejs + browser - - docs: how to use + usecases - - docs: examples for custom fetcher - - docs: example with tmdb search - - feature: choose an output environment (for browser or for native) - - feature: flixhq show support diff --git a/src/__test__/providers/checks.test.ts b/src/__test__/providers/checks.test.ts index bc89a20..dca4e73 100644 --- a/src/__test__/providers/checks.test.ts +++ b/src/__test__/providers/checks.test.ts @@ -5,6 +5,10 @@ import { vi, describe, it, expect, afterEach } from 'vitest'; const mocks = await vi.hoisted(async () => (await import('../providerTests.ts')).makeProviderMocks()); vi.mock('@/providers/all', () => mocks); +const features = { + requires: [], +} + describe('getProviders()', () => { afterEach(() => { vi.clearAllMocks(); @@ -13,7 +17,7 @@ describe('getProviders()', () => { it('should return providers', () => { mocks.gatherAllEmbeds.mockReturnValue([mockEmbeds.embedD]); mocks.gatherAllSources.mockReturnValue([mockSources.sourceA, mockSources.sourceB]); - expect(getProviders()).toEqual({ + expect(getProviders(features)).toEqual({ sources: [mockSources.sourceA, mockSources.sourceB], embeds: [mockEmbeds.embedD], }); @@ -22,7 +26,7 @@ describe('getProviders()', () => { it('should filter out disabled providers', () => { mocks.gatherAllEmbeds.mockReturnValue([mockEmbeds.embedD, mockEmbeds.embedEDisabled]); mocks.gatherAllSources.mockReturnValue([mockSources.sourceA, mockSources.sourceCDisabled, mockSources.sourceB]); - expect(getProviders()).toEqual({ + expect(getProviders(features)).toEqual({ sources: [mockSources.sourceA, mockSources.sourceB], embeds: [mockEmbeds.embedD], }); @@ -31,31 +35,31 @@ describe('getProviders()', () => { it('should throw on duplicate ids in sources', () => { mocks.gatherAllEmbeds.mockReturnValue([]); mocks.gatherAllSources.mockReturnValue([mockSources.sourceAHigherRank, mockSources.sourceA, mockSources.sourceB]); - expect(() => getProviders()).toThrowError(); + expect(() => getProviders(features)).toThrowError(); }); it('should throw on duplicate ids in embeds', () => { mocks.gatherAllEmbeds.mockReturnValue([mockEmbeds.embedD, mockEmbeds.embedDHigherRank, mockEmbeds.embedA]); mocks.gatherAllSources.mockReturnValue([]); - expect(() => getProviders()).toThrowError(); + expect(() => getProviders(features)).toThrowError(); }); it('should throw on duplicate ids between sources and embeds', () => { mocks.gatherAllEmbeds.mockReturnValue([mockEmbeds.embedD, mockEmbeds.embedA]); mocks.gatherAllSources.mockReturnValue([mockSources.sourceA, mockSources.sourceB]); - expect(() => getProviders()).toThrowError(); + expect(() => getProviders(features)).toThrowError(); }); it('should throw on duplicate rank between sources and embeds', () => { mocks.gatherAllEmbeds.mockReturnValue([mockEmbeds.embedD, mockEmbeds.embedA]); mocks.gatherAllSources.mockReturnValue([mockSources.sourceA, mockSources.sourceB]); - expect(() => getProviders()).toThrowError(); + expect(() => getProviders(features)).toThrowError(); }); it('should not throw with same rank between sources and embeds', () => { mocks.gatherAllEmbeds.mockReturnValue([mockEmbeds.embedD, mockEmbeds.embedHSameRankAsSourceA]); mocks.gatherAllSources.mockReturnValue([mockSources.sourceA, mockSources.sourceB]); - expect(getProviders()).toEqual({ + expect(getProviders(features)).toEqual({ sources: [mockSources.sourceA, mockSources.sourceB], embeds: [mockEmbeds.embedD, mockEmbeds.embedHSameRankAsSourceA], }); diff --git a/src/__test__/runner/list.test.ts b/src/__test__/runner/list.test.ts index b10eddb..483485e 100644 --- a/src/__test__/runner/list.test.ts +++ b/src/__test__/runner/list.test.ts @@ -1,5 +1,6 @@ import { mockEmbeds, mockSources } from '@/__test__/providerTests'; import { makeProviders } from '@/main/builder'; +import { targets } from '@/main/targets.ts'; import { afterEach, describe, expect, it, vi } from 'vitest'; const mocks = await vi.hoisted(async () => (await import('../providerTests.ts')).makeProviderMocks()); @@ -15,6 +16,7 @@ describe('ProviderControls.listSources()', () => { mocks.gatherAllEmbeds.mockReturnValue([]); const p = makeProviders({ fetcher: null as any, + target: targets.NATIVE, }); expect(p.listSources()).toEqual([ { @@ -32,6 +34,7 @@ describe('ProviderControls.listSources()', () => { mocks.gatherAllEmbeds.mockReturnValue([]); const p = makeProviders({ fetcher: null as any, + target: targets.NATIVE, }); expect(p.listSources()).toEqual([ { @@ -49,6 +52,7 @@ describe('ProviderControls.listSources()', () => { mocks.gatherAllEmbeds.mockReturnValue([]); const p = makeProviders({ fetcher: null as any, + target: targets.NATIVE, }); expect(p.listSources()).toEqual([ { @@ -66,6 +70,7 @@ describe('ProviderControls.listSources()', () => { mocks.gatherAllEmbeds.mockReturnValue([]); const p1 = makeProviders({ fetcher: null as any, + target: targets.NATIVE, }); const l1 = p1.listSources(); expect(l1.map((v) => v.id).join(',')).toEqual('z,y'); @@ -74,6 +79,7 @@ describe('ProviderControls.listSources()', () => { mocks.gatherAllEmbeds.mockReturnValue([]); const p2 = makeProviders({ fetcher: null as any, + target: targets.NATIVE, }); const l2 = p2.listSources(); expect(l2.map((v) => v.id).join(',')).toEqual('z,y'); @@ -90,6 +96,7 @@ describe('ProviderControls.getAllEmbedMetaSorted()', () => { mocks.gatherAllEmbeds.mockReturnValue([mockEmbeds.fullEmbedX]); const p = makeProviders({ fetcher: null as any, + target: targets.NATIVE, }); expect(p.listEmbeds()).toEqual([ { @@ -106,6 +113,7 @@ describe('ProviderControls.getAllEmbedMetaSorted()', () => { mocks.gatherAllEmbeds.mockReturnValue([mockEmbeds.fullEmbedX, mockEmbeds.fullEmbedZ]); const p1 = makeProviders({ fetcher: null as any, + target: targets.NATIVE, }); const l1 = p1.listEmbeds(); expect(l1.map((v) => v.id).join(',')).toEqual('z,x'); @@ -114,6 +122,7 @@ describe('ProviderControls.getAllEmbedMetaSorted()', () => { mocks.gatherAllEmbeds.mockReturnValue([mockEmbeds.fullEmbedZ, mockEmbeds.fullEmbedX]); const p2 = makeProviders({ fetcher: null as any, + target: targets.NATIVE, }); const l2 = p2.listEmbeds(); expect(l2.map((v) => v.id).join(',')).toEqual('z,x'); diff --git a/src/__test__/runner/meta.test.ts b/src/__test__/runner/meta.test.ts index 59c3a3a..4b0e55e 100644 --- a/src/__test__/runner/meta.test.ts +++ b/src/__test__/runner/meta.test.ts @@ -1,5 +1,6 @@ import { mockEmbeds, mockSources } from '@/__test__/providerTests'; import { makeProviders } from '@/main/builder'; +import { targets } from '@/main/targets.ts'; import { afterEach, describe, expect, it, vi } from 'vitest'; const mocks = await vi.hoisted(async () => (await import('../providerTests.ts')).makeProviderMocks()); @@ -15,6 +16,7 @@ describe('ProviderControls.getMetadata()', () => { mocks.gatherAllEmbeds.mockReturnValue([]); const p = makeProviders({ fetcher: null as any, + target: targets.NATIVE, }); expect(p.getMetadata(':)')).toEqual(null); }); @@ -24,6 +26,7 @@ describe('ProviderControls.getMetadata()', () => { mocks.gatherAllEmbeds.mockReturnValue([]); const p = makeProviders({ fetcher: null as any, + target: targets.NATIVE, }); expect(p.getMetadata(mockSources.fullSourceZBoth.id)).toEqual({ type: 'source', @@ -39,6 +42,7 @@ describe('ProviderControls.getMetadata()', () => { mocks.gatherAllEmbeds.mockReturnValue([mockEmbeds.fullEmbedX]); const p = makeProviders({ fetcher: null as any, + target: targets.NATIVE, }); expect(p.getMetadata(mockEmbeds.fullEmbedX.id)).toEqual({ type: 'embed', diff --git a/src/index.ts b/src/index.ts index 1721525..ea8b255 100644 --- a/src/index.ts +++ b/src/index.ts @@ -2,6 +2,7 @@ export type { EmbedOutput, SourcererOutput } from '@/providers/base'; export type { RunOutput } from '@/main/runner'; export type { MetaOutput } from '@/main/meta'; export type { FullScraperEvents } from '@/main/events'; +export type { Targets, Flags } from '@/main/targets'; export type { MediaTypes, ShowMedia, ScrapeMedia, MovieMedia } from '@/main/media'; export type { ProviderBuilderOptions, @@ -15,3 +16,4 @@ export { NotFoundError } from '@/utils/errors'; export { makeProviders } from '@/main/builder'; export { makeStandardFetcher } from '@/fetchers/standardFetch'; export { makeSimpleProxyFetcher } from '@/fetchers/simpleProxy'; +export { flags, targets } from '@/main/targets'; diff --git a/src/main/builder.ts b/src/main/builder.ts index de95679..0322dbd 100644 --- a/src/main/builder.ts +++ b/src/main/builder.ts @@ -5,6 +5,7 @@ import { scrapeIndividualEmbed, scrapeInvidualSource } from '@/main/individualRu import { ScrapeMedia } from '@/main/media'; import { MetaOutput, getAllEmbedMetaSorted, getAllSourceMetaSorted, getSpecificId } from '@/main/meta'; import { RunOutput, runAllProviders } from '@/main/runner'; +import { Targets, getTargetFeatures } from '@/main/targets'; import { EmbedOutput, SourcererOutput } from '@/providers/base'; import { getProviders } from '@/providers/get'; @@ -15,6 +16,9 @@ export interface ProviderBuilderOptions { // proxied fetcher, if the scraper needs to access a CORS proxy. this fetcher will be called instead // of the normal fetcher. Defaults to the normal fetcher. proxiedFetcher?: Fetcher; + + // target of where the streams will be used + target: Targets; } export interface RunnerOptions { @@ -77,8 +81,10 @@ export interface ProviderControls { } export function makeProviders(ops: ProviderBuilderOptions): ProviderControls { - const list = getProviders(); + const features = getTargetFeatures(ops.target); + const list = getProviders(features); const providerRunnerOps = { + features, fetcher: makeFullFetcher(ops.fetcher), proxiedFetcher: makeFullFetcher(ops.proxiedFetcher ?? ops.fetcher), }; diff --git a/src/main/individualRunner.ts b/src/main/individualRunner.ts index 2dc63fa..957edd0 100644 --- a/src/main/individualRunner.ts +++ b/src/main/individualRunner.ts @@ -1,11 +1,14 @@ import { UseableFetcher } from '@/fetchers/types'; import { IndividualScraperEvents } from '@/main/events'; import { ScrapeMedia } from '@/main/media'; +import { FeatureMap, flagsAllowedInFeatures } from '@/main/targets'; import { EmbedOutput, SourcererOutput } from '@/providers/base'; import { ProviderList } from '@/providers/get'; import { ScrapeContext } from '@/utils/context'; +import { NotFoundError } from '@/utils/errors'; export type IndividualSourceRunnerOptions = { + features: FeatureMap; fetcher: UseableFetcher; proxiedFetcher: UseableFetcher; media: ScrapeMedia; @@ -46,11 +49,17 @@ export async function scrapeInvidualSource( media: ops.media, }); + // stream doesn't satisfy the feature flags, so gets removed in output + if (output?.stream && !flagsAllowedInFeatures(ops.features, output.stream.flags)) { + output.stream = undefined; + } + if (!output) throw new Error('output is null'); return output; } export type IndividualEmbedRunnerOptions = { + features: FeatureMap; fetcher: UseableFetcher; proxiedFetcher: UseableFetcher; url: string; @@ -65,7 +74,7 @@ export async function scrapeIndividualEmbed( const embedScraper = list.embeds.find((v) => ops.id === v.id); if (!embedScraper) throw new Error('Embed with ID not found'); - return embedScraper.scrape({ + const output = await embedScraper.scrape({ fetcher: ops.fetcher, proxiedFetcher: ops.proxiedFetcher, url: ops.url, @@ -77,4 +86,8 @@ export async function scrapeIndividualEmbed( }); }, }); + + if (!flagsAllowedInFeatures(ops.features, output.stream.flags)) + throw new NotFoundError("stream doesn't satisfy target feature flags"); + return output; } diff --git a/src/main/runner.ts b/src/main/runner.ts index a438b01..0ecaee1 100644 --- a/src/main/runner.ts +++ b/src/main/runner.ts @@ -1,6 +1,7 @@ import { UseableFetcher } from '@/fetchers/types'; import { FullScraperEvents } from '@/main/events'; import { ScrapeMedia } from '@/main/media'; +import { FeatureMap, flagsAllowedInFeatures } from '@/main/targets'; import { EmbedOutput, SourcererOutput } from '@/providers/base'; import { ProviderList } from '@/providers/get'; import { Stream } from '@/providers/streams'; @@ -28,6 +29,7 @@ export type EmbedRunOutput = { export type ProviderRunnerOptions = { fetcher: UseableFetcher; proxiedFetcher: UseableFetcher; + features: FeatureMap; sourceOrder?: string[]; embedOrder?: string[]; events?: FullScraperEvents; @@ -77,6 +79,9 @@ export async function runAllProviders(list: ProviderList, ops: ProviderRunnerOpt ...contextBase, media: ops.media, }); + if (output?.stream && !flagsAllowedInFeatures(ops.features, output.stream.flags)) { + throw new NotFoundError("stream doesn't satisfy target feature flags"); + } } catch (err) { if (err instanceof NotFoundError) { ops.events?.update?.({ @@ -135,6 +140,9 @@ export async function runAllProviders(list: ProviderList, ops: ProviderRunnerOpt ...contextBase, url: e.url, }); + if (!flagsAllowedInFeatures(ops.features, embedOutput.stream.flags)) { + throw new NotFoundError("stream doesn't satisfy target feature flags"); + } } catch (err) { if (err instanceof NotFoundError) { ops.events?.update?.({ diff --git a/src/main/targets.ts b/src/main/targets.ts new file mode 100644 index 0000000..7b4a10f --- /dev/null +++ b/src/main/targets.ts @@ -0,0 +1,35 @@ +export const flags = { + NO_CORS: 'no-cors', +} as const; + +export type Flags = (typeof flags)[keyof typeof flags]; + +export const targets = { + BROWSER: 'browser', + NATIVE: 'native', +} as const; + +export type Targets = (typeof targets)[keyof typeof targets]; + +export type FeatureMap = { + requires: readonly Flags[]; +}; + +export const targetToFeatures: Record = { + browser: { + requires: [flags.NO_CORS], + }, + native: { + requires: [], + }, +} as const; + +export function getTargetFeatures(target: Targets): FeatureMap { + return targetToFeatures[target]; +} + +export function flagsAllowedInFeatures(features: FeatureMap, inputFlags: Flags[]): boolean { + const hasAllFlags = features.requires.every((v) => inputFlags.includes(v)); + if (!hasAllFlags) return false; + return true; +} diff --git a/src/providers/base.ts b/src/providers/base.ts index 4942ffd..902a5a6 100644 --- a/src/providers/base.ts +++ b/src/providers/base.ts @@ -1,4 +1,5 @@ import { MovieMedia, ShowMedia } from '@/main/media'; +import { Flags } from '@/main/targets'; import { Stream } from '@/providers/streams'; import { EmbedScrapeContext, ScrapeContext } from '@/utils/context'; @@ -15,6 +16,7 @@ export type Sourcerer = { name: string; // displayed in the UI rank: number; // the higher the number, the earlier it gets put on the queue disabled?: boolean; + flags: Flags[]; scrapeMovie?: (input: ScrapeContext & { media: MovieMedia }) => Promise; scrapeShow?: (input: ScrapeContext & { media: ShowMedia }) => Promise; }; diff --git a/src/providers/embeds/upcloud.ts b/src/providers/embeds/upcloud.ts index 36698e1..d2be1f9 100644 --- a/src/providers/embeds/upcloud.ts +++ b/src/providers/embeds/upcloud.ts @@ -1,5 +1,6 @@ import crypto from 'crypto-js'; +import { flags } from '@/main/targets'; import { makeEmbed } from '@/providers/base'; const { AES, enc } = crypto; @@ -69,6 +70,7 @@ export const upcloudScraper = makeEmbed({ stream: { type: 'hls', playlist: sources.file, + flags: [flags.NO_CORS], }, }; }, diff --git a/src/providers/get.ts b/src/providers/get.ts index 689617f..085aba5 100644 --- a/src/providers/get.ts +++ b/src/providers/get.ts @@ -1,3 +1,4 @@ +import { FeatureMap, flagsAllowedInFeatures } from '@/main/targets'; import { gatherAllEmbeds, gatherAllSources } from '@/providers/all'; import { Embed, Sourcerer } from '@/providers/base'; import { hasDuplicates } from '@/utils/predicates'; @@ -7,7 +8,7 @@ export interface ProviderList { embeds: Embed[]; } -export function getProviders(): ProviderList { +export function getProviders(features: FeatureMap): ProviderList { const sources = gatherAllSources().filter((v) => !v?.disabled); const embeds = gatherAllEmbeds().filter((v) => !v?.disabled); const combined = [...sources, ...embeds]; @@ -21,7 +22,7 @@ export function getProviders(): ProviderList { if (anyDuplicateEmbedRank) throw new Error('Duplicate rank found in embeds'); return { - sources, + sources: sources.filter((s) => flagsAllowedInFeatures(features, s.flags)), embeds, }; } diff --git a/src/providers/sources/flixhq/index.ts b/src/providers/sources/flixhq/index.ts index c8559bd..fcd504d 100644 --- a/src/providers/sources/flixhq/index.ts +++ b/src/providers/sources/flixhq/index.ts @@ -1,3 +1,4 @@ +import { flags } from '@/main/targets'; import { makeSourcerer } from '@/providers/base'; import { upcloudScraper } from '@/providers/embeds/upcloud'; import { getFlixhqSourceDetails, getFlixhqSources } from '@/providers/sources/flixhq/scrape'; @@ -9,6 +10,7 @@ export const flixhqScraper = makeSourcerer({ id: 'flixhq', name: 'FlixHQ', rank: 100, + flags: [flags.NO_CORS], async scrapeMovie(ctx) { const id = await getFlixhqId(ctx, ctx.media); if (!id) throw new NotFoundError('no search results match'); diff --git a/src/providers/streams.ts b/src/providers/streams.ts index 3c46567..54b3cb1 100644 --- a/src/providers/streams.ts +++ b/src/providers/streams.ts @@ -1,3 +1,5 @@ +import { Flags } from '@/main/targets'; + export type StreamFile = { type: 'mp4'; url: string; @@ -7,11 +9,13 @@ export type Qualities = '360' | '480' | '720' | '1080'; export type FileBasedStream = { type: 'file'; + flags: Flags[]; qualities: Partial>; }; export type HlsBasedStream = { type: 'hls'; + flags: Flags[]; playlist: string; }; diff --git a/test/index.js b/test/index.js index c0ff16f..11cdbe0 100644 --- a/test/index.js +++ b/test/index.js @@ -1,5 +1,6 @@ -const { makeProviders, makeStandardFetcher } = require('../lib/index.umd.js'); +const { makeProviders, makeStandardFetcher, targets } = require('../lib/index.umd.js'); const providers = makeProviders({ fetcher: makeStandardFetcher(fetch), + target: targets.BROWSER, }); diff --git a/test/index.mjs b/test/index.mjs index c5ba288..c4c0ad5 100644 --- a/test/index.mjs +++ b/test/index.mjs @@ -2,4 +2,5 @@ import { makeProviders, makeStandardFetcher } from '../lib/index.mjs'; const providers = makeProviders({ fetcher: makeStandardFetcher(fetch), + target: targets.BROWSER, });