From ef766936dd97c4f9b1fff32eb24aede68a933da0 Mon Sep 17 00:00:00 2001 From: mrjvs Date: Wed, 23 Aug 2023 20:07:39 +0200 Subject: [PATCH] Basically library structure --- src/fetchers/common.ts | 15 ++++++++++ src/fetchers/standardFetch.ts | 15 ++++++++++ src/fetchers/types.ts | 24 ++++++++++++++++ src/index.ts | 2 +- src/main/builder.ts | 54 +++++++++++++++++++++++++++-------- src/main/meta.ts | 37 ++++++++++++++++++++++++ src/main/runner.ts | 18 ++++++++++++ src/providers/all.ts | 23 ++++++++++++++- src/providers/base.ts | 7 ++++- src/utils/context.ts | 7 +++-- src/utils/fetcher.ts | 11 ------- src/utils/predicates.ts | 7 +++++ 12 files changed, 191 insertions(+), 29 deletions(-) create mode 100644 src/fetchers/common.ts create mode 100644 src/fetchers/standardFetch.ts create mode 100644 src/fetchers/types.ts create mode 100644 src/main/meta.ts create mode 100644 src/main/runner.ts delete mode 100644 src/utils/fetcher.ts create mode 100644 src/utils/predicates.ts diff --git a/src/fetchers/common.ts b/src/fetchers/common.ts new file mode 100644 index 0000000..6ce7c87 --- /dev/null +++ b/src/fetchers/common.ts @@ -0,0 +1,15 @@ +import { FetcherOptions } from '@/fetchers/types'; + +// make url with query params and base url used correctly +export function makeFullUrl(url: string, ops?: FetcherOptions): string { + // glue baseUrl and rest of url together + const fullUrl = ops?.baseUrl ?? ''; + // TODO make full url + + const parsedUrl = new URL(fullUrl); + Object.entries(ops?.query ?? {}).forEach(([k, v]) => { + parsedUrl.searchParams.set(k, v); + }); + + return parsedUrl.toString(); +} diff --git a/src/fetchers/standardFetch.ts b/src/fetchers/standardFetch.ts new file mode 100644 index 0000000..54ad91e --- /dev/null +++ b/src/fetchers/standardFetch.ts @@ -0,0 +1,15 @@ +import { makeFullUrl } from '@/fetchers/common'; +import { Fetcher } from '@/fetchers/types'; + +export function makeStandardFetcher(f: typeof fetch): Fetcher { + const normalFetch: Fetcher = (url, ops) => { + const fullUrl = makeFullUrl(url, ops); + + return f(fullUrl, { + method: ops.method, + body: JSON.stringify(ops.body), // TODO content type headers + proper serialization + }); + }; + + return normalFetch; +} diff --git a/src/fetchers/types.ts b/src/fetchers/types.ts new file mode 100644 index 0000000..62f78b9 --- /dev/null +++ b/src/fetchers/types.ts @@ -0,0 +1,24 @@ +export type FetcherOptions = { + baseUrl?: string; + headers?: Record; + query?: Record; + method?: 'GET' | 'POST'; + body?: Record | string | FormData; +}; + +export type DefaultedFetcherOptions = { + baseUrl?: string; + body?: Record | string | FormData; + headers: Record; + query: Record; + method: 'GET' | 'POST'; +}; + +export type Fetcher = { + (url: string, ops: DefaultedFetcherOptions): T; +}; + +// this feature has some quality of life features +export type UseableFetcher = { + (url: string, ops?: FetcherOptions): T; +}; diff --git a/src/index.ts b/src/index.ts index 89e547d..97173d5 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1 +1 @@ -export { ProviderBuilderOptions, Providers, makeProviders } from '@/main/builder'; +export { ProviderBuilderOptions, ProviderControls, makeProviders } from '@/main/builder'; diff --git a/src/main/builder.ts b/src/main/builder.ts index 65a6c26..7d594d0 100644 --- a/src/main/builder.ts +++ b/src/main/builder.ts @@ -1,4 +1,13 @@ -import { Fetcher } from '@/utils/fetcher'; +import { Fetcher } from '@/fetchers/types'; +import { MetaOutput, getAllEmbedMetaSorted, getAllSourceMetaSorted, getSpecificId } from '@/main/meta'; +import { EmbedRunOutput, RunOutput, SourceRunOutput } from '@/main/runner'; +import { getProviders } from '@/providers/all'; + +// TODO meta data input (tmdb id, imdb id, title, release year) +// TODO actually running scrapers +// TODO documentation: examples for nodejs + browser +// TODO documentation: how to use + usecases +// TODO documentation: examples on how to make a custom fetcher export interface ProviderBuilderOptions { // fetcher, every web request gets called through here @@ -9,18 +18,39 @@ export interface ProviderBuilderOptions { proxiedFetcher?: Fetcher; } -export interface Providers {} +export interface ProviderControls { + // Run all providers one by one. in order of rank (highest first) + // returns the stream, or null if none found + runAll(): Promise; -export function makeProviders(ops: ProviderBuilderOptions): Providers { - return {}; + // Run a source provider + runSource(id: string): Promise; + + // Run a embed provider + runEmbed(id: string): Promise; + + // get meta data about a source or embed. + getMetadata(id: string): MetaOutput | null; + + // return all sources. sorted by rank (highest first) + listSources(): MetaOutput[]; + + // return all embed scrapers. sorted by rank (highest first) + listEmbeds(): MetaOutput[]; } -// +export function makeProviders(_ops: ProviderBuilderOptions): ProviderControls { + const list = getProviders(); -// const scrapers = makeProviders({ -// fetcher: makeFetcher(fetch), -// }); - -// scrapers.scrape(); -// scrapers.callScraper(id); -// scrapers.getScraper(id); + return { + getMetadata(id) { + return getSpecificId(list, id); + }, + listSources() { + return getAllSourceMetaSorted(list); + }, + listEmbeds() { + return getAllEmbedMetaSorted(list); + }, + }; +} diff --git a/src/main/meta.ts b/src/main/meta.ts new file mode 100644 index 0000000..d2a05b4 --- /dev/null +++ b/src/main/meta.ts @@ -0,0 +1,37 @@ +import { ProviderList } from '@/providers/all'; + +export type MetaOutput = { + type: 'embed' | 'source'; + id: string; + rank: number; + name: string; +}; + +export function getAllSourceMetaSorted(list: ProviderList): MetaOutput[] { + return list.sources + .sort((a, b) => b.rank - a.rank) + .map((v) => ({ + type: 'source', + id: v.id, + name: v.name, + rank: v.rank, + })); +} + +export function getAllEmbedMetaSorted(_list: ProviderList): MetaOutput[] { + return []; +} + +export function getSpecificId(list: ProviderList, id: string): MetaOutput | null { + const foundSource = list.sources.find((v) => v.id === id); + if (foundSource) { + return { + type: 'source', + id: foundSource.id, + name: foundSource.name, + rank: foundSource.rank, + }; + } + + return null; +} diff --git a/src/main/runner.ts b/src/main/runner.ts new file mode 100644 index 0000000..f0295af --- /dev/null +++ b/src/main/runner.ts @@ -0,0 +1,18 @@ +import { Stream } from '@/providers/streams'; + +export type RunOutput = { + sourceId: string; + fromEmbed: boolean; + stream: Stream; +}; + +export type SourceRunOutput = { + sourceId: string; + stream?: Stream; + embeds: []; +}; + +export type EmbedRunOutput = { + embedId: string; + stream?: Stream; +}; diff --git a/src/providers/all.ts b/src/providers/all.ts index eb474bb..9ad3156 100644 --- a/src/providers/all.ts +++ b/src/providers/all.ts @@ -1,3 +1,24 @@ import { Sourcerer } from '@/providers/base'; +import { hasDuplicates, isNotNull } from '@/utils/predicates'; -export const sources: Array = []; +function gatherAllSources(): Array { + return []; +} + +export interface ProviderList { + sources: Sourcerer[]; +} + +export function getProviders(): ProviderList { + const sources = gatherAllSources().filter(isNotNull); + + const anyDuplicateId = hasDuplicates(sources.map((v) => v.id)); + const anyDuplicateRank = hasDuplicates(sources.map((v) => v.rank)); + + if (anyDuplicateId) throw new Error('Duplicate id found in sources'); + if (anyDuplicateRank) throw new Error('Duplicate rank found in sources'); + + return { + sources, + }; +} diff --git a/src/providers/base.ts b/src/providers/base.ts index 210ce3a..8292690 100644 --- a/src/providers/base.ts +++ b/src/providers/base.ts @@ -1,11 +1,16 @@ +import { Stream } from '@/providers/streams'; import { ScrapeContext } from '@/utils/context'; +export type SourcererOutput = { + stream?: Stream; +}; + export type Sourcerer = { id: string; name: string; // displayed in the UI rank: number; // the higher the number, the earlier it gets put on the queue disabled?: boolean; - scrape: (input: ScrapeContext) => void; + scrape: (input: ScrapeContext) => Promise; }; export function makeSourcerer(state: Sourcerer): Sourcerer | null { diff --git a/src/utils/context.ts b/src/utils/context.ts index 4bbe37d..3e42ffa 100644 --- a/src/utils/context.ts +++ b/src/utils/context.ts @@ -1,6 +1,7 @@ -import { Fetcher } from '@/utils/fetcher'; +import { UseableFetcher } from '@/fetchers/types'; export interface ScrapeContext { - proxiedFetcher: Fetcher; - fetcher: Fetcher; + proxiedFetcher: UseableFetcher; + fetcher: UseableFetcher; + progress(val: number): void; } diff --git a/src/utils/fetcher.ts b/src/utils/fetcher.ts deleted file mode 100644 index 83dd608..0000000 --- a/src/utils/fetcher.ts +++ /dev/null @@ -1,11 +0,0 @@ -export type FetcherOptions = { - baseUrl?: string; - headers?: Record; - query?: Record; - method?: 'GET' | 'POST'; - body?: Record | string | FormData; -}; - -export type Fetcher = { - (url: string, ops?: FetcherOptions): T; -}; diff --git a/src/utils/predicates.ts b/src/utils/predicates.ts new file mode 100644 index 0000000..03e7469 --- /dev/null +++ b/src/utils/predicates.ts @@ -0,0 +1,7 @@ +export function isNotNull(value: T | null | undefined): value is T { + return value !== null && value !== undefined; +} + +export function hasDuplicates(values: Array): boolean { + return new Set(values).size !== values.length; +}