Basically library structure

src/fetchers/common.ts

@@ -0,0 +1,15 @@
+import { FetcherOptions } from '@/fetchers/types';
+
+// make url with query params and base url used correctly
+export function makeFullUrl(url: string, ops?: FetcherOptions): string {
+  // glue baseUrl and rest of url together
+  const fullUrl = ops?.baseUrl ?? '';
+  // TODO make full url
+
+  const parsedUrl = new URL(fullUrl);
+  Object.entries(ops?.query ?? {}).forEach(([k, v]) => {
+    parsedUrl.searchParams.set(k, v);
+  });
+
+  return parsedUrl.toString();
+}

src/fetchers/standardFetch.ts

@@ -0,0 +1,15 @@
+import { makeFullUrl } from '@/fetchers/common';
+import { Fetcher } from '@/fetchers/types';
+
+export function makeStandardFetcher(f: typeof fetch): Fetcher {
+  const normalFetch: Fetcher = (url, ops) => {
+    const fullUrl = makeFullUrl(url, ops);
+
+    return f(fullUrl, {
+      method: ops.method,
+      body: JSON.stringify(ops.body), // TODO content type headers + proper serialization
+    });
+  };
+
+  return normalFetch;
+}

src/fetchers/types.ts

@@ -0,0 +1,24 @@
+export type FetcherOptions = {
+  baseUrl?: string;
+  headers?: Record<string, string>;
+  query?: Record<string, string>;
+  method?: 'GET' | 'POST';
+  body?: Record<string, any> | string | FormData;
+};
+
+export type DefaultedFetcherOptions = {
+  baseUrl?: string;
+  body?: Record<string, any> | string | FormData;
+  headers: Record<string, string>;
+  query: Record<string, string>;
+  method: 'GET' | 'POST';
+};
+
+export type Fetcher<T = any> = {
+  (url: string, ops: DefaultedFetcherOptions): T;
+};
+
+// this feature has some quality of life features
+export type UseableFetcher<T = any> = {
+  (url: string, ops?: FetcherOptions): T;
+};

src/index.ts

@@ -1 +1 @@
-export { ProviderBuilderOptions, Providers, makeProviders } from '@/main/builder';
+export { ProviderBuilderOptions, ProviderControls, makeProviders } from '@/main/builder';

src/main/builder.ts

@@ -1,4 +1,13 @@
-import { Fetcher } from '@/utils/fetcher';
+import { Fetcher } from '@/fetchers/types';
+import { MetaOutput, getAllEmbedMetaSorted, getAllSourceMetaSorted, getSpecificId } from '@/main/meta';
+import { EmbedRunOutput, RunOutput, SourceRunOutput } from '@/main/runner';
+import { getProviders } from '@/providers/all';
+
+// TODO meta data input (tmdb id, imdb id, title, release year)
+// TODO actually running scrapers
+// TODO documentation: examples for nodejs + browser
+// TODO documentation: how to use + usecases
+// TODO documentation: examples on how to make a custom fetcher
 
 export interface ProviderBuilderOptions {
   // fetcher, every web request gets called through here
@@ -9,18 +18,39 @@ export interface ProviderBuilderOptions {
   proxiedFetcher?: Fetcher;
 }
 
-export interface Providers {}
+export interface ProviderControls {
+  // Run all providers one by one. in order of rank (highest first)
+  // returns the stream, or null if none found
+  runAll(): Promise<RunOutput | null>;
 
-export function makeProviders(ops: ProviderBuilderOptions): Providers {
-  return {};
+  // Run a source provider
+  runSource(id: string): Promise<SourceRunOutput>;
+
+  // Run a embed provider
+  runEmbed(id: string): Promise<EmbedRunOutput>;
+
+  // get meta data about a source or embed.
+  getMetadata(id: string): MetaOutput | null;
+
+  // return all sources. sorted by rank (highest first)
+  listSources(): MetaOutput[];
+
+  // return all embed scrapers. sorted by rank (highest first)
+  listEmbeds(): MetaOutput[];
 }
 
-//
+export function makeProviders(_ops: ProviderBuilderOptions): ProviderControls {
+  const list = getProviders();
 
-// const scrapers = makeProviders({
-//   fetcher: makeFetcher(fetch),
-// });
-
-// scrapers.scrape();
-// scrapers.callScraper(id);
-// scrapers.getScraper(id);
+  return {
+    getMetadata(id) {
+      return getSpecificId(list, id);
+    },
+    listSources() {
+      return getAllSourceMetaSorted(list);
+    },
+    listEmbeds() {
+      return getAllEmbedMetaSorted(list);
+    },
+  };
+}

src/main/meta.ts

@@ -0,0 +1,37 @@
+import { ProviderList } from '@/providers/all';
+
+export type MetaOutput = {
+  type: 'embed' | 'source';
+  id: string;
+  rank: number;
+  name: string;
+};
+
+export function getAllSourceMetaSorted(list: ProviderList): MetaOutput[] {
+  return list.sources
+    .sort((a, b) => b.rank - a.rank)
+    .map((v) => ({
+      type: 'source',
+      id: v.id,
+      name: v.name,
+      rank: v.rank,
+    }));
+}
+
+export function getAllEmbedMetaSorted(_list: ProviderList): MetaOutput[] {
+  return [];
+}
+
+export function getSpecificId(list: ProviderList, id: string): MetaOutput | null {
+  const foundSource = list.sources.find((v) => v.id === id);
+  if (foundSource) {
+    return {
+      type: 'source',
+      id: foundSource.id,
+      name: foundSource.name,
+      rank: foundSource.rank,
+    };
+  }
+
+  return null;
+}

src/main/runner.ts

@@ -0,0 +1,18 @@
+import { Stream } from '@/providers/streams';
+
+export type RunOutput = {
+  sourceId: string;
+  fromEmbed: boolean;
+  stream: Stream;
+};
+
+export type SourceRunOutput = {
+  sourceId: string;
+  stream?: Stream;
+  embeds: [];
+};
+
+export type EmbedRunOutput = {
+  embedId: string;
+  stream?: Stream;
+};

src/providers/all.ts

@@ -1,3 +1,24 @@
 import { Sourcerer } from '@/providers/base';
+import { hasDuplicates, isNotNull } from '@/utils/predicates';
 
-export const sources: Array<Sourcerer | null> = [];
+function gatherAllSources(): Array<Sourcerer | null> {
+  return [];
+}
+
+export interface ProviderList {
+  sources: Sourcerer[];
+}
+
+export function getProviders(): ProviderList {
+  const sources = gatherAllSources().filter(isNotNull);
+
+  const anyDuplicateId = hasDuplicates(sources.map((v) => v.id));
+  const anyDuplicateRank = hasDuplicates(sources.map((v) => v.rank));
+
+  if (anyDuplicateId) throw new Error('Duplicate id found in sources');
+  if (anyDuplicateRank) throw new Error('Duplicate rank found in sources');
+
+  return {
+    sources,
+  };
+}

src/providers/base.ts

@@ -1,11 +1,16 @@
+import { Stream } from '@/providers/streams';
 import { ScrapeContext } from '@/utils/context';
 
+export type SourcererOutput = {
+  stream?: Stream;
+};
+
 export type Sourcerer = {
   id: string;
   name: string; // displayed in the UI
   rank: number; // the higher the number, the earlier it gets put on the queue
   disabled?: boolean;
-  scrape: (input: ScrapeContext) => void;
+  scrape: (input: ScrapeContext) => Promise<SourcererOutput>;
 };
 
 export function makeSourcerer(state: Sourcerer): Sourcerer | null {

src/utils/context.ts

@@ -1,6 +1,7 @@
-import { Fetcher } from '@/utils/fetcher';
+import { UseableFetcher } from '@/fetchers/types';
 
 export interface ScrapeContext {
-  proxiedFetcher: Fetcher;
-  fetcher: Fetcher;
+  proxiedFetcher: UseableFetcher;
+  fetcher: UseableFetcher;
+  progress(val: number): void;
 }

src/utils/fetcher.ts

@@ -1,11 +0,0 @@
-export type FetcherOptions = {
-  baseUrl?: string;
-  headers?: Record<string, string>;
-  query?: Record<string, string>;
-  method?: 'GET' | 'POST';
-  body?: Record<string, any> | string | FormData;
-};
-
-export type Fetcher<T = any> = {
-  (url: string, ops?: FetcherOptions): T;
-};

src/utils/predicates.ts

@@ -0,0 +1,7 @@
+export function isNotNull<T>(value: T | null | undefined): value is T {
+  return value !== null && value !== undefined;
+}
+
+export function hasDuplicates<T>(values: Array<T>): boolean {
+  return new Set(values).size !== values.length;
+}