Start building a provider scrape system

src/index.ts

@@ -1,5 +1 @@
-import { LOG } from '@/testing/oof';
-
-export function test() {
-  console.log(LOG);
-}
+export { ProviderBuilderOptions, Providers, makeProviders } from '@/main/builder';

src/main/builder.ts

@@ -0,0 +1,16 @@
+import { Fetcher } from '@/utils/fetcher';
+
+export interface ProviderBuilderOptions {
+  // fetcher, every web request gets called through here
+  fetcher: Fetcher;
+
+  // proxied fetcher, if the scraper needs to access a CORS proxy. this fetcher will be called instead
+  // of the normal fetcher. Defaults to the normal fetcher.
+  proxiedFetcher?: Fetcher;
+}
+
+export interface Providers {}
+
+export function makeProviders(ops: ProviderBuilderOptions): Providers {
+  return {};
+}

src/providers/all.ts

@@ -0,0 +1,3 @@
+import { Sourcerer } from '@/providers/base';
+
+export const sources: Array<Sourcerer | null> = [];

src/providers/base.ts

@@ -0,0 +1,14 @@
+import { ScrapeContext } from '@/utils/context';
+
+export type Sourcerer = {
+  id: string;
+  name: string; // displayed in the UI
+  rank: number; // the higher the number, the earlier it gets put on the queue
+  disabled?: boolean;
+  scrape: (input: ScrapeContext) => void;
+};
+
+export function makeSourcerer(state: Sourcerer): Sourcerer | null {
+  if (state.disabled) return null;
+  return state;
+}

src/providers/streams.ts

@@ -0,0 +1,18 @@
+export type StreamFile = {
+  type: 'mp4';
+  url: string;
+};
+
+export type Qualities = '360' | '480' | '720' | '1080';
+
+export type FileBasedStream = {
+  type: 'file';
+  qualities: Record<Qualities, StreamFile>;
+};
+
+export type HlsBasedStream = {
+  type: 'hls';
+  playlist: string;
+};
+
+export type Stream = FileBasedStream | HlsBasedStream;

src/testing/oof.ts

@@ -1 +0,0 @@
-export const LOG = 'hello world';

src/utils/context.ts

@@ -0,0 +1,6 @@
+import { Fetcher } from '@/utils/fetcher';
+
+export interface ScrapeContext {
+  proxiedFetcher: Fetcher;
+  fetcher: Fetcher;
+}

src/utils/fetcher.ts

@@ -0,0 +1,11 @@
+export type FetcherOptions = {
+  baseUrl?: string;
+  headers?: Record<string, string>;
+  query?: Record<string, string>;
+  method?: 'GET' | 'POST';
+  body?: Record<string, any> | string | FormData;
+};
+
+export type Fetcher<T = any> = {
+  (url: string, ops?: FetcherOptions): T;
+};