callback events

README.md

@@ -6,5 +6,12 @@ Feel free to use for your own projects.
 features:
  - scrape popular streaming websites
  - works in both browser and NodeJS server
+ - choose between all streams or non-protected stream (for browser use)
 
 > This package is still WIP
+
+> TODO documentation: examples for nodejs + browser
+
+> TODO documentation: how to use + usecases
+
+> TODO documentation: examples on how to make a custom fetcher

src/main/builder.ts

@@ -1,13 +1,11 @@
 import { Fetcher } from '@/fetchers/types';
+import { FullScraperEvents, SingleScraperEvents } from '@/main/events';
 import { MetaOutput, getAllEmbedMetaSorted, getAllSourceMetaSorted, getSpecificId } from '@/main/meta';
-import { EmbedRunOutput, RunOutput, SourceRunOutput } from '@/main/runner';
+import { ProviderRunnerOptions, RunOutput, SourceRunOutput, runAllProviders } from '@/main/runner';
 import { getProviders } from '@/providers/all';
 
 // TODO meta data input (tmdb id, imdb id, title, release year)
 // TODO actually running scrapers
-// TODO documentation: examples for nodejs + browser
-// TODO documentation: how to use + usecases
-// TODO documentation: examples on how to make a custom fetcher
 
 export interface ProviderBuilderOptions {
   // fetcher, every web request gets called through here
@@ -21,13 +19,10 @@ export interface ProviderBuilderOptions {
 export interface ProviderControls {
   // Run all providers one by one. in order of rank (highest first)
   // returns the stream, or null if none found
-  runAll(): Promise<RunOutput | null>;
+  runAll(cbs: FullScraperEvents): Promise<RunOutput | null>;
 
   // Run a source provider
-  runSource(id: string): Promise<SourceRunOutput>;
-
-  // Run a embed provider
-  runEmbed(id: string): Promise<EmbedRunOutput>;
+  runSource(id: string, cbs: SingleScraperEvents): Promise<SourceRunOutput>;
 
   // get meta data about a source or embed.
   getMetadata(id: string): MetaOutput | null;
@@ -39,10 +34,17 @@ export interface ProviderControls {
   listEmbeds(): MetaOutput[];
 }
 
-export function makeProviders(_ops: ProviderBuilderOptions): ProviderControls {
+export function makeProviders(ops: ProviderBuilderOptions): ProviderControls {
   const list = getProviders();
+  const providerRunnerOps: ProviderRunnerOptions = {
+    fetcher: ops.fetcher,
+    proxiedFetcher: ops.proxiedFetcher ?? ops.fetcher,
+  };
 
   return {
+    runAll(cbs) {
+      return runAllProviders(providerRunnerOps, cbs);
+    },
     getMetadata(id) {
       return getSpecificId(list, id);
     },

src/main/events.ts

@@ -0,0 +1,46 @@
+export type UpdateEventStatus = 'success' | 'failure' | 'notfound' | 'pending';
+
+export type UpdateEvent = {
+  percentage: number;
+  status: UpdateEventStatus;
+};
+
+export type InitEvent = {
+  sourceIds: string[]; // list of source ids
+};
+
+export type DiscoverEmbedsEvent = {
+  sourceId: string;
+
+  // list of embeds that will be scraped in order
+  embeds: Array<{
+    id: string;
+    embedScraperId: string;
+  }>;
+};
+
+export type StartScrapingEvent = {
+  sourceId: string;
+
+  // embed Id (not embedScraperId)
+  embedId?: string;
+};
+
+export type SingleScraperEvents = {
+  update?: (evt: UpdateEvent) => void;
+};
+
+export type FullScraperEvents = {
+  // update progress percentage and status of the currently scraping item
+  update?: (evt: UpdateEvent) => void;
+
+  // initial list of scrapers its running, only triggers once per run.
+  init?: (evt: InitEvent) => void;
+
+  // list of embeds are discovered for the currently running source scraper
+  // triggers once per source scraper
+  discoverEmbeds?: (evt: DiscoverEmbedsEvent) => void;
+
+  // start scraping an item.
+  start?: (id: string) => void;
+};

src/main/runner.ts

@@ -1,8 +1,10 @@
+import { Fetcher } from '@/fetchers/types';
+import { FullScraperEvents } from '@/main/events';
 import { Stream } from '@/providers/streams';
 
 export type RunOutput = {
   sourceId: string;
-  fromEmbed: boolean;
+  embedId?: string;
   stream: Stream;
 };
 
@@ -16,3 +18,18 @@ export type EmbedRunOutput = {
   embedId: string;
   stream?: Stream;
 };
+
+export type ProviderRunnerOptions = {
+  fetcher: Fetcher;
+  proxiedFetcher: Fetcher;
+};
+
+export async function runAllProviders(_ops: ProviderRunnerOptions, _cbs: FullScraperEvents): Promise<RunOutput | null> {
+  return {
+    sourceId: '123',
+    stream: {
+      type: 'file',
+      qualities: {},
+    },
+  };
+}

src/providers/base.ts

@@ -1,7 +1,11 @@
 import { Stream } from '@/providers/streams';
-import { ScrapeContext } from '@/utils/context';
+import { EmbedScrapeContext, ScrapeContext } from '@/utils/context';
 
 export type SourcererOutput = {
+  embeds: {
+    embedId: string;
+    url: string;
+  }[];
   stream?: Stream;
 };
 
@@ -17,3 +21,20 @@ export function makeSourcerer(state: Sourcerer): Sourcerer | null {
   if (state.disabled) return null;
   return state;
 }
+
+export type EmbedOutput = {
+  stream?: Stream;
+};
+
+export type Embed = {
+  id: string;
+  name: string; // displayed in the UI
+  rank: number; // the higher the number, the earlier it gets put on the queue
+  disabled?: boolean;
+  scrape: (input: EmbedScrapeContext) => Promise<EmbedOutput>;
+};
+
+export function makeEmbed(state: Embed): Embed | null {
+  if (state.disabled) return null;
+  return state;
+}

src/providers/streams.ts

@@ -7,7 +7,7 @@ export type Qualities = '360' | '480' | '720' | '1080';
 
 export type FileBasedStream = {
   type: 'file';
-  qualities: Record<Qualities, StreamFile>;
+  qualities: Partial<Record<Qualities, StreamFile>>;
 };
 
 export type HlsBasedStream = {

src/utils/context.ts

@@ -1,7 +1,13 @@
 import { UseableFetcher } from '@/fetchers/types';
 
-export interface ScrapeContext {
+export type ScrapeContext = {
   proxiedFetcher: UseableFetcher;
   fetcher: UseableFetcher;
   progress(val: number): void;
-}
+};
+
+export type EmbedInput = {
+  url: string;
+};
+
+export type EmbedScrapeContext = EmbedInput & ScrapeContext;

src/utils/errors.ts

@@ -0,0 +1,6 @@
+export class NotFoundError extends Error {
+  constructor(reason?: string) {
+    super(`Couldn't found a stream: ${reason ?? 'not found'}`);
+    this.name = 'NotFoundError';
+  }
+}