individual provider runs

README.md

@@ -16,16 +16,25 @@ Todos:
      - custom source or embed order
      - are fetchers called?
      - is proxiedFetcher properly defaulted back to normal fetcher?
+   - ProviderControls.runSourceScraper()
+     - is source scraper called?
+     - does it return as expected?
+     - does it error when invalid type or id?
+   - ProviderControls.runEmbedScraper()
+     - is embed scraper called?
+     - does it return as expected?
+     - does it error when invalid id?
    - makeStandardFetcher()
      - do all parameters get passed to real fetch as expected?
      - does serialisation work as expected? (formdata + json + string)
      - does json responses get automatically parsed?
- - running individual scrapers
  - add all real providers
+ - fetcher for MW's simple-proxy
 
 Future todos:
  - docs: examples for nodejs + browser
  - docs: how to use + usecases
  - docs: examples for custom fetcher
- - choose an output environment (for browser or for native)
+ - docs: example with tmdb search
- - flixhq show support
+ - feature: choose an output environment (for browser or for native)
+ - feature: flixhq show support

src/index.ts

@@ -1,8 +1,15 @@
+export type { EmbedOutput, SourcererOutput } from '@/providers/base';
 export type { RunOutput } from '@/main/runner';
 export type { MetaOutput } from '@/main/meta';
 export type { FullScraperEvents } from '@/main/events';
 export type { MediaTypes, ShowMedia, ScrapeMedia, MovieMedia } from '@/main/media';
-export type { ProviderBuilderOptions, ProviderControls, RunnerOptions } from '@/main/builder';
+export type {
+  ProviderBuilderOptions,
+  ProviderControls,
+  RunnerOptions,
+  EmbedRunnerOptions,
+  SourceRunnerOptions,
+} from '@/main/builder';
 
 export { NotFoundError } from '@/utils/errors';
 export { makeProviders } from '@/main/builder';

src/main/builder.ts

@@ -1,9 +1,11 @@
 import { makeFullFetcher } from '@/fetchers/common';
 import { Fetcher } from '@/fetchers/types';
-import { FullScraperEvents } from '@/main/events';
+import { FullScraperEvents, IndividualScraperEvents } from '@/main/events';
+import { scrapeIndividualEmbed, scrapeInvidualSource } from '@/main/individualRunner';
 import { ScrapeMedia } from '@/main/media';
 import { MetaOutput, getAllEmbedMetaSorted, getAllSourceMetaSorted, getSpecificId } from '@/main/meta';
 import { RunOutput, runAllProviders } from '@/main/runner';
+import { EmbedOutput, SourcererOutput } from '@/providers/base';
 import { getProviders } from '@/providers/get';
 
 export interface ProviderBuilderOptions {
@@ -31,11 +33,39 @@ export interface RunnerOptions {
   media: ScrapeMedia;
 }
 
+export interface SourceRunnerOptions {
+  // object of event functions
+  events?: IndividualScraperEvents;
+
+  // the media you want to see sources from
+  media: ScrapeMedia;
+
+  // id of the source scraper you want to scrape from
+  id: string;
+}
+
+export interface EmbedRunnerOptions {
+  // object of event functions
+  events?: IndividualScraperEvents;
+
+  // the embed url
+  url: string;
+
+  // id of the embed scraper you want to scrape from
+  id: string;
+}
+
 export interface ProviderControls {
   // Run all providers one by one. in order of rank (highest first)
   // returns the stream, or null if none found
   runAll(runnerOps: RunnerOptions): Promise<RunOutput | null>;
 
+  // Run a specific source scraper
+  runSourceScraper(runnerOps: SourceRunnerOptions): Promise<SourcererOutput>;
+
+  // Run a specific embed scraper
+  runEmbedScraper(runnerOps: EmbedRunnerOptions): Promise<EmbedOutput>;
+
   // get meta data about a source or embed.
   getMetadata(id: string): MetaOutput | null;
 
@@ -54,12 +84,24 @@ export function makeProviders(ops: ProviderBuilderOptions): ProviderControls {
   };
 
   return {
-    runAll(runnerOps: RunnerOptions) {
+    runAll(runnerOps) {
       return runAllProviders(list, {
         ...providerRunnerOps,
         ...runnerOps,
       });
     },
+    runSourceScraper(runnerOps) {
+      return scrapeInvidualSource(list, {
+        ...providerRunnerOps,
+        ...runnerOps,
+      });
+    },
+    runEmbedScraper(runnerOps) {
+      return scrapeIndividualEmbed(list, {
+        ...providerRunnerOps,
+        ...runnerOps,
+      });
+    },
     getMetadata(id) {
       return getSpecificId(list, id);
     },

src/main/events.ts

@@ -46,3 +46,8 @@ export type FullScraperEvents = {
   // start scraping an item.
   start?: (id: string) => void;
 };
+
+export type IndividualScraperEvents = {
+  // update progress percentage and status of the currently scraping item
+  update?: (evt: UpdateEvent) => void;
+};

src/main/individualRunner.ts

@@ -1,5 +1,5 @@
 import { UseableFetcher } from '@/fetchers/types';
-import { UpdateEvent } from '@/main/events';
+import { IndividualScraperEvents } from '@/main/events';
 import { ScrapeMedia } from '@/main/media';
 import { EmbedOutput, SourcererOutput } from '@/providers/base';
 import { ProviderList } from '@/providers/get';
@@ -10,9 +10,7 @@ export type IndividualSourceRunnerOptions = {
   proxiedFetcher: UseableFetcher;
   media: ScrapeMedia;
   id: string;
-  events: {
+  events?: IndividualScraperEvents;
-    update?: (evt: UpdateEvent) => void;
-  };
 };
 
 export async function scrapeInvidualSource(
@@ -56,9 +54,7 @@ export type IndividualEmbedRunnerOptions = {
   proxiedFetcher: UseableFetcher;
   url: string;
   id: string;
-  events?: {
+  events?: IndividualScraperEvents;
-    update?: (evt: UpdateEvent) => void;
-  };
 };
 
 export async function scrapeIndividualEmbed(