fix flixhq scraper and organize todos

README.md

@@ -9,16 +9,19 @@ features:
 
 > This package is still WIP
 
-> TODO documentation: examples for nodejs + browser
+Todos:
+ - add tests (integration, unit tests)
+ - running individual scrapers
+ - finish fetchers:
+   - make baseUrl param work
+   - proper serialization (with content-type headers) for standard fetcher
+   - automatically parse json
+ - error logging for failed scrapers
+ - make the lib not compile into one file, keep dependency structure
 
-> TODO documentation: how to use + usecases
+Future todos:
-
+ - docs: examples for nodejs + browser
-> TODO documentation: examples on how to make a custom fetcher
+ - docs: how to use + usecases
-
+ - docs: examples for custom fetcher
-> TODO functionality: running individual scrapers
+ - choose an output environment (for browser or for native)
-
+ - flixhq show support
-> TODO functionality: choose environment (for browser, for native)
-
-> TODO content: add all scrapers/providers
-
-> TODO tests: add tests

src/providers/all.ts

@@ -1,14 +1,14 @@
 import { Embed, Sourcerer } from '@/providers/base';
 import { upcloudScraper } from '@/providers/embeds/upcloud';
 import { flixhqScraper } from '@/providers/sources/flixhq/index';
-import { hasDuplicates, isNotNull } from '@/utils/predicates';
+import { hasDuplicates } from '@/utils/predicates';
 
-function gatherAllSources(): Array<Sourcerer | null> {
+function gatherAllSources(): Array<Sourcerer> {
   // all sources are gathered here
   return [flixhqScraper];
 }
 
-function gatherAllEmbeds(): Array<Embed | null> {
+function gatherAllEmbeds(): Array<Embed> {
   // all embeds are gathered here
   return [upcloudScraper];
 }
@@ -19,8 +19,8 @@ export interface ProviderList {
 }
 
 export function getProviders(): ProviderList {
-  const sources = gatherAllSources().filter(isNotNull);
+  const sources = gatherAllSources().filter((v) => !v?.disabled);
-  const embeds = gatherAllEmbeds().filter(isNotNull);
+  const embeds = gatherAllEmbeds().filter((v) => !v?.disabled);
   const combined = [...sources, ...embeds];
 
   const anyDuplicateId = hasDuplicates(combined.map((v) => v.id));

src/providers/base.ts

@@ -19,8 +19,7 @@ export type Sourcerer = {
   scrapeShow?: (input: ScrapeContext & { media: ShowMedia }) => Promise<SourcererOutput>;
 };
 
-export function makeSourcerer(state: Sourcerer): Sourcerer | null {
+export function makeSourcerer(state: Sourcerer): Sourcerer {
-  if (state.disabled) return null;
   return state;
 }
 
@@ -36,7 +35,6 @@ export type Embed = {
   scrape: (input: EmbedScrapeContext) => Promise<EmbedOutput>;
 };
 
-export function makeEmbed(state: Embed): Embed | null {
+export function makeEmbed(state: Embed): Embed {
-  if (state.disabled) return null;
   return state;
 }

src/providers/sources/flixhq/index.ts

@@ -1,4 +1,5 @@
 import { makeSourcerer } from '@/providers/base';
+import { upcloudScraper } from '@/providers/embeds/upcloud';
 import { getFlixhqSourceDetails, getFlixhqSources } from '@/providers/sources/flixhq/scrape';
 import { getFlixhqId } from '@/providers/sources/flixhq/search';
 import { NotFoundError } from '@/utils/errors';
@@ -19,7 +20,7 @@ export const flixhqScraper = makeSourcerer({
     return {
       embeds: [
         {
-          embedId: '', // TODO embed id
+          embedId: upcloudScraper.id,
           url: await getFlixhqSourceDetails(ctx, upcloudStream.episodeId),
         },
       ],

src/utils/predicates.ts

@@ -1,7 +1,3 @@
-export function isNotNull<T>(value: T | null | undefined): value is T {
-  return value !== null && value !== undefined;
-}
-
 export function hasDuplicates<T>(values: Array<T>): boolean {
   return new Set(values).size !== values.length;
 }