Merge branch 'dev' into pr-14-v2

src/dev-cli/validate.ts

@@ -81,6 +81,7 @@ export async function processOptions(sources: Array<Embed | Sourcerer>, options:
   const providerOptions: ProviderMakerOptions = {
     fetcher,
     target: targets.ANY,
+    consistentIpForRequests: true,
   };
 
   return {

src/entrypoint/providers.ts

@@ -2,9 +2,9 @@ import { gatherAllEmbeds, gatherAllSources } from '@/providers/all';
 import { Embed, Sourcerer } from '@/providers/base';
 
 export function getBuiltinSources(): Sourcerer[] {
-  return gatherAllSources();
+  return gatherAllSources().filter((v) => !v.disabled);
 }
 
 export function getBuiltinEmbeds(): Embed[] {
-  return gatherAllEmbeds();
+  return gatherAllEmbeds().filter((v) => !v.disabled);
 }

src/index.ts

@@ -1,6 +1,6 @@
 export type { EmbedOutput, SourcererOutput } from '@/providers/base';
 export type { Stream, StreamFile, FileBasedStream, HlsBasedStream, Qualities } from '@/providers/streams';
-export type { Fetcher, FetcherOptions, FetcherResponse } from '@/fetchers/types';
+export type { Fetcher, DefaultedFetcherOptions, FetcherOptions, FetcherResponse } from '@/fetchers/types';
 export type { RunOutput } from '@/runners/runner';
 export type { MetaOutput } from '@/entrypoint/utils/meta';
 export type { FullScraperEvents } from '@/entrypoint/utils/events';
@@ -9,6 +9,8 @@ export type { MediaTypes, ShowMedia, ScrapeMedia, MovieMedia } from '@/entrypoin
 export type { ProviderControls, RunnerOptions, EmbedRunnerOptions, SourceRunnerOptions } from '@/entrypoint/controls';
 export type { ProviderBuilder } from '@/entrypoint/builder';
 export type { ProviderMakerOptions } from '@/entrypoint/declare';
+export type { MovieScrapeContext, ShowScrapeContext, EmbedScrapeContext, ScrapeContext } from '@/utils/context';
+export type { SourcererOptions, EmbedOptions } from '@/providers/base';
 
 export { NotFoundError } from '@/utils/errors';
 export { makeProviders } from '@/entrypoint/declare';

src/providers/embeds/febbox/hls.ts

@@ -1,5 +1,4 @@
 import { MediaTypes } from '@/entrypoint/utils/media';
-import { flags } from '@/entrypoint/utils/targets';
 import { makeEmbed } from '@/providers/base';
 import { parseInputUrl } from '@/providers/embeds/febbox/common';
 import { getStreams } from '@/providers/embeds/febbox/fileList';
@@ -16,6 +15,7 @@ export const febboxHlsScraper = makeEmbed({
   id: 'febbox-hls',
   name: 'Febbox (HLS)',
   rank: 160,
+  disabled: true,
   async scrape(ctx) {
     const { type, id, season, episode } = parseInputUrl(ctx.url);
     const sharelinkResult = await ctx.proxiedFetcher<{
@@ -40,7 +40,7 @@ export const febboxHlsScraper = makeEmbed({
         {
           id: 'primary',
           type: 'hls',
-          flags: [flags.CORS_ALLOWED],
+          flags: [],
           captions: await getSubtitles(ctx, id, firstStream.fid, type as MediaTypes, season, episode),
           playlist: `https://www.febbox.com/hls/main/${firstStream.oss_fid}.m3u8`,
         },

src/providers/embeds/febbox/qualities.ts

@@ -4,24 +4,35 @@ import { ScrapeContext } from '@/utils/context';
 
 const allowedQualities = ['360', '480', '720', '1080', '4k'];
 
-export async function getStreamQualities(ctx: ScrapeContext, apiQuery: object) {
-  const mediaRes: { list: { path: string; quality: string; fid?: number }[] } = (await sendRequest(ctx, apiQuery)).data;
+interface FebboxQuality {
+  path: string;
+  real_quality: string;
+  fid?: number;
+}
 
-  const qualityMap = mediaRes.list
-    .filter((file) => allowedQualities.includes(file.quality.replace('p', '')))
-    .map((file) => ({
-      url: file.path,
-      quality: file.quality.replace('p', ''),
-    }));
+function mapToQuality(quality: FebboxQuality): FebboxQuality | null {
+  const q = quality.real_quality.replace('p', '').toLowerCase();
+  if (!allowedQualities.includes(q)) return null;
+  return {
+    real_quality: q,
+    path: quality.path,
+    fid: quality.fid,
+  };
+}
+
+export async function getStreamQualities(ctx: ScrapeContext, apiQuery: object) {
+  const mediaRes: { list: FebboxQuality[] } = (await sendRequest(ctx, apiQuery)).data;
+
+  const qualityMap = mediaRes.list.map((v) => mapToQuality(v)).filter((v): v is FebboxQuality => !!v);
 
   const qualities: Record<string, StreamFile> = {};
 
   allowedQualities.forEach((quality) => {
-    const foundQuality = qualityMap.find((q) => q.quality === quality);
-    if (foundQuality && foundQuality.url) {
+    const foundQuality = qualityMap.find((q) => q.real_quality === quality && q.path);
+    if (foundQuality) {
       qualities[quality] = {
         type: 'mp4',
-        url: foundQuality.url,
+        url: foundQuality.path,
       };
     }
   });

src/providers/embeds/febbox/subtitles.ts

@@ -37,22 +37,28 @@ export async function getSubtitles(
   const subResult = (await sendRequest(ctx, subtitleApiQuery)) as CaptionApiResponse;
   const subtitleList = subResult.data.list;
   const output: Caption[] = [];
+  const languagesAdded: Record<string, true> = {};
 
   subtitleList.forEach((sub) => {
     const subtitle = sub.subtitles.sort((a, b) => b.order - a.order)[0];
     if (!subtitle) return;
+
     const subtitleFilePath = subtitle.file_path
       .replace(captionsDomains[0], captionsDomains[1])
       .replace(/\s/g, '+')
       .replace(/[()]/g, (c) => {
         return `%${c.charCodeAt(0).toString(16)}`;
       });
+
     const subtitleType = getCaptionTypeFromUrl(subtitleFilePath);
     if (!subtitleType) return;
 
     const validCode = isValidLanguageCode(subtitle.lang);
     if (!validCode) return;
 
+    if (languagesAdded[subtitle.lang]) return;
+    languagesAdded[subtitle.lang] = true;
+
     output.push({
       id: subtitleFilePath,
       language: subtitle.lang,

src/providers/sources/lookmovie/index.ts

@@ -10,8 +10,8 @@ async function universalScraper(ctx: MovieScrapeContext | ShowScrapeContext): Pr
   if (!lookmovieData) throw new NotFoundError('Media not found');
 
   ctx.progress(30);
-  const videoUrl = await scrape(ctx, ctx.media, lookmovieData);
-  if (!videoUrl) throw new NotFoundError('No video found');
+  const video = await scrape(ctx, ctx.media, lookmovieData);
+  if (!video.playlist) throw new NotFoundError('No video found');
 
   ctx.progress(60);
 
@@ -20,10 +20,10 @@ async function universalScraper(ctx: MovieScrapeContext | ShowScrapeContext): Pr
     stream: [
       {
         id: 'primary',
-        playlist: videoUrl,
+        playlist: video.playlist,
         type: 'hls',
         flags: [flags.IP_LOCKED],
-        captions: [],
+        captions: video.captions,
       },
     ],
   };
@@ -33,6 +33,7 @@ export const lookmovieScraper = makeSourcerer({
   id: 'lookmovie',
   name: 'LookMovie',
   rank: 1,
+  disabled: true,
   flags: [flags.IP_LOCKED],
   scrapeShow: universalScraper,
   scrapeMovie: universalScraper,

src/providers/sources/lookmovie/type.ts

@@ -39,8 +39,17 @@ interface VideoSources {
   [key: string]: string;
 }
 
+interface VideoSubtitles {
+  id?: number;
+  id_movie?: number;
+  url: string;
+  language: string;
+  shard?: string;
+}
+
 export interface StreamsDataResult {
   streams: VideoSources;
+  subtitles: VideoSubtitles[];
 }
 
 export interface ResultItem {

src/providers/sources/lookmovie/util.ts

@@ -4,7 +4,9 @@ import { ScrapeContext } from '@/utils/context';
 import { NotFoundError } from '@/utils/errors';
 
 import { Result, ResultItem, ShowDataResult, episodeObj } from './type';
-import { getVideoUrl } from './video';
+import { getVideo } from './video';
+
+export const baseUrl = 'https://lmscript.xyz';
 
 export async function searchAndFindMedia(
   ctx: ScrapeContext,
@@ -12,7 +14,7 @@ export async function searchAndFindMedia(
 ): Promise<ResultItem | undefined> {
   if (media.type === 'show') {
     const searchRes = await ctx.fetcher<Result>(`/v1/shows`, {
-      baseUrl: 'https://lmscript.xyz',
+      baseUrl,
       query: { 'filters[q]': media.title },
     });
 
@@ -23,7 +25,7 @@ export async function searchAndFindMedia(
   }
   if (media.type === 'movie') {
     const searchRes = await ctx.fetcher<Result>(`/v1/movies`, {
-      baseUrl: 'https://lmscript.xyz',
+      baseUrl,
       query: { 'filters[q]': media.title },
     });
 
@@ -40,7 +42,7 @@ export async function scrape(ctx: ScrapeContext, media: MovieMedia | ShowMedia,
     id = result.id_movie;
   } else if (media.type === 'show') {
     const data = await ctx.fetcher<ShowDataResult>(`/v1/shows`, {
-      baseUrl: 'https://lmscript.xyz',
+      baseUrl,
       query: { expand: 'episodes', id: result.id_show },
     });
 
@@ -54,6 +56,6 @@ export async function scrape(ctx: ScrapeContext, media: MovieMedia | ShowMedia,
   // Check ID
   if (id === null) throw new NotFoundError('Not found');
 
-  const videoUrl = await getVideoUrl(ctx, id, media);
-  return videoUrl;
+  const video = await getVideo(ctx, id, media);
+  return video;
 }

src/providers/sources/lookmovie/video.ts

@@ -1,7 +1,9 @@
 import { MovieMedia, ShowMedia } from '@/entrypoint/utils/media';
+import { Caption } from '@/providers/captions';
 import { ScrapeContext } from '@/utils/context';
 
 import { StreamsDataResult } from './type';
+import { baseUrl } from './util';
 
 export async function getVideoSources(
   ctx: ScrapeContext,
@@ -17,17 +19,17 @@ export async function getVideoSources(
     path = `/v1/movies/view`;
   }
   const data = await ctx.fetcher<StreamsDataResult>(path, {
-    baseUrl: 'https://lmscript.xyz',
-    query: { expand: 'streams', id },
+    baseUrl,
+    query: { expand: 'streams,subtitles', id },
   });
   return data;
 }
 
-export async function getVideoUrl(
+export async function getVideo(
   ctx: ScrapeContext,
   id: string,
   media: MovieMedia | ShowMedia,
-): Promise<string | null> {
+): Promise<{ playlist: string | null; captions: Caption[] }> {
   // Get sources
   const data = await getVideoSources(ctx, id, media);
   const videoSources = data.streams;
@@ -42,5 +44,16 @@ export async function getVideoUrl(
     }
   }
 
-  return videoUrl;
+  const captions: Caption[] = data.subtitles.map((sub) => ({
+    id: sub.url,
+    type: 'vtt',
+    url: `${baseUrl}${sub.url}`,
+    hasCorsRestrictions: false,
+    language: sub.language,
+  }));
+
+  return {
+    playlist: videoUrl,
+    captions,
+  };
 }

src/providers/sources/showbox/index.ts

@@ -1,6 +1,5 @@
 import { flags } from '@/entrypoint/utils/targets';
 import { SourcererOutput, makeSourcerer } from '@/providers/base';
-import { febboxHlsScraper } from '@/providers/embeds/febbox/hls';
 import { febboxMp4Scraper } from '@/providers/embeds/febbox/mp4';
 import { compareTitle } from '@/utils/compare';
 import { MovieScrapeContext, ShowScrapeContext } from '@/utils/context';
@@ -31,10 +30,6 @@ async function comboScraper(ctx: ShowScrapeContext | MovieScrapeContext): Promis
 
   return {
     embeds: [
-      {
-        embedId: febboxHlsScraper.id,
-        url: `/${ctx.media.type}/${id}/${season}/${episode}`,
-      },
       {
         embedId: febboxMp4Scraper.id,
         url: `/${ctx.media.type}/${id}/${season}/${episode}`,

src/providers/sources/zoechip/common.ts

@@ -38,7 +38,7 @@ export async function formatSource(
         embed.embedId = mixdropScraper.id;
         break;
       default:
-        throw new Error(`Failed to find ZoeChip embed source for ${link}`);
+        return null;
     }
 
     return embed;

src/providers/streams.ts

@@ -4,7 +4,6 @@ import { Caption } from '@/providers/captions';
 export type StreamFile = {
   type: 'mp4';
   url: string;
-  headers?: Record<string, string>;
 };
 
 export type Qualities = 'unknown' | '360' | '480' | '720' | '1080' | '4k';

src/runners/runner.ts

@@ -116,9 +116,12 @@ export async function runAllProviders(list: ProviderList, ops: ProviderRunnerOpt
       };
     }
 
-    if (output.embeds.length > 0) {
+    // run embed scrapers on listed embeds
+    const sortedEmbeds = output.embeds.sort((a, b) => embedIds.indexOf(a.embedId) - embedIds.indexOf(b.embedId));
+
+    if (sortedEmbeds.length > 0) {
       ops.events?.discoverEmbeds?.({
-        embeds: output.embeds.map((v, i) => ({
+        embeds: sortedEmbeds.map((v, i) => ({
           id: [s.id, i].join('-'),
           embedScraperId: v.embedId,
         })),
@@ -126,10 +129,6 @@ export async function runAllProviders(list: ProviderList, ops: ProviderRunnerOpt
       });
     }
 
-    // run embed scrapers on listed embeds
-    const sortedEmbeds = output.embeds;
-    sortedEmbeds.sort((a, b) => embedIds.indexOf(a.embedId) - embedIds.indexOf(b.embedId));
-
     for (const ind in sortedEmbeds) {
       if (!Object.prototype.hasOwnProperty.call(sortedEmbeds, ind)) continue;
       const e = sortedEmbeds[ind];