From d31deeb907ca8af416060a4da546e1605882c361 Mon Sep 17 00:00:00 2001 From: mrjvs Date: Sat, 18 Nov 2023 21:26:53 +0100 Subject: [PATCH 1/6] introduce captions --- src/providers/captions.ts | 19 ++++++++++ src/providers/embeds/febBox.ts | 1 + src/providers/embeds/mixdrop.ts | 1 + src/providers/embeds/mp4upload.ts | 1 + src/providers/embeds/streamsb.ts | 1 + src/providers/embeds/upcloud.ts | 1 + src/providers/embeds/upstream.ts | 1 + src/providers/sources/remotestream.ts | 2 + .../sources/superstream/getStreamQualities.ts | 13 +++++-- src/providers/sources/superstream/index.ts | 14 ++++++- .../sources/superstream/subtitles.ts | 37 +++++++++++++++++++ src/providers/streams.ts | 2 + 12 files changed, 87 insertions(+), 6 deletions(-) create mode 100644 src/providers/captions.ts create mode 100644 src/providers/sources/superstream/subtitles.ts diff --git a/src/providers/captions.ts b/src/providers/captions.ts new file mode 100644 index 0000000..be3bf81 --- /dev/null +++ b/src/providers/captions.ts @@ -0,0 +1,19 @@ +export const captionTypes = { + srt: 'srt', + vtt: 'vtt', +}; +export type CaptionType = keyof typeof captionTypes; + +export type Caption = { + type: CaptionType; + url: string; + hasCorsRestrictions: boolean; + language: string; +}; + +export function getCaptionTypeFromUrl(url: string): CaptionType | null { + const extensions = Object.keys(captionTypes) as CaptionType[]; + const type = extensions.find((v) => url.endsWith(`.${v}`)); + if (!type) return null; + return type; +} diff --git a/src/providers/embeds/febBox.ts b/src/providers/embeds/febBox.ts index 06a3414..7855745 100644 --- a/src/providers/embeds/febBox.ts +++ b/src/providers/embeds/febBox.ts @@ -65,6 +65,7 @@ export const febBoxScraper = makeEmbed({ return { stream: { type: 'file', + captions: [], flags: [flags.NO_CORS], qualities: embedQualities, }, diff --git a/src/providers/embeds/mixdrop.ts b/src/providers/embeds/mixdrop.ts index e7df2cf..007738a 100644 --- a/src/providers/embeds/mixdrop.ts +++ b/src/providers/embeds/mixdrop.ts @@ -36,6 +36,7 @@ export const mixdropScraper = makeEmbed({ stream: { type: 'file', flags: [], + captions: [], qualities: { unknown: { type: 'mp4', diff --git a/src/providers/embeds/mp4upload.ts b/src/providers/embeds/mp4upload.ts index 4524018..81c2f94 100644 --- a/src/providers/embeds/mp4upload.ts +++ b/src/providers/embeds/mp4upload.ts @@ -18,6 +18,7 @@ export const mp4uploadScraper = makeEmbed({ stream: { type: 'file', flags: [flags.NO_CORS], + captions: [], qualities: { '1080': { type: 'mp4', diff --git a/src/providers/embeds/streamsb.ts b/src/providers/embeds/streamsb.ts index a42355a..d15b320 100644 --- a/src/providers/embeds/streamsb.ts +++ b/src/providers/embeds/streamsb.ts @@ -159,6 +159,7 @@ export const streamsbScraper = makeEmbed({ type: 'file', flags: [flags.NO_CORS], qualities, + captions: [], }, }; }, diff --git a/src/providers/embeds/upcloud.ts b/src/providers/embeds/upcloud.ts index 617c15a..683327f 100644 --- a/src/providers/embeds/upcloud.ts +++ b/src/providers/embeds/upcloud.ts @@ -101,6 +101,7 @@ export const upcloudScraper = makeEmbed({ type: 'hls', playlist: sources.file, flags: [flags.NO_CORS], + captions: [], }, }; }, diff --git a/src/providers/embeds/upstream.ts b/src/providers/embeds/upstream.ts index 62d2f01..852aac7 100644 --- a/src/providers/embeds/upstream.ts +++ b/src/providers/embeds/upstream.ts @@ -25,6 +25,7 @@ export const upstreamScraper = makeEmbed({ type: 'hls', playlist: link[1], flags: [flags.NO_CORS], + captions: [], }, }; } diff --git a/src/providers/sources/remotestream.ts b/src/providers/sources/remotestream.ts index 80a8621..696d897 100644 --- a/src/providers/sources/remotestream.ts +++ b/src/providers/sources/remotestream.ts @@ -23,6 +23,7 @@ export const remotestreamScraper = makeSourcerer({ return { embeds: [], stream: { + captions: [], playlist: playlistLink, type: 'hls', flags: [flags.NO_CORS], @@ -40,6 +41,7 @@ export const remotestreamScraper = makeSourcerer({ return { embeds: [], stream: { + captions: [], playlist: playlistLink, type: 'hls', flags: [flags.NO_CORS], diff --git a/src/providers/sources/superstream/getStreamQualities.ts b/src/providers/sources/superstream/getStreamQualities.ts index f968a18..655964d 100644 --- a/src/providers/sources/superstream/getStreamQualities.ts +++ b/src/providers/sources/superstream/getStreamQualities.ts @@ -6,14 +6,16 @@ import { sendRequest } from './sendRequest'; const allowedQualities = ['360', '480', '720', '1080']; export async function getStreamQualities(ctx: ScrapeContext, apiQuery: object) { - const mediaRes: { list: { path: string; real_quality: string }[] } = (await sendRequest(ctx, apiQuery)).data; + const mediaRes: { list: { path: string; quality: string; fid?: number }[] } = (await sendRequest(ctx, apiQuery)).data; ctx.progress(66); + console.log(mediaRes); + const qualityMap = mediaRes.list - .filter((file) => allowedQualities.includes(file.real_quality.replace('p', ''))) + .filter((file) => allowedQualities.includes(file.quality.replace('p', ''))) .map((file) => ({ url: file.path, - quality: file.real_quality.replace('p', ''), + quality: file.quality.replace('p', ''), })); const qualities: Record = {}; @@ -28,5 +30,8 @@ export async function getStreamQualities(ctx: ScrapeContext, apiQuery: object) { } }); - return qualities; + return { + qualities, + fid: mediaRes.list[0]?.fid, + }; } diff --git a/src/providers/sources/superstream/index.ts b/src/providers/sources/superstream/index.ts index 25286da..8e78167 100644 --- a/src/providers/sources/superstream/index.ts +++ b/src/providers/sources/superstream/index.ts @@ -1,5 +1,6 @@ import { flags } from '@/main/targets'; import { makeSourcerer } from '@/providers/base'; +import { getSubtitles } from '@/providers/sources/superstream/subtitles'; import { compareTitle } from '@/utils/compare'; import { NotFoundError } from '@/utils/errors'; @@ -41,11 +42,19 @@ export const superStreamScraper = makeSourcerer({ group: '', }; - const qualities = await getStreamQualities(ctx, apiQuery); + const { qualities, fid } = await getStreamQualities(ctx, apiQuery); return { embeds: [], stream: { + captions: await getSubtitles( + ctx, + superstreamId, + fid, + 'show', + ctx.media.episode.number, + ctx.media.season.number, + ), qualities, type: 'file', flags: [flags.NO_CORS], @@ -80,11 +89,12 @@ export const superStreamScraper = makeSourcerer({ group: '', }; - const qualities = await getStreamQualities(ctx, apiQuery); + const { qualities, fid } = await getStreamQualities(ctx, apiQuery); return { embeds: [], stream: { + captions: await getSubtitles(ctx, superstreamId, fid, 'movie'), qualities, type: 'file', flags: [flags.NO_CORS], diff --git a/src/providers/sources/superstream/subtitles.ts b/src/providers/sources/superstream/subtitles.ts new file mode 100644 index 0000000..89cdc39 --- /dev/null +++ b/src/providers/sources/superstream/subtitles.ts @@ -0,0 +1,37 @@ +import { Caption } from '@/providers/captions'; +import { sendRequest } from '@/providers/sources/superstream/sendRequest'; +import { ScrapeContext } from '@/utils/context'; + +interface CaptionApiResponse { + data: { + list: { + language: string; + subtitles: { + file_path: string; + }[]; + }[]; + }; +} + +export async function getSubtitles( + ctx: ScrapeContext, + id: string, + fid: number | undefined, + type: 'show' | 'movie', + episodeId?: number, + seasonId?: number, +): Promise { + const module = type === 'movie' ? 'Movie_srt_list_v2' : 'TV_srt_list_v2'; + const subtitleApiQuery = { + fid, + uid: '', + module, + mid: id, + episode: episodeId?.toString(), + season: seasonId?.toString(), + group: episodeId ? '' : undefined, + }; + // eslint-disable-next-line @typescript-eslint/no-unused-vars + const _subtitleList = ((await sendRequest(ctx, subtitleApiQuery)) as CaptionApiResponse).data.list; + return []; +} diff --git a/src/providers/streams.ts b/src/providers/streams.ts index 8a4674c..25847b2 100644 --- a/src/providers/streams.ts +++ b/src/providers/streams.ts @@ -1,4 +1,5 @@ import { Flags } from '@/main/targets'; +import { Caption } from '@/providers/captions'; export type StreamFile = { type: 'mp4'; @@ -18,6 +19,7 @@ export type HlsBasedStream = { type: 'hls'; flags: Flags[]; playlist: string; + captions: Caption[]; }; export type Stream = FileBasedStream | HlsBasedStream; From 4ce41e828dc0e7c8a0025777fda4020ed0f6330b Mon Sep 17 00:00:00 2001 From: mrjvs Date: Sat, 18 Nov 2023 22:44:34 +0100 Subject: [PATCH 2/6] add linked subs to superstream --- .../sources/superstream/subtitles.ts | 27 +++++++++++++++---- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/src/providers/sources/superstream/subtitles.ts b/src/providers/sources/superstream/subtitles.ts index 89cdc39..20f8647 100644 --- a/src/providers/sources/superstream/subtitles.ts +++ b/src/providers/sources/superstream/subtitles.ts @@ -1,12 +1,13 @@ -import { Caption } from '@/providers/captions'; +import { Caption, getCaptionTypeFromUrl } from '@/providers/captions'; import { sendRequest } from '@/providers/sources/superstream/sendRequest'; import { ScrapeContext } from '@/utils/context'; interface CaptionApiResponse { data: { list: { - language: string; subtitles: { + order: number; + lang: string; file_path: string; }[]; }[]; @@ -31,7 +32,23 @@ export async function getSubtitles( season: seasonId?.toString(), group: episodeId ? '' : undefined, }; - // eslint-disable-next-line @typescript-eslint/no-unused-vars - const _subtitleList = ((await sendRequest(ctx, subtitleApiQuery)) as CaptionApiResponse).data.list; - return []; + + const subtitleList = ((await sendRequest(ctx, subtitleApiQuery)) as CaptionApiResponse).data.list; + const output: Caption[] = []; + + subtitleList.forEach((sub) => { + const subtitle = sub.subtitles.sort((a, b) => a.order - b.order)[0]; + if (!subtitle) return; + const subtitleType = getCaptionTypeFromUrl(subtitle.file_path); + if (!subtitleType) return; + + output.push({ + language: subtitle.lang, + hasCorsRestrictions: true, + type: subtitleType, + url: subtitle.file_path, + }); + }); + + return output; } From fe4882b43e148341fc61e507bbcc77ff70d5b0f4 Mon Sep 17 00:00:00 2001 From: mrjvs Date: Sat, 18 Nov 2023 23:00:50 +0100 Subject: [PATCH 3/6] add upcloud captions --- src/providers/embeds/upcloud.ts | 16 +++++++++++++++- src/providers/sources/gomovies/index.ts | 8 ++------ 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/src/providers/embeds/upcloud.ts b/src/providers/embeds/upcloud.ts index 683327f..1ebe106 100644 --- a/src/providers/embeds/upcloud.ts +++ b/src/providers/embeds/upcloud.ts @@ -2,6 +2,7 @@ import crypto from 'crypto-js'; import { flags } from '@/main/targets'; import { makeEmbed } from '@/providers/base'; +import { Caption, getCaptionTypeFromUrl } from '@/providers/captions'; const { AES, enc } = crypto; @@ -96,12 +97,25 @@ export const upcloudScraper = makeEmbed({ if (!sources) throw new Error('upcloud source not found'); + const captions: Caption[] = []; + streamRes.tracks.forEach((track) => { + if (track.kind !== 'captions') return; + const type = getCaptionTypeFromUrl(track.file); + if (!type) return; + captions.push({ + language: track.label, // TODO Turn language name into ISO code + hasCorsRestrictions: false, + type, + url: track.file, + }); + }); + return { stream: { type: 'hls', playlist: sources.file, flags: [flags.NO_CORS], - captions: [], + captions, }, }; }, diff --git a/src/providers/sources/gomovies/index.ts b/src/providers/sources/gomovies/index.ts index e09cf8e..5240b02 100644 --- a/src/providers/sources/gomovies/index.ts +++ b/src/providers/sources/gomovies/index.ts @@ -17,9 +17,7 @@ export const goMoviesScraper = makeSourcerer({ async scrapeShow(ctx) { const search = await ctx.proxiedFetcher(`/ajax/search`, { method: 'POST', - body: JSON.stringify({ - keyword: ctx.media.title, - }), + body: new URLSearchParams({ keyword: ctx.media.title }), headers: { 'X-Requested-With': 'XMLHttpRequest', }, @@ -104,9 +102,7 @@ export const goMoviesScraper = makeSourcerer({ async scrapeMovie(ctx) { const search = await ctx.proxiedFetcher(`ajax/search`, { method: 'POST', - body: JSON.stringify({ - keyword: ctx.media.title, - }), + body: new URLSearchParams({ keyword: ctx.media.title }), headers: { 'X-Requested-With': 'XMLHttpRequest', }, From ab5dcc7b4289dc27a236e41f1e618e0d0e067314 Mon Sep 17 00:00:00 2001 From: mrjvs Date: Sat, 18 Nov 2023 23:14:06 +0100 Subject: [PATCH 4/6] Subtitle label conversion --- package.json | 1 + src/providers/captions.ts | 13 +++++++++++++ src/providers/embeds/upcloud.ts | 6 ++++-- src/providers/sources/superstream/subtitles.ts | 5 ++++- 4 files changed, 22 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index 232a641..76238f5 100644 --- a/package.json +++ b/package.json @@ -79,6 +79,7 @@ "cheerio": "^1.0.0-rc.12", "crypto-js": "^4.1.1", "form-data": "^4.0.0", + "iso-639-1": "^3.1.0", "nanoid": "^3.3.6", "node-fetch": "^2.7.0", "unpacker": "^1.0.1" diff --git a/src/providers/captions.ts b/src/providers/captions.ts index be3bf81..791bfa6 100644 --- a/src/providers/captions.ts +++ b/src/providers/captions.ts @@ -1,3 +1,5 @@ +import ISO6391 from 'iso-639-1'; + export const captionTypes = { srt: 'srt', vtt: 'vtt', @@ -17,3 +19,14 @@ export function getCaptionTypeFromUrl(url: string): CaptionType | null { if (!type) return null; return type; } + +export function labelToLanguageCode(label: string): string | null { + const code = ISO6391.getCode(label); + if (code.length === 0) return null; + return code; +} + +export function isValidLanguageCode(code: string | null): boolean { + if (!code) return false; + return ISO6391.validate(code); +} diff --git a/src/providers/embeds/upcloud.ts b/src/providers/embeds/upcloud.ts index 1ebe106..96cd14c 100644 --- a/src/providers/embeds/upcloud.ts +++ b/src/providers/embeds/upcloud.ts @@ -2,7 +2,7 @@ import crypto from 'crypto-js'; import { flags } from '@/main/targets'; import { makeEmbed } from '@/providers/base'; -import { Caption, getCaptionTypeFromUrl } from '@/providers/captions'; +import { Caption, getCaptionTypeFromUrl, labelToLanguageCode } from '@/providers/captions'; const { AES, enc } = crypto; @@ -102,8 +102,10 @@ export const upcloudScraper = makeEmbed({ if (track.kind !== 'captions') return; const type = getCaptionTypeFromUrl(track.file); if (!type) return; + const language = labelToLanguageCode(track.label); + if (!language) return; captions.push({ - language: track.label, // TODO Turn language name into ISO code + language, hasCorsRestrictions: false, type, url: track.file, diff --git a/src/providers/sources/superstream/subtitles.ts b/src/providers/sources/superstream/subtitles.ts index 20f8647..d49171d 100644 --- a/src/providers/sources/superstream/subtitles.ts +++ b/src/providers/sources/superstream/subtitles.ts @@ -1,4 +1,4 @@ -import { Caption, getCaptionTypeFromUrl } from '@/providers/captions'; +import { Caption, getCaptionTypeFromUrl, isValidLanguageCode } from '@/providers/captions'; import { sendRequest } from '@/providers/sources/superstream/sendRequest'; import { ScrapeContext } from '@/utils/context'; @@ -42,6 +42,9 @@ export async function getSubtitles( const subtitleType = getCaptionTypeFromUrl(subtitle.file_path); if (!subtitleType) return; + const validCode = isValidLanguageCode(subtitle.lang); + if (!validCode) return; + output.push({ language: subtitle.lang, hasCorsRestrictions: true, From 7d803f950619cf5f26068c6e880723c7d7ab063a Mon Sep 17 00:00:00 2001 From: mrjvs Date: Sat, 18 Nov 2023 23:20:11 +0100 Subject: [PATCH 5/6] bump version --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 76238f5..de7772c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@movie-web/providers", - "version": "1.0.5", + "version": "1.0.6", "description": "Package that contains all the providers of movie-web", "main": "./lib/index.umd.js", "types": "./lib/index.d.ts", From 78e0150944dbd1adfadf71b76e26e993c0008c00 Mon Sep 17 00:00:00 2001 From: mrjvs Date: Sat, 18 Nov 2023 23:21:31 +0100 Subject: [PATCH 6/6] actually a bigger bump is probably better --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index de7772c..7bc474d 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@movie-web/providers", - "version": "1.0.6", + "version": "1.1.0", "description": "Package that contains all the providers of movie-web", "main": "./lib/index.umd.js", "types": "./lib/index.d.ts",