diff --git a/.docs/content/0.index.md b/.docs/content/0.index.md index 581634c..29d9522 100644 --- a/.docs/content/0.index.md +++ b/.docs/content/0.index.md @@ -8,7 +8,7 @@ layout: page --- cta: - Get Started - - /guide/usage + - /get-started/introduction secondary: - Open on GitHub → - https://github.com/movie-web/providers diff --git a/.docs/content/1.Guide/1.targets.md b/.docs/content/1.Guide/1.targets.md deleted file mode 100644 index d59e6d8..0000000 --- a/.docs/content/1.Guide/1.targets.md +++ /dev/null @@ -1,13 +0,0 @@ -# Targets - -When making an instance of the library using `makeProviders()`. It will immediately require choosing a target. - -::alert{type="info"} -A target is the device where the stream will be played on. -**Where the scraping is run has nothing to do with the target**, only where the stream is finally played in the end is significant in choosing a target. -:: - -#### Possible targets -- **`targets.BROWSER`** Stream will be played in a browser with CORS -- **`targets.NATIVE`** Stream will be played natively -- **`targets.ALL`** Stream will be played on a device with no restrictions of any kind diff --git a/.docs/content/1.Guide/_dir.yml b/.docs/content/1.Guide/_dir.yml deleted file mode 100644 index ad46c6b..0000000 --- a/.docs/content/1.Guide/_dir.yml +++ /dev/null @@ -1,2 +0,0 @@ -icon: ph:book-open-fill -navigation.redirect: /guide/usage diff --git a/.docs/content/1.get-started/0.introduction.md b/.docs/content/1.get-started/0.introduction.md new file mode 100644 index 0000000..eb21056 --- /dev/null +++ b/.docs/content/1.get-started/0.introduction.md @@ -0,0 +1,14 @@ +# Introduction + +## What is `@movie-web/providers`? + +`@movie-web/providers` is the soul of [movie-web.app](https://movie-web.app). It's a collection of scrapers of various streaming sites. It extracts the raw streams from those sites, so you can watch them without any extra fluff from the original sites. + +## What can I use this on? + +We support many different environments, here are a few examples: + - In browser, watch streams without needing a server to scrape (does need a proxy) + - In a native app, scrape in the app itself + - In a backend server, scrape on the server and give the streams to the client to watch. + +To find out how to configure the library for your environment, You can read [How to use on X](../2.essentials/0.usage-on-x.md). diff --git a/.docs/content/1.Guide/0.usage.md b/.docs/content/1.get-started/1.quick-start.md similarity index 56% rename from .docs/content/1.Guide/0.usage.md rename to .docs/content/1.get-started/1.quick-start.md index a0781d7..8804083 100644 --- a/.docs/content/1.Guide/0.usage.md +++ b/.docs/content/1.get-started/1.quick-start.md @@ -1,4 +1,6 @@ -# Usage +# Quick start + +## Installation Let's get started with `@movie-web/providers`. First lets install the package. @@ -18,11 +20,15 @@ Let's get started with `@movie-web/providers`. First lets install the package. To get started with scraping on the **server**, first you have to make an instance of the providers. -```ts -import { makeProviders, makeDefaultFetcher, targets } from '@movie-web/providers'; +::alert{type="warning"} +This snippet will only work on a **server**, for other environments, check out [Usage on X](../2.essentials/0.usage-on-x.md). +:: + +```ts [index.ts (server)] +import { makeProviders, makeStandardFetcher, targets } from '@movie-web/providers'; // this is how the library will make http requests -const myFetcher = makeDefaultFetcher(fetch); +const myFetcher = makeStandardFetcher(fetch); // make an instance of the providers library const providers = makeProviders({ @@ -33,7 +39,8 @@ const providers = makeProviders({ }) ``` -Perfect, now we can start scraping a stream: +Perfect, this instance of the providers you can reuse everywhere where you need to. +Now lets actually scrape an item: ```ts [index.ts (server)] // fetch some data from TMDB @@ -47,7 +54,7 @@ const media = { const output = await providers.runAll({ media: media }) - -if (!output) console.log("No stream found") -console.log(`stream url: ${output.stream.playlist}`) ``` + +Now we have our stream in the output variable. (If the output is `null` then nothing could be found.) +To find out how to use the streams, check out [Using streams](../2.essentials/4.using-streams.md). diff --git a/.docs/content/1.get-started/3.examples.md b/.docs/content/1.get-started/3.examples.md new file mode 100644 index 0000000..a2a90db --- /dev/null +++ b/.docs/content/1.get-started/3.examples.md @@ -0,0 +1,5 @@ +# Examples + +::alert{type="warning"} +There are no examples yet, stay tuned! +:: diff --git a/.docs/content/1.get-started/4.changelog.md b/.docs/content/1.get-started/4.changelog.md new file mode 100644 index 0000000..90dce1b --- /dev/null +++ b/.docs/content/1.get-started/4.changelog.md @@ -0,0 +1,30 @@ +--- +title: 'Changelog' +--- + +# Version 2.0.0 + +::alert{type="warning"} +There are breaking changes in this list, make sure to read them thoroughly if you plan on updating. +:: + +**Development tooling:** +- Added integration test for browser. To make sure the package keeps working in the browser +- Add type checking when building, previously it ignored them +- Refactored the main folder, now called entrypoint. +- Dev-cli code has been split up a bit more, a bit cleaner to navigate +- Dev-cli is now moved to `npm run cli` +- Dev-cli has now has support for running in a headless browser using a proxy URL. +- Fetchers can now return a full response with headers and everything + +**New features:** +- Added system to allow scraping ip locked sources through the consistentIpforRequests option. +- There is now a `buildProviders()` function that gives a builder for the `ProviderControls`. It's an alternative to `makeProviders()`. +- Streams can now return a headers object and a `preferredHeaders` object. which is required and optional headers for when using the stream. + +**Notable changes:** +- Renamed the NO_CORS flag to CORS_ALLOWED (meaning that resource sharing is allowed) +- Export Fetcher and Stream types with all types related to it +- Providers can now return a list of streams instead of just one. +- Captions now have identifiers returned with them. Just generally useful to have +- New targets and some of them renamed diff --git a/.docs/content/1.get-started/_dir.yml b/.docs/content/1.get-started/_dir.yml new file mode 100644 index 0000000..d43345e --- /dev/null +++ b/.docs/content/1.get-started/_dir.yml @@ -0,0 +1,2 @@ +icon: ph:shooting-star-fill +navigation.redirect: /get-started/introduction diff --git a/.docs/content/2.Api/7.makeStandardFetcher.md b/.docs/content/2.Api/7.makeStandardFetcher.md deleted file mode 100644 index 8a35842..0000000 --- a/.docs/content/2.Api/7.makeStandardFetcher.md +++ /dev/null @@ -1,20 +0,0 @@ -# `makeStandardFetcher` - -Make a fetcher from a `fetch()` API. It is used for making a instance of providers with `makeProviders()`. - -## Example - -```ts -import { targets, makeProviders, makeDefaultFetcher } from "@movie-web/providers"; - -const providers = makeProviders({ - fetcher: makeDefaultFetcher(fetch), - target: targets.NATIVE, -}); -``` - -## Type - -```ts -function makeDefaultFetcher(fetchApi: typeof fetch): Fetcher; -``` diff --git a/.docs/content/2.Api/_dir.yml b/.docs/content/2.Api/_dir.yml deleted file mode 100644 index 821aa66..0000000 --- a/.docs/content/2.Api/_dir.yml +++ /dev/null @@ -1,2 +0,0 @@ -icon: ph:file-code-fill -navigation.redirect: /api/makeproviders diff --git a/.docs/content/2.essentials/0.usage-on-x.md b/.docs/content/2.essentials/0.usage-on-x.md new file mode 100644 index 0000000..a8c3e80 --- /dev/null +++ b/.docs/content/2.essentials/0.usage-on-x.md @@ -0,0 +1,50 @@ +# How to use on X + +The library can run in many environments, so it can be tricky to figure out how to set it up. + +So here is a checklist, for more specific environments, keep reading below: + - When requests are very restricted (like browser client-side). Configure a proxied fetcher. + - When your requests come from the same device it will be streamed on (Not compatible with proxied fetcher). Set `consistentIpForRequests: true`. + - To set a target. Consult [Targets](./1.targets.md). + +To make use of the examples below, You check check out the following pages: + - [Quick start](../1.get-started/1.quick-start.md) + - [Using streams](../2.essentials/4.using-streams.md) + +## NodeJs server +```ts +import { makeProviders, makeStandardFetcher, targets } from '@movie-web/providers'; + +const providers = makeProviders({ + fetcher: makeStandardFetcher(fetch), + target: chooseYourself, // check out https://providers.docs.movie-web.app/essentials/targets +}) +``` + +## Browser client-side + +Using the provider package client-side requires a hosted version of simple-proxy. +Read more [about proxy fetchers](./2.fetchers.md#using-fetchers-on-the-browser). + +```ts +import { makeProviders, makeStandardFetcher, targets } from '@movie-web/providers'; + +const proxyUrl = "https://your.proxy.workers.dev/"; + +const providers = makeProviders({ + fetcher: makeStandardFetcher(fetch), + proxiedFetcher: makeSimpleProxyFetcher(proxyUrl, fetch), + target: target.BROWSER, +}) +``` + +## React native +```ts +import { makeProviders, makeStandardFetcher, targets } from '@movie-web/providers'; + +const providers = makeProviders({ + fetcher: makeStandardFetcher(fetch), + target: target.NATIVE, + consistentIpForRequests: true, +}) +``` diff --git a/.docs/content/2.essentials/1.targets.md b/.docs/content/2.essentials/1.targets.md new file mode 100644 index 0000000..cf1858b --- /dev/null +++ b/.docs/content/2.essentials/1.targets.md @@ -0,0 +1,14 @@ +# Targets + +When creating provider controls, you will immediately be required to choose a target. + +::alert{type="warning"} +A target is the device where the stream will be played on. +**Where the scraping is run has nothing to do with the target**, only where the stream is finally played in the end is significant in choosing a target. +:: + +#### Possible targets +- **`targets.BROWSER`** Stream will be played in a browser with CORS +- **`targets.BROWSER_EXTENSION`** Stream will be played in a browser using the movie-web extension (WIP) +- **`targets.NATIVE`** Stream will be played on a native video player +- **`targets.ANY`** No restrictions for selecting streams, will just give all of them diff --git a/.docs/content/1.Guide/2.fetchers.md b/.docs/content/2.essentials/2.fetchers.md similarity index 55% rename from .docs/content/1.Guide/2.fetchers.md rename to .docs/content/2.essentials/2.fetchers.md index 7b3c0cf..3fe0738 100644 --- a/.docs/content/1.Guide/2.fetchers.md +++ b/.docs/content/2.essentials/2.fetchers.md @@ -1,13 +1,13 @@ # Fetchers -When making an instance of the library using `makeProviders()`. It will immediately make a fetcher. +When creating provider controls, it will need you to configure a fetcher. This comes with some considerations depending on the environment youre running. ## Using `fetch()` In most cases, you can use the `fetch()` API. This will work in newer versions of Node.js (18 and above) and on the browser. ```ts -const fetcher = makeDefaultFetcher(fetch); +const fetcher = makeStandardFetcher(fetch); ``` If you using older version of Node.js. You can use the npm package `node-fetch` to polyfill fetch: @@ -15,7 +15,7 @@ If you using older version of Node.js. You can use the npm package `node-fetch` ```ts import fetch from "node-fetch"; -const fetcher = makeDefaultFetcher(fetch); +const fetcher = makeStandardFetcher(fetch); ``` ## Using fetchers on the browser @@ -29,7 +29,7 @@ const fetcher = makeSimpleProxyFetcher("https://your.proxy.workers.dev/", fetch) If you aren't able to use this specific proxy and need to use a different one, you can make your own fetcher in the next section. -## Making a custom fetcher +## Making a derived fetcher In some rare cases, a custom fetcher will need to be made. This can be quite difficult to do from scratch so it's recommended to base it off an existing fetcher and building your own functionality around it. @@ -37,6 +37,7 @@ In some rare cases, a custom fetcher will need to be made. This can be quite dif export function makeCustomFetcher(): Fetcher { const fetcher = makeStandardFetcher(f); const customFetcher: Fetcher = (url, ops) => { + // Do something with the options and url here return fetcher(url, ops); }; @@ -44,4 +45,30 @@ export function makeCustomFetcher(): Fetcher { } ``` -If you need to make your own fetcher for a proxy. Make sure you make it compatible with the following headers: `Cookie`, `Referer`, `Origin`. Proxied fetchers need to be able to write those headers when making a request. +If you need to make your own fetcher for a proxy. Make sure you make it compatible with the following headers: `Set-Cookie`, `Cookie`, `Referer`, `Origin`. Proxied fetchers need to be able to write/read those headers when making a request. + + +## Making a fetcher from scratch + +In some even rare cases, you need to make one completely from scratch. +This is the list of features it needs: + - Send/read every header + - Parse JSON, otherwise parse as text + - Send JSON, Formdata or normal strings + - get final destination url + +It's not recommended to do this at all, but if you have to. You can base your code on the original implementation of `makeStandardFetcher`. Check the out [source code for it here](https://github.com/movie-web/providers/blob/dev/src/fetchers/standardFetch.ts). + +Here is a basic template on how to make your own custom fetcher: + +```ts +const myFetcher: Fetcher = (url, ops) => { + // Do some fetching + return { + body: {}, + finalUrl: '', + headers: new Headers(), // should only contain headers from ops.readHeaders + statusCode: 200, + }; +} +``` diff --git a/.docs/content/2.essentials/3.customize-providers.md b/.docs/content/2.essentials/3.customize-providers.md new file mode 100644 index 0000000..a37c846 --- /dev/null +++ b/.docs/content/2.essentials/3.customize-providers.md @@ -0,0 +1,74 @@ +# Customize providers + +You make a provider controls in two ways. Either with `makeProviders()` (the simpler option) or with `buildProviders()` (more elaborate and extensive option). + +## `makeProviders()` (simple) + +To know what to set the configuration to, you can read [How to use on X](./0.usage-on-x.md) for a detailed guide on how to configure your controls. + +```ts +const providers = makeProviders({ + // fetcher, every web request gets called through here + fetcher: makeStandardFetcher(fetch), + + // proxied fetcher, if the scraper needs to access a CORS proxy. this fetcher will be called instead + // of the normal fetcher. Defaults to the normal fetcher. + proxiedFetcher: undefined; + + // target of where the streams will be used + target: targets.NATIVE; + + // Set this to true, if the requests will have the same IP as + // the device that the stream will be played on. + consistentIpForRequests: false; +}) + +``` + +## `buildProviders()` (advanced) + +To know what to set the configuration to, you can read [How to use on X](./0.usage-on-x.md) for a detailed guide on how to configure your controls. + +### Standard setup + +```ts +const providers = buildProviders() + .setTarget(targets.NATIVE) // target of where the streams will be used + .setFetcher(makeStandardFetcher(fetch)) // fetcher, every web request gets called through here + .addBuiltinProviders() // add all builtin providers, if this is not called, no providers will be added to the controls + .build(); +``` + +### Adding only select few providers + +Not all providers are great quality, so you can make a instance of the controls with only the providers you want. + +```ts +const providers = buildProviders() + .setTarget(targets.NATIVE) // target of where the streams will be used + .setFetcher(makeStandardFetcher(fetch)) // fetcher, every web request gets called through here + .addSource('showbox') // only add showbox source + .addEmbed('febbox-hls') // add febbox-hls embed, which is returned by showbox + .build(); +``` + + +### Adding your own scrapers to the providers + +If you have your own scraper and still want to use the nice utils of the provider library or just want to add on to the builtin providers. You can add your own custom source. + +```ts +const providers = buildProviders() + .setTarget(targets.NATIVE) // target of where the streams will be used + .setFetcher(makeStandardFetcher(fetch)) // fetcher, every web request gets called through here + .addSource({ // add your own source + id: 'my-scraper', + name: 'My scraper', + rank: 800, + flags: [], + scrapeMovie(ctx) { + throw new Error('Not implemented'); + } + }) + .build(); +``` diff --git a/.docs/content/2.essentials/4.using-streams.md b/.docs/content/2.essentials/4.using-streams.md new file mode 100644 index 0000000..64ec16f --- /dev/null +++ b/.docs/content/2.essentials/4.using-streams.md @@ -0,0 +1,84 @@ +# Using streams + +Streams can sometimes be quite picky on how they can be used. So here is a guide on how to use them. + +## Essentials + +All streams have the same common parameters: + - `Stream.type`: The type of stream. Either `hls` or `file` + - `Stream.id`: The id of this stream, unique per scraper output. + - `Stream.flags`: A list of flags that apply to this stream. Most people won't need to use it. + - `Stream.captions`: A list of captions/subtitles for this stream. + - `Stream.headers`: Either undefined or a key value object of headers you must set to use the stream. + - `Stream.preferredHeaders`: Either undefined or a key value object of headers you may want to set if you want optimal playback - but not required. + +Now let's delve deeper into how to actually watch these streams! + +## Streams with type `hls` + +HLS streams can be tough to watch, it's not a normal file you can just use. +These streams have an extra property `Stream.playlist` which contains the m3u8 playlist. + +Here is a code sample of how to use HLS streams in web context using hls.js + +```html + + + + +``` + +## Streams with type `file` + +File streams are quite easy to use, it just returns a new property: `Stream.qualities`. +This property is a map of quality and a stream file. So if you want to get 1080p quality you do `stream["1080"]` to get your stream file. It will return undefined if there is no quality like that. + +The possibly qualities are: `unknown`, `360`, `480`, `720`, `1080`, `4k`. +File based streams are garuanteed to always have one quality. + +Once you get a streamfile, you have the following parameters: + - `StreamFile.type`: Right now it can only be `mp4`. + - `StreamFile.url`: The URL linking to the video file. + +Here is a code sample of how to watch a file based stream the video in a browser: + +```html + + +``` + +## Streams with headers + +Streams have both a `Stream.headers` and a `Stream.preferredHeaders`. +The difference between the two is that `Stream.headers` **must** be set in other for the stream to work. While the other one is optional, and can only enhance the quality or performance. + +If your target is set to `BROWSER`. There will never be required headers, as it's not possible to do. + +## Using captions/subtitles + +All streams have a list of captions at `Stream.captions`. The structure looks like this: +```ts +type Caption = { + type: CaptionType; // Language type, either "srt" or "vtt" + id: string; // Unique per stream + url: string; // The url pointing to the subtitle file + hasCorsRestrictions: boolean; // If true, you will need to proxy it if you're running in a browser + language: string; // Language code of the caption +}; +``` diff --git a/.docs/content/2.essentials/_dir.yml b/.docs/content/2.essentials/_dir.yml new file mode 100644 index 0000000..a2dbf9c --- /dev/null +++ b/.docs/content/2.essentials/_dir.yml @@ -0,0 +1,3 @@ +icon: ph:info-fill +navigation.redirect: /essentials/usage +navigation.title: "Get started" diff --git a/.docs/content/3.in-depth/0.sources-and-embeds.md b/.docs/content/3.in-depth/0.sources-and-embeds.md new file mode 100644 index 0000000..265e528 --- /dev/null +++ b/.docs/content/3.in-depth/0.sources-and-embeds.md @@ -0,0 +1,11 @@ +# Sources vs embeds + +::alert{type="warning"} +This page isn't quite done yet, stay tuned! +:: + + diff --git a/.docs/content/3.in-depth/1.new-providers.md b/.docs/content/3.in-depth/1.new-providers.md new file mode 100644 index 0000000..2397bf8 --- /dev/null +++ b/.docs/content/3.in-depth/1.new-providers.md @@ -0,0 +1,12 @@ +# New providers + +::alert{type="warning"} +This page isn't quite done yet, stay tuned! +:: + + diff --git a/.docs/content/3.in-depth/2.flags.md b/.docs/content/3.in-depth/2.flags.md new file mode 100644 index 0000000..6ec7ffa --- /dev/null +++ b/.docs/content/3.in-depth/2.flags.md @@ -0,0 +1,10 @@ +# Flags + +Flags is the primary way the library seperates entities between different environments. +For example some sources only give back content that has the CORS headers set to allow anyone, so that source gets the flag `CORS_ALLOWED`. Now if you set your target to `BROWSER`, sources without that flag won't even get listed. + +This concept is applied in multiple away across the library. + +## Flag options + - `CORS_ALLOWED`: Headers from the output streams are set to allow any origin. + - `IP_LOCKED`: The streams are locked by ip, requester and watcher must be the same. diff --git a/.docs/content/3.in-depth/_dir.yml b/.docs/content/3.in-depth/_dir.yml new file mode 100644 index 0000000..03f39fc --- /dev/null +++ b/.docs/content/3.in-depth/_dir.yml @@ -0,0 +1,3 @@ +icon: ph:atom-fill +navigation.redirect: /in-depth/sources-and-embeds +navigation.title: "In-depth" diff --git a/.docs/content/4.extra-topics/0.development.md b/.docs/content/4.extra-topics/0.development.md new file mode 100644 index 0000000..8bf65a0 --- /dev/null +++ b/.docs/content/4.extra-topics/0.development.md @@ -0,0 +1,72 @@ +# Development / contributing + +::alert{type="warning"} +This page isn't quite done yet, stay tuned! +:: + + + +## Testing using the CLI + +Testing can be quite difficult for this library, unit tests can't really be made because of the unreliable nature of scrapers. +But manually testing by writing an entrypoint is also really annoying. + +Our solution is to make a CLI that you can use to run the scrapers, for everything else there are unit tests. + +### Setup +Make a `.env` file in the root of the repository and add a TMDB api key: `MOVIE_WEB_TMDB_API_KEY=KEY_HERE`. +Then make sure you've ran `npm i` to get all the dependencies. + +### Mode 1 - interactive + +To run the CLI without needing to learn all the arguments, simply run the following command and go with the flow. + +```sh +npm run cli +``` + +### Mode 2 - arguments + +For repeatability, it can be useful to specify the arguments one by one. +To see all the arguments, you can run the help command: +```sh +npm run cli -- -h +``` + +Then just run it with your arguments, for example: +```sh +npm run cli -- -sid showbox -tid 556574 +``` + +### Examples + +```sh +# Spirited away - showbox +npm run cli -- -sid showbox -tid 129 + +# Hamilton - flixhq +npm run cli -- -sid flixhq -tid 556574 + +# Arcane S1E1 - showbox +npm run cli -- -sid zoechip -tid 94605 -s 1 -e 1 + +# febbox mp4 - get streams from an embed (gotten from a source output) +npm run cli -- -sid febbox-mp4 -u URL_HERE +``` + +### Fetcher options + +The CLI comes with a few built-in fetchers: + - `node-fetch`: Fetch using the "node-fetch" library. + - `native`: Use the new fetch built into Node.JS (undici). + - `browser`: Start up headless chrome, and run the library in that context using a proxied fetcher. + +::alert{type="warning"} +The browser fetcher will require you to run `npm run build` before running the CLI. Otherwise you will get outdated results. +:: diff --git a/.docs/content/4.extra-topics/_dir.yml b/.docs/content/4.extra-topics/_dir.yml new file mode 100644 index 0000000..87faebd --- /dev/null +++ b/.docs/content/4.extra-topics/_dir.yml @@ -0,0 +1,3 @@ +icon: ph:aperture-fill +navigation.redirect: /extra-topics/development +navigation.title: "Extra topics" diff --git a/.docs/content/2.Api/0.makeProviders.md b/.docs/content/5.api-reference/0.makeProviders.md similarity index 75% rename from .docs/content/2.Api/0.makeProviders.md rename to .docs/content/5.api-reference/0.makeProviders.md index 2d2cdba..c76794a 100644 --- a/.docs/content/2.Api/0.makeProviders.md +++ b/.docs/content/5.api-reference/0.makeProviders.md @@ -1,12 +1,12 @@ # `makeProviders` -Make an instance of providers with configuration. +Make an instance of provider controls with configuration. This is the main entrypoint of the library. It is recommended to make one instance globally and reuse it throughout your application. ## Example ```ts -import { targets, makeProviders, makeDefaultFetcher } from "@movie-web/providers"; +import { targets, makeProviders, makeDefaultFetcher } from '@movie-web/providers'; const providers = makeProviders({ fetcher: makeDefaultFetcher(fetch), @@ -25,7 +25,7 @@ interface ProviderBuilderOptions { // instance of a fetcher, in case the request has cors restrictions. // this fetcher will be called instead of normal fetcher. - // if your environment doesnt have cors restrictions (like nodejs), there is no need to set this. + // if your environment doesnt have cors restrictions (like Node.JS), there is no need to set this. proxiedFetcher?: Fetcher; // target to get streams for diff --git a/.docs/content/2.Api/1.ProviderControlsRunAll.md b/.docs/content/5.api-reference/1.ProviderControlsRunAll.md similarity index 97% rename from .docs/content/2.Api/1.ProviderControlsRunAll.md rename to .docs/content/5.api-reference/1.ProviderControlsRunAll.md index 7633381..21999ef 100644 --- a/.docs/content/2.Api/1.ProviderControlsRunAll.md +++ b/.docs/content/5.api-reference/1.ProviderControlsRunAll.md @@ -9,9 +9,9 @@ You can attach events if you need to know what is going on while its processing. // media from TMDB const media = { type: 'movie', - title: "Hamilton", + title: 'Hamilton', releaseYear: 2020, - tmdbId: "556574" + tmdbId: '556574' } // scrape a stream diff --git a/.docs/content/2.Api/2.ProviderControlsrunSourceScraper.md b/.docs/content/5.api-reference/2.ProviderControlsrunSourceScraper.md similarity index 82% rename from .docs/content/2.Api/2.ProviderControlsrunSourceScraper.md rename to .docs/content/5.api-reference/2.ProviderControlsrunSourceScraper.md index e369880..341d77b 100644 --- a/.docs/content/2.Api/2.ProviderControlsrunSourceScraper.md +++ b/.docs/content/5.api-reference/2.ProviderControlsrunSourceScraper.md @@ -5,14 +5,14 @@ Run a specific source scraper and get its outputted streams. ## Example ```ts -import { SourcererOutput, NotFoundError } from "@movie-web/providers"; +import { SourcererOutput, NotFoundError } from '@movie-web/providers'; // media from TMDB const media = { type: 'movie', - title: "Hamilton", + title: 'Hamilton', releaseYear: 2020, - tmdbId: "556574" + tmdbId: '556574' } // scrape a stream from flixhq @@ -24,15 +24,15 @@ try { }) } catch (err) { if (err instanceof NotFoundError) { - console.log("source doesnt have this media"); + console.log('source doesnt have this media'); } else { - console.log("failed to scrape") + console.log('failed to scrape') } return; } if (!output.stream && output.embeds.length === 0) { - console.log("no streams found"); + console.log('no streams found'); } ``` diff --git a/.docs/content/2.Api/3.ProviderControlsrunEmbedScraper.md b/.docs/content/5.api-reference/3.ProviderControlsrunEmbedScraper.md similarity index 82% rename from .docs/content/2.Api/3.ProviderControlsrunEmbedScraper.md rename to .docs/content/5.api-reference/3.ProviderControlsrunEmbedScraper.md index 4a877df..ceb9870 100644 --- a/.docs/content/2.Api/3.ProviderControlsrunEmbedScraper.md +++ b/.docs/content/5.api-reference/3.ProviderControlsrunEmbedScraper.md @@ -5,17 +5,17 @@ Run a specific embed scraper and get its outputted streams. ## Example ```ts -import { SourcererOutput } from "@movie-web/providers"; +import { SourcererOutput } from '@movie-web/providers'; // scrape a stream from upcloud let output: EmbedOutput; try { - output = await providers.runSourceScraper({ + output = await providers.runEmbedScraper({ id: 'upcloud', url: 'https://example.com/123', }) } catch (err) { - console.log("failed to scrape") + console.log('failed to scrape') return; } diff --git a/.docs/content/2.Api/4.ProviderControlslistSources.md b/.docs/content/5.api-reference/4.ProviderControlslistSources.md similarity index 100% rename from .docs/content/2.Api/4.ProviderControlslistSources.md rename to .docs/content/5.api-reference/4.ProviderControlslistSources.md diff --git a/.docs/content/2.Api/5.ProviderControlslistEmbeds.md b/.docs/content/5.api-reference/5.ProviderControlslistEmbeds.md similarity index 100% rename from .docs/content/2.Api/5.ProviderControlslistEmbeds.md rename to .docs/content/5.api-reference/5.ProviderControlslistEmbeds.md diff --git a/.docs/content/2.Api/6.ProviderControlsgetMetadata.md b/.docs/content/5.api-reference/6.ProviderControlsgetMetadata.md similarity index 100% rename from .docs/content/2.Api/6.ProviderControlsgetMetadata.md rename to .docs/content/5.api-reference/6.ProviderControlsgetMetadata.md diff --git a/.docs/content/5.api-reference/7.makeStandardFetcher.md b/.docs/content/5.api-reference/7.makeStandardFetcher.md new file mode 100644 index 0000000..a7e5e76 --- /dev/null +++ b/.docs/content/5.api-reference/7.makeStandardFetcher.md @@ -0,0 +1,20 @@ +# `makeStandardFetcher` + +Make a fetcher from a `fetch()` API. It is used for making a instance of provider controls. + +## Example + +```ts +import { targets, makeProviders, makeDefaultFetcher } from '@movie-web/providers'; + +const providers = makeProviders({ + fetcher: makeStandardFetcher(fetch), + target: targets.ANY, +}); +``` + +## Type + +```ts +function makeStandardFetcher(fetchApi: typeof fetch): Fetcher; +``` diff --git a/.docs/content/2.Api/8.makeSimpleProxyFetcher.md b/.docs/content/5.api-reference/8.makeSimpleProxyFetcher.md similarity index 85% rename from .docs/content/2.Api/8.makeSimpleProxyFetcher.md rename to .docs/content/5.api-reference/8.makeSimpleProxyFetcher.md index 81dd010..3f5f76a 100644 --- a/.docs/content/2.Api/8.makeSimpleProxyFetcher.md +++ b/.docs/content/5.api-reference/8.makeSimpleProxyFetcher.md @@ -5,9 +5,9 @@ Make a fetcher to use with [movie-web/simple-proxy](https://github.com/movie-web ## Example ```ts -import { targets, makeProviders, makeDefaultFetcher, makeSimpleProxyFetcher } from "@movie-web/providers"; +import { targets, makeProviders, makeDefaultFetcher, makeSimpleProxyFetcher } from '@movie-web/providers'; -const proxyUrl = "https://your.proxy.workers.dev/" +const proxyUrl = 'https://your.proxy.workers.dev/' const providers = makeProviders({ fetcher: makeDefaultFetcher(fetch), diff --git a/.docs/content/5.api-reference/_dir.yml b/.docs/content/5.api-reference/_dir.yml new file mode 100644 index 0000000..1432790 --- /dev/null +++ b/.docs/content/5.api-reference/_dir.yml @@ -0,0 +1,3 @@ +icon: ph:code-simple-fill +navigation.redirect: /api/makeproviders +navigation.title: "Api reference" diff --git a/.eslintrc.js b/.eslintrc.js index 7939452..0e7322b 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -3,7 +3,7 @@ module.exports = { browser: true, }, extends: ['airbnb-base', 'plugin:@typescript-eslint/recommended', 'plugin:prettier/recommended'], - ignorePatterns: ['lib/*', 'tests/*', '/*.js', '/*.ts', '/**/*.test.ts', 'test/*'], + ignorePatterns: ['lib/*', 'tests/*', '/*.js', '/*.ts', '/src/__test__/*', '/**/*.test.ts', 'test/*'], parser: '@typescript-eslint/parser', parserOptions: { project: './tsconfig.json', diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..2c2b7c4 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2023 movie-web + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md index d30d021..a0cadad 100644 --- a/README.md +++ b/README.md @@ -9,27 +9,6 @@ features: Visit documentation here: https://providers.docs.movie-web.app/ -## Development -To make testing scrapers easier during development a CLI tool is available to run specific sources. To run the CLI testing tool, use `npm run test:dev`. The script supports 2 execution modes +## How to run locally or test my changes -- CLI Mode, for passing in arguments directly to the script -- Question Mode, where the script asks you questions about which source you wish to test - -The following CLI Mode arguments are available - -| Argument | Alias | Description | Default | -|---------------|--------|-------------------------------------------------------------------------|--------------| -| `--fetcher` | `-f` | Fetcher type. Either `node-fetch` or `native` | `node-fetch` | -| `--source-id` | `-sid` | Source ID for the source to be tested | | -| `--tmdb-id` | `-tid` | TMDB ID for the media to scrape. Only used if source is a provider | | -| `--type` | `-t` | Media type. Either `movie` or `show`. Only used if source is a provider | `movie` | -| `--season` | `-s` | Season number. Only used if type is `show` | `0` | -| `--episode` | `-e` | Episode number. Only used if type is `show` | `0` | -| `--url` | `-u` | URL to a video embed. Only used if source is an embed | | -| `--help` | `-h` | Shows help for the command arguments | | - -Example testing the FlixHQ source on the movie "Spirited Away" - -```bash -npm run test:dev -- -sid flixhq -tid 129 -t movie -``` +These topics are also covered in the documentation, [read about it here](https://providers.docs.movie-web.app/extra-topics/development). diff --git a/package-lock.json b/package-lock.json index 9231ec0..6a4523d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@movie-web/providers", - "version": "1.1.4", + "version": "1.1.5", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@movie-web/providers", - "version": "1.1.4", + "version": "1.1.5", "license": "MIT", "dependencies": { "cheerio": "^1.0.0-rc.12", @@ -36,6 +36,7 @@ "eslint-plugin-prettier": "^4.2.1", "node-fetch": "^2.7.0", "prettier": "^2.6.2", + "puppeteer": "^21.6.1", "spinnies": "^0.5.1", "ts-node": "^10.9.1", "tsc-alias": "^1.6.7", @@ -67,6 +68,184 @@ "node": ">=6.0.0" } }, + "node_modules/@babel/code-frame": { + "version": "7.23.5", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.23.5.tgz", + "integrity": "sha512-CgH3s1a96LipHCmSUmYFPwY7MNx8C3avkq7i4Wl3cfa662ldtUe4VM1TPXX70pfmrlWTb6jLqTYrZyT2ZTJBgA==", + "dev": true, + "dependencies": { + "@babel/highlight": "^7.23.4", + "chalk": "^2.4.2" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/code-frame/node_modules/ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dev": true, + "dependencies": { + "color-convert": "^1.9.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/@babel/code-frame/node_modules/chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dev": true, + "dependencies": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/@babel/code-frame/node_modules/color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dev": true, + "dependencies": { + "color-name": "1.1.3" + } + }, + "node_modules/@babel/code-frame/node_modules/color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==", + "dev": true + }, + "node_modules/@babel/code-frame/node_modules/escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", + "dev": true, + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/@babel/code-frame/node_modules/has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/@babel/code-frame/node_modules/supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dev": true, + "dependencies": { + "has-flag": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/@babel/helper-validator-identifier": { + "version": "7.22.20", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.22.20.tgz", + "integrity": "sha512-Y4OZ+ytlatR8AI+8KZfKuL5urKp7qey08ha31L8b3BwewJAoJamTzyvxPR/5D+KkdJCGPq/+8TukHBlY10FX9A==", + "dev": true, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/highlight": { + "version": "7.23.4", + "resolved": "https://registry.npmjs.org/@babel/highlight/-/highlight-7.23.4.tgz", + "integrity": "sha512-acGdbYSfp2WheJoJm/EBBBLh/ID8KDc64ISZ9DYtBmC8/Q204PZJLHyzeB5qMzJ5trcOkybd78M4x2KWsUq++A==", + "dev": true, + "dependencies": { + "@babel/helper-validator-identifier": "^7.22.20", + "chalk": "^2.4.2", + "js-tokens": "^4.0.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/highlight/node_modules/ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dev": true, + "dependencies": { + "color-convert": "^1.9.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/@babel/highlight/node_modules/chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dev": true, + "dependencies": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/@babel/highlight/node_modules/color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dev": true, + "dependencies": { + "color-name": "1.1.3" + } + }, + "node_modules/@babel/highlight/node_modules/color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==", + "dev": true + }, + "node_modules/@babel/highlight/node_modules/escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", + "dev": true, + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/@babel/highlight/node_modules/has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/@babel/highlight/node_modules/supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dev": true, + "dependencies": { + "has-flag": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/@babel/parser": { "version": "7.23.5", "dev": true, @@ -366,6 +545,27 @@ "node": ">= 8" } }, + "node_modules/@puppeteer/browsers": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/@puppeteer/browsers/-/browsers-1.9.0.tgz", + "integrity": "sha512-QwguOLy44YBGC8vuPP2nmpX4MUN2FzWbsnvZJtiCzecU3lHmVZkaC1tq6rToi9a200m8RzlVtDyxCS0UIDrxUg==", + "dev": true, + "dependencies": { + "debug": "4.3.4", + "extract-zip": "2.0.1", + "progress": "2.0.3", + "proxy-agent": "6.3.1", + "tar-fs": "3.0.4", + "unbzip2-stream": "1.4.3", + "yargs": "17.7.2" + }, + "bin": { + "browsers": "lib/cjs/main-cli.js" + }, + "engines": { + "node": ">=16.3.0" + } + }, "node_modules/@rollup/pluginutils": { "version": "5.1.0", "dev": true, @@ -442,6 +642,12 @@ "dev": true, "license": "MIT" }, + "node_modules/@tootallnate/quickjs-emscripten": { + "version": "0.23.0", + "resolved": "https://registry.npmjs.org/@tootallnate/quickjs-emscripten/-/quickjs-emscripten-0.23.0.tgz", + "integrity": "sha512-C5Mc6rdnsaJDjO3UpGW/CQTHtCKaYlScZTly4JIu97Jxo/odCiH0ITnDXSJPTOrEKk/ycSZ0AOgTmkDtkOsvIA==", + "dev": true + }, "node_modules/@tsconfig/node10": { "version": "1.0.9", "dev": true, @@ -549,6 +755,16 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/yauzl": { + "version": "2.10.3", + "resolved": "https://registry.npmjs.org/@types/yauzl/-/yauzl-2.10.3.tgz", + "integrity": "sha512-oJoftv0LSuaDZE3Le4DbKX+KS9G36NzOeSap90UIK0yMA/NhKJhqlSGtNDORNRaIbQfzjXDrQa0ytJ6mNRGz/Q==", + "dev": true, + "optional": true, + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@typescript-eslint/eslint-plugin": { "version": "5.62.0", "dev": true, @@ -969,6 +1185,18 @@ "node": ">=0.4.0" } }, + "node_modules/agent-base": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.0.tgz", + "integrity": "sha512-o/zjMZRhJxny7OyEF+Op8X+efiELC7k7yOjMzgfzVqOzXqkBkWI79YoTdOtsuWd5BWhAGAuOY/Xa6xpiaWXiNg==", + "dev": true, + "dependencies": { + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/ajv": { "version": "6.12.6", "dev": true, @@ -1154,6 +1382,24 @@ "node": "*" } }, + "node_modules/ast-types": { + "version": "0.13.4", + "resolved": "https://registry.npmjs.org/ast-types/-/ast-types-0.13.4.tgz", + "integrity": "sha512-x1FCFnFifvYDDzTaLII71vG5uvDwgtmDTEVWAxrgeiR8VjMONcCXJx7E+USjDtHlwFmt9MysbqgF9b9Vjr6w+w==", + "dev": true, + "dependencies": { + "tslib": "^2.0.1" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/ast-types/node_modules/tslib": { + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz", + "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==", + "dev": true + }, "node_modules/asynckit": { "version": "0.4.0", "license": "MIT" @@ -1169,11 +1415,46 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/b4a": { + "version": "1.6.4", + "resolved": "https://registry.npmjs.org/b4a/-/b4a-1.6.4.tgz", + "integrity": "sha512-fpWrvyVHEKyeEvbKZTVOeZF3VSKKWtJxFIxX/jaVPf+cLbGUSitjb49pHLqPV2BUNNZ0LcoeEGfE/YCpyDYHIw==", + "dev": true + }, "node_modules/balanced-match": { "version": "1.0.2", "dev": true, "license": "MIT" }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, + "node_modules/basic-ftp": { + "version": "5.0.4", + "resolved": "https://registry.npmjs.org/basic-ftp/-/basic-ftp-5.0.4.tgz", + "integrity": "sha512-8PzkB0arJFV4jJWSGOYR+OEic6aeKMu/osRhBULN6RY0ykby6LKhbmuQ5ublvaas5BOwboah5D87nrHyuh8PPA==", + "dev": true, + "engines": { + "node": ">=10.0.0" + } + }, "node_modules/binary-extensions": { "version": "2.2.0", "dev": true, @@ -1206,6 +1487,39 @@ "node": ">=8" } }, + "node_modules/buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, + "node_modules/buffer-crc32": { + "version": "0.2.13", + "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", + "integrity": "sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==", + "dev": true, + "engines": { + "node": "*" + } + }, "node_modules/cac": { "version": "6.7.14", "dev": true, @@ -1349,6 +1663,19 @@ "node": ">= 6" } }, + "node_modules/chromium-bidi": { + "version": "0.5.1", + "resolved": "https://registry.npmjs.org/chromium-bidi/-/chromium-bidi-0.5.1.tgz", + "integrity": "sha512-dcCqOgq9fHKExc2R4JZs/oKbOghWpUNFAJODS8WKRtLhp3avtIH5UDCBrutdqZdh3pARogH8y1ObXm87emwb3g==", + "dev": true, + "dependencies": { + "mitt": "3.0.1", + "urlpattern-polyfill": "9.0.0" + }, + "peerDependencies": { + "devtools-protocol": "*" + } + }, "node_modules/cli-cursor": { "version": "3.1.0", "dev": true, @@ -1360,6 +1687,20 @@ "node": ">=8" } }, + "node_modules/cliui": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", + "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", + "dev": true, + "dependencies": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.1", + "wrap-ansi": "^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, "node_modules/color-convert": { "version": "2.0.1", "dev": true, @@ -1422,11 +1763,46 @@ "dev": true, "license": "MIT" }, + "node_modules/cosmiconfig": { + "version": "8.3.6", + "resolved": "https://registry.npmjs.org/cosmiconfig/-/cosmiconfig-8.3.6.tgz", + "integrity": "sha512-kcZ6+W5QzcJ3P1Mt+83OUv/oHFqZHIx8DuxG6eZ5RGMERoLqp4BuGjhHLYGK+Kf5XVkQvqBSmAy/nGWN3qDgEA==", + "dev": true, + "dependencies": { + "import-fresh": "^3.3.0", + "js-yaml": "^4.1.0", + "parse-json": "^5.2.0", + "path-type": "^4.0.0" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/d-fischer" + }, + "peerDependencies": { + "typescript": ">=4.9.5" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } + } + }, "node_modules/create-require": { "version": "1.1.1", "dev": true, "license": "MIT" }, + "node_modules/cross-fetch": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-4.0.0.tgz", + "integrity": "sha512-e4a5N8lVvuLgAWgnCrLr2PP0YyDOTHa9H/Rj54dirp61qXnNq46m82bRhNqIA5VccJtWBvPTFRV3TtvHUKPB1g==", + "dev": true, + "dependencies": { + "node-fetch": "^2.6.12" + } + }, "node_modules/cross-spawn": { "version": "7.0.3", "dev": true, @@ -1469,6 +1845,15 @@ "url": "https://github.com/sponsors/fb55" } }, + "node_modules/data-uri-to-buffer": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-6.0.1.tgz", + "integrity": "sha512-MZd3VlchQkp8rdend6vrx7MmVDJzSNTBvghvKjirLkD+WTChA3KUf0jkE68Q4UyctNqI11zZO9/x2Yx+ub5Cvg==", + "dev": true, + "engines": { + "node": ">= 14" + } + }, "node_modules/de-indent": { "version": "1.0.2", "dev": true, @@ -1535,6 +1920,20 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/degenerator": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/degenerator/-/degenerator-5.0.1.tgz", + "integrity": "sha512-TllpMR/t0M5sqCXfj85i4XaAzxmS5tVA16dqvdkMwGmzI+dXLXnw3J+3Vdv7VKw+ThlTMboK6i9rnZ6Nntj5CQ==", + "dev": true, + "dependencies": { + "ast-types": "^0.13.4", + "escodegen": "^2.1.0", + "esprima": "^4.0.1" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/delayed-stream": { "version": "1.0.0", "license": "MIT", @@ -1542,6 +1941,12 @@ "node": ">=0.4.0" } }, + "node_modules/devtools-protocol": { + "version": "0.0.1203626", + "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.1203626.tgz", + "integrity": "sha512-nEzHZteIUZfGCZtTiS1fRpC8UZmsfD1SiyPvaUNvS13dvKf666OAm8YTi0+Ca3n1nLEyu49Cy4+dPWpaHFJk9g==", + "dev": true + }, "node_modules/diff": { "version": "4.0.2", "dev": true, @@ -1638,6 +2043,21 @@ "url": "https://github.com/motdotla/dotenv?sponsor=1" } }, + "node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true + }, + "node_modules/end-of-stream": { + "version": "1.4.4", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", + "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", + "dev": true, + "dependencies": { + "once": "^1.4.0" + } + }, "node_modules/enhanced-resolve": { "version": "5.15.0", "dev": true, @@ -1672,6 +2092,15 @@ "url": "https://github.com/fb55/entities?sponsor=1" } }, + "node_modules/error-ex": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.2.tgz", + "integrity": "sha512-7dFHNmqeFSEt2ZBsCriorKnn3Z2pj+fd9kmI6QoWw4//DL+icEBfc0U7qJCisqrTsKTjw4fNFy2pW9OqStD84g==", + "dev": true, + "dependencies": { + "is-arrayish": "^0.2.1" + } + }, "node_modules/es-abstract": { "version": "1.22.3", "dev": true, @@ -1797,6 +2226,15 @@ "@esbuild/win32-x64": "0.18.20" } }, + "node_modules/escalade": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", + "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==", + "dev": true, + "engines": { + "node": ">=6" + } + }, "node_modules/escape-string-regexp": { "version": "4.0.0", "dev": true, @@ -1808,6 +2246,36 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/escodegen": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-2.1.0.tgz", + "integrity": "sha512-2NlIDTwUWJN0mRPQOdtQBzbUHvdGY2P1VXSyU83Q3xKxM7WHX2Ql8dKq782Q9TgQUNOLEzEYu9bzLNj1q88I5w==", + "dev": true, + "dependencies": { + "esprima": "^4.0.1", + "estraverse": "^5.2.0", + "esutils": "^2.0.2" + }, + "bin": { + "escodegen": "bin/escodegen.js", + "esgenerate": "bin/esgenerate.js" + }, + "engines": { + "node": ">=6.0" + }, + "optionalDependencies": { + "source-map": "~0.6.1" + } + }, + "node_modules/escodegen/node_modules/estraverse": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", + "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", + "dev": true, + "engines": { + "node": ">=4.0" + } + }, "node_modules/eslint": { "version": "8.55.0", "dev": true, @@ -2126,6 +2594,19 @@ "url": "https://opencollective.com/eslint" } }, + "node_modules/esprima": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", + "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", + "dev": true, + "bin": { + "esparse": "bin/esparse.js", + "esvalidate": "bin/esvalidate.js" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/esquery": { "version": "1.5.0", "dev": true, @@ -2185,6 +2666,26 @@ "node": ">=0.10.0" } }, + "node_modules/extract-zip": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/extract-zip/-/extract-zip-2.0.1.tgz", + "integrity": "sha512-GDhU9ntwuKyGXdZBUgTIe+vXnWj0fppUEtMDL0+idd5Sta8TGpHssn/eusA9mrPr9qNDym6SxAYZjNvCn/9RBg==", + "dev": true, + "dependencies": { + "debug": "^4.1.1", + "get-stream": "^5.1.0", + "yauzl": "^2.10.0" + }, + "bin": { + "extract-zip": "cli.js" + }, + "engines": { + "node": ">= 10.17.0" + }, + "optionalDependencies": { + "@types/yauzl": "^2.9.1" + } + }, "node_modules/fast-deep-equal": { "version": "3.1.3", "dev": true, @@ -2195,6 +2696,12 @@ "dev": true, "license": "Apache-2.0" }, + "node_modules/fast-fifo": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/fast-fifo/-/fast-fifo-1.3.2.tgz", + "integrity": "sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==", + "dev": true + }, "node_modules/fast-glob": { "version": "3.3.2", "dev": true, @@ -2239,6 +2746,15 @@ "reusify": "^1.0.4" } }, + "node_modules/fd-slicer": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.1.0.tgz", + "integrity": "sha512-cE1qsB/VwyQozZ+q1dGxR8LBYNZeofhEdUNGSMbQD3Gw2lAzX9Zb3uIU6Ebc/Fmyjo9AWWfnn0AUCHqtevs/8g==", + "dev": true, + "dependencies": { + "pend": "~1.2.0" + } + }, "node_modules/file-entry-cache": { "version": "6.0.1", "dev": true, @@ -2365,6 +2881,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "dev": true, + "engines": { + "node": "6.* || 8.* || >= 10.*" + } + }, "node_modules/get-func-name": { "version": "2.0.2", "dev": true, @@ -2387,6 +2912,21 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/get-stream": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.2.0.tgz", + "integrity": "sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA==", + "dev": true, + "dependencies": { + "pump": "^3.0.0" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/get-symbol-description": { "version": "1.0.0", "dev": true, @@ -2413,6 +2953,35 @@ "url": "https://github.com/privatenumber/get-tsconfig?sponsor=1" } }, + "node_modules/get-uri": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/get-uri/-/get-uri-6.0.2.tgz", + "integrity": "sha512-5KLucCJobh8vBY1K07EFV4+cPZH3mrV9YeAruUseCQKHB58SGjjT2l9/eA9LD082IiuMjSlFJEcdJ27TXvbZNw==", + "dev": true, + "dependencies": { + "basic-ftp": "^5.0.2", + "data-uri-to-buffer": "^6.0.0", + "debug": "^4.3.4", + "fs-extra": "^8.1.0" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/get-uri/node_modules/fs-extra": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-8.1.0.tgz", + "integrity": "sha512-yhlQgA6mnOJUKOsRUFsgJdQCvkKhcz8tlZG5HBQfReYZy46OwLcY+Zia0mtdHsOo9y/hP+CxMN0TU9QxoOtG4g==", + "dev": true, + "dependencies": { + "graceful-fs": "^4.2.0", + "jsonfile": "^4.0.0", + "universalify": "^0.1.0" + }, + "engines": { + "node": ">=6 <7 || >=8" + } + }, "node_modules/glob": { "version": "7.2.3", "dev": true, @@ -2615,6 +3184,52 @@ "entities": "^4.4.0" } }, + "node_modules/http-proxy-agent": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.0.tgz", + "integrity": "sha512-+ZT+iBxVUQ1asugqnD6oWoRiS25AkjNfG085dKJGtGxkdwLQrMKU5wJr2bOOFAXzKcTuqq+7fZlTMgG3SRfIYQ==", + "dev": true, + "dependencies": { + "agent-base": "^7.1.0", + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/https-proxy-agent": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.2.tgz", + "integrity": "sha512-NmLNjm6ucYwtcUmL7JQC1ZQ57LmHP4lT15FQ8D61nak1rO6DH+fz5qNK2Ap5UN4ZapYICE3/0KodcLYSPsPbaA==", + "dev": true, + "dependencies": { + "agent-base": "^7.0.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, "node_modules/ignore": { "version": "5.3.0", "dev": true, @@ -2681,6 +3296,12 @@ "node": ">= 0.4" } }, + "node_modules/ip": { + "version": "1.1.8", + "resolved": "https://registry.npmjs.org/ip/-/ip-1.1.8.tgz", + "integrity": "sha512-PuExPYUiu6qMBQb4l06ecm6T6ujzhmh+MeJcW9wa89PoAz5pvd4zPgN5WJV104mb6S2T1AwNIAaB70JNrLQWhg==", + "dev": true + }, "node_modules/is-array-buffer": { "version": "3.0.2", "dev": true, @@ -2694,6 +3315,12 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/is-arrayish": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", + "integrity": "sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==", + "dev": true + }, "node_modules/is-bigint": { "version": "1.0.4", "dev": true, @@ -2790,6 +3417,15 @@ "node": ">=0.10.0" } }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "dev": true, + "engines": { + "node": ">=8" + } + }, "node_modules/is-glob": { "version": "4.0.3", "dev": true, @@ -3011,6 +3647,12 @@ "dev": true, "license": "MIT" }, + "node_modules/js-tokens": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", + "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", + "dev": true + }, "node_modules/js-yaml": { "version": "4.1.0", "dev": true, @@ -3027,6 +3669,12 @@ "dev": true, "license": "MIT" }, + "node_modules/json-parse-even-better-errors": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz", + "integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==", + "dev": true + }, "node_modules/json-schema-traverse": { "version": "0.4.1", "dev": true, @@ -3087,6 +3735,12 @@ "node": ">= 0.8.0" } }, + "node_modules/lines-and-columns": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz", + "integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==", + "dev": true + }, "node_modules/local-pkg": { "version": "0.4.3", "dev": true, @@ -3245,6 +3899,18 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/mitt": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/mitt/-/mitt-3.0.1.tgz", + "integrity": "sha512-vKivATfr97l2/QBCYAkXYDbrIWPM2IIKEl7YPhjCvKlG3kE2gm+uBo6nEXK3M5/Ffh/FLpKExzOQ3JJoJGFKBw==", + "dev": true + }, + "node_modules/mkdirp-classic": { + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", + "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==", + "dev": true + }, "node_modules/mlly": { "version": "1.4.2", "dev": true, @@ -3304,6 +3970,15 @@ "dev": true, "license": "MIT" }, + "node_modules/netmask": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/netmask/-/netmask-2.0.2.tgz", + "integrity": "sha512-dBpDMdxv9Irdq66304OLfEmQ9tbNRFnFTuZiLo+bD+r332bBmMJ8GBLXklIXXgxd3+v9+KUnZaUR5PJMa75Gsg==", + "dev": true, + "engines": { + "node": ">= 0.4.0" + } + }, "node_modules/node-fetch": { "version": "2.7.0", "dev": true, @@ -3496,6 +4171,39 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/pac-proxy-agent": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/pac-proxy-agent/-/pac-proxy-agent-7.0.1.tgz", + "integrity": "sha512-ASV8yU4LLKBAjqIPMbrgtaKIvxQri/yh2OpI+S6hVa9JRkUI3Y3NPFbfngDtY7oFtSMD3w31Xns89mDa3Feo5A==", + "dev": true, + "dependencies": { + "@tootallnate/quickjs-emscripten": "^0.23.0", + "agent-base": "^7.0.2", + "debug": "^4.3.4", + "get-uri": "^6.0.1", + "http-proxy-agent": "^7.0.0", + "https-proxy-agent": "^7.0.2", + "pac-resolver": "^7.0.0", + "socks-proxy-agent": "^8.0.2" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/pac-resolver": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/pac-resolver/-/pac-resolver-7.0.0.tgz", + "integrity": "sha512-Fd9lT9vJbHYRACT8OhCbZBbxr6KRSawSovFpy8nDGshaK99S/EBhVIHp9+crhxrsZOuvLpgL1n23iyPg6Rl2hg==", + "dev": true, + "dependencies": { + "degenerator": "^5.0.0", + "ip": "^1.1.8", + "netmask": "^2.0.2" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/parent-module": { "version": "1.0.1", "dev": true, @@ -3507,6 +4215,24 @@ "node": ">=6" } }, + "node_modules/parse-json": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-5.2.0.tgz", + "integrity": "sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg==", + "dev": true, + "dependencies": { + "@babel/code-frame": "^7.0.0", + "error-ex": "^1.3.1", + "json-parse-even-better-errors": "^2.3.0", + "lines-and-columns": "^1.1.6" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/parse5": { "version": "7.1.2", "license": "MIT", @@ -3583,6 +4309,12 @@ "node": "*" } }, + "node_modules/pend": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", + "integrity": "sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==", + "dev": true + }, "node_modules/picocolors": { "version": "1.0.0", "dev": true, @@ -3706,6 +4438,59 @@ "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, + "node_modules/progress": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz", + "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==", + "dev": true, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/proxy-agent": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/proxy-agent/-/proxy-agent-6.3.1.tgz", + "integrity": "sha512-Rb5RVBy1iyqOtNl15Cw/llpeLH8bsb37gM1FUfKQ+Wck6xHlbAhWGUFiTRHtkjqGTA5pSHz6+0hrPW/oECihPQ==", + "dev": true, + "dependencies": { + "agent-base": "^7.0.2", + "debug": "^4.3.4", + "http-proxy-agent": "^7.0.0", + "https-proxy-agent": "^7.0.2", + "lru-cache": "^7.14.1", + "pac-proxy-agent": "^7.0.1", + "proxy-from-env": "^1.1.0", + "socks-proxy-agent": "^8.0.2" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/proxy-agent/node_modules/lru-cache": { + "version": "7.18.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-7.18.3.tgz", + "integrity": "sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA==", + "dev": true, + "engines": { + "node": ">=12" + } + }, + "node_modules/proxy-from-env": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==", + "dev": true + }, + "node_modules/pump": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", + "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==", + "dev": true, + "dependencies": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, "node_modules/punycode": { "version": "2.3.1", "dev": true, @@ -3714,6 +4499,41 @@ "node": ">=6" } }, + "node_modules/puppeteer": { + "version": "21.6.1", + "resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-21.6.1.tgz", + "integrity": "sha512-O+pbc61oj8ln6m8EJKncrsQFmytgRyFYERtk190PeLbJn5JKpmmynn2p1PiFrlhCitAQXLJ0MOy7F0TeyCRqBg==", + "dev": true, + "hasInstallScript": true, + "dependencies": { + "@puppeteer/browsers": "1.9.0", + "cosmiconfig": "8.3.6", + "puppeteer-core": "21.6.1" + }, + "bin": { + "puppeteer": "lib/esm/puppeteer/node/cli.js" + }, + "engines": { + "node": ">=16.13.2" + } + }, + "node_modules/puppeteer-core": { + "version": "21.6.1", + "resolved": "https://registry.npmjs.org/puppeteer-core/-/puppeteer-core-21.6.1.tgz", + "integrity": "sha512-0chaaK/RL9S1U3bsyR4fUeUfoj51vNnjWvXgG6DcsyMjwYNpLcAThv187i1rZCo7QhJP0wZN8plQkjNyrq2h+A==", + "dev": true, + "dependencies": { + "@puppeteer/browsers": "1.9.0", + "chromium-bidi": "0.5.1", + "cross-fetch": "4.0.0", + "debug": "4.3.4", + "devtools-protocol": "0.0.1203626", + "ws": "8.15.1" + }, + "engines": { + "node": ">=16.13.2" + } + }, "node_modules/queue-lit": { "version": "1.5.2", "dev": true, @@ -3741,6 +4561,12 @@ ], "license": "MIT" }, + "node_modules/queue-tick": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/queue-tick/-/queue-tick-1.0.1.tgz", + "integrity": "sha512-kJt5qhMxoszgU/62PLP1CJytzd2NKetjSRnyuj31fDd3Rlcz3fzlFdFLD1SItunPwyqEOkca6GbV612BWfaBag==", + "dev": true + }, "node_modules/react-is": { "version": "18.2.0", "dev": true, @@ -3773,6 +4599,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/resolve": { "version": "1.22.8", "dev": true, @@ -3998,6 +4833,50 @@ "node": ">=8" } }, + "node_modules/smart-buffer": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz", + "integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==", + "dev": true, + "engines": { + "node": ">= 6.0.0", + "npm": ">= 3.0.0" + } + }, + "node_modules/socks": { + "version": "2.7.1", + "resolved": "https://registry.npmjs.org/socks/-/socks-2.7.1.tgz", + "integrity": "sha512-7maUZy1N7uo6+WVEX6psASxtNlKaNVMlGQKkG/63nEDdLOWNbiUMoLK7X4uYoLhQstau72mLgfEWcXcwsaHbYQ==", + "dev": true, + "dependencies": { + "ip": "^2.0.0", + "smart-buffer": "^4.2.0" + }, + "engines": { + "node": ">= 10.13.0", + "npm": ">= 3.0.0" + } + }, + "node_modules/socks-proxy-agent": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-8.0.2.tgz", + "integrity": "sha512-8zuqoLv1aP/66PHF5TqwJ7Czm3Yv32urJQHrVyhD7mmA6d61Zv8cIXQYPTWwmg6qlupnPvs/QKDmfa4P/qct2g==", + "dev": true, + "dependencies": { + "agent-base": "^7.0.2", + "debug": "^4.3.4", + "socks": "^2.7.1" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/socks/node_modules/ip": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ip/-/ip-2.0.0.tgz", + "integrity": "sha512-WKa+XuLG1A1R0UWhl2+1XQSi+fZWMsYKffMZTTYsiZaUD8k2yDAj5atimTUD2TZkyCkNEeYE5NhFZmupOGtjYQ==", + "dev": true + }, "node_modules/source-map": { "version": "0.6.1", "dev": true, @@ -4122,6 +5001,16 @@ "dev": true, "license": "MIT" }, + "node_modules/streamx": { + "version": "2.15.6", + "resolved": "https://registry.npmjs.org/streamx/-/streamx-2.15.6.tgz", + "integrity": "sha512-q+vQL4AAz+FdfT137VF69Cc/APqUbxy+MDOImRrMvchJpigHj9GksgDU2LYbO9rx7RX6osWgxJB2WxhYv4SZAw==", + "dev": true, + "dependencies": { + "fast-fifo": "^1.1.0", + "queue-tick": "^1.0.1" + } + }, "node_modules/string-argv": { "version": "0.3.2", "dev": true, @@ -4130,6 +5019,20 @@ "node": ">=0.6.19" } }, + "node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/string.prototype.trim": { "version": "1.2.8", "dev": true, @@ -4243,6 +5146,28 @@ "node": ">=6" } }, + "node_modules/tar-fs": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-3.0.4.tgz", + "integrity": "sha512-5AFQU8b9qLfZCX9zp2duONhPmZv0hGYiBPJsyUdqMjzq/mqVpy/rEUSeHk1+YitmxugaptgBh5oDGU3VsAJq4w==", + "dev": true, + "dependencies": { + "mkdirp-classic": "^0.5.2", + "pump": "^3.0.0", + "tar-stream": "^3.1.5" + } + }, + "node_modules/tar-stream": { + "version": "3.1.6", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-3.1.6.tgz", + "integrity": "sha512-B/UyjYwPpMBv+PaFSWAmtYjwdrlEaZQEhMIBFNC5oEG8lpiW8XjcSdmEaClj28ArfKScKHs2nshz3k2le6crsg==", + "dev": true, + "dependencies": { + "b4a": "^1.6.4", + "fast-fifo": "^1.2.0", + "streamx": "^2.15.0" + } + }, "node_modules/test-exclude": { "version": "6.0.0", "dev": true, @@ -4261,6 +5186,12 @@ "dev": true, "license": "MIT" }, + "node_modules/through": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", + "integrity": "sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==", + "dev": true + }, "node_modules/tinybench": { "version": "2.5.1", "dev": true, @@ -4518,6 +5449,16 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/unbzip2-stream": { + "version": "1.4.3", + "resolved": "https://registry.npmjs.org/unbzip2-stream/-/unbzip2-stream-1.4.3.tgz", + "integrity": "sha512-mlExGW4w71ebDJviH16lQLtZS32VKqsSfk80GCfUlwT/4/hNRFsoscrF/c++9xinkMzECL1uL9DDwXqFWkruPg==", + "dev": true, + "dependencies": { + "buffer": "^5.2.1", + "through": "^2.3.8" + } + }, "node_modules/undici-types": { "version": "5.26.5", "dev": true, @@ -4543,6 +5484,12 @@ "punycode": "^2.1.0" } }, + "node_modules/urlpattern-polyfill": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/urlpattern-polyfill/-/urlpattern-polyfill-9.0.0.tgz", + "integrity": "sha512-WHN8KDQblxd32odxeIgo83rdVDE2bvdkb86it7bMhYZwWKJz0+O0RK/eZiHYnM+zgt/U7hAHOlCQGfjjvSkw2g==", + "dev": true + }, "node_modules/v8-compile-cache-lib": { "version": "3.0.1", "dev": true, @@ -4887,16 +5834,100 @@ "node": ">=8" } }, + "node_modules/wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "dev": true, + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, "node_modules/wrappy": { "version": "1.0.2", "dev": true, "license": "ISC" }, + "node_modules/ws": { + "version": "8.15.1", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.15.1.tgz", + "integrity": "sha512-W5OZiCjXEmk0yZ66ZN82beM5Sz7l7coYxpRkzS+p9PP+ToQry8szKh+61eNktr7EA9DOwvFGhfC605jDHbP6QQ==", + "dev": true, + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "node_modules/y18n": { + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", + "dev": true, + "engines": { + "node": ">=10" + } + }, "node_modules/yallist": { "version": "4.0.0", "dev": true, "license": "ISC" }, + "node_modules/yargs": { + "version": "17.7.2", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", + "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", + "dev": true, + "dependencies": { + "cliui": "^8.0.1", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.3", + "y18n": "^5.0.5", + "yargs-parser": "^21.1.1" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/yargs-parser": { + "version": "21.1.1", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", + "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", + "dev": true, + "engines": { + "node": ">=12" + } + }, + "node_modules/yauzl": { + "version": "2.10.0", + "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.10.0.tgz", + "integrity": "sha512-p4a9I6X6nu6IhoGmBqAcbJy1mlC4j27vEPZX9F4L4/vZT3Lyq1VkFHw/V/PUcB9Buo+DG3iHkT0x3Qya58zc3g==", + "dev": true, + "dependencies": { + "buffer-crc32": "~0.2.3", + "fd-slicer": "~1.1.0" + } + }, "node_modules/yn": { "version": "3.1.1", "dev": true, diff --git a/package.json b/package.json index 17915a7..cbf5a5f 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@movie-web/providers", - "version": "1.1.5", + "version": "2.0.0", "description": "Package that contains all the providers of movie-web", "main": "./lib/index.umd.js", "types": "./lib/index.d.ts", @@ -34,11 +34,11 @@ }, "homepage": "https://providers.docs.movie-web.app/", "scripts": { - "build": "vite build", + "build": "vite build && tsc --noEmit", + "cli": "ts-node ./src/dev-cli/index.ts", "test": "vitest run", - "test:dev": "ts-node ./src/dev-cli.ts", "test:watch": "vitest", - "test:integration": "node ./tests/cjs && node ./tests/esm", + "test:integration": "node ./tests/cjs && node ./tests/esm && node ./tests/browser", "test:coverage": "vitest run --coverage", "lint": "eslint --ext .ts,.js src/", "lint:fix": "eslint --fix --ext .ts,.js src/", @@ -65,6 +65,7 @@ "eslint-plugin-prettier": "^4.2.1", "node-fetch": "^2.7.0", "prettier": "^2.6.2", + "puppeteer": "^21.6.1", "spinnies": "^0.5.1", "ts-node": "^10.9.1", "tsc-alias": "^1.6.7", diff --git a/src/__test__/fetchers/simpleProxy.test.ts b/src/__test__/fetchers/simpleProxy.test.ts index 5066e63..fc1137a 100644 --- a/src/__test__/fetchers/simpleProxy.test.ts +++ b/src/__test__/fetchers/simpleProxy.test.ts @@ -16,6 +16,8 @@ describe("makeSimpleProxyFetcher()", () => { headers: new Headers({ "content-type": "text/plain", }), + status: 204, + url: "test123", text() { return Promise.resolve(value); }, @@ -24,6 +26,8 @@ describe("makeSimpleProxyFetcher()", () => { headers: new Headers({ "content-type": "application/json", }), + status: 204, + url: "test123", json() { return Promise.resolve(value); }, @@ -31,7 +35,11 @@ describe("makeSimpleProxyFetcher()", () => { } function expectFetchCall(ops: { inputUrl: string, input: DefaultedFetcherOptions, outputUrl?: string, output: any, outputBody: any }) { - expect(fetcher(ops.inputUrl, ops.input)).resolves.toEqual(ops.outputBody); + const prom = fetcher(ops.inputUrl, ops.input); + expect((async () => (await prom).body)()).resolves.toEqual(ops.outputBody); + expect((async () => (await prom).headers.entries())()).resolves.toEqual((new Headers()).entries()); + expect((async () => (await prom).statusCode)()).resolves.toEqual(204); + expect((async () => (await prom).finalUrl)()).resolves.toEqual("test123"); expect(fetch).toBeCalledWith(ops.outputUrl ?? ops.inputUrl, ops.output); vi.clearAllMocks(); } @@ -43,6 +51,7 @@ describe("makeSimpleProxyFetcher()", () => { input: { method: "GET", query: {}, + readHeaders: [], headers: { "X-Hello": "world", }, @@ -62,6 +71,7 @@ describe("makeSimpleProxyFetcher()", () => { input: { method: "GET", headers: {}, + readHeaders: [], query: { "a": 'b', } @@ -79,6 +89,7 @@ describe("makeSimpleProxyFetcher()", () => { input: { method: "GET", query: {}, + readHeaders: [], headers: {}, }, outputUrl: `https://example.com/proxy?destination=${encodeURIComponent('https://google.com/')}`, @@ -97,6 +108,7 @@ describe("makeSimpleProxyFetcher()", () => { input: { method: "POST", query: {}, + readHeaders: [], headers: {}, }, outputUrl: `https://example.com/proxy?destination=${encodeURIComponent('https://google.com/')}`, @@ -112,6 +124,7 @@ describe("makeSimpleProxyFetcher()", () => { input: { method: "POST", query: {}, + readHeaders: [], headers: {}, }, outputUrl: `https://example.com/proxy?destination=${encodeURIComponent('https://google.com/')}`, diff --git a/src/__test__/fetchers/standard.test.ts b/src/__test__/fetchers/standard.test.ts index 6bad99a..6753500 100644 --- a/src/__test__/fetchers/standard.test.ts +++ b/src/__test__/fetchers/standard.test.ts @@ -16,6 +16,8 @@ describe("makeStandardFetcher()", () => { headers: new Headers({ "content-type": "text/plain", }), + status: 204, + url: "test123", text() { return Promise.resolve(value); }, @@ -24,6 +26,8 @@ describe("makeStandardFetcher()", () => { headers: new Headers({ "content-type": "application/json", }), + status: 204, + url: "test123", json() { return Promise.resolve(value); }, @@ -31,7 +35,11 @@ describe("makeStandardFetcher()", () => { } function expectFetchCall(ops: { inputUrl: string, input: DefaultedFetcherOptions, outputUrl?: string, output: any, outputBody: any }) { - expect(fetcher(ops.inputUrl, ops.input)).resolves.toEqual(ops.outputBody); + const prom = fetcher(ops.inputUrl, ops.input); + expect((async () => (await prom).body)()).resolves.toEqual(ops.outputBody); + expect((async () => (await prom).headers.entries())()).resolves.toEqual((new Headers()).entries()); + expect((async () => (await prom).statusCode)()).resolves.toEqual(204); + expect((async () => (await prom).finalUrl)()).resolves.toEqual("test123"); expect(fetch).toBeCalledWith(ops.outputUrl ?? ops.inputUrl, ops.output); vi.clearAllMocks(); } @@ -43,6 +51,7 @@ describe("makeStandardFetcher()", () => { input: { method: "GET", query: {}, + readHeaders: [], headers: { "X-Hello": "world", }, @@ -53,6 +62,7 @@ describe("makeStandardFetcher()", () => { headers: { "X-Hello": "world", }, + body: undefined, }, outputBody: "hello world" }) @@ -62,6 +72,7 @@ describe("makeStandardFetcher()", () => { input: { method: "GET", headers: {}, + readHeaders: [], query: { "a": 'b', } @@ -79,6 +90,7 @@ describe("makeStandardFetcher()", () => { input: { query: {}, headers: {}, + readHeaders: [], method: "GET" }, outputUrl: "https://google.com/", @@ -97,6 +109,7 @@ describe("makeStandardFetcher()", () => { input: { query: {}, headers: {}, + readHeaders: [], method: "POST" }, outputUrl: "https://google.com/", @@ -112,6 +125,7 @@ describe("makeStandardFetcher()", () => { input: { query: {}, headers: {}, + readHeaders: [], method: "POST" }, outputUrl: "https://google.com/", diff --git a/src/__test__/providerTests.ts b/src/__test__/providerTests.ts index a8c0fa8..551a3ec 100644 --- a/src/__test__/providerTests.ts +++ b/src/__test__/providerTests.ts @@ -15,40 +15,52 @@ export function makeProviderMocks() { const sourceA = { id: 'a', + name: 'A', rank: 1, disabled: false, + flags: [], } as Sourcerer; const sourceB = { id: 'b', + name: 'B', rank: 2, disabled: false, + flags: [], } as Sourcerer; const sourceCDisabled = { id: 'c', + name: 'C', rank: 3, disabled: true, + flags: [], } as Sourcerer; const sourceAHigherRank = { id: 'a', + name: 'A', rank: 100, disabled: false, + flags: [], } as Sourcerer; const sourceGSameRankAsA = { id: 'g', + name: 'G', rank: 1, disabled: false, + flags: [], } as Sourcerer; const fullSourceYMovie = { id: 'y', name: 'Y', rank: 105, scrapeMovie: vi.fn(), + flags: [], } as Sourcerer; const fullSourceYShow = { id: 'y', name: 'Y', rank: 105, scrapeShow: vi.fn(), + flags: [], } as Sourcerer; const fullSourceZBoth = { id: 'z', @@ -56,6 +68,7 @@ const fullSourceZBoth = { rank: 106, scrapeMovie: vi.fn(), scrapeShow: vi.fn(), + flags: [], } as Sourcerer; const embedD = { diff --git a/src/__test__/providers/checks.test.ts b/src/__test__/providers/checks.test.ts index dca4e73..404fb31 100644 --- a/src/__test__/providers/checks.test.ts +++ b/src/__test__/providers/checks.test.ts @@ -1,12 +1,15 @@ import { mockEmbeds, mockSources } from '@/__test__/providerTests'; +import { getBuiltinEmbeds, getBuiltinSources } from '@/entrypoint/providers'; +import { FeatureMap } from '@/entrypoint/utils/targets'; import { getProviders } from '@/providers/get'; import { vi, describe, it, expect, afterEach } from 'vitest'; -const mocks = await vi.hoisted(async () => (await import('../providerTests.ts')).makeProviderMocks()); +const mocks = await vi.hoisted(async () => (await import('../providerTests')).makeProviderMocks()); vi.mock('@/providers/all', () => mocks); -const features = { +const features: FeatureMap = { requires: [], + disallowed: [] } describe('getProviders()', () => { @@ -17,7 +20,10 @@ describe('getProviders()', () => { it('should return providers', () => { mocks.gatherAllEmbeds.mockReturnValue([mockEmbeds.embedD]); mocks.gatherAllSources.mockReturnValue([mockSources.sourceA, mockSources.sourceB]); - expect(getProviders(features)).toEqual({ + expect(getProviders(features, { + embeds: getBuiltinEmbeds(), + sources: getBuiltinSources(), + })).toEqual({ sources: [mockSources.sourceA, mockSources.sourceB], embeds: [mockEmbeds.embedD], }); @@ -26,7 +32,10 @@ describe('getProviders()', () => { it('should filter out disabled providers', () => { mocks.gatherAllEmbeds.mockReturnValue([mockEmbeds.embedD, mockEmbeds.embedEDisabled]); mocks.gatherAllSources.mockReturnValue([mockSources.sourceA, mockSources.sourceCDisabled, mockSources.sourceB]); - expect(getProviders(features)).toEqual({ + expect(getProviders(features,{ + embeds: getBuiltinEmbeds(), + sources: getBuiltinSources(), + })).toEqual({ sources: [mockSources.sourceA, mockSources.sourceB], embeds: [mockEmbeds.embedD], }); @@ -35,31 +44,46 @@ describe('getProviders()', () => { it('should throw on duplicate ids in sources', () => { mocks.gatherAllEmbeds.mockReturnValue([]); mocks.gatherAllSources.mockReturnValue([mockSources.sourceAHigherRank, mockSources.sourceA, mockSources.sourceB]); - expect(() => getProviders(features)).toThrowError(); + expect(() => getProviders(features,{ + embeds: getBuiltinEmbeds(), + sources: getBuiltinSources(), + })).toThrowError(); }); it('should throw on duplicate ids in embeds', () => { mocks.gatherAllEmbeds.mockReturnValue([mockEmbeds.embedD, mockEmbeds.embedDHigherRank, mockEmbeds.embedA]); mocks.gatherAllSources.mockReturnValue([]); - expect(() => getProviders(features)).toThrowError(); + expect(() => getProviders(features,{ + embeds: getBuiltinEmbeds(), + sources: getBuiltinSources(), + })).toThrowError(); }); it('should throw on duplicate ids between sources and embeds', () => { mocks.gatherAllEmbeds.mockReturnValue([mockEmbeds.embedD, mockEmbeds.embedA]); mocks.gatherAllSources.mockReturnValue([mockSources.sourceA, mockSources.sourceB]); - expect(() => getProviders(features)).toThrowError(); + expect(() => getProviders(features,{ + embeds: getBuiltinEmbeds(), + sources: getBuiltinSources(), + })).toThrowError(); }); it('should throw on duplicate rank between sources and embeds', () => { mocks.gatherAllEmbeds.mockReturnValue([mockEmbeds.embedD, mockEmbeds.embedA]); mocks.gatherAllSources.mockReturnValue([mockSources.sourceA, mockSources.sourceB]); - expect(() => getProviders(features)).toThrowError(); + expect(() => getProviders(features,{ + embeds: getBuiltinEmbeds(), + sources: getBuiltinSources(), + })).toThrowError(); }); it('should not throw with same rank between sources and embeds', () => { mocks.gatherAllEmbeds.mockReturnValue([mockEmbeds.embedD, mockEmbeds.embedHSameRankAsSourceA]); mocks.gatherAllSources.mockReturnValue([mockSources.sourceA, mockSources.sourceB]); - expect(getProviders(features)).toEqual({ + expect(getProviders(features,{ + embeds: getBuiltinEmbeds(), + sources: getBuiltinSources(), + })).toEqual({ sources: [mockSources.sourceA, mockSources.sourceB], embeds: [mockEmbeds.embedD, mockEmbeds.embedHSameRankAsSourceA], }); diff --git a/src/__test__/runner/list.test.ts b/src/__test__/runner/list.test.ts index 483485e..336de44 100644 --- a/src/__test__/runner/list.test.ts +++ b/src/__test__/runner/list.test.ts @@ -1,6 +1,6 @@ import { mockEmbeds, mockSources } from '@/__test__/providerTests'; -import { makeProviders } from '@/main/builder'; -import { targets } from '@/main/targets.ts'; +import { makeProviders } from '@/entrypoint/declare'; +import { targets } from '@/entrypoint/utils/targets'; import { afterEach, describe, expect, it, vi } from 'vitest'; const mocks = await vi.hoisted(async () => (await import('../providerTests.ts')).makeProviderMocks()); diff --git a/src/__test__/runner/meta.test.ts b/src/__test__/runner/meta.test.ts index 4b0e55e..423a8e6 100644 --- a/src/__test__/runner/meta.test.ts +++ b/src/__test__/runner/meta.test.ts @@ -1,6 +1,6 @@ import { mockEmbeds, mockSources } from '@/__test__/providerTests'; -import { makeProviders } from '@/main/builder'; -import { targets } from '@/main/targets.ts'; +import { makeProviders } from '@/entrypoint/declare'; +import { targets } from '@/entrypoint/utils/targets'; import { afterEach, describe, expect, it, vi } from 'vitest'; const mocks = await vi.hoisted(async () => (await import('../providerTests.ts')).makeProviderMocks()); diff --git a/src/__test__/utils/features.test.ts b/src/__test__/utils/features.test.ts new file mode 100644 index 0000000..0da4019 --- /dev/null +++ b/src/__test__/utils/features.test.ts @@ -0,0 +1,77 @@ +import { FeatureMap, Flags, flags, flagsAllowedInFeatures } from "@/entrypoint/utils/targets"; +import { describe, it, expect } from "vitest"; + +describe('flagsAllowedInFeatures()', () => { + function checkFeatures(featureMap: FeatureMap, flags: Flags[], output: boolean) { + expect(flagsAllowedInFeatures(featureMap, flags)).toEqual(output); + } + + it('should check required correctly', () => { + checkFeatures({ + requires: [], + disallowed: [] + }, [], true); + checkFeatures({ + requires: [flags.CORS_ALLOWED], + disallowed: [] + }, [flags.CORS_ALLOWED], true); + checkFeatures({ + requires: [flags.CORS_ALLOWED], + disallowed: [] + }, [], false); + checkFeatures({ + requires: [flags.CORS_ALLOWED, flags.IP_LOCKED], + disallowed: [] + }, [flags.CORS_ALLOWED, flags.IP_LOCKED], true); + checkFeatures({ + requires: [flags.IP_LOCKED], + disallowed: [] + }, [flags.CORS_ALLOWED], false); + checkFeatures({ + requires: [flags.IP_LOCKED], + disallowed: [] + }, [], false); + }); + + it('should check disallowed correctly', () => { + checkFeatures({ + requires: [], + disallowed: [] + }, [], true); + checkFeatures({ + requires: [], + disallowed: [flags.CORS_ALLOWED] + }, [], true); + checkFeatures({ + requires: [], + disallowed: [flags.CORS_ALLOWED] + }, [flags.CORS_ALLOWED], false); + checkFeatures({ + requires: [], + disallowed: [flags.CORS_ALLOWED] + }, [flags.IP_LOCKED], true); + checkFeatures({ + requires: [], + disallowed: [flags.CORS_ALLOWED, flags.IP_LOCKED] + }, [flags.CORS_ALLOWED], false); + }); + + it('should pass mixed tests', () => { + checkFeatures({ + requires: [flags.CORS_ALLOWED], + disallowed: [flags.IP_LOCKED] + }, [], false); + checkFeatures({ + requires: [flags.CORS_ALLOWED], + disallowed: [flags.IP_LOCKED] + }, [flags.CORS_ALLOWED], true); + checkFeatures({ + requires: [flags.CORS_ALLOWED], + disallowed: [flags.IP_LOCKED] + }, [flags.IP_LOCKED], false); + checkFeatures({ + requires: [flags.CORS_ALLOWED], + disallowed: [flags.IP_LOCKED] + }, [flags.IP_LOCKED, flags.CORS_ALLOWED], false); + }); +}); diff --git a/src/__test__/utils/valid.test.ts b/src/__test__/utils/valid.test.ts index 540fbf3..eab835d 100644 --- a/src/__test__/utils/valid.test.ts +++ b/src/__test__/utils/valid.test.ts @@ -9,7 +9,9 @@ describe('isValidStream()', () => { it('should pass valid streams', () => { expect(isValidStream({ type: "file", + id: "a", flags: [], + captions: [], qualities: { "1080": { type: "mp4", @@ -19,7 +21,9 @@ describe('isValidStream()', () => { })).toBe(true); expect(isValidStream({ type: "hls", + id: "a", flags: [], + captions: [], playlist: "hello-world" })).toBe(true); }); @@ -27,7 +31,9 @@ describe('isValidStream()', () => { it('should detect empty qualities', () => { expect(isValidStream({ type: "file", + id: "a", flags: [], + captions: [], qualities: {} })).toBe(false); }); @@ -35,7 +41,9 @@ describe('isValidStream()', () => { it('should detect empty stream urls', () => { expect(isValidStream({ type: "file", + id: "a", flags: [], + captions: [], qualities: { "1080": { type: "mp4", @@ -48,7 +56,9 @@ describe('isValidStream()', () => { it('should detect emtpy HLS playlists', () => { expect(isValidStream({ type: "hls", + id: "a", flags: [], + captions: [], playlist: "", })).toBe(false); }); diff --git a/src/dev-cli.ts b/src/dev-cli.ts deleted file mode 100644 index 54d663a..0000000 --- a/src/dev-cli.ts +++ /dev/null @@ -1,423 +0,0 @@ -/* eslint import/no-extraneous-dependencies: ["error", {"devDependencies": true}] */ - -import util from 'node:util'; - -import { program } from 'commander'; -import dotenv from 'dotenv'; -import { prompt } from 'enquirer'; -import nodeFetch from 'node-fetch'; -import Spinnies from 'spinnies'; - -import { MetaOutput, MovieMedia, ProviderControls, ShowMedia, makeProviders, makeStandardFetcher, targets } from '.'; - -dotenv.config(); - -type ProviderSourceAnswers = { - id: string; - type: string; -}; - -type EmbedSourceAnswers = { - url: string; -}; - -type CommonAnswers = { - fetcher: string; - source: string; -}; - -type ShowAnswers = { - season: string; - episode: string; -}; - -type CommandLineArguments = { - fetcher: string; - sourceId: string; - tmdbId: string; - type: string; - season: string; - episode: string; - url: string; -}; - -const TMDB_API_KEY = process.env.MOVIE_WEB_TMDB_API_KEY ?? ''; - -if (!TMDB_API_KEY?.trim()) { - throw new Error('Missing MOVIE_WEB_TMDB_API_KEY environment variable'); -} - -function logDeepObject(object: Record) { - console.log(util.inspect(object, { showHidden: false, depth: null, colors: true })); -} - -function getAllSources() { - // * The only way to get a list of all sources is to - // * create all these things. Maybe this should change - const providers = makeProviders({ - fetcher: makeStandardFetcher(nodeFetch), - target: targets.NATIVE, - }); - - const combined = [...providers.listSources(), ...providers.listEmbeds()]; - - // * Remove dupes - const map = new Map(combined.map((source) => [source.id, source])); - - return [...map.values()]; -} - -// * Defined here cuz ESLint didn't like the order these were defined in -const sources = getAllSources(); - -async function makeTMDBRequest(url: string): Promise { - const headers: { - accept: 'application/json'; - authorization?: string; - } = { - accept: 'application/json', - }; - - // * Used to get around ESLint - // * Assignment to function parameter 'url'. eslint (no-param-reassign) - let requestURL = url; - - // * JWT keys always start with ey and are ONLY valid as a header. - // * All other keys are ONLY valid as a query param. - // * Thanks TMDB. - if (TMDB_API_KEY.startsWith('ey')) { - headers.authorization = `Bearer ${TMDB_API_KEY}`; - } else { - requestURL += `?api_key=${TMDB_API_KEY}`; - } - - return fetch(requestURL, { - method: 'GET', - headers, - }); -} - -async function getMovieMediaDetails(id: string): Promise { - const response = await makeTMDBRequest(`https://api.themoviedb.org/3/movie/${id}`); - const movie = await response.json(); - - if (movie.success === false) { - throw new Error(movie.status_message); - } - - if (!movie.release_date) { - throw new Error(`${movie.title} has no release_date. Assuming unreleased`); - } - - return { - type: 'movie', - title: movie.title, - releaseYear: Number(movie.release_date.split('-')[0]), - tmdbId: id, - }; -} - -async function getShowMediaDetails(id: string, seasonNumber: string, episodeNumber: string): Promise { - // * TV shows require the TMDB ID for the series, season, and episode - // * and the name of the series. Needs multiple requests - let response = await makeTMDBRequest(`https://api.themoviedb.org/3/tv/${id}`); - const series = await response.json(); - - if (series.success === false) { - throw new Error(series.status_message); - } - - if (!series.first_air_date) { - throw new Error(`${series.name} has no first_air_date. Assuming unaired`); - } - - response = await makeTMDBRequest(`https://api.themoviedb.org/3/tv/${id}/season/${seasonNumber}`); - const season = await response.json(); - - if (season.success === false) { - throw new Error(season.status_message); - } - - response = await makeTMDBRequest( - `https://api.themoviedb.org/3/tv/${id}/season/${seasonNumber}/episode/${episodeNumber}`, - ); - const episode = await response.json(); - - if (episode.success === false) { - throw new Error(episode.status_message); - } - - return { - type: 'show', - title: series.name, - releaseYear: Number(series.first_air_date.split('-')[0]), - tmdbId: id, - episode: { - number: episode.episode_number, - tmdbId: episode.id, - }, - season: { - number: season.season_number, - tmdbId: season.id, - }, - }; -} - -function joinMediaTypes(mediaTypes: string[] | undefined) { - if (mediaTypes) { - const formatted = mediaTypes - .map((type: string) => { - return `${type[0].toUpperCase() + type.substring(1).toLowerCase()}s`; - }) - .join(' / '); - - return `(${formatted})`; - } - return ''; // * Embed sources pass through here too -} - -async function runScraper(providers: ProviderControls, source: MetaOutput, options: CommandLineArguments) { - const spinnies = new Spinnies(); - - if (source.type === 'embed') { - spinnies.add('scrape', { text: `Running ${source.name} scraper on ${options.url}` }); - try { - const result = await providers.runEmbedScraper({ - url: options.url, - id: source.id, - }); - spinnies.succeed('scrape', { text: 'Done!' }); - logDeepObject(result); - } catch (error) { - let message = 'Unknown error'; - if (error instanceof Error) { - message = error.message; - } - - spinnies.fail('scrape', { text: `ERROR: ${message}` }); - } - } else { - let media; - - if (options.type === 'movie') { - media = await getMovieMediaDetails(options.tmdbId); - } else { - media = await getShowMediaDetails(options.tmdbId, options.season, options.episode); - } - - spinnies.add('scrape', { text: `Running ${source.name} scraper on ${media.title}` }); - try { - const result = await providers.runSourceScraper({ - media, - id: source.id, - }); - spinnies.succeed('scrape', { text: 'Done!' }); - logDeepObject(result); - } catch (error) { - let message = 'Unknown error'; - if (error instanceof Error) { - message = error.message; - } - - spinnies.fail('scrape', { text: `ERROR: ${message}` }); - } - } -} - -async function processOptions(options: CommandLineArguments) { - if (options.fetcher !== 'node-fetch' && options.fetcher !== 'native') { - throw new Error("Fetcher must be either 'native' or 'node-fetch'"); - } - - if (!options.sourceId.trim()) { - throw new Error('Source ID must be provided'); - } - - const source = sources.find(({ id }) => id === options.sourceId); - - if (!source) { - throw new Error('Invalid source ID. No source found'); - } - - if (source.type === 'embed' && !options.url.trim()) { - throw new Error('Must provide an embed URL for embed sources'); - } - - if (source.type === 'source') { - if (!options.tmdbId.trim()) { - throw new Error('Must provide a TMDB ID for provider sources'); - } - - if (Number.isNaN(Number(options.tmdbId)) || Number(options.tmdbId) < 0) { - throw new Error('TMDB ID must be a number greater than 0'); - } - - if (!options.type.trim()) { - throw new Error('Must provide a type for provider sources'); - } - - if (options.type !== 'movie' && options.type !== 'show') { - throw new Error("Invalid media type. Must be either 'movie' or 'show'"); - } - - if (options.type === 'show') { - if (!options.season.trim()) { - throw new Error('Must provide a season number for TV shows'); - } - - if (!options.episode.trim()) { - throw new Error('Must provide an episode number for TV shows'); - } - - if (Number.isNaN(Number(options.season)) || Number(options.season) <= 0) { - throw new Error('Season number must be a number greater than 0'); - } - - if (Number.isNaN(Number(options.episode)) || Number(options.episode) <= 0) { - throw new Error('Episode number must be a number greater than 0'); - } - } - } - - let fetcher; - - if (options.fetcher === 'native') { - fetcher = makeStandardFetcher(fetch); - } else { - fetcher = makeStandardFetcher(nodeFetch); - } - - const providers = makeProviders({ - fetcher, - target: targets.NATIVE, - }); - - await runScraper(providers, source, options); -} - -async function runQuestions() { - const options = { - fetcher: 'node-fetch', - sourceId: '', - tmdbId: '', - type: 'movie', - season: '0', - episode: '0', - url: '', - }; - - const answers = await prompt([ - { - type: 'select', - name: 'fetcher', - message: 'Select a fetcher', - choices: [ - { - message: 'Native', - name: 'native', - }, - { - message: 'Node fetch', - name: 'node-fetch', - }, - ], - }, - { - type: 'select', - name: 'source', - message: 'Select a source', - choices: sources.map((source) => ({ - message: `[${source.type.toLocaleUpperCase()}] ${source.name} ${joinMediaTypes(source.mediaTypes)}`.trim(), - name: source.id, - })), - }, - ]); - - options.fetcher = answers.fetcher; - options.sourceId = answers.source; - - const source = sources.find(({ id }) => id === answers.source); - - if (!source) { - throw new Error(`No source with ID ${answers.source} found`); - } - - if (source.type === 'embed') { - const sourceAnswers = await prompt([ - { - type: 'input', - name: 'url', - message: 'Embed URL', - }, - ]); - - options.url = sourceAnswers.url; - } else { - const sourceAnswers = await prompt([ - { - type: 'input', - name: 'id', - message: 'TMDB ID', - }, - { - type: 'select', - name: 'type', - message: 'Media type', - choices: [ - { - message: 'Movie', - name: 'movie', - }, - { - message: 'TV Show', - name: 'show', - }, - ], - }, - ]); - - options.tmdbId = sourceAnswers.id; - options.type = sourceAnswers.type; - - if (sourceAnswers.type === 'show') { - const seriesAnswers = await prompt([ - { - type: 'input', - name: 'season', - message: 'Season', - }, - { - type: 'input', - name: 'episode', - message: 'Episode', - }, - ]); - - options.season = seriesAnswers.season; - options.episode = seriesAnswers.episode; - } - } - - await processOptions(options); -} - -async function runCommandLine() { - program - .option('-f, --fetcher ', "Fetcher to use. Either 'native' or 'node-fetch'", 'node-fetch') - .option('-sid, --source-id ', 'ID for the source to use. Either an embed or provider', '') - .option('-tid, --tmdb-id ', 'TMDB ID for the media to scrape. Only used if source is a provider', '') - .option('-t, --type ', "Media type. Either 'movie' or 'show'. Only used if source is a provider", 'movie') - .option('-s, --season ', "Season number. Only used if type is 'show'", '0') - .option('-e, --episode ', "Episode number. Only used if type is 'show'", '0') - .option('-u, --url ', 'URL to a video embed. Only used if source is an embed', ''); - - program.parse(); - - await processOptions(program.opts()); -} - -if (process.argv.length === 2) { - runQuestions(); -} else { - runCommandLine(); -} diff --git a/src/dev-cli/browser/.gitignore b/src/dev-cli/browser/.gitignore new file mode 100644 index 0000000..1521c8b --- /dev/null +++ b/src/dev-cli/browser/.gitignore @@ -0,0 +1 @@ +dist diff --git a/src/dev-cli/browser/index.html b/src/dev-cli/browser/index.html new file mode 100644 index 0000000..7709f4b --- /dev/null +++ b/src/dev-cli/browser/index.html @@ -0,0 +1,11 @@ + + + + + + Scraper CLI + + + + + diff --git a/src/dev-cli/browser/index.ts b/src/dev-cli/browser/index.ts new file mode 100644 index 0000000..d1f6494 --- /dev/null +++ b/src/dev-cli/browser/index.ts @@ -0,0 +1,17 @@ +import { makeProviders, makeSimpleProxyFetcher, makeStandardFetcher, targets } from '../../../lib'; + +(window as any).scrape = (proxyUrl: string, type: 'source' | 'embed', input: any) => { + const providers = makeProviders({ + fetcher: makeStandardFetcher(fetch), + target: targets.BROWSER, + proxiedFetcher: makeSimpleProxyFetcher(proxyUrl, fetch), + }); + if (type === 'source') { + return providers.runSourceScraper(input); + } + if (type === 'embed') { + return providers.runEmbedScraper(input); + } + + throw new Error('Input input type'); +}; diff --git a/src/dev-cli/config.ts b/src/dev-cli/config.ts new file mode 100644 index 0000000..38f8039 --- /dev/null +++ b/src/dev-cli/config.ts @@ -0,0 +1,16 @@ +export function getConfig() { + let tmdbApiKey = process.env.MOVIE_WEB_TMDB_API_KEY ?? ''; + tmdbApiKey = tmdbApiKey.trim(); + + if (!tmdbApiKey) { + throw new Error('Missing MOVIE_WEB_TMDB_API_KEY environment variable'); + } + + let proxyUrl: undefined | string = process.env.MOVIE_WEB_PROXY_URL; + proxyUrl = !proxyUrl ? undefined : proxyUrl; + + return { + tmdbApiKey, + proxyUrl, + }; +} diff --git a/src/dev-cli/index.ts b/src/dev-cli/index.ts new file mode 100644 index 0000000..bd95599 --- /dev/null +++ b/src/dev-cli/index.ts @@ -0,0 +1,185 @@ +/* eslint import/no-extraneous-dependencies: ["error", {"devDependencies": true}] */ + +import { program } from 'commander'; +import dotenv from 'dotenv'; +import { prompt } from 'enquirer'; + +import { runScraper } from '@/dev-cli/scraper'; +import { processOptions } from '@/dev-cli/validate'; + +import { getBuiltinEmbeds, getBuiltinSources } from '..'; + +dotenv.config(); + +type ProviderSourceAnswers = { + id: string; + type: string; +}; + +type EmbedSourceAnswers = { + url: string; +}; + +type CommonAnswers = { + fetcher: string; + source: string; +}; + +type ShowAnswers = { + season: string; + episode: string; +}; + +const sourceScrapers = getBuiltinSources().sort((a, b) => b.rank - a.rank); +const embedScrapers = getBuiltinEmbeds().sort((a, b) => b.rank - a.rank); +const sources = [...sourceScrapers, ...embedScrapers]; + +function joinMediaTypes(mediaTypes: string[] | undefined) { + if (mediaTypes) { + const formatted = mediaTypes + .map((type: string) => { + return `${type[0].toUpperCase() + type.substring(1).toLowerCase()}s`; + }) + .join(' / '); + + return `(${formatted})`; + } + return ''; // * Embed sources pass through here too +} + +async function runQuestions() { + const options = { + fetcher: 'node-fetch', + sourceId: '', + tmdbId: '', + type: 'movie', + season: '0', + episode: '0', + url: '', + }; + + const answers = await prompt([ + { + type: 'select', + name: 'fetcher', + message: 'Select a fetcher mode', + choices: [ + { + message: 'Native', + name: 'native', + }, + { + message: 'Node fetch', + name: 'node-fetch', + }, + { + message: 'Browser', + name: 'browser', + }, + ], + }, + { + type: 'select', + name: 'source', + message: 'Select a source', + choices: sources.map((source) => ({ + message: `[${source.type.toLocaleUpperCase()}] ${source.name} ${joinMediaTypes(source.mediaTypes)}`.trim(), + name: source.id, + })), + }, + ]); + + options.fetcher = answers.fetcher; + options.sourceId = answers.source; + + const source = sources.find(({ id }) => id === answers.source); + + if (!source) { + throw new Error(`No source with ID ${answers.source} found`); + } + + if (source.type === 'embed') { + const sourceAnswers = await prompt([ + { + type: 'input', + name: 'url', + message: 'Embed URL', + }, + ]); + + options.url = sourceAnswers.url; + } else { + const sourceAnswers = await prompt([ + { + type: 'input', + name: 'id', + message: 'TMDB ID', + }, + { + type: 'select', + name: 'type', + message: 'Media type', + choices: [ + { + message: 'Movie', + name: 'movie', + }, + { + message: 'TV Show', + name: 'show', + }, + ], + }, + ]); + + options.tmdbId = sourceAnswers.id; + options.type = sourceAnswers.type; + + if (sourceAnswers.type === 'show') { + const seriesAnswers = await prompt([ + { + type: 'input', + name: 'season', + message: 'Season', + }, + { + type: 'input', + name: 'episode', + message: 'Episode', + }, + ]); + + options.season = seriesAnswers.season; + options.episode = seriesAnswers.episode; + } + } + + const { providerOptions, source: validatedSource, options: validatedOps } = await processOptions(sources, options); + await runScraper(providerOptions, validatedSource, validatedOps); +} + +async function runCommandLine() { + program + .option('-f, --fetcher ', "Fetcher to use. Either 'native' or 'node-fetch'", 'node-fetch') + .option('-sid, --source-id ', 'ID for the source to use. Either an embed or provider', '') + .option('-tid, --tmdb-id ', 'TMDB ID for the media to scrape. Only used if source is a provider', '') + .option('-t, --type ', "Media type. Either 'movie' or 'show'. Only used if source is a provider", 'movie') + .option('-s, --season ', "Season number. Only used if type is 'show'", '0') + .option('-e, --episode ', "Episode number. Only used if type is 'show'", '0') + .option('-u, --url ', 'URL to a video embed. Only used if source is an embed', ''); + + program.parse(); + + const { + providerOptions, + source: validatedSource, + options: validatedOps, + } = await processOptions(sources, program.opts()); + await runScraper(providerOptions, validatedSource, validatedOps); +} + +if (process.argv.length === 2) { + runQuestions().catch(() => console.error('Exited.')); +} else { + runCommandLine().catch(() => console.error('Exited.')); +} diff --git a/src/dev-cli/logging.ts b/src/dev-cli/logging.ts new file mode 100644 index 0000000..1a519f4 --- /dev/null +++ b/src/dev-cli/logging.ts @@ -0,0 +1,5 @@ +import { inspect } from 'node:util'; + +export function logDeepObject(object: Record) { + console.log(inspect(object, { showHidden: false, depth: null, colors: true })); +} diff --git a/src/dev-cli/scraper.ts b/src/dev-cli/scraper.ts new file mode 100644 index 0000000..882d321 --- /dev/null +++ b/src/dev-cli/scraper.ts @@ -0,0 +1,136 @@ +/* eslint import/no-extraneous-dependencies: ["error", {"devDependencies": true}] */ + +import { existsSync } from 'fs'; +import { join } from 'path'; + +import puppeteer, { Browser } from 'puppeteer'; +import Spinnies from 'spinnies'; +import { PreviewServer, build, preview } from 'vite'; + +import { getConfig } from '@/dev-cli/config'; +import { logDeepObject } from '@/dev-cli/logging'; +import { getMovieMediaDetails, getShowMediaDetails } from '@/dev-cli/tmdb'; +import { CommandLineArguments } from '@/dev-cli/validate'; + +import { MetaOutput, ProviderMakerOptions, makeProviders } from '..'; + +async function runBrowserScraping( + providerOptions: ProviderMakerOptions, + source: MetaOutput, + options: CommandLineArguments, +) { + if (!existsSync(join(__dirname, '../../lib/index.mjs'))) + throw new Error('Please compile before running cli in browser mode'); + const config = getConfig(); + if (!config.proxyUrl) + throw new Error('Simple proxy url must be set in the environment (MOVIE_WEB_PROXY_URL) for browser mode to work'); + + const root = join(__dirname, 'browser'); + let server: PreviewServer | undefined; + let browser: Browser | undefined; + try { + // setup browser + await build({ + root, + }); + server = await preview({ + root, + }); + browser = await puppeteer.launch({ + headless: 'new', + args: ['--no-sandbox', '--disable-setuid-sandbox'], + }); + const page = await browser.newPage(); + await page.goto(server.resolvedUrls.local[0]); + await page.waitForFunction('!!window.scrape', { timeout: 5000 }); + + // get input media + let input: any; + if (source.type === 'embed') { + input = { + url: options.url, + id: source.id, + }; + } else if (source.type === 'source') { + let media; + if (options.type === 'movie') { + media = await getMovieMediaDetails(options.tmdbId); + } else { + media = await getShowMediaDetails(options.tmdbId, options.season, options.episode); + } + input = { + media, + id: source.id, + }; + } else { + throw new Error('Wrong source input type'); + } + + return await page.evaluate( + async (proxy, type, inp) => { + return (window as any).scrape(proxy, type, inp); + }, + config.proxyUrl, + source.type, + input, + ); + } finally { + server?.httpServer.close(); + await browser?.close(); + } +} + +async function runActualScraping( + providerOptions: ProviderMakerOptions, + source: MetaOutput, + options: CommandLineArguments, +): Promise { + if (options.fetcher === 'browser') return runBrowserScraping(providerOptions, source, options); + const providers = makeProviders(providerOptions); + + if (source.type === 'embed') { + return providers.runEmbedScraper({ + url: options.url, + id: source.id, + }); + } + + if (source.type === 'source') { + let media; + + if (options.type === 'movie') { + media = await getMovieMediaDetails(options.tmdbId); + } else { + media = await getShowMediaDetails(options.tmdbId, options.season, options.episode); + } + + return providers.runSourceScraper({ + media, + id: source.id, + }); + } + + throw new Error('Invalid source type'); +} + +export async function runScraper( + providerOptions: ProviderMakerOptions, + source: MetaOutput, + options: CommandLineArguments, +) { + const spinnies = new Spinnies(); + + spinnies.add('scrape', { text: `Running ${source.name} scraper` }); + try { + const result = await runActualScraping(providerOptions, source, options); + spinnies.succeed('scrape', { text: 'Done!' }); + logDeepObject(result); + } catch (error) { + let message = 'Unknown error'; + if (error instanceof Error) { + message = error.message; + } + spinnies.fail('scrape', { text: `ERROR: ${message}` }); + console.error(error); + } +} diff --git a/src/dev-cli/tmdb.ts b/src/dev-cli/tmdb.ts new file mode 100644 index 0000000..7490336 --- /dev/null +++ b/src/dev-cli/tmdb.ts @@ -0,0 +1,95 @@ +import { getConfig } from '@/dev-cli/config'; + +import { MovieMedia, ShowMedia } from '..'; + +export async function makeTMDBRequest(url: string): Promise { + const headers: { + accept: 'application/json'; + authorization?: string; + } = { + accept: 'application/json', + }; + + let requestURL = url; + const key = getConfig().tmdbApiKey; + + // * JWT keys always start with ey and are ONLY valid as a header. + // * All other keys are ONLY valid as a query param. + // * Thanks TMDB. + if (key.startsWith('ey')) { + headers.authorization = `Bearer ${key}`; + } else { + requestURL += `?api_key=${key}`; + } + + return fetch(requestURL, { + method: 'GET', + headers, + }); +} + +export async function getMovieMediaDetails(id: string): Promise { + const response = await makeTMDBRequest(`https://api.themoviedb.org/3/movie/${id}`); + const movie = await response.json(); + + if (movie.success === false) { + throw new Error(movie.status_message); + } + + if (!movie.release_date) { + throw new Error(`${movie.title} has no release_date. Assuming unreleased`); + } + + return { + type: 'movie', + title: movie.title, + releaseYear: Number(movie.release_date.split('-')[0]), + tmdbId: id, + }; +} + +export async function getShowMediaDetails(id: string, seasonNumber: string, episodeNumber: string): Promise { + // * TV shows require the TMDB ID for the series, season, and episode + // * and the name of the series. Needs multiple requests + let response = await makeTMDBRequest(`https://api.themoviedb.org/3/tv/${id}`); + const series = await response.json(); + + if (series.success === false) { + throw new Error(series.status_message); + } + + if (!series.first_air_date) { + throw new Error(`${series.name} has no first_air_date. Assuming unaired`); + } + + response = await makeTMDBRequest(`https://api.themoviedb.org/3/tv/${id}/season/${seasonNumber}`); + const season = await response.json(); + + if (season.success === false) { + throw new Error(season.status_message); + } + + response = await makeTMDBRequest( + `https://api.themoviedb.org/3/tv/${id}/season/${seasonNumber}/episode/${episodeNumber}`, + ); + const episode = await response.json(); + + if (episode.success === false) { + throw new Error(episode.status_message); + } + + return { + type: 'show', + title: series.name, + releaseYear: Number(series.first_air_date.split('-')[0]), + tmdbId: id, + episode: { + number: episode.episode_number, + tmdbId: episode.id, + }, + season: { + number: season.season_number, + tmdbId: season.id, + }, + }; +} diff --git a/src/dev-cli/validate.ts b/src/dev-cli/validate.ts new file mode 100644 index 0000000..b600454 --- /dev/null +++ b/src/dev-cli/validate.ts @@ -0,0 +1,91 @@ +import nodeFetch from 'node-fetch'; + +import { Embed, Sourcerer } from '@/providers/base'; + +import { ProviderMakerOptions, makeStandardFetcher, targets } from '..'; + +export type CommandLineArguments = { + fetcher: string; + sourceId: string; + tmdbId: string; + type: string; + season: string; + episode: string; + url: string; +}; + +export async function processOptions(sources: Array, options: CommandLineArguments) { + const fetcherOptions = ['node-fetch', 'native', 'browser']; + if (!fetcherOptions.includes(options.fetcher)) { + throw new Error(`Fetcher must be any of: ${fetcherOptions.join()}`); + } + + if (!options.sourceId.trim()) { + throw new Error('Source ID must be provided'); + } + + const source = sources.find(({ id }) => id === options.sourceId); + + if (!source) { + throw new Error('Invalid source ID. No source found'); + } + + if (source.type === 'embed' && !options.url.trim()) { + throw new Error('Must provide an embed URL for embed sources'); + } + + if (source.type === 'source') { + if (!options.tmdbId.trim()) { + throw new Error('Must provide a TMDB ID for provider sources'); + } + + if (Number.isNaN(Number(options.tmdbId)) || Number(options.tmdbId) < 0) { + throw new Error('TMDB ID must be a number greater than 0'); + } + + if (!options.type.trim()) { + throw new Error('Must provide a type for provider sources'); + } + + if (options.type !== 'movie' && options.type !== 'show') { + throw new Error("Invalid media type. Must be either 'movie' or 'show'"); + } + + if (options.type === 'show') { + if (!options.season.trim()) { + throw new Error('Must provide a season number for TV shows'); + } + + if (!options.episode.trim()) { + throw new Error('Must provide an episode number for TV shows'); + } + + if (Number.isNaN(Number(options.season)) || Number(options.season) <= 0) { + throw new Error('Season number must be a number greater than 0'); + } + + if (Number.isNaN(Number(options.episode)) || Number(options.episode) <= 0) { + throw new Error('Episode number must be a number greater than 0'); + } + } + } + + let fetcher; + + if (options.fetcher === 'native') { + fetcher = makeStandardFetcher(fetch); + } else { + fetcher = makeStandardFetcher(nodeFetch); + } + + const providerOptions: ProviderMakerOptions = { + fetcher, + target: targets.ANY, + }; + + return { + providerOptions, + options, + source, + }; +} diff --git a/src/entrypoint/builder.ts b/src/entrypoint/builder.ts new file mode 100644 index 0000000..abf8288 --- /dev/null +++ b/src/entrypoint/builder.ts @@ -0,0 +1,93 @@ +import { ProviderControls, makeControls } from '@/entrypoint/controls'; +import { getBuiltinEmbeds, getBuiltinSources } from '@/entrypoint/providers'; +import { Targets, getTargetFeatures } from '@/entrypoint/utils/targets'; +import { Fetcher } from '@/fetchers/types'; +import { Embed, Sourcerer } from '@/providers/base'; +import { getProviders } from '@/providers/get'; + +export type ProviderBuilder = { + setTarget(target: Targets): ProviderBuilder; + setFetcher(fetcher: Fetcher): ProviderBuilder; + setProxiedFetcher(fetcher: Fetcher): ProviderBuilder; + addSource(scraper: Sourcerer): ProviderBuilder; + addSource(name: string): ProviderBuilder; + addEmbed(scraper: Embed): ProviderBuilder; + addEmbed(name: string): ProviderBuilder; + addBuiltinProviders(): ProviderBuilder; + enableConsistentIpForRequests(): ProviderBuilder; + build(): ProviderControls; +}; + +export function buildProviders(): ProviderBuilder { + let consistentIpForRequests = false; + let target: Targets | null = null; + let fetcher: Fetcher | null = null; + let proxiedFetcher: Fetcher | null = null; + const embeds: Embed[] = []; + const sources: Sourcerer[] = []; + const builtinSources = getBuiltinSources(); + const builtinEmbeds = getBuiltinEmbeds(); + + return { + enableConsistentIpForRequests() { + consistentIpForRequests = true; + return this; + }, + setFetcher(f) { + fetcher = f; + return this; + }, + setProxiedFetcher(f) { + proxiedFetcher = f; + return this; + }, + setTarget(t) { + target = t; + return this; + }, + addSource(input) { + if (typeof input !== 'string') { + sources.push(input); + return this; + } + + const matchingSource = builtinSources.find((v) => v.id === input); + if (!matchingSource) throw new Error('Source not found'); + sources.push(matchingSource); + return this; + }, + addEmbed(input) { + if (typeof input !== 'string') { + embeds.push(input); + return this; + } + + const matchingEmbed = builtinEmbeds.find((v) => v.id === input); + if (!matchingEmbed) throw new Error('Embed not found'); + embeds.push(matchingEmbed); + return this; + }, + addBuiltinProviders() { + sources.push(...builtinSources); + embeds.push(...builtinEmbeds); + return this; + }, + build() { + if (!target) throw new Error('Target not set'); + if (!fetcher) throw new Error('Fetcher not set'); + const features = getTargetFeatures(target, consistentIpForRequests); + const list = getProviders(features, { + embeds, + sources, + }); + + return makeControls({ + fetcher, + proxiedFetcher: proxiedFetcher ?? undefined, + embeds: list.embeds, + sources: list.sources, + features, + }); + }, + }; +} diff --git a/src/main/builder.ts b/src/entrypoint/controls.ts similarity index 68% rename from src/main/builder.ts rename to src/entrypoint/controls.ts index 0322dbd..5ff400b 100644 --- a/src/main/builder.ts +++ b/src/entrypoint/controls.ts @@ -1,24 +1,19 @@ -import { makeFullFetcher } from '@/fetchers/common'; +import { FullScraperEvents, IndividualScraperEvents } from '@/entrypoint/utils/events'; +import { ScrapeMedia } from '@/entrypoint/utils/media'; +import { MetaOutput, getAllEmbedMetaSorted, getAllSourceMetaSorted, getSpecificId } from '@/entrypoint/utils/meta'; +import { FeatureMap } from '@/entrypoint/utils/targets'; +import { makeFetcher } from '@/fetchers/common'; import { Fetcher } from '@/fetchers/types'; -import { FullScraperEvents, IndividualScraperEvents } from '@/main/events'; -import { scrapeIndividualEmbed, scrapeInvidualSource } from '@/main/individualRunner'; -import { ScrapeMedia } from '@/main/media'; -import { MetaOutput, getAllEmbedMetaSorted, getAllSourceMetaSorted, getSpecificId } from '@/main/meta'; -import { RunOutput, runAllProviders } from '@/main/runner'; -import { Targets, getTargetFeatures } from '@/main/targets'; -import { EmbedOutput, SourcererOutput } from '@/providers/base'; -import { getProviders } from '@/providers/get'; +import { Embed, EmbedOutput, Sourcerer, SourcererOutput } from '@/providers/base'; +import { scrapeIndividualEmbed, scrapeInvidualSource } from '@/runners/individualRunner'; +import { RunOutput, runAllProviders } from '@/runners/runner'; -export interface ProviderBuilderOptions { - // fetcher, every web request gets called through here +export interface ProviderControlsInput { fetcher: Fetcher; - - // proxied fetcher, if the scraper needs to access a CORS proxy. this fetcher will be called instead - // of the normal fetcher. Defaults to the normal fetcher. proxiedFetcher?: Fetcher; - - // target of where the streams will be used - target: Targets; + features: FeatureMap; + sources: Sourcerer[]; + embeds: Embed[]; } export interface RunnerOptions { @@ -80,13 +75,16 @@ export interface ProviderControls { listEmbeds(): MetaOutput[]; } -export function makeProviders(ops: ProviderBuilderOptions): ProviderControls { - const features = getTargetFeatures(ops.target); - const list = getProviders(features); +export function makeControls(ops: ProviderControlsInput): ProviderControls { + const list = { + embeds: ops.embeds, + sources: ops.sources, + }; + const providerRunnerOps = { - features, - fetcher: makeFullFetcher(ops.fetcher), - proxiedFetcher: makeFullFetcher(ops.proxiedFetcher ?? ops.fetcher), + features: ops.features, + fetcher: makeFetcher(ops.fetcher), + proxiedFetcher: makeFetcher(ops.proxiedFetcher ?? ops.fetcher), }; return { diff --git a/src/entrypoint/declare.ts b/src/entrypoint/declare.ts new file mode 100644 index 0000000..152ee87 --- /dev/null +++ b/src/entrypoint/declare.ts @@ -0,0 +1,37 @@ +import { makeControls } from '@/entrypoint/controls'; +import { getBuiltinEmbeds, getBuiltinSources } from '@/entrypoint/providers'; +import { Targets, getTargetFeatures } from '@/entrypoint/utils/targets'; +import { Fetcher } from '@/fetchers/types'; +import { getProviders } from '@/providers/get'; + +export interface ProviderMakerOptions { + // fetcher, every web request gets called through here + fetcher: Fetcher; + + // proxied fetcher, if the scraper needs to access a CORS proxy. this fetcher will be called instead + // of the normal fetcher. Defaults to the normal fetcher. + proxiedFetcher?: Fetcher; + + // target of where the streams will be used + target: Targets; + + // Set this to true, if the requests will have the same IP as + // the device that the stream will be played on + consistentIpForRequests?: boolean; +} + +export function makeProviders(ops: ProviderMakerOptions) { + const features = getTargetFeatures(ops.target, ops.consistentIpForRequests ?? false); + const list = getProviders(features, { + embeds: getBuiltinEmbeds(), + sources: getBuiltinSources(), + }); + + return makeControls({ + embeds: list.embeds, + sources: list.sources, + features, + fetcher: ops.fetcher, + proxiedFetcher: ops.proxiedFetcher, + }); +} diff --git a/src/entrypoint/providers.ts b/src/entrypoint/providers.ts new file mode 100644 index 0000000..b306417 --- /dev/null +++ b/src/entrypoint/providers.ts @@ -0,0 +1,10 @@ +import { gatherAllEmbeds, gatherAllSources } from '@/providers/all'; +import { Embed, Sourcerer } from '@/providers/base'; + +export function getBuiltinSources(): Sourcerer[] { + return gatherAllSources(); +} + +export function getBuiltinEmbeds(): Embed[] { + return gatherAllEmbeds(); +} diff --git a/src/main/events.ts b/src/entrypoint/utils/events.ts similarity index 100% rename from src/main/events.ts rename to src/entrypoint/utils/events.ts diff --git a/src/main/media.ts b/src/entrypoint/utils/media.ts similarity index 100% rename from src/main/media.ts rename to src/entrypoint/utils/media.ts diff --git a/src/main/meta.ts b/src/entrypoint/utils/meta.ts similarity index 96% rename from src/main/meta.ts rename to src/entrypoint/utils/meta.ts index 5696183..5e54b2a 100644 --- a/src/main/meta.ts +++ b/src/entrypoint/utils/meta.ts @@ -1,4 +1,4 @@ -import { MediaTypes } from '@/main/media'; +import { MediaTypes } from '@/entrypoint/utils/media'; import { Embed, Sourcerer } from '@/providers/base'; import { ProviderList } from '@/providers/get'; diff --git a/src/entrypoint/utils/targets.ts b/src/entrypoint/utils/targets.ts new file mode 100644 index 0000000..16a02da --- /dev/null +++ b/src/entrypoint/utils/targets.ts @@ -0,0 +1,64 @@ +export const flags = { + // CORS are set to allow any origin + CORS_ALLOWED: 'cors-allowed', + + // the stream is locked on IP, so only works if + // request maker is same as player (not compatible with proxies) + IP_LOCKED: 'ip-locked', +} as const; + +export type Flags = (typeof flags)[keyof typeof flags]; + +export const targets = { + // browser with CORS restrictions + BROWSER: 'browser', + + // browser, but no CORS restrictions through a browser extension + BROWSER_EXTENSION: 'browser-extension', + + // native app, so no restrictions in what can be played + NATIVE: 'native', + + // any target, no target restrictions + ANY: 'any', +} as const; + +export type Targets = (typeof targets)[keyof typeof targets]; + +export type FeatureMap = { + requires: Flags[]; + disallowed: Flags[]; +}; + +export const targetToFeatures: Record = { + browser: { + requires: [flags.CORS_ALLOWED], + disallowed: [], + }, + 'browser-extension': { + requires: [], + disallowed: [], + }, + native: { + requires: [], + disallowed: [], + }, + any: { + requires: [], + disallowed: [], + }, +}; + +export function getTargetFeatures(target: Targets, consistentIpForRequests: boolean): FeatureMap { + const features = targetToFeatures[target]; + if (!consistentIpForRequests) features.disallowed.push(flags.IP_LOCKED); + return features; +} + +export function flagsAllowedInFeatures(features: FeatureMap, inputFlags: Flags[]): boolean { + const hasAllFlags = features.requires.every((v) => inputFlags.includes(v)); + if (!hasAllFlags) return false; + const hasDisallowedFlag = features.disallowed.some((v) => inputFlags.includes(v)); + if (hasDisallowedFlag) return false; + return true; +} diff --git a/src/fetchers/common.ts b/src/fetchers/common.ts index e31b6d1..71956ba 100644 --- a/src/fetchers/common.ts +++ b/src/fetchers/common.ts @@ -26,14 +26,18 @@ export function makeFullUrl(url: string, ops?: FullUrlOptions): string { return parsedUrl.toString(); } -export function makeFullFetcher(fetcher: Fetcher): UseableFetcher { - return (url, ops) => { +export function makeFetcher(fetcher: Fetcher): UseableFetcher { + const newFetcher = (url: string, ops?: FetcherOptions) => { return fetcher(url, { headers: ops?.headers ?? {}, method: ops?.method ?? 'GET', query: ops?.query ?? {}, baseUrl: ops?.baseUrl ?? '', + readHeaders: ops?.readHeaders ?? [], body: ops?.body, }); }; + const output: UseableFetcher = async (url, ops) => (await newFetcher(url, ops)).body; + output.full = newFetcher; + return output; } diff --git a/src/fetchers/fetch.ts b/src/fetchers/fetch.ts index 1d419f0..d2156d0 100644 --- a/src/fetchers/fetch.ts +++ b/src/fetchers/fetch.ts @@ -11,12 +11,17 @@ export type FetchOps = { export type FetchHeaders = { get(key: string): string | null; + set(key: string, value: string): void; }; export type FetchReply = { text(): Promise; json(): Promise; + extraHeaders?: FetchHeaders; + extraUrl?: string; headers: FetchHeaders; + url: string; + status: number; }; export type FetchLike = (url: string, ops?: FetchOps | undefined) => Promise; diff --git a/src/fetchers/simpleProxy.ts b/src/fetchers/simpleProxy.ts index 07b048e..21ed5ca 100644 --- a/src/fetchers/simpleProxy.ts +++ b/src/fetchers/simpleProxy.ts @@ -9,9 +9,28 @@ const headerMap: Record = { origin: 'X-Origin', }; +const responseHeaderMap: Record = { + 'x-set-cookie': 'Set-Cookie', +}; + export function makeSimpleProxyFetcher(proxyUrl: string, f: FetchLike): Fetcher { - const fetcher = makeStandardFetcher(f); const proxiedFetch: Fetcher = async (url, ops) => { + const fetcher = makeStandardFetcher(async (a, b) => { + const res = await f(a, b); + + // set extra headers that cant normally be accessed + res.extraHeaders = new Headers(); + Object.entries(responseHeaderMap).forEach((entry) => { + const value = res.headers.get(entry[0]); + if (!value) return; + res.extraHeaders?.set(entry[0].toLowerCase(), value); + }); + + // set correct final url + res.extraUrl = res.headers.get('X-Final-Destination') ?? res.url; + return res; + }); + const fullUrl = makeFullUrl(url, ops); const headerEntries = Object.entries(ops.headers).map((entry) => { diff --git a/src/fetchers/standardFetch.ts b/src/fetchers/standardFetch.ts index dd84893..9fb6afa 100644 --- a/src/fetchers/standardFetch.ts +++ b/src/fetchers/standardFetch.ts @@ -1,8 +1,20 @@ import { serializeBody } from '@/fetchers/body'; import { makeFullUrl } from '@/fetchers/common'; -import { FetchLike } from '@/fetchers/fetch'; +import { FetchLike, FetchReply } from '@/fetchers/fetch'; import { Fetcher } from '@/fetchers/types'; +function getHeaders(list: string[], res: FetchReply): Headers { + const output = new Headers(); + list.forEach((header) => { + const realHeader = header.toLowerCase(); + const value = res.headers.get(realHeader); + const extraValue = res.extraHeaders?.get(realHeader); + if (!value) return; + output.set(realHeader, extraValue ?? value); + }); + return output; +} + export function makeStandardFetcher(f: FetchLike): Fetcher { const normalFetch: Fetcher = async (url, ops) => { const fullUrl = makeFullUrl(url, ops); @@ -17,9 +29,17 @@ export function makeStandardFetcher(f: FetchLike): Fetcher { body: seralizedBody.body, }); + let body: any; const isJson = res.headers.get('content-type')?.includes('application/json'); - if (isJson) return res.json(); - return res.text(); + if (isJson) body = await res.json(); + else body = await res.text(); + + return { + body, + finalUrl: res.extraUrl ?? res.url, + headers: getHeaders(ops.readHeaders, res), + statusCode: res.status, + }; }; return normalFetch; diff --git a/src/fetchers/types.ts b/src/fetchers/types.ts index 2d14748..7daa5df 100644 --- a/src/fetchers/types.ts +++ b/src/fetchers/types.ts @@ -5,22 +5,35 @@ export type FetcherOptions = { headers?: Record; query?: Record; method?: 'GET' | 'POST'; + readHeaders?: string[]; body?: Record | string | FormData | URLSearchParams; }; +// Version of the options that always has the defaults set +// This is to make making fetchers yourself easier export type DefaultedFetcherOptions = { baseUrl?: string; body?: Record | string | FormData; headers: Record; query: Record; + readHeaders: string[]; method: 'GET' | 'POST'; }; -export type Fetcher = { - (url: string, ops: DefaultedFetcherOptions): Promise; +export type FetcherResponse = { + statusCode: number; + headers: Headers; + finalUrl: string; + body: T; }; -// this feature has some quality of life features -export type UseableFetcher = { - (url: string, ops?: FetcherOptions): Promise; +// This is the version that will be inputted by library users +export type Fetcher = { + (url: string, ops: DefaultedFetcherOptions): Promise>; +}; + +// This is the version that scrapers will be interacting with +export type UseableFetcher = { + (url: string, ops?: FetcherOptions): Promise; + full: (url: string, ops?: FetcherOptions) => Promise>; }; diff --git a/src/index.ts b/src/index.ts index ea8b255..8191052 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,19 +1,21 @@ export type { EmbedOutput, SourcererOutput } from '@/providers/base'; -export type { RunOutput } from '@/main/runner'; -export type { MetaOutput } from '@/main/meta'; -export type { FullScraperEvents } from '@/main/events'; -export type { Targets, Flags } from '@/main/targets'; -export type { MediaTypes, ShowMedia, ScrapeMedia, MovieMedia } from '@/main/media'; -export type { - ProviderBuilderOptions, - ProviderControls, - RunnerOptions, - EmbedRunnerOptions, - SourceRunnerOptions, -} from '@/main/builder'; +export type { Stream, StreamFile, FileBasedStream, HlsBasedStream, Qualities } from '@/providers/streams'; +export type { Fetcher, DefaultedFetcherOptions, FetcherOptions, FetcherResponse } from '@/fetchers/types'; +export type { RunOutput } from '@/runners/runner'; +export type { MetaOutput } from '@/entrypoint/utils/meta'; +export type { FullScraperEvents } from '@/entrypoint/utils/events'; +export type { Targets, Flags } from '@/entrypoint/utils/targets'; +export type { MediaTypes, ShowMedia, ScrapeMedia, MovieMedia } from '@/entrypoint/utils/media'; +export type { ProviderControls, RunnerOptions, EmbedRunnerOptions, SourceRunnerOptions } from '@/entrypoint/controls'; +export type { ProviderBuilder } from '@/entrypoint/builder'; +export type { ProviderMakerOptions } from '@/entrypoint/declare'; +export type { MovieScrapeContext, ShowScrapeContext, EmbedScrapeContext, ScrapeContext } from '@/utils/context'; +export type { SourcererOptions, EmbedOptions } from '@/providers/base'; export { NotFoundError } from '@/utils/errors'; -export { makeProviders } from '@/main/builder'; +export { makeProviders } from '@/entrypoint/declare'; +export { buildProviders } from '@/entrypoint/builder'; +export { getBuiltinEmbeds, getBuiltinSources } from '@/entrypoint/providers'; export { makeStandardFetcher } from '@/fetchers/standardFetch'; export { makeSimpleProxyFetcher } from '@/fetchers/simpleProxy'; -export { flags, targets } from '@/main/targets'; +export { flags, targets } from '@/entrypoint/utils/targets'; diff --git a/src/main/targets.ts b/src/main/targets.ts deleted file mode 100644 index eb791c2..0000000 --- a/src/main/targets.ts +++ /dev/null @@ -1,40 +0,0 @@ -export const flags = { - NO_CORS: 'no-cors', - IP_LOCKED: 'ip-locked', -} as const; - -export type Flags = (typeof flags)[keyof typeof flags]; - -export const targets = { - BROWSER: 'browser', - NATIVE: 'native', - ALL: 'all', -} as const; - -export type Targets = (typeof targets)[keyof typeof targets]; - -export type FeatureMap = { - requires: readonly Flags[]; -}; - -export const targetToFeatures: Record = { - browser: { - requires: [flags.NO_CORS], - }, - native: { - requires: [], - }, - all: { - requires: [], - }, -} as const; - -export function getTargetFeatures(target: Targets): FeatureMap { - return targetToFeatures[target]; -} - -export function flagsAllowedInFeatures(features: FeatureMap, inputFlags: Flags[]): boolean { - const hasAllFlags = features.requires.every((v) => inputFlags.includes(v)); - if (!hasAllFlags) return false; - return true; -} diff --git a/src/providers/all.ts b/src/providers/all.ts index 8181e94..d1e7885 100644 --- a/src/providers/all.ts +++ b/src/providers/all.ts @@ -1,5 +1,6 @@ import { Embed, Sourcerer } from '@/providers/base'; -import { febBoxScraper } from '@/providers/embeds/febBox'; +import { febboxHlsScraper } from '@/providers/embeds/febbox/hls'; +import { febboxMp4Scraper } from '@/providers/embeds/febbox/mp4'; import { mixdropScraper } from '@/providers/embeds/mixdrop'; import { mp4uploadScraper } from '@/providers/embeds/mp4upload'; import { streamsbScraper } from '@/providers/embeds/streamsb'; @@ -10,10 +11,12 @@ import { goMoviesScraper } from '@/providers/sources/gomovies/index'; import { kissAsianScraper } from '@/providers/sources/kissasian/index'; import { lookmovieScraper } from '@/providers/sources/lookmovie'; import { remotestreamScraper } from '@/providers/sources/remotestream'; -import { superStreamScraper } from '@/providers/sources/superstream/index'; +import { showboxScraper } from '@/providers/sources/showbox/index'; import { zoechipScraper } from '@/providers/sources/zoechip'; -import { showBoxScraper } from './sources/showbox'; +import { smashyStreamDScraper } from './embeds/smashystream/dued'; +import { smashyStreamFScraper } from './embeds/smashystream/video1'; +import { smashyStreamScraper } from './sources/smashystream'; export function gatherAllSources(): Array { // all sources are gathered here @@ -21,15 +24,25 @@ export function gatherAllSources(): Array { flixhqScraper, remotestreamScraper, kissAsianScraper, - superStreamScraper, + showboxScraper, goMoviesScraper, zoechipScraper, lookmovieScraper, - showBoxScraper, + smashyStreamScraper, ]; } export function gatherAllEmbeds(): Array { // all embeds are gathered here - return [upcloudScraper, mp4uploadScraper, streamsbScraper, upstreamScraper, febBoxScraper, mixdropScraper]; + return [ + upcloudScraper, + mp4uploadScraper, + streamsbScraper, + upstreamScraper, + febboxMp4Scraper, + febboxHlsScraper, + mixdropScraper, + smashyStreamFScraper, + smashyStreamDScraper, + ]; } diff --git a/src/providers/base.ts b/src/providers/base.ts index 902a5a6..0d43895 100644 --- a/src/providers/base.ts +++ b/src/providers/base.ts @@ -1,35 +1,52 @@ -import { MovieMedia, ShowMedia } from '@/main/media'; -import { Flags } from '@/main/targets'; +import { Flags } from '@/entrypoint/utils/targets'; import { Stream } from '@/providers/streams'; -import { EmbedScrapeContext, ScrapeContext } from '@/utils/context'; +import { EmbedScrapeContext, MovieScrapeContext, ShowScrapeContext } from '@/utils/context'; -export type SourcererOutput = { - embeds: { - embedId: string; - url: string; - }[]; - stream?: Stream; +export type MediaScraperTypes = 'show' | 'movie'; + +export type SourcererEmbed = { + embedId: string; + url: string; }; -export type Sourcerer = { +export type SourcererOutput = { + embeds: SourcererEmbed[]; + stream?: Stream[]; +}; + +export type SourcererOptions = { id: string; name: string; // displayed in the UI rank: number; // the higher the number, the earlier it gets put on the queue disabled?: boolean; flags: Flags[]; - scrapeMovie?: (input: ScrapeContext & { media: MovieMedia }) => Promise; - scrapeShow?: (input: ScrapeContext & { media: ShowMedia }) => Promise; + scrapeMovie?: (input: MovieScrapeContext) => Promise; + scrapeShow?: (input: ShowScrapeContext) => Promise; }; -export function makeSourcerer(state: Sourcerer): Sourcerer { - return state; +export type Sourcerer = SourcererOptions & { + type: 'source'; + disabled: boolean; + mediaTypes: MediaScraperTypes[]; +}; + +export function makeSourcerer(state: SourcererOptions): Sourcerer { + const mediaTypes: MediaScraperTypes[] = []; + if (state.scrapeMovie) mediaTypes.push('movie'); + if (state.scrapeShow) mediaTypes.push('show'); + return { + ...state, + type: 'source', + disabled: state.disabled ?? false, + mediaTypes, + }; } export type EmbedOutput = { - stream: Stream; + stream: Stream[]; }; -export type Embed = { +export type EmbedOptions = { id: string; name: string; // displayed in the UI rank: number; // the higher the number, the earlier it gets put on the queue @@ -37,6 +54,17 @@ export type Embed = { scrape: (input: EmbedScrapeContext) => Promise; }; -export function makeEmbed(state: Embed): Embed { - return state; +export type Embed = EmbedOptions & { + type: 'embed'; + disabled: boolean; + mediaTypes: undefined; +}; + +export function makeEmbed(state: EmbedOptions): Embed { + return { + ...state, + type: 'embed', + disabled: state.disabled ?? false, + mediaTypes: undefined, + }; } diff --git a/src/providers/captions.ts b/src/providers/captions.ts index 791bfa6..ce3f398 100644 --- a/src/providers/captions.ts +++ b/src/providers/captions.ts @@ -8,6 +8,7 @@ export type CaptionType = keyof typeof captionTypes; export type Caption = { type: CaptionType; + id: string; // only unique per stream url: string; hasCorsRestrictions: boolean; language: string; diff --git a/src/providers/embeds/febBox.ts b/src/providers/embeds/febBox.ts deleted file mode 100644 index 7855745..0000000 --- a/src/providers/embeds/febBox.ts +++ /dev/null @@ -1,74 +0,0 @@ -import { flags } from '@/main/targets'; -import { makeEmbed } from '@/providers/base'; -import { StreamFile } from '@/providers/streams'; -import { NotFoundError } from '@/utils/errors'; - -const febBoxBase = `https://www.febbox.com`; - -const allowedQualities = ['360', '480', '720', '1080']; - -export const febBoxScraper = makeEmbed({ - id: 'febbox', - name: 'FebBox', - rank: 160, - async scrape(ctx) { - const shareKey = ctx.url.split('/')[4]; - const streams = await ctx.proxiedFetcher<{ - data?: { - file_list?: { - fid?: string; - }[]; - }; - }>('/file/file_share_list', { - headers: { - 'accept-language': 'en', // without this header, the request is marked as a webscraper - }, - baseUrl: febBoxBase, - query: { - share_key: shareKey, - pwd: '', - }, - }); - - const fid = streams?.data?.file_list?.[0]?.fid; - if (!fid) throw new NotFoundError('no result found'); - - const formParams = new URLSearchParams(); - formParams.append('fid', fid); - formParams.append('share_key', shareKey); - - const player = await ctx.proxiedFetcher('/file/player', { - baseUrl: febBoxBase, - body: formParams, - method: 'POST', - headers: { - 'accept-language': 'en', // without this header, the request is marked as a webscraper - }, - }); - - const sourcesMatch = player?.match(/var sources = (\[[^\]]+\]);/); - const qualities = sourcesMatch ? JSON.parse(sourcesMatch[0].replace('var sources = ', '').replace(';', '')) : null; - - const embedQualities: Record = {}; - - qualities.forEach((quality: { file: string; label: string }) => { - const normalizedLabel = quality.label.toLowerCase().replace('p', ''); - if (allowedQualities.includes(normalizedLabel)) { - if (!quality.file) return; - embedQualities[normalizedLabel] = { - type: 'mp4', - url: quality.file, - }; - } - }); - - return { - stream: { - type: 'file', - captions: [], - flags: [flags.NO_CORS], - qualities: embedQualities, - }, - }; - }, -}); diff --git a/src/providers/embeds/febbox/common.ts b/src/providers/embeds/febbox/common.ts new file mode 100644 index 0000000..5d902db --- /dev/null +++ b/src/providers/embeds/febbox/common.ts @@ -0,0 +1,24 @@ +import { MediaTypes } from '@/entrypoint/utils/media'; + +export const febBoxBase = `https://www.febbox.com`; + +export interface FebboxFileList { + file_name: string; + ext: string; + fid: number; + oss_fid: number; + is_dir: 0 | 1; +} + +export function parseInputUrl(url: string) { + const [type, id, seasonId, episodeId] = url.slice(1).split('/'); + const season = seasonId ? parseInt(seasonId, 10) : undefined; + const episode = episodeId ? parseInt(episodeId, 10) : undefined; + + return { + type: type as MediaTypes, + id, + season, + episode, + }; +} diff --git a/src/providers/embeds/febbox/fileList.ts b/src/providers/embeds/febbox/fileList.ts new file mode 100644 index 0000000..593fc77 --- /dev/null +++ b/src/providers/embeds/febbox/fileList.ts @@ -0,0 +1,69 @@ +import { MediaTypes } from '@/entrypoint/utils/media'; +import { FebboxFileList, febBoxBase } from '@/providers/embeds/febbox/common'; +import { EmbedScrapeContext } from '@/utils/context'; + +export async function getFileList( + ctx: EmbedScrapeContext, + shareKey: string, + parentId?: number, +): Promise { + const query: Record = { + share_key: shareKey, + pwd: '', + }; + if (parentId) { + query.parent_id = parentId.toString(); + query.page = '1'; + } + + const streams = await ctx.proxiedFetcher<{ + data?: { + file_list?: FebboxFileList[]; + }; + }>('/file/file_share_list', { + headers: { + 'accept-language': 'en', // without this header, the request is marked as a webscraper + }, + baseUrl: febBoxBase, + query, + }); + + return streams.data?.file_list ?? []; +} + +function isValidStream(file: FebboxFileList): boolean { + return file.ext === 'mp4' || file.ext === 'mkv'; +} + +export async function getStreams( + ctx: EmbedScrapeContext, + shareKey: string, + type: MediaTypes, + season?: number, + episode?: number, +): Promise { + const streams = await getFileList(ctx, shareKey); + + if (type === 'show') { + const seasonFolder = streams.find((v) => { + if (!v.is_dir) return false; + return v.file_name.toLowerCase() === `season ${season}`; + }); + if (!seasonFolder) return []; + + const episodes = await getFileList(ctx, shareKey, seasonFolder.fid); + const s = season?.toString() ?? '0'; + const e = episode?.toString() ?? '0'; + const episodeRegex = new RegExp(`[Ss]0*${s}[Ee]0*${e}`); + return episodes + .filter((file) => { + if (file.is_dir) return false; + const match = file.file_name.match(episodeRegex); + if (!match) return false; + return true; + }) + .filter(isValidStream); + } + + return streams.filter((v) => !v.is_dir).filter(isValidStream); +} diff --git a/src/providers/embeds/febbox/hls.ts b/src/providers/embeds/febbox/hls.ts new file mode 100644 index 0000000..d9fa54f --- /dev/null +++ b/src/providers/embeds/febbox/hls.ts @@ -0,0 +1,50 @@ +import { MediaTypes } from '@/entrypoint/utils/media'; +import { flags } from '@/entrypoint/utils/targets'; +import { makeEmbed } from '@/providers/base'; +import { parseInputUrl } from '@/providers/embeds/febbox/common'; +import { getStreams } from '@/providers/embeds/febbox/fileList'; +import { getSubtitles } from '@/providers/embeds/febbox/subtitles'; +import { showboxBase } from '@/providers/sources/showbox/common'; + +// structure: https://www.febbox.com/share/ +export function extractShareKey(url: string): string { + const parsedUrl = new URL(url); + const shareKey = parsedUrl.pathname.split('/')[2]; + return shareKey; +} +export const febboxHlsScraper = makeEmbed({ + id: 'febbox-hls', + name: 'Febbox (HLS)', + rank: 160, + async scrape(ctx) { + const { type, id, season, episode } = parseInputUrl(ctx.url); + const sharelinkResult = await ctx.proxiedFetcher<{ + data?: { link?: string }; + }>('/index/share_link', { + baseUrl: showboxBase, + query: { + id, + type: type === 'movie' ? '1' : '2', + }, + }); + if (!sharelinkResult?.data?.link) throw new Error('No embed url found'); + ctx.progress(30); + const shareKey = extractShareKey(sharelinkResult.data.link); + const fileList = await getStreams(ctx, shareKey, type, season, episode); + const firstStream = fileList[0]; + if (!firstStream) throw new Error('No playable mp4 stream found'); + ctx.progress(70); + + return { + stream: [ + { + id: 'primary', + type: 'hls', + flags: [flags.CORS_ALLOWED], + captions: await getSubtitles(ctx, id, firstStream.fid, type as MediaTypes, season, episode), + playlist: `https://www.febbox.com/hls/main/${firstStream.oss_fid}.m3u8`, + }, + ], + }; + }, +}); diff --git a/src/providers/embeds/febbox/mp4.ts b/src/providers/embeds/febbox/mp4.ts new file mode 100644 index 0000000..1122e53 --- /dev/null +++ b/src/providers/embeds/febbox/mp4.ts @@ -0,0 +1,53 @@ +import { flags } from '@/entrypoint/utils/targets'; +import { makeEmbed } from '@/providers/base'; +import { parseInputUrl } from '@/providers/embeds/febbox/common'; +import { getStreamQualities } from '@/providers/embeds/febbox/qualities'; +import { getSubtitles } from '@/providers/embeds/febbox/subtitles'; + +export const febboxMp4Scraper = makeEmbed({ + id: 'febbox-mp4', + name: 'Febbox (MP4)', + rank: 190, + async scrape(ctx) { + const { type, id, season, episode } = parseInputUrl(ctx.url); + let apiQuery: object | null = null; + + if (type === 'movie') { + apiQuery = { + uid: '', + module: 'Movie_downloadurl_v3', + mid: id, + oss: '1', + group: '', + }; + } else if (type === 'show') { + apiQuery = { + uid: '', + module: 'TV_downloadurl_v3', + tid: id, + season, + episode, + oss: '1', + group: '', + }; + } + + if (!apiQuery) throw Error('Incorrect type'); + + const { qualities, fid } = await getStreamQualities(ctx, apiQuery); + if (fid === undefined) throw new Error('No streamable file found'); + ctx.progress(70); + + return { + stream: [ + { + id: 'primary', + captions: await getSubtitles(ctx, id, fid, type, episode, season), + qualities, + type: 'file', + flags: [flags.CORS_ALLOWED], + }, + ], + }; + }, +}); diff --git a/src/providers/sources/superstream/getStreamQualities.ts b/src/providers/embeds/febbox/qualities.ts similarity index 87% rename from src/providers/sources/superstream/getStreamQualities.ts rename to src/providers/embeds/febbox/qualities.ts index 5e82b4c..54f8866 100644 --- a/src/providers/sources/superstream/getStreamQualities.ts +++ b/src/providers/embeds/febbox/qualities.ts @@ -1,13 +1,11 @@ +import { sendRequest } from '@/providers/sources/showbox/sendRequest'; import { StreamFile } from '@/providers/streams'; import { ScrapeContext } from '@/utils/context'; -import { sendRequest } from './sendRequest'; - -const allowedQualities = ['360', '480', '720', '1080']; +const allowedQualities = ['360', '480', '720', '1080', '4k']; export async function getStreamQualities(ctx: ScrapeContext, apiQuery: object) { const mediaRes: { list: { path: string; quality: string; fid?: number }[] } = (await sendRequest(ctx, apiQuery)).data; - ctx.progress(66); const qualityMap = mediaRes.list .filter((file) => allowedQualities.includes(file.quality.replace('p', ''))) diff --git a/src/providers/sources/superstream/subtitles.ts b/src/providers/embeds/febbox/subtitles.ts similarity index 90% rename from src/providers/sources/superstream/subtitles.ts rename to src/providers/embeds/febbox/subtitles.ts index 36be8de..b1b3064 100644 --- a/src/providers/sources/superstream/subtitles.ts +++ b/src/providers/embeds/febbox/subtitles.ts @@ -1,9 +1,8 @@ import { Caption, getCaptionTypeFromUrl, isValidLanguageCode } from '@/providers/captions'; -import { sendRequest } from '@/providers/sources/superstream/sendRequest'; +import { captionsDomains } from '@/providers/sources/showbox/common'; +import { sendRequest } from '@/providers/sources/showbox/sendRequest'; import { ScrapeContext } from '@/utils/context'; -import { captionsDomains } from './common'; - interface CaptionApiResponse { data: { list: { @@ -55,6 +54,7 @@ export async function getSubtitles( if (!validCode) return; output.push({ + id: subtitleFilePath, language: subtitle.lang, hasCorsRestrictions: true, type: subtitleType, diff --git a/src/providers/embeds/mixdrop.ts b/src/providers/embeds/mixdrop.ts index 007738a..71fd0ac 100644 --- a/src/providers/embeds/mixdrop.ts +++ b/src/providers/embeds/mixdrop.ts @@ -33,21 +33,24 @@ export const mixdropScraper = makeEmbed({ const url = link[1]; return { - stream: { - type: 'file', - flags: [], - captions: [], - qualities: { - unknown: { - type: 'mp4', - url: url.startsWith('http') ? url : `https:${url}`, // URLs don't always start with the protocol - headers: { - // MixDrop requires this header on all streams - Referer: 'https://mixdrop.co/', + stream: [ + { + id: 'primary', + type: 'file', + flags: [], + captions: [], + qualities: { + unknown: { + type: 'mp4', + url: url.startsWith('http') ? url : `https:${url}`, // URLs don't always start with the protocol + headers: { + // MixDrop requires this header on all streams + Referer: 'https://mixdrop.co/', + }, }, }, }, - }, + ], }; }, }); diff --git a/src/providers/embeds/mp4upload.ts b/src/providers/embeds/mp4upload.ts index 81c2f94..5bc8576 100644 --- a/src/providers/embeds/mp4upload.ts +++ b/src/providers/embeds/mp4upload.ts @@ -1,4 +1,4 @@ -import { flags } from '@/main/targets'; +import { flags } from '@/entrypoint/utils/targets'; import { makeEmbed } from '@/providers/base'; export const mp4uploadScraper = makeEmbed({ @@ -15,17 +15,20 @@ export const mp4uploadScraper = makeEmbed({ if (!streamUrl) throw new Error('Stream url not found in embed code'); return { - stream: { - type: 'file', - flags: [flags.NO_CORS], - captions: [], - qualities: { - '1080': { - type: 'mp4', - url: streamUrl, + stream: [ + { + id: 'primary', + type: 'file', + flags: [flags.CORS_ALLOWED], + captions: [], + qualities: { + '1080': { + type: 'mp4', + url: streamUrl, + }, }, }, - }, + ], }; }, }); diff --git a/src/providers/embeds/smashystream/dued.ts b/src/providers/embeds/smashystream/dued.ts new file mode 100644 index 0000000..c7a1d1d --- /dev/null +++ b/src/providers/embeds/smashystream/dued.ts @@ -0,0 +1,71 @@ +import { load } from 'cheerio'; + +import { flags } from '@/entrypoint/utils/targets'; +import { makeEmbed } from '@/providers/base'; + +type DPlayerSourcesResponse = { + title: string; + id: string; + file: string; +}[]; + +export const smashyStreamDScraper = makeEmbed({ + id: 'smashystream-d', + name: 'SmashyStream (D)', + rank: 71, + async scrape(ctx) { + const mainPageRes = await ctx.proxiedFetcher(ctx.url, { + headers: { + Referer: ctx.url, + }, + }); + const mainPageRes$ = load(mainPageRes); + const iframeUrl = mainPageRes$('iframe').attr('src'); + if (!iframeUrl) throw new Error(`[${this.name}] failed to find iframe url`); + const mainUrl = new URL(iframeUrl); + const iframeRes = await ctx.proxiedFetcher(iframeUrl, { + headers: { + Referer: ctx.url, + }, + }); + const textFilePath = iframeRes.match(/"file":"([^"]+)"/)?.[1]; + const csrfToken = iframeRes.match(/"key":"([^"]+)"/)?.[1]; + if (!textFilePath || !csrfToken) throw new Error(`[${this.name}] failed to find text file url or token`); + const textFileUrl = `${mainUrl.origin}${textFilePath}`; + const textFileRes = await ctx.proxiedFetcher(textFileUrl, { + method: 'POST', + headers: { + 'Content-Type': 'application/x-www-form-urlencoded', + 'X-CSRF-TOKEN': csrfToken, + Referer: iframeUrl, + }, + }); + // Playlists in Hindi, English, Tamil and Telugu are available. We only get the english one. + const textFilePlaylist = textFileRes.find((x) => x.title === 'English')?.file; + if (!textFilePlaylist) throw new Error(`[${this.name}] failed to find an english playlist`); + + const playlistRes = await ctx.proxiedFetcher( + `${mainUrl.origin}/playlist/${textFilePlaylist.slice(1)}.txt`, + { + method: 'POST', + headers: { + 'Content-Type': 'application/x-www-form-urlencoded', + 'X-CSRF-TOKEN': csrfToken, + Referer: iframeUrl, + }, + }, + ); + + return { + stream: [ + { + id: 'primary', + playlist: playlistRes, + type: 'hls', + flags: [flags.CORS_ALLOWED], + captions: [], + }, + ], + }; + }, +}); diff --git a/src/providers/embeds/smashystream/video1.ts b/src/providers/embeds/smashystream/video1.ts new file mode 100644 index 0000000..1fc2edf --- /dev/null +++ b/src/providers/embeds/smashystream/video1.ts @@ -0,0 +1,57 @@ +import { flags } from '@/entrypoint/utils/targets'; +import { makeEmbed } from '@/providers/base'; +import { Caption, getCaptionTypeFromUrl, labelToLanguageCode } from '@/providers/captions'; + +type FPlayerResponse = { + sourceUrls: string[]; + subtitleUrls: string; +}; + +export const smashyStreamFScraper = makeEmbed({ + id: 'smashystream-f', + name: 'SmashyStream (F)', + rank: 70, + async scrape(ctx) { + const res = await ctx.proxiedFetcher(ctx.url, { + headers: { + Referer: ctx.url, + }, + }); + + const captions: Caption[] = + res.subtitleUrls + .match(/\[([^\]]+)\](https?:\/\/\S+?)(?=,\[|$)/g) + ?.map((entry: string) => { + const match = entry.match(/\[([^\]]+)\](https?:\/\/\S+?)(?=,\[|$)/); + if (match) { + const [, language, url] = match; + if (language && url) { + const languageCode = labelToLanguageCode(language); + const captionType = getCaptionTypeFromUrl(url); + if (!languageCode || !captionType) return null; + return { + id: url, + url: url.replace(',', ''), + language: languageCode, + type: captionType, + hasCorsRestrictions: false, + }; + } + } + return null; + }) + .filter((x): x is Caption => x !== null) ?? []; + + return { + stream: [ + { + id: 'primary', + playlist: res.sourceUrls[0], + type: 'hls', + flags: [flags.CORS_ALLOWED], + captions, + }, + ], + }; + }, +}); diff --git a/src/providers/embeds/streamsb.ts b/src/providers/embeds/streamsb.ts index d15b320..2314be4 100644 --- a/src/providers/embeds/streamsb.ts +++ b/src/providers/embeds/streamsb.ts @@ -3,7 +3,7 @@ import Base64 from 'crypto-js/enc-base64'; import Utf8 from 'crypto-js/enc-utf8'; import FormData from 'form-data'; -import { flags } from '@/main/targets'; +import { flags } from '@/entrypoint/utils/targets'; import { makeEmbed } from '@/providers/base'; import { StreamFile } from '@/providers/streams'; import { EmbedScrapeContext } from '@/utils/context'; @@ -155,12 +155,15 @@ export const streamsbScraper = makeEmbed({ }, {} as Record); return { - stream: { - type: 'file', - flags: [flags.NO_CORS], - qualities, - captions: [], - }, + stream: [ + { + id: 'primary', + type: 'file', + flags: [flags.CORS_ALLOWED], + qualities, + captions: [], + }, + ], }; }, }); diff --git a/src/providers/embeds/upcloud.ts b/src/providers/embeds/upcloud.ts index d31cf72..7880cc4 100644 --- a/src/providers/embeds/upcloud.ts +++ b/src/providers/embeds/upcloud.ts @@ -1,6 +1,6 @@ import crypto from 'crypto-js'; -import { flags } from '@/main/targets'; +import { flags } from '@/entrypoint/utils/targets'; import { makeEmbed } from '@/providers/base'; import { Caption, getCaptionTypeFromUrl, labelToLanguageCode } from '@/providers/captions'; @@ -110,6 +110,7 @@ export const upcloudScraper = makeEmbed({ const language = labelToLanguageCode(track.label); if (!language) return; captions.push({ + id: track.file, language, hasCorsRestrictions: false, type, @@ -118,12 +119,15 @@ export const upcloudScraper = makeEmbed({ }); return { - stream: { - type: 'hls', - playlist: sources.file, - flags: [flags.NO_CORS], - captions, - }, + stream: [ + { + id: 'primary', + type: 'hls', + playlist: sources.file, + flags: [flags.CORS_ALLOWED], + captions, + }, + ], }; }, }); diff --git a/src/providers/embeds/upstream.ts b/src/providers/embeds/upstream.ts index 852aac7..8becf22 100644 --- a/src/providers/embeds/upstream.ts +++ b/src/providers/embeds/upstream.ts @@ -1,6 +1,6 @@ import * as unpacker from 'unpacker'; -import { flags } from '@/main/targets'; +import { flags } from '@/entrypoint/utils/targets'; import { makeEmbed } from '@/providers/base'; const packedRegex = /(eval\(function\(p,a,c,k,e,d\).*\)\)\))/; @@ -21,12 +21,15 @@ export const upstreamScraper = makeEmbed({ if (link) { return { - stream: { - type: 'hls', - playlist: link[1], - flags: [flags.NO_CORS], - captions: [], - }, + stream: [ + { + id: 'primary', + type: 'hls', + playlist: link[1], + flags: [flags.CORS_ALLOWED], + captions: [], + }, + ], }; } } diff --git a/src/providers/get.ts b/src/providers/get.ts index 085aba5..900bbd1 100644 --- a/src/providers/get.ts +++ b/src/providers/get.ts @@ -1,5 +1,4 @@ -import { FeatureMap, flagsAllowedInFeatures } from '@/main/targets'; -import { gatherAllEmbeds, gatherAllSources } from '@/providers/all'; +import { FeatureMap, flagsAllowedInFeatures } from '@/entrypoint/utils/targets'; import { Embed, Sourcerer } from '@/providers/base'; import { hasDuplicates } from '@/utils/predicates'; @@ -8,9 +7,9 @@ export interface ProviderList { embeds: Embed[]; } -export function getProviders(features: FeatureMap): ProviderList { - const sources = gatherAllSources().filter((v) => !v?.disabled); - const embeds = gatherAllEmbeds().filter((v) => !v?.disabled); +export function getProviders(features: FeatureMap, list: ProviderList): ProviderList { + const sources = list.sources.filter((v) => !v?.disabled); + const embeds = list.embeds.filter((v) => !v?.disabled); const combined = [...sources, ...embeds]; const anyDuplicateId = hasDuplicates(combined.map((v) => v.id)); diff --git a/src/providers/sources/flixhq/index.ts b/src/providers/sources/flixhq/index.ts index 90fe251..e9fbaef 100644 --- a/src/providers/sources/flixhq/index.ts +++ b/src/providers/sources/flixhq/index.ts @@ -1,4 +1,4 @@ -import { flags } from '@/main/targets'; +import { flags } from '@/entrypoint/utils/targets'; import { makeSourcerer } from '@/providers/base'; import { upcloudScraper } from '@/providers/embeds/upcloud'; import { getFlixhqMovieSources, getFlixhqShowSources, getFlixhqSourceDetails } from '@/providers/sources/flixhq/scrape'; @@ -9,7 +9,7 @@ export const flixhqScraper = makeSourcerer({ id: 'flixhq', name: 'FlixHQ', rank: 100, - flags: [flags.NO_CORS], + flags: [flags.CORS_ALLOWED], async scrapeMovie(ctx) { const id = await getFlixhqId(ctx, ctx.media); if (!id) throw new NotFoundError('no search results match'); diff --git a/src/providers/sources/flixhq/scrape.ts b/src/providers/sources/flixhq/scrape.ts index a73916c..3f42e9a 100644 --- a/src/providers/sources/flixhq/scrape.ts +++ b/src/providers/sources/flixhq/scrape.ts @@ -1,6 +1,6 @@ import { load } from 'cheerio'; -import { MovieMedia, ShowMedia } from '@/main/media'; +import { MovieMedia, ShowMedia } from '@/entrypoint/utils/media'; import { flixHqBase } from '@/providers/sources/flixhq/common'; import { ScrapeContext } from '@/utils/context'; import { NotFoundError } from '@/utils/errors'; diff --git a/src/providers/sources/flixhq/search.ts b/src/providers/sources/flixhq/search.ts index 90517f6..bcab033 100644 --- a/src/providers/sources/flixhq/search.ts +++ b/src/providers/sources/flixhq/search.ts @@ -1,8 +1,8 @@ import { load } from 'cheerio'; -import { MovieMedia, ShowMedia } from '@/main/media'; +import { MovieMedia, ShowMedia } from '@/entrypoint/utils/media'; import { flixHqBase } from '@/providers/sources/flixhq/common'; -import { compareMedia } from '@/utils/compare'; +import { compareMedia, compareTitle } from '@/utils/compare'; import { ScrapeContext } from '@/utils/context'; export async function getFlixhqId(ctx: ScrapeContext, media: MovieMedia | ShowMedia): Promise { @@ -18,16 +18,26 @@ export async function getFlixhqId(ctx: ScrapeContext, media: MovieMedia | ShowMe const id = query.find('div.film-poster > a').attr('href')?.slice(1); const title = query.find('div.film-detail > h2 > a').attr('title'); const year = query.find('div.film-detail > div.fd-infor > span:nth-child(1)').text(); + const seasons = year.includes('SS') ? year.split('SS')[1] : '0'; if (!id || !title || !year) return null; return { id, title, year: parseInt(year, 10), + seasons: parseInt(seasons, 10), }; }); - const matchingItem = items.find((v) => v && compareMedia(media, v.title, v.year)); + const matchingItem = items.find((v) => { + if (!v) return false; + + if (media.type === 'movie') { + return compareMedia(media, v.title, v.year); + } + + return compareTitle(media.title, v.title) && media.season.number < v.seasons + 1; + }); if (!matchingItem) return null; return matchingItem.id; diff --git a/src/providers/sources/gomovies/index.ts b/src/providers/sources/gomovies/index.ts index ce4a428..9bec3e0 100644 --- a/src/providers/sources/gomovies/index.ts +++ b/src/providers/sources/gomovies/index.ts @@ -1,6 +1,6 @@ import { load } from 'cheerio'; -import { flags } from '@/main/targets'; +import { flags } from '@/entrypoint/utils/targets'; import { makeSourcerer } from '@/providers/base'; import { upcloudScraper } from '@/providers/embeds/upcloud'; import { NotFoundError } from '@/utils/errors'; @@ -13,7 +13,7 @@ export const goMoviesScraper = makeSourcerer({ id: 'gomovies', name: 'GOmovies', rank: 110, - flags: [flags.NO_CORS], + flags: [flags.CORS_ALLOWED], async scrapeShow(ctx) { const search = await ctx.proxiedFetcher(`/ajax/search`, { method: 'POST', diff --git a/src/providers/sources/kissasian/index.ts b/src/providers/sources/kissasian/index.ts index 67111ca..4b8032b 100644 --- a/src/providers/sources/kissasian/index.ts +++ b/src/providers/sources/kissasian/index.ts @@ -1,6 +1,6 @@ import { load } from 'cheerio'; -import { flags } from '@/main/targets'; +import { flags } from '@/entrypoint/utils/targets'; import { makeSourcerer } from '@/providers/base'; import { NotFoundError } from '@/utils/errors'; @@ -13,7 +13,7 @@ export const kissAsianScraper = makeSourcerer({ id: 'kissasian', name: 'KissAsian', rank: 130, - flags: [flags.NO_CORS], + flags: [flags.CORS_ALLOWED], disabled: true, async scrapeShow(ctx) { diff --git a/src/providers/sources/lookmovie/index.ts b/src/providers/sources/lookmovie/index.ts index 101dae9..5cd82e9 100644 --- a/src/providers/sources/lookmovie/index.ts +++ b/src/providers/sources/lookmovie/index.ts @@ -1,11 +1,11 @@ -import { flags } from '@/main/targets'; +import { flags } from '@/entrypoint/utils/targets'; import { SourcererOutput, makeSourcerer } from '@/providers/base'; +import { MovieScrapeContext, ShowScrapeContext } from '@/utils/context'; import { NotFoundError } from '@/utils/errors'; import { scrape, searchAndFindMedia } from './util'; -import { MovieContext, ShowContext } from '../zoechip/common'; -async function universalScraper(ctx: ShowContext | MovieContext): Promise { +async function universalScraper(ctx: MovieScrapeContext | ShowScrapeContext): Promise { const lookmovieData = await searchAndFindMedia(ctx, ctx.media); if (!lookmovieData) throw new NotFoundError('Media not found'); @@ -17,12 +17,15 @@ async function universalScraper(ctx: ShowContext | MovieContext): Promise('/search', { - baseUrl: showboxBase, - query: { - keyword: ctx.media.title, +async function comboScraper(ctx: ShowScrapeContext | MovieScrapeContext): Promise { + const searchQuery = { + module: 'Search4', + page: '1', + type: 'all', + keyword: ctx.media.title, + pagelimit: '20', + }; + + const searchRes = (await sendRequest(ctx, searchQuery, true)).data.list; + ctx.progress(50); + + const showboxEntry = searchRes.find( + (res: any) => compareTitle(res.title, ctx.media.title) && res.year === Number(ctx.media.releaseYear), + ); + if (!showboxEntry) throw new NotFoundError('No entry found'); + + const id = showboxEntry.id; + const season = ctx.media.type === 'show' ? ctx.media.season.number : ''; + const episode = ctx.media.type === 'show' ? ctx.media.episode.number : ''; + + return { + embeds: [ + { + embedId: febboxHlsScraper.id, + url: `/${ctx.media.type}/${id}/${season}/${episode}`, }, - }); - - const searchPage = load(search); - const result = searchPage('.film-name > a') - .toArray() - .map((el) => { - const titleContainer = el.parent?.parent; - if (!titleContainer) return; - const year = searchPage(titleContainer).find('.fdi-item').first().text(); - - return { - title: el.attribs.title, - path: el.attribs.href, - year: !year.includes('SS') ? parseInt(year, 10) : undefined, - }; - }) - .find((v) => v && compareMedia(ctx.media, v.title, v.year ? v.year : undefined)); - - if (!result?.path) throw new NotFoundError('no result found'); - - const febboxResult = await ctx.proxiedFetcher<{ - data?: { link?: string }; - }>('/index/share_link', { - baseUrl: showboxBase, - query: { - id: result.path.split('/')[3], - type: '1', + { + embedId: febboxMp4Scraper.id, + url: `/${ctx.media.type}/${id}/${season}/${episode}`, }, - }); + ], + }; +} - if (!febboxResult?.data?.link) throw new NotFoundError('no result found'); - - return { - embeds: [ - { - embedId: febBoxScraper.id, - url: febboxResult.data.link, - }, - ], - }; - }, +export const showboxScraper = makeSourcerer({ + id: 'showbox', + name: 'Showbox', + rank: 300, + flags: [flags.CORS_ALLOWED], + scrapeShow: comboScraper, + scrapeMovie: comboScraper, }); diff --git a/src/providers/sources/superstream/sendRequest.ts b/src/providers/sources/showbox/sendRequest.ts similarity index 100% rename from src/providers/sources/superstream/sendRequest.ts rename to src/providers/sources/showbox/sendRequest.ts diff --git a/src/providers/sources/smashystream/index.ts b/src/providers/sources/smashystream/index.ts new file mode 100644 index 0000000..62102ea --- /dev/null +++ b/src/providers/sources/smashystream/index.ts @@ -0,0 +1,64 @@ +import { load } from 'cheerio'; + +import { flags } from '@/entrypoint/utils/targets'; +import { SourcererEmbed, SourcererOutput, makeSourcerer } from '@/providers/base'; +import { smashyStreamDScraper } from '@/providers/embeds/smashystream/dued'; +import { smashyStreamFScraper } from '@/providers/embeds/smashystream/video1'; +import { MovieScrapeContext, ShowScrapeContext } from '@/utils/context'; + +const smashyStreamBase = 'https://embed.smashystream.com'; +const referer = 'https://smashystream.com/'; + +const universalScraper = async (ctx: ShowScrapeContext | MovieScrapeContext): Promise => { + const mainPage = await ctx.proxiedFetcher('/playere.php', { + query: { + tmdb: ctx.media.tmdbId, + ...(ctx.media.type === 'show' && { + season: ctx.media.season.number.toString(), + episode: ctx.media.episode.number.toString(), + }), + }, + headers: { + Referer: referer, + }, + baseUrl: smashyStreamBase, + }); + + ctx.progress(30); + + const mainPage$ = load(mainPage); + const sourceUrls = mainPage$('.dropdown-menu a[data-url]') + .map((_, el) => mainPage$(el).attr('data-url')) + .get(); + + const embeds: SourcererEmbed[] = []; + for (const sourceUrl of sourceUrls) { + if (sourceUrl.includes('video1d.php')) { + embeds.push({ + embedId: smashyStreamFScraper.id, + url: sourceUrl, + }); + } + if (sourceUrl.includes('dued.php')) { + embeds.push({ + embedId: smashyStreamDScraper.id, + url: sourceUrl, + }); + } + } + + ctx.progress(60); + + return { + embeds, + }; +}; + +export const smashyStreamScraper = makeSourcerer({ + id: 'smashystream', + name: 'SmashyStream', + rank: 70, + flags: [flags.CORS_ALLOWED], + scrapeMovie: universalScraper, + scrapeShow: universalScraper, +}); diff --git a/src/providers/sources/superstream/index.ts b/src/providers/sources/superstream/index.ts deleted file mode 100644 index 173f849..0000000 --- a/src/providers/sources/superstream/index.ts +++ /dev/null @@ -1,106 +0,0 @@ -import { flags } from '@/main/targets'; -import { makeSourcerer } from '@/providers/base'; -import { getSubtitles } from '@/providers/sources/superstream/subtitles'; -import { compareTitle } from '@/utils/compare'; -import { NotFoundError } from '@/utils/errors'; - -import { getStreamQualities } from './getStreamQualities'; -import { sendRequest } from './sendRequest'; - -export const superStreamScraper = makeSourcerer({ - id: 'superstream', - name: 'Superstream', - rank: 300, - flags: [flags.NO_CORS], - async scrapeShow(ctx) { - const searchQuery = { - module: 'Search4', - page: '1', - type: 'all', - keyword: ctx.media.title, - pagelimit: '20', - }; - - const searchRes = (await sendRequest(ctx, searchQuery, true)).data.list; - ctx.progress(33); - - const superstreamEntry = searchRes.find( - (res: any) => compareTitle(res.title, ctx.media.title) && res.year === Number(ctx.media.releaseYear), - ); - - if (!superstreamEntry) throw new NotFoundError('No entry found'); - const superstreamId = superstreamEntry.id; - - // Fetch requested episode - const apiQuery = { - uid: '', - module: 'TV_downloadurl_v3', - tid: superstreamId, - season: ctx.media.season.number, - episode: ctx.media.episode.number, - oss: '1', - group: '', - }; - - const { qualities, fid } = await getStreamQualities(ctx, apiQuery); - if (fid === undefined) throw new NotFoundError('No streamable file found'); - - return { - embeds: [], - stream: { - captions: await getSubtitles( - ctx, - superstreamId, - fid, - 'show', - ctx.media.episode.number, - ctx.media.season.number, - ), - qualities, - type: 'file', - flags: [flags.NO_CORS], - }, - }; - }, - async scrapeMovie(ctx) { - const searchQuery = { - module: 'Search4', - page: '1', - type: 'all', - keyword: ctx.media.title, - pagelimit: '20', - }; - - const searchRes = (await sendRequest(ctx, searchQuery, true)).data.list; - ctx.progress(33); - - const superstreamEntry = searchRes.find( - (res: any) => compareTitle(res.title, ctx.media.title) && res.year === Number(ctx.media.releaseYear), - ); - - if (!superstreamEntry) throw new NotFoundError('No entry found'); - const superstreamId = superstreamEntry.id; - - // Fetch requested episode - const apiQuery = { - uid: '', - module: 'Movie_downloadurl_v3', - mid: superstreamId, - oss: '1', - group: '', - }; - - const { qualities, fid } = await getStreamQualities(ctx, apiQuery); - if (fid === undefined) throw new NotFoundError('No streamable file found'); - - return { - embeds: [], - stream: { - captions: await getSubtitles(ctx, superstreamId, fid, 'movie'), - qualities, - type: 'file', - flags: [flags.NO_CORS], - }, - }; - }, -}); diff --git a/src/providers/sources/zoechip/common.ts b/src/providers/sources/zoechip/common.ts index 7ca1b2e..a860125 100644 --- a/src/providers/sources/zoechip/common.ts +++ b/src/providers/sources/zoechip/common.ts @@ -1,20 +1,11 @@ -import { MovieMedia, ShowMedia } from '@/main/media'; import { mixdropScraper } from '@/providers/embeds/mixdrop'; import { upcloudScraper } from '@/providers/embeds/upcloud'; import { upstreamScraper } from '@/providers/embeds/upstream'; import { getZoeChipSourceURL, getZoeChipSources } from '@/providers/sources/zoechip/scrape'; -import { ScrapeContext } from '@/utils/context'; +import { MovieScrapeContext, ShowScrapeContext } from '@/utils/context'; export const zoeBase = 'https://zoechip.cc'; -export type MovieContext = ScrapeContext & { - media: MovieMedia; -}; - -export type ShowContext = ScrapeContext & { - media: ShowMedia; -}; - export type ZoeChipSourceDetails = { type: string; // Only seen "iframe" so far link: string; @@ -23,7 +14,10 @@ export type ZoeChipSourceDetails = { title: string; }; -export async function formatSource(ctx: MovieContext | ShowContext, source: { embed: string; episodeId: string }) { +export async function formatSource( + ctx: MovieScrapeContext | ShowScrapeContext, + source: { embed: string; episodeId: string }, +) { const link = await getZoeChipSourceURL(ctx, source.episodeId); if (link) { const embed = { @@ -44,14 +38,14 @@ export async function formatSource(ctx: MovieContext | ShowContext, source: { em embed.embedId = mixdropScraper.id; break; default: - throw new Error(`Failed to find ZoeChip embed source for ${link}`); + return null; } return embed; } } -export async function createZoeChipStreamData(ctx: MovieContext | ShowContext, id: string) { +export async function createZoeChipStreamData(ctx: MovieScrapeContext | ShowScrapeContext, id: string) { const sources = await getZoeChipSources(ctx, id); const embeds: { embedId: string; diff --git a/src/providers/sources/zoechip/index.ts b/src/providers/sources/zoechip/index.ts index ab4d00f..ee7f3a6 100644 --- a/src/providers/sources/zoechip/index.ts +++ b/src/providers/sources/zoechip/index.ts @@ -1,4 +1,4 @@ -import { flags } from '@/main/targets'; +import { flags } from '@/entrypoint/utils/targets'; import { makeSourcerer } from '@/providers/base'; import { scrapeMovie } from '@/providers/sources/zoechip/scrape-movie'; import { scrapeShow } from '@/providers/sources/zoechip/scrape-show'; @@ -7,7 +7,7 @@ export const zoechipScraper = makeSourcerer({ id: 'zoechip', name: 'ZoeChip', rank: 200, - flags: [flags.NO_CORS], + flags: [flags.CORS_ALLOWED], scrapeMovie, scrapeShow, }); diff --git a/src/providers/sources/zoechip/scrape-movie.ts b/src/providers/sources/zoechip/scrape-movie.ts index 448af0b..86161fc 100644 --- a/src/providers/sources/zoechip/scrape-movie.ts +++ b/src/providers/sources/zoechip/scrape-movie.ts @@ -1,8 +1,9 @@ -import { MovieContext, createZoeChipStreamData } from '@/providers/sources/zoechip/common'; +import { createZoeChipStreamData } from '@/providers/sources/zoechip/common'; import { getZoeChipMovieID } from '@/providers/sources/zoechip/search'; +import { MovieScrapeContext } from '@/utils/context'; import { NotFoundError } from '@/utils/errors'; -export async function scrapeMovie(ctx: MovieContext) { +export async function scrapeMovie(ctx: MovieScrapeContext) { const movieID = await getZoeChipMovieID(ctx, ctx.media); if (!movieID) { throw new NotFoundError('no search results match'); diff --git a/src/providers/sources/zoechip/scrape-show.ts b/src/providers/sources/zoechip/scrape-show.ts index eb21d13..fe9f4eb 100644 --- a/src/providers/sources/zoechip/scrape-show.ts +++ b/src/providers/sources/zoechip/scrape-show.ts @@ -1,9 +1,10 @@ -import { ShowContext, createZoeChipStreamData } from '@/providers/sources/zoechip/common'; +import { createZoeChipStreamData } from '@/providers/sources/zoechip/common'; import { getZoeChipEpisodeID, getZoeChipSeasonID } from '@/providers/sources/zoechip/scrape'; import { getZoeChipShowID } from '@/providers/sources/zoechip/search'; +import { ShowScrapeContext } from '@/utils/context'; import { NotFoundError } from '@/utils/errors'; -export async function scrapeShow(ctx: ShowContext) { +export async function scrapeShow(ctx: ShowScrapeContext) { const showID = await getZoeChipShowID(ctx, ctx.media); if (!showID) { throw new NotFoundError('no search results match'); diff --git a/src/providers/sources/zoechip/scrape.ts b/src/providers/sources/zoechip/scrape.ts index 5be0edd..d3eb183 100644 --- a/src/providers/sources/zoechip/scrape.ts +++ b/src/providers/sources/zoechip/scrape.ts @@ -1,10 +1,10 @@ import { load } from 'cheerio'; -import { ShowMedia } from '@/main/media'; -import { MovieContext, ShowContext, ZoeChipSourceDetails, zoeBase } from '@/providers/sources/zoechip/common'; -import { ScrapeContext } from '@/utils/context'; +import { ShowMedia } from '@/entrypoint/utils/media'; +import { ZoeChipSourceDetails, zoeBase } from '@/providers/sources/zoechip/common'; +import { MovieScrapeContext, ScrapeContext, ShowScrapeContext } from '@/utils/context'; -export async function getZoeChipSources(ctx: MovieContext | ShowContext, id: string) { +export async function getZoeChipSources(ctx: MovieScrapeContext | ShowScrapeContext, id: string) { // Movies use /ajax/episode/list/ID // Shows use /ajax/episode/servers/ID const endpoint = ctx.media.type === 'movie' ? 'list' : 'servers'; diff --git a/src/providers/sources/zoechip/search.ts b/src/providers/sources/zoechip/search.ts index 6297bc4..f3a838d 100644 --- a/src/providers/sources/zoechip/search.ts +++ b/src/providers/sources/zoechip/search.ts @@ -1,6 +1,6 @@ import { load } from 'cheerio'; -import { MovieMedia, ShowMedia } from '@/main/media'; +import { MovieMedia, ShowMedia } from '@/entrypoint/utils/media'; import { zoeBase } from '@/providers/sources/zoechip/common'; import { compareMedia } from '@/utils/compare'; import { ScrapeContext } from '@/utils/context'; diff --git a/src/providers/streams.ts b/src/providers/streams.ts index 34863dd..f4674c6 100644 --- a/src/providers/streams.ts +++ b/src/providers/streams.ts @@ -1,26 +1,29 @@ -import { Flags } from '@/main/targets'; +import { Flags } from '@/entrypoint/utils/targets'; import { Caption } from '@/providers/captions'; export type StreamFile = { type: 'mp4'; url: string; - headers?: Record; }; -export type Qualities = 'unknown' | '360' | '480' | '720' | '1080'; +export type Qualities = 'unknown' | '360' | '480' | '720' | '1080' | '4k'; -export type FileBasedStream = { +type StreamCommon = { + id: string; // only unique per output + flags: Flags[]; + captions: Caption[]; + headers?: Record; // these headers HAVE to be set to watch the stream + preferredHeaders?: Record; // these headers are optional, would improve the stream +}; + +export type FileBasedStream = StreamCommon & { type: 'file'; - flags: Flags[]; qualities: Partial>; - captions: Caption[]; }; -export type HlsBasedStream = { +export type HlsBasedStream = StreamCommon & { type: 'hls'; - flags: Flags[]; playlist: string; - captions: Caption[]; }; export type Stream = FileBasedStream | HlsBasedStream; diff --git a/src/main/individualRunner.ts b/src/runners/individualRunner.ts similarity index 75% rename from src/main/individualRunner.ts rename to src/runners/individualRunner.ts index ac563ea..2befd84 100644 --- a/src/main/individualRunner.ts +++ b/src/runners/individualRunner.ts @@ -1,7 +1,7 @@ +import { IndividualScraperEvents } from '@/entrypoint/utils/events'; +import { ScrapeMedia } from '@/entrypoint/utils/media'; +import { FeatureMap, flagsAllowedInFeatures } from '@/entrypoint/utils/targets'; import { UseableFetcher } from '@/fetchers/types'; -import { IndividualScraperEvents } from '@/main/events'; -import { ScrapeMedia } from '@/main/media'; -import { FeatureMap, flagsAllowedInFeatures } from '@/main/targets'; import { EmbedOutput, SourcererOutput } from '@/providers/base'; import { ProviderList } from '@/providers/get'; import { ScrapeContext } from '@/utils/context'; @@ -50,12 +50,16 @@ export async function scrapeInvidualSource( media: ops.media, }); - // stream doesn't satisfy the feature flags, so gets removed in output - if (output?.stream && (!isValidStream(output.stream) || !flagsAllowedInFeatures(ops.features, output.stream.flags))) { - output.stream = undefined; + // filter output with only valid streams + if (output?.stream) { + output.stream = output.stream + .filter((stream) => isValidStream(stream)) + .filter((stream) => flagsAllowedInFeatures(ops.features, stream.flags)); } if (!output) throw new Error('output is null'); + if ((!output.stream || output.stream.length === 0) && output.embeds.length === 0) + throw new NotFoundError('No streams found'); return output; } @@ -88,9 +92,10 @@ export async function scrapeIndividualEmbed( }, }); - if (!isValidStream(output.stream)) throw new NotFoundError('stream is incomplete'); - if (!flagsAllowedInFeatures(ops.features, output.stream.flags)) - throw new NotFoundError("stream doesn't satisfy target feature flags"); + output.stream = output.stream + .filter((stream) => isValidStream(stream)) + .filter((stream) => flagsAllowedInFeatures(ops.features, stream.flags)); + if (output.stream.length === 0) throw new NotFoundError('No streams found'); return output; } diff --git a/src/main/runner.ts b/src/runners/runner.ts similarity index 80% rename from src/main/runner.ts rename to src/runners/runner.ts index 700dbe4..1774213 100644 --- a/src/main/runner.ts +++ b/src/runners/runner.ts @@ -1,7 +1,7 @@ +import { FullScraperEvents } from '@/entrypoint/utils/events'; +import { ScrapeMedia } from '@/entrypoint/utils/media'; +import { FeatureMap, flagsAllowedInFeatures } from '@/entrypoint/utils/targets'; import { UseableFetcher } from '@/fetchers/types'; -import { FullScraperEvents } from '@/main/events'; -import { ScrapeMedia } from '@/main/media'; -import { FeatureMap, flagsAllowedInFeatures } from '@/main/targets'; import { EmbedOutput, SourcererOutput } from '@/providers/base'; import { ProviderList } from '@/providers/get'; import { Stream } from '@/providers/streams'; @@ -18,13 +18,13 @@ export type RunOutput = { export type SourceRunOutput = { sourceId: string; - stream?: Stream; + stream: Stream[]; embeds: []; }; export type EmbedRunOutput = { embedId: string; - stream?: Stream; + stream: Stream[]; }; export type ProviderRunnerOptions = { @@ -80,12 +80,14 @@ export async function runAllProviders(list: ProviderList, ops: ProviderRunnerOpt ...contextBase, media: ops.media, }); - if (output?.stream && !isValidStream(output?.stream)) { - throw new NotFoundError('stream is incomplete'); - } - if (output?.stream && !flagsAllowedInFeatures(ops.features, output.stream.flags)) { - throw new NotFoundError("stream doesn't satisfy target feature flags"); + if (output) { + output.stream = (output.stream ?? []) + .filter((stream) => isValidStream(stream)) + .filter((stream) => flagsAllowedInFeatures(ops.features, stream.flags)); } + if (!output) throw Error('No output'); + if ((!output.stream || output.stream.length === 0) && output.embeds.length === 0) + throw new NotFoundError('No streams found'); } catch (err) { if (err instanceof NotFoundError) { ops.events?.update?.({ @@ -107,10 +109,10 @@ export async function runAllProviders(list: ProviderList, ops: ProviderRunnerOpt if (!output) throw new Error('Invalid media type'); // return stream is there are any - if (output.stream) { + if (output.stream?.[0]) { return { sourceId: s.id, - stream: output.stream, + stream: output.stream[0], }; } @@ -144,9 +146,10 @@ export async function runAllProviders(list: ProviderList, ops: ProviderRunnerOpt ...contextBase, url: e.url, }); - if (!flagsAllowedInFeatures(ops.features, embedOutput.stream.flags)) { - throw new NotFoundError("stream doesn't satisfy target feature flags"); - } + embedOutput.stream = embedOutput.stream + .filter((stream) => isValidStream(stream)) + .filter((stream) => flagsAllowedInFeatures(ops.features, stream.flags)); + if (embedOutput.stream.length === 0) throw new NotFoundError('No streams found'); } catch (err) { if (err instanceof NotFoundError) { ops.events?.update?.({ @@ -169,7 +172,7 @@ export async function runAllProviders(list: ProviderList, ops: ProviderRunnerOpt return { sourceId: s.id, embedId: scraper.id, - stream: embedOutput.stream, + stream: embedOutput.stream[0], }; } } diff --git a/src/utils/compare.ts b/src/utils/compare.ts index 8cce7da..cceffb8 100644 --- a/src/utils/compare.ts +++ b/src/utils/compare.ts @@ -1,4 +1,4 @@ -import { CommonMedia } from '@/main/media'; +import { CommonMedia } from '@/entrypoint/utils/media'; export function normalizeTitle(title: string): string { return title diff --git a/src/utils/context.ts b/src/utils/context.ts index 5a9664a..6f16bca 100644 --- a/src/utils/context.ts +++ b/src/utils/context.ts @@ -1,8 +1,9 @@ +import { MovieMedia, ShowMedia } from '@/entrypoint/utils/media'; import { UseableFetcher } from '@/fetchers/types'; export type ScrapeContext = { - proxiedFetcher: (...params: Parameters>) => ReturnType>; - fetcher: (...params: Parameters>) => ReturnType>; + proxiedFetcher: UseableFetcher; + fetcher: UseableFetcher; progress(val: number): void; }; @@ -11,3 +12,11 @@ export type EmbedInput = { }; export type EmbedScrapeContext = EmbedInput & ScrapeContext; + +export type MovieScrapeContext = ScrapeContext & { + media: MovieMedia; +}; + +export type ShowScrapeContext = ScrapeContext & { + media: ShowMedia; +}; diff --git a/tests/browser/.gitignore b/tests/browser/.gitignore new file mode 100644 index 0000000..1521c8b --- /dev/null +++ b/tests/browser/.gitignore @@ -0,0 +1 @@ +dist diff --git a/tests/browser/index.html b/tests/browser/index.html new file mode 100644 index 0000000..a27e53d --- /dev/null +++ b/tests/browser/index.html @@ -0,0 +1,11 @@ + + + + + + Browser integration test + + + + + diff --git a/tests/browser/index.ts b/tests/browser/index.ts new file mode 100644 index 0000000..aab14cc --- /dev/null +++ b/tests/browser/index.ts @@ -0,0 +1,8 @@ +import { makeProviders, makeStandardFetcher, targets } from '../../lib/index.mjs'; + +(window as any).TEST = () => { + makeProviders({ + fetcher: makeStandardFetcher(fetch), + target: targets.ANY, + }); +} diff --git a/tests/browser/package.json b/tests/browser/package.json new file mode 100644 index 0000000..727f329 --- /dev/null +++ b/tests/browser/package.json @@ -0,0 +1,3 @@ +{ + "main": "startup.mjs" +} diff --git a/tests/browser/startup.mjs b/tests/browser/startup.mjs new file mode 100644 index 0000000..fc43bd5 --- /dev/null +++ b/tests/browser/startup.mjs @@ -0,0 +1,31 @@ +import { build, preview } from 'vite'; +import puppeteer from 'puppeteer'; +import { fileURLToPath } from 'url'; +import { dirname } from 'path'; + +const root = dirname(fileURLToPath(import.meta.url)); + +await build({ + root, +}); +const server = await preview({ + root, +}); +let browser; +try { + browser = await puppeteer.launch({ + headless: 'new', + args: ['--no-sandbox', '--disable-setuid-sandbox'], + }); + const page = await browser.newPage(); + await page.goto(server.resolvedUrls.local[0]); + await page.waitForFunction('!!window.TEST', { timeout: 5000 }); + await page.evaluate(() => { + window.TEST(); + }); +} finally { + server.httpServer.close(); + await browser.close(); +} + +console.log('Success!'); diff --git a/tsconfig.json b/tsconfig.json index a57441a..9ac853e 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -17,7 +17,7 @@ } }, "include": ["src"], - "exclude": ["node_modules", "**/__tests__/*"], + "exclude": ["node_modules", "**/__test__"], "ts-node": { "require": ["tsconfig-paths/register"] }