Merge pull request #39 from movie-web/dev

Backend update v1.3.1

.docker/development/docker-compose.yml

@@ -12,6 +12,8 @@ services:
       POSTGRES_PASSWORD: postgres
     volumes:
      - 'postgres_data:/var/lib/postgresql/data'
+  redis:
+    image: redis
 
   # custom services
   backend:
@@ -39,7 +41,7 @@ services:
     links: 
       - postgres:postgres
     environment:
-      - DATABASE_URL=postgres://postgres:postgres@postgres:5432/postgres?sslmode=disable
+      - PGWEB_DATABASE_URL=postgres://postgres:postgres@postgres:5432/postgres?sslmode=disable
     depends_on:
       - postgres
 

.github/CODEOWNERS

@@ -1,3 +1 @@
-* @movie-web/core
+* @movie-web/project-leads
-
-.github @binaryoverload

.github/workflows/linting_testing.yml

@@ -14,16 +14,16 @@ jobs:
 
     steps:
       - name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
                   
       - uses: pnpm/action-setup@v2
         with:
           version: 8
 
       - name: Install Node.js
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
-          node-version: 18
+          node-version: 20
           cache: 'pnpm'
       
       - name: Install packages
@@ -38,16 +38,16 @@ jobs:
 
     steps:
       - name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
                   
       - uses: pnpm/action-setup@v2
         with:
           version: 8
 
       - name: Install Node.js
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
-          node-version: 18
+          node-version: 20
           cache: 'pnpm'
       
       - name: Install packages
@@ -62,10 +62,10 @@ jobs:
 
     steps:
       - name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
 
       - name: Build
-        uses: docker/build-push-action@v4
+        uses: docker/build-push-action@v5

.github/workflows/release.yml

@@ -12,7 +12,7 @@ jobs:
 
     steps:
     - name: Checkout code
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     - name: Get version
       id: package-version
@@ -42,10 +42,10 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Setup Docker buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
 
       - name: Get version
         id: package-version
@@ -70,9 +70,12 @@ jobs:
 
       - name: Build and push Docker image
         id: build-and-push
-        uses: docker/build-push-action@v4
+        uses: docker/build-push-action@v5
         with:
           push: true
+          platforms: linux/amd64,linux/arm64
           context: .
           labels: ${{ steps.meta.outputs.labels }}
           tags: ${{ steps.meta.outputs.tags }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

package.json

@@ -1,6 +1,6 @@
 {
   "name": "backend",
-  "version": "1.1.4",
+  "version": "1.3.1",
   "private": true,
   "homepage": "https://github.com/movie-web/backend",
   "engines": {
@@ -8,16 +8,16 @@
   },
   "scripts": {
     "dev": "nodemon -r tsconfig-paths/register src/main.ts",
-    "build": "npm run build:pre && npm run build:compile",
+    "build": "pnpm run build:pre && pnpm run build:compile",
     "start": "node dist/main.js",
     "lint": "eslint --ext .ts,.js,.json,.tsx src/",
     "lint:fix": "eslint --fix --ext .ts,.js,.json,.tsx src/",
     "build:pre": "rimraf dist/",
     "build:compile": "tsc && tsc-alias",
     "preinstall": "npx -y only-allow pnpm",
-    "migration:create": "npx -y mikro-orm migration:create",
+    "migration:create": "pnpm exec mikro-orm migration:create",
-    "migration:up": "npx -y mikro-orm migration:up",
+    "migration:up": "pnpm exec mikro-orm migration:up",
-    "migration:down": "npx -y mikro-orm migration:down"
+    "migration:down": "pnpm exec mikro-orm migration:down"
   },
   "mikro-orm": {
     "useTsNode": true,

src/config/fragments/docker.ts

@@ -4,4 +4,8 @@ export const dockerFragment: FragmentSchema = {
   postgres: {
     connection: 'postgres://postgres:postgres@postgres:5432/postgres',
   },
+  ratelimits: {
+    enabled: true,
+    redisUrl: 'redis://redis:6379',
+  },
 };

src/config/orm.ts

@@ -1,10 +1,20 @@
+import { devFragment } from '@/config/fragments/dev';
+import { dockerFragment } from '@/config/fragments/docker';
 import { createConfigLoader } from 'neat-config';
 import { z } from 'zod';
+import { booleanSchema } from './schema';
+
+const fragments = {
+  dev: devFragment,
+  dockerdev: dockerFragment,
+};
 
 export const ormConfigSchema = z.object({
   postgres: z.object({
     // connection URL for postgres database
     connection: z.string(),
+    // whether to use SSL for the connection
+    ssl: booleanSchema.default(false),
   }),
 });
 
@@ -15,6 +25,8 @@ export const ormConf = createConfigLoader()
     prefix: 'MWB_',
   })
   .addFromFile('config.json')
+  .setFragmentKey('usePresets')
+  .addConfigFragments(fragments)
   .addZodSchema(ormConfigSchema)
   .freeze()
   .load();

src/config/schema.ts

@@ -1,5 +1,7 @@
 import { z } from 'zod';
 
+export const booleanSchema = z.preprocess((val) => val === 'true', z.boolean());
+
 export const configSchema = z.object({
   server: z
     .object({
@@ -11,10 +13,13 @@ export const configSchema = z.object({
 
       // disable cross origin restrictions, allow any site.
       // overwrites the cors option above
-      allowAnySite: z.coerce.boolean().default(false),
+      allowAnySite: booleanSchema.default(false),
 
       // should it trust reverse proxy headers? (for ip gathering)
-      trustProxy: z.coerce.boolean().default(false),
+      trustProxy: booleanSchema.default(false),
+
+      // should it trust cloudflare headers? (for ip gathering, cloudflare has priority)
+      trustCloudflare: booleanSchema.default(false),
 
       // prefix for where the instance is run on. for example set it to /backend if you're hosting it on example.com/backend
       // if this is set, do not apply url rewriting before proxing
@@ -27,7 +32,7 @@ export const configSchema = z.object({
       format: z.enum(['json', 'pretty']).default('pretty'),
 
       // show debug logs?
-      debug: z.coerce.boolean().default(false),
+      debug: booleanSchema.default(false),
     })
     .default({}),
   postgres: z.object({
@@ -35,16 +40,19 @@ export const configSchema = z.object({
     connection: z.string(),
 
     // run all migrations on boot of the application
-    migrateOnBoot: z.coerce.boolean().default(false),
+    migrateOnBoot: booleanSchema.default(false),
 
     // try to sync the schema on boot, useful for development
     // will always keep the database schema in sync with the connected database
     // it is extremely destructive, do not use it EVER in production
-    syncSchema: z.coerce.boolean().default(false),
+    syncSchema: booleanSchema.default(false),
 
     // Enable debug logging for MikroORM - Outputs queries and entity management logs
     // Do NOT use in production, leaks all sensitive data
-    debugLogging: z.coerce.boolean().default(false),
+    debugLogging: booleanSchema.default(false),
+
+    // Enable SSL for the postgres connection
+    ssl: booleanSchema.default(false),
   }),
   crypto: z.object({
     // session secret. used for signing session tokens
@@ -59,7 +67,7 @@ export const configSchema = z.object({
   captcha: z
     .object({
       // enabled captchas on register
-      enabled: z.coerce.boolean().default(false),
+      enabled: booleanSchema.default(false),
 
       // captcha secret
       secret: z.string().min(1).optional(),
@@ -70,7 +78,7 @@ export const configSchema = z.object({
   ratelimits: z
     .object({
       // enabled captchas on register
-      enabled: z.coerce.boolean().default(false),
+      enabled: booleanSchema.default(false),
       redisUrl: z.string().optional(),
     })
     .default({}),

src/db/migrations/.snapshot-postgres.json

@@ -268,143 +268,6 @@
       "checks": [],
       "foreignKeys": {}
     },
-    {
-      "columns": {
-        "id": {
-          "name": "id",
-          "type": "uuid",
-          "unsigned": false,
-          "autoincrement": false,
-          "primary": false,
-          "nullable": false,
-          "mappedType": "uuid"
-        },
-        "tmdb_id": {
-          "name": "tmdb_id",
-          "type": "varchar(255)",
-          "unsigned": false,
-          "autoincrement": false,
-          "primary": false,
-          "nullable": false,
-          "mappedType": "string"
-        },
-        "type": {
-          "name": "type",
-          "type": "varchar(255)",
-          "unsigned": false,
-          "autoincrement": false,
-          "primary": false,
-          "nullable": false,
-          "mappedType": "string"
-        },
-        "title": {
-          "name": "title",
-          "type": "varchar(255)",
-          "unsigned": false,
-          "autoincrement": false,
-          "primary": false,
-          "nullable": false,
-          "mappedType": "string"
-        },
-        "season_id": {
-          "name": "season_id",
-          "type": "varchar(255)",
-          "unsigned": false,
-          "autoincrement": false,
-          "primary": false,
-          "nullable": true,
-          "mappedType": "string"
-        },
-        "episode_id": {
-          "name": "episode_id",
-          "type": "varchar(255)",
-          "unsigned": false,
-          "autoincrement": false,
-          "primary": false,
-          "nullable": true,
-          "mappedType": "string"
-        },
-        "created_at": {
-          "name": "created_at",
-          "type": "timestamptz(0)",
-          "unsigned": false,
-          "autoincrement": false,
-          "primary": false,
-          "nullable": false,
-          "length": 0,
-          "mappedType": "datetime"
-        },
-        "status": {
-          "name": "status",
-          "type": "varchar(255)",
-          "unsigned": false,
-          "autoincrement": false,
-          "primary": false,
-          "nullable": false,
-          "mappedType": "string"
-        },
-        "provider_id": {
-          "name": "provider_id",
-          "type": "varchar(255)",
-          "unsigned": false,
-          "autoincrement": false,
-          "primary": false,
-          "nullable": false,
-          "mappedType": "string"
-        },
-        "embed_id": {
-          "name": "embed_id",
-          "type": "varchar(255)",
-          "unsigned": false,
-          "autoincrement": false,
-          "primary": false,
-          "nullable": true,
-          "mappedType": "string"
-        },
-        "error_message": {
-          "name": "error_message",
-          "type": "text",
-          "unsigned": false,
-          "autoincrement": false,
-          "primary": false,
-          "nullable": true,
-          "mappedType": "text"
-        },
-        "full_error": {
-          "name": "full_error",
-          "type": "text",
-          "unsigned": false,
-          "autoincrement": false,
-          "primary": false,
-          "nullable": true,
-          "mappedType": "text"
-        },
-        "hostname": {
-          "name": "hostname",
-          "type": "varchar(255)",
-          "unsigned": false,
-          "autoincrement": false,
-          "primary": false,
-          "nullable": false,
-          "mappedType": "string"
-        }
-      },
-      "name": "provider_metrics",
-      "schema": "public",
-      "indexes": [
-        {
-          "keyName": "provider_metrics_pkey",
-          "columnNames": [
-            "id"
-          ],
-          "composite": false,
-          "primary": true,
-          "unique": true
-        }
-      ],
-      "checks": [],
-      "foreignKeys": {}
-    },
     {
       "columns": {
         "id": {
@@ -620,6 +483,15 @@
           "primary": false,
           "nullable": true,
           "mappedType": "string"
+        },
+        "proxy_urls": {
+          "name": "proxy_urls",
+          "type": "text[]",
+          "unsigned": false,
+          "autoincrement": false,
+          "primary": false,
+          "nullable": true,
+          "mappedType": "array"
         }
       },
       "name": "user_settings",

src/db/migrations/Migration20231221185725.ts

@@ -0,0 +1,13 @@
+import { Migration } from '@mikro-orm/migrations';
+
+export class Migration20231221185725 extends Migration {
+
+  async up(): Promise<void> {
+    this.addSql('drop table if exists "provider_metrics" cascade;');
+  }
+
+  async down(): Promise<void> {
+    this.addSql('create table "provider_metrics" ("id" uuid not null default null, "tmdb_id" varchar not null default null, "type" varchar not null default null, "title" varchar not null default null, "season_id" varchar null default null, "episode_id" varchar null default null, "created_at" timestamptz not null default null, "status" varchar not null default null, "provider_id" varchar not null default null, "embed_id" varchar null default null, "error_message" text null default null, "full_error" text null default null, "hostname" varchar not null default null, constraint "provider_metrics_pkey" primary key ("id"));');
+  }
+
+}

src/db/migrations/Migration20231229214215.ts

@@ -0,0 +1,13 @@
+import { Migration } from '@mikro-orm/migrations';
+
+export class Migration20231229214215 extends Migration {
+
+  async up(): Promise<void> {
+    this.addSql('alter table "user_settings" add column "proxy_urls" text[] null;');
+  }
+
+  async down(): Promise<void> {
+    this.addSql('alter table "user_settings" drop column "proxy_urls";');
+  }
+
+}

src/db/models/ProviderMetrics.ts

@@ -1,51 +0,0 @@
-import { Entity, PrimaryKey, Property } from '@mikro-orm/core';
-import { randomUUID } from 'crypto';
-
-export const status = {
-  failed: 'failed',
-  notfound: 'notfound',
-  success: 'success',
-} as const;
-type Status = keyof typeof status;
-
-@Entity({ tableName: 'provider_metrics' })
-export class ProviderMetric {
-  @PrimaryKey({ name: 'id', type: 'uuid' })
-  id: string = randomUUID();
-
-  @Property({ name: 'tmdb_id' })
-  tmdbId!: string;
-
-  @Property({ name: 'type' })
-  type!: string;
-
-  @Property({ name: 'title' })
-  title!: string;
-
-  @Property({ name: 'season_id', nullable: true })
-  seasonId?: string;
-
-  @Property({ name: 'episode_id', nullable: true })
-  episodeId?: string;
-
-  @Property({ name: 'created_at', type: 'date' })
-  createdAt = new Date();
-
-  @Property({ name: 'status' })
-  status!: Status;
-
-  @Property({ name: 'provider_id' })
-  providerId!: string;
-
-  @Property({ name: 'embed_id', nullable: true })
-  embedId?: string;
-
-  @Property({ name: 'error_message', nullable: true, type: 'text' })
-  errorMessage?: string;
-
-  @Property({ name: 'full_error', nullable: true, type: 'text' })
-  fullError?: string;
-
-  @Property({ name: 'hostname' })
-  hostname!: string;
-}

src/db/models/UserSettings.ts

@@ -1,4 +1,4 @@
-import { Entity, PrimaryKey, Property } from '@mikro-orm/core';
+import { ArrayType, Entity, PrimaryKey, Property } from '@mikro-orm/core';
 
 @Entity({ tableName: 'user_settings' })
 export class UserSettings {
@@ -13,6 +13,9 @@ export class UserSettings {
 
   @Property({ name: 'default_subtitle_language', nullable: true })
   defaultSubtitleLanguage?: string | null;
+
+  @Property({ name: 'proxy_urls', type: ArrayType, nullable: true })
+  proxyUrls?: string[] | null;
 }
 
 export interface UserSettingsDTO {
@@ -20,6 +23,7 @@ export interface UserSettingsDTO {
   applicationTheme?: string | null;
   applicationLanguage?: string | null;
   defaultSubtitleLanguage?: string | null;
+  proxyUrls?: string[] | null;
 }
 
 export function formatUserSettings(
@@ -30,5 +34,6 @@ export function formatUserSettings(
     applicationTheme: userSettings.applicationTheme,
     applicationLanguage: userSettings.applicationLanguage,
     defaultSubtitleLanguage: userSettings.defaultSubtitleLanguage,
+    proxyUrls: userSettings.proxyUrls,
   };
 }

src/mikro-orm.config.ts

@@ -1,4 +1,4 @@
 import { ormConf } from '@/config/orm';
 import { makeOrmConfig } from '@/modules/mikro/orm';
 
-export default makeOrmConfig(ormConf.postgres.connection);
+export default makeOrmConfig(ormConf.postgres.connection, ormConf.postgres.ssl);

src/modules/jobs/index.ts

@@ -1,11 +1,9 @@
 import { challengeCodeJob } from '@/modules/jobs/list/challengeCode';
 import { sessionExpiryJob } from '@/modules/jobs/list/sessionExpiry';
 import { userDeletionJob } from '@/modules/jobs/list/userDeletion';
-import { providerMetricCleanupJob } from '@/modules/jobs/list/providerMetricCleanup';
 
 export async function setupJobs() {
   challengeCodeJob.start();
   sessionExpiryJob.start();
   userDeletionJob.start();
-  providerMetricCleanupJob.start();
 }

src/modules/jobs/list/providerMetricCleanup.ts

@@ -1,27 +0,0 @@
-import { ProviderMetric } from '@/db/models/ProviderMetrics';
-import { job } from '@/modules/jobs/job';
-import ms from 'ms';
-
-// every day at 12:00:00
-export const providerMetricCleanupJob = job(
-  'provider-metric-cleanup',
-  '0 12 * * *',
-  async ({ em, log }) => {
-    const now = new Date();
-    const thirtyDaysAgo = new Date(now.getTime() - ms('30d'));
-
-    const deletedMetrics = await em
-      .createQueryBuilder(ProviderMetric)
-      .delete()
-      .where({
-        createdAt: {
-          $lt: thirtyDaysAgo,
-        },
-      })
-      .execute<{ affectedRows: number }>('run');
-
-    log.info(
-      `Removed ${deletedMetrics.affectedRows} metrics that were older than 30 days`,
-    );
-  },
-);

src/modules/metrics/index.ts

@@ -9,17 +9,11 @@ const log = scopedLogger('metrics');
 
 export type Metrics = {
   user: Counter<'namespace'>;
-  providerMetrics: Counter<
+  captchaSolves: Counter<'success'>;
-    | 'title'
+  providerHostnames: Counter<'hostname'>;
-    | 'tmdb_id'
+  providerStatuses: Counter<'provider_id' | 'status'>;
-    | 'season_id'
+  watchMetrics: Counter<'title' | 'tmdb_full_id' | 'provider_id' | 'success'>;
-    | 'episode_id'
+  toolMetrics: Counter<'tool'>;
-    | 'status'
-    | 'type'
-    | 'provider_id'
-    | 'embed_id'
-    | 'hostname'
-  >;
 };
 
 let metrics: null | Metrics = null;
@@ -42,31 +36,45 @@ export async function setupMetrics(app: FastifyInstance) {
 
   metrics = {
     user: new Counter({
-      name: 'user_count',
+      name: 'mw_user_count',
-      help: 'user_help',
+      help: 'mw_user_help',
       labelNames: ['namespace'],
     }),
-    providerMetrics: new Counter({
+    captchaSolves: new Counter({
-      name: 'provider_metrics',
+      name: 'mw_captcha_solves',
-      help: 'provider_metrics',
+      help: 'mw_captcha_solves',
-      labelNames: [
+      labelNames: ['success'],
-        'episode_id',
+    }),
-        'provider_id',
+    providerHostnames: new Counter({
-        'season_id',
+      name: 'mw_provider_hostname_count',
-        'status',
+      help: 'mw_provider_hostname_count',
-        'title',
+      labelNames: ['hostname'],
-        'tmdb_id',
+    }),
-        'type',
+    providerStatuses: new Counter({
-        'embed_id',
+      name: 'mw_provider_status_count',
-        'hostname',
+      help: 'mw_provider_status_count',
-      ],
+      labelNames: ['provider_id', 'status'],
+    }),
+    watchMetrics: new Counter({
+      name: 'mw_media_watch_count',
+      help: 'mw_media_watch_count',
+      labelNames: ['title', 'tmdb_full_id', 'provider_id', 'success'],
+    }),
+    toolMetrics: new Counter({
+      name: 'mw_provider_tool_count',
+      help: 'mw_provider_tool_count',
+      labelNames: ['tool'],
     }),
   };
 
   const promClient = app.metrics.client;
 
   promClient.register.registerMetric(metrics.user);
-  promClient.register.registerMetric(metrics.providerMetrics);
+  promClient.register.registerMetric(metrics.providerHostnames);
+  promClient.register.registerMetric(metrics.providerStatuses);
+  promClient.register.registerMetric(metrics.watchMetrics);
+  promClient.register.registerMetric(metrics.captchaSolves);
+  promClient.register.registerMetric(metrics.toolMetrics);
 
   const orm = getORM();
   const em = orm.em.fork();

src/modules/mikro/index.ts

@@ -18,6 +18,7 @@ export async function setupMikroORM() {
     conf.postgres.connection,
     conf.postgres.debugLogging,
     (msg) => log.info(msg),
+    conf.postgres.ssl,
   );
 
   if (conf.postgres.syncSchema) {

src/modules/mikro/orm.ts

@@ -2,7 +2,10 @@ import { Options } from '@mikro-orm/core';
 import { MikroORM, PostgreSqlDriver } from '@mikro-orm/postgresql';
 import path from 'path';
 
-export function makeOrmConfig(url: string): Options<PostgreSqlDriver> {
+export function makeOrmConfig(
+  url: string,
+  ssl: boolean,
+): Options<PostgreSqlDriver> {
   return {
     type: 'postgresql',
     clientUrl: url,
@@ -13,6 +16,11 @@ export function makeOrmConfig(url: string): Options<PostgreSqlDriver> {
       pathTs: './migrations',
       path: './migrations',
     },
+    driverOptions: {
+      connection: {
+        ssl,
+      },
+    },
   };
 }
 
@@ -20,9 +28,10 @@ export async function createORM(
   url: string,
   debug: boolean,
   log: (msg: string) => void,
+  ssl: boolean,
 ) {
   return await MikroORM.init<PostgreSqlDriver>({
-    ...makeOrmConfig(url),
+    ...makeOrmConfig(url, ssl),
     logger: log,
     debug,
   });

src/modules/ratelimits/limiter.ts

@@ -2,6 +2,7 @@ import Redis from 'ioredis';
 import RateLimiter from 'async-ratelimiter';
 import ms from 'ms';
 import { StatusError } from '@/services/error';
+import { IpReq, getIp } from '@/services/ip';
 
 export interface LimiterOptions {
   redis: Redis;
@@ -26,8 +27,8 @@ export class Limiter {
     this.redis = ops.redis;
   }
 
-  async bump(req: { ip: string }, ops: BucketOptions) {
+  async bump(req: IpReq, ops: BucketOptions) {
-    const ip = req.ip;
+    const ip = getIp(req);
     if (!this.buckets[ops.id]) {
       this.buckets[ops.id] = {
         limiter: new RateLimiter({
@@ -54,7 +55,7 @@ export class Limiter {
     };
   }
 
-  async assertAndBump(req: { ip: string }, ops: BucketOptions) {
+  async assertAndBump(req: IpReq, ops: BucketOptions) {
     const { hasBeenLimited } = await this.bump(req, ops);
     if (hasBeenLimited) {
       throw new StatusError('Ratelimited', 429);

src/routes/metrics.ts

@@ -1,8 +1,8 @@
 import { handle } from '@/services/handler';
 import { makeRouter } from '@/services/router';
 import { z } from 'zod';
-import { ProviderMetric, status } from '@/db/models/ProviderMetrics';
 import { getMetrics } from '@/modules/metrics';
+import { status } from '@/routes/statuses';
 
 const metricsProviderSchema = z.object({
   tmdbId: z.string(),
@@ -19,6 +19,7 @@ const metricsProviderSchema = z.object({
 
 const metricsProviderInputSchema = z.object({
   items: z.array(metricsProviderSchema).max(10).min(1),
+  tool: z.string().optional(),
 });
 
 export const metricsRouter = makeRouter((app) => {
@@ -29,7 +30,7 @@ export const metricsRouter = makeRouter((app) => {
         body: metricsProviderInputSchema,
       },
     },
-    handle(async ({ em, body, req, limiter }) => {
+    handle(async ({ body, req, limiter }) => {
       await limiter?.assertAndBump(req, {
         id: 'provider_metrics',
         max: 300,
@@ -37,43 +38,65 @@ export const metricsRouter = makeRouter((app) => {
         window: '30m',
       });
 
-      const hostname = req.headers.origin?.slice(0, 255) ?? 'unknown origin';
+      const hostname = req.headers.origin?.slice(0, 255) ?? '<UNKNOWN>';
-
+      getMetrics().providerHostnames.inc({
-      const entities = body.items.map((v) => {
-        const errorMessage = v.errorMessage?.slice(0, 200);
-        const truncatedFullError = v.fullError?.slice(0, 2000);
-
-        const metric = new ProviderMetric();
-        em.assign(metric, {
-          providerId: v.providerId,
-          embedId: v.embedId,
-          fullError: truncatedFullError,
-          errorMessage: errorMessage,
-          episodeId: v.episodeId,
-          seasonId: v.seasonId,
-          status: v.status,
-          title: v.title,
-          tmdbId: v.tmdbId,
-          type: v.type,
         hostname,
       });
-        return metric;
-      });
 
-      entities.forEach((entity) => {
+      body.items.forEach((item) => {
-        getMetrics().providerMetrics.inc({
+        getMetrics().providerStatuses.inc({
-          episode_id: entity.episodeId,
+          provider_id: item.embedId ?? item.providerId,
-          provider_id: entity.providerId,
+          status: item.status,
-          season_id: entity.seasonId,
-          status: entity.status,
-          title: entity.title,
-          tmdb_id: entity.tmdbId,
-          type: entity.type,
-          hostname,
         });
       });
 
-      await em.persistAndFlush(entities);
+      const itemList = [...body.items];
+      itemList.reverse();
+      const lastSuccessfulItem = body.items.find(
+        (v) => v.status === status.success,
+      );
+      const lastItem = itemList[0];
+
+      if (lastItem) {
+        getMetrics().watchMetrics.inc({
+          tmdb_full_id: lastItem.type + '-' + lastItem.tmdbId,
+          provider_id: lastSuccessfulItem?.providerId ?? lastItem.providerId,
+          title: lastItem.title,
+          success: (!!lastSuccessfulItem).toString(),
+        });
+      }
+
+      if (body.tool) {
+        getMetrics().toolMetrics.inc({
+          tool: body.tool,
+        });
+      }
+
+      return true;
+    }),
+  );
+
+  app.post(
+    '/metrics/captcha',
+    {
+      schema: {
+        body: z.object({
+          success: z.boolean(),
+        }),
+      },
+    },
+    handle(async ({ body, req, limiter }) => {
+      await limiter?.assertAndBump(req, {
+        id: 'captcha_solves',
+        max: 300,
+        inc: 1,
+        window: '30m',
+      });
+
+      getMetrics().captchaSolves.inc({
+        success: body.success.toString(),
+      });
+
       return true;
     }),
   );

src/routes/statuses.ts

@@ -0,0 +1,6 @@
+export const status = {
+  failed: 'failed',
+  notfound: 'notfound',
+  success: 'success',
+} as const;
+export type Status = keyof typeof status;

src/routes/users/bookmark.ts

@@ -6,7 +6,6 @@ import {
 import { StatusError } from '@/services/error';
 import { handle } from '@/services/handler';
 import { makeRouter } from '@/services/router';
-import { randomUUID } from 'crypto';
 import { z } from 'zod';
 
 const bookmarkDataSchema = z.object({

src/routes/users/progress.ts

@@ -6,6 +6,7 @@ import {
 import { StatusError } from '@/services/error';
 import { handle } from '@/services/handler';
 import { makeRouter } from '@/services/router';
+import { FilterQuery } from '@mikro-orm/core';
 import { randomUUID } from 'crypto';
 import { z } from 'zod';
 
@@ -18,6 +19,7 @@ const progressItemSchema = z.object({
   episodeId: z.string().optional(),
   seasonNumber: z.number().optional(),
   episodeNumber: z.number().optional(),
+  updatedAt: z.string().datetime({ offset: true }).optional(),
 });
 
 export const userProgressRouter = makeRouter((app) => {
@@ -100,7 +102,9 @@ export const userProgressRouter = makeRouter((app) => {
         if (newItemIndex > -1) {
           const newItem = newItems[newItemIndex];
           if (existingItem.watched < newItem.watched) {
-            existingItem.updatedAt = new Date();
+            existingItem.updatedAt = defaultAndCoerceDateTime(
+              newItem.updatedAt,
+            );
             existingItem.watched = newItem.watched;
           }
           itemsUpserted.push(existingItem);
@@ -123,7 +127,7 @@ export const userProgressRouter = makeRouter((app) => {
           tmdbId: newItem.tmdbId,
           userId: params.uid,
           watched: newItem.watched,
-          updatedAt: new Date(),
+          updatedAt: defaultAndCoerceDateTime(newItem.updatedAt),
         });
       }
 
@@ -161,22 +165,28 @@ export const userProgressRouter = makeRouter((app) => {
       if (auth.user.id !== params.uid)
         throw new StatusError('Cannot modify user other than yourself', 403);
 
-      const progressItem = await em.findOne(ProgressItem, {
+      const query: FilterQuery<ProgressItem> = {
         userId: params.uid,
         tmdbId: params.tmdbid,
-        episodeId: body.episodeId,
+      };
-        seasonId: body.seasonId,
+      if (body.seasonId) query.seasonId = body.seasonId;
-      });
+      if (body.episodeId) query.episodeId = body.episodeId;
-      if (!progressItem) {
+      const progressItems = await em.find(ProgressItem, query);
+
+      if (progressItems.length === 0) {
         return {
+          count: 0,
           tmdbId: params.tmdbid,
           episodeId: body.episodeId,
           seasonId: body.seasonId,
         };
       }
 
-      await em.removeAndFlush(progressItem);
+      progressItems.forEach((v) => em.remove(v));
+      await em.flush();
+
       return {
+        count: progressItems.length,
         tmdbId: params.tmdbid,
         episodeId: body.episodeId,
         seasonId: body.seasonId,
@@ -207,3 +217,14 @@ export const userProgressRouter = makeRouter((app) => {
     }),
   );
 });
+
+// 13th July 2021 - movie-web epoch
+const minEpoch = 1626134400000;
+
+function defaultAndCoerceDateTime(dateTime: string | undefined) {
+  const epoch = dateTime ? new Date(dateTime).getTime() : Date.now();
+
+  const clampedEpoch = Math.max(minEpoch, Math.min(epoch, Date.now()));
+
+  return new Date(clampedEpoch);
+}

src/routes/users/settings.ts

@@ -41,6 +41,7 @@ export const userSettingsRouter = makeRouter((app) => {
           applicationLanguage: z.string().nullable().optional(),
           applicationTheme: z.string().nullable().optional(),
           defaultSubtitleLanguage: z.string().nullable().optional(),
+          proxyUrls: z.string().array().nullable().optional(),
         }),
       },
     },
@@ -64,6 +65,7 @@ export const userSettingsRouter = makeRouter((app) => {
         settings.defaultSubtitleLanguage = body.defaultSubtitleLanguage;
       if (body.applicationTheme !== undefined)
         settings.applicationTheme = body.applicationTheme;
+      if (body.proxyUrls !== undefined) settings.proxyUrls = body.proxyUrls;
 
       await em.persistAndFlush(settings);
       return formatUserSettings(settings);

src/services/captcha.ts

@@ -4,16 +4,14 @@ import { StatusError } from '@/services/error';
 export async function isValidCaptcha(token: string): Promise<boolean> {
   if (!conf.captcha.secret)
     throw new Error('isValidCaptcha() is called but no secret set');
+  const formData = new URLSearchParams();
+  formData.append('secret', conf.captcha.secret);
+  formData.append('response', token);
   const res = await fetch('https://www.google.com/recaptcha/api/siteverify', {
     method: 'POST',
-    body: JSON.stringify({
+    body: formData,
-      secret: conf.captcha.secret,
-      response: token,
-    }),
-    headers: {
-      'content-type': 'application/json',
-    },
   });
+
   const json = await res.json();
   return !!json.success;
 }

src/services/ip.ts

@@ -0,0 +1,27 @@
+import { conf } from '@/config';
+import { IncomingHttpHeaders } from 'http';
+
+export type IpReq = {
+  ip: string;
+  headers: IncomingHttpHeaders;
+};
+
+const trustCloudflare = conf.server.trustCloudflare;
+
+function getSingleHeader(
+  headers: IncomingHttpHeaders,
+  key: string,
+): string | undefined {
+  const header = headers[key];
+  if (Array.isArray(header)) return header[0];
+  return header;
+}
+
+export function getIp(req: IpReq) {
+  const cfIp = getSingleHeader(req.headers, 'cf-connecting-ip');
+  if (trustCloudflare && cfIp) {
+    return cfIp;
+  }
+
+  return req.ip;
+}