Merge pull request #26 from movie-web/dev

Backend v1.2.0

.docker/development/docker-compose.yml

@@ -12,6 +12,8 @@ services:
       POSTGRES_PASSWORD: postgres
     volumes:
      - 'postgres_data:/var/lib/postgresql/data'
+  redis:
+    image: redis
 
   # custom services
   backend:

package.json

@@ -1,6 +1,6 @@
 {
   "name": "backend",
-  "version": "1.1.4",
+  "version": "1.2.0",
   "private": true,
   "homepage": "https://github.com/movie-web/backend",
   "engines": {

src/config/fragments/docker.ts

@@ -4,4 +4,8 @@ export const dockerFragment: FragmentSchema = {
   postgres: {
     connection: 'postgres://postgres:postgres@postgres:5432/postgres',
   },
+  ratelimits: {
+    enabled: true,
+    redisUrl: 'redis://redis:6379',
+  },
 };

src/config/orm.ts

@@ -1,6 +1,13 @@
+import { devFragment } from '@/config/fragments/dev';
+import { dockerFragment } from '@/config/fragments/docker';
 import { createConfigLoader } from 'neat-config';
 import { z } from 'zod';
 
+const fragments = {
+  dev: devFragment,
+  dockerdev: dockerFragment,
+};
+
 export const ormConfigSchema = z.object({
   postgres: z.object({
     // connection URL for postgres database
@@ -15,6 +22,8 @@ export const ormConf = createConfigLoader()
     prefix: 'MWB_',
   })
   .addFromFile('config.json')
+  .setFragmentKey('usePresets')
+  .addConfigFragments(fragments)
   .addZodSchema(ormConfigSchema)
   .freeze()
   .load();

src/config/schema.ts

@@ -16,6 +16,9 @@ export const configSchema = z.object({
       // should it trust reverse proxy headers? (for ip gathering)
       trustProxy: z.coerce.boolean().default(false),
 
+      // should it trust cloudflare headers? (for ip gathering, cloudflare has priority)
+      trustCloudflare: z.coerce.boolean().default(false),
+
       // prefix for where the instance is run on. for example set it to /backend if you're hosting it on example.com/backend
       // if this is set, do not apply url rewriting before proxing
       basePath: z.string().default('/'),

src/db/migrations/.snapshot-postgres.json

@@ -268,143 +268,6 @@
       "checks": [],
       "foreignKeys": {}
     },
-    {
-      "columns": {
-        "id": {
-          "name": "id",
-          "type": "uuid",
-          "unsigned": false,
-          "autoincrement": false,
-          "primary": false,
-          "nullable": false,
-          "mappedType": "uuid"
-        },
-        "tmdb_id": {
-          "name": "tmdb_id",
-          "type": "varchar(255)",
-          "unsigned": false,
-          "autoincrement": false,
-          "primary": false,
-          "nullable": false,
-          "mappedType": "string"
-        },
-        "type": {
-          "name": "type",
-          "type": "varchar(255)",
-          "unsigned": false,
-          "autoincrement": false,
-          "primary": false,
-          "nullable": false,
-          "mappedType": "string"
-        },
-        "title": {
-          "name": "title",
-          "type": "varchar(255)",
-          "unsigned": false,
-          "autoincrement": false,
-          "primary": false,
-          "nullable": false,
-          "mappedType": "string"
-        },
-        "season_id": {
-          "name": "season_id",
-          "type": "varchar(255)",
-          "unsigned": false,
-          "autoincrement": false,
-          "primary": false,
-          "nullable": true,
-          "mappedType": "string"
-        },
-        "episode_id": {
-          "name": "episode_id",
-          "type": "varchar(255)",
-          "unsigned": false,
-          "autoincrement": false,
-          "primary": false,
-          "nullable": true,
-          "mappedType": "string"
-        },
-        "created_at": {
-          "name": "created_at",
-          "type": "timestamptz(0)",
-          "unsigned": false,
-          "autoincrement": false,
-          "primary": false,
-          "nullable": false,
-          "length": 0,
-          "mappedType": "datetime"
-        },
-        "status": {
-          "name": "status",
-          "type": "varchar(255)",
-          "unsigned": false,
-          "autoincrement": false,
-          "primary": false,
-          "nullable": false,
-          "mappedType": "string"
-        },
-        "provider_id": {
-          "name": "provider_id",
-          "type": "varchar(255)",
-          "unsigned": false,
-          "autoincrement": false,
-          "primary": false,
-          "nullable": false,
-          "mappedType": "string"
-        },
-        "embed_id": {
-          "name": "embed_id",
-          "type": "varchar(255)",
-          "unsigned": false,
-          "autoincrement": false,
-          "primary": false,
-          "nullable": true,
-          "mappedType": "string"
-        },
-        "error_message": {
-          "name": "error_message",
-          "type": "text",
-          "unsigned": false,
-          "autoincrement": false,
-          "primary": false,
-          "nullable": true,
-          "mappedType": "text"
-        },
-        "full_error": {
-          "name": "full_error",
-          "type": "text",
-          "unsigned": false,
-          "autoincrement": false,
-          "primary": false,
-          "nullable": true,
-          "mappedType": "text"
-        },
-        "hostname": {
-          "name": "hostname",
-          "type": "varchar(255)",
-          "unsigned": false,
-          "autoincrement": false,
-          "primary": false,
-          "nullable": false,
-          "mappedType": "string"
-        }
-      },
-      "name": "provider_metrics",
-      "schema": "public",
-      "indexes": [
-        {
-          "keyName": "provider_metrics_pkey",
-          "columnNames": [
-            "id"
-          ],
-          "composite": false,
-          "primary": true,
-          "unique": true
-        }
-      ],
-      "checks": [],
-      "foreignKeys": {}
-    },
     {
       "columns": {
         "id": {

src/db/migrations/Migration20231221185725.ts

@@ -0,0 +1,13 @@
+import { Migration } from '@mikro-orm/migrations';
+
+export class Migration20231221185725 extends Migration {
+
+  async up(): Promise<void> {
+    this.addSql('drop table if exists "provider_metrics" cascade;');
+  }
+
+  async down(): Promise<void> {
+    this.addSql('create table "provider_metrics" ("id" uuid not null default null, "tmdb_id" varchar not null default null, "type" varchar not null default null, "title" varchar not null default null, "season_id" varchar null default null, "episode_id" varchar null default null, "created_at" timestamptz not null default null, "status" varchar not null default null, "provider_id" varchar not null default null, "embed_id" varchar null default null, "error_message" text null default null, "full_error" text null default null, "hostname" varchar not null default null, constraint "provider_metrics_pkey" primary key ("id"));');
+  }
+
+}

src/db/models/ProviderMetrics.ts

@@ -1,51 +0,0 @@
-import { Entity, PrimaryKey, Property } from '@mikro-orm/core';
-import { randomUUID } from 'crypto';
-
-export const status = {
-  failed: 'failed',
-  notfound: 'notfound',
-  success: 'success',
-} as const;
-type Status = keyof typeof status;
-
-@Entity({ tableName: 'provider_metrics' })
-export class ProviderMetric {
-  @PrimaryKey({ name: 'id', type: 'uuid' })
-  id: string = randomUUID();
-
-  @Property({ name: 'tmdb_id' })
-  tmdbId!: string;
-
-  @Property({ name: 'type' })
-  type!: string;
-
-  @Property({ name: 'title' })
-  title!: string;
-
-  @Property({ name: 'season_id', nullable: true })
-  seasonId?: string;
-
-  @Property({ name: 'episode_id', nullable: true })
-  episodeId?: string;
-
-  @Property({ name: 'created_at', type: 'date' })
-  createdAt = new Date();
-
-  @Property({ name: 'status' })
-  status!: Status;
-
-  @Property({ name: 'provider_id' })
-  providerId!: string;
-
-  @Property({ name: 'embed_id', nullable: true })
-  embedId?: string;
-
-  @Property({ name: 'error_message', nullable: true, type: 'text' })
-  errorMessage?: string;
-
-  @Property({ name: 'full_error', nullable: true, type: 'text' })
-  fullError?: string;
-
-  @Property({ name: 'hostname' })
-  hostname!: string;
-}

src/modules/jobs/index.ts

@@ -1,11 +1,9 @@
 import { challengeCodeJob } from '@/modules/jobs/list/challengeCode';
 import { sessionExpiryJob } from '@/modules/jobs/list/sessionExpiry';
 import { userDeletionJob } from '@/modules/jobs/list/userDeletion';
-import { providerMetricCleanupJob } from '@/modules/jobs/list/providerMetricCleanup';
 
 export async function setupJobs() {
   challengeCodeJob.start();
   sessionExpiryJob.start();
   userDeletionJob.start();
-  providerMetricCleanupJob.start();
 }

src/modules/jobs/list/providerMetricCleanup.ts

@@ -1,27 +0,0 @@
-import { ProviderMetric } from '@/db/models/ProviderMetrics';
-import { job } from '@/modules/jobs/job';
-import ms from 'ms';
-
-// every day at 12:00:00
-export const providerMetricCleanupJob = job(
-  'provider-metric-cleanup',
-  '0 12 * * *',
-  async ({ em, log }) => {
-    const now = new Date();
-    const thirtyDaysAgo = new Date(now.getTime() - ms('30d'));
-
-    const deletedMetrics = await em
-      .createQueryBuilder(ProviderMetric)
-      .delete()
-      .where({
-        createdAt: {
-          $lt: thirtyDaysAgo,
-        },
-      })
-      .execute<{ affectedRows: number }>('run');
-
-    log.info(
-      `Removed ${deletedMetrics.affectedRows} metrics that were older than 30 days`,
-    );
-  },
-);

src/modules/metrics/index.ts

@@ -9,17 +9,10 @@ const log = scopedLogger('metrics');
 
 export type Metrics = {
   user: Counter<'namespace'>;
-  providerMetrics: Counter<
+  captchaSolves: Counter<'success'>;
-    | 'title'
+  providerHostnames: Counter<'hostname'>;
-    | 'tmdb_id'
+  providerStatuses: Counter<'provider_id' | 'status'>;
-    | 'season_id'
+  watchMetrics: Counter<'title' | 'tmdb_full_id' | 'provider_id' | 'success'>;
-    | 'episode_id'
-    | 'status'
-    | 'type'
-    | 'provider_id'
-    | 'embed_id'
-    | 'hostname'
-  >;
 };
 
 let metrics: null | Metrics = null;
@@ -42,31 +35,38 @@ export async function setupMetrics(app: FastifyInstance) {
 
   metrics = {
     user: new Counter({
-      name: 'user_count',
+      name: 'mw_user_count',
-      help: 'user_help',
+      help: 'mw_user_help',
       labelNames: ['namespace'],
     }),
-    providerMetrics: new Counter({
+    captchaSolves: new Counter({
-      name: 'provider_metrics',
+      name: 'mw_captcha_solves',
-      help: 'provider_metrics',
+      help: 'mw_captcha_solves',
-      labelNames: [
+      labelNames: ['success'],
-        'episode_id',
+    }),
-        'provider_id',
+    providerHostnames: new Counter({
-        'season_id',
+      name: 'mw_provider_hostname_count',
-        'status',
+      help: 'mw_provider_hostname_count',
-        'title',
+      labelNames: ['hostname'],
-        'tmdb_id',
+    }),
-        'type',
+    providerStatuses: new Counter({
-        'embed_id',
+      name: 'mw_provider_status_count',
-        'hostname',
+      help: 'mw_provider_status_count',
-      ],
+      labelNames: ['provider_id', 'status'],
+    }),
+    watchMetrics: new Counter({
+      name: 'mw_media_watch_count',
+      help: 'mw_media_watch_count',
+      labelNames: ['title', 'tmdb_full_id', 'provider_id', 'success'],
     }),
   };
 
   const promClient = app.metrics.client;
 
   promClient.register.registerMetric(metrics.user);
-  promClient.register.registerMetric(metrics.providerMetrics);
+  promClient.register.registerMetric(metrics.providerHostnames);
+  promClient.register.registerMetric(metrics.providerStatuses);
+  promClient.register.registerMetric(metrics.watchMetrics);
 
   const orm = getORM();
   const em = orm.em.fork();

src/modules/ratelimits/limiter.ts

@@ -2,6 +2,7 @@ import Redis from 'ioredis';
 import RateLimiter from 'async-ratelimiter';
 import ms from 'ms';
 import { StatusError } from '@/services/error';
+import { IpReq, getIp } from '@/services/ip';
 
 export interface LimiterOptions {
   redis: Redis;
@@ -26,8 +27,8 @@ export class Limiter {
     this.redis = ops.redis;
   }
 
-  async bump(req: { ip: string }, ops: BucketOptions) {
+  async bump(req: IpReq, ops: BucketOptions) {
-    const ip = req.ip;
+    const ip = getIp(req);
     if (!this.buckets[ops.id]) {
       this.buckets[ops.id] = {
         limiter: new RateLimiter({
@@ -54,7 +55,7 @@ export class Limiter {
     };
   }
 
-  async assertAndBump(req: { ip: string }, ops: BucketOptions) {
+  async assertAndBump(req: IpReq, ops: BucketOptions) {
     const { hasBeenLimited } = await this.bump(req, ops);
     if (hasBeenLimited) {
       throw new StatusError('Ratelimited', 429);

src/routes/metrics.ts

@@ -1,8 +1,8 @@
 import { handle } from '@/services/handler';
 import { makeRouter } from '@/services/router';
 import { z } from 'zod';
-import { ProviderMetric, status } from '@/db/models/ProviderMetrics';
 import { getMetrics } from '@/modules/metrics';
+import { status } from '@/routes/statuses';
 
 const metricsProviderSchema = z.object({
   tmdbId: z.string(),
@@ -29,7 +29,7 @@ export const metricsRouter = makeRouter((app) => {
         body: metricsProviderInputSchema,
       },
     },
-    handle(async ({ em, body, req, limiter }) => {
+    handle(async ({ body, req, limiter }) => {
       await limiter?.assertAndBump(req, {
         id: 'provider_metrics',
         max: 300,
@@ -37,43 +37,59 @@ export const metricsRouter = makeRouter((app) => {
         window: '30m',
       });
 
-      const hostname = req.headers.origin?.slice(0, 255) ?? 'unknown origin';
+      const hostname = req.headers.origin?.slice(0, 255) ?? '<UNKNOWN>';
-
+      getMetrics().providerHostnames.inc({
-      const entities = body.items.map((v) => {
-        const errorMessage = v.errorMessage?.slice(0, 200);
-        const truncatedFullError = v.fullError?.slice(0, 2000);
-
-        const metric = new ProviderMetric();
-        em.assign(metric, {
-          providerId: v.providerId,
-          embedId: v.embedId,
-          fullError: truncatedFullError,
-          errorMessage: errorMessage,
-          episodeId: v.episodeId,
-          seasonId: v.seasonId,
-          status: v.status,
-          title: v.title,
-          tmdbId: v.tmdbId,
-          type: v.type,
         hostname,
       });
-        return metric;
-      });
 
-      entities.forEach((entity) => {
+      body.items.forEach((item) => {
-        getMetrics().providerMetrics.inc({
+        getMetrics().providerStatuses.inc({
-          episode_id: entity.episodeId,
+          provider_id: item.embedId ?? item.providerId,
-          provider_id: entity.providerId,
+          status: item.status,
-          season_id: entity.seasonId,
-          status: entity.status,
-          title: entity.title,
-          tmdb_id: entity.tmdbId,
-          type: entity.type,
-          hostname,
         });
       });
 
-      await em.persistAndFlush(entities);
+      const itemList = [...body.items];
+      itemList.reverse();
+      const lastSuccessfulItem = body.items.find(
+        (v) => v.status === status.success,
+      );
+      const lastItem = itemList[0];
+
+      if (lastItem) {
+        getMetrics().watchMetrics.inc({
+          tmdb_full_id: lastItem.type + '-' + lastItem.tmdbId,
+          provider_id: lastSuccessfulItem?.providerId ?? lastItem.providerId,
+          title: lastItem.title,
+          success: (!!lastSuccessfulItem).toString(),
+        });
+      }
+
+      return true;
+    }),
+  );
+
+  app.post(
+    '/metrics/captcha',
+    {
+      schema: {
+        body: z.object({
+          success: z.boolean(),
+        }),
+      },
+    },
+    handle(async ({ body, req, limiter }) => {
+      await limiter?.assertAndBump(req, {
+        id: 'captcha_solves',
+        max: 300,
+        inc: 1,
+        window: '30m',
+      });
+
+      getMetrics().captchaSolves.inc({
+        success: body.success.toString(),
+      });
+
       return true;
     }),
   );

src/routes/statuses.ts

@@ -0,0 +1,6 @@
+export const status = {
+  failed: 'failed',
+  notfound: 'notfound',
+  success: 'success',
+} as const;
+export type Status = keyof typeof status;

src/routes/users/bookmark.ts

@@ -6,7 +6,6 @@ import {
 import { StatusError } from '@/services/error';
 import { handle } from '@/services/handler';
 import { makeRouter } from '@/services/router';
-import { randomUUID } from 'crypto';
 import { z } from 'zod';
 
 const bookmarkDataSchema = z.object({

src/routes/users/progress.ts

@@ -6,6 +6,7 @@ import {
 import { StatusError } from '@/services/error';
 import { handle } from '@/services/handler';
 import { makeRouter } from '@/services/router';
+import { FilterQuery } from '@mikro-orm/core';
 import { randomUUID } from 'crypto';
 import { z } from 'zod';
 
@@ -18,6 +19,7 @@ const progressItemSchema = z.object({
   episodeId: z.string().optional(),
   seasonNumber: z.number().optional(),
   episodeNumber: z.number().optional(),
+  updatedAt: z.string().datetime({ offset: true }).optional(),
 });
 
 export const userProgressRouter = makeRouter((app) => {
@@ -100,7 +102,9 @@ export const userProgressRouter = makeRouter((app) => {
         if (newItemIndex > -1) {
           const newItem = newItems[newItemIndex];
           if (existingItem.watched < newItem.watched) {
-            existingItem.updatedAt = new Date();
+            existingItem.updatedAt = defaultAndCoerceDateTime(
+              newItem.updatedAt,
+            );
             existingItem.watched = newItem.watched;
           }
           itemsUpserted.push(existingItem);
@@ -123,7 +127,7 @@ export const userProgressRouter = makeRouter((app) => {
           tmdbId: newItem.tmdbId,
           userId: params.uid,
           watched: newItem.watched,
-          updatedAt: new Date(),
+          updatedAt: defaultAndCoerceDateTime(newItem.updatedAt),
         });
       }
 
@@ -161,22 +165,28 @@ export const userProgressRouter = makeRouter((app) => {
       if (auth.user.id !== params.uid)
         throw new StatusError('Cannot modify user other than yourself', 403);
 
-      const progressItem = await em.findOne(ProgressItem, {
+      const query: FilterQuery<ProgressItem> = {
         userId: params.uid,
         tmdbId: params.tmdbid,
-        episodeId: body.episodeId,
+      };
-        seasonId: body.seasonId,
+      if (body.seasonId) query.seasonId = body.seasonId;
-      });
+      if (body.episodeId) query.episodeId = body.episodeId;
-      if (!progressItem) {
+      const progressItems = await em.find(ProgressItem, query);
+
+      if (progressItems.length === 0) {
         return {
+          count: 0,
           tmdbId: params.tmdbid,
           episodeId: body.episodeId,
           seasonId: body.seasonId,
         };
       }
 
-      await em.removeAndFlush(progressItem);
+      progressItems.forEach((v) => em.remove(v));
+      await em.flush();
+
       return {
+        count: progressItems.length,
         tmdbId: params.tmdbid,
         episodeId: body.episodeId,
         seasonId: body.seasonId,
@@ -207,3 +217,14 @@ export const userProgressRouter = makeRouter((app) => {
     }),
   );
 });
+
+// 13th July 2021 - movie-web epoch
+const minEpoch = 1626134400000;
+
+function defaultAndCoerceDateTime(dateTime: string | undefined) {
+  const epoch = dateTime ? new Date(dateTime).getTime() : Date.now();
+
+  const clampedEpoch = Math.max(minEpoch, Math.min(epoch, Date.now()));
+
+  return new Date(clampedEpoch);
+}

src/services/ip.ts

@@ -0,0 +1,27 @@
+import { conf } from '@/config';
+import { IncomingHttpHeaders } from 'http';
+
+export type IpReq = {
+  ip: string;
+  headers: IncomingHttpHeaders;
+};
+
+const trustCloudflare = conf.server.trustCloudflare;
+
+function getSingleHeader(
+  headers: IncomingHttpHeaders,
+  key: string,
+): string | undefined {
+  const header = headers[key];
+  if (Array.isArray(header)) return header[0];
+  return header;
+}
+
+export function getIp(req: IpReq) {
+  const cfIp = getSingleHeader(req.headers, 'cf-connecting-ip');
+  if (trustCloudflare && cfIp) {
+    return cfIp;
+  }
+
+  return req.ip;
+}