Merge pull request #39 from movie-web/dev

Backend update v1.3.1

.docker/development/docker-compose.yml

@@ -12,6 +12,8 @@ services:
       POSTGRES_PASSWORD: postgres
     volumes:
      - 'postgres_data:/var/lib/postgresql/data'
+  redis:
+    image: redis
 
   # custom services
   backend:
@@ -39,7 +41,7 @@ services:
     links: 
       - postgres:postgres
     environment:
-      - DATABASE_URL=postgres://postgres:postgres@postgres:5432/postgres?sslmode=disable
+      - PGWEB_DATABASE_URL=postgres://postgres:postgres@postgres:5432/postgres?sslmode=disable
     depends_on:
       - postgres
 

.eslintrc.js

@@ -13,7 +13,7 @@ module.exports = {
     sourceType: 'module',
   },
   plugins: ['@typescript-eslint'],
-  ignorePatterns: ['./src/db/migrations/**/*'],
+  ignorePatterns: ['src/db/migrations/**/*'],
   rules: {
     '@typescript-eslint/interface-name-prefix': 'off',
     '@typescript-eslint/explicit-function-return-type': 'off',

.github/CODEOWNERS

@@ -1,3 +1 @@
-* @movie-web/core
+* @movie-web/project-leads
-
-.github @binaryoverload

.github/workflows/linting_testing.yml

@@ -14,16 +14,16 @@ jobs:
 
     steps:
       - name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
                   
       - uses: pnpm/action-setup@v2
         with:
           version: 8
 
       - name: Install Node.js
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
-          node-version: 18
+          node-version: 20
           cache: 'pnpm'
       
       - name: Install packages
@@ -38,16 +38,16 @@ jobs:
 
     steps:
       - name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
                   
       - uses: pnpm/action-setup@v2
         with:
           version: 8
 
       - name: Install Node.js
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
-          node-version: 18
+          node-version: 20
           cache: 'pnpm'
       
       - name: Install packages
@@ -62,10 +62,10 @@ jobs:
 
     steps:
       - name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
 
       - name: Build
-        uses: docker/build-push-action@v4
+        uses: docker/build-push-action@v5

.github/workflows/release.yml

@@ -12,7 +12,7 @@ jobs:
 
     steps:
     - name: Checkout code
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     - name: Get version
       id: package-version
@@ -42,10 +42,10 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Setup Docker buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
 
       - name: Get version
         id: package-version
@@ -70,9 +70,12 @@ jobs:
 
       - name: Build and push Docker image
         id: build-and-push
-        uses: docker/build-push-action@v4
+        uses: docker/build-push-action@v5
         with:
           push: true
+          platforms: linux/amd64,linux/arm64
           context: .
           labels: ${{ steps.meta.outputs.labels }}
           tags: ${{ steps.meta.outputs.tags }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

package.json

@@ -1,6 +1,6 @@
 {
   "name": "backend",
-  "version": "1.0.3",
+  "version": "1.3.1",
   "private": true,
   "homepage": "https://github.com/movie-web/backend",
   "engines": {
@@ -8,16 +8,16 @@
   },
   "scripts": {
     "dev": "nodemon -r tsconfig-paths/register src/main.ts",
-    "build": "npm run build:pre && npm run build:compile",
+    "build": "pnpm run build:pre && pnpm run build:compile",
     "start": "node dist/main.js",
     "lint": "eslint --ext .ts,.js,.json,.tsx src/",
     "lint:fix": "eslint --fix --ext .ts,.js,.json,.tsx src/",
     "build:pre": "rimraf dist/",
     "build:compile": "tsc && tsc-alias",
     "preinstall": "npx -y only-allow pnpm",
-    "migration:create": "npx -y mikro-orm migration:create",
+    "migration:create": "pnpm exec mikro-orm migration:create",
-    "migration:up": "npx -y mikro-orm migration:up",
+    "migration:up": "pnpm exec mikro-orm migration:up",
-    "migration:down": "npx -y mikro-orm migration:down"
+    "migration:down": "pnpm exec mikro-orm migration:down"
   },
   "mikro-orm": {
     "useTsNode": true,

src/config/fragments/docker.ts

@@ -4,4 +4,8 @@ export const dockerFragment: FragmentSchema = {
   postgres: {
     connection: 'postgres://postgres:postgres@postgres:5432/postgres',
   },
+  ratelimits: {
+    enabled: true,
+    redisUrl: 'redis://redis:6379',
+  },
 };

src/config/index.ts

@@ -8,7 +8,7 @@ const fragments = {
   dockerdev: dockerFragment,
 };
 
-export const version = '1.0.3';
+export const version = process.env.npm_package_version ?? 'unknown';
 
 export const conf = createConfigLoader()
   .addFromEnvironment('MWB_')

src/config/orm.ts

@@ -1,10 +1,20 @@
+import { devFragment } from '@/config/fragments/dev';
+import { dockerFragment } from '@/config/fragments/docker';
 import { createConfigLoader } from 'neat-config';
 import { z } from 'zod';
+import { booleanSchema } from './schema';
+
+const fragments = {
+  dev: devFragment,
+  dockerdev: dockerFragment,
+};
 
 export const ormConfigSchema = z.object({
   postgres: z.object({
     // connection URL for postgres database
     connection: z.string(),
+    // whether to use SSL for the connection
+    ssl: booleanSchema.default(false),
   }),
 });
 
@@ -15,6 +25,8 @@ export const ormConf = createConfigLoader()
     prefix: 'MWB_',
   })
   .addFromFile('config.json')
+  .setFragmentKey('usePresets')
+  .addConfigFragments(fragments)
   .addZodSchema(ormConfigSchema)
   .freeze()
   .load();

src/config/schema.ts

@@ -1,5 +1,7 @@
 import { z } from 'zod';
 
+export const booleanSchema = z.preprocess((val) => val === 'true', z.boolean());
+
 export const configSchema = z.object({
   server: z
     .object({
@@ -11,10 +13,13 @@ export const configSchema = z.object({
 
       // disable cross origin restrictions, allow any site.
       // overwrites the cors option above
-      allowAnySite: z.coerce.boolean().default(false),
+      allowAnySite: booleanSchema.default(false),
 
       // should it trust reverse proxy headers? (for ip gathering)
-      trustProxy: z.coerce.boolean().default(false),
+      trustProxy: booleanSchema.default(false),
+
+      // should it trust cloudflare headers? (for ip gathering, cloudflare has priority)
+      trustCloudflare: booleanSchema.default(false),
 
       // prefix for where the instance is run on. for example set it to /backend if you're hosting it on example.com/backend
       // if this is set, do not apply url rewriting before proxing
@@ -27,7 +32,7 @@ export const configSchema = z.object({
       format: z.enum(['json', 'pretty']).default('pretty'),
 
       // show debug logs?
-      debug: z.coerce.boolean().default(false),
+      debug: booleanSchema.default(false),
     })
     .default({}),
   postgres: z.object({
@@ -35,12 +40,19 @@ export const configSchema = z.object({
     connection: z.string(),
 
     // run all migrations on boot of the application
-    migrateOnBoot: z.coerce.boolean().default(false),
+    migrateOnBoot: booleanSchema.default(false),
 
     // try to sync the schema on boot, useful for development
     // will always keep the database schema in sync with the connected database
     // it is extremely destructive, do not use it EVER in production
-    syncSchema: z.coerce.boolean().default(false),
+    syncSchema: booleanSchema.default(false),
+
+    // Enable debug logging for MikroORM - Outputs queries and entity management logs
+    // Do NOT use in production, leaks all sensitive data
+    debugLogging: booleanSchema.default(false),
+
+    // Enable SSL for the postgres connection
+    ssl: booleanSchema.default(false),
   }),
   crypto: z.object({
     // session secret. used for signing session tokens
@@ -55,7 +67,7 @@ export const configSchema = z.object({
   captcha: z
     .object({
       // enabled captchas on register
-      enabled: z.coerce.boolean().default(false),
+      enabled: booleanSchema.default(false),
 
       // captcha secret
       secret: z.string().min(1).optional(),
@@ -66,7 +78,7 @@ export const configSchema = z.object({
   ratelimits: z
     .object({
       // enabled captchas on register
-      enabled: z.coerce.boolean().default(false),
+      enabled: booleanSchema.default(false),
       redisUrl: z.string().optional(),
     })
     .default({}),

src/db/migrations/.snapshot-postgres.json

@@ -268,134 +268,6 @@
       "checks": [],
       "foreignKeys": {}
     },
-    {
-      "columns": {
-        "id": {
-          "name": "id",
-          "type": "uuid",
-          "unsigned": false,
-          "autoincrement": false,
-          "primary": false,
-          "nullable": false,
-          "mappedType": "uuid"
-        },
-        "tmdb_id": {
-          "name": "tmdb_id",
-          "type": "varchar(255)",
-          "unsigned": false,
-          "autoincrement": false,
-          "primary": false,
-          "nullable": false,
-          "mappedType": "string"
-        },
-        "type": {
-          "name": "type",
-          "type": "varchar(255)",
-          "unsigned": false,
-          "autoincrement": false,
-          "primary": false,
-          "nullable": false,
-          "mappedType": "string"
-        },
-        "title": {
-          "name": "title",
-          "type": "varchar(255)",
-          "unsigned": false,
-          "autoincrement": false,
-          "primary": false,
-          "nullable": false,
-          "mappedType": "string"
-        },
-        "season_id": {
-          "name": "season_id",
-          "type": "varchar(255)",
-          "unsigned": false,
-          "autoincrement": false,
-          "primary": false,
-          "nullable": true,
-          "mappedType": "string"
-        },
-        "episode_id": {
-          "name": "episode_id",
-          "type": "varchar(255)",
-          "unsigned": false,
-          "autoincrement": false,
-          "primary": false,
-          "nullable": true,
-          "mappedType": "string"
-        },
-        "created_at": {
-          "name": "created_at",
-          "type": "timestamptz(0)",
-          "unsigned": false,
-          "autoincrement": false,
-          "primary": false,
-          "nullable": false,
-          "length": 0,
-          "mappedType": "datetime"
-        },
-        "status": {
-          "name": "status",
-          "type": "varchar(255)",
-          "unsigned": false,
-          "autoincrement": false,
-          "primary": false,
-          "nullable": false,
-          "mappedType": "string"
-        },
-        "provider_id": {
-          "name": "provider_id",
-          "type": "varchar(255)",
-          "unsigned": false,
-          "autoincrement": false,
-          "primary": false,
-          "nullable": false,
-          "mappedType": "string"
-        },
-        "embed_id": {
-          "name": "embed_id",
-          "type": "varchar(255)",
-          "unsigned": false,
-          "autoincrement": false,
-          "primary": false,
-          "nullable": true,
-          "mappedType": "string"
-        },
-        "error_message": {
-          "name": "error_message",
-          "type": "varchar(255)",
-          "unsigned": false,
-          "autoincrement": false,
-          "primary": false,
-          "nullable": true,
-          "mappedType": "string"
-        },
-        "full_error": {
-          "name": "full_error",
-          "type": "varchar(255)",
-          "unsigned": false,
-          "autoincrement": false,
-          "primary": false,
-          "nullable": true,
-          "mappedType": "string"
-        }
-      },
-      "name": "provider_metrics",
-      "schema": "public",
-      "indexes": [
-        {
-          "keyName": "provider_metrics_pkey",
-          "columnNames": [
-            "id"
-          ],
-          "composite": false,
-          "primary": true,
-          "unique": true
-        }
-      ],
-      "checks": [],
-      "foreignKeys": {}
-    },
     {
       "columns": {
         "id": {
@@ -578,12 +450,12 @@
       "columns": {
         "id": {
           "name": "id",
-          "type": "uuid",
+          "type": "text",
           "unsigned": false,
           "autoincrement": false,
           "primary": false,
           "nullable": false,
-          "mappedType": "uuid"
+          "mappedType": "text"
         },
         "application_theme": {
           "name": "application_theme",
@@ -611,6 +483,15 @@
           "primary": false,
           "nullable": true,
           "mappedType": "string"
+        },
+        "proxy_urls": {
+          "name": "proxy_urls",
+          "type": "text[]",
+          "unsigned": false,
+          "autoincrement": false,
+          "primary": false,
+          "nullable": true,
+          "mappedType": "array"
         }
       },
       "name": "user_settings",

src/db/migrations/Migration20231111160045.ts

@@ -0,0 +1,25 @@
+import { Migration } from '@mikro-orm/migrations';
+
+export class Migration20231111160045 extends Migration {
+  async up(): Promise<void> {
+    this.addSql(
+      'alter table "provider_metrics" add column "hostname" varchar(255) not null;',
+    );
+    this.addSql(
+      'alter table "provider_metrics" alter column "error_message" type text using ("error_message"::text);',
+    );
+    this.addSql(
+      'alter table "provider_metrics" alter column "full_error" type text using ("full_error"::text);',
+    );
+  }
+
+  async down(): Promise<void> {
+    this.addSql(
+      'alter table "provider_metrics" alter column "error_message" type varchar(255) using ("error_message"::varchar(255));',
+    );
+    this.addSql(
+      'alter table "provider_metrics" alter column "full_error" type varchar(255) using ("full_error"::varchar(255));',
+    );
+    this.addSql('alter table "provider_metrics" drop column "hostname";');
+  }
+}

src/db/migrations/Migration20231122231620.ts

@@ -0,0 +1,14 @@
+import { Migration } from '@mikro-orm/migrations';
+
+export class Migration20231122231620 extends Migration {
+
+  async up(): Promise<void> {
+    this.addSql('alter table "user_settings" alter column "id" type text using ("id"::text);');
+  }
+
+  async down(): Promise<void> {
+    this.addSql('alter table "user_settings" alter column "id" drop default;');
+    this.addSql('alter table "user_settings" alter column "id" type uuid using ("id"::text::uuid);');
+  }
+
+}

src/db/migrations/Migration20231221185725.ts

@@ -0,0 +1,13 @@
+import { Migration } from '@mikro-orm/migrations';
+
+export class Migration20231221185725 extends Migration {
+
+  async up(): Promise<void> {
+    this.addSql('drop table if exists "provider_metrics" cascade;');
+  }
+
+  async down(): Promise<void> {
+    this.addSql('create table "provider_metrics" ("id" uuid not null default null, "tmdb_id" varchar not null default null, "type" varchar not null default null, "title" varchar not null default null, "season_id" varchar null default null, "episode_id" varchar null default null, "created_at" timestamptz not null default null, "status" varchar not null default null, "provider_id" varchar not null default null, "embed_id" varchar null default null, "error_message" text null default null, "full_error" text null default null, "hostname" varchar not null default null, constraint "provider_metrics_pkey" primary key ("id"));');
+  }
+
+}

src/db/migrations/Migration20231229214215.ts

@@ -0,0 +1,13 @@
+import { Migration } from '@mikro-orm/migrations';
+
+export class Migration20231229214215 extends Migration {
+
+  async up(): Promise<void> {
+    this.addSql('alter table "user_settings" add column "proxy_urls" text[] null;');
+  }
+
+  async down(): Promise<void> {
+    this.addSql('alter table "user_settings" drop column "proxy_urls";');
+  }
+
+}

src/db/models/ProgressItem.ts

@@ -53,6 +53,7 @@ export class ProgressItem {
 }
 
 export interface ProgressItemDTO {
+  id: string;
   tmdbId: string;
   season: {
     id?: string;
@@ -77,6 +78,7 @@ export function formatProgressItem(
   progressItem: ProgressItem,
 ): ProgressItemDTO {
   return {
+    id: progressItem.id,
     tmdbId: progressItem.tmdbId,
     episode: {
       id: progressItem.episodeId,

src/db/models/ProviderMetrics.ts

@@ -1,48 +0,0 @@
-import { Entity, PrimaryKey, Property } from '@mikro-orm/core';
-import { randomUUID } from 'crypto';
-
-export const status = {
-  failed: 'failed',
-  notfound: 'notfound',
-  success: 'success',
-} as const;
-type Status = keyof typeof status;
-
-@Entity({ tableName: 'provider_metrics' })
-export class ProviderMetric {
-  @PrimaryKey({ name: 'id', type: 'uuid' })
-  id: string = randomUUID();
-
-  @Property({ name: 'tmdb_id' })
-  tmdbId!: string;
-
-  @Property({ name: 'type' })
-  type!: string;
-
-  @Property({ name: 'title' })
-  title!: string;
-
-  @Property({ name: 'season_id', nullable: true })
-  seasonId?: string;
-
-  @Property({ name: 'episode_id', nullable: true })
-  episodeId?: string;
-
-  @Property({ name: 'created_at', type: 'date' })
-  createdAt = new Date();
-
-  @Property({ name: 'status' })
-  status!: Status;
-
-  @Property({ name: 'provider_id' })
-  providerId!: string;
-
-  @Property({ name: 'embed_id', nullable: true })
-  embedId?: string;
-
-  @Property({ name: 'error_message', nullable: true })
-  errorMessage?: string;
-
-  @Property({ name: 'full_error', nullable: true })
-  fullError?: string;
-}

src/db/models/UserSettings.ts

@@ -1,10 +1,9 @@
-import { Entity, PrimaryKey, Property } from '@mikro-orm/core';
+import { ArrayType, Entity, PrimaryKey, Property } from '@mikro-orm/core';
-import { randomUUID } from 'crypto';
 
 @Entity({ tableName: 'user_settings' })
 export class UserSettings {
-  @PrimaryKey({ name: 'id', type: 'uuid' })
+  @PrimaryKey({ name: 'id', type: 'text' })
-  id: string = randomUUID();
+  id!: string;
 
   @Property({ name: 'application_theme', nullable: true })
   applicationTheme?: string | null;
@@ -14,6 +13,9 @@ export class UserSettings {
 
   @Property({ name: 'default_subtitle_language', nullable: true })
   defaultSubtitleLanguage?: string | null;
+
+  @Property({ name: 'proxy_urls', type: ArrayType, nullable: true })
+  proxyUrls?: string[] | null;
 }
 
 export interface UserSettingsDTO {
@@ -21,6 +23,7 @@ export interface UserSettingsDTO {
   applicationTheme?: string | null;
   applicationLanguage?: string | null;
   defaultSubtitleLanguage?: string | null;
+  proxyUrls?: string[] | null;
 }
 
 export function formatUserSettings(
@@ -31,5 +34,6 @@ export function formatUserSettings(
     applicationTheme: userSettings.applicationTheme,
     applicationLanguage: userSettings.applicationLanguage,
     defaultSubtitleLanguage: userSettings.defaultSubtitleLanguage,
+    proxyUrls: userSettings.proxyUrls,
   };
 }

src/mikro-orm.config.ts

@@ -1,4 +1,4 @@
 import { ormConf } from '@/config/orm';
 import { makeOrmConfig } from '@/modules/mikro/orm';
 
-export default makeOrmConfig(ormConf.postgres.connection);
+export default makeOrmConfig(ormConf.postgres.connection, ormConf.postgres.ssl);

src/modules/fastify/routes.ts

@@ -2,7 +2,7 @@ import { loginAuthRouter } from '@/routes/auth/login';
 import { manageAuthRouter } from '@/routes/auth/manage';
 import { metaRouter } from '@/routes/meta';
 import { metricsRouter } from '@/routes/metrics';
-import { sessionsRouter } from '@/routes/sessions';
+import { sessionsRouter } from '@/routes/sessions/sessions';
 import { userBookmarkRouter } from '@/routes/users/bookmark';
 import { userDeleteRouter } from '@/routes/users/delete';
 import { userEditRouter } from '@/routes/users/edit';

src/modules/jobs/index.ts

@@ -1,11 +1,9 @@
 import { challengeCodeJob } from '@/modules/jobs/list/challengeCode';
 import { sessionExpiryJob } from '@/modules/jobs/list/sessionExpiry';
 import { userDeletionJob } from '@/modules/jobs/list/userDeletion';
-import { providerMetricCleanupJob } from '@/modules/jobs/list/providerMetricCleanup';
 
 export async function setupJobs() {
   challengeCodeJob.start();
   sessionExpiryJob.start();
   userDeletionJob.start();
-  providerMetricCleanupJob.start();
 }

src/modules/jobs/list/providerMetricCleanup.ts

@@ -1,27 +0,0 @@
-import { ProviderMetric } from '@/db/models/ProviderMetrics';
-import { job } from '@/modules/jobs/job';
-import ms from 'ms';
-
-// every day at 12:00:00
-export const providerMetricCleanupJob = job(
-  'provider-metric-cleanup',
-  '0 12 * * *',
-  async ({ em, log }) => {
-    const now = new Date();
-    const thirtyDaysAgo = new Date(now.getTime() - ms('30d'));
-
-    const deletedMetrics = await em
-      .createQueryBuilder(ProviderMetric)
-      .delete()
-      .where({
-        createdAt: {
-          $lt: thirtyDaysAgo,
-        },
-      })
-      .execute<{ affectedRows: number }>('run');
-
-    log.info(
-      `Removed ${deletedMetrics.affectedRows} metrics that were older than 30 days`,
-    );
-  },
-);

src/modules/metrics/index.ts

@@ -9,16 +9,11 @@ const log = scopedLogger('metrics');
 
 export type Metrics = {
   user: Counter<'namespace'>;
-  providerMetrics: Counter<
+  captchaSolves: Counter<'success'>;
-    | 'title'
+  providerHostnames: Counter<'hostname'>;
-    | 'tmdb_id'
+  providerStatuses: Counter<'provider_id' | 'status'>;
-    | 'season_id'
+  watchMetrics: Counter<'title' | 'tmdb_full_id' | 'provider_id' | 'success'>;
-    | 'episode_id'
+  toolMetrics: Counter<'tool'>;
-    | 'status'
-    | 'type'
-    | 'provider_id'
-    | 'embed_id'
-  >;
 };
 
 let metrics: null | Metrics = null;
@@ -41,30 +36,45 @@ export async function setupMetrics(app: FastifyInstance) {
 
   metrics = {
     user: new Counter({
-      name: 'user_count',
+      name: 'mw_user_count',
-      help: 'user_help',
+      help: 'mw_user_help',
       labelNames: ['namespace'],
     }),
-    providerMetrics: new Counter({
+    captchaSolves: new Counter({
-      name: 'provider_metrics',
+      name: 'mw_captcha_solves',
-      help: 'provider_metrics',
+      help: 'mw_captcha_solves',
-      labelNames: [
+      labelNames: ['success'],
-        'episode_id',
+    }),
-        'provider_id',
+    providerHostnames: new Counter({
-        'season_id',
+      name: 'mw_provider_hostname_count',
-        'status',
+      help: 'mw_provider_hostname_count',
-        'title',
+      labelNames: ['hostname'],
-        'tmdb_id',
+    }),
-        'type',
+    providerStatuses: new Counter({
-        'embed_id',
+      name: 'mw_provider_status_count',
-      ],
+      help: 'mw_provider_status_count',
+      labelNames: ['provider_id', 'status'],
+    }),
+    watchMetrics: new Counter({
+      name: 'mw_media_watch_count',
+      help: 'mw_media_watch_count',
+      labelNames: ['title', 'tmdb_full_id', 'provider_id', 'success'],
+    }),
+    toolMetrics: new Counter({
+      name: 'mw_provider_tool_count',
+      help: 'mw_provider_tool_count',
+      labelNames: ['tool'],
     }),
   };
 
   const promClient = app.metrics.client;
 
   promClient.register.registerMetric(metrics.user);
-  promClient.register.registerMetric(metrics.providerMetrics);
+  promClient.register.registerMetric(metrics.providerHostnames);
+  promClient.register.registerMetric(metrics.providerStatuses);
+  promClient.register.registerMetric(metrics.watchMetrics);
+  promClient.register.registerMetric(metrics.captchaSolves);
+  promClient.register.registerMetric(metrics.toolMetrics);
 
   const orm = getORM();
   const em = orm.em.fork();

src/modules/mikro/index.ts

@@ -14,8 +14,11 @@ export function getORM() {
 
 export async function setupMikroORM() {
   log.info(`Connecting to postgres`, { evt: 'connecting' });
-  const mikro = await createORM(conf.postgres.connection, (msg) =>
+  const mikro = await createORM(
-    log.info(msg),
+    conf.postgres.connection,
+    conf.postgres.debugLogging,
+    (msg) => log.info(msg),
+    conf.postgres.ssl,
   );
 
   if (conf.postgres.syncSchema) {

src/modules/mikro/orm.ts

@@ -2,7 +2,10 @@ import { Options } from '@mikro-orm/core';
 import { MikroORM, PostgreSqlDriver } from '@mikro-orm/postgresql';
 import path from 'path';
 
-export function makeOrmConfig(url: string): Options<PostgreSqlDriver> {
+export function makeOrmConfig(
+  url: string,
+  ssl: boolean,
+): Options<PostgreSqlDriver> {
   return {
     type: 'postgresql',
     clientUrl: url,
@@ -13,12 +16,23 @@ export function makeOrmConfig(url: string): Options<PostgreSqlDriver> {
       pathTs: './migrations',
       path: './migrations',
     },
+    driverOptions: {
+      connection: {
+        ssl,
+      },
+    },
   };
 }
 
-export async function createORM(url: string, log: (msg: string) => void) {
+export async function createORM(
+  url: string,
+  debug: boolean,
+  log: (msg: string) => void,
+  ssl: boolean,
+) {
   return await MikroORM.init<PostgreSqlDriver>({
-    ...makeOrmConfig(url),
+    ...makeOrmConfig(url, ssl),
     logger: log,
+    debug,
   });
 }

src/modules/ratelimits/limiter.ts

@@ -2,6 +2,7 @@ import Redis from 'ioredis';
 import RateLimiter from 'async-ratelimiter';
 import ms from 'ms';
 import { StatusError } from '@/services/error';
+import { IpReq, getIp } from '@/services/ip';
 
 export interface LimiterOptions {
   redis: Redis;
@@ -26,8 +27,8 @@ export class Limiter {
     this.redis = ops.redis;
   }
 
-  async bump(req: { ip: string }, ops: BucketOptions) {
+  async bump(req: IpReq, ops: BucketOptions) {
-    const ip = req.ip;
+    const ip = getIp(req);
     if (!this.buckets[ops.id]) {
       this.buckets[ops.id] = {
         limiter: new RateLimiter({
@@ -54,7 +55,7 @@ export class Limiter {
     };
   }
 
-  async assertAndBump(req: { ip: string }, ops: BucketOptions) {
+  async assertAndBump(req: IpReq, ops: BucketOptions) {
     const { hasBeenLimited } = await this.bump(req, ops);
     if (hasBeenLimited) {
       throw new StatusError('Ratelimited', 429);

src/routes/metrics.ts

@@ -1,8 +1,8 @@
 import { handle } from '@/services/handler';
 import { makeRouter } from '@/services/router';
 import { z } from 'zod';
-import { ProviderMetric, status } from '@/db/models/ProviderMetrics';
 import { getMetrics } from '@/modules/metrics';
+import { status } from '@/routes/statuses';
 
 const metricsProviderSchema = z.object({
   tmdbId: z.string(),
@@ -19,6 +19,7 @@ const metricsProviderSchema = z.object({
 
 const metricsProviderInputSchema = z.object({
   items: z.array(metricsProviderSchema).max(10).min(1),
+  tool: z.string().optional(),
 });
 
 export const metricsRouter = makeRouter((app) => {
@@ -29,7 +30,7 @@ export const metricsRouter = makeRouter((app) => {
         body: metricsProviderInputSchema,
       },
     },
-    handle(async ({ em, body, req, limiter }) => {
+    handle(async ({ body, req, limiter }) => {
       await limiter?.assertAndBump(req, {
         id: 'provider_metrics',
         max: 300,
@@ -37,36 +38,65 @@ export const metricsRouter = makeRouter((app) => {
         window: '30m',
       });
 
-      const entities = body.items.map((v) => {
+      const hostname = req.headers.origin?.slice(0, 255) ?? '<UNKNOWN>';
-        const metric = new ProviderMetric();
+      getMetrics().providerHostnames.inc({
-        em.assign(metric, {
+        hostname,
-          providerId: v.providerId,
-          embedId: v.embedId,
-          fullError: v.fullError,
-          errorMessage: v.errorMessage,
-          episodeId: v.episodeId,
-          seasonId: v.seasonId,
-          status: v.status,
-          title: v.title,
-          tmdbId: v.tmdbId,
-          type: v.type,
-        });
-        return metric;
       });
 
-      entities.forEach((entity) => {
+      body.items.forEach((item) => {
-        getMetrics().providerMetrics.inc({
+        getMetrics().providerStatuses.inc({
-          episode_id: entity.episodeId,
+          provider_id: item.embedId ?? item.providerId,
-          provider_id: entity.providerId,
+          status: item.status,
-          season_id: entity.seasonId,
-          status: entity.status,
-          title: entity.title,
-          tmdb_id: entity.tmdbId,
-          type: entity.type,
         });
       });
 
-      await em.persistAndFlush(entities);
+      const itemList = [...body.items];
+      itemList.reverse();
+      const lastSuccessfulItem = body.items.find(
+        (v) => v.status === status.success,
+      );
+      const lastItem = itemList[0];
+
+      if (lastItem) {
+        getMetrics().watchMetrics.inc({
+          tmdb_full_id: lastItem.type + '-' + lastItem.tmdbId,
+          provider_id: lastSuccessfulItem?.providerId ?? lastItem.providerId,
+          title: lastItem.title,
+          success: (!!lastSuccessfulItem).toString(),
+        });
+      }
+
+      if (body.tool) {
+        getMetrics().toolMetrics.inc({
+          tool: body.tool,
+        });
+      }
+
+      return true;
+    }),
+  );
+
+  app.post(
+    '/metrics/captcha',
+    {
+      schema: {
+        body: z.object({
+          success: z.boolean(),
+        }),
+      },
+    },
+    handle(async ({ body, req, limiter }) => {
+      await limiter?.assertAndBump(req, {
+        id: 'captcha_solves',
+        max: 300,
+        inc: 1,
+        window: '30m',
+      });
+
+      getMetrics().captchaSolves.inc({
+        success: body.success.toString(),
+      });
+
       return true;
     }),
   );

src/routes/sessions/session.ts

@@ -1,35 +0,0 @@
-import { Session } from '@/db/models/Session';
-import { StatusError } from '@/services/error';
-import { handle } from '@/services/handler';
-import { makeRouter } from '@/services/router';
-import { z } from 'zod';
-
-export const sessionRouter = makeRouter((app) => {
-  app.delete(
-    '/sessions/:sid',
-    {
-      schema: {
-        params: z.object({
-          sid: z.string(),
-        }),
-      },
-    },
-    handle(async ({ auth, params, em }) => {
-      await auth.assert();
-
-      const targetedSession = await em.findOne(Session, { id: params.sid });
-      if (!targetedSession)
-        return {
-          id: params.sid,
-        };
-
-      if (targetedSession.user !== auth.user.id)
-        throw new StatusError('Cannot delete sessions you do not own', 401);
-
-      await em.removeAndFlush(targetedSession);
-      return {
-        id: params.sid,
-      };
-    }),
-  );
-});

src/routes/sessions/sessions.ts

@@ -13,7 +13,7 @@ export const sessionsRouter = makeRouter((app) => {
           sid: z.string(),
         }),
         body: z.object({
-          name: z.string().max(500).min(1).optional(),
+          deviceName: z.string().max(500).min(1).optional(),
         }),
       },
     },
@@ -25,10 +25,10 @@ export const sessionsRouter = makeRouter((app) => {
       if (!targetedSession)
         throw new StatusError('Session cannot be found', 404);
 
-      if (targetedSession.user !== auth.user.id)
+      if (targetedSession.id !== params.sid)
-        throw new StatusError('Cannot modify sessions you do not own', 401);
+        throw new StatusError('Cannot edit sessions other than your own', 401);
 
-      if (body.name) targetedSession.device = body.name;
+      if (body.deviceName) targetedSession.device = body.deviceName;
 
       await em.persistAndFlush(targetedSession);
 

src/routes/statuses.ts

@@ -0,0 +1,6 @@
+export const status = {
+  failed: 'failed',
+  notfound: 'notfound',
+  success: 'success',
+} as const;
+export type Status = keyof typeof status;

src/routes/users/bookmark.ts

@@ -8,6 +8,11 @@ import { handle } from '@/services/handler';
 import { makeRouter } from '@/services/router';
 import { z } from 'zod';
 
+const bookmarkDataSchema = z.object({
+  tmdbId: z.string(),
+  meta: bookmarkMetaSchema,
+});
+
 export const userBookmarkRouter = makeRouter((app) => {
   app.get(
     '/users/:uid/bookmarks',
@@ -40,9 +45,7 @@ export const userBookmarkRouter = makeRouter((app) => {
           uid: z.string(),
           tmdbid: z.string(),
         }),
-        body: z.object({
+        body: bookmarkDataSchema,
-          meta: bookmarkMetaSchema,
-        }),
       },
     },
     handle(async ({ auth, params, body, em }) => {
@@ -70,6 +73,40 @@ export const userBookmarkRouter = makeRouter((app) => {
     }),
   );
 
+  app.put(
+    '/users/:uid/bookmarks',
+    {
+      schema: {
+        params: z.object({
+          uid: z.string(),
+        }),
+        body: z.array(bookmarkDataSchema),
+      },
+    },
+    handle(async ({ auth, params, body, em }) => {
+      await auth.assert();
+
+      if (auth.user.id !== params.uid)
+        throw new StatusError('Cannot modify user other than yourself', 403);
+
+      const bookmarks = await em.upsertMany(
+        Bookmark,
+        body.map((item) => ({
+          userId: params.uid,
+          tmdbId: item.tmdbId,
+          meta: item.meta,
+          updatedAt: new Date(),
+        })),
+        {
+          onConflictFields: ['tmdbId', 'userId'],
+        },
+      );
+
+      await em.flush();
+      return bookmarks.map(formatBookmark);
+    }),
+  );
+
   app.delete(
     '/users/:uid/bookmarks/:tmdbid',
     {

src/routes/users/get.ts

@@ -1,3 +1,4 @@
+import { formatSession } from '@/db/models/Session';
 import { User, formatUser } from '@/db/models/User';
 import { StatusError } from '@/services/error';
 import { handle } from '@/services/handler';
@@ -5,6 +6,24 @@ import { makeRouter } from '@/services/router';
 import { z } from 'zod';
 
 export const userGetRouter = makeRouter((app) => {
+  app.get(
+    '/users/@me',
+    handle(async ({ auth, em }) => {
+      await auth.assert();
+
+      const user = await em.findOne(User, { id: auth.user.id });
+      if (!user) throw new StatusError('User does not exist', 404);
+
+      const session = await auth.getSession();
+      if (!session) throw new StatusError('Session does not exist', 400);
+
+      return {
+        user: formatUser(user),
+        session: formatSession(session),
+      };
+    }),
+  );
+
   app.get(
     '/users/:uid',
     {
@@ -16,16 +35,16 @@ export const userGetRouter = makeRouter((app) => {
     },
     handle(async ({ auth, params, em }) => {
       await auth.assert();
-      let uid = params.uid;
-      if (uid === '@me') uid = auth.user.id;
 
-      if (auth.user.id !== uid)
+      if (auth.user.id !== params.uid)
         throw new StatusError('Cannot access users other than yourself', 403);
 
-      const user = await em.findOne(User, { id: uid });
+      const user = await em.findOne(User, { id: params.uid });
       if (!user) throw new StatusError('User does not exist', 404);
 
-      return formatUser(user);
+      return {
+        user: formatUser(user),
+      };
     }),
   );
 });

src/routes/users/progress.ts

@@ -6,8 +6,22 @@ import {
 import { StatusError } from '@/services/error';
 import { handle } from '@/services/handler';
 import { makeRouter } from '@/services/router';
+import { FilterQuery } from '@mikro-orm/core';
+import { randomUUID } from 'crypto';
 import { z } from 'zod';
 
+const progressItemSchema = z.object({
+  meta: progressMetaSchema,
+  tmdbId: z.string(),
+  duration: z.number().transform((n) => Math.round(n)),
+  watched: z.number().transform((n) => Math.round(n)),
+  seasonId: z.string().optional(),
+  episodeId: z.string().optional(),
+  seasonNumber: z.number().optional(),
+  episodeNumber: z.number().optional(),
+  updatedAt: z.string().datetime({ offset: true }).optional(),
+});
+
 export const userProgressRouter = makeRouter((app) => {
   app.put(
     '/users/:uid/progress/:tmdbid',
@@ -17,15 +31,7 @@ export const userProgressRouter = makeRouter((app) => {
           uid: z.string(),
           tmdbid: z.string(),
         }),
-        body: z.object({
+        body: progressItemSchema,
-          meta: progressMetaSchema,
-          duration: z.number(),
-          watched: z.number(),
-          seasonId: z.string().optional(),
-          episodeId: z.string().optional(),
-          seasonNumber: z.number().optional(),
-          episodeNumber: z.number().optional(),
-        }),
       },
     },
     handle(async ({ auth, params, body, em }) => {
@@ -63,6 +69,82 @@ export const userProgressRouter = makeRouter((app) => {
     }),
   );
 
+  app.put(
+    '/users/:uid/progress/import',
+    {
+      schema: {
+        params: z.object({
+          uid: z.string(),
+        }),
+        body: z.array(progressItemSchema),
+      },
+    },
+    handle(async ({ auth, params, body, em, req, limiter }) => {
+      await auth.assert();
+
+      if (auth.user.id !== params.uid)
+        throw new StatusError('Cannot modify user other than yourself', 403);
+
+      const itemsUpserted: ProgressItem[] = [];
+
+      const newItems = [...body];
+
+      for (const existingItem of await em.find(ProgressItem, {
+        userId: params.uid,
+      })) {
+        const newItemIndex = newItems.findIndex(
+          (item) =>
+            item.tmdbId == existingItem.tmdbId &&
+            item.seasonId == existingItem.seasonId &&
+            item.episodeId == existingItem.episodeId,
+        );
+
+        if (newItemIndex > -1) {
+          const newItem = newItems[newItemIndex];
+          if (existingItem.watched < newItem.watched) {
+            existingItem.updatedAt = defaultAndCoerceDateTime(
+              newItem.updatedAt,
+            );
+            existingItem.watched = newItem.watched;
+          }
+          itemsUpserted.push(existingItem);
+
+          // Remove the item from the array, we have processed it
+          newItems.splice(newItemIndex, 1);
+        }
+      }
+
+      // All unprocessed items, aka all items that don't already exist
+      for (const newItem of newItems) {
+        itemsUpserted.push({
+          id: randomUUID(),
+          duration: newItem.duration,
+          episodeId: newItem.episodeId,
+          episodeNumber: newItem.episodeNumber,
+          meta: newItem.meta,
+          seasonId: newItem.seasonId,
+          seasonNumber: newItem.seasonNumber,
+          tmdbId: newItem.tmdbId,
+          userId: params.uid,
+          watched: newItem.watched,
+          updatedAt: defaultAndCoerceDateTime(newItem.updatedAt),
+        });
+      }
+
+      const progressItems = await em.upsertMany(ProgressItem, itemsUpserted);
+
+      await em.flush();
+
+      await limiter?.assertAndBump(req, {
+        id: 'progress_import',
+        max: 5,
+        window: '10m',
+      });
+
+      return progressItems.map(formatProgressItem);
+    }),
+  );
+
   app.delete(
     '/users/:uid/progress/:tmdbid',
     {
@@ -83,22 +165,28 @@ export const userProgressRouter = makeRouter((app) => {
       if (auth.user.id !== params.uid)
         throw new StatusError('Cannot modify user other than yourself', 403);
 
-      const progressItem = await em.findOne(ProgressItem, {
+      const query: FilterQuery<ProgressItem> = {
         userId: params.uid,
         tmdbId: params.tmdbid,
-        episodeId: body.episodeId,
+      };
-        seasonId: body.seasonId,
+      if (body.seasonId) query.seasonId = body.seasonId;
-      });
+      if (body.episodeId) query.episodeId = body.episodeId;
-      if (!progressItem) {
+      const progressItems = await em.find(ProgressItem, query);
+
+      if (progressItems.length === 0) {
         return {
+          count: 0,
           tmdbId: params.tmdbid,
           episodeId: body.episodeId,
           seasonId: body.seasonId,
         };
       }
 
-      await em.removeAndFlush(progressItem);
+      progressItems.forEach((v) => em.remove(v));
+      await em.flush();
+
       return {
+        count: progressItems.length,
         tmdbId: params.tmdbid,
         episodeId: body.episodeId,
         seasonId: body.seasonId,
@@ -129,3 +217,14 @@ export const userProgressRouter = makeRouter((app) => {
     }),
   );
 });
+
+// 13th July 2021 - movie-web epoch
+const minEpoch = 1626134400000;
+
+function defaultAndCoerceDateTime(dateTime: string | undefined) {
+  const epoch = dateTime ? new Date(dateTime).getTime() : Date.now();
+
+  const clampedEpoch = Math.max(minEpoch, Math.min(epoch, Date.now()));
+
+  return new Date(clampedEpoch);
+}

src/routes/users/settings.ts

@@ -38,9 +38,10 @@ export const userSettingsRouter = makeRouter((app) => {
           uid: z.string(),
         }),
         body: z.object({
-          applicationLanguage: z.string().optional(),
+          applicationLanguage: z.string().nullable().optional(),
-          applicationTheme: z.string().optional(),
+          applicationTheme: z.string().nullable().optional(),
-          defaultSubtitleLanguage: z.string().optional(),
+          defaultSubtitleLanguage: z.string().nullable().optional(),
+          proxyUrls: z.string().array().nullable().optional(),
         }),
       },
     },
@@ -58,12 +59,13 @@ export const userSettingsRouter = makeRouter((app) => {
         settings.id = params.uid;
       }
 
-      if (body.applicationLanguage)
+      if (body.applicationLanguage !== undefined)
         settings.applicationLanguage = body.applicationLanguage;
-      if (body.applicationTheme)
+      if (body.defaultSubtitleLanguage !== undefined)
-        settings.applicationTheme = body.applicationTheme;
-      if (body.defaultSubtitleLanguage)
         settings.defaultSubtitleLanguage = body.defaultSubtitleLanguage;
+      if (body.applicationTheme !== undefined)
+        settings.applicationTheme = body.applicationTheme;
+      if (body.proxyUrls !== undefined) settings.proxyUrls = body.proxyUrls;
 
       await em.persistAndFlush(settings);
       return formatUserSettings(settings);

src/services/captcha.ts

@@ -4,16 +4,14 @@ import { StatusError } from '@/services/error';
 export async function isValidCaptcha(token: string): Promise<boolean> {
   if (!conf.captcha.secret)
     throw new Error('isValidCaptcha() is called but no secret set');
+  const formData = new URLSearchParams();
+  formData.append('secret', conf.captcha.secret);
+  formData.append('response', token);
   const res = await fetch('https://www.google.com/recaptcha/api/siteverify', {
     method: 'POST',
-    body: JSON.stringify({
+    body: formData,
-      secret: conf.captcha.secret,
-      response: token,
-    }),
-    headers: {
-      'content-type': 'application/json',
-    },
   });
+
   const json = await res.json();
   return !!json.success;
 }

src/services/ip.ts

@@ -0,0 +1,27 @@
+import { conf } from '@/config';
+import { IncomingHttpHeaders } from 'http';
+
+export type IpReq = {
+  ip: string;
+  headers: IncomingHttpHeaders;
+};
+
+const trustCloudflare = conf.server.trustCloudflare;
+
+function getSingleHeader(
+  headers: IncomingHttpHeaders,
+  key: string,
+): string | undefined {
+  const header = headers[key];
+  if (Array.isArray(header)) return header[0];
+  return header;
+}
+
+export function getIp(req: IpReq) {
+  const cfIp = getSingleHeader(req.headers, 'cf-connecting-ip');
+  if (trustCloudflare && cfIp) {
+    return cfIp;
+  }
+
+  return req.ip;
+}