Merge pull request #27 from movie-web/dev

Register Captcha Solves Metric

.docker/development/docker-compose.yml

@@ -12,6 +12,8 @@ services:
       POSTGRES_PASSWORD: postgres
     volumes:
      - 'postgres_data:/var/lib/postgresql/data'
+  redis:
+    image: redis
 
   # custom services
   backend:

.eslintrc.js

@@ -13,7 +13,7 @@ module.exports = {
     sourceType: 'module',
   },
   plugins: ['@typescript-eslint'],
-  ignorePatterns: ['./src/db/migrations/**/*'],
+  ignorePatterns: ['src/db/migrations/**/*'],
   rules: {
     '@typescript-eslint/interface-name-prefix': 'off',
     '@typescript-eslint/explicit-function-return-type': 'off',

package.json

@@ -1,6 +1,6 @@
 {
   "name": "backend",
-  "version": "1.0.1",
+  "version": "1.2.1",
   "private": true,
   "homepage": "https://github.com/movie-web/backend",
   "engines": {
@@ -15,7 +15,9 @@
     "build:pre": "rimraf dist/",
     "build:compile": "tsc && tsc-alias",
     "preinstall": "npx -y only-allow pnpm",
-    "migration:create": "npx -y mikro-orm migration:create"
+    "migration:create": "npx -y mikro-orm migration:create",
+    "migration:up": "npx -y mikro-orm migration:up",
+    "migration:down": "npx -y mikro-orm migration:down"
   },
   "mikro-orm": {
     "useTsNode": true,

src/config/fragments/dev.ts

@@ -2,7 +2,7 @@ import { FragmentSchema } from '@/config/fragments/types';
 
 export const devFragment: FragmentSchema = {
   server: {
-    cors: 'http://localhost:5173',
+    allowAnySite: true,
     trustProxy: true,
   },
   logging: {

src/config/fragments/docker.ts

@@ -4,4 +4,8 @@ export const dockerFragment: FragmentSchema = {
   postgres: {
     connection: 'postgres://postgres:postgres@postgres:5432/postgres',
   },
+  ratelimits: {
+    enabled: true,
+    redisUrl: 'redis://redis:6379',
+  },
 };

src/config/index.ts

@@ -8,7 +8,7 @@ const fragments = {
   dockerdev: dockerFragment,
 };
 
-export const version = '1.0.0';
+export const version = process.env.npm_package_version ?? 'unknown';
 
 export const conf = createConfigLoader()
   .addFromEnvironment('MWB_')

src/config/orm.ts

@@ -0,0 +1,29 @@
+import { devFragment } from '@/config/fragments/dev';
+import { dockerFragment } from '@/config/fragments/docker';
+import { createConfigLoader } from 'neat-config';
+import { z } from 'zod';
+
+const fragments = {
+  dev: devFragment,
+  dockerdev: dockerFragment,
+};
+
+export const ormConfigSchema = z.object({
+  postgres: z.object({
+    // connection URL for postgres database
+    connection: z.string(),
+  }),
+});
+
+export const ormConf = createConfigLoader()
+  .addFromEnvironment('MWB_')
+  .addFromCLI('mwb-')
+  .addFromFile('.env', {
+    prefix: 'MWB_',
+  })
+  .addFromFile('config.json')
+  .setFragmentKey('usePresets')
+  .addConfigFragments(fragments)
+  .addZodSchema(ormConfigSchema)
+  .freeze()
+  .load();

src/config/schema.ts

@@ -9,9 +9,16 @@ export const configSchema = z.object({
       // space seperated list of allowed cors domains
       cors: z.string().default(''),
 
+      // disable cross origin restrictions, allow any site.
+      // overwrites the cors option above
+      allowAnySite: z.coerce.boolean().default(false),
+
       // should it trust reverse proxy headers? (for ip gathering)
       trustProxy: z.coerce.boolean().default(false),
 
+      // should it trust cloudflare headers? (for ip gathering, cloudflare has priority)
+      trustCloudflare: z.coerce.boolean().default(false),
+
       // prefix for where the instance is run on. for example set it to /backend if you're hosting it on example.com/backend
       // if this is set, do not apply url rewriting before proxing
       basePath: z.string().default('/'),
@@ -37,6 +44,10 @@ export const configSchema = z.object({
     // will always keep the database schema in sync with the connected database
     // it is extremely destructive, do not use it EVER in production
     syncSchema: z.coerce.boolean().default(false),
+
+    // Enable debug logging for MikroORM - Outputs queries and entity management logs
+    // Do NOT use in production, leaks all sensitive data
+    debugLogging: z.coerce.boolean().default(false),
   }),
   crypto: z.object({
     // session secret. used for signing session tokens

src/db/migrations/.snapshot-postgres.json

@@ -184,6 +184,24 @@
           "nullable": true,
           "mappedType": "string"
         },
+        "season_number": {
+          "name": "season_number",
+          "type": "int",
+          "unsigned": false,
+          "autoincrement": false,
+          "primary": false,
+          "nullable": true,
+          "mappedType": "integer"
+        },
+        "episode_number": {
+          "name": "episode_number",
+          "type": "int",
+          "unsigned": false,
+          "autoincrement": false,
+          "primary": false,
+          "nullable": true,
+          "mappedType": "integer"
+        },
         "meta": {
           "name": "meta",
           "type": "jsonb",
@@ -250,134 +268,6 @@
       "checks": [],
       "foreignKeys": {}
     },
-    {
-      "columns": {
-        "id": {
-          "name": "id",
-          "type": "uuid",
-          "unsigned": false,
-          "autoincrement": false,
-          "primary": false,
-          "nullable": false,
-          "mappedType": "uuid"
-        },
-        "tmdb_id": {
-          "name": "tmdb_id",
-          "type": "varchar(255)",
-          "unsigned": false,
-          "autoincrement": false,
-          "primary": false,
-          "nullable": false,
-          "mappedType": "string"
-        },
-        "type": {
-          "name": "type",
-          "type": "varchar(255)",
-          "unsigned": false,
-          "autoincrement": false,
-          "primary": false,
-          "nullable": false,
-          "mappedType": "string"
-        },
-        "title": {
-          "name": "title",
-          "type": "varchar(255)",
-          "unsigned": false,
-          "autoincrement": false,
-          "primary": false,
-          "nullable": false,
-          "mappedType": "string"
-        },
-        "season_id": {
-          "name": "season_id",
-          "type": "varchar(255)",
-          "unsigned": false,
-          "autoincrement": false,
-          "primary": false,
-          "nullable": true,
-          "mappedType": "string"
-        },
-        "episode_id": {
-          "name": "episode_id",
-          "type": "varchar(255)",
-          "unsigned": false,
-          "autoincrement": false,
-          "primary": false,
-          "nullable": true,
-          "mappedType": "string"
-        },
-        "created_at": {
-          "name": "created_at",
-          "type": "timestamptz(0)",
-          "unsigned": false,
-          "autoincrement": false,
-          "primary": false,
-          "nullable": false,
-          "length": 0,
-          "mappedType": "datetime"
-        },
-        "status": {
-          "name": "status",
-          "type": "varchar(255)",
-          "unsigned": false,
-          "autoincrement": false,
-          "primary": false,
-          "nullable": false,
-          "mappedType": "string"
-        },
-        "provider_id": {
-          "name": "provider_id",
-          "type": "varchar(255)",
-          "unsigned": false,
-          "autoincrement": false,
-          "primary": false,
-          "nullable": false,
-          "mappedType": "string"
-        },
-        "embed_id": {
-          "name": "embed_id",
-          "type": "varchar(255)",
-          "unsigned": false,
-          "autoincrement": false,
-          "primary": false,
-          "nullable": true,
-          "mappedType": "string"
-        },
-        "error_message": {
-          "name": "error_message",
-          "type": "varchar(255)",
-          "unsigned": false,
-          "autoincrement": false,
-          "primary": false,
-          "nullable": true,
-          "mappedType": "string"
-        },
-        "full_error": {
-          "name": "full_error",
-          "type": "varchar(255)",
-          "unsigned": false,
-          "autoincrement": false,
-          "primary": false,
-          "nullable": true,
-          "mappedType": "string"
-        }
-      },
-      "name": "provider_metrics",
-      "schema": "public",
-      "indexes": [
-        {
-          "keyName": "provider_metrics_pkey",
-          "columnNames": [
-            "id"
-          ],
-          "composite": false,
-          "primary": true,
-          "unique": true
-        }
-      ],
-      "checks": [],
-      "foreignKeys": {}
-    },
     {
       "columns": {
         "id": {
@@ -560,12 +450,12 @@
       "columns": {
         "id": {
           "name": "id",
-          "type": "uuid",
+          "type": "text",
           "unsigned": false,
           "autoincrement": false,
           "primary": false,
           "nullable": false,
-          "mappedType": "uuid"
+          "mappedType": "text"
         },
         "application_theme": {
           "name": "application_theme",

src/db/migrations/Migration20231105150807.ts

@@ -0,0 +1,14 @@
+import { Migration } from '@mikro-orm/migrations';
+
+export class Migration20231105150807 extends Migration {
+  async up(): Promise<void> {
+    this.addSql(
+      'alter table "progress_items" add column "season_number" int null, add column "episode_number" int null;',
+    );
+  }
+
+  async down(): Promise<void> {
+    this.addSql('alter table "progress_items" drop column "season_number";');
+    this.addSql('alter table "progress_items" drop column "episode_number";');
+  }
+}

src/db/migrations/Migration20231111160045.ts

@@ -0,0 +1,25 @@
+import { Migration } from '@mikro-orm/migrations';
+
+export class Migration20231111160045 extends Migration {
+  async up(): Promise<void> {
+    this.addSql(
+      'alter table "provider_metrics" add column "hostname" varchar(255) not null;',
+    );
+    this.addSql(
+      'alter table "provider_metrics" alter column "error_message" type text using ("error_message"::text);',
+    );
+    this.addSql(
+      'alter table "provider_metrics" alter column "full_error" type text using ("full_error"::text);',
+    );
+  }
+
+  async down(): Promise<void> {
+    this.addSql(
+      'alter table "provider_metrics" alter column "error_message" type varchar(255) using ("error_message"::varchar(255));',
+    );
+    this.addSql(
+      'alter table "provider_metrics" alter column "full_error" type varchar(255) using ("full_error"::varchar(255));',
+    );
+    this.addSql('alter table "provider_metrics" drop column "hostname";');
+  }
+}

src/db/migrations/Migration20231122231620.ts

@@ -0,0 +1,14 @@
+import { Migration } from '@mikro-orm/migrations';
+
+export class Migration20231122231620 extends Migration {
+
+  async up(): Promise<void> {
+    this.addSql('alter table "user_settings" alter column "id" type text using ("id"::text);');
+  }
+
+  async down(): Promise<void> {
+    this.addSql('alter table "user_settings" alter column "id" drop default;');
+    this.addSql('alter table "user_settings" alter column "id" type uuid using ("id"::text::uuid);');
+  }
+
+}

src/db/migrations/Migration20231221185725.ts

@@ -0,0 +1,13 @@
+import { Migration } from '@mikro-orm/migrations';
+
+export class Migration20231221185725 extends Migration {
+
+  async up(): Promise<void> {
+    this.addSql('drop table if exists "provider_metrics" cascade;');
+  }
+
+  async down(): Promise<void> {
+    this.addSql('create table "provider_metrics" ("id" uuid not null default null, "tmdb_id" varchar not null default null, "type" varchar not null default null, "title" varchar not null default null, "season_id" varchar null default null, "episode_id" varchar null default null, "created_at" timestamptz not null default null, "status" varchar not null default null, "provider_id" varchar not null default null, "embed_id" varchar null default null, "error_message" text null default null, "full_error" text null default null, "hostname" varchar not null default null, constraint "provider_metrics_pkey" primary key ("id"));');
+  }
+
+}

src/db/models/ProgressItem.ts

@@ -6,7 +6,7 @@ export const progressMetaSchema = z.object({
   title: z.string(),
   year: z.number(),
   poster: z.string().optional(),
-  type: z.string(),
+  type: z.union([z.literal('show'), z.literal('movie')]),
 });
 
 export type ProgressMeta = z.infer<typeof progressMetaSchema>;
@@ -29,6 +29,12 @@ export class ProgressItem {
   @Property({ name: 'episode_id', nullable: true })
   episodeId?: string;
 
+  @Property({ name: 'season_number', nullable: true })
+  seasonNumber?: number;
+
+  @Property({ name: 'episode_number', nullable: true })
+  episodeNumber?: number;
+
   @Property({
     name: 'meta',
     type: types.json,
@@ -47,9 +53,16 @@ export class ProgressItem {
 }
 
 export interface ProgressItemDTO {
+  id: string;
   tmdbId: string;
-  seasonId?: string;
+  season: {
-  episodeId?: string;
+    id?: string;
+    number?: number;
+  };
+  episode: {
+    id?: string;
+    number?: number;
+  };
   meta: {
     title: string;
     year: number;
@@ -65,9 +78,16 @@ export function formatProgressItem(
   progressItem: ProgressItem,
 ): ProgressItemDTO {
   return {
+    id: progressItem.id,
     tmdbId: progressItem.tmdbId,
-    seasonId: progressItem.seasonId,
+    episode: {
-    episodeId: progressItem.episodeId,
+      id: progressItem.episodeId,
+      number: progressItem.episodeNumber,
+    },
+    season: {
+      id: progressItem.seasonId,
+      number: progressItem.seasonNumber,
+    },
     meta: {
       title: progressItem.meta.title,
       year: progressItem.meta.year,

src/db/models/ProviderMetrics.ts

@@ -1,48 +0,0 @@
-import { Entity, PrimaryKey, Property } from '@mikro-orm/core';
-import { randomUUID } from 'crypto';
-
-export const status = {
-  failed: 'failed',
-  notfound: 'notfound',
-  success: 'success',
-} as const;
-type Status = keyof typeof status;
-
-@Entity({ tableName: 'provider_metrics' })
-export class ProviderMetric {
-  @PrimaryKey({ name: 'id', type: 'uuid' })
-  id: string = randomUUID();
-
-  @Property({ name: 'tmdb_id' })
-  tmdbId!: string;
-
-  @Property({ name: 'type' })
-  type!: string;
-
-  @Property({ name: 'title' })
-  title!: string;
-
-  @Property({ name: 'season_id', nullable: true })
-  seasonId?: string;
-
-  @Property({ name: 'episode_id', nullable: true })
-  episodeId?: string;
-
-  @Property({ name: 'created_at', type: 'date' })
-  createdAt = new Date();
-
-  @Property({ name: 'status' })
-  status!: Status;
-
-  @Property({ name: 'provider_id' })
-  providerId!: string;
-
-  @Property({ name: 'embed_id', nullable: true })
-  embedId?: string;
-
-  @Property({ name: 'error_message', nullable: true })
-  errorMessage?: string;
-
-  @Property({ name: 'full_error', nullable: true })
-  fullError?: string;
-}

src/db/models/UserSettings.ts

@@ -1,10 +1,9 @@
 import { Entity, PrimaryKey, Property } from '@mikro-orm/core';
-import { randomUUID } from 'crypto';
 
 @Entity({ tableName: 'user_settings' })
 export class UserSettings {
-  @PrimaryKey({ name: 'id', type: 'uuid' })
+  @PrimaryKey({ name: 'id', type: 'text' })
-  id: string = randomUUID();
+  id!: string;
 
   @Property({ name: 'application_theme', nullable: true })
   applicationTheme?: string | null;

src/mikro-orm.config.ts

@@ -1,4 +1,4 @@
+import { ormConf } from '@/config/orm';
 import { makeOrmConfig } from '@/modules/mikro/orm';
-import { conf } from '@/config';
 
-export default makeOrmConfig(conf.postgres.connection);
+export default makeOrmConfig(ormConf.postgres.connection);

src/modules/fastify/index.ts

@@ -56,8 +56,10 @@ export async function setupFastify(): Promise<FastifyInstance> {
 
   // plugins
   log.info(`setting up plugins`, { evt: 'setup-plugins' });
+  const corsDomains = conf.server.cors.split(' ').filter((v) => v.length > 0);
+  const corsSetting = conf.server.allowAnySite ? true : corsDomains;
   await app.register(cors, {
-    origin: conf.server.cors.split(' ').filter((v) => v.length > 0),
+    origin: corsSetting,
     credentials: true,
   });
 

src/modules/fastify/routes.ts

@@ -2,7 +2,7 @@ import { loginAuthRouter } from '@/routes/auth/login';
 import { manageAuthRouter } from '@/routes/auth/manage';
 import { metaRouter } from '@/routes/meta';
 import { metricsRouter } from '@/routes/metrics';
-import { sessionsRouter } from '@/routes/sessions';
+import { sessionsRouter } from '@/routes/sessions/sessions';
 import { userBookmarkRouter } from '@/routes/users/bookmark';
 import { userDeleteRouter } from '@/routes/users/delete';
 import { userEditRouter } from '@/routes/users/edit';

src/modules/jobs/index.ts

@@ -1,11 +1,9 @@
 import { challengeCodeJob } from '@/modules/jobs/list/challengeCode';
 import { sessionExpiryJob } from '@/modules/jobs/list/sessionExpiry';
 import { userDeletionJob } from '@/modules/jobs/list/userDeletion';
-import { providerMetricCleanupJob } from '@/modules/jobs/list/providerMetricCleanup';
 
 export async function setupJobs() {
   challengeCodeJob.start();
   sessionExpiryJob.start();
   userDeletionJob.start();
-  providerMetricCleanupJob.start();
 }

src/modules/jobs/list/providerMetricCleanup.ts

@@ -1,27 +0,0 @@
-import { ProviderMetric } from '@/db/models/ProviderMetrics';
-import { job } from '@/modules/jobs/job';
-import ms from 'ms';
-
-// every day at 12:00:00
-export const providerMetricCleanupJob = job(
-  'provider-metric-cleanup',
-  '0 12 * * *',
-  async ({ em, log }) => {
-    const now = new Date();
-    const thirtyDaysAgo = new Date(now.getTime() - ms('30d'));
-
-    const deletedMetrics = await em
-      .createQueryBuilder(ProviderMetric)
-      .delete()
-      .where({
-        createdAt: {
-          $lt: thirtyDaysAgo,
-        },
-      })
-      .execute<{ affectedRows: number }>('run');
-
-    log.info(
-      `Removed ${deletedMetrics.affectedRows} metrics that were older than 30 days`,
-    );
-  },
-);

src/modules/metrics/index.ts

@@ -9,16 +9,10 @@ const log = scopedLogger('metrics');
 
 export type Metrics = {
   user: Counter<'namespace'>;
-  providerMetrics: Counter<
+  captchaSolves: Counter<'success'>;
-    | 'title'
+  providerHostnames: Counter<'hostname'>;
-    | 'tmdb_id'
+  providerStatuses: Counter<'provider_id' | 'status'>;
-    | 'season_id'
+  watchMetrics: Counter<'title' | 'tmdb_full_id' | 'provider_id' | 'success'>;
-    | 'episode_id'
-    | 'status'
-    | 'type'
-    | 'provider_id'
-    | 'embed_id'
-  >;
 };
 
 let metrics: null | Metrics = null;
@@ -41,30 +35,39 @@ export async function setupMetrics(app: FastifyInstance) {
 
   metrics = {
     user: new Counter({
-      name: 'user_count',
+      name: 'mw_user_count',
-      help: 'user_help',
+      help: 'mw_user_help',
       labelNames: ['namespace'],
     }),
-    providerMetrics: new Counter({
+    captchaSolves: new Counter({
-      name: 'provider_metrics',
+      name: 'mw_captcha_solves',
-      help: 'provider_metrics',
+      help: 'mw_captcha_solves',
-      labelNames: [
+      labelNames: ['success'],
-        'episode_id',
+    }),
-        'provider_id',
+    providerHostnames: new Counter({
-        'season_id',
+      name: 'mw_provider_hostname_count',
-        'status',
+      help: 'mw_provider_hostname_count',
-        'title',
+      labelNames: ['hostname'],
-        'tmdb_id',
+    }),
-        'type',
+    providerStatuses: new Counter({
-        'embed_id',
+      name: 'mw_provider_status_count',
-      ],
+      help: 'mw_provider_status_count',
+      labelNames: ['provider_id', 'status'],
+    }),
+    watchMetrics: new Counter({
+      name: 'mw_media_watch_count',
+      help: 'mw_media_watch_count',
+      labelNames: ['title', 'tmdb_full_id', 'provider_id', 'success'],
     }),
   };
 
   const promClient = app.metrics.client;
 
   promClient.register.registerMetric(metrics.user);
-  promClient.register.registerMetric(metrics.providerMetrics);
+  promClient.register.registerMetric(metrics.providerHostnames);
+  promClient.register.registerMetric(metrics.providerStatuses);
+  promClient.register.registerMetric(metrics.watchMetrics);
+  promClient.register.registerMetric(metrics.captchaSolves);
 
   const orm = getORM();
   const em = orm.em.fork();

src/modules/mikro/index.ts

@@ -14,8 +14,10 @@ export function getORM() {
 
 export async function setupMikroORM() {
   log.info(`Connecting to postgres`, { evt: 'connecting' });
-  const mikro = await createORM(conf.postgres.connection, (msg) =>
+  const mikro = await createORM(
-    log.info(msg),
+    conf.postgres.connection,
+    conf.postgres.debugLogging,
+    (msg) => log.info(msg),
   );
 
   if (conf.postgres.syncSchema) {

src/modules/mikro/orm.ts

@@ -16,9 +16,14 @@ export function makeOrmConfig(url: string): Options<PostgreSqlDriver> {
   };
 }
 
-export async function createORM(url: string, log: (msg: string) => void) {
+export async function createORM(
+  url: string,
+  debug: boolean,
+  log: (msg: string) => void,
+) {
   return await MikroORM.init<PostgreSqlDriver>({
     ...makeOrmConfig(url),
     logger: log,
+    debug,
   });
 }

src/modules/ratelimits/limiter.ts

@@ -2,6 +2,7 @@ import Redis from 'ioredis';
 import RateLimiter from 'async-ratelimiter';
 import ms from 'ms';
 import { StatusError } from '@/services/error';
+import { IpReq, getIp } from '@/services/ip';
 
 export interface LimiterOptions {
   redis: Redis;
@@ -26,8 +27,8 @@ export class Limiter {
     this.redis = ops.redis;
   }
 
-  async bump(req: { ip: string }, ops: BucketOptions) {
+  async bump(req: IpReq, ops: BucketOptions) {
-    const ip = req.ip;
+    const ip = getIp(req);
     if (!this.buckets[ops.id]) {
       this.buckets[ops.id] = {
         limiter: new RateLimiter({
@@ -54,7 +55,7 @@ export class Limiter {
     };
   }
 
-  async assertAndBump(req: { ip: string }, ops: BucketOptions) {
+  async assertAndBump(req: IpReq, ops: BucketOptions) {
     const { hasBeenLimited } = await this.bump(req, ops);
     if (hasBeenLimited) {
       throw new StatusError('Ratelimited', 429);

src/routes/auth/login.ts

@@ -1,6 +1,6 @@
 import { ChallengeCode } from '@/db/models/ChallengeCode';
 import { formatSession } from '@/db/models/Session';
-import { User } from '@/db/models/User';
+import { User, formatUser } from '@/db/models/User';
 import { assertChallengeCode } from '@/services/challenge';
 import { StatusError } from '@/services/error';
 import { handle } from '@/services/handler';
@@ -85,6 +85,7 @@ export const loginAuthRouter = makeRouter((app) => {
         await em.persistAndFlush([session, user]);
 
         return {
+          user: formatUser(user),
           session: formatSession(session),
           token: makeSessionToken(session),
         };

src/routes/meta.ts

@@ -1,17 +1,21 @@
-import { conf } from '@/config';
+import { conf, version } from '@/config';
 import { handle } from '@/services/handler';
 import { makeRouter } from '@/services/router';
 
 export const metaRouter = makeRouter((app) => {
   app.get(
     '/healthcheck',
-    handle(async ({ em }) => {
+    handle(async ({ em, res }) => {
       const databaseConnected = await em.config
         .getDriver()
         .getConnection()
         .isConnected();
+
+      const healthy = databaseConnected;
+      if (!healthy) res.status(503);
+
       return {
-        healthy: databaseConnected,
+        healthy,
         databaseConnected,
       };
     }),
@@ -22,6 +26,7 @@ export const metaRouter = makeRouter((app) => {
       return {
         name: conf.meta.name,
         description: conf.meta.description,
+        version: version,
         hasCaptcha: conf.captcha.enabled,
         captchaClientKey: conf.captcha.clientKey,
       };

src/routes/metrics.ts

@@ -1,8 +1,8 @@
 import { handle } from '@/services/handler';
 import { makeRouter } from '@/services/router';
 import { z } from 'zod';
-import { ProviderMetric, status } from '@/db/models/ProviderMetrics';
 import { getMetrics } from '@/modules/metrics';
+import { status } from '@/routes/statuses';
 
 const metricsProviderSchema = z.object({
   tmdbId: z.string(),
@@ -29,7 +29,7 @@ export const metricsRouter = makeRouter((app) => {
         body: metricsProviderInputSchema,
       },
     },
-    handle(async ({ em, body, req, limiter }) => {
+    handle(async ({ body, req, limiter }) => {
       await limiter?.assertAndBump(req, {
         id: 'provider_metrics',
         max: 300,
@@ -37,36 +37,59 @@ export const metricsRouter = makeRouter((app) => {
         window: '30m',
       });
 
-      const entities = body.items.map((v) => {
+      const hostname = req.headers.origin?.slice(0, 255) ?? '<UNKNOWN>';
-        const metric = new ProviderMetric();
+      getMetrics().providerHostnames.inc({
-        em.assign(metric, {
+        hostname,
-          providerId: v.providerId,
-          embedId: v.embedId,
-          fullError: v.fullError,
-          errorMessage: v.errorMessage,
-          episodeId: v.episodeId,
-          seasonId: v.seasonId,
-          status: v.status,
-          title: v.title,
-          tmdbId: v.tmdbId,
-          type: v.type,
-        });
-        return metric;
       });
 
-      entities.forEach((entity) => {
+      body.items.forEach((item) => {
-        getMetrics().providerMetrics.inc({
+        getMetrics().providerStatuses.inc({
-          episode_id: entity.episodeId,
+          provider_id: item.embedId ?? item.providerId,
-          provider_id: entity.providerId,
+          status: item.status,
-          season_id: entity.seasonId,
-          status: entity.status,
-          title: entity.title,
-          tmdb_id: entity.tmdbId,
-          type: entity.type,
         });
       });
 
-      await em.persistAndFlush(entities);
+      const itemList = [...body.items];
+      itemList.reverse();
+      const lastSuccessfulItem = body.items.find(
+        (v) => v.status === status.success,
+      );
+      const lastItem = itemList[0];
+
+      if (lastItem) {
+        getMetrics().watchMetrics.inc({
+          tmdb_full_id: lastItem.type + '-' + lastItem.tmdbId,
+          provider_id: lastSuccessfulItem?.providerId ?? lastItem.providerId,
+          title: lastItem.title,
+          success: (!!lastSuccessfulItem).toString(),
+        });
+      }
+
+      return true;
+    }),
+  );
+
+  app.post(
+    '/metrics/captcha',
+    {
+      schema: {
+        body: z.object({
+          success: z.boolean(),
+        }),
+      },
+    },
+    handle(async ({ body, req, limiter }) => {
+      await limiter?.assertAndBump(req, {
+        id: 'captcha_solves',
+        max: 300,
+        inc: 1,
+        window: '30m',
+      });
+
+      getMetrics().captchaSolves.inc({
+        success: body.success.toString(),
+      });
+
       return true;
     }),
   );

src/routes/sessions/session.ts

@@ -1,35 +0,0 @@
-import { Session } from '@/db/models/Session';
-import { StatusError } from '@/services/error';
-import { handle } from '@/services/handler';
-import { makeRouter } from '@/services/router';
-import { z } from 'zod';
-
-export const sessionRouter = makeRouter((app) => {
-  app.delete(
-    '/sessions/:sid',
-    {
-      schema: {
-        params: z.object({
-          sid: z.string(),
-        }),
-      },
-    },
-    handle(async ({ auth, params, em }) => {
-      await auth.assert();
-
-      const targetedSession = await em.findOne(Session, { id: params.sid });
-      if (!targetedSession)
-        return {
-          id: params.sid,
-        };
-
-      if (targetedSession.user !== auth.user.id)
-        throw new StatusError('Cannot delete sessions you do not own', 401);
-
-      await em.removeAndFlush(targetedSession);
-      return {
-        id: params.sid,
-      };
-    }),
-  );
-});

src/routes/sessions/sessions.ts

@@ -13,7 +13,7 @@ export const sessionsRouter = makeRouter((app) => {
           sid: z.string(),
         }),
         body: z.object({
-          name: z.string().max(500).min(1).optional(),
+          deviceName: z.string().max(500).min(1).optional(),
         }),
       },
     },
@@ -25,10 +25,10 @@ export const sessionsRouter = makeRouter((app) => {
       if (!targetedSession)
         throw new StatusError('Session cannot be found', 404);
 
-      if (targetedSession.user !== auth.user.id)
+      if (targetedSession.id !== params.sid)
-        throw new StatusError('Cannot modify sessions you do not own', 401);
+        throw new StatusError('Cannot edit sessions other than your own', 401);
 
-      if (body.name) targetedSession.device = body.name;
+      if (body.deviceName) targetedSession.device = body.deviceName;
 
       await em.persistAndFlush(targetedSession);
 

src/routes/statuses.ts

@@ -0,0 +1,6 @@
+export const status = {
+  failed: 'failed',
+  notfound: 'notfound',
+  success: 'success',
+} as const;
+export type Status = keyof typeof status;

src/routes/users/bookmark.ts

@@ -8,6 +8,11 @@ import { handle } from '@/services/handler';
 import { makeRouter } from '@/services/router';
 import { z } from 'zod';
 
+const bookmarkDataSchema = z.object({
+  tmdbId: z.string(),
+  meta: bookmarkMetaSchema,
+});
+
 export const userBookmarkRouter = makeRouter((app) => {
   app.get(
     '/users/:uid/bookmarks',
@@ -40,9 +45,7 @@ export const userBookmarkRouter = makeRouter((app) => {
           uid: z.string(),
           tmdbid: z.string(),
         }),
-        body: z.object({
+        body: bookmarkDataSchema,
-          meta: bookmarkMetaSchema,
-        }),
       },
     },
     handle(async ({ auth, params, body, em }) => {
@@ -70,6 +73,40 @@ export const userBookmarkRouter = makeRouter((app) => {
     }),
   );
 
+  app.put(
+    '/users/:uid/bookmarks',
+    {
+      schema: {
+        params: z.object({
+          uid: z.string(),
+        }),
+        body: z.array(bookmarkDataSchema),
+      },
+    },
+    handle(async ({ auth, params, body, em }) => {
+      await auth.assert();
+
+      if (auth.user.id !== params.uid)
+        throw new StatusError('Cannot modify user other than yourself', 403);
+
+      const bookmarks = await em.upsertMany(
+        Bookmark,
+        body.map((item) => ({
+          userId: params.uid,
+          tmdbId: item.tmdbId,
+          meta: item.meta,
+          updatedAt: new Date(),
+        })),
+        {
+          onConflictFields: ['tmdbId', 'userId'],
+        },
+      );
+
+      await em.flush();
+      return bookmarks.map(formatBookmark);
+    }),
+  );
+
   app.delete(
     '/users/:uid/bookmarks/:tmdbid',
     {

src/routes/users/get.ts

@@ -1,3 +1,4 @@
+import { formatSession } from '@/db/models/Session';
 import { User, formatUser } from '@/db/models/User';
 import { StatusError } from '@/services/error';
 import { handle } from '@/services/handler';
@@ -5,6 +6,24 @@ import { makeRouter } from '@/services/router';
 import { z } from 'zod';
 
 export const userGetRouter = makeRouter((app) => {
+  app.get(
+    '/users/@me',
+    handle(async ({ auth, em }) => {
+      await auth.assert();
+
+      const user = await em.findOne(User, { id: auth.user.id });
+      if (!user) throw new StatusError('User does not exist', 404);
+
+      const session = await auth.getSession();
+      if (!session) throw new StatusError('Session does not exist', 400);
+
+      return {
+        user: formatUser(user),
+        session: formatSession(session),
+      };
+    }),
+  );
+
   app.get(
     '/users/:uid',
     {
@@ -16,16 +35,16 @@ export const userGetRouter = makeRouter((app) => {
     },
     handle(async ({ auth, params, em }) => {
       await auth.assert();
-      let uid = params.uid;
-      if (uid === '@me') uid = auth.user.id;
 
-      if (auth.user.id !== uid)
+      if (auth.user.id !== params.uid)
         throw new StatusError('Cannot access users other than yourself', 403);
 
-      const user = await em.findOne(User, { id: uid });
+      const user = await em.findOne(User, { id: params.uid });
       if (!user) throw new StatusError('User does not exist', 404);
 
-      return formatUser(user);
+      return {
+        user: formatUser(user),
+      };
     }),
   );
 });

src/routes/users/progress.ts

@@ -6,8 +6,22 @@ import {
 import { StatusError } from '@/services/error';
 import { handle } from '@/services/handler';
 import { makeRouter } from '@/services/router';
+import { FilterQuery } from '@mikro-orm/core';
+import { randomUUID } from 'crypto';
 import { z } from 'zod';
 
+const progressItemSchema = z.object({
+  meta: progressMetaSchema,
+  tmdbId: z.string(),
+  duration: z.number().transform((n) => Math.round(n)),
+  watched: z.number().transform((n) => Math.round(n)),
+  seasonId: z.string().optional(),
+  episodeId: z.string().optional(),
+  seasonNumber: z.number().optional(),
+  episodeNumber: z.number().optional(),
+  updatedAt: z.string().datetime({ offset: true }).optional(),
+});
+
 export const userProgressRouter = makeRouter((app) => {
   app.put(
     '/users/:uid/progress/:tmdbid',
@@ -17,13 +31,7 @@ export const userProgressRouter = makeRouter((app) => {
           uid: z.string(),
           tmdbid: z.string(),
         }),
-        body: z.object({
+        body: progressItemSchema,
-          meta: progressMetaSchema,
-          seasonId: z.string().optional(),
-          episodeId: z.string().optional(),
-          duration: z.number(),
-          watched: z.number(),
-        }),
       },
     },
     handle(async ({ auth, params, body, em }) => {
@@ -38,12 +46,15 @@ export const userProgressRouter = makeRouter((app) => {
         episodeId: body.episodeId,
         seasonId: body.seasonId,
       });
+
       if (!progressItem) {
         progressItem = new ProgressItem();
         progressItem.tmdbId = params.tmdbid;
         progressItem.userId = params.uid;
         progressItem.episodeId = body.episodeId;
         progressItem.seasonId = body.seasonId;
+        progressItem.episodeNumber = body.episodeNumber;
+        progressItem.seasonNumber = body.seasonNumber;
       }
 
       em.assign(progressItem, {
@@ -58,6 +69,82 @@ export const userProgressRouter = makeRouter((app) => {
     }),
   );
 
+  app.put(
+    '/users/:uid/progress/import',
+    {
+      schema: {
+        params: z.object({
+          uid: z.string(),
+        }),
+        body: z.array(progressItemSchema),
+      },
+    },
+    handle(async ({ auth, params, body, em, req, limiter }) => {
+      await auth.assert();
+
+      if (auth.user.id !== params.uid)
+        throw new StatusError('Cannot modify user other than yourself', 403);
+
+      const itemsUpserted: ProgressItem[] = [];
+
+      const newItems = [...body];
+
+      for (const existingItem of await em.find(ProgressItem, {
+        userId: params.uid,
+      })) {
+        const newItemIndex = newItems.findIndex(
+          (item) =>
+            item.tmdbId == existingItem.tmdbId &&
+            item.seasonId == existingItem.seasonId &&
+            item.episodeId == existingItem.episodeId,
+        );
+
+        if (newItemIndex > -1) {
+          const newItem = newItems[newItemIndex];
+          if (existingItem.watched < newItem.watched) {
+            existingItem.updatedAt = defaultAndCoerceDateTime(
+              newItem.updatedAt,
+            );
+            existingItem.watched = newItem.watched;
+          }
+          itemsUpserted.push(existingItem);
+
+          // Remove the item from the array, we have processed it
+          newItems.splice(newItemIndex, 1);
+        }
+      }
+
+      // All unprocessed items, aka all items that don't already exist
+      for (const newItem of newItems) {
+        itemsUpserted.push({
+          id: randomUUID(),
+          duration: newItem.duration,
+          episodeId: newItem.episodeId,
+          episodeNumber: newItem.episodeNumber,
+          meta: newItem.meta,
+          seasonId: newItem.seasonId,
+          seasonNumber: newItem.seasonNumber,
+          tmdbId: newItem.tmdbId,
+          userId: params.uid,
+          watched: newItem.watched,
+          updatedAt: defaultAndCoerceDateTime(newItem.updatedAt),
+        });
+      }
+
+      const progressItems = await em.upsertMany(ProgressItem, itemsUpserted);
+
+      await em.flush();
+
+      await limiter?.assertAndBump(req, {
+        id: 'progress_import',
+        max: 5,
+        window: '10m',
+      });
+
+      return progressItems.map(formatProgressItem);
+    }),
+  );
+
   app.delete(
     '/users/:uid/progress/:tmdbid',
     {
@@ -78,22 +165,28 @@ export const userProgressRouter = makeRouter((app) => {
       if (auth.user.id !== params.uid)
         throw new StatusError('Cannot modify user other than yourself', 403);
 
-      const progressItem = await em.findOne(ProgressItem, {
+      const query: FilterQuery<ProgressItem> = {
         userId: params.uid,
         tmdbId: params.tmdbid,
-        episodeId: body.episodeId,
+      };
-        seasonId: body.seasonId,
+      if (body.seasonId) query.seasonId = body.seasonId;
-      });
+      if (body.episodeId) query.episodeId = body.episodeId;
-      if (!progressItem) {
+      const progressItems = await em.find(ProgressItem, query);
+
+      if (progressItems.length === 0) {
         return {
+          count: 0,
           tmdbId: params.tmdbid,
           episodeId: body.episodeId,
           seasonId: body.seasonId,
         };
       }
 
-      await em.removeAndFlush(progressItem);
+      progressItems.forEach((v) => em.remove(v));
+      await em.flush();
+
       return {
+        count: progressItems.length,
         tmdbId: params.tmdbid,
         episodeId: body.episodeId,
         seasonId: body.seasonId,
@@ -124,3 +217,14 @@ export const userProgressRouter = makeRouter((app) => {
     }),
   );
 });
+
+// 13th July 2021 - movie-web epoch
+const minEpoch = 1626134400000;
+
+function defaultAndCoerceDateTime(dateTime: string | undefined) {
+  const epoch = dateTime ? new Date(dateTime).getTime() : Date.now();
+
+  const clampedEpoch = Math.max(minEpoch, Math.min(epoch, Date.now()));
+
+  return new Date(clampedEpoch);
+}

src/routes/users/settings.ts

@@ -38,9 +38,9 @@ export const userSettingsRouter = makeRouter((app) => {
           uid: z.string(),
         }),
         body: z.object({
-          applicationLanguage: z.string().optional(),
+          applicationLanguage: z.string().nullable().optional(),
-          applicationTheme: z.string().optional(),
+          applicationTheme: z.string().nullable().optional(),
-          defaultSubtitleLanguage: z.string().optional(),
+          defaultSubtitleLanguage: z.string().nullable().optional(),
         }),
       },
     },
@@ -58,12 +58,12 @@ export const userSettingsRouter = makeRouter((app) => {
         settings.id = params.uid;
       }
 
-      if (body.applicationLanguage)
+      if (body.applicationLanguage !== undefined)
         settings.applicationLanguage = body.applicationLanguage;
-      if (body.applicationTheme)
+      if (body.defaultSubtitleLanguage !== undefined)
-        settings.applicationTheme = body.applicationTheme;
-      if (body.defaultSubtitleLanguage)
         settings.defaultSubtitleLanguage = body.defaultSubtitleLanguage;
+      if (body.applicationTheme !== undefined)
+        settings.applicationTheme = body.applicationTheme;
 
       await em.persistAndFlush(settings);
       return formatUserSettings(settings);

src/services/ip.ts

@@ -0,0 +1,27 @@
+import { conf } from '@/config';
+import { IncomingHttpHeaders } from 'http';
+
+export type IpReq = {
+  ip: string;
+  headers: IncomingHttpHeaders;
+};
+
+const trustCloudflare = conf.server.trustCloudflare;
+
+function getSingleHeader(
+  headers: IncomingHttpHeaders,
+  key: string,
+): string | undefined {
+  const header = headers[key];
+  if (Array.isArray(header)) return header[0];
+  return header;
+}
+
+export function getIp(req: IpReq) {
+  const cfIp = getSingleHeader(req.headers, 'cf-connecting-ip');
+  if (trustCloudflare && cfIp) {
+    return cfIp;
+  }
+
+  return req.ip;
+}

src/services/logger.ts

@@ -64,6 +64,8 @@ export function scopedLogger(service: string, meta: object = {}) {
   return logger;
 }
 
+const ignoredUrls = ['/healthcheck', '/metrics'];
+
 export function makeFastifyLogger(logger: winston.Logger) {
   logger.format = winston.format.combine(
     winston.format((info) => {
@@ -78,6 +80,9 @@ export function makeFastifyLogger(logger: winston.Logger) {
         let url = request.url;
         try {
           const pathParts = (request.url as string).split('?', 2);
+
+          if (ignoredUrls.includes(pathParts[0])) return false;
+
           if (pathParts[1]) {
             const searchParams = new URLSearchParams(pathParts[1]);
             pathParts[1] = searchParams.toString();