Remove provider metric table and replace prometheus metrics with something more efficient

Co-authored-by: William Oldham <github@binaryoverload.co.uk>

src/config/orm.ts

@@ -1,6 +1,13 @@
+import { devFragment } from '@/config/fragments/dev';
+import { dockerFragment } from '@/config/fragments/docker';
 import { createConfigLoader } from 'neat-config';
 import { z } from 'zod';
 
+const fragments = {
+  dev: devFragment,
+  dockerdev: dockerFragment,
+};
+
 export const ormConfigSchema = z.object({
   postgres: z.object({
     // connection URL for postgres database
@@ -15,6 +22,8 @@ export const ormConf = createConfigLoader()
     prefix: 'MWB_',
   })
   .addFromFile('config.json')
+  .setFragmentKey('usePresets')
+  .addConfigFragments(fragments)
   .addZodSchema(ormConfigSchema)
   .freeze()
   .load();

src/db/migrations/.snapshot-postgres.json

@@ -268,143 +268,6 @@
       "checks": [],
       "foreignKeys": {}
     },
-    {
-      "columns": {
-        "id": {
-          "name": "id",
-          "type": "uuid",
-          "unsigned": false,
-          "autoincrement": false,
-          "primary": false,
-          "nullable": false,
-          "mappedType": "uuid"
-        },
-        "tmdb_id": {
-          "name": "tmdb_id",
-          "type": "varchar(255)",
-          "unsigned": false,
-          "autoincrement": false,
-          "primary": false,
-          "nullable": false,
-          "mappedType": "string"
-        },
-        "type": {
-          "name": "type",
-          "type": "varchar(255)",
-          "unsigned": false,
-          "autoincrement": false,
-          "primary": false,
-          "nullable": false,
-          "mappedType": "string"
-        },
-        "title": {
-          "name": "title",
-          "type": "varchar(255)",
-          "unsigned": false,
-          "autoincrement": false,
-          "primary": false,
-          "nullable": false,
-          "mappedType": "string"
-        },
-        "season_id": {
-          "name": "season_id",
-          "type": "varchar(255)",
-          "unsigned": false,
-          "autoincrement": false,
-          "primary": false,
-          "nullable": true,
-          "mappedType": "string"
-        },
-        "episode_id": {
-          "name": "episode_id",
-          "type": "varchar(255)",
-          "unsigned": false,
-          "autoincrement": false,
-          "primary": false,
-          "nullable": true,
-          "mappedType": "string"
-        },
-        "created_at": {
-          "name": "created_at",
-          "type": "timestamptz(0)",
-          "unsigned": false,
-          "autoincrement": false,
-          "primary": false,
-          "nullable": false,
-          "length": 0,
-          "mappedType": "datetime"
-        },
-        "status": {
-          "name": "status",
-          "type": "varchar(255)",
-          "unsigned": false,
-          "autoincrement": false,
-          "primary": false,
-          "nullable": false,
-          "mappedType": "string"
-        },
-        "provider_id": {
-          "name": "provider_id",
-          "type": "varchar(255)",
-          "unsigned": false,
-          "autoincrement": false,
-          "primary": false,
-          "nullable": false,
-          "mappedType": "string"
-        },
-        "embed_id": {
-          "name": "embed_id",
-          "type": "varchar(255)",
-          "unsigned": false,
-          "autoincrement": false,
-          "primary": false,
-          "nullable": true,
-          "mappedType": "string"
-        },
-        "error_message": {
-          "name": "error_message",
-          "type": "text",
-          "unsigned": false,
-          "autoincrement": false,
-          "primary": false,
-          "nullable": true,
-          "mappedType": "text"
-        },
-        "full_error": {
-          "name": "full_error",
-          "type": "text",
-          "unsigned": false,
-          "autoincrement": false,
-          "primary": false,
-          "nullable": true,
-          "mappedType": "text"
-        },
-        "hostname": {
-          "name": "hostname",
-          "type": "varchar(255)",
-          "unsigned": false,
-          "autoincrement": false,
-          "primary": false,
-          "nullable": false,
-          "mappedType": "string"
-        }
-      },
-      "name": "provider_metrics",
-      "schema": "public",
-      "indexes": [
-        {
-          "keyName": "provider_metrics_pkey",
-          "columnNames": [
-            "id"
-          ],
-          "composite": false,
-          "primary": true,
-          "unique": true
-        }
-      ],
-      "checks": [],
-      "foreignKeys": {}
-    },
     {
       "columns": {
         "id": {

src/db/migrations/Migration20231221185725.ts

@@ -0,0 +1,13 @@
+import { Migration } from '@mikro-orm/migrations';
+
+export class Migration20231221185725 extends Migration {
+
+  async up(): Promise<void> {
+    this.addSql('drop table if exists "provider_metrics" cascade;');
+  }
+
+  async down(): Promise<void> {
+    this.addSql('create table "provider_metrics" ("id" uuid not null default null, "tmdb_id" varchar not null default null, "type" varchar not null default null, "title" varchar not null default null, "season_id" varchar null default null, "episode_id" varchar null default null, "created_at" timestamptz not null default null, "status" varchar not null default null, "provider_id" varchar not null default null, "embed_id" varchar null default null, "error_message" text null default null, "full_error" text null default null, "hostname" varchar not null default null, constraint "provider_metrics_pkey" primary key ("id"));');
+  }
+
+}

src/db/models/ProviderMetrics.ts

@@ -1,51 +0,0 @@
-import { Entity, PrimaryKey, Property } from '@mikro-orm/core';
-import { randomUUID } from 'crypto';
-
-export const status = {
-  failed: 'failed',
-  notfound: 'notfound',
-  success: 'success',
-} as const;
-type Status = keyof typeof status;
-
-@Entity({ tableName: 'provider_metrics' })
-export class ProviderMetric {
-  @PrimaryKey({ name: 'id', type: 'uuid' })
-  id: string = randomUUID();
-
-  @Property({ name: 'tmdb_id' })
-  tmdbId!: string;
-
-  @Property({ name: 'type' })
-  type!: string;
-
-  @Property({ name: 'title' })
-  title!: string;
-
-  @Property({ name: 'season_id', nullable: true })
-  seasonId?: string;
-
-  @Property({ name: 'episode_id', nullable: true })
-  episodeId?: string;
-
-  @Property({ name: 'created_at', type: 'date' })
-  createdAt = new Date();
-
-  @Property({ name: 'status' })
-  status!: Status;
-
-  @Property({ name: 'provider_id' })
-  providerId!: string;
-
-  @Property({ name: 'embed_id', nullable: true })
-  embedId?: string;
-
-  @Property({ name: 'error_message', nullable: true, type: 'text' })
-  errorMessage?: string;
-
-  @Property({ name: 'full_error', nullable: true, type: 'text' })
-  fullError?: string;
-
-  @Property({ name: 'hostname' })
-  hostname!: string;
-}

src/modules/jobs/index.ts

@@ -1,11 +1,9 @@
 import { challengeCodeJob } from '@/modules/jobs/list/challengeCode';
 import { sessionExpiryJob } from '@/modules/jobs/list/sessionExpiry';
 import { userDeletionJob } from '@/modules/jobs/list/userDeletion';
-import { providerMetricCleanupJob } from '@/modules/jobs/list/providerMetricCleanup';
 
 export async function setupJobs() {
   challengeCodeJob.start();
   sessionExpiryJob.start();
   userDeletionJob.start();
-  providerMetricCleanupJob.start();
 }

src/modules/jobs/list/providerMetricCleanup.ts

@@ -1,27 +0,0 @@
-import { ProviderMetric } from '@/db/models/ProviderMetrics';
-import { job } from '@/modules/jobs/job';
-import ms from 'ms';
-
-// every day at 12:00:00
-export const providerMetricCleanupJob = job(
-  'provider-metric-cleanup',
-  '0 12 * * *',
-  async ({ em, log }) => {
-    const now = new Date();
-    const thirtyDaysAgo = new Date(now.getTime() - ms('30d'));
-
-    const deletedMetrics = await em
-      .createQueryBuilder(ProviderMetric)
-      .delete()
-      .where({
-        createdAt: {
-          $lt: thirtyDaysAgo,
-        },
-      })
-      .execute<{ affectedRows: number }>('run');
-
-    log.info(
-      `Removed ${deletedMetrics.affectedRows} metrics that were older than 30 days`,
-    );
-  },
-);

src/modules/metrics/index.ts

@@ -9,17 +9,9 @@ const log = scopedLogger('metrics');
 
 export type Metrics = {
   user: Counter<'namespace'>;
-  providerMetrics: Counter<
+  providerHostnames: Counter<'hostname'>;
-    | 'title'
+  providerStatuses: Counter<'provider_id' | 'status'>;
-    | 'tmdb_id'
+  watchMetrics: Counter<'title' | 'tmdb_full_id' | 'provider_id' | 'success'>;
-    | 'season_id'
-    | 'episode_id'
-    | 'status'
-    | 'type'
-    | 'provider_id'
-    | 'embed_id'
-    | 'hostname'
-  >;
 };
 
 let metrics: null | Metrics = null;
@@ -42,31 +34,33 @@ export async function setupMetrics(app: FastifyInstance) {
 
   metrics = {
     user: new Counter({
-      name: 'user_count',
+      name: 'mw_user_count',
-      help: 'user_help',
+      help: 'mw_user_help',
       labelNames: ['namespace'],
     }),
-    providerMetrics: new Counter({
+    providerHostnames: new Counter({
-      name: 'provider_metrics',
+      name: 'mw_provider_hostname_count',
-      help: 'provider_metrics',
+      help: 'mw_provider_hostname_count',
-      labelNames: [
+      labelNames: ['hostname'],
-        'episode_id',
+    }),
-        'provider_id',
+    providerStatuses: new Counter({
-        'season_id',
+      name: 'mw_provider_status_count',
-        'status',
+      help: 'mw_provider_status_count',
-        'title',
+      labelNames: ['provider_id', 'status'],
-        'tmdb_id',
+    }),
-        'type',
+    watchMetrics: new Counter({
-        'embed_id',
+      name: 'mw_media_watch_count',
-        'hostname',
+      help: 'mw_media_watch_count',
-      ],
+      labelNames: ['title', 'tmdb_full_id', 'provider_id', 'success'],
     }),
   };
 
   const promClient = app.metrics.client;
 
   promClient.register.registerMetric(metrics.user);
-  promClient.register.registerMetric(metrics.providerMetrics);
+  promClient.register.registerMetric(metrics.providerHostnames);
+  promClient.register.registerMetric(metrics.providerStatuses);
+  promClient.register.registerMetric(metrics.watchMetrics);
 
   const orm = getORM();
   const em = orm.em.fork();

src/routes/metrics.ts

@@ -1,8 +1,8 @@
 import { handle } from '@/services/handler';
 import { makeRouter } from '@/services/router';
 import { z } from 'zod';
-import { ProviderMetric, status } from '@/db/models/ProviderMetrics';
 import { getMetrics } from '@/modules/metrics';
+import { status } from '@/routes/statuses';
 
 const metricsProviderSchema = z.object({
   tmdbId: z.string(),
@@ -29,7 +29,7 @@ export const metricsRouter = makeRouter((app) => {
         body: metricsProviderInputSchema,
       },
     },
-    handle(async ({ em, body, req, limiter }) => {
+    handle(async ({ body, req, limiter }) => {
       await limiter?.assertAndBump(req, {
         id: 'provider_metrics',
         max: 300,
@@ -37,43 +37,34 @@ export const metricsRouter = makeRouter((app) => {
         window: '30m',
       });
 
-      const hostname = req.headers.origin?.slice(0, 255) ?? 'unknown origin';
+      const hostname = req.headers.origin?.slice(0, 255) ?? '<UNKNOWN>';
-
+      getMetrics().providerHostnames.inc({
-      const entities = body.items.map((v) => {
+        hostname,
-        const errorMessage = v.errorMessage?.slice(0, 200);
-        const truncatedFullError = v.fullError?.slice(0, 2000);
-
-        const metric = new ProviderMetric();
-        em.assign(metric, {
-          providerId: v.providerId,
-          embedId: v.embedId,
-          fullError: truncatedFullError,
-          errorMessage: errorMessage,
-          episodeId: v.episodeId,
-          seasonId: v.seasonId,
-          status: v.status,
-          title: v.title,
-          tmdbId: v.tmdbId,
-          type: v.type,
-          hostname,
-        });
-        return metric;
       });
 
-      entities.forEach((entity) => {
+      body.items.forEach((item) => {
-        getMetrics().providerMetrics.inc({
+        getMetrics().providerStatuses.inc({
-          episode_id: entity.episodeId,
+          provider_id: item.embedId ?? item.providerId,
-          provider_id: entity.providerId,
+          status: item.status,
-          season_id: entity.seasonId,
-          status: entity.status,
-          title: entity.title,
-          tmdb_id: entity.tmdbId,
-          type: entity.type,
-          hostname,
         });
       });
 
-      await em.persistAndFlush(entities);
+      const itemList = [...body.items];
+      itemList.reverse();
+      const lastSuccessfulItem = body.items.find(
+        (v) => v.status === status.success,
+      );
+      const lastItem = itemList[0];
+
+      if (lastItem) {
+        getMetrics().watchMetrics.inc({
+          tmdb_full_id: lastItem.tmdbId,
+          provider_id: lastSuccessfulItem?.providerId ?? lastItem.providerId,
+          title: lastItem.title,
+          success: (!!lastSuccessfulItem).toString(),
+        });
+      }
+
       return true;
     }),
   );

src/routes/statuses.ts

@@ -0,0 +1,6 @@
+export const status = {
+  failed: 'failed',
+  notfound: 'notfound',
+  success: 'success',
+} as const;
+export type Status = keyof typeof status;