From 0c3de831fb774ea70113f33b8ed45cfb449e95d1 Mon Sep 17 00:00:00 2001 From: caio_nogueira_27 Date: Sun, 31 Dec 2023 13:03:32 +0000 Subject: [PATCH 01/11] update deprecated composer syntax; change body format in API request --- .docker/development/docker-compose.yml | 2 +- src/services/captcha.ts | 12 +++++------- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/.docker/development/docker-compose.yml b/.docker/development/docker-compose.yml index e094d89..01f76d6 100644 --- a/.docker/development/docker-compose.yml +++ b/.docker/development/docker-compose.yml @@ -41,7 +41,7 @@ services: links: - postgres:postgres environment: - - DATABASE_URL=postgres://postgres:postgres@postgres:5432/postgres?sslmode=disable + - PGWEB_DATABASE_URL=postgres://postgres:postgres@postgres:5432/postgres?sslmode=disable depends_on: - postgres diff --git a/src/services/captcha.ts b/src/services/captcha.ts index e6cf4ad..68d7c4d 100644 --- a/src/services/captcha.ts +++ b/src/services/captcha.ts @@ -4,16 +4,14 @@ import { StatusError } from '@/services/error'; export async function isValidCaptcha(token: string): Promise { if (!conf.captcha.secret) throw new Error('isValidCaptcha() is called but no secret set'); + const formData = new URLSearchParams(); + formData.append('secret', conf.captcha.secret); + formData.append('response', token); const res = await fetch('https://www.google.com/recaptcha/api/siteverify', { method: 'POST', - body: JSON.stringify({ - secret: conf.captcha.secret, - response: token, - }), - headers: { - 'content-type': 'application/json', - }, + body: formData, }); + const json = await res.json(); return !!json.success; } From e7c5d93cc38d47dd31d6bc794be7e4ce4e64230b Mon Sep 17 00:00:00 2001 From: weeryan17 Date: Sun, 7 Jan 2024 12:42:45 -0500 Subject: [PATCH 02/11] Update workflows to support arm --- .github/workflows/linting_testing.yml | 14 +++++++------- .github/workflows/release.yml | 11 +++++++---- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/.github/workflows/linting_testing.yml b/.github/workflows/linting_testing.yml index ed9f673..4847dde 100644 --- a/.github/workflows/linting_testing.yml +++ b/.github/workflows/linting_testing.yml @@ -14,7 +14,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - uses: pnpm/action-setup@v2 with: @@ -23,7 +23,7 @@ jobs: - name: Install Node.js uses: actions/setup-node@v3 with: - node-version: 18 + node-version: 20 cache: 'pnpm' - name: Install packages @@ -38,7 +38,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - uses: pnpm/action-setup@v2 with: @@ -47,7 +47,7 @@ jobs: - name: Install Node.js uses: actions/setup-node@v3 with: - node-version: 18 + node-version: 20 cache: 'pnpm' - name: Install packages @@ -62,10 +62,10 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 - name: Build - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v5 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e342cb7..dcdb309 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -12,7 +12,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Get version id: package-version @@ -42,10 +42,10 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup Docker buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 - name: Get version id: package-version @@ -70,9 +70,12 @@ jobs: - name: Build and push Docker image id: build-and-push - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v5 with: push: true + platforms: linux/amd64,linux/arm64,linux/arm context: . labels: ${{ steps.meta.outputs.labels }} tags: ${{ steps.meta.outputs.tags }} + cache-from: type=gha + cache-to: type=gha,mode=max From 78e6b6443f77904027ec9866c2dc571a71ef7fba Mon Sep 17 00:00:00 2001 From: weeryan17 Date: Sun, 7 Jan 2024 12:51:09 -0500 Subject: [PATCH 03/11] Update CODEOWNERS file --- .github/CODEOWNERS | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index d0f0ca6..7458772 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,3 +1 @@ -* @movie-web/core - -.github @binaryoverload +* @movie-web/project-leads From 9073010f4c2528d972806c598232b761da6eb8a8 Mon Sep 17 00:00:00 2001 From: weeryan17 Date: Sun, 7 Jan 2024 12:55:24 -0500 Subject: [PATCH 04/11] Update .github/workflows/linting_testing.yml Co-authored-by: William Oldham --- .github/workflows/linting_testing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/linting_testing.yml b/.github/workflows/linting_testing.yml index 4847dde..8f0fd5d 100644 --- a/.github/workflows/linting_testing.yml +++ b/.github/workflows/linting_testing.yml @@ -21,7 +21,7 @@ jobs: version: 8 - name: Install Node.js - uses: actions/setup-node@v3 + uses: actions/setup-node@v4 with: node-version: 20 cache: 'pnpm' From 5855943a56a2c533e2b9f7a1c7f81bc5496f2a7b Mon Sep 17 00:00:00 2001 From: weeryan17 Date: Sun, 7 Jan 2024 12:55:30 -0500 Subject: [PATCH 05/11] Update .github/workflows/linting_testing.yml Co-authored-by: William Oldham --- .github/workflows/linting_testing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/linting_testing.yml b/.github/workflows/linting_testing.yml index 8f0fd5d..8e98406 100644 --- a/.github/workflows/linting_testing.yml +++ b/.github/workflows/linting_testing.yml @@ -45,7 +45,7 @@ jobs: version: 8 - name: Install Node.js - uses: actions/setup-node@v3 + uses: actions/setup-node@v4 with: node-version: 20 cache: 'pnpm' From 14601b9010fafe2227d7d973ef054c763b9836e1 Mon Sep 17 00:00:00 2001 From: William Oldham Date: Tue, 9 Jan 2024 21:01:52 +0000 Subject: [PATCH 06/11] Remove ARM 32-bit from Docker Build --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index dcdb309..22da909 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -73,7 +73,7 @@ jobs: uses: docker/build-push-action@v5 with: push: true - platforms: linux/amd64,linux/arm64,linux/arm + platforms: linux/amd64,linux/arm64 context: . labels: ${{ steps.meta.outputs.labels }} tags: ${{ steps.meta.outputs.tags }} From 7b26b029de4ac7b23f5de9900e19748d639e5bd4 Mon Sep 17 00:00:00 2001 From: Kevin Carrera Calzado Date: Wed, 24 Jan 2024 11:11:18 -0500 Subject: [PATCH 07/11] Fix:Implement function to handle boolean strings --- src/config/orm.ts | 3 ++- src/config/schema.ts | 22 ++++++++++++---------- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/src/config/orm.ts b/src/config/orm.ts index bfbe038..c7e7be9 100644 --- a/src/config/orm.ts +++ b/src/config/orm.ts @@ -2,6 +2,7 @@ import { devFragment } from '@/config/fragments/dev'; import { dockerFragment } from '@/config/fragments/docker'; import { createConfigLoader } from 'neat-config'; import { z } from 'zod'; +import { booleanSchema } from './schema'; const fragments = { dev: devFragment, @@ -13,7 +14,7 @@ export const ormConfigSchema = z.object({ // connection URL for postgres database connection: z.string(), // whether to use SSL for the connection - ssl: z.coerce.boolean().default(false), + ssl: booleanSchema, }), }); diff --git a/src/config/schema.ts b/src/config/schema.ts index d42327e..640d035 100644 --- a/src/config/schema.ts +++ b/src/config/schema.ts @@ -1,5 +1,7 @@ import { z } from 'zod'; +export const booleanSchema = z.preprocess((val) => val === 'true', z.boolean()); + export const configSchema = z.object({ server: z .object({ @@ -11,13 +13,13 @@ export const configSchema = z.object({ // disable cross origin restrictions, allow any site. // overwrites the cors option above - allowAnySite: z.coerce.boolean().default(false), + allowAnySite: booleanSchema, // should it trust reverse proxy headers? (for ip gathering) - trustProxy: z.coerce.boolean().default(false), + trustProxy: booleanSchema, // should it trust cloudflare headers? (for ip gathering, cloudflare has priority) - trustCloudflare: z.coerce.boolean().default(false), + trustCloudflare: booleanSchema, // prefix for where the instance is run on. for example set it to /backend if you're hosting it on example.com/backend // if this is set, do not apply url rewriting before proxing @@ -30,7 +32,7 @@ export const configSchema = z.object({ format: z.enum(['json', 'pretty']).default('pretty'), // show debug logs? - debug: z.coerce.boolean().default(false), + debug: booleanSchema, }) .default({}), postgres: z.object({ @@ -38,19 +40,19 @@ export const configSchema = z.object({ connection: z.string(), // run all migrations on boot of the application - migrateOnBoot: z.coerce.boolean().default(false), + migrateOnBoot: booleanSchema, // try to sync the schema on boot, useful for development // will always keep the database schema in sync with the connected database // it is extremely destructive, do not use it EVER in production - syncSchema: z.coerce.boolean().default(false), + syncSchema: booleanSchema, // Enable debug logging for MikroORM - Outputs queries and entity management logs // Do NOT use in production, leaks all sensitive data - debugLogging: z.coerce.boolean().default(false), + debugLogging: booleanSchema, // Enable SSL for the postgres connection - ssl: z.coerce.boolean().default(false), + ssl: booleanSchema, }), crypto: z.object({ // session secret. used for signing session tokens @@ -65,7 +67,7 @@ export const configSchema = z.object({ captcha: z .object({ // enabled captchas on register - enabled: z.coerce.boolean().default(false), + enabled: booleanSchema, // captcha secret secret: z.string().min(1).optional(), @@ -76,7 +78,7 @@ export const configSchema = z.object({ ratelimits: z .object({ // enabled captchas on register - enabled: z.coerce.boolean().default(false), + enabled: booleanSchema, redisUrl: z.string().optional(), }) .default({}), From a911e52ddb3e200857dfbabf0151eb21518434c5 Mon Sep 17 00:00:00 2001 From: Kevin Carrera Calzado Date: Wed, 24 Jan 2024 12:54:24 -0500 Subject: [PATCH 08/11] Add default false value to boolean schema --- src/config/orm.ts | 2 +- src/config/schema.ts | 20 ++++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/config/orm.ts b/src/config/orm.ts index c7e7be9..a226a20 100644 --- a/src/config/orm.ts +++ b/src/config/orm.ts @@ -14,7 +14,7 @@ export const ormConfigSchema = z.object({ // connection URL for postgres database connection: z.string(), // whether to use SSL for the connection - ssl: booleanSchema, + ssl: booleanSchema.default(false), }), }); diff --git a/src/config/schema.ts b/src/config/schema.ts index 640d035..cb6d405 100644 --- a/src/config/schema.ts +++ b/src/config/schema.ts @@ -13,13 +13,13 @@ export const configSchema = z.object({ // disable cross origin restrictions, allow any site. // overwrites the cors option above - allowAnySite: booleanSchema, + allowAnySite: booleanSchema.default(false), // should it trust reverse proxy headers? (for ip gathering) - trustProxy: booleanSchema, + trustProxy: booleanSchema.default(false), // should it trust cloudflare headers? (for ip gathering, cloudflare has priority) - trustCloudflare: booleanSchema, + trustCloudflare: booleanSchema.default(false), // prefix for where the instance is run on. for example set it to /backend if you're hosting it on example.com/backend // if this is set, do not apply url rewriting before proxing @@ -32,7 +32,7 @@ export const configSchema = z.object({ format: z.enum(['json', 'pretty']).default('pretty'), // show debug logs? - debug: booleanSchema, + debug: booleanSchema.default(false), }) .default({}), postgres: z.object({ @@ -40,19 +40,19 @@ export const configSchema = z.object({ connection: z.string(), // run all migrations on boot of the application - migrateOnBoot: booleanSchema, + migrateOnBoot: booleanSchema.default(false), // try to sync the schema on boot, useful for development // will always keep the database schema in sync with the connected database // it is extremely destructive, do not use it EVER in production - syncSchema: booleanSchema, + syncSchema: booleanSchema.default(false), // Enable debug logging for MikroORM - Outputs queries and entity management logs // Do NOT use in production, leaks all sensitive data - debugLogging: booleanSchema, + debugLogging: booleanSchema.default(false), // Enable SSL for the postgres connection - ssl: booleanSchema, + ssl: booleanSchema.default(false), }), crypto: z.object({ // session secret. used for signing session tokens @@ -67,7 +67,7 @@ export const configSchema = z.object({ captcha: z .object({ // enabled captchas on register - enabled: booleanSchema, + enabled: booleanSchema.default(false), // captcha secret secret: z.string().min(1).optional(), @@ -78,7 +78,7 @@ export const configSchema = z.object({ ratelimits: z .object({ // enabled captchas on register - enabled: booleanSchema, + enabled: booleanSchema.default(false), redisUrl: z.string().optional(), }) .default({}), From 4af2d32b7276c0e95539736850479dd7cd6dd22b Mon Sep 17 00:00:00 2001 From: mrjvs Date: Thu, 25 Jan 2024 22:24:58 +0100 Subject: [PATCH 09/11] Add tool metrics --- src/modules/metrics/index.ts | 7 +++++++ src/routes/metrics.ts | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/src/modules/metrics/index.ts b/src/modules/metrics/index.ts index da35f11..61cc652 100644 --- a/src/modules/metrics/index.ts +++ b/src/modules/metrics/index.ts @@ -13,6 +13,7 @@ export type Metrics = { providerHostnames: Counter<'hostname'>; providerStatuses: Counter<'provider_id' | 'status'>; watchMetrics: Counter<'title' | 'tmdb_full_id' | 'provider_id' | 'success'>; + toolMetrics: Counter<'tool'>; }; let metrics: null | Metrics = null; @@ -59,6 +60,11 @@ export async function setupMetrics(app: FastifyInstance) { help: 'mw_media_watch_count', labelNames: ['title', 'tmdb_full_id', 'provider_id', 'success'], }), + toolMetrics: new Counter({ + name: 'mw_provider_tool_count', + help: 'mw_provider_tool_count', + labelNames: ['tool'], + }), }; const promClient = app.metrics.client; @@ -68,6 +74,7 @@ export async function setupMetrics(app: FastifyInstance) { promClient.register.registerMetric(metrics.providerStatuses); promClient.register.registerMetric(metrics.watchMetrics); promClient.register.registerMetric(metrics.captchaSolves); + promClient.register.registerMetric(metrics.toolMetrics); const orm = getORM(); const em = orm.em.fork(); diff --git a/src/routes/metrics.ts b/src/routes/metrics.ts index e560505..53cf7eb 100644 --- a/src/routes/metrics.ts +++ b/src/routes/metrics.ts @@ -19,6 +19,7 @@ const metricsProviderSchema = z.object({ const metricsProviderInputSchema = z.object({ items: z.array(metricsProviderSchema).max(10).min(1), + tool: z.string().optional(), }); export const metricsRouter = makeRouter((app) => { @@ -65,6 +66,12 @@ export const metricsRouter = makeRouter((app) => { }); } + if (body.tool) { + getMetrics().toolMetrics.inc({ + tool: body.tool + }); + } + return true; }), ); From 26b46876c53b51b477d1ca6e9e3a75df67a958cb Mon Sep 17 00:00:00 2001 From: mrjvs Date: Thu, 25 Jan 2024 22:27:54 +0100 Subject: [PATCH 10/11] fix linting --- src/routes/metrics.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/routes/metrics.ts b/src/routes/metrics.ts index 53cf7eb..e20e39f 100644 --- a/src/routes/metrics.ts +++ b/src/routes/metrics.ts @@ -68,7 +68,7 @@ export const metricsRouter = makeRouter((app) => { if (body.tool) { getMetrics().toolMetrics.inc({ - tool: body.tool + tool: body.tool, }); } From eb0f47ba53001c76c8a52f4fe89a50015962a700 Mon Sep 17 00:00:00 2001 From: mrjvs Date: Thu, 25 Jan 2024 22:28:08 +0100 Subject: [PATCH 11/11] Bump version --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 80acfde..18b344d 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "backend", - "version": "1.3.0", + "version": "1.3.1", "private": true, "homepage": "https://github.com/movie-web/backend", "engines": {