From c56f30ce15aa03c5635bbca746c5d3ede0218cd8 Mon Sep 17 00:00:00 2001 From: Tomi Turtiainen <10324676+tomi@users.noreply.github.com> Date: Mon, 28 Oct 2024 16:14:38 +0200 Subject: [PATCH] feat: N8N_RUNNERS_MAX_OLD_SPACE_SIZE configuration (no-changelog) (#11419) --- docker/images/n8n/n8n-task-runners.json | 3 +- .../@n8n/config/src/configs/runners.config.ts | 4 ++ packages/@n8n/config/test/config.test.ts | 1 + .../__tests__/task-runner-process.test.ts | 43 ++++++++++++++--- .../cli/src/runners/task-runner-process.ts | 47 +++++++++++++------ 5 files changed, 77 insertions(+), 21 deletions(-) diff --git a/docker/images/n8n/n8n-task-runners.json b/docker/images/n8n/n8n-task-runners.json index 56a48b2d09..4019189589 100644 --- a/docker/images/n8n/n8n-task-runners.json +++ b/docker/images/n8n/n8n-task-runners.json @@ -11,7 +11,8 @@ "N8N_RUNNERS_N8N_URI", "N8N_RUNNERS_MAX_PAYLOAD", "NODE_FUNCTION_ALLOW_BUILTIN", - "NODE_FUNCTION_ALLOW_EXTERNAL" + "NODE_FUNCTION_ALLOW_EXTERNAL", + "NODE_OPTIONS" ], "uid": 2000, "gid": 2000 diff --git a/packages/@n8n/config/src/configs/runners.config.ts b/packages/@n8n/config/src/configs/runners.config.ts index 648959e3f4..f26b96636c 100644 --- a/packages/@n8n/config/src/configs/runners.config.ts +++ b/packages/@n8n/config/src/configs/runners.config.ts @@ -42,4 +42,8 @@ export class TaskRunnersConfig { /** Which task runner to launch from the config */ @Env('N8N_RUNNERS_LAUNCHER_RUNNER') launcherRunner: string = 'javascript'; + + /** The --max-old-space-size option to use for the runner (in MB). Default means node.js will determine it based on the available memory. */ + @Env('N8N_RUNNERS_MAX_OLD_SPACE_SIZE') + maxOldSpaceSize: string = ''; } diff --git a/packages/@n8n/config/test/config.test.ts b/packages/@n8n/config/test/config.test.ts index f605006067..eabcd5c489 100644 --- a/packages/@n8n/config/test/config.test.ts +++ b/packages/@n8n/config/test/config.test.ts @@ -231,6 +231,7 @@ describe('GlobalConfig', () => { port: 5679, launcherPath: '', launcherRunner: 'javascript', + maxOldSpaceSize: '', }, sentry: { backendDsn: '', diff --git a/packages/cli/src/runners/__tests__/task-runner-process.test.ts b/packages/cli/src/runners/__tests__/task-runner-process.test.ts index 1bae991811..8940428789 100644 --- a/packages/cli/src/runners/__tests__/task-runner-process.test.ts +++ b/packages/cli/src/runners/__tests__/task-runner-process.test.ts @@ -23,7 +23,7 @@ describe('TaskRunnerProcess', () => { runnerConfig.disabled = false; runnerConfig.mode = 'internal_childprocess'; const authService = mock(); - const taskRunnerProcess = new TaskRunnerProcess(runnerConfig, authService); + let taskRunnerProcess = new TaskRunnerProcess(runnerConfig, authService); afterEach(async () => { spawnMock.mockClear(); @@ -40,10 +40,31 @@ describe('TaskRunnerProcess', () => { }); describe('start', () => { - it('should propagate NODE_FUNCTION_ALLOW_BUILTIN and NODE_FUNCTION_ALLOW_EXTERNAL from env', async () => { + beforeEach(() => { + taskRunnerProcess = new TaskRunnerProcess(runnerConfig, authService); + }); + + test.each(['PATH', 'NODE_FUNCTION_ALLOW_BUILTIN', 'NODE_FUNCTION_ALLOW_EXTERNAL'])( + 'should propagate %s from env as is', + async (envVar) => { + jest.spyOn(authService, 'createGrantToken').mockResolvedValue('grantToken'); + process.env[envVar] = 'custom value'; + + await taskRunnerProcess.start(); + + // @ts-expect-error The type is not correct + const options = spawnMock.mock.calls[0][2] as SpawnOptions; + expect(options.env).toEqual( + expect.objectContaining({ + [envVar]: 'custom value', + }), + ); + }, + ); + + it('should pass NODE_OPTIONS env if maxOldSpaceSize is configured', async () => { jest.spyOn(authService, 'createGrantToken').mockResolvedValue('grantToken'); - process.env.NODE_FUNCTION_ALLOW_BUILTIN = '*'; - process.env.NODE_FUNCTION_ALLOW_EXTERNAL = '*'; + runnerConfig.maxOldSpaceSize = '1024'; await taskRunnerProcess.start(); @@ -51,10 +72,20 @@ describe('TaskRunnerProcess', () => { const options = spawnMock.mock.calls[0][2] as SpawnOptions; expect(options.env).toEqual( expect.objectContaining({ - NODE_FUNCTION_ALLOW_BUILTIN: '*', - NODE_FUNCTION_ALLOW_EXTERNAL: '*', + NODE_OPTIONS: '--max-old-space-size=1024', }), ); }); + + it('should not pass NODE_OPTIONS env if maxOldSpaceSize is not configured', async () => { + jest.spyOn(authService, 'createGrantToken').mockResolvedValue('grantToken'); + runnerConfig.maxOldSpaceSize = ''; + + await taskRunnerProcess.start(); + + // @ts-expect-error The type is not correct + const options = spawnMock.mock.calls[0][2] as SpawnOptions; + expect(options.env).not.toHaveProperty('NODE_OPTIONS'); + }); }); }); diff --git a/packages/cli/src/runners/task-runner-process.ts b/packages/cli/src/runners/task-runner-process.ts index 413b74d725..0415b910b1 100644 --- a/packages/cli/src/runners/task-runner-process.ts +++ b/packages/cli/src/runners/task-runner-process.ts @@ -38,6 +38,12 @@ export class TaskRunnerProcess { private isShuttingDown = false; + private readonly passthroughEnvVars = [ + 'PATH', + 'NODE_FUNCTION_ALLOW_BUILTIN', + 'NODE_FUNCTION_ALLOW_EXTERNAL', + ] as const; + constructor( private readonly runnerConfig: TaskRunnersConfig, private readonly authService: TaskRunnerAuthService, @@ -68,26 +74,14 @@ export class TaskRunnerProcess { const startScript = require.resolve('@n8n/task-runner'); return spawn('node', [startScript], { - env: { - PATH: process.env.PATH, - N8N_RUNNERS_GRANT_TOKEN: grantToken, - N8N_RUNNERS_N8N_URI: n8nUri, - N8N_RUNNERS_MAX_PAYLOAD: this.runnerConfig.maxPayload.toString(), - NODE_FUNCTION_ALLOW_BUILTIN: process.env.NODE_FUNCTION_ALLOW_BUILTIN, - NODE_FUNCTION_ALLOW_EXTERNAL: process.env.NODE_FUNCTION_ALLOW_EXTERNAL, - }, + env: this.getProcessEnvVars(grantToken, n8nUri), }); } startLauncher(grantToken: string, n8nUri: string) { return spawn(this.runnerConfig.launcherPath, ['launch', this.runnerConfig.launcherRunner], { env: { - PATH: process.env.PATH, - N8N_RUNNERS_GRANT_TOKEN: grantToken, - N8N_RUNNERS_N8N_URI: n8nUri, - N8N_RUNNERS_MAX_PAYLOAD: this.runnerConfig.maxPayload.toString(), - NODE_FUNCTION_ALLOW_BUILTIN: process.env.NODE_FUNCTION_ALLOW_BUILTIN, - NODE_FUNCTION_ALLOW_EXTERNAL: process.env.NODE_FUNCTION_ALLOW_EXTERNAL, + ...this.getProcessEnvVars(grantToken, n8nUri), // For debug logging if enabled RUST_LOG: process.env.RUST_LOG, }, @@ -155,4 +149,29 @@ export class TaskRunnerProcess { setImmediate(async () => await this.start()); } } + + private getProcessEnvVars(grantToken: string, n8nUri: string) { + const envVars: Record = { + N8N_RUNNERS_GRANT_TOKEN: grantToken, + N8N_RUNNERS_N8N_URI: n8nUri, + N8N_RUNNERS_MAX_PAYLOAD: this.runnerConfig.maxPayload.toString(), + ...this.getPassthroughEnvVars(), + }; + + if (this.runnerConfig.maxOldSpaceSize) { + envVars.NODE_OPTIONS = `--max-old-space-size=${this.runnerConfig.maxOldSpaceSize}`; + } + + return envVars; + } + + private getPassthroughEnvVars() { + return this.passthroughEnvVars.reduce>((env, key) => { + if (process.env[key]) { + env[key] = process.env[key]; + } + + return env; + }, {}); + } }