From b028d81390dd6374b7effea07b8c1b086f3ffed9 Mon Sep 17 00:00:00 2001 From: Val <68596159+valya@users.noreply.github.com> Date: Mon, 14 Oct 2024 14:19:17 +0100 Subject: [PATCH] feat: Start Task Runner via Launcher (no-changelog) (#11071) --- docker/images/n8n-custom/Dockerfile | 24 ++++++ docker/images/n8n/Dockerfile | 24 ++++++ docker/images/n8n/n8n-task-runners.json | 19 +++++ .../@n8n/config/src/configs/runners.config.ts | 10 +++ packages/@n8n/config/test/config.test.ts | 3 + .../cli/src/runners/task-runner-process.ts | 77 ++++++++++++++++--- .../runners/task-runner-process.test.ts | 52 +++++++++++++ 7 files changed, 198 insertions(+), 11 deletions(-) create mode 100644 docker/images/n8n/n8n-task-runners.json diff --git a/docker/images/n8n-custom/Dockerfile b/docker/images/n8n-custom/Dockerfile index ba271017d1..a533cdbdab 100644 --- a/docker/images/n8n-custom/Dockerfile +++ b/docker/images/n8n-custom/Dockerfile @@ -31,6 +31,30 @@ WORKDIR /home/node COPY --from=builder /compiled /usr/local/lib/node_modules/n8n COPY docker/images/n8n/docker-entrypoint.sh / +# Setup the Task Runner Launcher +ARG TARGETPLATFORM +ARG LAUNCHER_VERSION=0.1.1 +ENV N8N_RUNNERS_USE_LAUNCHER=true \ + N8N_RUNNERS_LAUNCHER_PATH=/usr/local/bin/task-runner-launcher +COPY docker/images/n8n/n8n-task-runners.json /etc/n8n-task-runners.json +# First, download, verify, then extract the launcher binary +# Second, chmod with 4555 to allow the use of setuid +# Third, create a new user and group to execute the Task Runners under +RUN \ + if [[ "$TARGETPLATFORM" = "linux/amd64" ]]; then export ARCH_NAME="x86_64"; \ + elif [[ "$TARGETPLATFORM" = "linux/arm64" ]]; then export ARCH_NAME="aarch64"; fi; \ + mkdir /launcher-temp && \ + cd /launcher-temp && \ + wget https://github.com/n8n-io/task-runner-launcher/releases/download/${LAUNCHER_VERSION}/task-runner-launcher-$ARCH_NAME-unknown-linux-musl.zip && \ + wget https://github.com/n8n-io/task-runner-launcher/releases/download/${LAUNCHER_VERSION}/task-runner-launcher-$ARCH_NAME-unknown-linux-musl.sha256 && \ + sha256sum -c task-runner-launcher-$ARCH_NAME-unknown-linux-musl.sha256 && \ + unzip -d $(dirname ${N8N_RUNNERS_LAUNCHER_PATH}) task-runner-launcher-$ARCH_NAME-unknown-linux-musl.zip task-runner-launcher && \ + cd - && \ + rm -r /launcher-temp && \ + chmod 4555 ${N8N_RUNNERS_LAUNCHER_PATH} && \ + addgroup -g 2000 task-runner && \ + adduser -D -u 2000 -g "Task Runner User" -G task-runner task-runner + RUN \ cd /usr/local/lib/node_modules/n8n && \ npm rebuild sqlite3 && \ diff --git a/docker/images/n8n/Dockerfile b/docker/images/n8n/Dockerfile index 2da1bc1f47..08e031cf5f 100644 --- a/docker/images/n8n/Dockerfile +++ b/docker/images/n8n/Dockerfile @@ -22,6 +22,30 @@ RUN set -eux; \ find /usr/local/lib/node_modules/n8n -type f -name "*.ts" -o -name "*.js.map" -o -name "*.vue" | xargs rm -f && \ rm -rf /root/.npm +# Setup the Task Runner Launcher +ARG TARGETPLATFORM +ARG LAUNCHER_VERSION=0.1.1 +ENV N8N_RUNNERS_USE_LAUNCHER=true \ + N8N_RUNNERS_LAUNCHER_PATH=/usr/local/bin/task-runner-launcher +COPY n8n-task-runners.json /etc/n8n-task-runners.json +# First, download, verify, then extract the launcher binary +# Second, chmod with 4555 to allow the use of setuid +# Third, create a new user and group to execute the Task Runners under +RUN \ + if [[ "$TARGETPLATFORM" = "linux/amd64" ]]; then export ARCH_NAME="x86_64"; \ + elif [[ "$TARGETPLATFORM" = "linux/arm64" ]]; then export ARCH_NAME="aarch64"; fi; \ + mkdir /launcher-temp && \ + cd /launcher-temp && \ + wget https://github.com/n8n-io/task-runner-launcher/releases/download/${LAUNCHER_VERSION}/task-runner-launcher-$ARCH_NAME-unknown-linux-musl.zip && \ + wget https://github.com/n8n-io/task-runner-launcher/releases/download/${LAUNCHER_VERSION}/task-runner-launcher-$ARCH_NAME-unknown-linux-musl.sha256 && \ + sha256sum -c task-runner-launcher-$ARCH_NAME-unknown-linux-musl.sha256 && \ + unzip -d $(dirname ${N8N_RUNNERS_LAUNCHER_PATH}) task-runner-launcher-$ARCH_NAME-unknown-linux-musl.zip task-runner-launcher && \ + cd - && \ + rm -r /launcher-temp && \ + chmod 4555 ${N8N_RUNNERS_LAUNCHER_PATH} && \ + addgroup -g 2000 task-runner && \ + adduser -D -u 2000 -g "Task Runner User" -G task-runner task-runner + COPY docker-entrypoint.sh / RUN \ diff --git a/docker/images/n8n/n8n-task-runners.json b/docker/images/n8n/n8n-task-runners.json new file mode 100644 index 0000000000..2dd65b67c8 --- /dev/null +++ b/docker/images/n8n/n8n-task-runners.json @@ -0,0 +1,19 @@ +{ + "task-runners": [ + { + "runner-type": "javascript", + "workdir": "/home/task-runner", + "command": "/usr/local/bin/node", + "args": ["/usr/local/lib/node_modules/n8n/node_modules/@n8n/task-runner/dist/start.js"], + "allowed-env": [ + "PATH", + "N8N_RUNNERS_GRANT_TOKEN", + "N8N_RUNNERS_N8N_URI", + "NODE_FUNCTION_ALLOW_BUILTIN", + "NODE_FUNCTION_ALLOW_EXTERNAL" + ], + "uid": 2000, + "gid": 2000 + } + ] +} diff --git a/packages/@n8n/config/src/configs/runners.config.ts b/packages/@n8n/config/src/configs/runners.config.ts index e7335e8827..14d1b01d1a 100644 --- a/packages/@n8n/config/src/configs/runners.config.ts +++ b/packages/@n8n/config/src/configs/runners.config.ts @@ -19,4 +19,14 @@ export class TaskRunnersConfig { /** IP address task runners server should listen on */ @Env('N8N_RUNNERS_SERVER_LISTEN_ADDRESS') listen_address: string = '127.0.0.1'; + + @Env('N8N_RUNNERS_USE_LAUNCHER') + useLauncher: boolean = false; + + @Env('N8N_RUNNERS_LAUNCHER_PATH') + launcherPath: string = ''; + + /** Which task runner to launch from the config */ + @Env('N8N_RUNNERS_LAUNCHER_RUNNER') + launcherRunner: string = 'javascript'; } diff --git a/packages/@n8n/config/test/config.test.ts b/packages/@n8n/config/test/config.test.ts index 301022ca3e..56f3bc6de7 100644 --- a/packages/@n8n/config/test/config.test.ts +++ b/packages/@n8n/config/test/config.test.ts @@ -228,6 +228,9 @@ describe('GlobalConfig', () => { authToken: '', listen_address: '127.0.0.1', port: 5679, + useLauncher: false, + launcherPath: '', + launcherRunner: 'javascript', }, sentry: { backendDsn: '', diff --git a/packages/cli/src/runners/task-runner-process.ts b/packages/cli/src/runners/task-runner-process.ts index 9f570fcb38..857d581127 100644 --- a/packages/cli/src/runners/task-runner-process.ts +++ b/packages/cli/src/runners/task-runner-process.ts @@ -39,17 +39,11 @@ export class TaskRunnerProcess { a.ok(!this.process, 'Task Runner Process already running'); const grantToken = await this.authService.createGrantToken(); - const startScript = require.resolve('@n8n/task-runner'); - this.process = spawn('node', [startScript], { - env: { - PATH: process.env.PATH, - N8N_RUNNERS_GRANT_TOKEN: grantToken, - N8N_RUNNERS_N8N_URI: `127.0.0.1:${this.globalConfig.taskRunners.port}`, - NODE_FUNCTION_ALLOW_BUILTIN: process.env.NODE_FUNCTION_ALLOW_BUILTIN, - NODE_FUNCTION_ALLOW_EXTERNAL: process.env.NODE_FUNCTION_ALLOW_EXTERNAL, - }, - }); + const n8nUri = `127.0.0.1:${this.globalConfig.taskRunners.port}`; + this.process = this.globalConfig.taskRunners.useLauncher + ? this.startLauncher(grantToken, n8nUri) + : this.startNode(grantToken, n8nUri); this.process.stdout?.pipe(process.stdout); this.process.stderr?.pipe(process.stderr); @@ -57,6 +51,38 @@ export class TaskRunnerProcess { this.monitorProcess(this.process); } + startNode(grantToken: string, n8nUri: string) { + const startScript = require.resolve('@n8n/task-runner'); + + return spawn('node', [startScript], { + env: { + PATH: process.env.PATH, + N8N_RUNNERS_GRANT_TOKEN: grantToken, + N8N_RUNNERS_N8N_URI: n8nUri, + NODE_FUNCTION_ALLOW_BUILTIN: process.env.NODE_FUNCTION_ALLOW_BUILTIN, + NODE_FUNCTION_ALLOW_EXTERNAL: process.env.NODE_FUNCTION_ALLOW_EXTERNAL, + }, + }); + } + + startLauncher(grantToken: string, n8nUri: string) { + return spawn( + this.globalConfig.taskRunners.launcherPath, + ['launch', this.globalConfig.taskRunners.launcherRunner], + { + env: { + PATH: process.env.PATH, + N8N_RUNNERS_GRANT_TOKEN: grantToken, + N8N_RUNNERS_N8N_URI: n8nUri, + NODE_FUNCTION_ALLOW_BUILTIN: process.env.NODE_FUNCTION_ALLOW_BUILTIN, + NODE_FUNCTION_ALLOW_EXTERNAL: process.env.NODE_FUNCTION_ALLOW_EXTERNAL, + // For debug logging if enabled + RUST_LOG: process.env.RUST_LOG, + }, + }, + ); + } + @OnShutdown() async stop() { if (!this.process) { @@ -66,12 +92,41 @@ export class TaskRunnerProcess { this.isShuttingDown = true; // TODO: Timeout & force kill - this.process.kill(); + if (this.globalConfig.taskRunners.useLauncher) { + await this.killLauncher(); + } else { + this.killNode(); + } await this.runPromise; this.isShuttingDown = false; } + killNode() { + if (!this.process) { + return; + } + this.process.kill(); + } + + async killLauncher() { + if (!this.process?.pid) { + return; + } + + const killProcess = spawn(this.globalConfig.taskRunners.launcherPath, [ + 'kill', + this.globalConfig.taskRunners.launcherRunner, + this.process.pid.toString(), + ]); + + await new Promise((resolve) => { + killProcess.on('exit', () => { + resolve(); + }); + }); + } + private monitorProcess(taskRunnerProcess: ChildProcess) { this.runPromise = new Promise((resolve) => { taskRunnerProcess.on('exit', (code) => { diff --git a/packages/cli/test/integration/runners/task-runner-process.test.ts b/packages/cli/test/integration/runners/task-runner-process.test.ts index f517ee6398..e623d5f371 100644 --- a/packages/cli/test/integration/runners/task-runner-process.test.ts +++ b/packages/cli/test/integration/runners/task-runner-process.test.ts @@ -18,6 +18,11 @@ describe('TaskRunnerProcess', () => { const taskBroker = Container.get(TaskBroker); const taskRunnerService = Container.get(TaskRunnerService); + const startLauncherSpy = jest.spyOn(runnerProcess, 'startLauncher'); + const startNodeSpy = jest.spyOn(runnerProcess, 'startNode'); + const killLauncherSpy = jest.spyOn(runnerProcess, 'killLauncher'); + const killNodeSpy = jest.spyOn(runnerProcess, 'killNode'); + beforeAll(async () => { await taskRunnerServer.start(); // Set the port to the actually used port @@ -30,6 +35,11 @@ describe('TaskRunnerProcess', () => { afterEach(async () => { await runnerProcess.stop(); + + startLauncherSpy.mockClear(); + startNodeSpy.mockClear(); + killLauncherSpy.mockClear(); + killNodeSpy.mockClear(); }); const getNumConnectedRunners = () => taskRunnerService.runnerConnections.size; @@ -88,4 +98,46 @@ describe('TaskRunnerProcess', () => { expect(getNumConnectedRunners()).toBe(1); expect(getNumRegisteredRunners()).toBe(1); }); + + it('should launch runner directly if not using a launcher', async () => { + globalConfig.taskRunners.useLauncher = false; + + await runnerProcess.start(); + + expect(startLauncherSpy).toBeCalledTimes(0); + expect(startNodeSpy).toBeCalledTimes(1); + }); + + it('should use a launcher if configured', async () => { + globalConfig.taskRunners.useLauncher = true; + globalConfig.taskRunners.launcherPath = 'node'; + + await runnerProcess.start(); + + expect(startLauncherSpy).toBeCalledTimes(1); + expect(startNodeSpy).toBeCalledTimes(0); + globalConfig.taskRunners.useLauncher = false; + }); + + it('should kill the process directly if not using a launcher', async () => { + globalConfig.taskRunners.useLauncher = false; + + await runnerProcess.start(); + await runnerProcess.stop(); + + expect(killLauncherSpy).toBeCalledTimes(0); + expect(killNodeSpy).toBeCalledTimes(1); + }); + + it('should kill the process using a launcher if configured', async () => { + globalConfig.taskRunners.useLauncher = true; + globalConfig.taskRunners.launcherPath = 'node'; + + await runnerProcess.start(); + await runnerProcess.stop(); + + expect(killLauncherSpy).toBeCalledTimes(1); + expect(killNodeSpy).toBeCalledTimes(0); + globalConfig.taskRunners.useLauncher = false; + }); });