feat: Start Task Runner via Launcher (no-changelog) (#11071)

docker/images/n8n-custom/Dockerfile

@@ -31,6 +31,30 @@ WORKDIR /home/node
 COPY --from=builder /compiled /usr/local/lib/node_modules/n8n
 COPY docker/images/n8n/docker-entrypoint.sh /
 
+# Setup the Task Runner Launcher
+ARG TARGETPLATFORM
+ARG LAUNCHER_VERSION=0.1.1
+ENV N8N_RUNNERS_USE_LAUNCHER=true \
+    N8N_RUNNERS_LAUNCHER_PATH=/usr/local/bin/task-runner-launcher
+COPY docker/images/n8n/n8n-task-runners.json /etc/n8n-task-runners.json
+# First, download, verify, then extract the launcher binary
+# Second, chmod with 4555 to allow the use of setuid
+# Third, create a new user and group to execute the Task Runners under
+RUN \
+	if [[ "$TARGETPLATFORM" = "linux/amd64" ]]; then export ARCH_NAME="x86_64"; \
+	elif [[ "$TARGETPLATFORM" = "linux/arm64" ]]; then export ARCH_NAME="aarch64"; fi; \
+	mkdir /launcher-temp && \
+	cd /launcher-temp && \
+	wget https://github.com/n8n-io/task-runner-launcher/releases/download/${LAUNCHER_VERSION}/task-runner-launcher-$ARCH_NAME-unknown-linux-musl.zip && \
+	wget https://github.com/n8n-io/task-runner-launcher/releases/download/${LAUNCHER_VERSION}/task-runner-launcher-$ARCH_NAME-unknown-linux-musl.sha256 && \
+	sha256sum -c task-runner-launcher-$ARCH_NAME-unknown-linux-musl.sha256 && \
+	unzip -d $(dirname ${N8N_RUNNERS_LAUNCHER_PATH}) task-runner-launcher-$ARCH_NAME-unknown-linux-musl.zip task-runner-launcher && \
+	cd - && \
+	rm -r /launcher-temp && \
+	chmod 4555 ${N8N_RUNNERS_LAUNCHER_PATH} && \
+	addgroup -g 2000 task-runner && \
+	adduser -D -u 2000 -g "Task Runner User" -G task-runner task-runner
+
 RUN \
 	cd /usr/local/lib/node_modules/n8n && \
 	npm rebuild sqlite3 && \

docker/images/n8n/Dockerfile

@@ -22,6 +22,30 @@ RUN set -eux; \
 	find /usr/local/lib/node_modules/n8n -type f -name "*.ts" -o -name "*.js.map" -o -name "*.vue" | xargs rm -f && \
 	rm -rf /root/.npm
 
+# Setup the Task Runner Launcher
+ARG TARGETPLATFORM
+ARG LAUNCHER_VERSION=0.1.1
+ENV N8N_RUNNERS_USE_LAUNCHER=true \
+    N8N_RUNNERS_LAUNCHER_PATH=/usr/local/bin/task-runner-launcher
+COPY n8n-task-runners.json /etc/n8n-task-runners.json
+# First, download, verify, then extract the launcher binary
+# Second, chmod with 4555 to allow the use of setuid
+# Third, create a new user and group to execute the Task Runners under
+RUN \
+	if [[ "$TARGETPLATFORM" = "linux/amd64" ]]; then export ARCH_NAME="x86_64"; \
+	elif [[ "$TARGETPLATFORM" = "linux/arm64" ]]; then export ARCH_NAME="aarch64"; fi; \
+	mkdir /launcher-temp && \
+	cd /launcher-temp && \
+	wget https://github.com/n8n-io/task-runner-launcher/releases/download/${LAUNCHER_VERSION}/task-runner-launcher-$ARCH_NAME-unknown-linux-musl.zip && \
+	wget https://github.com/n8n-io/task-runner-launcher/releases/download/${LAUNCHER_VERSION}/task-runner-launcher-$ARCH_NAME-unknown-linux-musl.sha256 && \
+	sha256sum -c task-runner-launcher-$ARCH_NAME-unknown-linux-musl.sha256 && \
+	unzip -d $(dirname ${N8N_RUNNERS_LAUNCHER_PATH}) task-runner-launcher-$ARCH_NAME-unknown-linux-musl.zip task-runner-launcher && \
+	cd - && \
+	rm -r /launcher-temp && \
+	chmod 4555 ${N8N_RUNNERS_LAUNCHER_PATH} && \
+	addgroup -g 2000 task-runner && \
+	adduser -D -u 2000 -g "Task Runner User" -G task-runner task-runner
+
 COPY docker-entrypoint.sh /
 
 RUN \

docker/images/n8n/n8n-task-runners.json

@@ -0,0 +1,19 @@
+{
+	"task-runners": [
+		{
+			"runner-type": "javascript",
+			"workdir": "/home/task-runner",
+			"command": "/usr/local/bin/node",
+			"args": ["/usr/local/lib/node_modules/n8n/node_modules/@n8n/task-runner/dist/start.js"],
+			"allowed-env": [
+				"PATH",
+				"N8N_RUNNERS_GRANT_TOKEN",
+				"N8N_RUNNERS_N8N_URI",
+				"NODE_FUNCTION_ALLOW_BUILTIN",
+				"NODE_FUNCTION_ALLOW_EXTERNAL"
+			],
+			"uid": 2000,
+			"gid": 2000
+		}
+	]
+}

packages/@n8n/config/src/configs/runners.config.ts

@@ -19,4 +19,14 @@ export class TaskRunnersConfig {
 	/** IP address task runners server should listen on */
 	@Env('N8N_RUNNERS_SERVER_LISTEN_ADDRESS')
 	listen_address: string = '127.0.0.1';
+
+	@Env('N8N_RUNNERS_USE_LAUNCHER')
+	useLauncher: boolean = false;
+
+	@Env('N8N_RUNNERS_LAUNCHER_PATH')
+	launcherPath: string = '';
+
+	/** Which task runner to launch from the config */
+	@Env('N8N_RUNNERS_LAUNCHER_RUNNER')
+	launcherRunner: string = 'javascript';
 }

packages/@n8n/config/test/config.test.ts

@@ -228,6 +228,9 @@ describe('GlobalConfig', () => {
 			authToken: '',
 			listen_address: '127.0.0.1',
 			port: 5679,
+			useLauncher: false,
+			launcherPath: '',
+			launcherRunner: 'javascript',
 		},
 		sentry: {
 			backendDsn: '',

packages/cli/src/runners/task-runner-process.ts

@@ -39,17 +39,11 @@ export class TaskRunnerProcess {
 		a.ok(!this.process, 'Task Runner Process already running');
 
 		const grantToken = await this.authService.createGrantToken();
-		const startScript = require.resolve('@n8n/task-runner');
 
-		this.process = spawn('node', [startScript], {
+		const n8nUri = `127.0.0.1:${this.globalConfig.taskRunners.port}`;
-			env: {
+		this.process = this.globalConfig.taskRunners.useLauncher
-				PATH: process.env.PATH,
+			? this.startLauncher(grantToken, n8nUri)
-				N8N_RUNNERS_GRANT_TOKEN: grantToken,
+			: this.startNode(grantToken, n8nUri);
-				N8N_RUNNERS_N8N_URI: `127.0.0.1:${this.globalConfig.taskRunners.port}`,
-				NODE_FUNCTION_ALLOW_BUILTIN: process.env.NODE_FUNCTION_ALLOW_BUILTIN,
-				NODE_FUNCTION_ALLOW_EXTERNAL: process.env.NODE_FUNCTION_ALLOW_EXTERNAL,
-			},
-		});
 
 		this.process.stdout?.pipe(process.stdout);
 		this.process.stderr?.pipe(process.stderr);
@@ -57,6 +51,38 @@ export class TaskRunnerProcess {
 		this.monitorProcess(this.process);
 	}
 
+	startNode(grantToken: string, n8nUri: string) {
+		const startScript = require.resolve('@n8n/task-runner');
+
+		return spawn('node', [startScript], {
+			env: {
+				PATH: process.env.PATH,
+				N8N_RUNNERS_GRANT_TOKEN: grantToken,
+				N8N_RUNNERS_N8N_URI: n8nUri,
+				NODE_FUNCTION_ALLOW_BUILTIN: process.env.NODE_FUNCTION_ALLOW_BUILTIN,
+				NODE_FUNCTION_ALLOW_EXTERNAL: process.env.NODE_FUNCTION_ALLOW_EXTERNAL,
+			},
+		});
+	}
+
+	startLauncher(grantToken: string, n8nUri: string) {
+		return spawn(
+			this.globalConfig.taskRunners.launcherPath,
+			['launch', this.globalConfig.taskRunners.launcherRunner],
+			{
+				env: {
+					PATH: process.env.PATH,
+					N8N_RUNNERS_GRANT_TOKEN: grantToken,
+					N8N_RUNNERS_N8N_URI: n8nUri,
+					NODE_FUNCTION_ALLOW_BUILTIN: process.env.NODE_FUNCTION_ALLOW_BUILTIN,
+					NODE_FUNCTION_ALLOW_EXTERNAL: process.env.NODE_FUNCTION_ALLOW_EXTERNAL,
+					// For debug logging if enabled
+					RUST_LOG: process.env.RUST_LOG,
+				},
+			},
+		);
+	}
+
 	@OnShutdown()
 	async stop() {
 		if (!this.process) {
@@ -66,12 +92,41 @@ export class TaskRunnerProcess {
 		this.isShuttingDown = true;
 
 		// TODO: Timeout & force kill
-		this.process.kill();
+		if (this.globalConfig.taskRunners.useLauncher) {
+			await this.killLauncher();
+		} else {
+			this.killNode();
+		}
 		await this.runPromise;
 
 		this.isShuttingDown = false;
 	}
 
+	killNode() {
+		if (!this.process) {
+			return;
+		}
+		this.process.kill();
+	}
+
+	async killLauncher() {
+		if (!this.process?.pid) {
+			return;
+		}
+
+		const killProcess = spawn(this.globalConfig.taskRunners.launcherPath, [
+			'kill',
+			this.globalConfig.taskRunners.launcherRunner,
+			this.process.pid.toString(),
+		]);
+
+		await new Promise<void>((resolve) => {
+			killProcess.on('exit', () => {
+				resolve();
+			});
+		});
+	}
+
 	private monitorProcess(taskRunnerProcess: ChildProcess) {
 		this.runPromise = new Promise((resolve) => {
 			taskRunnerProcess.on('exit', (code) => {

packages/cli/test/integration/runners/task-runner-process.test.ts

@@ -18,6 +18,11 @@ describe('TaskRunnerProcess', () => {
 	const taskBroker = Container.get(TaskBroker);
 	const taskRunnerService = Container.get(TaskRunnerService);
 
+	const startLauncherSpy = jest.spyOn(runnerProcess, 'startLauncher');
+	const startNodeSpy = jest.spyOn(runnerProcess, 'startNode');
+	const killLauncherSpy = jest.spyOn(runnerProcess, 'killLauncher');
+	const killNodeSpy = jest.spyOn(runnerProcess, 'killNode');
+
 	beforeAll(async () => {
 		await taskRunnerServer.start();
 		// Set the port to the actually used port
@@ -30,6 +35,11 @@ describe('TaskRunnerProcess', () => {
 
 	afterEach(async () => {
 		await runnerProcess.stop();
+
+		startLauncherSpy.mockClear();
+		startNodeSpy.mockClear();
+		killLauncherSpy.mockClear();
+		killNodeSpy.mockClear();
 	});
 
 	const getNumConnectedRunners = () => taskRunnerService.runnerConnections.size;
@@ -88,4 +98,46 @@ describe('TaskRunnerProcess', () => {
 		expect(getNumConnectedRunners()).toBe(1);
 		expect(getNumRegisteredRunners()).toBe(1);
 	});
+
+	it('should launch runner directly if not using a launcher', async () => {
+		globalConfig.taskRunners.useLauncher = false;
+
+		await runnerProcess.start();
+
+		expect(startLauncherSpy).toBeCalledTimes(0);
+		expect(startNodeSpy).toBeCalledTimes(1);
+	});
+
+	it('should use a launcher if configured', async () => {
+		globalConfig.taskRunners.useLauncher = true;
+		globalConfig.taskRunners.launcherPath = 'node';
+
+		await runnerProcess.start();
+
+		expect(startLauncherSpy).toBeCalledTimes(1);
+		expect(startNodeSpy).toBeCalledTimes(0);
+		globalConfig.taskRunners.useLauncher = false;
+	});
+
+	it('should kill the process directly if not using a launcher', async () => {
+		globalConfig.taskRunners.useLauncher = false;
+
+		await runnerProcess.start();
+		await runnerProcess.stop();
+
+		expect(killLauncherSpy).toBeCalledTimes(0);
+		expect(killNodeSpy).toBeCalledTimes(1);
+	});
+
+	it('should kill the process using a launcher if configured', async () => {
+		globalConfig.taskRunners.useLauncher = true;
+		globalConfig.taskRunners.launcherPath = 'node';
+
+		await runnerProcess.start();
+		await runnerProcess.stop();
+
+		expect(killLauncherSpy).toBeCalledTimes(1);
+		expect(killNodeSpy).toBeCalledTimes(0);
+		globalConfig.taskRunners.useLauncher = false;
+	});
 });