fix(Code Node): Install python modules always in a user-writable folder (#6568)

* upgrade pyodide

* install pyodide modules to a custom user-writable path

* in `augmentObject` `newData` is never undefined

package.json

@@ -93,7 +93,8 @@
       "element-ui@2.15.12": "patches/element-ui@2.15.12.patch",
       "typedi@0.10.0": "patches/typedi@0.10.0.patch",
       "@sentry/cli@2.17.0": "patches/@sentry__cli@2.17.0.patch",
-      "pkce-challenge@3.0.0": "patches/pkce-challenge@3.0.0.patch"
+      "pkce-challenge@3.0.0": "patches/pkce-challenge@3.0.0.patch",
+      "pyodide@0.23.4": "patches/pyodide@0.23.4.patch"
     }
   }
 }

packages/core/src/NodeExecuteFunctions.ts

@@ -128,6 +128,7 @@ import {
 	setAllWorkflowExecutionMetadata,
 	setWorkflowExecutionMetadata,
 } from './WorkflowExecutionMetadata';
+import { getUserN8nFolderPath } from './UserSettings';
 
 axios.defaults.timeout = 300000;
 // Prevent axios from adding x-form-www-urlencoded headers by default
@@ -2245,6 +2246,9 @@ const getFileSystemHelperFunctions = (node: INode): FileSystemHelperFunctions =>
 		}
 		return createReadStream(filePath);
 	},
+	getStoragePath() {
+		return path.join(getUserN8nFolderPath(), `storage/${node.type}`);
+	},
 });
 
 const getNodeHelperFunctions = ({

packages/nodes-base/nodes/Code/Code.node.ts

@@ -108,10 +108,8 @@ export class Code implements INodeType {
 			}
 
 			if (language === 'python') {
-				const modules = this.getNodeParameter('modules', index) as string;
-				const moduleImports: string[] = modules ? modules.split(',').map((m) => m.trim()) : [];
 				context.printOverwrite = workflowMode === 'manual' ? this.sendMessageToUI : null;
-				return new PythonSandbox(context, code, moduleImports, index, this.helpers);
+				return new PythonSandbox(context, code, index, this.helpers);
 			} else {
 				const sandbox = new JavaScriptSandbox(context, code, index, workflowMode, this.helpers);
 				if (workflowMode === 'manual') {

packages/nodes-base/nodes/Code/Pyodide.ts

@@ -2,26 +2,15 @@ import type { PyodideInterface } from 'pyodide';
 
 let pyodideInstance: PyodideInterface | undefined;
 
-export async function LoadPyodide(): Promise<PyodideInterface> {
+export async function LoadPyodide(packageCacheDir: string): Promise<PyodideInterface> {
 	if (pyodideInstance === undefined) {
-		// TODO: Find better way to suppress warnings
-		//@ts-ignore
-		globalThis.Blob = (await import('node:buffer')).Blob;
-
-		// From: https://github.com/nodejs/node/issues/30810
-		const { emitWarning } = process;
-		process.emitWarning = (warning, ...args) => {
-			if (args[0] === 'ExperimentalWarning') {
-				return;
-			}
-			if (args[0] && typeof args[0] === 'object' && args[0].type === 'ExperimentalWarning') {
-				return;
-			}
-			return emitWarning(warning, ...(args as string[]));
-		};
-
 		const { loadPyodide } = await import('pyodide');
-		pyodideInstance = await loadPyodide();
+		pyodideInstance = await loadPyodide({ packageCacheDir });
+
+		await pyodideInstance.runPythonAsync(`
+from _pyodide_core import jsproxy_typedict
+from js import Object
+`);
 	}
 
 	return pyodideInstance;

packages/nodes-base/nodes/Code/PythonSandbox.ts

@@ -1,5 +1,5 @@
 import type { IExecuteFunctions, INodeExecutionData } from 'n8n-workflow';
-import type { PyProxyDict } from 'pyodide';
+import type { PyDict } from 'pyodide/ffi';
 import { LoadPyodide } from './Pyodide';
 import type { SandboxContext } from './Sandbox';
 import { Sandbox } from './Sandbox';
@@ -18,7 +18,6 @@ export class PythonSandbox extends Sandbox {
 	constructor(
 		context: SandboxContext,
 		private pythonCode: string,
-		private moduleImports: string[],
 		itemIndex: number | undefined,
 		helpers: IExecuteFunctions['helpers'],
 	) {
@@ -51,47 +50,35 @@ export class PythonSandbox extends Sandbox {
 	}
 
 	private async runCodeInPython<T>() {
-		// Below workaround from here:
-		// https://github.com/pyodide/pyodide/discussions/3537#discussioncomment-4864345
-		const runCode = `
-from _pyodide_core import jsproxy_typedict
-from js import Object
-jsproxy_typedict[0] = type(Object.new().as_object_map())
-
-if printOverwrite:
-  print = printOverwrite
-
-async def __main():
-${this.pythonCode
-	.split('\n')
-	.map((line) => '  ' + line)
-	.join('\n')}
-await __main()
-`;
-		const pyodide = await LoadPyodide();
-
-		const moduleImportsFiltered = this.moduleImports.filter(
-			(importModule) => !['asyncio', 'pyodide', 'math'].includes(importModule),
-		);
-
-		if (moduleImportsFiltered.length) {
-			await pyodide.loadPackage('micropip');
-			const micropip = pyodide.pyimport('micropip');
-			await Promise.all(
-				moduleImportsFiltered.map((importModule) => micropip.install(importModule)),
-			);
-		}
+		const packageCacheDir = this.helpers.getStoragePath();
+		const pyodide = await LoadPyodide(packageCacheDir);
 
 		let executionResult;
 		try {
+			await pyodide.runPythonAsync('jsproxy_typedict[0] = type(Object.new().as_object_map())');
+
+			await pyodide.loadPackagesFromImports(this.pythonCode);
+
 			const dict = pyodide.globals.get('dict');
-			const globalsDict: PyProxyDict = dict();
+			const globalsDict: PyDict = dict();
 			for (const key of Object.keys(this.context)) {
 				if ((key === '_env' && envAccessBlocked) || key === '_node') continue;
 				const value = this.context[key];
 				globalsDict.set(key, value);
 			}
 
+			await pyodide.runPythonAsync(`
+if 'printOverwrite' in globals():
+	print = printOverwrite
+			`);
+
+			const runCode = `
+async def __main():
+${this.pythonCode
+	.split('\n')
+	.map((line) => '  ' + line)
+	.join('\n')}
+await __main()`;
 			executionResult = await pyodide.runPythonAsync(runCode, { globals: globalsDict });
 			globalsDict.destroy();
 		} catch (error) {

packages/nodes-base/nodes/Code/Sandbox.ts

@@ -35,7 +35,7 @@ export abstract class Sandbox {
 	constructor(
 		private textKeys: SandboxTextKeys,
 		protected itemIndex: number | undefined,
-		private helpers: IExecuteFunctions['helpers'],
+		protected helpers: IExecuteFunctions['helpers'],
 	) {}
 
 	abstract runCodeAllItems(): Promise<INodeExecutionData[]>;

packages/nodes-base/nodes/Code/descriptions/PythonCodeDescription.ts

@@ -45,19 +45,4 @@ export const pythonCodeDescription: INodeProperties[] = [
 		},
 		default: '',
 	},
-	{
-		displayName: 'Python Modules',
-		name: 'modules',
-		displayOptions: {
-			show: {
-				language: ['python'],
-			},
-		},
-		type: 'string',
-		default: '',
-		placeholder: 'opencv-python',
-		description:
-			'Comma-separated list of Python modules to load. They have to be installed to be able to be loaded and imported.',
-		noDataExpression: true,
-	},
 ];

packages/nodes-base/package.json

@@ -831,7 +831,7 @@
     "pg-promise": "^10.5.8",
     "pretty-bytes": "^5.6.0",
     "promise-ftp": "^1.3.5",
-    "pyodide": "^0.22.1",
+    "pyodide": "^0.23.4",
     "redis": "^3.1.1",
     "rhea": "^1.0.11",
     "rss-parser": "^3.7.0",

packages/workflow/src/AugmentObject.ts

@@ -133,7 +133,11 @@ export function augmentObject<T extends object>(data: T): T {
 
 			return true;
 		},
-
+		has(target, key) {
+			if (deletedProperties.indexOf(key) !== -1) return false;
+			const newKeys = Object.keys(newData);
+			return Reflect.has(newKeys.length ? newData : target, key);
+		},
 		ownKeys(target) {
 			const originalKeys = Reflect.ownKeys(target);
 			const newKeys = Object.keys(newData);

packages/workflow/src/Interfaces.ts

@@ -671,6 +671,7 @@ interface JsonHelperFunctions {
 
 export interface FileSystemHelperFunctions {
 	createReadStream(path: PathLike): Promise<Readable>;
+	getStoragePath(): string;
 }
 
 export interface BinaryHelperFunctions {

packages/workflow/src/ObservableObject.ts

@@ -51,6 +51,9 @@ export function create(
 		get(target, name, receiver) {
 			return Reflect.get(target, name, receiver);
 		},
+		has(target, key) {
+			return Reflect.has(target, key);
+		},
 		set(target, name, value) {
 			if (parent === undefined) {
 				// If no parent is given mark current data as changed

packages/workflow/src/WorkflowDataProxy.ts

@@ -146,6 +146,7 @@ export class WorkflowDataProxy {
 		return new Proxy(
 			{},
 			{
+				has: () => true,
 				ownKeys(target) {
 					if (Reflect.ownKeys(target).length === 0) {
 						// Target object did not get set yet
@@ -178,6 +179,7 @@ export class WorkflowDataProxy {
 		return new Proxy(
 			{},
 			{
+				has: () => true,
 				ownKeys(target) {
 					return Reflect.ownKeys(target);
 				},
@@ -202,6 +204,7 @@ export class WorkflowDataProxy {
 		const node = this.workflow.nodes[nodeName];
 
 		return new Proxy(node.parameters, {
+			has: () => true,
 			ownKeys(target) {
 				return Reflect.ownKeys(target);
 			},
@@ -384,6 +387,7 @@ export class WorkflowDataProxy {
 		return new Proxy(
 			{ binary: undefined, data: undefined, json: undefined },
 			{
+				has: () => true,
 				get(target, name, receiver) {
 					if (name === 'isProxy') return true;
 					name = name.toString();
@@ -461,6 +465,7 @@ export class WorkflowDataProxy {
 		return new Proxy(
 			{},
 			{
+				has: () => true,
 				get(target, name, receiver) {
 					if (name === 'isProxy') return true;
 
@@ -491,6 +496,7 @@ export class WorkflowDataProxy {
 		return new Proxy(
 			{},
 			{
+				has: () => true,
 				ownKeys(target) {
 					return allowedValues;
 				},
@@ -538,6 +544,7 @@ export class WorkflowDataProxy {
 		return new Proxy(
 			{},
 			{
+				has: () => true,
 				ownKeys(target) {
 					return allowedValues;
 				},
@@ -580,6 +587,7 @@ export class WorkflowDataProxy {
 		return new Proxy(
 			{},
 			{
+				has: () => true,
 				get(target, name, receiver) {
 					if (name === 'isProxy') return true;
 
@@ -950,6 +958,7 @@ export class WorkflowDataProxy {
 				return new Proxy(
 					{},
 					{
+						has: () => true,
 						ownKeys(target) {
 							return [
 								'pairedItem',
@@ -1073,6 +1082,7 @@ export class WorkflowDataProxy {
 			},
 
 			$input: new Proxy({} as ProxyInput, {
+				has: () => true,
 				ownKeys(target) {
 					return ['all', 'context', 'first', 'item', 'last', 'params'];
 				},
@@ -1238,6 +1248,7 @@ export class WorkflowDataProxy {
 		};
 
 		return new Proxy(base, {
+			has: () => true,
 			get(target, name, receiver) {
 				if (name === 'isProxy') return true;
 

packages/workflow/test/AugmentObject.test.ts

@@ -557,5 +557,19 @@ describe('AugmentObject', () => {
 				writable: true,
 			});
 		});
+
+		test('should return valid values on `has` calls', () => {
+			const originalObject = {
+				x: {
+					y: {},
+				},
+			};
+			const augmentedObject = augmentObject(originalObject);
+			expect('y' in augmentedObject.x).toBe(true);
+			expect('z' in augmentedObject.x).toBe(false);
+
+			augmentedObject.x.z = 5;
+			expect('z' in augmentedObject.x).toBe(true);
+		});
 	});
 });

patches/pyodide@0.23.4.patch

@@ -0,0 +1,20 @@
+diff --git a/pyodide.d.ts b/pyodide.d.ts
+index d5ed46f6345855a75ec6f2b7ef73237a0af64a7e..e0087c83792558ca30713e9d07a9a37625f68d8d 100644
+--- a/pyodide.d.ts
++++ b/pyodide.d.ts
+@@ -1118,6 +1118,15 @@ export declare function loadPyodide(options?: {
+ 	 * (``pyodide.js`` or ``pyodide.mjs``) removed.
+ 	 */
+ 	indexURL?: string;
++	/**
++	 * The file path where packages will be cached in `node.js`. If a package
++	 * exists in `packageCacheDir` it is loaded from there, otherwise it is
++	 * downloaded from the JsDelivr CDN and then cached into `packageCacheDir`.
++	 * Only applies when running in node.js. Ignored in browsers.
++	 *
++	 * Default: same as indexURL
++	 */
++	packageCacheDir?: string;
+ 	/**
+ 	 * file. You can produce custom lock files with :py:func:`micropip.freeze`.
+ 	 * Default: ```${indexURL}/repodata.json```

pnpm-lock.yaml

@@ -35,6 +35,9 @@ patchedDependencies:
   pkce-challenge@3.0.0:
     hash: dypouzb3lve7vncq25i5fuanki
     path: patches/pkce-challenge@3.0.0.patch
+  pyodide@0.23.4:
+    hash: kzcwsjcayy5m6iezu7r4tdimjq
+    path: patches/pyodide@0.23.4.patch
   typedi@0.10.0:
     hash: 62r6bc2crgimafeyruodhqlgo4
     path: patches/typedi@0.10.0.patch
@@ -1142,8 +1145,8 @@ importers:
         specifier: ^1.3.5
         version: 1.3.5(promise-ftp-common@1.1.5)
       pyodide:
-        specifier: ^0.22.1
-        version: 0.22.1
+        specifier: ^0.23.4
+        version: 0.23.4(patch_hash=kzcwsjcayy5m6iezu7r4tdimjq)
       redis:
         specifier: ^3.1.1
         version: 3.1.2
@@ -18656,8 +18659,8 @@ packages:
     resolution: {integrity: sha512-t+x1zEHDjBwkDGY5v5ApnZ/utcd4XYDiJsaQQoptTXgUXX95sDg1elCdJghzicm7n2mbCBJ3uYWr6M22SO19rg==}
     dev: true
 
-  /pyodide@0.22.1:
-    resolution: {integrity: sha512-6+PkFLTC+kcBKtFQxYBxR44J5IBxLm8UGkobLgZv1SxzV9qOU2rb0YYf0qDtlnfDiN/IQd2uckf+D8Zwe88Mqg==}
+  /pyodide@0.23.4(patch_hash=kzcwsjcayy5m6iezu7r4tdimjq):
+    resolution: {integrity: sha512-WpQUHaIXQ1xede5BMqPAjBcmopxN22s5hEsYOR8T7/UW/fkNLFUn07SaemUgthbtvedD5JGymMMj4VpD9sGMTg==}
     dependencies:
       base-64: 1.0.0
       node-fetch: 2.6.8
@@ -18667,6 +18670,7 @@ packages:
       - encoding
       - utf-8-validate
     dev: false
+    patched: true
 
   /python-struct@1.1.3:
     resolution: {integrity: sha512-UsI/mNvk25jRpGKYI38Nfbv84z48oiIWwG67DLVvjRhy8B/0aIK+5Ju5WOHgw/o9rnEmbAS00v4rgKFQeC332Q==}