fix(HTTP Request Node): Use iconv-lite to decode http responses, to support more encoding types (#11930)

packages/@n8n/imap/package.json

@@ -20,7 +20,7 @@
     "dist/**/*"
   ],
   "dependencies": {
-    "iconv-lite": "0.6.3",
+    "iconv-lite": "catalog:",
     "imap": "0.8.19",
     "quoted-printable": "1.0.1",
     "utf8": "3.0.0",

packages/core/package.json

@@ -47,6 +47,7 @@
     "fast-glob": "catalog:",
     "file-type": "16.5.4",
     "form-data": "catalog:",
+    "iconv-lite": "catalog:",
     "lodash": "catalog:",
     "luxon": "catalog:",
     "mime-types": "2.1.35",

packages/core/src/NodeExecuteFunctions.ts

@@ -28,6 +28,7 @@ import { createReadStream } from 'fs';
 import { access as fsAccess, writeFile as fsWriteFile } from 'fs/promises';
 import { IncomingMessage } from 'http';
 import { Agent, type AgentOptions } from 'https';
+import iconv from 'iconv-lite';
 import get from 'lodash/get';
 import isEmpty from 'lodash/isEmpty';
 import merge from 'lodash/merge';
@@ -745,13 +746,13 @@ export function parseIncomingMessage(message: IncomingMessage) {
 	}
 }
 
-export async function binaryToString(body: Buffer | Readable, encoding?: BufferEncoding) {
+export async function binaryToString(body: Buffer | Readable, encoding?: string) {
-	const buffer = await binaryToBuffer(body);
 	if (!encoding && body instanceof IncomingMessage) {
 		parseIncomingMessage(body);
 		encoding = body.encoding;
 	}
-	return buffer.toString(encoding);
+	const buffer = await binaryToBuffer(body);
+	return iconv.decode(buffer, encoding ?? 'utf-8');
 }
 
 export async function proxyRequestToAxios(

packages/core/test/NodeExecuteFunctions.test.ts

@@ -1,5 +1,5 @@
 import { mkdtempSync, readFileSync } from 'fs';
-import type { IncomingMessage } from 'http';
+import { IncomingMessage } from 'http';
 import type { Agent } from 'https';
 import { mock } from 'jest-mock-extended';
 import type {
@@ -16,12 +16,14 @@ import type {
 import nock from 'nock';
 import { tmpdir } from 'os';
 import { join } from 'path';
+import { Readable } from 'stream';
 import type { SecureContextOptions } from 'tls';
 import Container from 'typedi';
 
 import { BinaryDataService } from '@/BinaryData/BinaryData.service';
 import { InstanceSettings } from '@/InstanceSettings';
 import {
+	binaryToString,
 	copyInputItems,
 	getBinaryDataBuffer,
 	isFilePathBlocked,
@@ -549,6 +551,101 @@ describe('NodeExecuteFunctions', () => {
 			},
 		);
 	});
+
+	describe('binaryToString', () => {
+		const ENCODING_SAMPLES = {
+			utf8: {
+				text: 'Hello, 世界! τεστ мир ⚡️ é à ü ñ',
+				buffer: Buffer.from([
+					0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x2c, 0x20, 0xe4, 0xb8, 0x96, 0xe7, 0x95, 0x8c, 0x21, 0x20,
+					0xcf, 0x84, 0xce, 0xb5, 0xcf, 0x83, 0xcf, 0x84, 0x20, 0xd0, 0xbc, 0xd0, 0xb8, 0xd1, 0x80,
+					0x20, 0xe2, 0x9a, 0xa1, 0xef, 0xb8, 0x8f, 0x20, 0xc3, 0xa9, 0x20, 0xc3, 0xa0, 0x20, 0xc3,
+					0xbc, 0x20, 0xc3, 0xb1,
+				]),
+			},
+
+			'iso-8859-15': {
+				text: 'Café € personnalité',
+				buffer: Buffer.from([
+					0x43, 0x61, 0x66, 0xe9, 0x20, 0xa4, 0x20, 0x70, 0x65, 0x72, 0x73, 0x6f, 0x6e, 0x6e, 0x61,
+					0x6c, 0x69, 0x74, 0xe9,
+				]),
+			},
+
+			latin1: {
+				text: 'señor année déjà',
+				buffer: Buffer.from([
+					0x73, 0x65, 0xf1, 0x6f, 0x72, 0x20, 0x61, 0x6e, 0x6e, 0xe9, 0x65, 0x20, 0x64, 0xe9, 0x6a,
+					0xe0,
+				]),
+			},
+
+			ascii: {
+				text: 'Hello, World! 123',
+				buffer: Buffer.from([
+					0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x2c, 0x20, 0x57, 0x6f, 0x72, 0x6c, 0x64, 0x21, 0x20, 0x31,
+					0x32, 0x33,
+				]),
+			},
+
+			'windows-1252': {
+				text: '€ Smart "quotes" • bullet',
+				buffer: Buffer.from([
+					0x80, 0x20, 0x53, 0x6d, 0x61, 0x72, 0x74, 0x20, 0x22, 0x71, 0x75, 0x6f, 0x74, 0x65, 0x73,
+					0x22, 0x20, 0x95, 0x20, 0x62, 0x75, 0x6c, 0x6c, 0x65, 0x74,
+				]),
+			},
+
+			'shift-jis': {
+				text: 'こんにちは世界',
+				buffer: Buffer.from([
+					0x82, 0xb1, 0x82, 0xf1, 0x82, 0xc9, 0x82, 0xbf, 0x82, 0xcd, 0x90, 0xa2, 0x8a, 0x45,
+				]),
+			},
+
+			big5: {
+				text: '哈囉世界',
+				buffer: Buffer.from([0xab, 0xa2, 0xc5, 0x6f, 0xa5, 0x40, 0xac, 0xc9]),
+			},
+
+			'koi8-r': {
+				text: 'Привет мир',
+				buffer: Buffer.from([0xf0, 0xd2, 0xc9, 0xd7, 0xc5, 0xd4, 0x20, 0xcd, 0xc9, 0xd2]),
+			},
+		};
+
+		describe('should handle Buffer', () => {
+			for (const [encoding, { text, buffer }] of Object.entries(ENCODING_SAMPLES)) {
+				test(`with ${encoding}`, async () => {
+					const data = await binaryToString(buffer, encoding);
+					expect(data).toBe(text);
+				});
+			}
+		});
+
+		describe('should handle streams', () => {
+			for (const [encoding, { text, buffer }] of Object.entries(ENCODING_SAMPLES)) {
+				test(`with ${encoding}`, async () => {
+					const stream = Readable.from(buffer);
+					const data = await binaryToString(stream, encoding);
+					expect(data).toBe(text);
+				});
+			}
+		});
+
+		describe('should handle IncomingMessage', () => {
+			for (const [encoding, { text, buffer }] of Object.entries(ENCODING_SAMPLES)) {
+				test(`with ${encoding}`, async () => {
+					const response = Readable.from(buffer) as IncomingMessage;
+					response.headers = { 'content-type': `application/json;charset=${encoding}` };
+					// @ts-expect-error need this hack to fake `instanceof IncomingMessage` checks
+					response.__proto__ = IncomingMessage.prototype;
+					const data = await binaryToString(response);
+					expect(data).toBe(text);
+				});
+			}
+		});
+	});
 });
 
 describe('isFilePathBlocked', () => {

packages/nodes-base/package.json

@@ -868,7 +868,7 @@
     "get-system-fonts": "2.0.2",
     "gm": "1.25.0",
     "html-to-text": "9.0.5",
-    "iconv-lite": "0.6.3",
+    "iconv-lite": "catalog:",
     "ics": "2.40.0",
     "isbot": "3.6.13",
     "iso-639-1": "2.1.15",

pnpm-lock.yaml

@@ -45,6 +45,9 @@ catalogs:
     form-data:
       specifier: 4.0.0
       version: 4.0.0
+    iconv-lite:
+      specifier: 0.6.3
+      version: 0.6.3
     lodash:
       specifier: 4.17.21
       version: 4.17.21
@@ -280,7 +283,7 @@ importers:
         version: 4.0.7
       axios:
         specifier: 'catalog:'
-        version: 1.7.4
+        version: 1.7.4(debug@4.3.7)
       dotenv:
         specifier: 8.6.0
         version: 8.6.0
@@ -348,7 +351,7 @@ importers:
     dependencies:
       axios:
         specifier: 'catalog:'
-        version: 1.7.4
+        version: 1.7.4(debug@4.3.7)
 
   packages/@n8n/codemirror-lang:
     dependencies:
@@ -378,7 +381,7 @@ importers:
   packages/@n8n/imap:
     dependencies:
       iconv-lite:
-        specifier: 0.6.3
+        specifier: 'catalog:'
         version: 0.6.3
       imap:
         specifier: 0.8.19
@@ -422,7 +425,7 @@ importers:
         version: 3.666.0(@aws-sdk/client-sts@3.666.0)
       '@getzep/zep-cloud':
         specifier: 1.0.12
-        version: 1.0.12(@langchain/core@0.3.15(openai@4.69.0(encoding@0.1.13)(zod@3.23.8)))(encoding@0.1.13)(langchain@0.3.5(7umjwzmwnymi4lyinuvazmp6ki))
+        version: 1.0.12(@langchain/core@0.3.15(openai@4.69.0(encoding@0.1.13)(zod@3.23.8)))(encoding@0.1.13)(langchain@0.3.5(4ubssgvn2k3t3hxnzmxuoc2aja))
       '@getzep/zep-js':
         specifier: 0.9.0
         version: 0.9.0
@@ -449,7 +452,7 @@ importers:
         version: 0.3.1(@aws-sdk/client-sso-oidc@3.666.0(@aws-sdk/client-sts@3.666.0))(@langchain/core@0.3.15(openai@4.69.0(encoding@0.1.13)(zod@3.23.8)))(encoding@0.1.13)
       '@langchain/community':
         specifier: 0.3.11
-        version: 0.3.11(ndajhtzj4xqxxpqz4t56suvqri)
+        version: 0.3.11(qw65fvnztmfuymkskopqa7kjky)
       '@langchain/core':
         specifier: 'catalog:'
         version: 0.3.15(openai@4.69.0(encoding@0.1.13)(zod@3.23.8))
@@ -536,7 +539,7 @@ importers:
         version: 23.0.1
       langchain:
         specifier: 0.3.5
-        version: 0.3.5(7umjwzmwnymi4lyinuvazmp6ki)
+        version: 0.3.5(4ubssgvn2k3t3hxnzmxuoc2aja)
       lodash:
         specifier: 'catalog:'
         version: 4.17.21
@@ -795,7 +798,7 @@ importers:
         version: 1.11.0
       axios:
         specifier: 'catalog:'
-        version: 1.7.4
+        version: 1.7.4(debug@4.3.7)
       bcryptjs:
         specifier: 2.4.3
         version: 2.4.3
@@ -1126,7 +1129,7 @@ importers:
         version: 1.11.0
       axios:
         specifier: 'catalog:'
-        version: 1.7.4
+        version: 1.7.4(debug@4.3.7)
       concat-stream:
         specifier: 2.0.0
         version: 2.0.0
@@ -1142,6 +1145,9 @@ importers:
       form-data:
         specifier: 'catalog:'
         version: 4.0.0
+      iconv-lite:
+        specifier: 'catalog:'
+        version: 0.6.3
       lodash:
         specifier: 'catalog:'
         version: 4.17.21
@@ -1416,7 +1422,7 @@ importers:
         version: 10.11.0(vue@3.5.11(typescript@5.7.2))
       axios:
         specifier: 'catalog:'
-        version: 1.7.4
+        version: 1.7.4(debug@4.3.7)
       bowser:
         specifier: 2.11.0
         version: 2.11.0
@@ -1678,7 +1684,7 @@ importers:
         specifier: 9.0.5
         version: 9.0.5
       iconv-lite:
-        specifier: 0.6.3
+        specifier: 'catalog:'
         version: 0.6.3
       ics:
         specifier: 2.40.0
@@ -1899,7 +1905,7 @@ importers:
         version: 0.15.2
       axios:
         specifier: 'catalog:'
-        version: 1.7.4
+        version: 1.7.4(debug@4.3.7)
       callsites:
         specifier: 3.1.0
         version: 3.1.0
@@ -14121,7 +14127,7 @@ snapshots:
   '@gar/promisify@1.1.3':
     optional: true
 
-  '@getzep/zep-cloud@1.0.12(@langchain/core@0.3.15(openai@4.69.0(encoding@0.1.13)(zod@3.23.8)))(encoding@0.1.13)(langchain@0.3.5(7umjwzmwnymi4lyinuvazmp6ki))':
+  '@getzep/zep-cloud@1.0.12(@langchain/core@0.3.15(openai@4.69.0(encoding@0.1.13)(zod@3.23.8)))(encoding@0.1.13)(langchain@0.3.5(4ubssgvn2k3t3hxnzmxuoc2aja))':
     dependencies:
       form-data: 4.0.0
       node-fetch: 2.7.0(encoding@0.1.13)
@@ -14130,7 +14136,7 @@ snapshots:
       zod: 3.23.8
     optionalDependencies:
       '@langchain/core': 0.3.15(openai@4.69.0(encoding@0.1.13)(zod@3.23.8))
-      langchain: 0.3.5(7umjwzmwnymi4lyinuvazmp6ki)
+      langchain: 0.3.5(4ubssgvn2k3t3hxnzmxuoc2aja)
     transitivePeerDependencies:
       - encoding
 
@@ -14597,7 +14603,7 @@ snapshots:
       - aws-crt
       - encoding
 
-  '@langchain/community@0.3.11(ndajhtzj4xqxxpqz4t56suvqri)':
+  '@langchain/community@0.3.11(qw65fvnztmfuymkskopqa7kjky)':
     dependencies:
       '@ibm-cloud/watsonx-ai': 1.1.2
       '@langchain/core': 0.3.15(openai@4.69.0(encoding@0.1.13)(zod@3.23.8))
@@ -14607,7 +14613,7 @@ snapshots:
       flat: 5.0.2
       ibm-cloud-sdk-core: 5.1.0
       js-yaml: 4.1.0
-      langchain: 0.3.5(7umjwzmwnymi4lyinuvazmp6ki)
+      langchain: 0.3.5(4ubssgvn2k3t3hxnzmxuoc2aja)
       langsmith: 0.2.3(openai@4.69.0(encoding@0.1.13)(zod@3.23.8))
       uuid: 10.0.0
       zod: 3.23.8
@@ -14620,7 +14626,7 @@ snapshots:
       '@aws-sdk/client-s3': 3.666.0
       '@aws-sdk/credential-provider-node': 3.666.0(@aws-sdk/client-sso-oidc@3.666.0(@aws-sdk/client-sts@3.666.0))(@aws-sdk/client-sts@3.666.0)
       '@azure/storage-blob': 12.18.0(encoding@0.1.13)
-      '@getzep/zep-cloud': 1.0.12(@langchain/core@0.3.15(openai@4.69.0(encoding@0.1.13)(zod@3.23.8)))(encoding@0.1.13)(langchain@0.3.5(7umjwzmwnymi4lyinuvazmp6ki))
+      '@getzep/zep-cloud': 1.0.12(@langchain/core@0.3.15(openai@4.69.0(encoding@0.1.13)(zod@3.23.8)))(encoding@0.1.13)(langchain@0.3.5(4ubssgvn2k3t3hxnzmxuoc2aja))
       '@getzep/zep-js': 0.9.0
       '@google-ai/generativelanguage': 2.6.0(encoding@0.1.13)
       '@google-cloud/storage': 7.12.1(encoding@0.1.13)
@@ -15301,7 +15307,7 @@ snapshots:
 
   '@rudderstack/rudder-sdk-node@2.0.9(tslib@2.6.2)':
     dependencies:
-      axios: 1.7.4
+      axios: 1.7.4(debug@4.3.7)
       axios-retry: 3.7.0
       component-type: 1.2.1
       join-component: 1.1.0
@@ -17557,17 +17563,9 @@ snapshots:
       '@babel/runtime': 7.24.7
       is-retry-allowed: 2.2.0
 
-  axios@1.7.4:
-    dependencies:
-      follow-redirects: 1.15.6(debug@4.3.6)
-      form-data: 4.0.0
-      proxy-from-env: 1.1.0
-    transitivePeerDependencies:
-      - debug
-
   axios@1.7.4(debug@4.3.7):
     dependencies:
-      follow-redirects: 1.15.6(debug@4.3.7)
+      follow-redirects: 1.15.6(debug@4.3.6)
       form-data: 4.0.0
       proxy-from-env: 1.1.0
     transitivePeerDependencies:
@@ -21409,7 +21407,7 @@ snapshots:
 
   kuler@2.0.0: {}
 
-  langchain@0.3.5(7umjwzmwnymi4lyinuvazmp6ki):
+  langchain@0.3.5(4ubssgvn2k3t3hxnzmxuoc2aja):
     dependencies:
       '@langchain/core': 0.3.15(openai@4.69.0(encoding@0.1.13)(zod@3.23.8))
       '@langchain/openai': 0.3.11(@langchain/core@0.3.15(openai@4.69.0(encoding@0.1.13)(zod@3.23.8)))(encoding@0.1.13)
@@ -21433,7 +21431,7 @@ snapshots:
       '@langchain/groq': 0.1.2(@langchain/core@0.3.15(openai@4.69.0(encoding@0.1.13)(zod@3.23.8)))(encoding@0.1.13)
       '@langchain/mistralai': 0.1.1(@langchain/core@0.3.15(openai@4.69.0(encoding@0.1.13)(zod@3.23.8)))(encoding@0.1.13)
       '@langchain/ollama': 0.1.1(@langchain/core@0.3.15(openai@4.69.0(encoding@0.1.13)(zod@3.23.8)))
-      axios: 1.7.4
+      axios: 1.7.4(debug@4.3.7)
       cheerio: 1.0.0
       handlebars: 4.7.8
     transitivePeerDependencies:

pnpm-workspace.yaml

@@ -18,6 +18,7 @@ catalog:
   fast-glob: 3.2.12
   flatted: 3.2.7
   form-data: 4.0.0
+  iconv-lite: 0.6.3
   lodash: 4.17.21
   luxon: 3.4.4
   nanoid: 3.3.6