mudhorn/n8n
1
0
Fork 0
mirror of https://github.com/n8n-io/n8n.git synced 2025-03-05 20:50:17 -08:00
Code Issues Actions Packages Projects Releases Wiki Activity

fix(HTTP Request Node): Use iconv-lite to decode http responses, to support more encoding types (#11930)

Browse source
This commit is contained in:
कारतोफ्फेलस्क्रिप्ट™ 2024-11-28 14:31:54 +01:00 committed by GitHub
parent eccd924f5e
commit 461b39c5df
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 132 additions and 34 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
packages/@n8n/imap/package.json
View file

@ -20,7 +20,7 @@
"dist/**/*"
],
"dependencies": {
"iconv-lite": "0.6.3",
"iconv-lite": "catalog:",
"imap": "0.8.19",
"quoted-printable": "1.0.1",
"utf8": "3.0.0",

1
packages/core/package.json
View file

@ -47,6 +47,7 @@
"fast-glob": "catalog:",
"file-type": "16.5.4",
"form-data": "catalog:",
"iconv-lite": "catalog:",
"lodash": "catalog:",
"luxon": "catalog:",
"mime-types": "2.1.35",

7
packages/core/src/NodeExecuteFunctions.ts
View file

@ -28,6 +28,7 @@ import { createReadStream } from 'fs';
import { access as fsAccess, writeFile as fsWriteFile } from 'fs/promises';
import { IncomingMessage } from 'http';
import { Agent, type AgentOptions } from 'https';
import iconv from 'iconv-lite';
import get from 'lodash/get';
import isEmpty from 'lodash/isEmpty';
import merge from 'lodash/merge';
@ -745,13 +746,13 @@ export function parseIncomingMessage(message: IncomingMessage) {
}
}
export async function binaryToString(body: Buffer | Readable, encoding?: BufferEncoding) {
const buffer = await binaryToBuffer(body);
export async function binaryToString(body: Buffer | Readable, encoding?: string) {
if (!encoding && body instanceof IncomingMessage) {
parseIncomingMessage(body);
encoding = body.encoding;
}
return buffer.toString(encoding);
const buffer = await binaryToBuffer(body);
return iconv.decode(buffer, encoding ?? 'utf-8');
}
export async function proxyRequestToAxios(

99
packages/core/test/NodeExecuteFunctions.test.ts
View file

@ -1,5 +1,5 @@
import { mkdtempSync, readFileSync } from 'fs';
import type { IncomingMessage } from 'http';
import { IncomingMessage } from 'http';
import type { Agent } from 'https';
import { mock } from 'jest-mock-extended';
import type {
@ -16,12 +16,14 @@ import type {
import nock from 'nock';
import { tmpdir } from 'os';
import { join } from 'path';
import { Readable } from 'stream';
import type { SecureContextOptions } from 'tls';
import Container from 'typedi';
import { BinaryDataService } from '@/BinaryData/BinaryData.service';
import { InstanceSettings } from '@/InstanceSettings';
import {
binaryToString,
copyInputItems,
getBinaryDataBuffer,
isFilePathBlocked,
@ -549,6 +551,101 @@ describe('NodeExecuteFunctions', () => {
},
);
});
describe('binaryToString', () => {
const ENCODING_SAMPLES = {
utf8: {
text: 'Hello, 世界! τεστ мир ⚡️ é à ü ñ',
buffer: Buffer.from([
0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x2c, 0x20, 0xe4, 0xb8, 0x96, 0xe7, 0x95, 0x8c, 0x21, 0x20,
0xcf, 0x84, 0xce, 0xb5, 0xcf, 0x83, 0xcf, 0x84, 0x20, 0xd0, 0xbc, 0xd0, 0xb8, 0xd1, 0x80,
0x20, 0xe2, 0x9a, 0xa1, 0xef, 0xb8, 0x8f, 0x20, 0xc3, 0xa9, 0x20, 0xc3, 0xa0, 0x20, 0xc3,
0xbc, 0x20, 0xc3, 0xb1,
]),
},
'iso-8859-15': {
text: 'Café € personnalité',
buffer: Buffer.from([
0x43, 0x61, 0x66, 0xe9, 0x20, 0xa4, 0x20, 0x70, 0x65, 0x72, 0x73, 0x6f, 0x6e, 0x6e, 0x61,
0x6c, 0x69, 0x74, 0xe9,
]),
},
latin1: {
text: 'señor année déjà',
buffer: Buffer.from([
0x73, 0x65, 0xf1, 0x6f, 0x72, 0x20, 0x61, 0x6e, 0x6e, 0xe9, 0x65, 0x20, 0x64, 0xe9, 0x6a,
0xe0,
]),
},
ascii: {
text: 'Hello, World! 123',
buffer: Buffer.from([
0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x2c, 0x20, 0x57, 0x6f, 0x72, 0x6c, 0x64, 0x21, 0x20, 0x31,
0x32, 0x33,
]),
},
'windows-1252': {
text: '€ Smart "quotes" • bullet',
buffer: Buffer.from([
0x80, 0x20, 0x53, 0x6d, 0x61, 0x72, 0x74, 0x20, 0x22, 0x71, 0x75, 0x6f, 0x74, 0x65, 0x73,
0x22, 0x20, 0x95, 0x20, 0x62, 0x75, 0x6c, 0x6c, 0x65, 0x74,
]),
},
'shift-jis': {
text: 'こんにちは世界',
buffer: Buffer.from([
0x82, 0xb1, 0x82, 0xf1, 0x82, 0xc9, 0x82, 0xbf, 0x82, 0xcd, 0x90, 0xa2, 0x8a, 0x45,
]),
},
big5: {
text: '哈囉世界',
buffer: Buffer.from([0xab, 0xa2, 0xc5, 0x6f, 0xa5, 0x40, 0xac, 0xc9]),
},
'koi8-r': {
text: 'Привет мир',
buffer: Buffer.from([0xf0, 0xd2, 0xc9, 0xd7, 0xc5, 0xd4, 0x20, 0xcd, 0xc9, 0xd2]),
},
};
describe('should handle Buffer', () => {
for (const [encoding, { text, buffer }] of Object.entries(ENCODING_SAMPLES)) {
test(`with ${encoding}`, async () => {
const data = await binaryToString(buffer, encoding);
expect(data).toBe(text);
});
}
});
describe('should handle streams', () => {
for (const [encoding, { text, buffer }] of Object.entries(ENCODING_SAMPLES)) {
test(`with ${encoding}`, async () => {
const stream = Readable.from(buffer);
const data = await binaryToString(stream, encoding);
expect(data).toBe(text);
});
}
});
describe('should handle IncomingMessage', () => {
for (const [encoding, { text, buffer }] of Object.entries(ENCODING_SAMPLES)) {
test(`with ${encoding}`, async () => {
const response = Readable.from(buffer) as IncomingMessage;
response.headers = { 'content-type': `application/json;charset=${encoding}` };
// @ts-expect-error need this hack to fake `instanceof IncomingMessage` checks
response.__proto__ = IncomingMessage.prototype;
const data = await binaryToString(response);
expect(data).toBe(text);
});
}
});
});
});
describe('isFilePathBlocked', () => {

2
packages/nodes-base/package.json
View file

@ -868,7 +868,7 @@
"get-system-fonts": "2.0.2",
"gm": "1.25.0",
"html-to-text": "9.0.5",
"iconv-lite": "0.6.3",
"iconv-lite": "catalog:",
"ics": "2.40.0",
"isbot": "3.6.13",
"iso-639-1": "2.1.15",

54
pnpm-lock.yaml
View file

@ -45,6 +45,9 @@ catalogs:
form-data:
specifier: 4.0.0
version: 4.0.0
iconv-lite:
specifier: 0.6.3
version: 0.6.3
lodash:
specifier: 4.17.21
version: 4.17.21
@ -280,7 +283,7 @@ importers:
version: 4.0.7
axios:
specifier: 'catalog:'
version: 1.7.4
version: 1.7.4(debug@4.3.7)
dotenv:
specifier: 8.6.0
version: 8.6.0
@ -348,7 +351,7 @@ importers:
dependencies:
axios:
specifier: 'catalog:'
version: 1.7.4
version: 1.7.4(debug@4.3.7)
packages/@n8n/codemirror-lang:
dependencies:
@ -378,7 +381,7 @@ importers:
packages/@n8n/imap:
dependencies:
iconv-lite:
specifier: 0.6.3
specifier: 'catalog:'
version: 0.6.3
imap:
specifier: 0.8.19
@ -422,7 +425,7 @@ importers:
version: 3.666.0(@aws-sdk/client-sts@3.666.0)
'@getzep/zep-cloud':
specifier: 1.0.12
version: 1.0.12(@langchain/core@0.3.15(openai@4.69.0(encoding@0.1.13)(zod@3.23.8)))(encoding@0.1.13)(langchain@0.3.5(7umjwzmwnymi4lyinuvazmp6ki))
version: 1.0.12(@langchain/core@0.3.15(openai@4.69.0(encoding@0.1.13)(zod@3.23.8)))(encoding@0.1.13)(langchain@0.3.5(4ubssgvn2k3t3hxnzmxuoc2aja))
'@getzep/zep-js':
specifier: 0.9.0
version: 0.9.0
@ -449,7 +452,7 @@ importers:
version: 0.3.1(@aws-sdk/client-sso-oidc@3.666.0(@aws-sdk/client-sts@3.666.0))(@langchain/core@0.3.15(openai@4.69.0(encoding@0.1.13)(zod@3.23.8)))(encoding@0.1.13)
'@langchain/community':
specifier: 0.3.11
version: 0.3.11(ndajhtzj4xqxxpqz4t56suvqri)
version: 0.3.11(qw65fvnztmfuymkskopqa7kjky)
'@langchain/core':
specifier: 'catalog:'
version: 0.3.15(openai@4.69.0(encoding@0.1.13)(zod@3.23.8))
@ -536,7 +539,7 @@ importers:
version: 23.0.1
langchain:
specifier: 0.3.5
version: 0.3.5(7umjwzmwnymi4lyinuvazmp6ki)
version: 0.3.5(4ubssgvn2k3t3hxnzmxuoc2aja)
lodash:
specifier: 'catalog:'
version: 4.17.21
@ -795,7 +798,7 @@ importers:
version: 1.11.0
axios:
specifier: 'catalog:'
version: 1.7.4
version: 1.7.4(debug@4.3.7)
bcryptjs:
specifier: 2.4.3
version: 2.4.3
@ -1126,7 +1129,7 @@ importers:
version: 1.11.0
axios:
specifier: 'catalog:'
version: 1.7.4
version: 1.7.4(debug@4.3.7)
concat-stream:
specifier: 2.0.0
version: 2.0.0
@ -1142,6 +1145,9 @@ importers:
form-data:
specifier: 'catalog:'
version: 4.0.0
iconv-lite:
specifier: 'catalog:'
version: 0.6.3
lodash:
specifier: 'catalog:'
version: 4.17.21
@ -1416,7 +1422,7 @@ importers:
version: 10.11.0(vue@3.5.11(typescript@5.7.2))
axios:
specifier: 'catalog:'
version: 1.7.4
version: 1.7.4(debug@4.3.7)
bowser:
specifier: 2.11.0
version: 2.11.0
@ -1678,7 +1684,7 @@ importers:
specifier: 9.0.5
version: 9.0.5
iconv-lite:
specifier: 0.6.3
specifier: 'catalog:'
version: 0.6.3
ics:
specifier: 2.40.0
@ -1899,7 +1905,7 @@ importers:
version: 0.15.2
axios:
specifier: 'catalog:'
version: 1.7.4
version: 1.7.4(debug@4.3.7)
callsites:
specifier: 3.1.0
version: 3.1.0
@ -14121,7 +14127,7 @@ snapshots:
'@gar/promisify@1.1.3':
optional: true
'@getzep/zep-cloud@1.0.12(@langchain/core@0.3.15(openai@4.69.0(encoding@0.1.13)(zod@3.23.8)))(encoding@0.1.13)(langchain@0.3.5(7umjwzmwnymi4lyinuvazmp6ki))':
'@getzep/zep-cloud@1.0.12(@langchain/core@0.3.15(openai@4.69.0(encoding@0.1.13)(zod@3.23.8)))(encoding@0.1.13)(langchain@0.3.5(4ubssgvn2k3t3hxnzmxuoc2aja))':
dependencies:
form-data: 4.0.0
node-fetch: 2.7.0(encoding@0.1.13)
@ -14130,7 +14136,7 @@ snapshots:
zod: 3.23.8
optionalDependencies:
'@langchain/core': 0.3.15(openai@4.69.0(encoding@0.1.13)(zod@3.23.8))
langchain: 0.3.5(7umjwzmwnymi4lyinuvazmp6ki)
langchain: 0.3.5(4ubssgvn2k3t3hxnzmxuoc2aja)
transitivePeerDependencies:
- encoding
@ -14597,7 +14603,7 @@ snapshots:
- aws-crt
- encoding
'@langchain/community@0.3.11(ndajhtzj4xqxxpqz4t56suvqri)':
'@langchain/community@0.3.11(qw65fvnztmfuymkskopqa7kjky)':
dependencies:
'@ibm-cloud/watsonx-ai': 1.1.2
'@langchain/core': 0.3.15(openai@4.69.0(encoding@0.1.13)(zod@3.23.8))
@ -14607,7 +14613,7 @@ snapshots:
flat: 5.0.2
ibm-cloud-sdk-core: 5.1.0
js-yaml: 4.1.0
langchain: 0.3.5(7umjwzmwnymi4lyinuvazmp6ki)
langchain: 0.3.5(4ubssgvn2k3t3hxnzmxuoc2aja)
langsmith: 0.2.3(openai@4.69.0(encoding@0.1.13)(zod@3.23.8))
uuid: 10.0.0
zod: 3.23.8
@ -14620,7 +14626,7 @@ snapshots:
'@aws-sdk/client-s3': 3.666.0
'@aws-sdk/credential-provider-node': 3.666.0(@aws-sdk/client-sso-oidc@3.666.0(@aws-sdk/client-sts@3.666.0))(@aws-sdk/client-sts@3.666.0)
'@azure/storage-blob': 12.18.0(encoding@0.1.13)
'@getzep/zep-cloud': 1.0.12(@langchain/core@0.3.15(openai@4.69.0(encoding@0.1.13)(zod@3.23.8)))(encoding@0.1.13)(langchain@0.3.5(7umjwzmwnymi4lyinuvazmp6ki))
'@getzep/zep-cloud': 1.0.12(@langchain/core@0.3.15(openai@4.69.0(encoding@0.1.13)(zod@3.23.8)))(encoding@0.1.13)(langchain@0.3.5(4ubssgvn2k3t3hxnzmxuoc2aja))
'@getzep/zep-js': 0.9.0
'@google-ai/generativelanguage': 2.6.0(encoding@0.1.13)
'@google-cloud/storage': 7.12.1(encoding@0.1.13)
@ -15301,7 +15307,7 @@ snapshots:
'@rudderstack/rudder-sdk-node@2.0.9(tslib@2.6.2)':
dependencies:
axios: 1.7.4
axios: 1.7.4(debug@4.3.7)
axios-retry: 3.7.0
component-type: 1.2.1
join-component: 1.1.0
@ -17557,17 +17563,9 @@ snapshots:
'@babel/runtime': 7.24.7
is-retry-allowed: 2.2.0
axios@1.7.4:
dependencies:
follow-redirects: 1.15.6(debug@4.3.6)
form-data: 4.0.0
proxy-from-env: 1.1.0
transitivePeerDependencies:
- debug
axios@1.7.4(debug@4.3.7):
dependencies:
follow-redirects: 1.15.6(debug@4.3.7)
follow-redirects: 1.15.6(debug@4.3.6)
form-data: 4.0.0
proxy-from-env: 1.1.0
transitivePeerDependencies:
@ -21409,7 +21407,7 @@ snapshots:
kuler@2.0.0: {}
langchain@0.3.5(7umjwzmwnymi4lyinuvazmp6ki):
langchain@0.3.5(4ubssgvn2k3t3hxnzmxuoc2aja):
dependencies:
'@langchain/core': 0.3.15(openai@4.69.0(encoding@0.1.13)(zod@3.23.8))
'@langchain/openai': 0.3.11(@langchain/core@0.3.15(openai@4.69.0(encoding@0.1.13)(zod@3.23.8)))(encoding@0.1.13)
@ -21433,7 +21431,7 @@ snapshots:
'@langchain/groq': 0.1.2(@langchain/core@0.3.15(openai@4.69.0(encoding@0.1.13)(zod@3.23.8)))(encoding@0.1.13)
'@langchain/mistralai': 0.1.1(@langchain/core@0.3.15(openai@4.69.0(encoding@0.1.13)(zod@3.23.8)))(encoding@0.1.13)
'@langchain/ollama': 0.1.1(@langchain/core@0.3.15(openai@4.69.0(encoding@0.1.13)(zod@3.23.8)))
axios: 1.7.4
axios: 1.7.4(debug@4.3.7)
cheerio: 1.0.0
handlebars: 4.7.8
transitivePeerDependencies:

1
pnpm-workspace.yaml
View file

@ -18,6 +18,7 @@ catalog:
fast-glob: 3.2.12
flatted: 3.2.7
form-data: 4.0.0
iconv-lite: 0.6.3
lodash: 4.17.21
luxon: 3.4.4
nanoid: 3.3.6