fix(HTTP Request Node): Sanitize secrets of predefined credentials (#9612)

This commit is contained in:
Elias Meire 2024-06-05 09:25:39 +02:00 committed by GitHub
parent 5361e9f69a
commit 84f091d3e5
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 222 additions and 71 deletions

View file

@ -2844,7 +2844,8 @@ const getCommonWorkflowFunctions = (
getInstanceBaseUrl: () => additionalData.instanceBaseUrl, getInstanceBaseUrl: () => additionalData.instanceBaseUrl,
getInstanceId: () => Container.get(InstanceSettings).instanceId, getInstanceId: () => Container.get(InstanceSettings).instanceId,
getTimezone: () => getTimezone(workflow), getTimezone: () => getTimezone(workflow),
getCredentialsProperties: (type: string) =>
additionalData.credentialsHelper.getCredentialsProperties(type),
prepareOutputData: async (outputData) => [outputData], prepareOutputData: async (outputData) => [outputData],
}); });

View file

@ -1,16 +1,20 @@
import type { SecureContextOptions } from 'tls'; import type { SecureContextOptions } from 'tls';
import type { import type {
ICredentialDataDecryptedObject,
IDataObject, IDataObject,
INodeExecutionData, INodeExecutionData,
INodeProperties,
IOAuth2Options, IOAuth2Options,
IRequestOptions, IRequestOptions,
} from 'n8n-workflow'; } from 'n8n-workflow';
import set from 'lodash/set'; import set from 'lodash/set';
import isPlainObject from 'lodash/isPlainObject';
import FormData from 'form-data'; import FormData from 'form-data';
import type { HttpSslAuthCredentials } from './interfaces';
import { formatPrivateKey } from '../../utils/utilities'; import { formatPrivateKey } from '../../utils/utilities';
import type { HttpSslAuthCredentials } from './interfaces';
import get from 'lodash/get';
export type BodyParameter = { export type BodyParameter = {
name: string; name: string;
@ -29,7 +33,33 @@ export const replaceNullValues = (item: INodeExecutionData) => {
return item; return item;
}; };
export function sanitizeUiMessage(request: IRequestOptions, authDataKeys: IAuthDataSanitizeKeys) { export const REDACTED = '**hidden**';
function isObject(obj: unknown): obj is IDataObject {
return isPlainObject(obj);
}
function redact<T = unknown>(obj: T, secrets: string[]): T {
if (typeof obj === 'string') {
return secrets.reduce((safe, secret) => safe.replace(secret, REDACTED), obj) as T;
}
if (Array.isArray(obj)) {
return obj.map((item) => redact(item, secrets)) as T;
} else if (isObject(obj)) {
for (const [key, value] of Object.entries(obj)) {
(obj as IDataObject)[key] = redact(value, secrets);
}
}
return obj;
}
export function sanitizeUiMessage(
request: IRequestOptions,
authDataKeys: IAuthDataSanitizeKeys,
secrets?: string[],
) {
let sendRequest = request as unknown as IDataObject; let sendRequest = request as unknown as IDataObject;
// Protect browser from sending large binary data // Protect browser from sending large binary data
@ -38,7 +68,7 @@ export function sanitizeUiMessage(request: IRequestOptions, authDataKeys: IAuthD
...request, ...request,
body: `Binary data got replaced with this text. Original was a Buffer with a size of ${ body: `Binary data got replaced with this text. Original was a Buffer with a size of ${
(request.body as string).length (request.body as string).length
} byte.`, } bytes.`,
}; };
} }
@ -50,7 +80,7 @@ export function sanitizeUiMessage(request: IRequestOptions, authDataKeys: IAuthD
// eslint-disable-next-line @typescript-eslint/no-loop-func // eslint-disable-next-line @typescript-eslint/no-loop-func
(acc: IDataObject, curr) => { (acc: IDataObject, curr) => {
acc[curr] = authDataKeys[requestProperty].includes(curr) acc[curr] = authDataKeys[requestProperty].includes(curr)
? '** hidden **' ? REDACTED
: (sendRequest[requestProperty] as IDataObject)[curr]; : (sendRequest[requestProperty] as IDataObject)[curr];
return acc; return acc;
}, },
@ -59,9 +89,33 @@ export function sanitizeUiMessage(request: IRequestOptions, authDataKeys: IAuthD
}; };
} }
if (secrets && secrets.length > 0) {
return redact(sendRequest, secrets);
}
return sendRequest; return sendRequest;
} }
export function getSecrets(
properties: INodeProperties[],
credentials: ICredentialDataDecryptedObject,
): string[] {
const sensitivePropNames = new Set(
properties.filter((prop) => prop.typeOptions?.password).map((prop) => prop.name),
);
const secrets = Object.entries(credentials)
.filter(([propName]) => sensitivePropNames.has(propName))
.map(([_, value]) => value)
.filter((value): value is string => typeof value === 'string');
const oauthAccessToken = get(credentials, 'oauthTokenData.access_token');
if (typeof oauthAccessToken === 'string') {
secrets.push(oauthAccessToken);
}
return secrets;
}
export const getOAuth2AdditionalParameters = (nodeCredentialType: string) => { export const getOAuth2AdditionalParameters = (nodeCredentialType: string) => {
const oAuth2Options: { [credentialType: string]: IOAuth2Options } = { const oAuth2Options: { [credentialType: string]: IOAuth2Options } = {
bitlyOAuth2Api: { bitlyOAuth2Api: {

View file

@ -29,14 +29,15 @@ import type { BodyParameter, IAuthDataSanitizeKeys } from '../GenericFunctions';
import { import {
binaryContentTypes, binaryContentTypes,
getOAuth2AdditionalParameters, getOAuth2AdditionalParameters,
getSecrets,
prepareRequestBody, prepareRequestBody,
reduceAsync, reduceAsync,
replaceNullValues, replaceNullValues,
sanitizeUiMessage, sanitizeUiMessage,
setAgentOptions, setAgentOptions,
} from '../GenericFunctions'; } from '../GenericFunctions';
import { keysToLowercase } from '@utils/utilities';
import type { HttpSslAuthCredentials } from '../interfaces'; import type { HttpSslAuthCredentials } from '../interfaces';
import { keysToLowercase } from '@utils/utilities';
function toText<T>(data: T) { function toText<T>(data: T) {
if (typeof data === 'object' && data !== null) { if (typeof data === 'object' && data !== null) {
@ -1299,7 +1300,12 @@ export class HttpRequestV3 implements INodeType {
requestInterval: number; requestInterval: number;
}; };
const sanitazedRequests: IDataObject[] = []; const requests: Array<{
options: IRequestOptions;
authKeys: IAuthDataSanitizeKeys;
credentialType?: string;
}> = [];
for (let itemIndex = 0; itemIndex < items.length; itemIndex++) { for (let itemIndex = 0; itemIndex < items.length; itemIndex++) {
if (authentication === 'genericCredentialType') { if (authentication === 'genericCredentialType') {
genericCredentialType = this.getNodeParameter('genericAuthType', 0) as string; genericCredentialType = this.getNodeParameter('genericAuthType', 0) as string;
@ -1696,11 +1702,11 @@ export class HttpRequestV3 implements INodeType {
} }
} }
try { requests.push({
const sanitazedRequestOptions = sanitizeUiMessage(requestOptions, authDataKeys); options: requestOptions,
this.sendMessageToUI(sanitazedRequestOptions); authKeys: authDataKeys,
sanitazedRequests.push(sanitazedRequestOptions); credentialType: nodeCredentialType,
} catch (e) {} });
if (pagination && pagination.paginationMode !== 'off') { if (pagination && pagination.paginationMode !== 'off') {
let continueExpression = '={{false}}'; let continueExpression = '={{false}}';
@ -1827,7 +1833,29 @@ export class HttpRequestV3 implements INodeType {
requestPromises.push(requestWithAuthentication); requestPromises.push(requestWithAuthentication);
} }
} }
const promisesResponses = await Promise.allSettled(requestPromises);
const sanitizedRequests: IDataObject[] = [];
const promisesResponses = await Promise.allSettled(
requestPromises.map(
async (requestPromise, itemIndex) =>
await requestPromise.finally(async () => {
try {
// Secrets need to be read after the request because secrets could have changed
// For example: OAuth token refresh, preAuthentication
const { options, authKeys, credentialType } = requests[itemIndex];
let secrets: string[] = [];
if (credentialType) {
const properties = this.getCredentialsProperties(credentialType);
const credentials = await this.getCredentials(credentialType, itemIndex);
secrets = getSecrets(properties, credentials);
}
const sanitizedRequestOptions = sanitizeUiMessage(options, authKeys, secrets);
sanitizedRequests.push(sanitizedRequestOptions);
this.sendMessageToUI(sanitizedRequestOptions);
} catch (e) {}
}),
),
);
let responseData: any; let responseData: any;
for (let itemIndex = 0; itemIndex < items.length; itemIndex++) { for (let itemIndex = 0; itemIndex < items.length; itemIndex++) {
@ -1842,7 +1870,7 @@ export class HttpRequestV3 implements INodeType {
responseData.reason.error = Buffer.from(responseData.reason.error as Buffer).toString(); responseData.reason.error = Buffer.from(responseData.reason.error as Buffer).toString();
} }
const error = new NodeApiError(this.getNode(), responseData as JsonObject, { itemIndex }); const error = new NodeApiError(this.getNode(), responseData as JsonObject, { itemIndex });
set(error, 'context.request', sanitazedRequests[itemIndex]); set(error, 'context.request', sanitizedRequests[itemIndex]);
throw error; throw error;
} else { } else {
removeCircularRefs(responseData.reason as JsonObject); removeCircularRefs(responseData.reason as JsonObject);

View file

@ -1,8 +1,14 @@
import type { IRequestOptions } from 'n8n-workflow'; import type { IRequestOptions } from 'n8n-workflow';
import { prepareRequestBody, setAgentOptions } from '../../GenericFunctions'; import {
REDACTED,
prepareRequestBody,
sanitizeUiMessage,
setAgentOptions,
} from '../../GenericFunctions';
import type { BodyParameter, BodyParametersReducer } from '../../GenericFunctions'; import type { BodyParameter, BodyParametersReducer } from '../../GenericFunctions';
describe('HTTP Node Utils, prepareRequestBody', () => { describe('HTTP Node Utils', () => {
describe('prepareRequestBody', () => {
it('should call default reducer', async () => { it('should call default reducer', async () => {
const bodyParameters: BodyParameter[] = [ const bodyParameters: BodyParameter[] = [
{ {
@ -35,7 +41,7 @@ describe('HTTP Node Utils, prepareRequestBody', () => {
}); });
}); });
describe('HTTP Node Utils, setAgentOptions', () => { describe('setAgentOptions', () => {
it("should not have agentOptions as it's undefined", async () => { it("should not have agentOptions as it's undefined", async () => {
const requestOptions: IRequestOptions = { const requestOptions: IRequestOptions = {
method: 'GET', method: 'GET',
@ -73,3 +79,62 @@ describe('HTTP Node Utils, setAgentOptions', () => {
}); });
}); });
}); });
describe('sanitizeUiMessage', () => {
it('should remove large Buffers', async () => {
const requestOptions: IRequestOptions = {
method: 'POST',
uri: 'https://example.com',
body: Buffer.alloc(900000),
};
expect(sanitizeUiMessage(requestOptions, {}).body).toEqual(
'Binary data got replaced with this text. Original was a Buffer with a size of 900000 bytes.',
);
});
it('should remove keys that contain sensitive data', async () => {
const requestOptions: IRequestOptions = {
method: 'POST',
uri: 'https://example.com',
body: { sessionToken: 'secret', other: 'foo' },
headers: { authorization: 'secret', other: 'foo' },
auth: { user: 'user', password: 'secret' },
};
expect(
sanitizeUiMessage(requestOptions, {
headers: ['authorization'],
body: ['sessionToken'],
auth: ['password'],
}),
).toEqual({
body: { sessionToken: REDACTED, other: 'foo' },
headers: { other: 'foo', authorization: REDACTED },
auth: { user: 'user', password: REDACTED },
method: 'POST',
uri: 'https://example.com',
});
});
it('should remove secrets', async () => {
const requestOptions: IRequestOptions = {
method: 'POST',
uri: 'https://example.com',
body: { nested: { secret: 'secretAccessToken' } },
headers: { authorization: 'secretAccessToken', other: 'foo' },
};
expect(sanitizeUiMessage(requestOptions, {}, ['secretAccessToken'])).toEqual({
body: {
nested: {
secret: REDACTED,
},
},
headers: { authorization: REDACTED, other: 'foo' },
method: 'POST',
uri: 'https://example.com',
});
});
});
});

View file

@ -216,6 +216,8 @@ export abstract class ICredentialsHelper {
type: string, type: string,
data: ICredentialDataDecryptedObject, data: ICredentialDataDecryptedObject,
): Promise<void>; ): Promise<void>;
abstract getCredentialsProperties(type: string): INodeProperties[];
} }
export interface IAuthenticateBase { export interface IAuthenticateBase {
@ -813,6 +815,7 @@ export type NodeTypeAndVersion = {
export interface FunctionsBase { export interface FunctionsBase {
logger: Logger; logger: Logger;
getCredentials(type: string, itemIndex?: number): Promise<ICredentialDataDecryptedObject>; getCredentials(type: string, itemIndex?: number): Promise<ICredentialDataDecryptedObject>;
getCredentialsProperties(type: string): INodeProperties[];
getExecutionId(): string; getExecutionId(): string;
getNode(): INode; getNode(): INode;
getWorkflow(): IWorkflowMetadata; getWorkflow(): IWorkflowMetadata;