import { Readability } from '@mozilla/readability'; import cheerio from 'cheerio'; import { convert } from 'html-to-text'; import { JSDOM } from 'jsdom'; import get from 'lodash/get'; import set from 'lodash/set'; import unset from 'lodash/unset'; import * as mime from 'mime-types'; import { getOAuth2AdditionalParameters } from 'n8n-nodes-base/dist/nodes/HttpRequest/GenericFunctions'; import type { IDataObject, IHttpRequestOptions, IRequestOptionsSimplified, ExecutionError, NodeApiError, ISupplyDataFunctions, } from 'n8n-workflow'; import { NodeConnectionType, NodeOperationError, jsonParse } from 'n8n-workflow'; import { z } from 'zod'; import type { ParameterInputType, ParametersValues, PlaceholderDefinition, ParametersValues as RawParametersValues, SendIn, ToolParameter, } from './interfaces'; import type { DynamicZodObject } from '../../../types/zod.types'; const genericCredentialRequest = async (ctx: ISupplyDataFunctions, itemIndex: number) => { const genericType = ctx.getNodeParameter('genericAuthType', itemIndex) as string; if (genericType === 'httpBasicAuth' || genericType === 'httpDigestAuth') { const basicAuth = await ctx.getCredentials('httpBasicAuth', itemIndex); const sendImmediately = genericType === 'httpDigestAuth' ? false : undefined; return async (options: IHttpRequestOptions) => { options.auth = { username: basicAuth.user as string, password: basicAuth.password as string, sendImmediately, }; return await ctx.helpers.httpRequest(options); }; } if (genericType === 'httpHeaderAuth') { const headerAuth = await ctx.getCredentials('httpHeaderAuth', itemIndex); return async (options: IHttpRequestOptions) => { if (!options.headers) options.headers = {}; options.headers[headerAuth.name as string] = headerAuth.value; return await ctx.helpers.httpRequest(options); }; } if (genericType === 'httpQueryAuth') { const queryAuth = await ctx.getCredentials('httpQueryAuth', itemIndex); return async (options: IHttpRequestOptions) => { if (!options.qs) options.qs = {}; options.qs[queryAuth.name as string] = queryAuth.value; return await ctx.helpers.httpRequest(options); }; } if (genericType === 'httpCustomAuth') { const customAuth = await ctx.getCredentials('httpCustomAuth', itemIndex); return async (options: IHttpRequestOptions) => { const auth = jsonParse((customAuth.json as string) || '{}', { errorMessage: 'Invalid Custom Auth JSON', }); if (auth.headers) { options.headers = { ...options.headers, ...auth.headers }; } if (auth.body) { options.body = { ...(options.body as IDataObject), ...auth.body }; } if (auth.qs) { options.qs = { ...options.qs, ...auth.qs }; } return await ctx.helpers.httpRequest(options); }; } if (genericType === 'oAuth1Api') { return async (options: IHttpRequestOptions) => { return await ctx.helpers.requestOAuth1.call(ctx, 'oAuth1Api', options); }; } if (genericType === 'oAuth2Api') { return async (options: IHttpRequestOptions) => { return await ctx.helpers.requestOAuth2.call(ctx, 'oAuth1Api', options, { tokenType: 'Bearer', }); }; } throw new NodeOperationError(ctx.getNode(), `The type ${genericType} is not supported`, { itemIndex, }); }; const predefinedCredentialRequest = async (ctx: ISupplyDataFunctions, itemIndex: number) => { const predefinedType = ctx.getNodeParameter('nodeCredentialType', itemIndex) as string; const additionalOptions = getOAuth2AdditionalParameters(predefinedType); return async (options: IHttpRequestOptions) => { return await ctx.helpers.httpRequestWithAuthentication.call( ctx, predefinedType, options, additionalOptions && { oauth2: additionalOptions }, ); }; }; export const configureHttpRequestFunction = async ( ctx: ISupplyDataFunctions, credentialsType: 'predefinedCredentialType' | 'genericCredentialType' | 'none', itemIndex: number, ) => { switch (credentialsType) { case 'genericCredentialType': return await genericCredentialRequest(ctx, itemIndex); case 'predefinedCredentialType': return await predefinedCredentialRequest(ctx, itemIndex); default: return async (options: IHttpRequestOptions) => { return await ctx.helpers.httpRequest(options); }; } }; const defaultOptimizer = (response: T) => { if (typeof response === 'string') { return response; } if (typeof response === 'object') { return JSON.stringify(response, null, 2); } return String(response); }; const htmlOptimizer = (ctx: ISupplyDataFunctions, itemIndex: number, maxLength: number) => { const cssSelector = ctx.getNodeParameter('cssSelector', itemIndex, '') as string; const onlyContent = ctx.getNodeParameter('onlyContent', itemIndex, false) as boolean; let elementsToOmit: string[] = []; if (onlyContent) { const elementsToOmitUi = ctx.getNodeParameter('elementsToOmit', itemIndex, '') as | string | string[]; if (typeof elementsToOmitUi === 'string') { elementsToOmit = elementsToOmitUi .split(',') .filter((s) => s) .map((s) => s.trim()); } } return (response: T) => { if (typeof response !== 'string') { throw new NodeOperationError( ctx.getNode(), `The response type must be a string. Received: ${typeof response}`, { itemIndex }, ); } const returnData: string[] = []; const html = cheerio.load(response); const htmlElements = html(cssSelector); htmlElements.each((_, el) => { let value = html(el).html() || ''; if (onlyContent) { let htmlToTextOptions; if (elementsToOmit?.length) { htmlToTextOptions = { selectors: elementsToOmit.map((selector) => ({ selector, format: 'skip', })), }; } value = convert(value, htmlToTextOptions); } value = value .trim() .replace(/^\s+|\s+$/g, '') .replace(/(\r\n|\n|\r)/gm, '') .replace(/\s+/g, ' '); returnData.push(value); }); const text = JSON.stringify(returnData, null, 2); if (maxLength > 0 && text.length > maxLength) { return text.substring(0, maxLength); } return text; }; }; const textOptimizer = (ctx: ISupplyDataFunctions, itemIndex: number, maxLength: number) => { return (response: string | IDataObject) => { if (typeof response === 'object') { try { response = JSON.stringify(response, null, 2); } catch (error) {} } if (typeof response !== 'string') { throw new NodeOperationError( ctx.getNode(), `The response type must be a string. Received: ${typeof response}`, { itemIndex }, ); } const dom = new JSDOM(response); const article = new Readability(dom.window.document, { keepClasses: true, }).parse(); const text = article?.textContent || ''; if (maxLength > 0 && text.length > maxLength) { return text.substring(0, maxLength); } return text; }; }; const jsonOptimizer = (ctx: ISupplyDataFunctions, itemIndex: number) => { return (response: string): string => { let responseData: IDataObject | IDataObject[] | string = response; if (typeof responseData === 'string') { responseData = jsonParse(response); } if (typeof responseData !== 'object' || !responseData) { throw new NodeOperationError( ctx.getNode(), 'The response type must be an object or an array of objects', { itemIndex }, ); } const dataField = ctx.getNodeParameter('dataField', itemIndex, '') as string; let returnData: IDataObject[] = []; if (!Array.isArray(responseData)) { if (dataField) { const data = responseData[dataField] as IDataObject | IDataObject[]; if (Array.isArray(data)) { responseData = data; } else { responseData = [data]; } } else { responseData = [responseData]; } } else { if (dataField) { responseData = responseData.map((data) => data[dataField]) as IDataObject[]; } } const fieldsToInclude = ctx.getNodeParameter('fieldsToInclude', itemIndex, 'all') as | 'all' | 'selected' | 'except'; let fields: string | string[] = []; if (fieldsToInclude !== 'all') { fields = ctx.getNodeParameter('fields', itemIndex, []) as string[] | string; if (typeof fields === 'string') { fields = fields.split(',').map((field) => field.trim()); } } else { returnData = responseData; } if (fieldsToInclude === 'selected') { for (const item of responseData) { const newItem: IDataObject = {}; for (const field of fields) { set(newItem, field, get(item, field)); } returnData.push(newItem); } } if (fieldsToInclude === 'except') { for (const item of responseData) { for (const field of fields) { unset(item, field); } returnData.push(item); } } return JSON.stringify(returnData, null, 2); }; }; export const configureResponseOptimizer = (ctx: ISupplyDataFunctions, itemIndex: number) => { const optimizeResponse = ctx.getNodeParameter('optimizeResponse', itemIndex, false) as boolean; if (optimizeResponse) { const responseType = ctx.getNodeParameter('responseType', itemIndex) as | 'json' | 'text' | 'html'; let maxLength = 0; const truncateResponse = ctx.getNodeParameter('truncateResponse', itemIndex, false) as boolean; if (truncateResponse) { maxLength = ctx.getNodeParameter('maxLength', itemIndex, 0) as number; } switch (responseType) { case 'html': return htmlOptimizer(ctx, itemIndex, maxLength); case 'text': return textOptimizer(ctx, itemIndex, maxLength); case 'json': return jsonOptimizer(ctx, itemIndex); } } return defaultOptimizer; }; const extractPlaceholders = (text: string): string[] => { const placeholder = /(\{[a-zA-Z0-9_-]+\})/g; const returnData: string[] = []; const matches = text.matchAll(placeholder); for (const match of matches) { returnData.push(match[0].replace(/{|}/g, '')); } return returnData; }; export const extractParametersFromText = ( placeholders: PlaceholderDefinition[], text: string, sendIn: SendIn, key?: string, ): ToolParameter[] => { if (typeof text !== 'string') return []; const parameters = extractPlaceholders(text); if (parameters.length) { // eslint-disable-next-line @typescript-eslint/no-use-before-define const inputParameters = prepareParameters( parameters.map((name) => ({ name, valueProvider: 'modelRequired', })), placeholders, 'keypair', sendIn, '', ); return key ? inputParameters.parameters.map((p) => ({ ...p, key })) : inputParameters.parameters; } return []; }; function prepareParameters( rawParameters: RawParametersValues, placeholders: PlaceholderDefinition[], parametersInputType: 'model' | 'keypair' | 'json', sendIn: SendIn, modelInputDescription: string, jsonWithPlaceholders?: string, ): { parameters: ToolParameter[]; values: IDataObject } { const parameters: ToolParameter[] = []; const values: IDataObject = {}; if (parametersInputType === 'model') { return { parameters: [ { name: sendIn, required: true, type: 'json', description: modelInputDescription, sendIn, }, ], values: {}, }; } if (parametersInputType === 'keypair') { for (const entry of rawParameters) { if (entry.valueProvider.includes('model')) { const placeholder = placeholders.find((p) => p.name === entry.name); const parameter: ToolParameter = { name: entry.name, required: entry.valueProvider === 'modelRequired', sendIn, }; if (placeholder) { parameter.type = placeholder.type; parameter.description = placeholder.description; } parameters.push(parameter); } else if (entry.value) { // if value has placeholders push them to parameters parameters.push( ...extractParametersFromText(placeholders, entry.value, sendIn, entry.name), ); values[entry.name] = entry.value; //push to user provided values } } } if (parametersInputType === 'json' && jsonWithPlaceholders) { parameters.push( ...extractParametersFromText(placeholders, jsonWithPlaceholders, sendIn, `${sendIn + 'Raw'}`), ); } return { parameters, values, }; } const MODEL_INPUT_DESCRIPTION = { qs: 'Query parameters for request as key value pairs', headers: 'Headers parameters for request as key value pairs', body: 'Body parameters for request as key value pairs', }; export const updateParametersAndOptions = (options: { ctx: ISupplyDataFunctions; itemIndex: number; toolParameters: ToolParameter[]; placeholdersDefinitions: PlaceholderDefinition[]; requestOptions: IHttpRequestOptions; rawRequestOptions: { [key: string]: string }; requestOptionsProperty: 'headers' | 'qs' | 'body'; inputTypePropertyName: string; jsonPropertyName: string; parametersPropertyName: string; }) => { const { ctx, itemIndex, toolParameters, placeholdersDefinitions, requestOptions, rawRequestOptions, requestOptionsProperty, inputTypePropertyName, jsonPropertyName, parametersPropertyName, } = options; const inputType = ctx.getNodeParameter( inputTypePropertyName, itemIndex, 'keypair', ) as ParameterInputType; let parametersValues: ParametersValues = []; if (inputType === 'json') { rawRequestOptions[requestOptionsProperty] = ctx.getNodeParameter( jsonPropertyName, itemIndex, '', ) as string; } else { parametersValues = ctx.getNodeParameter( parametersPropertyName, itemIndex, [], ) as ParametersValues; } const inputParameters = prepareParameters( parametersValues, placeholdersDefinitions, inputType, requestOptionsProperty, MODEL_INPUT_DESCRIPTION[requestOptionsProperty], rawRequestOptions[requestOptionsProperty], ); toolParameters.push(...inputParameters.parameters); requestOptions[requestOptionsProperty] = { ...(requestOptions[requestOptionsProperty] as IDataObject), ...inputParameters.values, }; }; const getParametersDescription = (parameters: ToolParameter[]) => parameters .map( (p) => `${p.name}: (description: ${p.description ?? ''}, type: ${p.type ?? 'string'}, required: ${!!p.required})`, ) .join(',\n '); export const prepareToolDescription = ( toolDescription: string, toolParameters: ToolParameter[], ) => { let description = `${toolDescription}`; if (toolParameters.length) { description += ` Tool expects valid stringified JSON object with ${toolParameters.length} properties. Property names with description, type and required status: ${getParametersDescription(toolParameters)} ALL parameters marked as required must be provided`; } return description; }; export const configureToolFunction = ( ctx: ISupplyDataFunctions, itemIndex: number, toolParameters: ToolParameter[], requestOptions: IHttpRequestOptions, rawRequestOptions: { [key: string]: string }, httpRequest: (options: IHttpRequestOptions) => Promise, optimizeResponse: (response: string) => string, ) => { return async (query: string | IDataObject): Promise => { const { index } = ctx.addInputData(NodeConnectionType.AiTool, [[{ json: { query } }]]); // Clone options and rawRequestOptions to avoid mutating the original objects const options: IHttpRequestOptions | null = structuredClone(requestOptions); const clonedRawRequestOptions: { [key: string]: string } = structuredClone(rawRequestOptions); let fullResponse: any; let response: string = ''; let executionError: Error | undefined = undefined; if (!toolParameters.length) { query = '{}'; } try { if (query) { let dataFromModel; if (typeof query === 'string') { try { dataFromModel = jsonParse(query); } catch (error) { if (toolParameters.length === 1) { dataFromModel = { [toolParameters[0].name]: query }; } else { throw new NodeOperationError( ctx.getNode(), `Input is not a valid JSON: ${error.message}`, { itemIndex }, ); } } } else { dataFromModel = query; } for (const parameter of toolParameters) { if ( parameter.required && (dataFromModel[parameter.name] === undefined || dataFromModel[parameter.name] === null) ) { throw new NodeOperationError( ctx.getNode(), `Model did not provide parameter '${parameter.name}' which is required and must be present in the input`, { itemIndex }, ); } } for (const parameter of toolParameters) { let argument = dataFromModel[parameter.name]; if ( argument && parameter.type === 'json' && !['qsRaw', 'headersRaw', 'bodyRaw'].includes(parameter.key ?? '') && typeof argument !== 'object' ) { try { argument = jsonParse(String(argument)); } catch (error) { throw new NodeOperationError( ctx.getNode(), `Parameter ${parameter.name} is not a valid JSON: ${error.message}`, { itemIndex, }, ); } } if (parameter.sendIn === 'path') { argument = String(argument); //remove " or ' from start or end argument = argument.replace(/^['"]+|['"]+$/g, ''); options.url = options.url.replace(`{${parameter.name}}`, argument); continue; } if (parameter.sendIn === parameter.name) { set(options, [parameter.sendIn], argument); continue; } if (['qsRaw', 'headersRaw', 'bodyRaw'].includes(parameter.key ?? '')) { //enclose string in quotes as user and model could omit them if (parameter.type === 'string') { argument = String(argument); if ( !argument.startsWith('"') && !clonedRawRequestOptions[parameter.sendIn].includes(`"{${parameter.name}}"`) ) { argument = `"${argument}"`; } } if (typeof argument === 'object') { argument = JSON.stringify(argument); } clonedRawRequestOptions[parameter.sendIn] = clonedRawRequestOptions[ parameter.sendIn ].replace(`{${parameter.name}}`, String(argument)); continue; } if (parameter.key) { let requestOptionsValue = get(options, [parameter.sendIn, parameter.key]); if (typeof requestOptionsValue === 'string') { requestOptionsValue = requestOptionsValue.replace( `{${parameter.name}}`, String(argument), ); } set(options, [parameter.sendIn, parameter.key], requestOptionsValue); continue; } set(options, [parameter.sendIn, parameter.name], argument); } for (const [key, value] of Object.entries(clonedRawRequestOptions)) { if (value) { let parsedValue; try { parsedValue = jsonParse(value); } catch (error) { let recoveredData = ''; try { recoveredData = value .replace(/'/g, '"') // Replace single quotes with double quotes .replace(/(['"])?([a-zA-Z0-9_]+)(['"])?:/g, '"$2":') // Wrap keys in double quotes .replace(/,\s*([\]}])/g, '$1') // Remove trailing commas from objects .replace(/,+$/, ''); // Remove trailing comma parsedValue = jsonParse(recoveredData); } catch (err) { throw new NodeOperationError( ctx.getNode(), `Could not replace placeholders in ${key}: ${error.message}`, ); } } options[key as 'qs' | 'headers' | 'body'] = parsedValue; } } } if (options) { options.url = encodeURI(options.url); if (options.headers && !Object.keys(options.headers).length) { delete options.headers; } if (options.qs && !Object.keys(options.qs).length) { delete options.qs; } if (options.body && !Object.keys(options.body).length) { delete options.body; } } } catch (error) { const errorMessage = 'Input provided by model is not valid'; if (error instanceof NodeOperationError) { executionError = error; } else { executionError = new NodeOperationError(ctx.getNode(), errorMessage, { itemIndex, }); } response = errorMessage; } if (options) { try { fullResponse = await httpRequest(options); } catch (error) { const httpCode = (error as NodeApiError).httpCode; response = `${httpCode ? `HTTP ${httpCode} ` : ''}There was an error: "${error.message}"`; } if (!response) { try { // Check if the response is binary data if (fullResponse?.headers?.['content-type']) { const contentType = fullResponse.headers['content-type'] as string; const mimeType = contentType.split(';')[0].trim(); if (mime.charset(mimeType) !== 'UTF-8') { throw new NodeOperationError(ctx.getNode(), 'Binary data is not supported'); } } response = optimizeResponse(fullResponse.body); } catch (error) { response = `There was an error: "${error.message}"`; } } } if (typeof response !== 'string') { executionError = new NodeOperationError(ctx.getNode(), 'Wrong output type returned', { description: `The response property should be a string, but it is an ${typeof response}`, }); response = `There was an error: "${executionError.message}"`; } if (executionError) { void ctx.addOutputData(NodeConnectionType.AiTool, index, executionError as ExecutionError); } else { void ctx.addOutputData(NodeConnectionType.AiTool, index, [[{ json: { response } }]]); } return response; }; }; function makeParameterZodSchema(parameter: ToolParameter) { let schema: z.ZodTypeAny; if (parameter.type === 'string') { schema = z.string(); } else if (parameter.type === 'number') { schema = z.number(); } else if (parameter.type === 'boolean') { schema = z.boolean(); } else if (parameter.type === 'json') { schema = z.record(z.any()); } else { schema = z.string(); } if (!parameter.required) { schema = schema.optional(); } if (parameter.description) { schema = schema.describe(parameter.description); } return schema; } export function makeToolInputSchema(parameters: ToolParameter[]): DynamicZodObject { const schemaEntries = parameters.map((parameter) => [ parameter.name, makeParameterZodSchema(parameter), ]); return z.object(Object.fromEntries(schemaEntries)); }