mudhorn/n8n
1
0
Fork 0
mirror of https://github.com/n8n-io/n8n.git synced 2025-01-11 21:07:28 -08:00
Code Issues Actions Packages Projects Releases Wiki Activity

refactor: Replace json-schema-to-zod with our own fork (#11229)

Browse source
This commit is contained in:
Tomi Turtiainen 2024-10-18 08:29:19 +02:00 committed by GitHub
parent c57cac9e4d
commit a042d5c8e6
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 55 additions and 121 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
packages/@n8n/nodes-langchain/nodes/chains/InformationExtractor/InformationExtractor.node.ts
View file

@ -1,3 +1,8 @@
import type { BaseLanguageModel } from '@langchain/core/language_models/base';
import { HumanMessage } from '@langchain/core/messages';
import { ChatPromptTemplate, SystemMessagePromptTemplate } from '@langchain/core/prompts';
import type { JSONSchema7 } from 'json-schema';
import { OutputFixingParser, StructuredOutputParser } from 'langchain/output_parsers';
import { jsonParse, NodeConnectionType, NodeOperationError } from 'n8n-workflow'; import { jsonParse, NodeConnectionType, NodeOperationError } from 'n8n-workflow';
import type { import type {
INodeType, INodeType,
@ -6,21 +11,17 @@ import type {
INodeExecutionData, INodeExecutionData,
INodePropertyOptions, INodePropertyOptions,
} from 'n8n-workflow'; } from 'n8n-workflow';
import type { JSONSchema7 } from 'json-schema';
import type { BaseLanguageModel } from '@langchain/core/language_models/base';
import { ChatPromptTemplate, SystemMessagePromptTemplate } from '@langchain/core/prompts';
import type { z } from 'zod'; import type { z } from 'zod';
import { OutputFixingParser, StructuredOutputParser } from 'langchain/output_parsers';
import { HumanMessage } from '@langchain/core/messages'; import { makeZodSchemaFromAttributes } from './helpers';
import { generateSchema, getSandboxWithZod } from '../../../utils/schemaParsing'; import type { AttributeDefinition } from './types';
import { import {
inputSchemaField, inputSchemaField,
jsonSchemaExampleField, jsonSchemaExampleField,
schemaTypeField, schemaTypeField,
} from '../../../utils/descriptions'; } from '../../../utils/descriptions';
import { convertJsonSchemaToZod, generateSchema } from '../../../utils/schemaParsing';
import { getTracingConfig } from '../../../utils/tracing'; import { getTracingConfig } from '../../../utils/tracing';
import type { AttributeDefinition } from './types';
import { makeZodSchemaFromAttributes } from './helpers';
const SYSTEM_PROMPT_TEMPLATE = `You are an expert extraction algorithm. const SYSTEM_PROMPT_TEMPLATE = `You are an expert extraction algorithm.
Only extract relevant information from the text. Only extract relevant information from the text.
@ -261,8 +262,7 @@ export class InformationExtractor implements INodeType {
jsonSchema = jsonParse<JSONSchema7>(inputSchema); jsonSchema = jsonParse<JSONSchema7>(inputSchema);
} }
const zodSchemaSandbox = getSandboxWithZod(this, jsonSchema, 0); const zodSchema = convertJsonSchemaToZod<z.ZodSchema<object>>(jsonSchema);
const zodSchema = await zodSchemaSandbox.runCode<z.ZodSchema<object>>();
parser = OutputFixingParser.fromLLM(llm, StructuredOutputParser.fromZodSchema(zodSchema)); parser = OutputFixingParser.fromLLM(llm, StructuredOutputParser.fromZodSchema(zodSchema));
} }

29
packages/@n8n/nodes-langchain/nodes/output_parser/OutputParserStructured/OutputParserStructured.node.ts
View file

@ -1,4 +1,8 @@
/* eslint-disable n8n-nodes-base/node-dirname-against-convention */ /* eslint-disable n8n-nodes-base/node-dirname-against-convention */
import { OutputParserException } from '@langchain/core/output_parsers';
import type { JSONSchema7 } from 'json-schema';
import { StructuredOutputParser } from 'langchain/output_parsers';
import get from 'lodash/get';
import { import {
jsonParse, jsonParse,
type IExecuteFunctions, type IExecuteFunctions,
@ -9,19 +13,15 @@ import {
NodeConnectionType, NodeConnectionType,
} from 'n8n-workflow'; } from 'n8n-workflow';
import { z } from 'zod'; import { z } from 'zod';
import type { JSONSchema7 } from 'json-schema';
import { StructuredOutputParser } from 'langchain/output_parsers';
import { OutputParserException } from '@langchain/core/output_parsers';
import get from 'lodash/get';
import type { JavaScriptSandbox } from 'n8n-nodes-base/dist/nodes/Code/JavaScriptSandbox';
import { getConnectionHintNoticeField } from '../../../utils/sharedFields';
import { logWrapper } from '../../../utils/logWrapper';
import { generateSchema, getSandboxWithZod } from '../../../utils/schemaParsing';
import { import {
inputSchemaField, inputSchemaField,
jsonSchemaExampleField, jsonSchemaExampleField,
schemaTypeField, schemaTypeField,
} from '../../../utils/descriptions'; } from '../../../utils/descriptions';
import { logWrapper } from '../../../utils/logWrapper';
import { convertJsonSchemaToZod, generateSchema } from '../../../utils/schemaParsing';
import { getConnectionHintNoticeField } from '../../../utils/sharedFields';
const STRUCTURED_OUTPUT_KEY = '__structured__output'; const STRUCTURED_OUTPUT_KEY = '__structured__output';
const STRUCTURED_OUTPUT_OBJECT_KEY = '__structured__output__object'; const STRUCTURED_OUTPUT_OBJECT_KEY = '__structured__output__object';
@ -44,12 +44,10 @@ export class N8nStructuredOutputParser<T extends z.ZodTypeAny> extends Structure
} }
} }
static async fromZedJsonSchema( static async fromZedSchema(
sandboxedSchema: JavaScriptSandbox, zodSchema: z.ZodSchema<object>,
nodeVersion: number, nodeVersion: number,
): Promise<StructuredOutputParser<z.ZodType<object, z.ZodTypeDef, object>>> { ): Promise<StructuredOutputParser<z.ZodType<object, z.ZodTypeDef, object>>> {
const zodSchema = await sandboxedSchema.runCode<z.ZodSchema<object>>();
let returnSchema: z.ZodSchema<object>; let returnSchema: z.ZodSchema<object>;
if (nodeVersion === 1) { if (nodeVersion === 1) {
returnSchema = z.object({ returnSchema = z.object({
@ -204,13 +202,10 @@ export class OutputParserStructured implements INodeType {
const jsonSchema = const jsonSchema =
schemaType === 'fromJson' ? generateSchema(jsonExample) : jsonParse<JSONSchema7>(inputSchema); schemaType === 'fromJson' ? generateSchema(jsonExample) : jsonParse<JSONSchema7>(inputSchema);
const zodSchemaSandbox = getSandboxWithZod(this, jsonSchema, 0); const zodSchema = convertJsonSchemaToZod<z.ZodSchema<object>>(jsonSchema);
const nodeVersion = this.getNode().typeVersion; const nodeVersion = this.getNode().typeVersion;
try { try {
const parser = await N8nStructuredOutputParser.fromZedJsonSchema( const parser = await N8nStructuredOutputParser.fromZedSchema(zodSchema, nodeVersion);
zodSchemaSandbox,
nodeVersion,
);
return { return {
response: logWrapper(parser, this), response: logWrapper(parser, this),
}; };

24
packages/@n8n/nodes-langchain/nodes/tools/ToolCode/ToolCode.node.ts
View file

@ -1,4 +1,10 @@
/* eslint-disable n8n-nodes-base/node-dirname-against-convention */ /* eslint-disable n8n-nodes-base/node-dirname-against-convention */
import { DynamicStructuredTool, DynamicTool } from '@langchain/core/tools';
import type { JSONSchema7 } from 'json-schema';
import { JavaScriptSandbox } from 'n8n-nodes-base/dist/nodes/Code/JavaScriptSandbox';
import { PythonSandbox } from 'n8n-nodes-base/dist/nodes/Code/PythonSandbox';
import type { Sandbox } from 'n8n-nodes-base/dist/nodes/Code/Sandbox';
import { getSandboxContext } from 'n8n-nodes-base/dist/nodes/Code/Sandbox';
import type { import type {
IExecuteFunctions, IExecuteFunctions,
INodeType, INodeType,
@ -7,23 +13,16 @@ import type {
ExecutionError, ExecutionError,
IDataObject, IDataObject,
} from 'n8n-workflow'; } from 'n8n-workflow';
import { jsonParse, NodeConnectionType, NodeOperationError } from 'n8n-workflow'; import { jsonParse, NodeConnectionType, NodeOperationError } from 'n8n-workflow';
import type { Sandbox } from 'n8n-nodes-base/dist/nodes/Code/Sandbox';
import { getSandboxContext } from 'n8n-nodes-base/dist/nodes/Code/Sandbox';
import { JavaScriptSandbox } from 'n8n-nodes-base/dist/nodes/Code/JavaScriptSandbox';
import { PythonSandbox } from 'n8n-nodes-base/dist/nodes/Code/PythonSandbox';
import { DynamicStructuredTool, DynamicTool } from '@langchain/core/tools'; import type { DynamicZodObject } from '../../../types/zod.types';
import { getConnectionHintNoticeField } from '../../../utils/sharedFields';
import { import {
inputSchemaField, inputSchemaField,
jsonSchemaExampleField, jsonSchemaExampleField,
schemaTypeField, schemaTypeField,
} from '../../../utils/descriptions'; } from '../../../utils/descriptions';
import { generateSchema, getSandboxWithZod } from '../../../utils/schemaParsing'; import { convertJsonSchemaToZod, generateSchema } from '../../../utils/schemaParsing';
import type { JSONSchema7 } from 'json-schema'; import { getConnectionHintNoticeField } from '../../../utils/sharedFields';
import type { DynamicZodObject } from '../../../types/zod.types';
export class ToolCode implements INodeType { export class ToolCode implements INodeType {
description: INodeTypeDescription = { description: INodeTypeDescription = {
@ -273,10 +272,9 @@ export class ToolCode implements INodeType {
? generateSchema(jsonExample) ? generateSchema(jsonExample)
: jsonParse<JSONSchema7>(inputSchema); : jsonParse<JSONSchema7>(inputSchema);
const zodSchemaSandbox = getSandboxWithZod(this, jsonSchema, 0); const zodSchema = convertJsonSchemaToZod<DynamicZodObject>(jsonSchema);
const zodSchema = await zodSchemaSandbox.runCode<DynamicZodObject>();
tool = new DynamicStructuredTool<typeof zodSchema>({ tool = new DynamicStructuredTool({
schema: zodSchema, schema: zodSchema,
...commonToolOptions, ...commonToolOptions,
}); });

24
packages/@n8n/nodes-langchain/nodes/tools/ToolWorkflow/ToolWorkflow.node.ts
View file

@ -1,3 +1,10 @@
import type { CallbackManagerForToolRun } from '@langchain/core/callbacks/manager';
import { DynamicStructuredTool, DynamicTool } from '@langchain/core/tools';
import type { JSONSchema7 } from 'json-schema';
import get from 'lodash/get';
import isObject from 'lodash/isObject';
import type { SetField, SetNodeOptions } from 'n8n-nodes-base/dist/nodes/Set/v2/helpers/interfaces';
import * as manual from 'n8n-nodes-base/dist/nodes/Set/v2/manual.mode';
import type { import type {
IExecuteFunctions, IExecuteFunctions,
IExecuteWorkflowInfo, IExecuteWorkflowInfo,
@ -11,22 +18,16 @@ import type {
INodeParameterResourceLocator, INodeParameterResourceLocator,
} from 'n8n-workflow'; } from 'n8n-workflow';
import { NodeConnectionType, NodeOperationError, jsonParse } from 'n8n-workflow'; import { NodeConnectionType, NodeOperationError, jsonParse } from 'n8n-workflow';
import type { SetField, SetNodeOptions } from 'n8n-nodes-base/dist/nodes/Set/v2/helpers/interfaces';
import * as manual from 'n8n-nodes-base/dist/nodes/Set/v2/manual.mode';
import { DynamicStructuredTool, DynamicTool } from '@langchain/core/tools';
import get from 'lodash/get';
import isObject from 'lodash/isObject';
import type { CallbackManagerForToolRun } from '@langchain/core/callbacks/manager';
import type { JSONSchema7 } from 'json-schema';
import { getConnectionHintNoticeField } from '../../../utils/sharedFields';
import type { DynamicZodObject } from '../../../types/zod.types'; import type { DynamicZodObject } from '../../../types/zod.types';
import { generateSchema, getSandboxWithZod } from '../../../utils/schemaParsing';
import { import {
jsonSchemaExampleField, jsonSchemaExampleField,
schemaTypeField, schemaTypeField,
inputSchemaField, inputSchemaField,
} from '../../../utils/descriptions'; } from '../../../utils/descriptions';
import { convertJsonSchemaToZod, generateSchema } from '../../../utils/schemaParsing';
import { getConnectionHintNoticeField } from '../../../utils/sharedFields';
export class ToolWorkflow implements INodeType { export class ToolWorkflow implements INodeType {
description: INodeTypeDescription = { description: INodeTypeDescription = {
displayName: 'Call n8n Workflow Tool', displayName: 'Call n8n Workflow Tool',
@ -529,10 +530,9 @@ export class ToolWorkflow implements INodeType {
? generateSchema(jsonExample) ? generateSchema(jsonExample)
: jsonParse<JSONSchema7>(inputSchema); : jsonParse<JSONSchema7>(inputSchema);
const zodSchemaSandbox = getSandboxWithZod(this, jsonSchema, 0); const zodSchema = convertJsonSchemaToZod<DynamicZodObject>(jsonSchema);
const zodSchema = await zodSchemaSandbox.runCode<DynamicZodObject>();
tool = new DynamicStructuredTool<typeof zodSchema>({ tool = new DynamicStructuredTool({
schema: zodSchema, schema: zodSchema,
...functionBase, ...functionBase,
}); });

2
packages/@n8n/nodes-langchain/package.json
View file

@ -168,7 +168,7 @@
"generate-schema": "2.6.0", "generate-schema": "2.6.0",
"html-to-text": "9.0.5", "html-to-text": "9.0.5",
"jsdom": "23.0.1", "jsdom": "23.0.1",
"json-schema-to-zod": "2.1.0", "@n8n/json-schema-to-zod": "workspace:*",
"langchain": "0.3.2", "langchain": "0.3.2",
"lodash": "catalog:", "lodash": "catalog:",
"mammoth": "1.7.2", "mammoth": "1.7.2",

65
packages/@n8n/nodes-langchain/utils/schemaParsing.ts
View file

@ -1,67 +1,10 @@
import { makeResolverFromLegacyOptions } from '@n8n/vm2'; import { jsonSchemaToZod } from '@n8n/json-schema-to-zod';
import { json as generateJsonSchema } from 'generate-schema'; import { json as generateJsonSchema } from 'generate-schema';
import type { SchemaObject } from 'generate-schema'; import type { SchemaObject } from 'generate-schema';
import type { JSONSchema7 } from 'json-schema'; import type { JSONSchema7 } from 'json-schema';
import { JavaScriptSandbox } from 'n8n-nodes-base/dist/nodes/Code/JavaScriptSandbox';
import { getSandboxContext } from 'n8n-nodes-base/dist/nodes/Code/Sandbox';
import type { IExecuteFunctions } from 'n8n-workflow'; import type { IExecuteFunctions } from 'n8n-workflow';
import { NodeOperationError, jsonParse } from 'n8n-workflow'; import { NodeOperationError, jsonParse } from 'n8n-workflow';
import type { z } from 'zod';
const vmResolver = makeResolverFromLegacyOptions({
external: {
modules: ['json-schema-to-zod', 'zod'],
transitive: false,
},
resolve(moduleName, parentDirname) {
if (moduleName === 'json-schema-to-zod') {
return require.resolve(
'@n8n/n8n-nodes-langchain/node_modules/json-schema-to-zod/dist/cjs/jsonSchemaToZod.js',
{
paths: [parentDirname],
},
);
}
if (moduleName === 'zod') {
return require.resolve('@n8n/n8n-nodes-langchain/node_modules/zod.cjs', {
paths: [parentDirname],
});
}
return;
},
builtin: [],
});
export function getSandboxWithZod(ctx: IExecuteFunctions, schema: JSONSchema7, itemIndex: number) {
const context = getSandboxContext.call(ctx, itemIndex);
let itemSchema: JSONSchema7 = schema;
try {
// If the root type is not defined, we assume it's an object
if (itemSchema.type === undefined) {
itemSchema = {
type: 'object',
properties: itemSchema.properties ?? (itemSchema as { [key: string]: JSONSchema7 }),
};
}
} catch (error) {
throw new NodeOperationError(ctx.getNode(), 'Error during parsing of JSON Schema.');
}
// Make sure to remove the description from root schema
const { description, ...restOfSchema } = itemSchema;
const sandboxedSchema = new JavaScriptSandbox(
context,
`
const { z } = require('zod');
const { parseSchema } = require('json-schema-to-zod');
const zodSchema = parseSchema(${JSON.stringify(restOfSchema)});
const itemSchema = new Function('z', 'return (' + zodSchema + ')')(z)
return itemSchema
`,
ctx.helpers,
{ resolver: vmResolver },
);
return sandboxedSchema;
}
export function generateSchema(schemaString: string): JSONSchema7 { export function generateSchema(schemaString: string): JSONSchema7 {
const parsedSchema = jsonParse<SchemaObject>(schemaString); const parsedSchema = jsonParse<SchemaObject>(schemaString);
@ -69,6 +12,10 @@ export function generateSchema(schemaString: string): JSONSchema7 {
return generateJsonSchema(parsedSchema) as JSONSchema7; return generateJsonSchema(parsedSchema) as JSONSchema7;
} }
export function convertJsonSchemaToZod<T extends z.ZodTypeAny = z.ZodTypeAny>(schema: JSONSchema7) {
return jsonSchemaToZod<T>(schema);
}
export function throwIfToolSchema(ctx: IExecuteFunctions, error: Error) { export function throwIfToolSchema(ctx: IExecuteFunctions, error: Error) {
if (error?.message?.includes('tool input did not match expected schema')) { if (error?.message?.includes('tool input did not match expected schema')) {
throw new NodeOperationError( throw new NodeOperationError(

12
pnpm-lock.yaml
View file

@ -477,6 +477,9 @@ importers:
'@mozilla/readability': '@mozilla/readability':
specifier: 0.5.0 specifier: 0.5.0
version: 0.5.0 version: 0.5.0
'@n8n/json-schema-to-zod':
specifier: workspace:*
version: link:../json-schema-to-zod
'@n8n/typeorm': '@n8n/typeorm':
specifier: 0.3.20-12 specifier: 0.3.20-12
version: 0.3.20-12(@sentry/node@7.87.0)(ioredis@5.3.2)(mssql@10.0.2)(mysql2@3.11.0)(pg@8.12.0)(redis@4.6.12)(sqlite3@5.1.7)(ts-node@10.9.2(@types/node@18.16.16)(typescript@5.6.2)) version: 0.3.20-12(@sentry/node@7.87.0)(ioredis@5.3.2)(mssql@10.0.2)(mysql2@3.11.0)(pg@8.12.0)(redis@4.6.12)(sqlite3@5.1.7)(ts-node@10.9.2(@types/node@18.16.16)(typescript@5.6.2))
@ -522,9 +525,6 @@ importers:
jsdom: jsdom:
specifier: 23.0.1 specifier: 23.0.1
version: 23.0.1 version: 23.0.1
json-schema-to-zod:
specifier: 2.1.0
version: 2.1.0
langchain: langchain:
specifier: 0.3.2 specifier: 0.3.2
version: 0.3.2(u4cmnaniapk3e37ytin75vjstm) version: 0.3.2(u4cmnaniapk3e37ytin75vjstm)
@ -8602,10 +8602,6 @@ packages:
json-pointer@0.6.2: json-pointer@0.6.2:
resolution: {integrity: sha512-vLWcKbOaXlO+jvRy4qNd+TI1QUPZzfJj1tpJ3vAXDych5XJf93ftpUKe5pKCrzyIIwgBJcOcCVRUfqQP25afBw==} resolution: {integrity: sha512-vLWcKbOaXlO+jvRy4qNd+TI1QUPZzfJj1tpJ3vAXDych5XJf93ftpUKe5pKCrzyIIwgBJcOcCVRUfqQP25afBw==}
json-schema-to-zod@2.1.0:
resolution: {integrity: sha512-7ishNgYY+AbIKeeHcp5xCOdJbdVwSfDx/4V2ktc16LUusCJJbz2fEKdWUmAxhKIiYzhZ9Fp4E8OsAoM/h9cOLA==}
hasBin: true
json-schema-traverse@0.4.1: json-schema-traverse@0.4.1:
resolution: {integrity: sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==} resolution: {integrity: sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==}
@ -21340,8 +21336,6 @@ snapshots:
dependencies: dependencies:
foreach: 2.0.6 foreach: 2.0.6
json-schema-to-zod@2.1.0: {}
json-schema-traverse@0.4.1: {} json-schema-traverse@0.4.1: {}
json-schema-traverse@1.0.0: {} json-schema-traverse@1.0.0: {}