mudhorn/n8n
1
0
Fork 0
mirror of https://github.com/n8n-io/n8n.git synced 2025-01-11 21:07:28 -08:00
Code Issues Actions Packages Projects Releases Wiki Activity

refactor: Replace json-schema-to-zod with our own fork (#11229)

Browse source
This commit is contained in:
Tomi Turtiainen 2024-10-18 08:29:19 +02:00 committed by GitHub
parent c57cac9e4d
commit a042d5c8e6
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 55 additions and 121 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
packages/@n8n/nodes-langchain/nodes/chains/InformationExtractor/InformationExtractor.node.ts
View file

@ -1,3 +1,8 @@
import type { BaseLanguageModel } from '@langchain/core/language_models/base';
import { HumanMessage } from '@langchain/core/messages';
import { ChatPromptTemplate, SystemMessagePromptTemplate } from '@langchain/core/prompts';
import type { JSONSchema7 } from 'json-schema';
import { OutputFixingParser, StructuredOutputParser } from 'langchain/output_parsers';
import { jsonParse, NodeConnectionType, NodeOperationError } from 'n8n-workflow';
import type {
INodeType,
@ -6,21 +11,17 @@ import type {
INodeExecutionData,
INodePropertyOptions,
} from 'n8n-workflow';
import type { JSONSchema7 } from 'json-schema';
import type { BaseLanguageModel } from '@langchain/core/language_models/base';
import { ChatPromptTemplate, SystemMessagePromptTemplate } from '@langchain/core/prompts';
import type { z } from 'zod';
import { OutputFixingParser, StructuredOutputParser } from 'langchain/output_parsers';
import { HumanMessage } from '@langchain/core/messages';
import { generateSchema, getSandboxWithZod } from '../../../utils/schemaParsing';
import { makeZodSchemaFromAttributes } from './helpers';
import type { AttributeDefinition } from './types';
import {
inputSchemaField,
jsonSchemaExampleField,
schemaTypeField,
} from '../../../utils/descriptions';
import { convertJsonSchemaToZod, generateSchema } from '../../../utils/schemaParsing';
import { getTracingConfig } from '../../../utils/tracing';
import type { AttributeDefinition } from './types';
import { makeZodSchemaFromAttributes } from './helpers';
const SYSTEM_PROMPT_TEMPLATE = `You are an expert extraction algorithm.
Only extract relevant information from the text.
@ -261,8 +262,7 @@ export class InformationExtractor implements INodeType {
jsonSchema = jsonParse<JSONSchema7>(inputSchema);
}
const zodSchemaSandbox = getSandboxWithZod(this, jsonSchema, 0);
const zodSchema = await zodSchemaSandbox.runCode<z.ZodSchema<object>>();
const zodSchema = convertJsonSchemaToZod<z.ZodSchema<object>>(jsonSchema);
parser = OutputFixingParser.fromLLM(llm, StructuredOutputParser.fromZodSchema(zodSchema));
}

29
packages/@n8n/nodes-langchain/nodes/output_parser/OutputParserStructured/OutputParserStructured.node.ts
View file

@ -1,4 +1,8 @@
/* eslint-disable n8n-nodes-base/node-dirname-against-convention */
import { OutputParserException } from '@langchain/core/output_parsers';
import type { JSONSchema7 } from 'json-schema';
import { StructuredOutputParser } from 'langchain/output_parsers';
import get from 'lodash/get';
import {
jsonParse,
type IExecuteFunctions,
@ -9,19 +13,15 @@ import {
NodeConnectionType,
} from 'n8n-workflow';
import { z } from 'zod';
import type { JSONSchema7 } from 'json-schema';
import { StructuredOutputParser } from 'langchain/output_parsers';
import { OutputParserException } from '@langchain/core/output_parsers';
import get from 'lodash/get';
import type { JavaScriptSandbox } from 'n8n-nodes-base/dist/nodes/Code/JavaScriptSandbox';
import { getConnectionHintNoticeField } from '../../../utils/sharedFields';
import { logWrapper } from '../../../utils/logWrapper';
import { generateSchema, getSandboxWithZod } from '../../../utils/schemaParsing';
import {
inputSchemaField,
jsonSchemaExampleField,
schemaTypeField,
} from '../../../utils/descriptions';
import { logWrapper } from '../../../utils/logWrapper';
import { convertJsonSchemaToZod, generateSchema } from '../../../utils/schemaParsing';
import { getConnectionHintNoticeField } from '../../../utils/sharedFields';
const STRUCTURED_OUTPUT_KEY = '__structured__output';
const STRUCTURED_OUTPUT_OBJECT_KEY = '__structured__output__object';
@ -44,12 +44,10 @@ export class N8nStructuredOutputParser<T extends z.ZodTypeAny> extends Structure
}
}
static async fromZedJsonSchema(
sandboxedSchema: JavaScriptSandbox,
static async fromZedSchema(
zodSchema: z.ZodSchema<object>,
nodeVersion: number,
): Promise<StructuredOutputParser<z.ZodType<object, z.ZodTypeDef, object>>> {
const zodSchema = await sandboxedSchema.runCode<z.ZodSchema<object>>();
let returnSchema: z.ZodSchema<object>;
if (nodeVersion === 1) {
returnSchema = z.object({
@ -204,13 +202,10 @@ export class OutputParserStructured implements INodeType {
const jsonSchema =
schemaType === 'fromJson' ? generateSchema(jsonExample) : jsonParse<JSONSchema7>(inputSchema);
const zodSchemaSandbox = getSandboxWithZod(this, jsonSchema, 0);
const zodSchema = convertJsonSchemaToZod<z.ZodSchema<object>>(jsonSchema);
const nodeVersion = this.getNode().typeVersion;
try {
const parser = await N8nStructuredOutputParser.fromZedJsonSchema(
zodSchemaSandbox,
nodeVersion,
);
const parser = await N8nStructuredOutputParser.fromZedSchema(zodSchema, nodeVersion);
return {
response: logWrapper(parser, this),
};

24
packages/@n8n/nodes-langchain/nodes/tools/ToolCode/ToolCode.node.ts
View file

@ -1,4 +1,10 @@
/* eslint-disable n8n-nodes-base/node-dirname-against-convention */
import { DynamicStructuredTool, DynamicTool } from '@langchain/core/tools';
import type { JSONSchema7 } from 'json-schema';
import { JavaScriptSandbox } from 'n8n-nodes-base/dist/nodes/Code/JavaScriptSandbox';
import { PythonSandbox } from 'n8n-nodes-base/dist/nodes/Code/PythonSandbox';
import type { Sandbox } from 'n8n-nodes-base/dist/nodes/Code/Sandbox';
import { getSandboxContext } from 'n8n-nodes-base/dist/nodes/Code/Sandbox';
import type {
IExecuteFunctions,
INodeType,
@ -7,23 +13,16 @@ import type {
ExecutionError,
IDataObject,
} from 'n8n-workflow';
import { jsonParse, NodeConnectionType, NodeOperationError } from 'n8n-workflow';
import type { Sandbox } from 'n8n-nodes-base/dist/nodes/Code/Sandbox';
import { getSandboxContext } from 'n8n-nodes-base/dist/nodes/Code/Sandbox';
import { JavaScriptSandbox } from 'n8n-nodes-base/dist/nodes/Code/JavaScriptSandbox';
import { PythonSandbox } from 'n8n-nodes-base/dist/nodes/Code/PythonSandbox';
import { DynamicStructuredTool, DynamicTool } from '@langchain/core/tools';
import { getConnectionHintNoticeField } from '../../../utils/sharedFields';
import type { DynamicZodObject } from '../../../types/zod.types';
import {
inputSchemaField,
jsonSchemaExampleField,
schemaTypeField,
} from '../../../utils/descriptions';
import { generateSchema, getSandboxWithZod } from '../../../utils/schemaParsing';
import type { JSONSchema7 } from 'json-schema';
import type { DynamicZodObject } from '../../../types/zod.types';
import { convertJsonSchemaToZod, generateSchema } from '../../../utils/schemaParsing';
import { getConnectionHintNoticeField } from '../../../utils/sharedFields';
export class ToolCode implements INodeType {
description: INodeTypeDescription = {
@ -273,10 +272,9 @@ export class ToolCode implements INodeType {
? generateSchema(jsonExample)
: jsonParse<JSONSchema7>(inputSchema);
const zodSchemaSandbox = getSandboxWithZod(this, jsonSchema, 0);
const zodSchema = await zodSchemaSandbox.runCode<DynamicZodObject>();
const zodSchema = convertJsonSchemaToZod<DynamicZodObject>(jsonSchema);
tool = new DynamicStructuredTool<typeof zodSchema>({
tool = new DynamicStructuredTool({
schema: zodSchema,
...commonToolOptions,
});

24
packages/@n8n/nodes-langchain/nodes/tools/ToolWorkflow/ToolWorkflow.node.ts
View file

@ -1,3 +1,10 @@
import type { CallbackManagerForToolRun } from '@langchain/core/callbacks/manager';
import { DynamicStructuredTool, DynamicTool } from '@langchain/core/tools';
import type { JSONSchema7 } from 'json-schema';
import get from 'lodash/get';
import isObject from 'lodash/isObject';
import type { SetField, SetNodeOptions } from 'n8n-nodes-base/dist/nodes/Set/v2/helpers/interfaces';
import * as manual from 'n8n-nodes-base/dist/nodes/Set/v2/manual.mode';
import type {
IExecuteFunctions,
IExecuteWorkflowInfo,
@ -11,22 +18,16 @@ import type {
INodeParameterResourceLocator,
} from 'n8n-workflow';
import { NodeConnectionType, NodeOperationError, jsonParse } from 'n8n-workflow';
import type { SetField, SetNodeOptions } from 'n8n-nodes-base/dist/nodes/Set/v2/helpers/interfaces';
import * as manual from 'n8n-nodes-base/dist/nodes/Set/v2/manual.mode';
import { DynamicStructuredTool, DynamicTool } from '@langchain/core/tools';
import get from 'lodash/get';
import isObject from 'lodash/isObject';
import type { CallbackManagerForToolRun } from '@langchain/core/callbacks/manager';
import type { JSONSchema7 } from 'json-schema';
import { getConnectionHintNoticeField } from '../../../utils/sharedFields';
import type { DynamicZodObject } from '../../../types/zod.types';
import { generateSchema, getSandboxWithZod } from '../../../utils/schemaParsing';
import {
jsonSchemaExampleField,
schemaTypeField,
inputSchemaField,
} from '../../../utils/descriptions';
import { convertJsonSchemaToZod, generateSchema } from '../../../utils/schemaParsing';
import { getConnectionHintNoticeField } from '../../../utils/sharedFields';
export class ToolWorkflow implements INodeType {
description: INodeTypeDescription = {
displayName: 'Call n8n Workflow Tool',
@ -529,10 +530,9 @@ export class ToolWorkflow implements INodeType {
? generateSchema(jsonExample)
: jsonParse<JSONSchema7>(inputSchema);
const zodSchemaSandbox = getSandboxWithZod(this, jsonSchema, 0);
const zodSchema = await zodSchemaSandbox.runCode<DynamicZodObject>();
const zodSchema = convertJsonSchemaToZod<DynamicZodObject>(jsonSchema);
tool = new DynamicStructuredTool<typeof zodSchema>({
tool = new DynamicStructuredTool({
schema: zodSchema,
...functionBase,
});

2
packages/@n8n/nodes-langchain/package.json
View file

@ -168,7 +168,7 @@
"generate-schema": "2.6.0",
"html-to-text": "9.0.5",
"jsdom": "23.0.1",
"json-schema-to-zod": "2.1.0",
"@n8n/json-schema-to-zod": "workspace:*",
"langchain": "0.3.2",
"lodash": "catalog:",
"mammoth": "1.7.2",

65
packages/@n8n/nodes-langchain/utils/schemaParsing.ts
View file

@ -1,67 +1,10 @@
import { makeResolverFromLegacyOptions } from '@n8n/vm2';
import { jsonSchemaToZod } from '@n8n/json-schema-to-zod';
import { json as generateJsonSchema } from 'generate-schema';
import type { SchemaObject } from 'generate-schema';
import type { JSONSchema7 } from 'json-schema';
import { JavaScriptSandbox } from 'n8n-nodes-base/dist/nodes/Code/JavaScriptSandbox';
import { getSandboxContext } from 'n8n-nodes-base/dist/nodes/Code/Sandbox';
import type { IExecuteFunctions } from 'n8n-workflow';
import { NodeOperationError, jsonParse } from 'n8n-workflow';
const vmResolver = makeResolverFromLegacyOptions({
external: {
modules: ['json-schema-to-zod', 'zod'],
transitive: false,
},
resolve(moduleName, parentDirname) {
if (moduleName === 'json-schema-to-zod') {
return require.resolve(
'@n8n/n8n-nodes-langchain/node_modules/json-schema-to-zod/dist/cjs/jsonSchemaToZod.js',
{
paths: [parentDirname],
},
);
}
if (moduleName === 'zod') {
return require.resolve('@n8n/n8n-nodes-langchain/node_modules/zod.cjs', {
paths: [parentDirname],
});
}
return;
},
builtin: [],
});
export function getSandboxWithZod(ctx: IExecuteFunctions, schema: JSONSchema7, itemIndex: number) {
const context = getSandboxContext.call(ctx, itemIndex);
let itemSchema: JSONSchema7 = schema;
try {
// If the root type is not defined, we assume it's an object
if (itemSchema.type === undefined) {
itemSchema = {
type: 'object',
properties: itemSchema.properties ?? (itemSchema as { [key: string]: JSONSchema7 }),
};
}
} catch (error) {
throw new NodeOperationError(ctx.getNode(), 'Error during parsing of JSON Schema.');
}
// Make sure to remove the description from root schema
const { description, ...restOfSchema } = itemSchema;
const sandboxedSchema = new JavaScriptSandbox(
context,
`
const { z } = require('zod');
const { parseSchema } = require('json-schema-to-zod');
const zodSchema = parseSchema(${JSON.stringify(restOfSchema)});
const itemSchema = new Function('z', 'return (' + zodSchema + ')')(z)
return itemSchema
`,
ctx.helpers,
{ resolver: vmResolver },
);
return sandboxedSchema;
}
import type { z } from 'zod';
export function generateSchema(schemaString: string): JSONSchema7 {
const parsedSchema = jsonParse<SchemaObject>(schemaString);
@ -69,6 +12,10 @@ export function generateSchema(schemaString: string): JSONSchema7 {
return generateJsonSchema(parsedSchema) as JSONSchema7;
}
export function convertJsonSchemaToZod<T extends z.ZodTypeAny = z.ZodTypeAny>(schema: JSONSchema7) {
return jsonSchemaToZod<T>(schema);
}
export function throwIfToolSchema(ctx: IExecuteFunctions, error: Error) {
if (error?.message?.includes('tool input did not match expected schema')) {
throw new NodeOperationError(

12
pnpm-lock.yaml
View file

@ -477,6 +477,9 @@ importers:
'@mozilla/readability':
specifier: 0.5.0
version: 0.5.0
'@n8n/json-schema-to-zod':
specifier: workspace:*
version: link:../json-schema-to-zod
'@n8n/typeorm':
specifier: 0.3.20-12
version: 0.3.20-12(@sentry/node@7.87.0)(ioredis@5.3.2)(mssql@10.0.2)(mysql2@3.11.0)(pg@8.12.0)(redis@4.6.12)(sqlite3@5.1.7)(ts-node@10.9.2(@types/node@18.16.16)(typescript@5.6.2))
@ -522,9 +525,6 @@ importers:
jsdom:
specifier: 23.0.1
version: 23.0.1
json-schema-to-zod:
specifier: 2.1.0
version: 2.1.0
langchain:
specifier: 0.3.2
version: 0.3.2(u4cmnaniapk3e37ytin75vjstm)
@ -8602,10 +8602,6 @@ packages:
json-pointer@0.6.2:
resolution: {integrity: sha512-vLWcKbOaXlO+jvRy4qNd+TI1QUPZzfJj1tpJ3vAXDych5XJf93ftpUKe5pKCrzyIIwgBJcOcCVRUfqQP25afBw==}
json-schema-to-zod@2.1.0:
resolution: {integrity: sha512-7ishNgYY+AbIKeeHcp5xCOdJbdVwSfDx/4V2ktc16LUusCJJbz2fEKdWUmAxhKIiYzhZ9Fp4E8OsAoM/h9cOLA==}
hasBin: true
json-schema-traverse@0.4.1:
resolution: {integrity: sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==}
@ -21340,8 +21336,6 @@ snapshots:
dependencies:
foreach: 2.0.6
json-schema-to-zod@2.1.0: {}
json-schema-traverse@0.4.1: {}
json-schema-traverse@1.0.0: {}