From 1e333b7310f2d19c9621b2bf6d02fe7049ff0e9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iv=C3=A1n=20Ovejero?= Date: Thu, 26 Sep 2024 09:35:08 +0200 Subject: [PATCH] WIP --- packages/cli/package.json | 2 + .../src/databases/database-schema.service.ts | 23 +- .../src/databases/import-export/constants.ts | 23 +- .../import-export/database-export.service.ts | 118 ++++---- .../import-export/database-import.service.ts | 10 +- .../import-export/manifest.schema.ts | 5 +- .../cli/src/databases/import-export/types.ts | 41 ++- pnpm-lock.yaml | 252 ++++++++++-------- 8 files changed, 244 insertions(+), 230 deletions(-) diff --git a/packages/cli/package.json b/packages/cli/package.json index b178512965..3b03bbf0ab 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -101,6 +101,7 @@ "@rudderstack/rudder-sdk-node": "2.0.9", "@sentry/integrations": "7.87.0", "@sentry/node": "7.87.0", + "@types/tar-stream": "^3.1.3", "archiver": "7.0.1", "aws4": "1.11.0", "axios": "catalog:", @@ -170,6 +171,7 @@ "sshpk": "1.17.0", "swagger-ui-express": "5.0.0", "syslog-client": "1.1.1", + "tar-stream": "^3.1.7", "typedi": "catalog:", "unzip-stream": "0.3.4", "uuid": "catalog:", diff --git a/packages/cli/src/databases/database-schema.service.ts b/packages/cli/src/databases/database-schema.service.ts index bd6beb80d2..990efae293 100644 --- a/packages/cli/src/databases/database-schema.service.ts +++ b/packages/cli/src/databases/database-schema.service.ts @@ -7,7 +7,7 @@ import { Service } from 'typedi'; import { NonEmptyTableError } from '@/errors/non-empty-table.error'; -import type { Sequence } from './import-export/types'; +import type { Sequence, SequenceRow } from './import-export/types'; import { LastMigrationNotFoundError } from '../errors/last-migration-not-found.error'; /** @@ -72,24 +72,39 @@ export class DatabaseSchemaService { /** Get the names and values of all incremental ID sequences. */ async getSequences() { if (this.dbType === 'sqlite') { - return await this.dataSource.query( + const result = await this.dataSource.query( "SELECT name, seq AS value FROM sqlite_sequence WHERE name != 'migrations';", ); + + return result.reduce((acc, cur) => { + acc[cur.name] = cur.value; + return acc; + }, {}); } if (this.dbType === 'postgresdb') { - return await this.dataSource.query( + const result = await this.dataSource.query( "SELECT sequencename AS name, start_value AS value FROM pg_sequences WHERE sequencename != 'migrations_id_seq';", ); + + return result.reduce((acc, cur) => { + acc[cur.name] = cur.value; + return acc; + }, {}); } // @TODO: Does this work for MariaDB? if (this.dbType === 'mysqldb' || this.dbType === 'mariadb') { const schema = this.globalConfig.database.mysqldb.database; // @TODO: Why deprecated? How to filter otherwise? - return await this.dataSource.query( + const result = await this.dataSource.query( `SELECT table_name AS name, ordinal_position AS value FROM information_schema.columns WHERE table_schema = '${schema}' AND extra = 'auto_increment' AND table_name != 'migrations';`, ); + + return result.reduce((acc, cur) => { + acc[cur.name] = cur.value; + return acc; + }, {}); } throw new ApplicationError('Unknown database type', { extra: { dbType: this.dbType } }); diff --git a/packages/cli/src/databases/import-export/constants.ts b/packages/cli/src/databases/import-export/constants.ts index 13d1ed4602..b5ae95a940 100644 --- a/packages/cli/src/databases/import-export/constants.ts +++ b/packages/cli/src/databases/import-export/constants.ts @@ -1,5 +1,18 @@ -/** - * Name of the file containing metadata about exported files. - * Underscored to prevent accidental match with table name. - */ -export const MANIFEST_FILENAME = '_manifest.json'; +/** Base filename for the tarball, to be suffixed with `-{timestamp}.zip`. */ +export const ZIP_BASE_FILE_NAME = 'n8n-db-export'; + +/** Name of the file describing the export. */ +export const MANIFEST_FILENAME = 'manifest.json'; + +/** Default number of rows to retrieve from DB and write to a `.jsonl` file at a time. */ +export const BATCH_SIZE = 500; + +/** Tables to exclude from the export in lightweight mode. */ +export const EXCLUDE_LIST = [ + 'execution_annotation_tags', + 'execution_annotations', + 'execution_data', + 'execution_entity', + 'execution_metadata', + 'annotation_tag_entity', +]; diff --git a/packages/cli/src/databases/import-export/database-export.service.ts b/packages/cli/src/databases/import-export/database-export.service.ts index e79b3ac255..56232d4c43 100644 --- a/packages/cli/src/databases/import-export/database-export.service.ts +++ b/packages/cli/src/databases/import-export/database-export.service.ts @@ -1,54 +1,34 @@ import { GlobalConfig } from '@n8n/config'; import type { ColumnMetadata } from '@n8n/typeorm/metadata/ColumnMetadata'; -import archiver from 'archiver'; import { jsonParse } from 'n8n-workflow'; import fs from 'node:fs'; import path from 'node:path'; -import { PassThrough } from 'node:stream'; +import { createGzip } from 'node:zlib'; +import tar from 'tar-stream'; import { Service } from 'typedi'; import { Logger } from '@/logger'; -import { MANIFEST_FILENAME } from './constants'; -import type { Manifest } from './manifest.schema'; -import type { DatabaseExportConfig, Row } from './types'; -import { FilesystemService } from '../../filesystem/filesystem.service'; +import { BATCH_SIZE, EXCLUDE_LIST, MANIFEST_FILENAME, ZIP_BASE_FILE_NAME } from './constants'; +import type { DatabaseExportConfig, Manifest, Row } from './types'; import { DatabaseSchemaService } from '../database-schema.service'; import type { DatabaseType } from '../types'; // @TODO: Check minimum version for each DB type? -// @TODO: Optional table exclude list @Service() export class DatabaseExportService { private config: DatabaseExportConfig = { - storageDirPath: '/tmp/backup', - tarballBaseFileName: 'n8n-db-export', - batchSize: 500, + outDir: '/tmp/backup', // @TODO: Update to cwd + mode: 'full', }; - /** Paths to the files to include in the tarball. */ - private readonly exportFilePaths: string[] = []; - - /** Number of rows in tables being exported. */ - private readonly rowCounts: { [tableName: string]: number } = {}; + private readonly rowCounts: Manifest['rowCounts'] = {}; private readonly dbType: DatabaseType; - get tarballPath() { - const now = new Date(); - const year = now.getFullYear(); - const month = String(now.getMonth() + 1).padStart(2, '0'); - const day = String(now.getDate()).padStart(2, '0'); - - const tarballFileName = `${this.config.tarballBaseFileName}-${year}-${month}-${day}.tar.gz`; - - return path.join(this.config.storageDirPath, tarballFileName); - } - constructor( private readonly globalConfig: GlobalConfig, - private readonly fsService: FilesystemService, private readonly schemaService: DatabaseSchemaService, private readonly logger: Logger, ) { @@ -59,22 +39,31 @@ export class DatabaseExportService { this.config = { ...this.config, ...config }; } + get tarballPath() { + const now = new Date(); + const year = now.getFullYear(); + const month = String(now.getMonth() + 1).padStart(2, '0'); + const day = String(now.getDate()).padStart(2, '0'); + + const tarballFileName = `${ZIP_BASE_FILE_NAME}-${year}-${month}-${day}.tar.gz`; + + return path.join(this.config.outDir, tarballFileName); + } + // #region Export - /** Export DB tables into a tarball of `.jsonl` files plus a `.json` metadata file. */ async export() { - await this.fsService.ensureDir(this.config.storageDirPath); + this.logger.info('[ExportService] Starting export', { outDir: this.config.outDir }); - this.logger.info('[ExportService] Starting export', { - dbType: this.dbType, - storageDirPath: this.config.storageDirPath, - }); + try { + await fs.promises.access(this.config.outDir); + } catch { + await fs.promises.mkdir(this.config.outDir, { recursive: true }); + } await this.writeTarball(); - await this.postExportCleanup(); - - this.logger.info('[ExportService] Completed export', { tarballPath: this.tarballPath }); + this.logger.info('[ExportService] Completed export', { zipPath: this.tarballPath }); } // #endregion @@ -82,48 +71,44 @@ export class DatabaseExportService { // #region Export steps private async writeTarball() { - const tarballPath = path.join(this.config.storageDirPath, this.tarballPath); + const pack = tar.pack(); - const archive = archiver('zip', { zlib: { level: 9 } }); + // DB row -> entryStream -> tarStream -> gzipStream -> writeStream - archive.pipe(fs.createWriteStream(tarballPath)); + const tables = + this.config.mode === 'full' + ? this.schemaService.getTables() + : this.schemaService.getTables().filter((t) => !EXCLUDE_LIST.includes(t.tableName)); - const writeStream = new PassThrough(); - - for (const { tableName, columns } of this.schemaService.getTables()) { - archive.append(writeStream, { name: `${tableName}.jsonl` }); + for (const { tableName, columns } of tables) { + const entry = pack.entry({ name: `${tableName}.jsonl` }); let offset = 0; let totalRows = 0; while (true) { - const rows = await this.schemaService + const batch = await this.schemaService .getDataSource() - .query( - `SELECT * FROM ${tableName} LIMIT ${this.config.batchSize} OFFSET ${offset};`, - ); // @TODO: Double-quotes for column in Postgres but not for other DB types? + .query(`SELECT * FROM ${tableName} LIMIT ${BATCH_SIZE} OFFSET ${offset};`); + // @TODO: Double quotes for Postgres but not others - if (rows.length === 0) break; + if (batch.length === 0) break; - for (const row of rows) { + for (const row of batch) { for (const column of columns) { this.normalizeRow(row, { column, tableName }); } - - const json = JSON.stringify(row); - writeStream.write(json); - writeStream.write('\n'); + entry.write(JSON.stringify(row) + '\n'); } - totalRows += rows.length; - offset += this.config.batchSize; - - this.logger.info(`[ExportService] Exported ${totalRows} rows from ${tableName}`); - - writeStream.end(); + totalRows += batch.length; + offset += BATCH_SIZE; } + if (totalRows === 0) continue; + this.rowCounts[tableName] = totalRows; + this.logger.info(`[ExportService] Exported ${totalRows} rows from ${tableName}`); } const manifest: Manifest = { @@ -136,12 +121,14 @@ export class DatabaseExportService { const manifestBuffer = Buffer.from(JSON.stringify(manifest, null, 2), 'utf-8'); - archive.append(manifestBuffer, { name: MANIFEST_FILENAME }); + pack.entry({ name: MANIFEST_FILENAME }, manifestBuffer); - await archive.finalize(); + pack.finalize(); + + // pack.pipe(process.stdout); + pack.pipe(createGzip()).pipe(fs.createWriteStream(this.tarballPath)); } - /** Make values in SQLite and MySQL rows compatible with Postgres. */ private normalizeRow( row: Row, { column, tableName }: { column: ColumnMetadata; tableName: string }, @@ -166,13 +153,6 @@ export class DatabaseExportService { // @TODO: MySQL and MariaDB normalizations } - /** Clear all `.jsonl` and `.json` files from the storage dir. */ - async postExportCleanup() { - await this.fsService.removeFiles(this.exportFilePaths); - - this.exportFilePaths.length = 0; - } - // #endregion // #region Utils diff --git a/packages/cli/src/databases/import-export/database-import.service.ts b/packages/cli/src/databases/import-export/database-import.service.ts index 8ce868dc67..b84558bd31 100644 --- a/packages/cli/src/databases/import-export/database-import.service.ts +++ b/packages/cli/src/databases/import-export/database-import.service.ts @@ -13,9 +13,8 @@ import { Logger } from '@/logger'; import { isObjectLiteral } from '@/utils'; import { MANIFEST_FILENAME } from './constants'; -import type { Manifest } from './manifest.schema'; import { manifestSchema } from './manifest.schema'; -import type { DatabaseImportConfig } from './types'; +import type { DatabaseImportConfig, Manifest } from './types'; import { MalformedManifestError } from '../../errors/malformed-manifest.error'; import { MigrationsMismatchError } from '../../errors/migrations-mismatch.error'; import { UnsupportedDestinationError } from '../../errors/unsupported-destination.error'; @@ -167,10 +166,11 @@ export class DatabaseImportService { * Adjust incremental ID sequences in Postgres to match the source database. */ private async adjustSequences() { - for (const { name, value } of this.manifest.sequences) { + for (const [rawSeqName, rawSeqValue] of Object.entries(this.manifest.sequences)) { // `execution_metadata` has abnormally named and numbered sequence - const sequenceName = name === 'execution_metadata' ? `${name}_temp_id_seq` : `${name}_id_seq`; - const sequenceValue = value <= 0 ? 1 : value; + const sequenceName = + rawSeqName === 'execution_metadata' ? `${rawSeqName}_temp_id_seq` : `${rawSeqName}_id_seq`; + const sequenceValue = rawSeqValue <= 0 ? 1 : rawSeqValue; await this.schemaService .getDataSource() diff --git a/packages/cli/src/databases/import-export/manifest.schema.ts b/packages/cli/src/databases/import-export/manifest.schema.ts index f570dada06..290257dd13 100644 --- a/packages/cli/src/databases/import-export/manifest.schema.ts +++ b/packages/cli/src/databases/import-export/manifest.schema.ts @@ -31,8 +31,5 @@ export const manifestSchema = z.object({ * Incremental ID sequences in tables being exported. * @example [ { name: 'workflow_entity', value: 123 }, { name: 'credentials_entity', value: 456 } ] */ - sequences: z.array(z.object({ name: z.string(), value: z.number() })), + sequences: z.record(z.string(), z.number()), }); - -/** Manifest describing the export, included as JSON file in the tarball. */ -export type Manifest = z.infer; diff --git a/packages/cli/src/databases/import-export/types.ts b/packages/cli/src/databases/import-export/types.ts index 41384cc720..b984c1e252 100644 --- a/packages/cli/src/databases/import-export/types.ts +++ b/packages/cli/src/databases/import-export/types.ts @@ -1,39 +1,28 @@ +import type { z } from 'zod'; + +import type { manifestSchema } from './manifest.schema'; + export type Row = Record; -/** Name and value of incremental ID sequence for column. */ -export type Sequence = { name: string; value: number }; // @TODO: Refactor as { [tableName: string]: number } +export type SequenceRow = { name: string; value: number }; + +export type Sequence = { [tableName: string]: number }; + +export type Manifest = z.infer; export type DatabaseExportConfig = { - /** - * Path to the dir to place the export in. - * @default '/tmp/backup' - */ - storageDirPath: string; + /** Dir to place the export in. By default, the current working directory. */ + outDir: string; - /** - * Base filename for the tarball, to be suffixed with `-{timestamp}.tar.gz`. - * @default 'n8n-db-export' - */ - tarballBaseFileName: string; - - /** - * Number of rows to retrieve from DB and write to a `.jsonl` file at a time. - * @default 500 - */ - batchSize: number; + /** Whether to export all data or only a smaller subset of data. */ + mode: 'full' | 'lightweight'; }; export type DatabaseImportConfig = { - /** - * Path to the file to import. Unset by default. - * @example '/tmp/backup/n8n-db-export-2021-01-01.tar.gz' - */ + /** Absolute path to the file to import. */ importFilePath: string; - /** - * Path to the directory to extract the tarball into. - * @default '/tmp/backup' - */ + // REMOVE extractDirPath: string; /** diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 1c2c90385c..37deecfef7 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -258,7 +258,7 @@ importers: version: 4.0.7 axios: specifier: 'catalog:' - version: 1.7.4(debug@4.3.6) + version: 1.7.4 dotenv: specifier: 8.6.0 version: 8.6.0 @@ -335,7 +335,7 @@ importers: dependencies: axios: specifier: 'catalog:' - version: 1.7.4(debug@4.3.6) + version: 1.7.4 packages/@n8n/codemirror-lang: dependencies: @@ -424,7 +424,7 @@ importers: version: 0.3.0(@aws-sdk/client-sso-oidc@3.654.0(@aws-sdk/client-sts@3.654.0))(@langchain/core@0.3.3(openai@4.63.0(encoding@0.1.13)(zod@3.23.8)))(encoding@0.1.13) '@langchain/community': specifier: 0.3.2 - version: 0.3.2(w76mtnfu5u6rwfka7oawoz7cbe) + version: 0.3.2(civqmlpli3jn6p47akgofyjajm) '@langchain/core': specifier: 'catalog:' version: 0.3.3(openai@4.63.0(encoding@0.1.13)(zod@3.23.8)) @@ -719,6 +719,9 @@ importers: '@sentry/node': specifier: 7.87.0 version: 7.87.0 + '@types/tar-stream': + specifier: ^3.1.3 + version: 3.1.3 archiver: specifier: 7.0.1 version: 7.0.1 @@ -727,7 +730,7 @@ importers: version: 1.11.0 axios: specifier: 'catalog:' - version: 1.7.4(debug@4.3.6) + version: 1.7.4 bcryptjs: specifier: 2.4.3 version: 2.4.3 @@ -926,6 +929,9 @@ importers: syslog-client: specifier: 1.1.1 version: 1.1.1 + tar-stream: + specifier: ^3.1.7 + version: 3.1.7 typedi: specifier: 'catalog:' version: 0.10.0(patch_hash=sk6omkefrosihg7lmqbzh7vfxe) @@ -1061,7 +1067,7 @@ importers: dependencies: '@langchain/core': specifier: 'catalog:' - version: 0.3.3(openai@4.63.0) + version: 0.3.3(openai@4.63.0(encoding@0.1.13)(zod@3.23.8)) '@n8n/client-oauth2': specifier: workspace:* version: link:../@n8n/client-oauth2 @@ -1070,7 +1076,7 @@ importers: version: 1.11.0 axios: specifier: 'catalog:' - version: 1.7.4(debug@4.3.6) + version: 1.7.4 concat-stream: specifier: 2.0.0 version: 2.0.0 @@ -1357,7 +1363,7 @@ importers: version: 10.11.0(vue@3.4.21(typescript@5.6.2)) axios: specifier: 'catalog:' - version: 1.7.4(debug@4.3.6) + version: 1.7.4 bowser: specifier: 2.11.0 version: 2.11.0 @@ -1831,7 +1837,7 @@ importers: version: 0.15.2 axios: specifier: 'catalog:' - version: 1.7.4(debug@4.3.6) + version: 1.7.4 callsites: specifier: 3.1.0 version: 3.1.0 @@ -1877,7 +1883,7 @@ importers: devDependencies: '@langchain/core': specifier: 'catalog:' - version: 0.3.3(openai@4.63.0) + version: 0.3.3(openai@4.63.0(encoding@0.1.13)(zod@3.23.8)) '@types/deep-equal': specifier: ^1.0.1 version: 1.0.1 @@ -5516,6 +5522,9 @@ packages: '@types/syslog-client@1.1.2': resolution: {integrity: sha512-X8MwGedXYNmYltPDaZQCM9X6cSdfFbJZWhrU81gWKsg+Q6mSgRWs/12Mq9nHaUV4wqMYDNrnytbwbMUiVnWegw==} + '@types/tar-stream@3.1.3': + resolution: {integrity: sha512-Zbnx4wpkWBMBSu5CytMbrT5ZpMiF55qgM+EpHzR4yIDu7mv52cej8hTkOc6K+LzpkOAbxwn/m7j3iO+/l42YkQ==} + '@types/tedious@4.0.9': resolution: {integrity: sha512-ipwFvfy9b2m0gjHsIX0D6NAAwGCKokzf5zJqUZHUGt+7uWVlBIy6n2eyMgiKQ8ChLFVxic/zwQUhjLYNzbHDRA==} @@ -6218,6 +6227,9 @@ packages: axios-retry@3.7.0: resolution: {integrity: sha512-ZTnCkJbRtfScvwiRnoVskFAfvU0UG3xNcsjwTR0mawSbIJoothxn67gKsMaNAFHRXJ1RmuLhmZBzvyXi3+9WyQ==} + axios@1.7.3: + resolution: {integrity: sha512-Ar7ND9pU99eJ9GpoGQKhKf58GpUOgnzuaB7ueNQ5BMi0p+LZ5oaEnfF999fAArcTIBwXTCHAmGcHOZJaWPq9Nw==} + axios@1.7.4: resolution: {integrity: sha512-DukmaFRnY6AzAALSH4J2M3k6PkaC+MfaAGdEERRWcC9q3/TWQwLpHR8ZRLKTdQ3aBDL64EdluRDjJqKw+BPZEw==} @@ -13056,7 +13068,7 @@ snapshots: '@aws-sdk/client-sso-oidc': 3.654.0(@aws-sdk/client-sts@3.645.0) '@aws-sdk/client-sts': 3.654.0 '@aws-sdk/core': 3.654.0 - '@aws-sdk/credential-provider-node': 3.654.0(@aws-sdk/client-sso-oidc@3.654.0(@aws-sdk/client-sts@3.654.0))(@aws-sdk/client-sts@3.645.0) + '@aws-sdk/credential-provider-node': 3.654.0(@aws-sdk/client-sso-oidc@3.654.0(@aws-sdk/client-sts@3.654.0))(@aws-sdk/client-sts@3.654.0) '@aws-sdk/middleware-host-header': 3.654.0 '@aws-sdk/middleware-logger': 3.654.0 '@aws-sdk/middleware-recursion-detection': 3.654.0 @@ -13201,7 +13213,7 @@ snapshots: '@aws-sdk/client-sso-oidc': 3.654.0(@aws-sdk/client-sts@3.654.0) '@aws-sdk/client-sts': 3.654.0 '@aws-sdk/core': 3.654.0 - '@aws-sdk/credential-provider-node': 3.654.0(@aws-sdk/client-sso-oidc@3.654.0(@aws-sdk/client-sts@3.654.0))(@aws-sdk/client-sts@3.645.0) + '@aws-sdk/credential-provider-node': 3.654.0(@aws-sdk/client-sso-oidc@3.654.0(@aws-sdk/client-sts@3.654.0))(@aws-sdk/client-sts@3.654.0) '@aws-sdk/middleware-host-header': 3.654.0 '@aws-sdk/middleware-logger': 3.654.0 '@aws-sdk/middleware-recursion-detection': 3.654.0 @@ -13448,7 +13460,7 @@ snapshots: '@aws-crypto/sha256-js': 5.2.0 '@aws-sdk/client-sts': 3.654.0 '@aws-sdk/core': 3.654.0 - '@aws-sdk/credential-provider-node': 3.654.0(@aws-sdk/client-sso-oidc@3.654.0(@aws-sdk/client-sts@3.654.0))(@aws-sdk/client-sts@3.645.0) + '@aws-sdk/credential-provider-node': 3.654.0(@aws-sdk/client-sso-oidc@3.654.0(@aws-sdk/client-sts@3.654.0))(@aws-sdk/client-sts@3.654.0) '@aws-sdk/middleware-host-header': 3.654.0 '@aws-sdk/middleware-logger': 3.654.0 '@aws-sdk/middleware-recursion-detection': 3.654.0 @@ -13711,7 +13723,7 @@ snapshots: '@aws-crypto/sha256-js': 5.2.0 '@aws-sdk/client-sso-oidc': 3.654.0(@aws-sdk/client-sts@3.654.0) '@aws-sdk/core': 3.654.0 - '@aws-sdk/credential-provider-node': 3.654.0(@aws-sdk/client-sso-oidc@3.654.0(@aws-sdk/client-sts@3.654.0))(@aws-sdk/client-sts@3.645.0) + '@aws-sdk/credential-provider-node': 3.654.0(@aws-sdk/client-sso-oidc@3.654.0(@aws-sdk/client-sts@3.654.0))(@aws-sdk/client-sts@3.654.0) '@aws-sdk/middleware-host-header': 3.654.0 '@aws-sdk/middleware-logger': 3.654.0 '@aws-sdk/middleware-recursion-detection': 3.654.0 @@ -13891,19 +13903,19 @@ snapshots: - '@aws-sdk/client-sso-oidc' - aws-crt - '@aws-sdk/credential-provider-ini@3.645.0(@aws-sdk/client-sts@3.645.0)': + '@aws-sdk/credential-provider-ini@3.654.0(@aws-sdk/client-sso-oidc@3.654.0(@aws-sdk/client-sts@3.645.0))(@aws-sdk/client-sts@3.645.0)': dependencies: '@aws-sdk/client-sts': 3.645.0 - '@aws-sdk/credential-provider-env': 3.620.1 - '@aws-sdk/credential-provider-http': 3.635.0 - '@aws-sdk/credential-provider-process': 3.620.1 - '@aws-sdk/credential-provider-sso': 3.645.0(@aws-sdk/client-sso-oidc@3.645.0(@aws-sdk/client-sts@3.645.0)) - '@aws-sdk/credential-provider-web-identity': 3.621.0(@aws-sdk/client-sts@3.645.0) - '@aws-sdk/types': 3.609.0 - '@smithy/credential-provider-imds': 3.2.0 - '@smithy/property-provider': 3.1.3 - '@smithy/shared-ini-file-loader': 3.1.4 - '@smithy/types': 3.3.0 + '@aws-sdk/credential-provider-env': 3.654.0 + '@aws-sdk/credential-provider-http': 3.654.0 + '@aws-sdk/credential-provider-process': 3.654.0 + '@aws-sdk/credential-provider-sso': 3.654.0(@aws-sdk/client-sso-oidc@3.654.0(@aws-sdk/client-sts@3.645.0)) + '@aws-sdk/credential-provider-web-identity': 3.654.0(@aws-sdk/client-sts@3.645.0) + '@aws-sdk/types': 3.654.0 + '@smithy/credential-provider-imds': 3.2.3 + '@smithy/property-provider': 3.1.6 + '@smithy/shared-ini-file-loader': 3.1.7 + '@smithy/types': 3.4.2 tslib: 2.6.2 transitivePeerDependencies: - '@aws-sdk/client-sso-oidc' @@ -13928,6 +13940,24 @@ snapshots: - '@aws-sdk/client-sso-oidc' - aws-crt + '@aws-sdk/credential-provider-ini@3.654.0(@aws-sdk/client-sso-oidc@3.654.0(@aws-sdk/client-sts@3.654.0))(@aws-sdk/client-sts@3.654.0)': + dependencies: + '@aws-sdk/client-sts': 3.654.0 + '@aws-sdk/credential-provider-env': 3.654.0 + '@aws-sdk/credential-provider-http': 3.654.0 + '@aws-sdk/credential-provider-process': 3.654.0 + '@aws-sdk/credential-provider-sso': 3.654.0(@aws-sdk/client-sso-oidc@3.654.0(@aws-sdk/client-sts@3.654.0)) + '@aws-sdk/credential-provider-web-identity': 3.654.0(@aws-sdk/client-sts@3.654.0) + '@aws-sdk/types': 3.654.0 + '@smithy/credential-provider-imds': 3.2.3 + '@smithy/property-provider': 3.1.6 + '@smithy/shared-ini-file-loader': 3.1.7 + '@smithy/types': 3.4.2 + tslib: 2.6.2 + transitivePeerDependencies: + - '@aws-sdk/client-sso-oidc' + - aws-crt + '@aws-sdk/credential-provider-node@3.478.0': dependencies: '@aws-sdk/credential-provider-env': 3.468.0 @@ -13982,19 +14012,19 @@ snapshots: - '@aws-sdk/client-sts' - aws-crt - '@aws-sdk/credential-provider-node@3.645.0(@aws-sdk/client-sts@3.645.0)': + '@aws-sdk/credential-provider-node@3.654.0(@aws-sdk/client-sso-oidc@3.654.0(@aws-sdk/client-sts@3.645.0))(@aws-sdk/client-sts@3.645.0)': dependencies: - '@aws-sdk/credential-provider-env': 3.620.1 - '@aws-sdk/credential-provider-http': 3.635.0 - '@aws-sdk/credential-provider-ini': 3.645.0(@aws-sdk/client-sts@3.645.0) - '@aws-sdk/credential-provider-process': 3.620.1 - '@aws-sdk/credential-provider-sso': 3.645.0(@aws-sdk/client-sso-oidc@3.645.0(@aws-sdk/client-sts@3.645.0)) - '@aws-sdk/credential-provider-web-identity': 3.621.0(@aws-sdk/client-sts@3.645.0) - '@aws-sdk/types': 3.609.0 - '@smithy/credential-provider-imds': 3.2.0 - '@smithy/property-provider': 3.1.3 - '@smithy/shared-ini-file-loader': 3.1.4 - '@smithy/types': 3.3.0 + '@aws-sdk/credential-provider-env': 3.654.0 + '@aws-sdk/credential-provider-http': 3.654.0 + '@aws-sdk/credential-provider-ini': 3.654.0(@aws-sdk/client-sso-oidc@3.654.0(@aws-sdk/client-sts@3.645.0))(@aws-sdk/client-sts@3.645.0) + '@aws-sdk/credential-provider-process': 3.654.0 + '@aws-sdk/credential-provider-sso': 3.654.0(@aws-sdk/client-sso-oidc@3.654.0(@aws-sdk/client-sts@3.645.0)) + '@aws-sdk/credential-provider-web-identity': 3.654.0(@aws-sdk/client-sts@3.645.0) + '@aws-sdk/types': 3.654.0 + '@smithy/credential-provider-imds': 3.2.3 + '@smithy/property-provider': 3.1.6 + '@smithy/shared-ini-file-loader': 3.1.7 + '@smithy/types': 3.4.2 tslib: 2.6.2 transitivePeerDependencies: - '@aws-sdk/client-sso-oidc' @@ -14021,6 +14051,25 @@ snapshots: - '@aws-sdk/client-sts' - aws-crt + '@aws-sdk/credential-provider-node@3.654.0(@aws-sdk/client-sso-oidc@3.654.0(@aws-sdk/client-sts@3.654.0))(@aws-sdk/client-sts@3.654.0)': + dependencies: + '@aws-sdk/credential-provider-env': 3.654.0 + '@aws-sdk/credential-provider-http': 3.654.0 + '@aws-sdk/credential-provider-ini': 3.654.0(@aws-sdk/client-sso-oidc@3.654.0(@aws-sdk/client-sts@3.654.0))(@aws-sdk/client-sts@3.654.0) + '@aws-sdk/credential-provider-process': 3.654.0 + '@aws-sdk/credential-provider-sso': 3.654.0(@aws-sdk/client-sso-oidc@3.654.0(@aws-sdk/client-sts@3.654.0)) + '@aws-sdk/credential-provider-web-identity': 3.654.0(@aws-sdk/client-sts@3.654.0) + '@aws-sdk/types': 3.654.0 + '@smithy/credential-provider-imds': 3.2.3 + '@smithy/property-provider': 3.1.6 + '@smithy/shared-ini-file-loader': 3.1.7 + '@smithy/types': 3.4.2 + tslib: 2.6.2 + transitivePeerDependencies: + - '@aws-sdk/client-sso-oidc' + - '@aws-sdk/client-sts' + - aws-crt + '@aws-sdk/credential-provider-process@3.468.0': dependencies: '@aws-sdk/types': 3.468.0 @@ -14083,6 +14132,20 @@ snapshots: - '@aws-sdk/client-sso-oidc' - aws-crt + '@aws-sdk/credential-provider-sso@3.654.0(@aws-sdk/client-sso-oidc@3.654.0(@aws-sdk/client-sts@3.645.0))': + dependencies: + '@aws-sdk/client-sso': 3.654.0 + '@aws-sdk/token-providers': 3.654.0(@aws-sdk/client-sso-oidc@3.654.0(@aws-sdk/client-sts@3.645.0)) + '@aws-sdk/types': 3.654.0 + '@smithy/property-provider': 3.1.6 + '@smithy/shared-ini-file-loader': 3.1.7 + '@smithy/types': 3.4.2 + tslib: 2.6.2 + transitivePeerDependencies: + - '@aws-sdk/client-sso-oidc' + - aws-crt + optional: true + '@aws-sdk/credential-provider-sso@3.654.0(@aws-sdk/client-sso-oidc@3.654.0(@aws-sdk/client-sts@3.654.0))': dependencies: '@aws-sdk/client-sso': 3.654.0 @@ -14119,28 +14182,13 @@ snapshots: '@smithy/types': 3.4.2 tslib: 2.6.2 - '@aws-sdk/credential-providers@3.645.0': + '@aws-sdk/credential-provider-web-identity@3.654.0(@aws-sdk/client-sts@3.654.0)': dependencies: - '@aws-sdk/client-cognito-identity': 3.645.0 - '@aws-sdk/client-sso': 3.645.0 - '@aws-sdk/client-sts': 3.645.0 - '@aws-sdk/credential-provider-cognito-identity': 3.645.0 - '@aws-sdk/credential-provider-env': 3.620.1 - '@aws-sdk/credential-provider-http': 3.635.0 - '@aws-sdk/credential-provider-ini': 3.645.0(@aws-sdk/client-sts@3.645.0) - '@aws-sdk/credential-provider-node': 3.645.0(@aws-sdk/client-sts@3.645.0) - '@aws-sdk/credential-provider-process': 3.620.1 - '@aws-sdk/credential-provider-sso': 3.645.0(@aws-sdk/client-sso-oidc@3.645.0(@aws-sdk/client-sts@3.645.0)) - '@aws-sdk/credential-provider-web-identity': 3.621.0(@aws-sdk/client-sts@3.645.0) - '@aws-sdk/types': 3.609.0 - '@smithy/credential-provider-imds': 3.2.3 + '@aws-sdk/client-sts': 3.654.0 + '@aws-sdk/types': 3.654.0 '@smithy/property-provider': 3.1.6 '@smithy/types': 3.4.2 tslib: 2.6.2 - transitivePeerDependencies: - - '@aws-sdk/client-sso-oidc' - - aws-crt - optional: true '@aws-sdk/credential-providers@3.645.0(@aws-sdk/client-sso-oidc@3.654.0(@aws-sdk/client-sts@3.654.0))': dependencies: @@ -14420,7 +14468,7 @@ snapshots: '@smithy/types': 3.3.0 tslib: 2.6.2 - '@aws-sdk/token-providers@3.654.0(@aws-sdk/client-sso-oidc@3.654.0(@aws-sdk/client-sts@3.654.0))': + '@aws-sdk/token-providers@3.654.0(@aws-sdk/client-sso-oidc@3.654.0(@aws-sdk/client-sts@3.645.0))': dependencies: '@aws-sdk/client-sso-oidc': 3.654.0(@aws-sdk/client-sts@3.645.0) '@aws-sdk/types': 3.654.0 @@ -14428,6 +14476,16 @@ snapshots: '@smithy/shared-ini-file-loader': 3.1.7 '@smithy/types': 3.4.2 tslib: 2.6.2 + optional: true + + '@aws-sdk/token-providers@3.654.0(@aws-sdk/client-sso-oidc@3.654.0(@aws-sdk/client-sts@3.654.0))': + dependencies: + '@aws-sdk/client-sso-oidc': 3.654.0(@aws-sdk/client-sts@3.654.0) + '@aws-sdk/types': 3.654.0 + '@smithy/property-provider': 3.1.6 + '@smithy/shared-ini-file-loader': 3.1.7 + '@smithy/types': 3.4.2 + tslib: 2.6.2 '@aws-sdk/types@3.468.0': dependencies: @@ -15763,7 +15821,7 @@ snapshots: - aws-crt - encoding - '@langchain/community@0.3.2(w76mtnfu5u6rwfka7oawoz7cbe)': + '@langchain/community@0.3.2(civqmlpli3jn6p47akgofyjajm)': dependencies: '@langchain/core': 0.3.3(openai@4.63.0(encoding@0.1.13)(zod@3.23.8)) '@langchain/openai': 0.3.0(@langchain/core@0.3.3(openai@4.63.0(encoding@0.1.13)(zod@3.23.8)))(encoding@0.1.13) @@ -15782,7 +15840,7 @@ snapshots: '@aws-sdk/client-bedrock-runtime': 3.645.0 '@aws-sdk/client-kendra': 3.654.0 '@aws-sdk/client-s3': 3.478.0 - '@aws-sdk/credential-provider-node': 3.654.0(@aws-sdk/client-sso-oidc@3.654.0(@aws-sdk/client-sts@3.654.0))(@aws-sdk/client-sts@3.645.0) + '@aws-sdk/credential-provider-node': 3.654.0(@aws-sdk/client-sso-oidc@3.654.0(@aws-sdk/client-sts@3.645.0))(@aws-sdk/client-sts@3.645.0) '@azure/storage-blob': 12.18.0(encoding@0.1.13) '@getzep/zep-cloud': 1.0.11(@langchain/core@0.3.3(openai@4.63.0(encoding@0.1.13)(zod@3.23.8)))(encoding@0.1.13)(langchain@0.3.2(jteet444uhz3pk3ib4eb2yc7pm)) '@getzep/zep-js': 0.9.0 @@ -15848,22 +15906,6 @@ snapshots: transitivePeerDependencies: - openai - '@langchain/core@0.3.3(openai@4.63.0)': - dependencies: - ansi-styles: 5.2.0 - camelcase: 6.3.0 - decamelize: 1.2.0 - js-tiktoken: 1.0.12 - langsmith: 0.1.59(openai@4.63.0) - mustache: 4.2.0 - p-queue: 6.6.2 - p-retry: 4.6.2 - uuid: 10.0.0 - zod: 3.23.8 - zod-to-json-schema: 3.23.3(zod@3.23.8) - transitivePeerDependencies: - - openai - '@langchain/google-common@0.1.1(@langchain/core@0.3.3(openai@4.63.0(encoding@0.1.13)(zod@3.23.8)))(zod@3.23.8)': dependencies: '@langchain/core': 0.3.3(openai@4.63.0(encoding@0.1.13)(zod@3.23.8)) @@ -16091,7 +16133,7 @@ snapshots: '@n8n/localtunnel@3.0.0': dependencies: - axios: 1.7.4(debug@4.3.6) + axios: 1.7.3(debug@4.3.6) debug: 4.3.6(supports-color@8.1.1) transitivePeerDependencies: - supports-color @@ -16475,7 +16517,7 @@ snapshots: '@rudderstack/rudder-sdk-node@2.0.9(tslib@2.6.2)': dependencies: - axios: 1.7.4(debug@4.3.6) + axios: 1.7.4 axios-retry: 3.7.0 component-type: 1.2.1 join-component: 1.1.0 @@ -18422,6 +18464,10 @@ snapshots: dependencies: '@types/node': 18.16.16 + '@types/tar-stream@3.1.3': + dependencies: + '@types/node': 18.16.16 + '@types/tedious@4.0.9': dependencies: '@types/node': 18.16.16 @@ -19279,7 +19325,7 @@ snapshots: '@babel/runtime': 7.24.7 is-retry-allowed: 2.2.0 - axios@1.7.4(debug@4.3.6): + axios@1.7.3(debug@4.3.6): dependencies: follow-redirects: 1.15.6(debug@4.3.6) form-data: 4.0.0 @@ -19287,14 +19333,13 @@ snapshots: transitivePeerDependencies: - debug - axios@1.7.7: + axios@1.7.4: dependencies: - follow-redirects: 1.15.6(debug@4.3.6) + follow-redirects: 1.15.6(debug@4.3.7) form-data: 4.0.0 proxy-from-env: 1.1.0 transitivePeerDependencies: - debug - optional: true axios@1.7.7(debug@4.3.7): dependencies: @@ -20849,7 +20894,7 @@ snapshots: eslint-import-resolver-node@0.3.9: dependencies: - debug: 3.2.7(supports-color@5.5.0) + debug: 3.2.7(supports-color@8.1.1) is-core-module: 2.13.1 resolve: 1.22.8 transitivePeerDependencies: @@ -20874,7 +20919,7 @@ snapshots: eslint-module-utils@2.8.0(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.2))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.6.2))(eslint-plugin-import@2.29.1)(eslint@8.57.0))(eslint@8.57.0): dependencies: - debug: 3.2.7(supports-color@5.5.0) + debug: 3.2.7(supports-color@8.1.1) optionalDependencies: '@typescript-eslint/parser': 7.2.0(eslint@8.57.0)(typescript@5.6.2) eslint: 8.57.0 @@ -20894,7 +20939,7 @@ snapshots: array.prototype.findlastindex: 1.2.3 array.prototype.flat: 1.3.2 array.prototype.flatmap: 1.3.2 - debug: 3.2.7(supports-color@5.5.0) + debug: 3.2.7(supports-color@8.1.1) doctrine: 2.1.0 eslint: 8.57.0 eslint-import-resolver-node: 0.3.9 @@ -21703,7 +21748,7 @@ snapshots: array-parallel: 0.1.3 array-series: 0.1.5 cross-spawn: 4.0.2 - debug: 3.2.7(supports-color@5.5.0) + debug: 3.2.7(supports-color@8.1.1) transitivePeerDependencies: - supports-color @@ -22055,7 +22100,7 @@ snapshots: infisical-node@1.3.0: dependencies: - axios: 1.7.4(debug@4.3.6) + axios: 1.7.4 dotenv: 16.3.1 tweetnacl: 1.0.3 tweetnacl-util: 0.15.1 @@ -23014,7 +23059,7 @@ snapshots: '@langchain/groq': 0.1.2(@langchain/core@0.3.3(openai@4.63.0(encoding@0.1.13)(zod@3.23.8)))(encoding@0.1.13) '@langchain/mistralai': 0.1.1(@langchain/core@0.3.3(openai@4.63.0(encoding@0.1.13)(zod@3.23.8)))(encoding@0.1.13) '@langchain/ollama': 0.1.0(@langchain/core@0.3.3(openai@4.63.0(encoding@0.1.13)(zod@3.23.8))) - axios: 1.7.7 + axios: 1.7.7(debug@4.3.7) cheerio: 1.0.0-rc.12 handlebars: 4.7.8 transitivePeerDependencies: @@ -23033,17 +23078,6 @@ snapshots: optionalDependencies: openai: 4.63.0(encoding@0.1.13)(zod@3.23.8) - langsmith@0.1.59(openai@4.63.0): - dependencies: - '@types/uuid': 10.0.0 - commander: 10.0.1 - p-queue: 6.6.2 - p-retry: 4.6.2 - semver: 7.6.0 - uuid: 10.0.0 - optionalDependencies: - openai: 4.63.0(zod@3.23.8) - lazy-ass@1.6.0: {} lazystream@1.0.1: @@ -23921,7 +23955,7 @@ snapshots: bson: 6.3.0 mongodb-connection-string-url: 3.0.0 optionalDependencies: - '@aws-sdk/credential-providers': 3.645.0 + '@aws-sdk/credential-providers': 3.645.0(@aws-sdk/client-sso-oidc@3.654.0(@aws-sdk/client-sts@3.654.0)) gcp-metadata: 5.3.0(encoding@0.1.13) socks: 2.7.1 @@ -24362,22 +24396,6 @@ snapshots: - encoding - supports-color - openai@4.63.0(zod@3.23.8): - dependencies: - '@types/node': 18.16.16 - '@types/node-fetch': 2.6.4 - abort-controller: 3.0.0 - agentkeepalive: 4.2.1 - form-data-encoder: 1.7.2 - formdata-node: 4.4.1 - node-fetch: 2.7.0(encoding@0.1.13) - optionalDependencies: - zod: 3.23.8 - transitivePeerDependencies: - - encoding - - supports-color - optional: true - openapi-sampler@1.4.0: dependencies: '@types/json-schema': 7.0.15 @@ -24561,7 +24579,7 @@ snapshots: pdf-parse@1.1.1: dependencies: - debug: 3.2.7(supports-color@5.5.0) + debug: 3.2.7(supports-color@8.1.1) node-ensure: 0.0.0 transitivePeerDependencies: - supports-color @@ -24763,7 +24781,7 @@ snapshots: posthog-node@3.2.1: dependencies: - axios: 1.7.4(debug@4.3.6) + axios: 1.7.4 rusha: 0.8.14 transitivePeerDependencies: - debug @@ -25374,7 +25392,7 @@ snapshots: rhea@1.0.24: dependencies: - debug: 3.2.7(supports-color@5.5.0) + debug: 3.2.7(supports-color@8.1.1) transitivePeerDependencies: - supports-color @@ -25745,7 +25763,7 @@ snapshots: asn1.js: 5.4.1 asn1.js-rfc2560: 5.0.1(asn1.js@5.4.1) asn1.js-rfc5280: 3.0.0 - axios: 1.7.4(debug@4.3.6) + axios: 1.7.4 big-integer: 1.6.51 bignumber.js: 9.1.2 binascii: 0.0.2