mirror of
https://github.com/n8n-io/n8n.git
synced 2024-11-09 22:24:05 -08:00
feat(core): Add support for SQLite connection pooling (#8722)
This commit is contained in:
parent
d85d0ecf45
commit
c4c319d7cf
27
.github/workflows/ci-postgres-mysql.yml
vendored
27
.github/workflows/ci-postgres-mysql.yml
vendored
|
@ -34,6 +34,33 @@ jobs:
|
||||||
path: ./packages/**/dist
|
path: ./packages/**/dist
|
||||||
key: ${{ github.sha }}:db-tests
|
key: ${{ github.sha }}:db-tests
|
||||||
|
|
||||||
|
sqlite-pooled:
|
||||||
|
name: SQLite Pooled
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
needs: build
|
||||||
|
timeout-minutes: 20
|
||||||
|
env:
|
||||||
|
DB_TYPE: sqlite
|
||||||
|
DB_SQLITE_POOL_SIZE: 4
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4.1.1
|
||||||
|
- run: corepack enable
|
||||||
|
- uses: actions/setup-node@v4.0.1
|
||||||
|
with:
|
||||||
|
node-version: 18.x
|
||||||
|
cache: 'pnpm'
|
||||||
|
- run: pnpm install --frozen-lockfile
|
||||||
|
|
||||||
|
- name: Restore cached build artifacts
|
||||||
|
uses: actions/cache/restore@v4.0.0
|
||||||
|
with:
|
||||||
|
path: ./packages/**/dist
|
||||||
|
key: ${{ github.sha }}:db-tests
|
||||||
|
|
||||||
|
- name: Test SQLite Pooled
|
||||||
|
working-directory: packages/cli
|
||||||
|
run: pnpm jest --coverage
|
||||||
|
|
||||||
mysql:
|
mysql:
|
||||||
name: MySQL
|
name: MySQL
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
|
|
|
@ -140,7 +140,7 @@
|
||||||
"@langchain/openai": "^0.0.16",
|
"@langchain/openai": "^0.0.16",
|
||||||
"@langchain/pinecone": "^0.0.3",
|
"@langchain/pinecone": "^0.0.3",
|
||||||
"@langchain/redis": "^0.0.2",
|
"@langchain/redis": "^0.0.2",
|
||||||
"@n8n/typeorm": "0.3.20-3",
|
"@n8n/typeorm": "0.3.20-7",
|
||||||
"@n8n/vm2": "3.9.20",
|
"@n8n/vm2": "3.9.20",
|
||||||
"@pinecone-database/pinecone": "2.1.0",
|
"@pinecone-database/pinecone": "2.1.0",
|
||||||
"@qdrant/js-client-rest": "1.7.0",
|
"@qdrant/js-client-rest": "1.7.0",
|
||||||
|
|
|
@ -97,7 +97,7 @@
|
||||||
"@n8n/localtunnel": "2.1.0",
|
"@n8n/localtunnel": "2.1.0",
|
||||||
"@n8n/n8n-nodes-langchain": "workspace:*",
|
"@n8n/n8n-nodes-langchain": "workspace:*",
|
||||||
"@n8n/permissions": "workspace:*",
|
"@n8n/permissions": "workspace:*",
|
||||||
"@n8n/typeorm": "0.3.20-3",
|
"@n8n/typeorm": "0.3.20-7",
|
||||||
"@n8n_io/license-sdk": "2.10.0",
|
"@n8n_io/license-sdk": "2.10.0",
|
||||||
"@oclif/core": "3.18.1",
|
"@oclif/core": "3.18.1",
|
||||||
"@rudderstack/rudder-sdk-node": "2.0.7",
|
"@rudderstack/rudder-sdk-node": "2.0.7",
|
||||||
|
|
|
@ -171,11 +171,17 @@ export const schema = {
|
||||||
env: 'DB_SQLITE_DATABASE',
|
env: 'DB_SQLITE_DATABASE',
|
||||||
},
|
},
|
||||||
enableWAL: {
|
enableWAL: {
|
||||||
doc: 'Enable SQLite WAL mode',
|
doc: 'Enable SQLite WAL mode (Always enabled for pool-size > 1)',
|
||||||
format: Boolean,
|
format: Boolean,
|
||||||
default: false,
|
default: false,
|
||||||
env: 'DB_SQLITE_ENABLE_WAL',
|
env: 'DB_SQLITE_ENABLE_WAL',
|
||||||
},
|
},
|
||||||
|
poolSize: {
|
||||||
|
doc: 'SQLite Pool Size (Setting this to 0 disables pooling)',
|
||||||
|
format: Number,
|
||||||
|
default: 0,
|
||||||
|
env: 'DB_SQLITE_POOL_SIZE',
|
||||||
|
},
|
||||||
executeVacuumOnStartup: {
|
executeVacuumOnStartup: {
|
||||||
doc: 'Runs VACUUM operation on startup to rebuild the database. Reduces filesize and optimizes indexes. WARNING: This is a long running blocking operation. Will increase start-up time.',
|
doc: 'Runs VACUUM operation on startup to rebuild the database. Reduces filesize and optimizes indexes. WARNING: This is a long running blocking operation. Will increase start-up time.',
|
||||||
format: Boolean,
|
format: Boolean,
|
||||||
|
|
|
@ -3,6 +3,7 @@ import { Container } from 'typedi';
|
||||||
import type { TlsOptions } from 'tls';
|
import type { TlsOptions } from 'tls';
|
||||||
import type { DataSourceOptions, LoggerOptions } from '@n8n/typeorm';
|
import type { DataSourceOptions, LoggerOptions } from '@n8n/typeorm';
|
||||||
import type { SqliteConnectionOptions } from '@n8n/typeorm/driver/sqlite/SqliteConnectionOptions';
|
import type { SqliteConnectionOptions } from '@n8n/typeorm/driver/sqlite/SqliteConnectionOptions';
|
||||||
|
import type { SqlitePooledConnectionOptions } from '@n8n/typeorm/driver/sqlite-pooled/SqlitePooledConnectionOptions';
|
||||||
import type { PostgresConnectionOptions } from '@n8n/typeorm/driver/postgres/PostgresConnectionOptions';
|
import type { PostgresConnectionOptions } from '@n8n/typeorm/driver/postgres/PostgresConnectionOptions';
|
||||||
import type { MysqlConnectionOptions } from '@n8n/typeorm/driver/mysql/MysqlConnectionOptions';
|
import type { MysqlConnectionOptions } from '@n8n/typeorm/driver/mysql/MysqlConnectionOptions';
|
||||||
import { InstanceSettings } from 'n8n-core';
|
import { InstanceSettings } from 'n8n-core';
|
||||||
|
@ -47,16 +48,26 @@ export const getOptionOverrides = (dbType: 'postgresdb' | 'mysqldb') => ({
|
||||||
password: config.getEnv(`database.${dbType}.password`),
|
password: config.getEnv(`database.${dbType}.password`),
|
||||||
});
|
});
|
||||||
|
|
||||||
const getSqliteConnectionOptions = (): SqliteConnectionOptions => ({
|
const getSqliteConnectionOptions = (): SqliteConnectionOptions | SqlitePooledConnectionOptions => {
|
||||||
type: 'sqlite',
|
const poolSize = config.getEnv('database.sqlite.poolSize');
|
||||||
|
const commonOptions = {
|
||||||
...getCommonOptions(),
|
...getCommonOptions(),
|
||||||
database: path.resolve(
|
database: path.resolve(
|
||||||
Container.get(InstanceSettings).n8nFolder,
|
Container.get(InstanceSettings).n8nFolder,
|
||||||
config.getEnv('database.sqlite.database'),
|
config.getEnv('database.sqlite.database'),
|
||||||
),
|
),
|
||||||
enableWAL: config.getEnv('database.sqlite.enableWAL'),
|
|
||||||
migrations: sqliteMigrations,
|
migrations: sqliteMigrations,
|
||||||
});
|
};
|
||||||
|
if (poolSize > 0) {
|
||||||
|
return { type: 'sqlite-pooled', poolSize, enableWAL: true, ...commonOptions };
|
||||||
|
} else {
|
||||||
|
return {
|
||||||
|
type: 'sqlite',
|
||||||
|
enableWAL: config.getEnv('database.sqlite.enableWAL'),
|
||||||
|
...commonOptions,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
const getPostgresConnectionOptions = (): PostgresConnectionOptions => {
|
const getPostgresConnectionOptions = (): PostgresConnectionOptions => {
|
||||||
const sslCa = config.getEnv('database.postgresdb.ssl.ca');
|
const sslCa = config.getEnv('database.postgresdb.ssl.ca');
|
||||||
|
|
|
@ -218,8 +218,8 @@ importers:
|
||||||
specifier: ^0.0.2
|
specifier: ^0.0.2
|
||||||
version: 0.0.2
|
version: 0.0.2
|
||||||
'@n8n/typeorm':
|
'@n8n/typeorm':
|
||||||
specifier: 0.3.20-3
|
specifier: 0.3.20-7
|
||||||
version: 0.3.20-3(pg@8.11.3)(redis@4.6.12)(sqlite3@5.1.7)
|
version: 0.3.20-7(pg@8.11.3)(redis@4.6.12)(sqlite3@5.1.7)
|
||||||
'@n8n/vm2':
|
'@n8n/vm2':
|
||||||
specifier: 3.9.20
|
specifier: 3.9.20
|
||||||
version: 3.9.20
|
version: 3.9.20
|
||||||
|
@ -443,8 +443,8 @@ importers:
|
||||||
specifier: workspace:*
|
specifier: workspace:*
|
||||||
version: link:../@n8n/permissions
|
version: link:../@n8n/permissions
|
||||||
'@n8n/typeorm':
|
'@n8n/typeorm':
|
||||||
specifier: 0.3.20-3
|
specifier: 0.3.20-7
|
||||||
version: 0.3.20-3(@sentry/node@7.87.0)(ioredis@5.3.2)(mysql2@2.3.3)(pg@8.11.3)(sqlite3@5.1.7)
|
version: 0.3.20-7(@sentry/node@7.87.0)(ioredis@5.3.2)(mysql2@2.3.3)(pg@8.11.3)(sqlite3@5.1.7)
|
||||||
'@n8n_io/license-sdk':
|
'@n8n_io/license-sdk':
|
||||||
specifier: 2.10.0
|
specifier: 2.10.0
|
||||||
version: 2.10.0
|
version: 2.10.0
|
||||||
|
@ -4642,6 +4642,11 @@ packages:
|
||||||
resolution: {integrity: sha512-ooWCrlZP11i8GImSjTHYHLkvFDP48nS4+204nGb1RiX/WXYHmJA2III9/e2DWVabCESdW7hBAEzHRqUn9OUVvQ==}
|
resolution: {integrity: sha512-ooWCrlZP11i8GImSjTHYHLkvFDP48nS4+204nGb1RiX/WXYHmJA2III9/e2DWVabCESdW7hBAEzHRqUn9OUVvQ==}
|
||||||
engines: {node: '>=0.1.90'}
|
engines: {node: '>=0.1.90'}
|
||||||
|
|
||||||
|
/@common.js/is-network-error@1.0.1:
|
||||||
|
resolution: {integrity: sha512-dkk7FX8L/JLia5pi+IQ11lCw2D6FTmbWL2iWTHgCbP40/deeXgknlkEQcQ/rOkjwQbqp8RZ4ey/anR17K66sqw==}
|
||||||
|
engines: {node: '>=16'}
|
||||||
|
dev: false
|
||||||
|
|
||||||
/@ctrl/tinycolor@3.6.0:
|
/@ctrl/tinycolor@3.6.0:
|
||||||
resolution: {integrity: sha512-/Z3l6pXthq0JvMYdUFyX9j0MaCltlIn6mfh9jLyQwg5aPKxkyNa0PTHtU1AlFXLNk55ZuAeJRcpvq+tmLfKmaQ==}
|
resolution: {integrity: sha512-/Z3l6pXthq0JvMYdUFyX9j0MaCltlIn6mfh9jLyQwg5aPKxkyNa0PTHtU1AlFXLNk55ZuAeJRcpvq+tmLfKmaQ==}
|
||||||
engines: {node: '>=10'}
|
engines: {node: '>=10'}
|
||||||
|
@ -6437,6 +6442,15 @@ packages:
|
||||||
- supports-color
|
- supports-color
|
||||||
dev: false
|
dev: false
|
||||||
|
|
||||||
|
/@n8n/p-retry@6.2.0-2:
|
||||||
|
resolution: {integrity: sha512-rbnMnSdEwq2yuYMgzOQ4jTXm+oH7yjN/0ISfB/7O6pUcEPsZt9UW60BYfQ1WWHkKa/evI8vgER2zV5/RC1BupQ==}
|
||||||
|
engines: {node: '>=18.10'}
|
||||||
|
dependencies:
|
||||||
|
'@common.js/is-network-error': 1.0.1
|
||||||
|
'@types/retry': 0.12.5
|
||||||
|
retry: 0.13.1
|
||||||
|
dev: false
|
||||||
|
|
||||||
/@n8n/tournament@1.0.2:
|
/@n8n/tournament@1.0.2:
|
||||||
resolution: {integrity: sha512-fTpi7F8ra5flGSVfRzohPyG7czAAKCZPlLjdKdwbLJivLoI/Ekhgodov1jfVSCVFVbwQ06gRQRxLEDzl2jl8ig==}
|
resolution: {integrity: sha512-fTpi7F8ra5flGSVfRzohPyG7czAAKCZPlLjdKdwbLJivLoI/Ekhgodov1jfVSCVFVbwQ06gRQRxLEDzl2jl8ig==}
|
||||||
engines: {node: '>=18.10', pnpm: '>=8.6'}
|
engines: {node: '>=18.10', pnpm: '>=8.6'}
|
||||||
|
@ -6447,8 +6461,8 @@ packages:
|
||||||
recast: 0.22.0
|
recast: 0.22.0
|
||||||
dev: false
|
dev: false
|
||||||
|
|
||||||
/@n8n/typeorm@0.3.20-3(@sentry/node@7.87.0)(ioredis@5.3.2)(mysql2@2.3.3)(pg@8.11.3)(sqlite3@5.1.7):
|
/@n8n/typeorm@0.3.20-7(@sentry/node@7.87.0)(ioredis@5.3.2)(mysql2@2.3.3)(pg@8.11.3)(sqlite3@5.1.7):
|
||||||
resolution: {integrity: sha512-eTaEBvaJZZCStqiLKSUIvFLMZJ3WMOFc+pnF+jedcQOc6AunWM57FqhPvnuB/Z6c67JmgIyINlnjbLDHtPgKZQ==}
|
resolution: {integrity: sha512-f4A9RGOnB3kCkusNAr1QDCGOVq1HU1YCBKoIGr2of+P3CVS3I+1vW7neOhlr/ic5S1F14Qy5TU8Lb78mRBYRSw==}
|
||||||
engines: {node: '>=16.13.0'}
|
engines: {node: '>=16.13.0'}
|
||||||
hasBin: true
|
hasBin: true
|
||||||
peerDependencies:
|
peerDependencies:
|
||||||
|
@ -6511,6 +6525,7 @@ packages:
|
||||||
typeorm-aurora-data-api-driver:
|
typeorm-aurora-data-api-driver:
|
||||||
optional: true
|
optional: true
|
||||||
dependencies:
|
dependencies:
|
||||||
|
'@n8n/p-retry': 6.2.0-2
|
||||||
'@sentry/node': 7.87.0
|
'@sentry/node': 7.87.0
|
||||||
'@sqltools/formatter': 1.2.5
|
'@sqltools/formatter': 1.2.5
|
||||||
app-root-path: 3.1.0
|
app-root-path: 3.1.0
|
||||||
|
@ -6527,6 +6542,7 @@ packages:
|
||||||
reflect-metadata: 0.2.1
|
reflect-metadata: 0.2.1
|
||||||
sha.js: 2.4.11
|
sha.js: 2.4.11
|
||||||
sqlite3: 5.1.7
|
sqlite3: 5.1.7
|
||||||
|
tarn: 3.0.2
|
||||||
tslib: 2.6.1
|
tslib: 2.6.1
|
||||||
uuid: 9.0.0
|
uuid: 9.0.0
|
||||||
yargs: 17.7.2
|
yargs: 17.7.2
|
||||||
|
@ -6534,8 +6550,8 @@ packages:
|
||||||
- supports-color
|
- supports-color
|
||||||
dev: false
|
dev: false
|
||||||
|
|
||||||
/@n8n/typeorm@0.3.20-3(pg@8.11.3)(redis@4.6.12)(sqlite3@5.1.7):
|
/@n8n/typeorm@0.3.20-7(pg@8.11.3)(redis@4.6.12)(sqlite3@5.1.7):
|
||||||
resolution: {integrity: sha512-eTaEBvaJZZCStqiLKSUIvFLMZJ3WMOFc+pnF+jedcQOc6AunWM57FqhPvnuB/Z6c67JmgIyINlnjbLDHtPgKZQ==}
|
resolution: {integrity: sha512-f4A9RGOnB3kCkusNAr1QDCGOVq1HU1YCBKoIGr2of+P3CVS3I+1vW7neOhlr/ic5S1F14Qy5TU8Lb78mRBYRSw==}
|
||||||
engines: {node: '>=16.13.0'}
|
engines: {node: '>=16.13.0'}
|
||||||
hasBin: true
|
hasBin: true
|
||||||
peerDependencies:
|
peerDependencies:
|
||||||
|
@ -6598,6 +6614,7 @@ packages:
|
||||||
typeorm-aurora-data-api-driver:
|
typeorm-aurora-data-api-driver:
|
||||||
optional: true
|
optional: true
|
||||||
dependencies:
|
dependencies:
|
||||||
|
'@n8n/p-retry': 6.2.0-2
|
||||||
'@sqltools/formatter': 1.2.5
|
'@sqltools/formatter': 1.2.5
|
||||||
app-root-path: 3.1.0
|
app-root-path: 3.1.0
|
||||||
buffer: 6.0.3
|
buffer: 6.0.3
|
||||||
|
@ -6612,6 +6629,7 @@ packages:
|
||||||
reflect-metadata: 0.2.1
|
reflect-metadata: 0.2.1
|
||||||
sha.js: 2.4.11
|
sha.js: 2.4.11
|
||||||
sqlite3: 5.1.7
|
sqlite3: 5.1.7
|
||||||
|
tarn: 3.0.2
|
||||||
tslib: 2.6.1
|
tslib: 2.6.1
|
||||||
uuid: 9.0.0
|
uuid: 9.0.0
|
||||||
yargs: 17.7.2
|
yargs: 17.7.2
|
||||||
|
@ -9703,6 +9721,10 @@ packages:
|
||||||
resolution: {integrity: sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA==}
|
resolution: {integrity: sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA==}
|
||||||
dev: false
|
dev: false
|
||||||
|
|
||||||
|
/@types/retry@0.12.5:
|
||||||
|
resolution: {integrity: sha512-3xSjTp3v03X/lSQLkczaN9UIEwJMoMCA1+Nb5HfbJEQWogdeQIyVtTvxPXDQjZ5zws8rFQfVfRdz03ARihPJgw==}
|
||||||
|
dev: false
|
||||||
|
|
||||||
/@types/rfc2047@2.0.1:
|
/@types/rfc2047@2.0.1:
|
||||||
resolution: {integrity: sha512-slgtykv+XXME7EperkdqfdBBUGcs28ru+a21BK0zOQY4IoxE7tEqvIcvAFAz5DJVxyOmoAUXo30Oxpm3KS+TBQ==}
|
resolution: {integrity: sha512-slgtykv+XXME7EperkdqfdBBUGcs28ru+a21BK0zOQY4IoxE7tEqvIcvAFAz5DJVxyOmoAUXo30Oxpm3KS+TBQ==}
|
||||||
dev: true
|
dev: true
|
||||||
|
|
Loading…
Reference in a new issue