From 8d235e94cb79f5c0c542b68e5a0c3718bd094e45 Mon Sep 17 00:00:00 2001 From: Omar Ajoue Date: Tue, 6 Jul 2021 23:25:25 +0200 Subject: [PATCH] :zap: Performance improvements for executions count on Postgres (#1888) * Performance improvements for executions count on Postgres As reported by a community member https://community.n8n.io/t/stress-load-testing/4846/5 and https://github.com/n8n-io/n8n/issues/1578, when using postgres with a big volume of executions, the executions list's performance degrades. This PR is aimed at Postgres specifically by querying postgres' stats collector instead of running a full table scan, providing a good estimate. More can be read here: https://www.citusdata.com/blog/2016/10/12/count-performance/ * Removed order of magnitude so we display closer numbers * Making count based on statistics only when not applying filters * :zap: Minor styling improvements Co-authored-by: Jan Oberhauser --- packages/cli/src/Interfaces.ts | 1 + packages/cli/src/Server.ts | 44 ++++++++++++++++--- packages/editor-ui/src/Interface.ts | 1 + .../src/components/ExecutionsList.vue | 13 ++++-- 4 files changed, 50 insertions(+), 9 deletions(-) diff --git a/packages/cli/src/Interfaces.ts b/packages/cli/src/Interfaces.ts index df965c50e7..e1c09cf55e 100644 --- a/packages/cli/src/Interfaces.ts +++ b/packages/cli/src/Interfaces.ts @@ -188,6 +188,7 @@ export interface IExecutionsListResponse { count: number; // results: IExecutionShortResponse[]; results: IExecutionsSummary[]; + estimated: boolean; } export interface IExecutionsStopData { diff --git a/packages/cli/src/Server.ts b/packages/cli/src/Server.ts index 0a31144999..ee2f50f2cf 100644 --- a/packages/cli/src/Server.ts +++ b/packages/cli/src/Server.ts @@ -33,6 +33,7 @@ import { CredentialsHelper, CredentialsOverwrites, CredentialTypes, + DatabaseType, Db, ExternalHooks, GenericHelpers, @@ -88,6 +89,7 @@ import { IRunData, IWorkflowBase, IWorkflowCredentials, + LoggerProxy, Workflow, WorkflowExecuteMode, } from 'n8n-workflow'; @@ -1612,8 +1614,7 @@ class App { executingWorkflowIds.push(...this.activeExecutionsInstance.getActiveExecutions().map(execution => execution.id.toString()) as string[]); const countFilter = JSON.parse(JSON.stringify(filter)); - countFilter.select = ['id']; - countFilter.where = {id: Not(In(executingWorkflowIds))}; + countFilter.id = Not(In(executingWorkflowIds)); const resultsQuery = await Db.collections.Execution! .createQueryBuilder("execution") @@ -1645,10 +1646,10 @@ class App { const resultsPromise = resultsQuery.getMany(); - const countPromise = Db.collections.Execution!.count(countFilter); + const countPromise = getExecutionsCount(countFilter); const results: IExecutionFlattedDb[] = await resultsPromise; - const count = await countPromise; + const countedObjects = await countPromise; const returnResults: IExecutionsSummary[] = []; @@ -1667,8 +1668,9 @@ class App { } return { - count, + count: countedObjects.count, results: returnResults, + estimated: countedObjects.estimate, }; })); @@ -2161,3 +2163,35 @@ export async function start(): Promise { await app.externalHooks.run('n8n.ready', [app]); }); } + +async function getExecutionsCount(countFilter: IDataObject): Promise<{ count: number; estimate: boolean; }> { + + const dbType = await GenericHelpers.getConfigValue('database.type') as DatabaseType; + const filteredFields = Object.keys(countFilter).filter(field => field !== 'id'); + + // Do regular count for other databases than pgsql and + // if we are filtering based on workflowId or finished fields. + if (dbType !== 'postgresdb' || filteredFields.length > 0) { + const count = await Db.collections.Execution!.count(countFilter); + return { count, estimate: false }; + } + + try { + // Get an estimate of rows count. + const estimateRowsNumberSql = "SELECT n_live_tup FROM pg_stat_all_tables WHERE relname = 'execution_entity';"; + const rows: Array<{ n_live_tup: string }> = await Db.collections.Execution!.query(estimateRowsNumberSql); + + const estimate = parseInt(rows[0].n_live_tup, 10); + // If over 100k, return just an estimate. + if (estimate > 100000) { + // if less than 100k, we get the real count as even a full + // table scan should not take so long. + return { count: estimate, estimate: true }; + } + } catch (err) { + LoggerProxy.warn('Unable to get executions count from postgres: ' + err); + } + + const count = await Db.collections.Execution!.count(countFilter); + return { count, estimate: false }; +} diff --git a/packages/editor-ui/src/Interface.ts b/packages/editor-ui/src/Interface.ts index f119699f6e..2fa997a545 100644 --- a/packages/editor-ui/src/Interface.ts +++ b/packages/editor-ui/src/Interface.ts @@ -325,6 +325,7 @@ export interface IExecutionShortResponse { export interface IExecutionsListResponse { count: number; results: IExecutionsSummary[]; + estimated: boolean; } export interface IExecutionsCurrentSummaryExtended { diff --git a/packages/editor-ui/src/components/ExecutionsList.vue b/packages/editor-ui/src/components/ExecutionsList.vue index e0e129a737..1ecfc4caf8 100644 --- a/packages/editor-ui/src/components/ExecutionsList.vue +++ b/packages/editor-ui/src/components/ExecutionsList.vue @@ -1,6 +1,6 @@