⚡ Performance improvements for executions count on Postgres (#1888)

* Performance improvements for executions count on Postgres

As reported by a community member https://community.n8n.io/t/stress-load-testing/4846/5
and https://github.com/n8n-io/n8n/issues/1578, when using postgres with
a big volume of executions, the executions list's performance degrades.

This PR is aimed at Postgres specifically by querying postgres' stats
collector instead of running a full table scan, providing a good
estimate.

More can be read here: https://www.citusdata.com/blog/2016/10/12/count-performance/

* Removed order of magnitude so we display closer numbers

* Making count based on statistics only when not applying filters

* ⚡ Minor styling improvements

Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>

packages/cli/src/Interfaces.ts

@@ -188,6 +188,7 @@ export interface IExecutionsListResponse {
 	count: number;
 	// results: IExecutionShortResponse[];
 	results: IExecutionsSummary[];
+	estimated: boolean;
 }
 
 export interface IExecutionsStopData {

packages/cli/src/Server.ts

@@ -33,6 +33,7 @@ import {
 	CredentialsHelper,
 	CredentialsOverwrites,
 	CredentialTypes,
+	DatabaseType,
 	Db,
 	ExternalHooks,
 	GenericHelpers,
@@ -88,6 +89,7 @@ import {
 	IRunData,
 	IWorkflowBase,
 	IWorkflowCredentials,
+	LoggerProxy,
 	Workflow,
 	WorkflowExecuteMode,
 } from 'n8n-workflow';
@@ -1612,8 +1614,7 @@ class App {
 			executingWorkflowIds.push(...this.activeExecutionsInstance.getActiveExecutions().map(execution => execution.id.toString()) as string[]);
 
 			const countFilter = JSON.parse(JSON.stringify(filter));
-			countFilter.select = ['id'];
+			countFilter.id = Not(In(executingWorkflowIds));
-			countFilter.where = {id: Not(In(executingWorkflowIds))};
 
 			const resultsQuery = await Db.collections.Execution!
 				.createQueryBuilder("execution")
@@ -1645,10 +1646,10 @@ class App {
 
 			const resultsPromise = resultsQuery.getMany();
 
-			const countPromise = Db.collections.Execution!.count(countFilter);
+			const countPromise = getExecutionsCount(countFilter);
 
 			const results: IExecutionFlattedDb[] = await resultsPromise;
-			const count = await countPromise;
+			const countedObjects = await countPromise;
 
 			const returnResults: IExecutionsSummary[] = [];
 
@@ -1667,8 +1668,9 @@ class App {
 			}
 
 			return {
-				count,
+				count: countedObjects.count,
 				results: returnResults,
+				estimated: countedObjects.estimate,
 			};
 		}));
 
@@ -2161,3 +2163,35 @@ export async function start(): Promise<void> {
 		await app.externalHooks.run('n8n.ready', [app]);
 	});
 }
+
+async function getExecutionsCount(countFilter: IDataObject): Promise<{ count: number; estimate: boolean; }> {
+
+	const dbType = await GenericHelpers.getConfigValue('database.type') as DatabaseType;
+	const filteredFields = Object.keys(countFilter).filter(field => field !== 'id');
+
+	// Do regular count for other databases than pgsql and
+	// if we are filtering based on workflowId or finished fields.
+	if (dbType !== 'postgresdb' || filteredFields.length > 0) {
+		const count = await Db.collections.Execution!.count(countFilter);
+		return { count, estimate: false };
+	}
+
+	try {
+		// Get an estimate of rows count.
+		const estimateRowsNumberSql = "SELECT n_live_tup FROM pg_stat_all_tables WHERE relname = 'execution_entity';";
+		const rows: Array<{ n_live_tup: string }> = await Db.collections.Execution!.query(estimateRowsNumberSql);
+
+		const estimate = parseInt(rows[0].n_live_tup, 10);
+		// If over 100k, return just an estimate.
+		if (estimate > 100000) {
+			// if less than 100k, we get the real count as even a full
+			// table scan should not take so long.
+			return { count: estimate, estimate: true };
+		}
+	} catch (err) {
+		LoggerProxy.warn('Unable to get executions count from postgres: ' + err);
+	}
+
+	const count = await Db.collections.Execution!.count(countFilter);
+	return { count, estimate: false };
+}

packages/editor-ui/src/Interface.ts

@@ -325,6 +325,7 @@ export interface IExecutionShortResponse {
 export interface IExecutionsListResponse {
 	count: number;
 	results: IExecutionsSummary[];
+	estimated: boolean;
 }
 
 export interface IExecutionsCurrentSummaryExtended {

packages/editor-ui/src/components/ExecutionsList.vue

@@ -1,6 +1,6 @@
 <template>
 	<span>
-		<el-dialog :visible="dialogVisible" append-to-body width="80%" :title="`Workflow Executions (${combinedExecutions.length}/${combinedExecutionsCount})`" :before-close="closeDialog">
+		<el-dialog :visible="dialogVisible" append-to-body width="80%" :title="`Workflow Executions ${combinedExecutions.length}/${finishedExecutionsCountEstimated === true ? '~' : ''}${combinedExecutionsCount}`" :before-close="closeDialog">
 			<div class="filters">
 				<el-row>
 					<el-col :span="4" class="filter-headline">
@@ -38,7 +38,7 @@
 
 			<div class="selection-options">
 				<span v-if="checkAll === true || isIndeterminate === true">
-					Selected: {{numSelected}}/{{finishedExecutionsCount}}
+					Selected: {{numSelected}} / <span v-if="finishedExecutionsCountEstimated === true">~</span>{{finishedExecutionsCount}}
 					<el-button type="danger" title="Delete Selected" icon="el-icon-delete" size="mini" @click="handleDeleteSelected" circle></el-button>
 				</span>
 			</div>
@@ -142,7 +142,7 @@
 				</el-table-column>
 			</el-table>
 
-			<div class="load-more" v-if="finishedExecutionsCount > finishedExecutions.length">
+			<div class="load-more" v-if="finishedExecutionsCount > finishedExecutions.length || finishedExecutionsCountEstimated === true">
 				<el-button title="Load More" @click="loadMore()" size="small" :disabled="isDataLoading">
 					<font-awesome-icon icon="sync" /> Load More
 				</el-button>
@@ -200,6 +200,7 @@ export default mixins(
 		return {
 			finishedExecutions: [] as IExecutionsSummary[],
 			finishedExecutionsCount: 0,
+			finishedExecutionsCountEstimated: false,
 
 			checkAll: false,
 			autoRefresh: true,
@@ -256,7 +257,7 @@ export default mixins(
 			return returnData;
 		},
 		combinedExecutionsCount (): number {
-			return this.activeExecutions.length + this.finishedExecutionsCount;
+			return 0 + this.activeExecutions.length + this.finishedExecutionsCount;
 		},
 		numSelected (): number {
 			if (this.checkAll === true) {
@@ -489,16 +490,19 @@ export default mixins(
 			}
 			this.finishedExecutions = this.finishedExecutions.filter(execution => !gaps.includes(parseInt(execution.id, 10)) && lastId >= parseInt(execution.id, 10));
 			this.finishedExecutionsCount = results[0].count;
+			this.finishedExecutionsCountEstimated = results[0].estimated;
 		},
 		async loadFinishedExecutions (): Promise<void> {
 			if (this.filter.status === 'running') {
 				this.finishedExecutions = [];
 				this.finishedExecutionsCount = 0;
+				this.finishedExecutionsCountEstimated = false;
 				return;
 			}
 			const data = await this.restApi().getPastExecutions(this.workflowFilterPast, this.requestItemsPerRequest);
 			this.finishedExecutions = data.results;
 			this.finishedExecutionsCount = data.count;
+			this.finishedExecutionsCountEstimated = data.estimated;
 		},
 		async loadMore () {
 			if (this.filter.status === 'running') {
@@ -526,6 +530,7 @@ export default mixins(
 
 			this.finishedExecutions.push.apply(this.finishedExecutions, data.results);
 			this.finishedExecutionsCount = data.count;
+			this.finishedExecutionsCountEstimated = data.estimated;
 
 			this.isDataLoading = false;
 		},