Performance improvements for executions count on Postgres (#1888)

* Performance improvements for executions count on Postgres

As reported by a community member https://community.n8n.io/t/stress-load-testing/4846/5
and https://github.com/n8n-io/n8n/issues/1578, when using postgres with
a big volume of executions, the executions list's performance degrades.

This PR is aimed at Postgres specifically by querying postgres' stats
collector instead of running a full table scan, providing a good
estimate.

More can be read here: https://www.citusdata.com/blog/2016/10/12/count-performance/

* Removed order of magnitude so we display closer numbers

* Making count based on statistics only when not applying filters

*  Minor styling improvements

Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com>
This commit is contained in:
Omar Ajoue 2021-07-06 23:25:25 +02:00 committed by GitHub
parent 297d9fe77d
commit 8d235e94cb
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 50 additions and 9 deletions

View file

@ -188,6 +188,7 @@ export interface IExecutionsListResponse {
count: number;
// results: IExecutionShortResponse[];
results: IExecutionsSummary[];
estimated: boolean;
}
export interface IExecutionsStopData {

View file

@ -33,6 +33,7 @@ import {
CredentialsHelper,
CredentialsOverwrites,
CredentialTypes,
DatabaseType,
Db,
ExternalHooks,
GenericHelpers,
@ -88,6 +89,7 @@ import {
IRunData,
IWorkflowBase,
IWorkflowCredentials,
LoggerProxy,
Workflow,
WorkflowExecuteMode,
} from 'n8n-workflow';
@ -1612,8 +1614,7 @@ class App {
executingWorkflowIds.push(...this.activeExecutionsInstance.getActiveExecutions().map(execution => execution.id.toString()) as string[]);
const countFilter = JSON.parse(JSON.stringify(filter));
countFilter.select = ['id'];
countFilter.where = {id: Not(In(executingWorkflowIds))};
countFilter.id = Not(In(executingWorkflowIds));
const resultsQuery = await Db.collections.Execution!
.createQueryBuilder("execution")
@ -1645,10 +1646,10 @@ class App {
const resultsPromise = resultsQuery.getMany();
const countPromise = Db.collections.Execution!.count(countFilter);
const countPromise = getExecutionsCount(countFilter);
const results: IExecutionFlattedDb[] = await resultsPromise;
const count = await countPromise;
const countedObjects = await countPromise;
const returnResults: IExecutionsSummary[] = [];
@ -1667,8 +1668,9 @@ class App {
}
return {
count,
count: countedObjects.count,
results: returnResults,
estimated: countedObjects.estimate,
};
}));
@ -2161,3 +2163,35 @@ export async function start(): Promise<void> {
await app.externalHooks.run('n8n.ready', [app]);
});
}
async function getExecutionsCount(countFilter: IDataObject): Promise<{ count: number; estimate: boolean; }> {
const dbType = await GenericHelpers.getConfigValue('database.type') as DatabaseType;
const filteredFields = Object.keys(countFilter).filter(field => field !== 'id');
// Do regular count for other databases than pgsql and
// if we are filtering based on workflowId or finished fields.
if (dbType !== 'postgresdb' || filteredFields.length > 0) {
const count = await Db.collections.Execution!.count(countFilter);
return { count, estimate: false };
}
try {
// Get an estimate of rows count.
const estimateRowsNumberSql = "SELECT n_live_tup FROM pg_stat_all_tables WHERE relname = 'execution_entity';";
const rows: Array<{ n_live_tup: string }> = await Db.collections.Execution!.query(estimateRowsNumberSql);
const estimate = parseInt(rows[0].n_live_tup, 10);
// If over 100k, return just an estimate.
if (estimate > 100000) {
// if less than 100k, we get the real count as even a full
// table scan should not take so long.
return { count: estimate, estimate: true };
}
} catch (err) {
LoggerProxy.warn('Unable to get executions count from postgres: ' + err);
}
const count = await Db.collections.Execution!.count(countFilter);
return { count, estimate: false };
}

View file

@ -325,6 +325,7 @@ export interface IExecutionShortResponse {
export interface IExecutionsListResponse {
count: number;
results: IExecutionsSummary[];
estimated: boolean;
}
export interface IExecutionsCurrentSummaryExtended {

View file

@ -1,6 +1,6 @@
<template>
<span>
<el-dialog :visible="dialogVisible" append-to-body width="80%" :title="`Workflow Executions (${combinedExecutions.length}/${combinedExecutionsCount})`" :before-close="closeDialog">
<el-dialog :visible="dialogVisible" append-to-body width="80%" :title="`Workflow Executions ${combinedExecutions.length}/${finishedExecutionsCountEstimated === true ? '~' : ''}${combinedExecutionsCount}`" :before-close="closeDialog">
<div class="filters">
<el-row>
<el-col :span="4" class="filter-headline">
@ -38,7 +38,7 @@
<div class="selection-options">
<span v-if="checkAll === true || isIndeterminate === true">
Selected: {{numSelected}}/{{finishedExecutionsCount}}
Selected: {{numSelected}} / <span v-if="finishedExecutionsCountEstimated === true">~</span>{{finishedExecutionsCount}}
<el-button type="danger" title="Delete Selected" icon="el-icon-delete" size="mini" @click="handleDeleteSelected" circle></el-button>
</span>
</div>
@ -142,7 +142,7 @@
</el-table-column>
</el-table>
<div class="load-more" v-if="finishedExecutionsCount > finishedExecutions.length">
<div class="load-more" v-if="finishedExecutionsCount > finishedExecutions.length || finishedExecutionsCountEstimated === true">
<el-button title="Load More" @click="loadMore()" size="small" :disabled="isDataLoading">
<font-awesome-icon icon="sync" /> Load More
</el-button>
@ -200,6 +200,7 @@ export default mixins(
return {
finishedExecutions: [] as IExecutionsSummary[],
finishedExecutionsCount: 0,
finishedExecutionsCountEstimated: false,
checkAll: false,
autoRefresh: true,
@ -256,7 +257,7 @@ export default mixins(
return returnData;
},
combinedExecutionsCount (): number {
return this.activeExecutions.length + this.finishedExecutionsCount;
return 0 + this.activeExecutions.length + this.finishedExecutionsCount;
},
numSelected (): number {
if (this.checkAll === true) {
@ -489,16 +490,19 @@ export default mixins(
}
this.finishedExecutions = this.finishedExecutions.filter(execution => !gaps.includes(parseInt(execution.id, 10)) && lastId >= parseInt(execution.id, 10));
this.finishedExecutionsCount = results[0].count;
this.finishedExecutionsCountEstimated = results[0].estimated;
},
async loadFinishedExecutions (): Promise<void> {
if (this.filter.status === 'running') {
this.finishedExecutions = [];
this.finishedExecutionsCount = 0;
this.finishedExecutionsCountEstimated = false;
return;
}
const data = await this.restApi().getPastExecutions(this.workflowFilterPast, this.requestItemsPerRequest);
this.finishedExecutions = data.results;
this.finishedExecutionsCount = data.count;
this.finishedExecutionsCountEstimated = data.estimated;
},
async loadMore () {
if (this.filter.status === 'running') {
@ -526,6 +530,7 @@ export default mixins(
this.finishedExecutions.push.apply(this.finishedExecutions, data.results);
this.finishedExecutionsCount = data.count;
this.finishedExecutionsCountEstimated = data.estimated;
this.isDataLoading = false;
},