perf: Improve workflows list performance (#5021)

* spike: Improve workflow list performance

* fix: Correcting override behavior

* refactor: Remove unnecessary promise

* remove duplicate code

* remove the `async` that is breaking the listings page

Co-authored-by: कारतोफ्फेलस्क्रिप्ट™ <aditya@netroy.in>

packages/cli/src/credentials/credentials.service.ts

@@ -125,17 +125,6 @@ export class CredentialsService {
 		return Db.collections.SharedCredentials.findOne(options);
 	}
 
-	static createCredentialsFromCredentialsEntity(
-		credential: CredentialsEntity,
-		encrypt = false,
-	): Credentials {
-		const { id, name, type, nodesAccess, data } = credential;
-		if (encrypt) {
-			return new Credentials({ id: null, name }, type, nodesAccess);
-		}
-		return new Credentials({ id: id.toString(), name }, type, nodesAccess, data);
-	}
-
 	static async prepareCreateData(
 		data: CredentialRequest.CredentialProperties,
 	): Promise<CredentialsEntity> {

packages/cli/src/workflows/workflows.controller.ee.ts

@@ -108,7 +108,7 @@ EEWorkflowController.get(
 
 		EEWorkflows.addOwnerAndSharings(workflow);
 		await EEWorkflows.addCredentialsToWorkflow(workflow, req.user);
-		return { ...workflow, id: workflow.id.toString() };
+		return EEWorkflows.entityToResponse(workflow);
 	}),
 );
 
@@ -189,12 +189,7 @@ EEWorkflowController.post(
 		await ExternalHooks().run('workflow.afterCreate', [savedWorkflow]);
 		void InternalHooksManager.getInstance().onWorkflowCreated(req.user.id, newWorkflow, false);
 
-		const { id, ...rest } = savedWorkflow;
+		return EEWorkflows.entityToResponse(savedWorkflow);
-
-		return {
-			id: id.toString(),
-			...rest,
-		};
 	}),
 );
 
@@ -204,19 +199,14 @@ EEWorkflowController.post(
 EEWorkflowController.get(
 	'/',
 	ResponseHelper.send(async (req: WorkflowRequest.GetAll) => {
-		const workflows = (await EEWorkflows.getMany(
+		const workflows = await EEWorkflows.getMany(req.user, req.query.filter);
-			req.user,
+		await EEWorkflows.addCredentialsToWorkflows(workflows, req.user);
-			req.query.filter,
-		)) as unknown as WorkflowEntity[];
 
-		return Promise.all(
+		return workflows.map((workflow) => {
-			workflows.map(async (workflow) => {
+			EEWorkflows.addOwnerAndSharings(workflow);
-				EEWorkflows.addOwnerAndSharings(workflow);
+			workflow.nodes = [];
-				await EEWorkflows.addCredentialsToWorkflow(workflow, req.user);
+			return EEWorkflows.entityToResponse(workflow);
-				workflow.nodes = [];
+		});
-				return { ...workflow, id: workflow.id.toString() };
-			}),
-		);
 	}),
 );
 
@@ -240,12 +230,7 @@ EEWorkflowController.patch(
 			forceSave,
 		);
 
-		const { id, ...remainder } = updatedWorkflow;
+		return EEWorkflows.entityToResponse(updatedWorkflow);
-
-		return {
-			id: id.toString(),
-			...remainder,
-		};
 	}),
 );
 

packages/cli/src/workflows/workflows.controller.ts

@@ -106,12 +106,7 @@ workflowsController.post(
 		await ExternalHooks().run('workflow.afterCreate', [savedWorkflow]);
 		void InternalHooksManager.getInstance().onWorkflowCreated(req.user.id, newWorkflow, false);
 
-		const { id, ...rest } = savedWorkflow;
+		return WorkflowsService.entityToResponse(savedWorkflow);
-
-		return {
-			id: id.toString(),
-			...rest,
-		};
 	}),
 );
 
@@ -121,7 +116,8 @@ workflowsController.post(
 workflowsController.get(
 	'/',
 	ResponseHelper.send(async (req: WorkflowRequest.GetAll) => {
-		return WorkflowsService.getMany(req.user, req.query.filter);
+		const workflows = await WorkflowsService.getMany(req.user, req.query.filter);
+		return workflows.map((workflow) => WorkflowsService.entityToResponse(workflow));
 	}),
 );
 
@@ -222,14 +218,7 @@ workflowsController.get(
 			);
 		}
 
-		const {
+		return WorkflowsService.entityToResponse(shared.workflow);
-			workflow: { id, ...rest },
-		} = shared;
-
-		return {
-			id: id.toString(),
-			...rest,
-		};
 	}),
 );
 
@@ -255,12 +244,7 @@ workflowsController.patch(
 			['owner'],
 		);
 
-		const { id, ...remainder } = updatedWorkflow;
+		return WorkflowsService.entityToResponse(updatedWorkflow);
-
-		return {
-			id: id.toString(),
-			...remainder,
-		};
 	}),
 );
 

packages/cli/src/workflows/workflows.services.ee.ts

@@ -89,7 +89,9 @@ export class EEWorkflowsService extends WorkflowsService {
 	static addOwnerAndSharings(workflow: WorkflowWithSharingsAndCredentials): void {
 		workflow.ownedBy = null;
 		workflow.sharedWith = [];
-		workflow.usedCredentials = [];
+		if (!workflow.usedCredentials) {
+			workflow.usedCredentials = [];
+		}
 
 		workflow.shared?.forEach(({ user, role }) => {
 			const { id, email, firstName, lastName } = user;
@@ -154,6 +156,71 @@ export class EEWorkflowsService extends WorkflowsService {
 		});
 	}
 
+	static async addCredentialsToWorkflows(
+		workflows: WorkflowWithSharingsAndCredentials[],
+		currentUser: User,
+	): Promise<void> {
+		// Create 2 maps: one with all the credential ids used by all workflows
+		// And another to match back workflow <> credentials
+		const allUsedCredentialIds = new Set<string>();
+		const mapsWorkflowsToUsedCredentials: string[][] = [];
+		workflows.forEach((workflow, idx) => {
+			workflow.nodes.forEach((node) => {
+				if (!node.credentials) {
+					return;
+				}
+				Object.keys(node.credentials).forEach((credentialType) => {
+					const credential = node.credentials?.[credentialType];
+					if (!credential?.id) {
+						return;
+					}
+					if (!mapsWorkflowsToUsedCredentials[idx]) {
+						mapsWorkflowsToUsedCredentials[idx] = [];
+					}
+					mapsWorkflowsToUsedCredentials[idx].push(credential.id);
+					allUsedCredentialIds.add(credential.id);
+				});
+			});
+		});
+
+		const usedWorkflowsCredentials = await EECredentials.getMany({
+			where: {
+				id: In(Array.from(allUsedCredentialIds)),
+			},
+			relations: ['shared', 'shared.user', 'shared.role'],
+		});
+		const userCredentials = await EECredentials.getAll(currentUser, { disableGlobalRole: true });
+		const userCredentialIds = userCredentials.map((credential) => credential.id.toString());
+		const credentialsMap: Record<string, CredentialUsedByWorkflow> = {};
+		usedWorkflowsCredentials.forEach((credential) => {
+			credentialsMap[credential.id.toString()] = {
+				id: credential.id.toString(),
+				name: credential.name,
+				type: credential.type,
+				currentUserHasAccess: userCredentialIds.includes(credential.id.toString()),
+				sharedWith: [],
+				ownedBy: null,
+			};
+			credential.shared?.forEach(({ user, role }) => {
+				const { id, email, firstName, lastName } = user;
+				if (role.name === 'owner') {
+					credentialsMap[credential.id.toString()].ownedBy = { id, email, firstName, lastName };
+				} else {
+					credentialsMap[credential.id.toString()].sharedWith?.push({
+						id,
+						email,
+						firstName,
+						lastName,
+					});
+				}
+			});
+		});
+
+		mapsWorkflowsToUsedCredentials.forEach((usedCredentialIds, idx) => {
+			workflows[idx].usedCredentials = usedCredentialIds.map((id) => credentialsMap[id]);
+		});
+	}
+
 	static validateCredentialPermissionsToUser(
 		workflow: WorkflowEntity,
 		allowedCredentials: ICredentialsDb[],

packages/cli/src/workflows/workflows.services.ts

@@ -16,7 +16,7 @@ import { validateEntity } from '@/GenericHelpers';
 import { ExternalHooks } from '@/ExternalHooks';
 import * as TagHelpers from '@/TagHelpers';
 import { WorkflowRequest } from '@/requests';
-import { IWorkflowDb, IWorkflowExecutionDataProcess } from '@/Interfaces';
+import { IWorkflowDb, IWorkflowExecutionDataProcess, IWorkflowResponse } from '@/Interfaces';
 import { NodeTypes } from '@/NodeTypes';
 import { WorkflowRunner } from '@/WorkflowRunner';
 import * as WorkflowExecuteAdditionalData from '@/WorkflowExecuteAdditionalData';
@@ -115,12 +115,17 @@ export class WorkflowsService {
 		return Db.collections.Workflow.findOne(workflow, options);
 	}
 
-	// Warning: this function is overriden by EE to disregard role list.
+	// Warning: this function is overridden by EE to disregard role list.
 	static async getWorkflowIdsForUser(user: User, roles?: string[]): Promise<string[]> {
 		return getSharedWorkflowIds(user, roles);
 	}
 
-	static async getMany(user: User, rawFilter: string) {
+	static entityToResponse(entity: WorkflowEntity): IWorkflowResponse {
+		const { id, ...rest } = entity;
+		return { ...rest, id: id.toString() };
+	}
+
+	static async getMany(user: User, rawFilter: string): Promise<WorkflowEntity[]> {
 		const sharedWorkflowIds = await this.getWorkflowIdsForUser(user, ['owner']);
 		if (sharedWorkflowIds.length === 0) {
 			// return early since without shared workflows there can be no hits
@@ -185,16 +190,7 @@ export class WorkflowsService {
 			},
 		};
 
-		const workflows = await Db.collections.Workflow.find(query);
+		return Db.collections.Workflow.find(query);
-
-		return workflows.map((workflow) => {
-			const { id, ...rest } = workflow;
-
-			return {
-				id: id.toString(),
-				...rest,
-			};
-		});
 	}
 
 	static async update(