Improve error messages

packages/cli/src/commands/Interfaces.d.ts

@@ -1,5 +1,6 @@
 interface IResult {
 	totalWorkflows: number;
+	slackMessage: string;
 	summary: {
 		failedExecutions: number;
 		successfulExecutions: number;

packages/cli/src/commands/executeBatch.ts

@@ -35,6 +35,8 @@ export class ExecuteBatch extends BaseCommand {
 
 	static concurrency = 1;
 
+	static githubWorkflow = false;
+
 	static debug = false;
 
 	static executionTimeout = 3 * 60 * 1000;
@@ -80,6 +82,12 @@ export class ExecuteBatch extends BaseCommand {
 			description:
 				'Compares only if attributes output from node are the same, with no regards to nested JSON objects.',
 		}),
+
+		githubWorkflow: flags.boolean({
+			description:
+				'Enables more lenient comparison for GitHub workflows. This is useful for reducing false positives when comparing Test workflows.',
+		}),
+
 		skipList: flags.string({
 			description: 'File containing a comma separated list of workflow IDs to skip.',
 		}),
@@ -175,7 +183,6 @@ export class ExecuteBatch extends BaseCommand {
 	async run() {
 		// eslint-disable-next-line @typescript-eslint/no-shadow
 		const { flags } = this.parse(ExecuteBatch);
-
 		ExecuteBatch.debug = flags.debug;
 		ExecuteBatch.concurrency = flags.concurrency || 1;
 
@@ -251,6 +258,10 @@ export class ExecuteBatch extends BaseCommand {
 			ExecuteBatch.shallow = true;
 		}
 
+		if (flags.githubWorkflow) {
+			ExecuteBatch.githubWorkflow = true;
+		}
+
 		ExecuteBatch.instanceOwner = await getInstanceOwner();
 
 		const query = Db.collections.Workflow.createQueryBuilder('workflows');
@@ -369,6 +380,7 @@ export class ExecuteBatch extends BaseCommand {
 	private async runTests(allWorkflows: IWorkflowDb[]): Promise<IResult> {
 		const result: IResult = {
 			totalWorkflows: allWorkflows.length,
+			slackMessage: '',
 			summary: {
 				failedExecutions: 0,
 				warningExecutions: 0,
@@ -472,7 +484,18 @@ export class ExecuteBatch extends BaseCommand {
 			}
 
 			await Promise.allSettled(promisesArray);
-
+			if (ExecuteBatch.githubWorkflow) {
+				if (result.summary.errors.length < 6) {
+					const errorMessage = result.summary.errors.map((error) => {
+						return `*${error.workflowId}*: ${error.error} \n`;
+					});
+					result.slackMessage = `*${
+						result.summary.errors.length
+					} Executions errors*. Workflows failing: \n ${errorMessage.join('\n')} `;
+				} else {
+					result.slackMessage = `*${result.summary.errors.length} Executions errors*`;
+				}
+			}
 			res(result);
 		});
 	}
@@ -756,8 +779,13 @@ export class ExecuteBatch extends BaseCommand {
 									// and search for the `__deleted` string
 									const changesJson = JSON.stringify(changes);
 									if (changesJson.includes('__deleted')) {
+										if (ExecuteBatch.githubWorkflow) {
+											const deletedChanges = changesJson.match(/__deleted/g) ?? [];
 											// we have structural changes. Report them.
+											executionResult.error = `Workflow contains ${deletedChanges.length} deleted data.`;
+										} else {
 											executionResult.error = 'Workflow may contain breaking changes';
+										}
 										executionResult.changes = changes;
 										executionResult.executionStatus = 'error';
 									} else {