ci: Improve Test-worklows messages (#6290)

* Update test-worklows action

* Improve error messages

* Fix output data to slack

* Fix slackmessage

* add github flag

* add slack config

* Add result to github output

* Fix output string

* Remove old comment for oauth2

* Remove console.logs

* Improve message

* Update to different slack action

* update payload

* Fix wrong json

* debugging

* Revert "update payload"

This reverts commit 4ea955c85e.

* Reverted to old slack action

* Update slack config

* Remove fallback value

* Change config name

* tests

* more tests

* tests

* Use single message

* message fixes

* Final fixes

* Correct channel

* delete slack.yml config

* keep unused Export credentials step for later

---------

Co-authored-by: Marcus <marcus@n8n.io>

.github/workflows/test-workflows.yml

@@ -69,8 +69,9 @@ jobs:
         shell: bash
 
       - name: Run tests
-        run: n8n/packages/cli/bin/n8n executeBatch --shallow --skipList=test-workflows/skipList.txt --shortOutput --concurrency=16 --compare=test-workflows/snapshots
+        run: n8n/packages/cli/bin/n8n executeBatch --shallow --skipList=test-workflows/skipList.txt --githubWorkflow --shortOutput --concurrency=16 --compare=test-workflows/snapshots
         shell: bash
+        id: tests
         env:
           N8N_ENCRYPTION_KEY: ${{secrets.ENCRYPTION_KEY}}
           SKIP_STATISTICS_EVENTS: true
@@ -98,4 +99,7 @@ jobs:
           status: ${{ job.status }}
           channel: '#updates-build-alerts'
           webhook-url: ${{ secrets.SLACK_WEBHOOK_URL }}
-          message: Test workflows failed (${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})
+          message: |
+            🛑 Workflow test failed 🛑:
+            ${{ steps.tests.outputs.slackMessage}}
+            Sent by *Github Action*: <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}| Test workflow>

packages/cli/src/commands/Interfaces.d.ts

@@ -1,5 +1,6 @@
 interface IResult {
 	totalWorkflows: number;
+	slackMessage: string;
 	summary: {
 		failedExecutions: number;
 		successfulExecutions: number;

packages/cli/src/commands/executeBatch.ts

@@ -1,5 +1,6 @@
 /* eslint-disable @typescript-eslint/no-loop-func */
 import fs from 'fs';
+import os from 'os';
 import { flags } from '@oclif/command';
 import type { ITaskData } from 'n8n-workflow';
 import { sleep } from 'n8n-workflow';
@@ -35,6 +36,8 @@ export class ExecuteBatch extends BaseCommand {
 
 	static concurrency = 1;
 
+	static githubWorkflow = false;
+
 	static debug = false;
 
 	static executionTimeout = 3 * 60 * 1000;
@@ -80,6 +83,12 @@ export class ExecuteBatch extends BaseCommand {
 			description:
 				'Compares only if attributes output from node are the same, with no regards to nested JSON objects.',
 		}),
+
+		githubWorkflow: flags.boolean({
+			description:
+				'Enables more lenient comparison for GitHub workflows. This is useful for reducing false positives when comparing Test workflows.',
+		}),
+
 		skipList: flags.string({
 			description: 'File containing a comma separated list of workflow IDs to skip.',
 		}),
@@ -175,7 +184,6 @@ export class ExecuteBatch extends BaseCommand {
 	async run() {
 		// eslint-disable-next-line @typescript-eslint/no-shadow
 		const { flags } = this.parse(ExecuteBatch);
-
 		ExecuteBatch.debug = flags.debug;
 		ExecuteBatch.concurrency = flags.concurrency || 1;
 
@@ -251,6 +259,10 @@ export class ExecuteBatch extends BaseCommand {
 			ExecuteBatch.shallow = true;
 		}
 
+		if (flags.githubWorkflow) {
+			ExecuteBatch.githubWorkflow = true;
+		}
+
 		ExecuteBatch.instanceOwner = await getInstanceOwner();
 
 		const query = Db.collections.Workflow.createQueryBuilder('workflows');
@@ -369,6 +381,7 @@ export class ExecuteBatch extends BaseCommand {
 	private async runTests(allWorkflows: IWorkflowDb[]): Promise<IResult> {
 		const result: IResult = {
 			totalWorkflows: allWorkflows.length,
+			slackMessage: '',
 			summary: {
 				failedExecutions: 0,
 				warningExecutions: 0,
@@ -472,11 +485,30 @@ export class ExecuteBatch extends BaseCommand {
 			}
 
 			await Promise.allSettled(promisesArray);
-
+			if (ExecuteBatch.githubWorkflow) {
+				if (result.summary.errors.length < 6) {
+					const errorMessage = result.summary.errors.map((error) => {
+						return `*${error.workflowId}*: ${error.error}`;
+					});
+					result.slackMessage = `*${
+						result.summary.errors.length
+					} Executions errors*. Workflows failing: ${errorMessage.join(' ')} `;
+				} else {
+					result.slackMessage = `*${result.summary.errors.length} Executions errors*`;
+				}
+				this.setOutput('slackMessage', JSON.stringify(result.slackMessage));
+			}
 			res(result);
 		});
 	}
 
+	setOutput(key: string, value: any) {
+		// Temporary hack until we move to the new action.
+		const output = process.env.GITHUB_OUTPUT;
+		// eslint-disable-next-line @typescript-eslint/restrict-template-expressions
+		fs.appendFileSync(output as unknown as fs.PathOrFileDescriptor, `${key}=${value}${os.EOL}`);
+	}
+
 	updateStatus() {
 		if (ExecuteBatch.cancelled) {
 			return;
@@ -756,8 +788,13 @@ export class ExecuteBatch extends BaseCommand {
 									// and search for the `__deleted` string
 									const changesJson = JSON.stringify(changes);
 									if (changesJson.includes('__deleted')) {
-										// we have structural changes. Report them.
+										if (ExecuteBatch.githubWorkflow) {
-										executionResult.error = 'Workflow may contain breaking changes';
+											const deletedChanges = changesJson.match(/__deleted/g) ?? [];
+											// we have structural changes. Report them.
+											executionResult.error = `Workflow contains ${deletedChanges.length} deleted data.`;
+										} else {
+											executionResult.error = 'Workflow may contain breaking changes';
+										}
 										executionResult.changes = changes;
 										executionResult.executionStatus = 'error';
 									} else {