diff --git a/packages/cli/src/commands/Interfaces.d.ts b/packages/cli/src/commands/Interfaces.d.ts index e06bb83abd..765b16d7aa 100644 --- a/packages/cli/src/commands/Interfaces.d.ts +++ b/packages/cli/src/commands/Interfaces.d.ts @@ -1,5 +1,6 @@ interface IResult { totalWorkflows: number; + slackMessage: string; summary: { failedExecutions: number; successfulExecutions: number; diff --git a/packages/cli/src/commands/executeBatch.ts b/packages/cli/src/commands/executeBatch.ts index f0c60bf20f..0c13f6e000 100644 --- a/packages/cli/src/commands/executeBatch.ts +++ b/packages/cli/src/commands/executeBatch.ts @@ -35,6 +35,8 @@ export class ExecuteBatch extends BaseCommand { static concurrency = 1; + static githubWorkflow = false; + static debug = false; static executionTimeout = 3 * 60 * 1000; @@ -80,6 +82,12 @@ export class ExecuteBatch extends BaseCommand { description: 'Compares only if attributes output from node are the same, with no regards to nested JSON objects.', }), + + githubWorkflow: flags.boolean({ + description: + 'Enables more lenient comparison for GitHub workflows. This is useful for reducing false positives when comparing Test workflows.', + }), + skipList: flags.string({ description: 'File containing a comma separated list of workflow IDs to skip.', }), @@ -175,7 +183,6 @@ export class ExecuteBatch extends BaseCommand { async run() { // eslint-disable-next-line @typescript-eslint/no-shadow const { flags } = this.parse(ExecuteBatch); - ExecuteBatch.debug = flags.debug; ExecuteBatch.concurrency = flags.concurrency || 1; @@ -251,6 +258,10 @@ export class ExecuteBatch extends BaseCommand { ExecuteBatch.shallow = true; } + if (flags.githubWorkflow) { + ExecuteBatch.githubWorkflow = true; + } + ExecuteBatch.instanceOwner = await getInstanceOwner(); const query = Db.collections.Workflow.createQueryBuilder('workflows'); @@ -369,6 +380,7 @@ export class ExecuteBatch extends BaseCommand { private async runTests(allWorkflows: IWorkflowDb[]): Promise { const result: IResult = { totalWorkflows: allWorkflows.length, + slackMessage: '', summary: { failedExecutions: 0, warningExecutions: 0, @@ -472,7 +484,18 @@ export class ExecuteBatch extends BaseCommand { } await Promise.allSettled(promisesArray); - + if (ExecuteBatch.githubWorkflow) { + if (result.summary.errors.length < 6) { + const errorMessage = result.summary.errors.map((error) => { + return `*${error.workflowId}*: ${error.error} \n`; + }); + result.slackMessage = `*${ + result.summary.errors.length + } Executions errors*. Workflows failing: \n ${errorMessage.join('\n')} `; + } else { + result.slackMessage = `*${result.summary.errors.length} Executions errors*`; + } + } res(result); }); } @@ -756,8 +779,13 @@ export class ExecuteBatch extends BaseCommand { // and search for the `__deleted` string const changesJson = JSON.stringify(changes); if (changesJson.includes('__deleted')) { - // we have structural changes. Report them. - executionResult.error = 'Workflow may contain breaking changes'; + if (ExecuteBatch.githubWorkflow) { + const deletedChanges = changesJson.match(/__deleted/g) ?? []; + // we have structural changes. Report them. + executionResult.error = `Workflow contains ${deletedChanges.length} deleted data.`; + } else { + executionResult.error = 'Workflow may contain breaking changes'; + } executionResult.changes = changes; executionResult.executionStatus = 'error'; } else {