ci: Improve Test-worklows messages (#6290)

* Update test-worklows action

* Improve error messages

* Fix output data to slack

* Fix slackmessage

* add github flag

* add slack config

* Add result to github output

* Fix output string

* Remove old comment for oauth2

* Remove console.logs

* Improve message

* Update to different slack action

* update payload

* Fix wrong json

* debugging

* Revert "update payload"

This reverts commit 4ea955c85e.

* Reverted to old slack action

* Update slack config

* Remove fallback value

* Change config name

* tests

* more tests

* tests

* Use single message

* message fixes

* Final fixes

* Correct channel

* delete slack.yml config

* keep unused Export credentials step for later

---------

Co-authored-by: Marcus <marcus@n8n.io>
This commit is contained in:
agobrech 2023-06-05 11:20:58 +02:00 committed by GitHub
parent 4162b7b38a
commit 3a1e96dc6d
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 48 additions and 6 deletions

View file

@ -69,8 +69,9 @@ jobs:
shell: bash shell: bash
- name: Run tests - name: Run tests
run: n8n/packages/cli/bin/n8n executeBatch --shallow --skipList=test-workflows/skipList.txt --shortOutput --concurrency=16 --compare=test-workflows/snapshots run: n8n/packages/cli/bin/n8n executeBatch --shallow --skipList=test-workflows/skipList.txt --githubWorkflow --shortOutput --concurrency=16 --compare=test-workflows/snapshots
shell: bash shell: bash
id: tests
env: env:
N8N_ENCRYPTION_KEY: ${{secrets.ENCRYPTION_KEY}} N8N_ENCRYPTION_KEY: ${{secrets.ENCRYPTION_KEY}}
SKIP_STATISTICS_EVENTS: true SKIP_STATISTICS_EVENTS: true
@ -98,4 +99,7 @@ jobs:
status: ${{ job.status }} status: ${{ job.status }}
channel: '#updates-build-alerts' channel: '#updates-build-alerts'
webhook-url: ${{ secrets.SLACK_WEBHOOK_URL }} webhook-url: ${{ secrets.SLACK_WEBHOOK_URL }}
message: Test workflows failed (${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) message: |
🛑 Workflow test failed 🛑:
${{ steps.tests.outputs.slackMessage}}
Sent by *Github Action*: <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}| Test workflow>

View file

@ -1,5 +1,6 @@
interface IResult { interface IResult {
totalWorkflows: number; totalWorkflows: number;
slackMessage: string;
summary: { summary: {
failedExecutions: number; failedExecutions: number;
successfulExecutions: number; successfulExecutions: number;

View file

@ -1,5 +1,6 @@
/* eslint-disable @typescript-eslint/no-loop-func */ /* eslint-disable @typescript-eslint/no-loop-func */
import fs from 'fs'; import fs from 'fs';
import os from 'os';
import { flags } from '@oclif/command'; import { flags } from '@oclif/command';
import type { ITaskData } from 'n8n-workflow'; import type { ITaskData } from 'n8n-workflow';
import { sleep } from 'n8n-workflow'; import { sleep } from 'n8n-workflow';
@ -35,6 +36,8 @@ export class ExecuteBatch extends BaseCommand {
static concurrency = 1; static concurrency = 1;
static githubWorkflow = false;
static debug = false; static debug = false;
static executionTimeout = 3 * 60 * 1000; static executionTimeout = 3 * 60 * 1000;
@ -80,6 +83,12 @@ export class ExecuteBatch extends BaseCommand {
description: description:
'Compares only if attributes output from node are the same, with no regards to nested JSON objects.', 'Compares only if attributes output from node are the same, with no regards to nested JSON objects.',
}), }),
githubWorkflow: flags.boolean({
description:
'Enables more lenient comparison for GitHub workflows. This is useful for reducing false positives when comparing Test workflows.',
}),
skipList: flags.string({ skipList: flags.string({
description: 'File containing a comma separated list of workflow IDs to skip.', description: 'File containing a comma separated list of workflow IDs to skip.',
}), }),
@ -175,7 +184,6 @@ export class ExecuteBatch extends BaseCommand {
async run() { async run() {
// eslint-disable-next-line @typescript-eslint/no-shadow // eslint-disable-next-line @typescript-eslint/no-shadow
const { flags } = this.parse(ExecuteBatch); const { flags } = this.parse(ExecuteBatch);
ExecuteBatch.debug = flags.debug; ExecuteBatch.debug = flags.debug;
ExecuteBatch.concurrency = flags.concurrency || 1; ExecuteBatch.concurrency = flags.concurrency || 1;
@ -251,6 +259,10 @@ export class ExecuteBatch extends BaseCommand {
ExecuteBatch.shallow = true; ExecuteBatch.shallow = true;
} }
if (flags.githubWorkflow) {
ExecuteBatch.githubWorkflow = true;
}
ExecuteBatch.instanceOwner = await getInstanceOwner(); ExecuteBatch.instanceOwner = await getInstanceOwner();
const query = Db.collections.Workflow.createQueryBuilder('workflows'); const query = Db.collections.Workflow.createQueryBuilder('workflows');
@ -369,6 +381,7 @@ export class ExecuteBatch extends BaseCommand {
private async runTests(allWorkflows: IWorkflowDb[]): Promise<IResult> { private async runTests(allWorkflows: IWorkflowDb[]): Promise<IResult> {
const result: IResult = { const result: IResult = {
totalWorkflows: allWorkflows.length, totalWorkflows: allWorkflows.length,
slackMessage: '',
summary: { summary: {
failedExecutions: 0, failedExecutions: 0,
warningExecutions: 0, warningExecutions: 0,
@ -472,11 +485,30 @@ export class ExecuteBatch extends BaseCommand {
} }
await Promise.allSettled(promisesArray); await Promise.allSettled(promisesArray);
if (ExecuteBatch.githubWorkflow) {
if (result.summary.errors.length < 6) {
const errorMessage = result.summary.errors.map((error) => {
return `*${error.workflowId}*: ${error.error}`;
});
result.slackMessage = `*${
result.summary.errors.length
} Executions errors*. Workflows failing: ${errorMessage.join(' ')} `;
} else {
result.slackMessage = `*${result.summary.errors.length} Executions errors*`;
}
this.setOutput('slackMessage', JSON.stringify(result.slackMessage));
}
res(result); res(result);
}); });
} }
setOutput(key: string, value: any) {
// Temporary hack until we move to the new action.
const output = process.env.GITHUB_OUTPUT;
// eslint-disable-next-line @typescript-eslint/restrict-template-expressions
fs.appendFileSync(output as unknown as fs.PathOrFileDescriptor, `${key}=${value}${os.EOL}`);
}
updateStatus() { updateStatus() {
if (ExecuteBatch.cancelled) { if (ExecuteBatch.cancelled) {
return; return;
@ -756,8 +788,13 @@ export class ExecuteBatch extends BaseCommand {
// and search for the `__deleted` string // and search for the `__deleted` string
const changesJson = JSON.stringify(changes); const changesJson = JSON.stringify(changes);
if (changesJson.includes('__deleted')) { if (changesJson.includes('__deleted')) {
if (ExecuteBatch.githubWorkflow) {
const deletedChanges = changesJson.match(/__deleted/g) ?? [];
// we have structural changes. Report them. // we have structural changes. Report them.
executionResult.error = `Workflow contains ${deletedChanges.length} deleted data.`;
} else {
executionResult.error = 'Workflow may contain breaking changes'; executionResult.error = 'Workflow may contain breaking changes';
}
executionResult.changes = changes; executionResult.changes = changes;
executionResult.executionStatus = 'error'; executionResult.executionStatus = 'error';
} else { } else {