mirror of
https://github.com/n8n-io/n8n.git
synced 2025-03-05 20:50:17 -08:00
feat(core): Keep track of test case executions during test run (no-changelog) (#12787)
This commit is contained in:
Eugene
GitHub
parent
d9d64083d3
commit
1ca6a9799a
|
|
@ -20,6 +20,7 @@ import { Settings } from './settings';
|
|||
import { SharedCredentials } from './shared-credentials';
|
||||
import { SharedWorkflow } from './shared-workflow';
|
||||
import { TagEntity } from './tag-entity';
|
||||
import { TestCaseExecution } from './test-case-execution.ee';
|
||||
import { TestDefinition } from './test-definition.ee';
|
||||
import { TestMetric } from './test-metric.ee';
|
||||
import { TestRun } from './test-run.ee';
|
||||
|
|
@ -64,4 +65,5 @@ export const entities = {
|
|||
TestDefinition,
|
||||
TestMetric,
|
||||
TestRun,
|
||||
TestCaseExecution,
|
||||
};
|
||||
|
|
|
|||
|
|
@ -0,0 +1,68 @@
|
|||
import { Column, Entity, ManyToOne, OneToOne } from '@n8n/typeorm';
|
||||
|
||||
import {
|
||||
datetimeColumnType,
|
||||
jsonColumnType,
|
||||
WithStringId,
|
||||
} from '@/databases/entities/abstract-entity';
|
||||
import type { ExecutionEntity } from '@/databases/entities/execution-entity';
|
||||
import { TestRun } from '@/databases/entities/test-run.ee';
|
||||
|
||||
export type TestCaseRunMetrics = Record<string, number | boolean>;
|
||||
|
||||
/**
|
||||
* This entity represents the linking between the test runs and individual executions.
|
||||
* It stores status, links to past, new and evaluation executions, and metrics produced by individual evaluation wf executions
|
||||
* Entries in this table are meant to outlive the execution entities, which might be pruned over time.
|
||||
* This allows us to keep track of the details of test runs' status and metrics even after the executions are deleted.
|
||||
*/
|
||||
@Entity({ name: 'test_case_execution' })
|
||||
export class TestCaseExecution extends WithStringId {
|
||||
@ManyToOne('TestRun')
|
||||
testRun: TestRun;
|
||||
|
||||
@ManyToOne('ExecutionEntity', {
|
||||
onDelete: 'SET NULL',
|
||||
nullable: true,
|
||||
})
|
||||
pastExecution: ExecutionEntity | null;
|
||||
|
||||
@Column({ type: 'varchar', nullable: true })
|
||||
pastExecutionId: string | null;
|
||||
|
||||
@OneToOne('ExecutionEntity', {
|
||||
onDelete: 'SET NULL',
|
||||
nullable: true,
|
||||
})
|
||||
execution: ExecutionEntity | null;
|
||||
|
||||
@Column({ type: 'varchar', nullable: true })
|
||||
executionId: string | null;
|
||||
|
||||
@OneToOne('ExecutionEntity', {
|
||||
onDelete: 'SET NULL',
|
||||
nullable: true,
|
||||
})
|
||||
evaluationExecution: ExecutionEntity | null;
|
||||
|
||||
@Column({ type: 'varchar', nullable: true })
|
||||
evaluationExecutionId: string | null;
|
||||
|
||||
@Column()
|
||||
status: 'new' | 'running' | 'evaluation_running' | 'success' | 'error' | 'cancelled';
|
||||
|
||||
@Column({ type: datetimeColumnType, nullable: true })
|
||||
runAt: Date | null;
|
||||
|
||||
@Column({ type: datetimeColumnType, nullable: true })
|
||||
completedAt: Date | null;
|
||||
|
||||
@Column('varchar', { nullable: true })
|
||||
errorCode: string | null;
|
||||
|
||||
@Column(jsonColumnType, { nullable: true })
|
||||
errorDetails: Record<string, unknown>;
|
||||
|
||||
@Column(jsonColumnType, { nullable: true })
|
||||
metrics: TestCaseRunMetrics;
|
||||
}
|
||||
|
|
@ -0,0 +1,47 @@
|
|||
import type { MigrationContext, ReversibleMigration } from '@/databases/types';
|
||||
|
||||
const testCaseExecutionTableName = 'test_case_execution';
|
||||
|
||||
export class CreateTestCaseExecutionTable1736947513045 implements ReversibleMigration {
|
||||
async up({ schemaBuilder: { createTable, column } }: MigrationContext) {
|
||||
await createTable(testCaseExecutionTableName)
|
||||
.withColumns(
|
||||
column('id').varchar(36).primary.notNull,
|
||||
column('testRunId').varchar(36).notNull,
|
||||
column('pastExecutionId').int, // Might be null if execution was deleted after the test run
|
||||
column('executionId').int, // Execution of the workflow under test. Might be null if execution was deleted after the test run
|
||||
column('evaluationExecutionId').int, // Execution of the evaluation workflow. Might be null if execution was deleted after the test run, or if the test run was cancelled
|
||||
column('status').varchar().notNull,
|
||||
column('runAt').timestamp(),
|
||||
column('completedAt').timestamp(),
|
||||
column('errorCode').varchar(),
|
||||
column('errorDetails').json,
|
||||
column('metrics').json,
|
||||
)
|
||||
.withIndexOn('testRunId')
|
||||
.withForeignKey('testRunId', {
|
||||
tableName: 'test_run',
|
||||
columnName: 'id',
|
||||
onDelete: 'CASCADE',
|
||||
})
|
||||
.withForeignKey('pastExecutionId', {
|
||||
tableName: 'execution_entity',
|
||||
columnName: 'id',
|
||||
onDelete: 'SET NULL',
|
||||
})
|
||||
.withForeignKey('executionId', {
|
||||
tableName: 'execution_entity',
|
||||
columnName: 'id',
|
||||
onDelete: 'SET NULL',
|
||||
})
|
||||
.withForeignKey('evaluationExecutionId', {
|
||||
tableName: 'execution_entity',
|
||||
columnName: 'id',
|
||||
onDelete: 'SET NULL',
|
||||
}).withTimestamps;
|
||||
}
|
||||
|
||||
async down({ schemaBuilder: { dropTable } }: MigrationContext) {
|
||||
await dropTable(testCaseExecutionTableName);
|
||||
}
|
||||
}
|
||||
|
|
@ -77,6 +77,7 @@ import { CreateTestRun1732549866705 } from '../common/1732549866705-CreateTestRu
|
|||
import { AddMockedNodesColumnToTestDefinition1733133775640 } from '../common/1733133775640-AddMockedNodesColumnToTestDefinition';
|
||||
import { AddManagedColumnToCredentialsTable1734479635324 } from '../common/1734479635324-AddManagedColumnToCredentialsTable';
|
||||
import { AddStatsColumnsToTestRun1736172058779 } from '../common/1736172058779-AddStatsColumnsToTestRun';
|
||||
import { CreateTestCaseExecutionTable1736947513045 } from '../common/1736947513045-CreateTestCaseExecutionTable';
|
||||
|
||||
export const mysqlMigrations: Migration[] = [
|
||||
InitialMigration1588157391238,
|
||||
|
|
@ -156,4 +157,5 @@ export const mysqlMigrations: Migration[] = [
|
|||
AddManagedColumnToCredentialsTable1734479635324,
|
||||
AddProjectIcons1729607673469,
|
||||
AddStatsColumnsToTestRun1736172058779,
|
||||
CreateTestCaseExecutionTable1736947513045,
|
||||
];
|
||||
|
|
|
|||
|
|
@ -77,6 +77,7 @@ import { CreateTestRun1732549866705 } from '../common/1732549866705-CreateTestRu
|
|||
import { AddMockedNodesColumnToTestDefinition1733133775640 } from '../common/1733133775640-AddMockedNodesColumnToTestDefinition';
|
||||
import { AddManagedColumnToCredentialsTable1734479635324 } from '../common/1734479635324-AddManagedColumnToCredentialsTable';
|
||||
import { AddStatsColumnsToTestRun1736172058779 } from '../common/1736172058779-AddStatsColumnsToTestRun';
|
||||
import { CreateTestCaseExecutionTable1736947513045 } from '../common/1736947513045-CreateTestCaseExecutionTable';
|
||||
|
||||
export const postgresMigrations: Migration[] = [
|
||||
InitialMigration1587669153312,
|
||||
|
|
@ -156,4 +157,5 @@ export const postgresMigrations: Migration[] = [
|
|||
AddManagedColumnToCredentialsTable1734479635324,
|
||||
AddProjectIcons1729607673469,
|
||||
AddStatsColumnsToTestRun1736172058779,
|
||||
CreateTestCaseExecutionTable1736947513045,
|
||||
];
|
||||
|
|
|
|||
|
|
@ -74,6 +74,7 @@ import { CreateTestRun1732549866705 } from '../common/1732549866705-CreateTestRu
|
|||
import { AddMockedNodesColumnToTestDefinition1733133775640 } from '../common/1733133775640-AddMockedNodesColumnToTestDefinition';
|
||||
import { AddManagedColumnToCredentialsTable1734479635324 } from '../common/1734479635324-AddManagedColumnToCredentialsTable';
|
||||
import { AddStatsColumnsToTestRun1736172058779 } from '../common/1736172058779-AddStatsColumnsToTestRun';
|
||||
import { CreateTestCaseExecutionTable1736947513045 } from '../common/1736947513045-CreateTestCaseExecutionTable';
|
||||
|
||||
const sqliteMigrations: Migration[] = [
|
||||
InitialMigration1588102412422,
|
||||
|
|
@ -150,6 +151,7 @@ const sqliteMigrations: Migration[] = [
|
|||
AddManagedColumnToCredentialsTable1734479635324,
|
||||
AddProjectIcons1729607673469,
|
||||
AddStatsColumnsToTestRun1736172058779,
|
||||
CreateTestCaseExecutionTable1736947513045,
|
||||
];
|
||||
|
||||
export { sqliteMigrations };
|
||||
|
|
|
|||
|
|
@ -0,0 +1,99 @@
|
|||
import { Service } from '@n8n/di';
|
||||
import type { EntityManager } from '@n8n/typeorm';
|
||||
import { DataSource, In, Not, Repository } from '@n8n/typeorm';
|
||||
import type { DeepPartial } from '@n8n/typeorm/common/DeepPartial';
|
||||
|
||||
import { TestCaseExecution } from '@/databases/entities/test-case-execution.ee';
|
||||
|
||||
@Service()
|
||||
export class TestCaseExecutionRepository extends Repository<TestCaseExecution> {
|
||||
constructor(dataSource: DataSource) {
|
||||
super(TestCaseExecution, dataSource.manager);
|
||||
}
|
||||
|
||||
async createBatch(testRunId: string, pastExecutionIds: string[]) {
|
||||
const mappings = this.create(
|
||||
pastExecutionIds.map<DeepPartial<TestCaseExecution>>((id) => ({
|
||||
testRun: {
|
||||
id: testRunId,
|
||||
},
|
||||
pastExecution: {
|
||||
id,
|
||||
},
|
||||
status: 'new',
|
||||
})),
|
||||
);
|
||||
|
||||
return await this.save(mappings);
|
||||
}
|
||||
|
||||
async markAsRunning(testRunId: string, pastExecutionId: string, executionId: string) {
|
||||
return await this.update(
|
||||
{ testRun: { id: testRunId }, pastExecutionId },
|
||||
{
|
||||
status: 'running',
|
||||
executionId,
|
||||
runAt: new Date(),
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
async markAsEvaluationRunning(
|
||||
testRunId: string,
|
||||
pastExecutionId: string,
|
||||
evaluationExecutionId: string,
|
||||
) {
|
||||
return await this.update(
|
||||
{ testRun: { id: testRunId }, pastExecutionId },
|
||||
{
|
||||
status: 'evaluation_running',
|
||||
evaluationExecutionId,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
async markAsCompleted(
|
||||
testRunId: string,
|
||||
pastExecutionId: string,
|
||||
metrics: Record<string, number>,
|
||||
trx?: EntityManager,
|
||||
) {
|
||||
trx = trx ?? this.manager;
|
||||
|
||||
return await trx.update(
|
||||
TestCaseExecution,
|
||||
{ testRun: { id: testRunId }, pastExecutionId },
|
||||
{
|
||||
status: 'success',
|
||||
completedAt: new Date(),
|
||||
metrics,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
async markAllPendingAsCancelled(testRunId: string, trx?: EntityManager) {
|
||||
trx = trx ?? this.manager;
|
||||
|
||||
return await trx.update(
|
||||
TestCaseExecution,
|
||||
{ testRun: { id: testRunId }, status: Not(In(['success', 'error', 'cancelled'])) },
|
||||
{
|
||||
status: 'cancelled',
|
||||
completedAt: new Date(),
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
async markAsFailed(testRunId: string, pastExecutionId: string, trx?: EntityManager) {
|
||||
trx = trx ?? this.manager;
|
||||
|
||||
return await trx.update(
|
||||
TestCaseExecution,
|
||||
{ testRun: { id: testRunId }, pastExecutionId },
|
||||
{
|
||||
status: 'error',
|
||||
completedAt: new Date(),
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
import { Service } from '@n8n/di';
|
||||
import type { FindManyOptions } from '@n8n/typeorm';
|
||||
import type { EntityManager, FindManyOptions } from '@n8n/typeorm';
|
||||
import { DataSource, Repository } from '@n8n/typeorm';
|
||||
|
||||
import type { AggregatedTestRunMetrics } from '@/databases/entities/test-run.ee';
|
||||
|
|
@ -35,16 +35,19 @@ export class TestRunRepository extends Repository<TestRun> {
|
|||
return await this.update(id, { status: 'completed', completedAt: new Date(), metrics });
|
||||
}
|
||||
|
||||
async markAsCancelled(id: string) {
|
||||
return await this.update(id, { status: 'cancelled' });
|
||||
async markAsCancelled(id: string, trx?: EntityManager) {
|
||||
trx = trx ?? this.manager;
|
||||
return await trx.update(TestRun, id, { status: 'cancelled' });
|
||||
}
|
||||
|
||||
async incrementPassed(id: string) {
|
||||
return await this.increment({ id }, 'passedCases', 1);
|
||||
async incrementPassed(id: string, trx?: EntityManager) {
|
||||
trx = trx ?? this.manager;
|
||||
return await trx.increment(TestRun, { id }, 'passedCases', 1);
|
||||
}
|
||||
|
||||
async incrementFailed(id: string) {
|
||||
return await this.increment({ id }, 'failedCases', 1);
|
||||
async incrementFailed(id: string, trx?: EntityManager) {
|
||||
trx = trx ?? this.manager;
|
||||
return await trx.increment(TestRun, { id }, 'failedCases', 1);
|
||||
}
|
||||
|
||||
async getMany(testDefinitionId: string, options: ListQuery.Options) {
|
||||
|
|
|
|||
|
|
@ -94,4 +94,6 @@ export declare namespace TestRunsRequest {
|
|||
type Delete = AuthenticatedRequest<RouteParams.TestId & RouteParams.TestRunId>;
|
||||
|
||||
type Cancel = AuthenticatedRequest<RouteParams.TestId & RouteParams.TestRunId>;
|
||||
|
||||
type GetCases = AuthenticatedRequest<RouteParams.TestId & RouteParams.TestRunId>;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ import type { TestMetric } from '@/databases/entities/test-metric.ee';
|
|||
import type { TestRun } from '@/databases/entities/test-run.ee';
|
||||
import type { User } from '@/databases/entities/user';
|
||||
import type { ExecutionRepository } from '@/databases/repositories/execution.repository';
|
||||
import type { TestCaseExecutionRepository } from '@/databases/repositories/test-case-execution.repository.ee';
|
||||
import type { TestMetricRepository } from '@/databases/repositories/test-metric.repository.ee';
|
||||
import type { TestRunRepository } from '@/databases/repositories/test-run.repository.ee';
|
||||
import type { WorkflowRepository } from '@/databases/repositories/workflow.repository';
|
||||
|
|
@ -25,6 +26,10 @@ import { mockNodeTypesData } from '@test-integration/utils/node-types-data';
|
|||
|
||||
import { TestRunnerService } from '../test-runner.service.ee';
|
||||
|
||||
jest.mock('@/db', () => ({
|
||||
transaction: (cb: any) => cb(),
|
||||
}));
|
||||
|
||||
const wfUnderTestJson = JSON.parse(
|
||||
readFileSync(path.join(__dirname, './mock-data/workflow.under-test.json'), { encoding: 'utf-8' }),
|
||||
);
|
||||
|
|
@ -147,6 +152,7 @@ describe('TestRunnerService', () => {
|
|||
const activeExecutions = mock<ActiveExecutions>();
|
||||
const testRunRepository = mock<TestRunRepository>();
|
||||
const testMetricRepository = mock<TestMetricRepository>();
|
||||
const testCaseExecutionRepository = mock<TestCaseExecutionRepository>();
|
||||
|
||||
const mockNodeTypes = mockInstance(NodeTypes);
|
||||
mockInstance(LoadNodesAndCredentials, {
|
||||
|
|
@ -190,6 +196,7 @@ describe('TestRunnerService', () => {
|
|||
executionRepository,
|
||||
activeExecutions,
|
||||
testRunRepository,
|
||||
testCaseExecutionRepository,
|
||||
testMetricRepository,
|
||||
mockNodeTypes,
|
||||
errorReporter,
|
||||
|
|
@ -207,6 +214,7 @@ describe('TestRunnerService', () => {
|
|||
executionRepository,
|
||||
activeExecutions,
|
||||
testRunRepository,
|
||||
testCaseExecutionRepository,
|
||||
testMetricRepository,
|
||||
mockNodeTypes,
|
||||
errorReporter,
|
||||
|
|
@ -247,6 +255,7 @@ describe('TestRunnerService', () => {
|
|||
executionRepository,
|
||||
activeExecutions,
|
||||
testRunRepository,
|
||||
testCaseExecutionRepository,
|
||||
testMetricRepository,
|
||||
mockNodeTypes,
|
||||
errorReporter,
|
||||
|
|
@ -350,6 +359,7 @@ describe('TestRunnerService', () => {
|
|||
executionRepository,
|
||||
activeExecutions,
|
||||
testRunRepository,
|
||||
testCaseExecutionRepository,
|
||||
testMetricRepository,
|
||||
mockNodeTypes,
|
||||
errorReporter,
|
||||
|
|
@ -410,6 +420,7 @@ describe('TestRunnerService', () => {
|
|||
executionRepository,
|
||||
activeExecutions,
|
||||
testRunRepository,
|
||||
testCaseExecutionRepository,
|
||||
testMetricRepository,
|
||||
mockNodeTypes,
|
||||
errorReporter,
|
||||
|
|
@ -466,6 +477,7 @@ describe('TestRunnerService', () => {
|
|||
executionRepository,
|
||||
activeExecutions,
|
||||
testRunRepository,
|
||||
testCaseExecutionRepository,
|
||||
testMetricRepository,
|
||||
mockNodeTypes,
|
||||
errorReporter,
|
||||
|
|
@ -526,6 +538,7 @@ describe('TestRunnerService', () => {
|
|||
executionRepository,
|
||||
activeExecutions,
|
||||
testRunRepository,
|
||||
testCaseExecutionRepository,
|
||||
testMetricRepository,
|
||||
mockNodeTypes,
|
||||
errorReporter,
|
||||
|
|
@ -602,6 +615,7 @@ describe('TestRunnerService', () => {
|
|||
executionRepository,
|
||||
activeExecutions,
|
||||
testRunRepository,
|
||||
testCaseExecutionRepository,
|
||||
testMetricRepository,
|
||||
mockNodeTypes,
|
||||
errorReporter,
|
||||
|
|
@ -629,6 +643,7 @@ describe('TestRunnerService', () => {
|
|||
executionRepository,
|
||||
activeExecutions,
|
||||
testRunRepository,
|
||||
testCaseExecutionRepository,
|
||||
testMetricRepository,
|
||||
mockNodeTypes,
|
||||
errorReporter,
|
||||
|
|
@ -661,9 +676,10 @@ describe('TestRunnerService', () => {
|
|||
executionRepository,
|
||||
activeExecutions,
|
||||
testRunRepository,
|
||||
testCaseExecutionRepository,
|
||||
testMetricRepository,
|
||||
mockNodeTypes,
|
||||
mock<ErrorReporter>(),
|
||||
errorReporter,
|
||||
);
|
||||
|
||||
workflowRepository.findById.calledWith('workflow-under-test-id').mockResolvedValueOnce({
|
||||
|
|
|
|||
|
|
@ -9,12 +9,17 @@ export class EvaluationMetrics {
|
|||
}
|
||||
}
|
||||
|
||||
addResults(result: IDataObject) {
|
||||
addResults(result: IDataObject): Record<string, number> {
|
||||
const addedMetrics: Record<string, number> = {};
|
||||
|
||||
for (const [metricName, metricValue] of Object.entries(result)) {
|
||||
if (typeof metricValue === 'number' && this.metricNames.has(metricName)) {
|
||||
addedMetrics[metricName] = metricValue;
|
||||
this.rawMetricsByName.get(metricName)!.push(metricValue);
|
||||
}
|
||||
}
|
||||
|
||||
return addedMetrics;
|
||||
}
|
||||
|
||||
getAggregatedMetrics() {
|
||||
|
|
|
|||
|
|
@ -19,9 +19,11 @@ import type { TestRun } from '@/databases/entities/test-run.ee';
|
|||
import type { User } from '@/databases/entities/user';
|
||||
import type { WorkflowEntity } from '@/databases/entities/workflow-entity';
|
||||
import { ExecutionRepository } from '@/databases/repositories/execution.repository';
|
||||
import { TestCaseExecutionRepository } from '@/databases/repositories/test-case-execution.repository.ee';
|
||||
import { TestMetricRepository } from '@/databases/repositories/test-metric.repository.ee';
|
||||
import { TestRunRepository } from '@/databases/repositories/test-run.repository.ee';
|
||||
import { WorkflowRepository } from '@/databases/repositories/workflow.repository';
|
||||
import * as Db from '@/db';
|
||||
import { NodeTypes } from '@/node-types';
|
||||
import { Telemetry } from '@/telemetry';
|
||||
import { getRunData } from '@/workflow-execute-additional-data';
|
||||
|
|
@ -30,6 +32,15 @@ import { WorkflowRunner } from '@/workflow-runner';
|
|||
import { EvaluationMetrics } from './evaluation-metrics.ee';
|
||||
import { createPinData, getPastExecutionTriggerNode } from './utils.ee';
|
||||
|
||||
interface TestRunMetadata {
|
||||
testRunId: string;
|
||||
userId: string;
|
||||
}
|
||||
|
||||
interface TestCaseRunMetadata extends TestRunMetadata {
|
||||
pastExecutionId: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* This service orchestrates the running of test cases.
|
||||
* It uses the test definitions to find
|
||||
|
|
@ -50,6 +61,7 @@ export class TestRunnerService {
|
|||
private readonly executionRepository: ExecutionRepository,
|
||||
private readonly activeExecutions: ActiveExecutions,
|
||||
private readonly testRunRepository: TestRunRepository,
|
||||
private readonly testCaseExecutionRepository: TestCaseExecutionRepository,
|
||||
private readonly testMetricRepository: TestMetricRepository,
|
||||
private readonly nodeTypes: NodeTypes,
|
||||
private readonly errorReporter: ErrorReporter,
|
||||
|
|
@ -105,7 +117,7 @@ export class TestRunnerService {
|
|||
pastExecutionData: IRunExecutionData,
|
||||
pastExecutionWorkflowData: IWorkflowBase,
|
||||
mockedNodes: MockedNodeItem[],
|
||||
userId: string,
|
||||
metadata: TestCaseRunMetadata,
|
||||
abortSignal: AbortSignal,
|
||||
): Promise<IRun | undefined> {
|
||||
// Do not run if the test run is cancelled
|
||||
|
|
@ -128,7 +140,7 @@ export class TestRunnerService {
|
|||
runData: {},
|
||||
pinData,
|
||||
workflowData: { ...workflow, pinData },
|
||||
userId,
|
||||
userId: metadata.userId,
|
||||
partialExecutionVersion: '1',
|
||||
};
|
||||
|
||||
|
|
@ -141,6 +153,13 @@ export class TestRunnerService {
|
|||
this.activeExecutions.stopExecution(executionId);
|
||||
});
|
||||
|
||||
// Update status of the test run execution mapping
|
||||
await this.testCaseExecutionRepository.markAsRunning(
|
||||
metadata.testRunId,
|
||||
metadata.pastExecutionId,
|
||||
executionId,
|
||||
);
|
||||
|
||||
// Wait for the execution to finish
|
||||
const executePromise = this.activeExecutions.getPostExecutePromise(executionId);
|
||||
|
||||
|
|
@ -155,7 +174,7 @@ export class TestRunnerService {
|
|||
expectedData: IRunData,
|
||||
actualData: IRunData,
|
||||
abortSignal: AbortSignal,
|
||||
testRunId?: string,
|
||||
metadata: TestCaseRunMetadata,
|
||||
) {
|
||||
// Do not run if the test run is cancelled
|
||||
if (abortSignal.aborted) {
|
||||
|
|
@ -173,13 +192,6 @@ export class TestRunnerService {
|
|||
|
||||
// Prepare the data to run the evaluation workflow
|
||||
const data = await getRunData(evaluationWorkflow, [evaluationInputData]);
|
||||
// FIXME: This is a hack to add the testRunId to the evaluation workflow execution data
|
||||
// So that we can fetch all execution runs for a test run
|
||||
if (testRunId && data.executionData) {
|
||||
data.executionData.resultData.metadata = {
|
||||
testRunId,
|
||||
};
|
||||
}
|
||||
data.executionMode = 'evaluation';
|
||||
|
||||
// Trigger the evaluation workflow
|
||||
|
|
@ -191,6 +203,13 @@ export class TestRunnerService {
|
|||
this.activeExecutions.stopExecution(executionId);
|
||||
});
|
||||
|
||||
// Update status of the test run execution mapping
|
||||
await this.testCaseExecutionRepository.markAsEvaluationRunning(
|
||||
metadata.testRunId,
|
||||
metadata.pastExecutionId,
|
||||
executionId,
|
||||
);
|
||||
|
||||
// Wait for the execution to finish
|
||||
const executePromise = this.activeExecutions.getPostExecutePromise(executionId);
|
||||
|
||||
|
|
@ -248,9 +267,18 @@ export class TestRunnerService {
|
|||
const abortController = new AbortController();
|
||||
this.abortControllers.set(testRun.id, abortController);
|
||||
|
||||
// 0.2 Initialize metadata
|
||||
// This will be passed to the test case executions
|
||||
const testRunMetadata = {
|
||||
testRunId: testRun.id,
|
||||
userId: user.id,
|
||||
};
|
||||
|
||||
const abortSignal = abortController.signal;
|
||||
try {
|
||||
///
|
||||
// 1. Make test cases from previous executions
|
||||
///
|
||||
|
||||
// Select executions with the annotation tag and workflow ID of the test.
|
||||
// Fetch only ids to reduce the data transfer.
|
||||
|
|
@ -266,13 +294,22 @@ export class TestRunnerService {
|
|||
|
||||
this.logger.debug('Found past executions', { count: pastExecutions.length });
|
||||
|
||||
// Add all past executions mappings to the test run.
|
||||
// This will be used to track the status of each test case and keep the connection between test run and all related executions (past, current, and evaluation).
|
||||
await this.testCaseExecutionRepository.createBatch(
|
||||
testRun.id,
|
||||
pastExecutions.map((e) => e.id),
|
||||
);
|
||||
|
||||
// Get the metrics to collect from the evaluation workflow
|
||||
const testMetricNames = await this.getTestMetricNames(test.id);
|
||||
|
||||
// 2. Run over all the test cases
|
||||
const pastExecutionIds = pastExecutions.map((e) => e.id);
|
||||
|
||||
// Update test run status
|
||||
await this.testRunRepository.markAsRunning(testRun.id, pastExecutions.length);
|
||||
|
||||
this.telemetry.track('User runs test', {
|
||||
user_id: user.id,
|
||||
test_id: test.id,
|
||||
|
|
@ -282,9 +319,13 @@ export class TestRunnerService {
|
|||
evaluation_workflow_id: test.evaluationWorkflowId,
|
||||
});
|
||||
|
||||
// Object to collect the results of the evaluation workflow executions
|
||||
// Initialize object to collect the results of the evaluation workflow executions
|
||||
const metrics = new EvaluationMetrics(testMetricNames);
|
||||
|
||||
///
|
||||
// 2. Run over all the test cases
|
||||
///
|
||||
|
||||
for (const pastExecutionId of pastExecutionIds) {
|
||||
if (abortSignal.aborted) {
|
||||
this.logger.debug('Test run was cancelled', {
|
||||
|
|
@ -306,13 +347,18 @@ export class TestRunnerService {
|
|||
|
||||
const executionData = parse(pastExecution.executionData.data) as IRunExecutionData;
|
||||
|
||||
const testCaseMetadata = {
|
||||
...testRunMetadata,
|
||||
pastExecutionId,
|
||||
};
|
||||
|
||||
// Run the test case and wait for it to finish
|
||||
const testCaseExecution = await this.runTestCase(
|
||||
workflow,
|
||||
executionData,
|
||||
pastExecution.executionData.workflowData,
|
||||
test.mockedNodes,
|
||||
user.id,
|
||||
testCaseMetadata,
|
||||
abortSignal,
|
||||
);
|
||||
|
||||
|
|
@ -321,10 +367,18 @@ export class TestRunnerService {
|
|||
// In case of a permission check issue, the test case execution will be undefined.
|
||||
// Skip them, increment the failed count and continue with the next test case
|
||||
if (!testCaseExecution) {
|
||||
await this.testRunRepository.incrementFailed(testRun.id);
|
||||
await Db.transaction(async (trx) => {
|
||||
await this.testRunRepository.incrementFailed(testRun.id, trx);
|
||||
await this.testCaseExecutionRepository.markAsFailed(testRun.id, pastExecutionId, trx);
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
// Update status of the test case execution mapping entry in case of an error
|
||||
if (testCaseExecution.data.resultData.error) {
|
||||
await this.testCaseExecutionRepository.markAsFailed(testRun.id, pastExecutionId);
|
||||
}
|
||||
|
||||
// Collect the results of the test case execution
|
||||
const testCaseRunData = testCaseExecution.data.resultData.runData;
|
||||
|
||||
|
|
@ -337,23 +391,37 @@ export class TestRunnerService {
|
|||
originalRunData,
|
||||
testCaseRunData,
|
||||
abortSignal,
|
||||
testRun.id,
|
||||
testCaseMetadata,
|
||||
);
|
||||
assert(evalExecution);
|
||||
|
||||
this.logger.debug('Evaluation execution finished', { pastExecutionId });
|
||||
|
||||
// Extract the output of the last node executed in the evaluation workflow
|
||||
metrics.addResults(this.extractEvaluationResult(evalExecution));
|
||||
const addedMetrics = metrics.addResults(this.extractEvaluationResult(evalExecution));
|
||||
|
||||
if (evalExecution.data.resultData.error) {
|
||||
await this.testRunRepository.incrementFailed(testRun.id);
|
||||
await Db.transaction(async (trx) => {
|
||||
await this.testRunRepository.incrementFailed(testRun.id, trx);
|
||||
await this.testCaseExecutionRepository.markAsFailed(testRun.id, pastExecutionId, trx);
|
||||
});
|
||||
} else {
|
||||
await this.testRunRepository.incrementPassed(testRun.id);
|
||||
await Db.transaction(async (trx) => {
|
||||
await this.testRunRepository.incrementPassed(testRun.id, trx);
|
||||
await this.testCaseExecutionRepository.markAsCompleted(
|
||||
testRun.id,
|
||||
pastExecutionId,
|
||||
addedMetrics,
|
||||
trx,
|
||||
);
|
||||
});
|
||||
}
|
||||
} catch (e) {
|
||||
// In case of an unexpected error, increment the failed count and continue with the next test case
|
||||
await this.testRunRepository.incrementFailed(testRun.id);
|
||||
await Db.transaction(async (trx) => {
|
||||
await this.testRunRepository.incrementFailed(testRun.id, trx);
|
||||
await this.testCaseExecutionRepository.markAsFailed(testRun.id, pastExecutionId, trx);
|
||||
});
|
||||
|
||||
this.errorReporter.error(e);
|
||||
}
|
||||
|
|
@ -361,7 +429,10 @@ export class TestRunnerService {
|
|||
|
||||
// Mark the test run as completed or cancelled
|
||||
if (abortSignal.aborted) {
|
||||
await this.testRunRepository.markAsCancelled(testRun.id);
|
||||
await Db.transaction(async (trx) => {
|
||||
await this.testRunRepository.markAsCancelled(testRun.id, trx);
|
||||
await this.testCaseExecutionRepository.markAllPendingAsCancelled(testRun.id, trx);
|
||||
});
|
||||
} else {
|
||||
const aggregatedMetrics = metrics.getAggregatedMetrics();
|
||||
await this.testRunRepository.markAsCompleted(testRun.id, aggregatedMetrics);
|
||||
|
|
@ -375,7 +446,10 @@ export class TestRunnerService {
|
|||
stoppedOn: e.extra?.executionId,
|
||||
});
|
||||
|
||||
await this.testRunRepository.markAsCancelled(testRun.id);
|
||||
await Db.transaction(async (trx) => {
|
||||
await this.testRunRepository.markAsCancelled(testRun.id, trx);
|
||||
await this.testCaseExecutionRepository.markAllPendingAsCancelled(testRun.id, trx);
|
||||
});
|
||||
} else {
|
||||
throw e;
|
||||
}
|
||||
|
|
@ -402,8 +476,11 @@ export class TestRunnerService {
|
|||
abortController.abort();
|
||||
this.abortControllers.delete(testRunId);
|
||||
} else {
|
||||
// If there is no abort controller - just mark the test run as cancelled
|
||||
await this.testRunRepository.markAsCancelled(testRunId);
|
||||
// If there is no abort controller - just mark the test run and all its' pending test case executions as cancelled
|
||||
await Db.transaction(async (trx) => {
|
||||
await this.testRunRepository.markAsCancelled(testRunId, trx);
|
||||
await this.testCaseExecutionRepository.markAllPendingAsCancelled(testRunId, trx);
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import express from 'express';
|
||||
import { InstanceSettings } from 'n8n-core';
|
||||
|
||||
import { TestCaseExecutionRepository } from '@/databases/repositories/test-case-execution.repository.ee';
|
||||
import { TestRunRepository } from '@/databases/repositories/test-run.repository.ee';
|
||||
import { Delete, Get, Post, RestController } from '@/decorators';
|
||||
import { ConflictError } from '@/errors/response-errors/conflict.error';
|
||||
|
|
@ -18,6 +19,7 @@ export class TestRunsController {
|
|||
constructor(
|
||||
private readonly testDefinitionService: TestDefinitionService,
|
||||
private readonly testRunRepository: TestRunRepository,
|
||||
private readonly testCaseExecutionRepository: TestCaseExecutionRepository,
|
||||
private readonly testRunnerService: TestRunnerService,
|
||||
private readonly instanceSettings: InstanceSettings,
|
||||
) {}
|
||||
|
|
@ -76,6 +78,16 @@ export class TestRunsController {
|
|||
return await this.getTestRun(req);
|
||||
}
|
||||
|
||||
@Get('/:testDefinitionId/runs/:id/cases')
|
||||
async getTestCases(req: TestRunsRequest.GetCases) {
|
||||
await this.getTestDefinition(req);
|
||||
await this.getTestRun(req);
|
||||
|
||||
return await this.testCaseExecutionRepository.find({
|
||||
where: { testRun: { id: req.params.id } },
|
||||
});
|
||||
}
|
||||
|
||||
@Delete('/:testDefinitionId/runs/:id')
|
||||
async delete(req: TestRunsRequest.Delete) {
|
||||
const { id: testRunId } = req.params;
|
||||
|
|
|
|||
|
|
@ -61,6 +61,21 @@ interface DeleteTestRunParams {
|
|||
runId: string;
|
||||
}
|
||||
|
||||
export interface TestCaseExecutionRecord {
|
||||
id: string;
|
||||
testRunId: string;
|
||||
executionId: string;
|
||||
pastExecutionId: string;
|
||||
evaluationExecutionId: string;
|
||||
status: 'running' | 'completed' | 'error';
|
||||
createdAt: string;
|
||||
updatedAt: string;
|
||||
runAt: string;
|
||||
metrics?: Record<string, number>;
|
||||
errorCode?: string;
|
||||
errorDetails?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
const endpoint = '/evaluation/test-definitions';
|
||||
const getMetricsEndpoint = (testDefinitionId: string, metricId?: string) =>
|
||||
`${endpoint}/${testDefinitionId}/metrics${metricId ? `/${metricId}` : ''}`;
|
||||
|
|
@ -244,3 +259,19 @@ export const deleteTestRun = async (context: IRestApiContext, params: DeleteTest
|
|||
getRunsEndpoint(params.testDefinitionId, params.runId),
|
||||
);
|
||||
};
|
||||
|
||||
const getRunExecutionsEndpoint = (testDefinitionId: string, runId: string) =>
|
||||
`${endpoint}/${testDefinitionId}/runs/${runId}/cases`;
|
||||
|
||||
// Get all test cases of a test run
|
||||
export const getTestCaseExecutions = async (
|
||||
context: IRestApiContext,
|
||||
testDefinitionId: string,
|
||||
runId: string,
|
||||
) => {
|
||||
return await makeRestApiRequest<TestCaseExecutionRecord[]>(
|
||||
context,
|
||||
'GET',
|
||||
getRunExecutionsEndpoint(testDefinitionId, runId),
|
||||
);
|
||||
};
|
||||
|
|
|
|||
|
|
@ -23,8 +23,9 @@ function hasStatus(row: unknown): row is WithStatus {
|
|||
}
|
||||
|
||||
const statusThemeMap: Record<string, string> = {
|
||||
new: 'info',
|
||||
new: 'default',
|
||||
running: 'warning',
|
||||
evaluation_running: 'warning',
|
||||
completed: 'success',
|
||||
error: 'danger',
|
||||
success: 'success',
|
||||
|
|
@ -34,6 +35,7 @@ const statusThemeMap: Record<string, string> = {
|
|||
const statusLabelMap: Record<string, string> = {
|
||||
new: locale.baseText('testDefinition.listRuns.status.new'),
|
||||
running: locale.baseText('testDefinition.listRuns.status.running'),
|
||||
evaluation_running: locale.baseText('testDefinition.listRuns.status.evaluating'),
|
||||
completed: locale.baseText('testDefinition.listRuns.status.completed'),
|
||||
error: locale.baseText('testDefinition.listRuns.status.error'),
|
||||
success: locale.baseText('testDefinition.listRuns.status.success'),
|
||||
|
|
@ -53,11 +55,11 @@ const getCellContent = (column: TestTableColumn<T>, row: T) => {
|
|||
</script>
|
||||
|
||||
<template>
|
||||
<div v-if="column.route">
|
||||
<a v-if="column.openInNewTab" :href="router.resolve(column.route(row)).href" target="_blank">
|
||||
<div v-if="column.route?.(row)">
|
||||
<a v-if="column.openInNewTab" :href="router.resolve(column.route(row)!).href" target="_blank">
|
||||
{{ getCellContent(column, row) }}
|
||||
</a>
|
||||
<router-link v-else :to="column.route(row)">
|
||||
<router-link v-else :to="column.route(row)!">
|
||||
{{ getCellContent(column, row) }}
|
||||
</router-link>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@ export type TestTableColumn<TRow> = {
|
|||
sortable?: boolean;
|
||||
filters?: Array<{ text: string; value: string }>;
|
||||
filterMethod?: (value: string, row: TRow) => boolean;
|
||||
route?: (row: TRow) => RouteLocationRaw;
|
||||
route?: (row: TRow) => RouteLocationRaw | undefined;
|
||||
sortMethod?: (a: TRow, b: TRow) => number;
|
||||
openInNewTab?: boolean;
|
||||
formatter?: (row: TRow) => string;
|
||||
|
|
|
|||
|
|
@ -2854,6 +2854,7 @@
|
|||
"testDefinition.list.loadError": "Failed to load tests",
|
||||
"testDefinition.listRuns.status.new": "New",
|
||||
"testDefinition.listRuns.status.running": "Running",
|
||||
"testDefinition.listRuns.status.evaluating": "Evaluating",
|
||||
"testDefinition.listRuns.status.completed": "Completed",
|
||||
"testDefinition.listRuns.status.cancelled": "Cancelled",
|
||||
"testDefinition.listRuns.status.error": "Error",
|
||||
|
|
|
|||
|
|
@ -2,7 +2,11 @@ import { defineStore } from 'pinia';
|
|||
import { computed, ref } from 'vue';
|
||||
import { useRootStore } from './root.store';
|
||||
import * as testDefinitionsApi from '@/api/testDefinition.ee';
|
||||
import type { TestDefinitionRecord, TestRunRecord } from '@/api/testDefinition.ee';
|
||||
import type {
|
||||
TestCaseExecutionRecord,
|
||||
TestDefinitionRecord,
|
||||
TestRunRecord,
|
||||
} from '@/api/testDefinition.ee';
|
||||
import { usePostHog } from './posthog.store';
|
||||
import { STORES, WORKFLOW_EVALUATION_EXPERIMENT } from '@/constants';
|
||||
import { useAnnotationTagsStore } from './tags.store';
|
||||
|
|
@ -19,6 +23,7 @@ export const useTestDefinitionStore = defineStore(
|
|||
const fetchedAll = ref(false);
|
||||
const metricsById = ref<Record<string, testDefinitionsApi.TestMetricRecord>>({});
|
||||
const testRunsById = ref<Record<string, TestRunRecord>>({});
|
||||
const testCaseExecutionsById = ref<Record<string, TestCaseExecutionRecord>>({});
|
||||
const pollingTimeouts = ref<Record<string, NodeJS.Timeout>>({});
|
||||
const fieldsIssues = ref<Record<string, FieldIssue[]>>({});
|
||||
|
||||
|
|
@ -167,6 +172,20 @@ export const useTestDefinitionStore = defineStore(
|
|||
return testDefinition;
|
||||
};
|
||||
|
||||
const fetchTestCaseExecutions = async (params: { testDefinitionId: string; runId: string }) => {
|
||||
const testCaseExecutions = await testDefinitionsApi.getTestCaseExecutions(
|
||||
rootStore.restApiContext,
|
||||
params.testDefinitionId,
|
||||
params.runId,
|
||||
);
|
||||
|
||||
testCaseExecutions.forEach((testCaseExecution) => {
|
||||
testCaseExecutionsById.value[testCaseExecution.id] = testCaseExecution;
|
||||
});
|
||||
|
||||
return testCaseExecutions;
|
||||
};
|
||||
|
||||
/**
|
||||
* Fetches all test definitions from the API.
|
||||
* @param {boolean} force - If true, fetches the definitions from the API even if they were already fetched before.
|
||||
|
|
@ -421,6 +440,7 @@ export const useTestDefinitionStore = defineStore(
|
|||
fetchedAll,
|
||||
testDefinitionsById,
|
||||
testRunsById,
|
||||
testCaseExecutionsById,
|
||||
|
||||
// Computed
|
||||
allTestDefinitions,
|
||||
|
|
@ -435,6 +455,7 @@ export const useTestDefinitionStore = defineStore(
|
|||
|
||||
// Methods
|
||||
fetchTestDefinition,
|
||||
fetchTestCaseExecutions,
|
||||
fetchAll,
|
||||
create,
|
||||
update,
|
||||
|
|
|
|||
|
|
@ -7,24 +7,17 @@ import { useI18n } from '@/composables/useI18n';
|
|||
import { N8nCard, N8nText } from 'n8n-design-system';
|
||||
import TestTableBase from '@/components/TestDefinition/shared/TestTableBase.vue';
|
||||
import type { TestTableColumn } from '@/components/TestDefinition/shared/TestTableBase.vue';
|
||||
import { useExecutionsStore } from '@/stores/executions.store';
|
||||
import { get } from 'lodash-es';
|
||||
import type { ExecutionSummaryWithScopes } from '@/Interface';
|
||||
import { VIEWS } from '@/constants';
|
||||
import { useToast } from '@/composables/useToast';
|
||||
|
||||
interface TestCase extends ExecutionSummaryWithScopes {
|
||||
metrics: Record<string, number>;
|
||||
}
|
||||
import type { TestCaseExecutionRecord } from '@/api/testDefinition.ee';
|
||||
|
||||
const router = useRouter();
|
||||
const toast = useToast();
|
||||
const testDefinitionStore = useTestDefinitionStore();
|
||||
const executionsStore = useExecutionsStore();
|
||||
const locale = useI18n();
|
||||
|
||||
const isLoading = ref(true);
|
||||
const testCases = ref<TestCase[]>([]);
|
||||
const testCases = ref<TestCaseExecutionRecord[]>([]);
|
||||
|
||||
const runId = computed(() => router.currentRoute.value.params.runId as string);
|
||||
const testId = computed(() => router.currentRoute.value.params.testId as string);
|
||||
|
|
@ -36,17 +29,26 @@ const filteredTestCases = computed(() => {
|
|||
});
|
||||
|
||||
const columns = computed(
|
||||
(): Array<TestTableColumn<TestCase>> => [
|
||||
(): Array<TestTableColumn<TestCaseExecutionRecord>> => [
|
||||
{
|
||||
prop: 'id',
|
||||
width: 200,
|
||||
label: locale.baseText('testDefinition.runDetail.testCase'),
|
||||
sortable: true,
|
||||
route: (row: TestCase) => ({
|
||||
name: VIEWS.EXECUTION_PREVIEW,
|
||||
params: { name: row.workflowId, executionId: row.id },
|
||||
}),
|
||||
formatter: (row: TestCase) => `[${row.id}] ${row.workflowName}`,
|
||||
route: (row: TestCaseExecutionRecord) => {
|
||||
if (test.value?.evaluationWorkflowId && row.evaluationExecutionId) {
|
||||
return {
|
||||
name: VIEWS.EXECUTION_PREVIEW,
|
||||
params: {
|
||||
name: test.value?.evaluationWorkflowId,
|
||||
executionId: row.evaluationExecutionId,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
return undefined;
|
||||
},
|
||||
formatter: (row: TestCaseExecutionRecord) => `${row.id}`,
|
||||
openInNewTab: true,
|
||||
},
|
||||
{
|
||||
|
|
@ -58,14 +60,14 @@ const columns = computed(
|
|||
{ text: locale.baseText('testDefinition.listRuns.status.success'), value: 'success' },
|
||||
{ text: locale.baseText('testDefinition.listRuns.status.error'), value: 'error' },
|
||||
],
|
||||
filterMethod: (value: string, row: TestCase) => row.status === value,
|
||||
filterMethod: (value: string, row: TestCaseExecutionRecord) => row.status === value,
|
||||
},
|
||||
...Object.keys(run.value?.metrics ?? {}).map((metric) => ({
|
||||
prop: `metrics.${metric}`,
|
||||
label: metric,
|
||||
sortable: true,
|
||||
filter: true,
|
||||
formatter: (row: TestCase) => row.metrics[metric]?.toFixed(2) ?? '-',
|
||||
formatter: (row: TestCaseExecutionRecord) => row.metrics?.[metric]?.toFixed(2) ?? '-',
|
||||
})),
|
||||
],
|
||||
);
|
||||
|
|
@ -73,57 +75,24 @@ const columns = computed(
|
|||
const metrics = computed(() => run.value?.metrics ?? {});
|
||||
|
||||
// Temporary workaround to fetch test cases by manually getting workflow executions
|
||||
// TODO: Replace with dedicated API endpoint once available
|
||||
const fetchExecutionTestCases = async () => {
|
||||
if (!runId.value || !testId.value) return;
|
||||
|
||||
isLoading.value = true;
|
||||
try {
|
||||
await testDefinitionStore.fetchTestDefinition(testId.value);
|
||||
|
||||
const testRun = await testDefinitionStore.getTestRun({
|
||||
testDefinitionId: testId.value,
|
||||
runId: runId.value,
|
||||
});
|
||||
const testDefinition = await testDefinitionStore.fetchTestDefinition(testId.value);
|
||||
|
||||
// Fetch workflow executions that match this test run
|
||||
const evaluationWorkflowExecutions = await executionsStore.fetchExecutions({
|
||||
workflowId: testDefinition.evaluationWorkflowId ?? '',
|
||||
metadata: [{ key: 'testRunId', value: testRun.id }],
|
||||
const testCaseEvaluationExecutions = await testDefinitionStore.fetchTestCaseExecutions({
|
||||
testDefinitionId: testId.value,
|
||||
runId: testRun.id,
|
||||
});
|
||||
|
||||
// For each execution, fetch full details and extract metrics
|
||||
const executionsData = await Promise.all(
|
||||
evaluationWorkflowExecutions?.results.map(async (execution) => {
|
||||
const executionData = await executionsStore.fetchExecution(execution.id);
|
||||
const lastExecutedNode = executionData?.data?.resultData?.lastNodeExecuted;
|
||||
if (!lastExecutedNode) {
|
||||
throw new Error('Last executed node is required');
|
||||
}
|
||||
const metricsData = get(
|
||||
executionData,
|
||||
[
|
||||
'data',
|
||||
'resultData',
|
||||
'runData',
|
||||
lastExecutedNode,
|
||||
'0',
|
||||
'data',
|
||||
'main',
|
||||
'0',
|
||||
'0',
|
||||
'json',
|
||||
],
|
||||
{},
|
||||
);
|
||||
|
||||
return {
|
||||
...execution,
|
||||
metrics: metricsData,
|
||||
};
|
||||
}),
|
||||
);
|
||||
|
||||
testCases.value = executionsData ?? [];
|
||||
testCases.value = testCaseEvaluationExecutions ?? [];
|
||||
} catch (error) {
|
||||
toast.showError(error, 'Failed to load run details');
|
||||
} finally {
|
||||
|
|
|
|||
|
|
@ -7,7 +7,6 @@ import TestDefinitionRunDetailView from '@/views/TestDefinition/TestDefinitionRu
|
|||
import { useRoute, useRouter } from 'vue-router';
|
||||
import { useToast } from '@/composables/useToast';
|
||||
import { useTestDefinitionStore } from '@/stores/testDefinition.store.ee';
|
||||
import { useExecutionsStore } from '@/stores/executions.store';
|
||||
import { useWorkflowsStore } from '@/stores/workflows.store';
|
||||
import { nextTick, ref } from 'vue';
|
||||
import { mockedStore, waitAllPromises } from '@/__tests__/utils';
|
||||
|
|
@ -23,8 +22,7 @@ describe('TestDefinitionRunDetailView', () => {
|
|||
|
||||
let showErrorMock: Mock;
|
||||
let getTestRunMock: Mock;
|
||||
let fetchExecutionsMock: Mock;
|
||||
let fetchExecutionMock: Mock;
|
||||
let fetchTestCaseExecutionsMock: Mock;
|
||||
|
||||
const mockTestRun: TestRunRecord = {
|
||||
id: 'run1',
|
||||
|
|
@ -52,12 +50,10 @@ describe('TestDefinitionRunDetailView', () => {
|
|||
name: 'Evaluation Workflow',
|
||||
};
|
||||
|
||||
const mockExecutions = {
|
||||
results: [
|
||||
{ id: 'exec1', status: 'success' },
|
||||
{ id: 'exec2', status: 'error' },
|
||||
],
|
||||
};
|
||||
const mockTestCaseExecutions = [
|
||||
{ id: 'exec1', status: 'success' },
|
||||
{ id: 'exec2', status: 'error' },
|
||||
];
|
||||
|
||||
beforeEach(() => {
|
||||
setActivePinia(createPinia());
|
||||
|
|
@ -78,17 +74,7 @@ describe('TestDefinitionRunDetailView', () => {
|
|||
|
||||
showErrorMock = vi.fn();
|
||||
getTestRunMock = vi.fn().mockResolvedValue(mockTestRun);
|
||||
fetchExecutionsMock = vi.fn().mockResolvedValue(mockExecutions);
|
||||
fetchExecutionMock = vi.fn().mockResolvedValue({
|
||||
data: {
|
||||
resultData: {
|
||||
lastNodeExecuted: 'Node1',
|
||||
runData: {
|
||||
Node1: [{ data: { main: [[{ json: { accuracy: 0.95 } }]] } }],
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
fetchTestCaseExecutionsMock = vi.fn().mockResolvedValue(mockTestCaseExecutions);
|
||||
|
||||
vi.mocked(useToast).mockReturnValue({
|
||||
showError: showErrorMock,
|
||||
|
|
@ -108,10 +94,6 @@ describe('TestDefinitionRunDetailView', () => {
|
|||
testDefinitionStore.testDefinitionsById = { test1: mockTestDefinition };
|
||||
testDefinitionStore.getTestRun = getTestRunMock;
|
||||
|
||||
const executionsStore = mockedStore(useExecutionsStore);
|
||||
executionsStore.fetchExecutions = fetchExecutionsMock;
|
||||
executionsStore.fetchExecution = fetchExecutionMock;
|
||||
|
||||
const workflowsStore = mockedStore(useWorkflowsStore);
|
||||
workflowsStore.workflowsById = { workflow1: mockWorkflow as IWorkflowDb };
|
||||
|
||||
|
|
@ -151,7 +133,7 @@ describe('TestDefinitionRunDetailView', () => {
|
|||
testDefinitionStore.getTestRun = vi.fn().mockRejectedValue(new Error('Failed to load'));
|
||||
|
||||
renderComponent({ pinia });
|
||||
await nextTick();
|
||||
await waitAllPromises();
|
||||
|
||||
expect(showErrorMock).toHaveBeenCalledWith(expect.any(Error), 'Failed to load run details');
|
||||
});
|
||||
|
|
@ -243,7 +225,6 @@ describe('TestDefinitionRunDetailView', () => {
|
|||
setActivePinia(pinia);
|
||||
|
||||
const testDefinitionStore = mockedStore(useTestDefinitionStore);
|
||||
const executionsStore = mockedStore(useExecutionsStore);
|
||||
|
||||
// Mock all required store methods
|
||||
testDefinitionStore.testRunsById = { run1: mockTestRun };
|
||||
|
|
@ -251,17 +232,14 @@ describe('TestDefinitionRunDetailView', () => {
|
|||
testDefinitionStore.getTestRun = getTestRunMock;
|
||||
// Add this mock for fetchTestDefinition
|
||||
testDefinitionStore.fetchTestDefinition = vi.fn().mockResolvedValue(mockTestDefinition);
|
||||
|
||||
executionsStore.fetchExecutions = fetchExecutionsMock;
|
||||
executionsStore.fetchExecution = fetchExecutionMock;
|
||||
testDefinitionStore.fetchTestCaseExecutions = fetchTestCaseExecutionsMock;
|
||||
|
||||
const { container } = renderComponent({ pinia });
|
||||
await nextTick();
|
||||
|
||||
// Wait for all promises to resolve
|
||||
await waitAllPromises();
|
||||
|
||||
const tableRows = container.querySelectorAll('.el-table__row');
|
||||
expect(tableRows.length).toBe(mockExecutions.results.length);
|
||||
expect(tableRows.length).toBe(mockTestCaseExecutions.length);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
Loading…
Reference in a new issue