test(editor): Add AI Assistant e2e tests (no-changelog) (#10476)
Some checks are pending
Test Master / install-and-build (push) Waiting to run
Test Master / Unit tests (18.x) (push) Blocked by required conditions
Test Master / Unit tests (20.x) (push) Blocked by required conditions
Test Master / Unit tests (22.4) (push) Blocked by required conditions
Test Master / Lint (push) Blocked by required conditions
Test Master / Notify Slack on failure (push) Blocked by required conditions

This commit is contained in:
Milorad FIlipović 2024-08-20 13:05:09 +02:00 committed by GitHub
parent d0fc9dee0e
commit eecb80400d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
14 changed files with 513 additions and 13 deletions

View file

@ -0,0 +1,247 @@
import { NDV, WorkflowPage } from '../pages';
import { AIAssistant } from '../pages/features/ai-assistant';
const wf = new WorkflowPage();
const ndv = new NDV();
const aiAssistant = new AIAssistant();
describe('AI Assistant::disabled', () => {
beforeEach(() => {
aiAssistant.actions.disableAssistant();
wf.actions.visit();
});
it('does not show assistant button if feature is disabled', () => {
aiAssistant.getters.askAssistantFloatingButton().should('not.exist');
});
});
describe('AI Assistant::enabled', () => {
beforeEach(() => {
aiAssistant.actions.enableAssistant();
wf.actions.visit();
});
after(() => {
aiAssistant.actions.disableAssistant();
});
it('renders placeholder UI', () => {
aiAssistant.getters.askAssistantFloatingButton().should('be.visible');
aiAssistant.getters.askAssistantFloatingButton().click();
aiAssistant.getters.askAssistantChat().should('be.visible');
aiAssistant.getters.placeholderMessage().should('be.visible');
aiAssistant.getters.chatInputWrapper().should('not.exist');
aiAssistant.getters.closeChatButton().should('be.visible');
aiAssistant.getters.closeChatButton().click();
aiAssistant.getters.askAssistantChat().should('not.exist');
});
it('should resize assistant chat up', () => {
aiAssistant.getters.askAssistantFloatingButton().click();
aiAssistant.getters.askAssistantSidebarResizer().should('be.visible');
aiAssistant.getters.askAssistantChat().then((element) => {
const { width, left } = element[0].getBoundingClientRect();
cy.drag(aiAssistant.getters.askAssistantSidebarResizer(), [left - 10, 0], {
abs: true,
clickToFinish: true,
});
aiAssistant.getters.askAssistantChat().then((newElement) => {
const newWidth = newElement[0].getBoundingClientRect().width;
expect(newWidth).to.be.greaterThan(width);
});
});
});
it('should resize assistant chat down', () => {
aiAssistant.getters.askAssistantFloatingButton().click();
aiAssistant.getters.askAssistantSidebarResizer().should('be.visible');
aiAssistant.getters.askAssistantChat().then((element) => {
const { width, left } = element[0].getBoundingClientRect();
cy.drag(aiAssistant.getters.askAssistantSidebarResizer(), [left + 10, 0], {
abs: true,
clickToFinish: true,
});
aiAssistant.getters.askAssistantChat().then((newElement) => {
const newWidth = newElement[0].getBoundingClientRect().width;
expect(newWidth).to.be.lessThan(width);
});
});
});
it('should start chat session from node error view', () => {
cy.intercept('POST', '/rest/ai-assistant/chat', {
statusCode: 200,
fixture: 'aiAssistant/simple_message_response.json',
}).as('chatRequest');
cy.createFixtureWorkflow('aiAssistant/test_workflow.json');
wf.actions.openNode('Stop and Error');
ndv.getters.nodeExecuteButton().click();
aiAssistant.getters.nodeErrorViewAssistantButton().click();
cy.wait('@chatRequest');
aiAssistant.getters.chatMessagesAll().should('have.length', 1);
aiAssistant.getters
.chatMessagesAll()
.eq(0)
.should('contain.text', 'Hey, this is an assistant message');
aiAssistant.getters.nodeErrorViewAssistantButton().should('be.disabled');
});
it('should render chat input correctly', () => {
cy.intercept('POST', '/rest/ai-assistant/chat', {
statusCode: 200,
fixture: 'aiAssistant/simple_message_response.json',
}).as('chatRequest');
cy.createFixtureWorkflow('aiAssistant/test_workflow.json');
wf.actions.openNode('Stop and Error');
ndv.getters.nodeExecuteButton().click();
aiAssistant.getters.nodeErrorViewAssistantButton().click();
cy.wait('@chatRequest');
// Send button should be disabled when input is empty
aiAssistant.getters.sendMessageButton().should('be.disabled');
aiAssistant.getters.chatInput().type('Yo ');
aiAssistant.getters.sendMessageButton().should('not.be.disabled');
aiAssistant.getters.chatInput().then((element) => {
const { height } = element[0].getBoundingClientRect();
// Shift + Enter should add a new line
aiAssistant.getters.chatInput().type('Hello{shift+enter}there');
aiAssistant.getters.chatInput().then((newElement) => {
const newHeight = newElement[0].getBoundingClientRect().height;
// Chat input should grow as user adds new lines
expect(newHeight).to.be.greaterThan(height);
aiAssistant.getters.sendMessageButton().click();
cy.wait('@chatRequest');
// New lines should be rendered as <br> in the chat
aiAssistant.getters.chatMessagesUser().should('have.length', 1);
aiAssistant.getters.chatMessagesUser().eq(0).find('br').should('have.length', 1);
// Chat input should be cleared now
aiAssistant.getters.chatInput().should('have.value', '');
});
});
});
it('should render and handle quick replies', () => {
cy.intercept('POST', '/rest/ai-assistant/chat', {
statusCode: 200,
fixture: 'aiAssistant/quick_reply_message_response.json',
}).as('chatRequest');
cy.createFixtureWorkflow('aiAssistant/test_workflow.json');
wf.actions.openNode('Stop and Error');
ndv.getters.nodeExecuteButton().click();
aiAssistant.getters.nodeErrorViewAssistantButton().click();
cy.wait('@chatRequest');
aiAssistant.getters.quickReplies().should('have.length', 2);
aiAssistant.getters.quickReplies().eq(0).click();
cy.wait('@chatRequest');
aiAssistant.getters.chatMessagesUser().should('have.length', 1);
aiAssistant.getters.chatMessagesUser().eq(0).should('contain.text', "Sure, let's do it");
});
it('should send message to assistant when node is executed', () => {
cy.intercept('POST', '/rest/ai-assistant/chat', {
statusCode: 200,
fixture: 'aiAssistant/simple_message_response.json',
}).as('chatRequest');
cy.createFixtureWorkflow('aiAssistant/test_workflow.json');
wf.actions.openNode('Edit Fields');
ndv.getters.nodeExecuteButton().click();
aiAssistant.getters.nodeErrorViewAssistantButton().click();
cy.wait('@chatRequest');
aiAssistant.getters.chatMessagesAssistant().should('have.length', 1);
// Executing the same node should sende a new message to the assistant automatically
ndv.getters.nodeExecuteButton().click();
cy.wait('@chatRequest');
aiAssistant.getters.chatMessagesAssistant().should('have.length', 2);
});
it('should warn before starting a new session', () => {
cy.intercept('POST', '/rest/ai-assistant/chat', {
statusCode: 200,
fixture: 'aiAssistant/simple_message_response.json',
}).as('chatRequest');
cy.createFixtureWorkflow('aiAssistant/test_workflow.json');
wf.actions.openNode('Edit Fields');
ndv.getters.nodeExecuteButton().click();
aiAssistant.getters.nodeErrorViewAssistantButton().click();
cy.wait('@chatRequest');
aiAssistant.getters.closeChatButton().click();
ndv.getters.backToCanvas().click();
wf.actions.openNode('Stop and Error');
ndv.getters.nodeExecuteButton().click();
aiAssistant.getters.nodeErrorViewAssistantButton().click();
// Since we already have an active session, a warning should be shown
aiAssistant.getters.newAssistantSessionModal().should('be.visible');
aiAssistant.getters
.newAssistantSessionModal()
.find('button')
.contains('Start new session')
.click();
cy.wait('@chatRequest');
// New session should start with initial assistant message
aiAssistant.getters.chatMessagesAll().should('have.length', 1);
});
it('should apply code diff to code node', () => {
cy.intercept('POST', '/rest/ai-assistant/chat', {
statusCode: 200,
fixture: 'aiAssistant/code_diff_suggestion_response.json',
}).as('chatRequest');
cy.intercept('POST', '/rest/ai-assistant/chat/apply-suggestion', {
statusCode: 200,
fixture: 'aiAssistant/apply_code_diff_response.json',
}).as('applySuggestion');
cy.createFixtureWorkflow('aiAssistant/test_workflow.json');
wf.actions.openNode('Code');
ndv.getters.nodeExecuteButton().click();
aiAssistant.getters.nodeErrorViewAssistantButton().click({ force: true });
cy.wait('@chatRequest');
// Should have two assistant messages
aiAssistant.getters.chatMessagesAll().should('have.length', 2);
aiAssistant.getters.codeDiffs().should('have.length', 1);
aiAssistant.getters.applyCodeDiffButtons().should('have.length', 1);
aiAssistant.getters.applyCodeDiffButtons().first().click();
cy.wait('@applySuggestion');
aiAssistant.getters.applyCodeDiffButtons().should('have.length', 0);
aiAssistant.getters.undoReplaceCodeButtons().should('have.length', 1);
aiAssistant.getters.codeReplacedMessage().should('be.visible');
ndv.getters
.parameterInput('jsCode')
.get('.cm-content')
.should('contain.text', 'item.json.myNewField = 1');
// Clicking undo should revert the code back but not call the assistant
aiAssistant.getters.undoReplaceCodeButtons().first().click();
aiAssistant.getters.applyCodeDiffButtons().should('have.length', 1);
aiAssistant.getters.codeReplacedMessage().should('not.exist');
cy.get('@applySuggestion.all').then((interceptions) => {
expect(interceptions).to.have.length(1);
});
ndv.getters
.parameterInput('jsCode')
.get('.cm-content')
.should('contain.text', 'item.json.myNewField = 1aaa');
// Replacing the code again should also not call the assistant
cy.get('@applySuggestion.all').then((interceptions) => {
expect(interceptions).to.have.length(1);
});
aiAssistant.getters.applyCodeDiffButtons().should('have.length', 1);
aiAssistant.getters.applyCodeDiffButtons().first().click();
ndv.getters
.parameterInput('jsCode')
.get('.cm-content')
.should('contain.text', 'item.json.myNewField = 1');
});
it('should end chat session when `end_session` event is received', () => {
cy.intercept('POST', '/rest/ai-assistant/chat', {
statusCode: 200,
fixture: 'aiAssistant/end_session_response.json',
}).as('chatRequest');
cy.createFixtureWorkflow('aiAssistant/test_workflow.json');
wf.actions.openNode('Stop and Error');
ndv.getters.nodeExecuteButton().click();
aiAssistant.getters.nodeErrorViewAssistantButton().click();
cy.wait('@chatRequest');
aiAssistant.getters.chatMessagesSystem().should('have.length', 1);
aiAssistant.getters.chatMessagesSystem().first().should('contain.text', 'session has ended');
});
});

View file

@ -0,0 +1,8 @@
{
"data": {
"sessionId": "f9130bd7-c078-4862-a38a-369b27b0ff20-e96eb9f7-d581-4684-b6a9-fd3dfe9fe1fb-emTezIGat7bQsDdtIlbti",
"parameters": {
"jsCode": "// Loop over input items and add a new field called 'myNewField' to the JSON of each one\nfor (const item of $input.all()) {\n item.json.myNewField = 1;\n}\n\nreturn $input.all();"
}
}
}

View file

@ -0,0 +1,23 @@
{
"sessionId": "1",
"messages": [
{
"role": "assistant",
"type": "message",
"text": "Hi there! Here is my top solution to fix the error in your **Code** node 👇"
},
{
"type": "code-diff",
"description": "Fix the syntax error by changing '1asd' to a valid value. In this case, it seems like '1' was intended.",
"suggestionId": "1",
"codeDiff": "@@ -2,2 +2,2 @@\n item.json.myNewField = 1asd;\n+ item.json.myNewField = 1;\n",
"role": "assistant",
"quickReplies": [
{
"text": "Give me another solution",
"type": "new-suggestion"
}
]
}
]
}

View file

@ -0,0 +1,16 @@
{
"sessionId": "f9130bd7-c078-4862-a38a-369b27b0ff20-e96eb9f7-d581-4684-b6a9-fd3dfe9fe1fb-XCldJLlusGrEVku5I9cYT",
"messages": [
{
"role": "assistant",
"type": "agent-suggestion",
"title": "Glad to Help",
"text": "I'm glad I could help. If you have any more questions or need further assistance with your n8n workflows, feel free to ask!"
},
{
"role": "assistant",
"type": "event",
"eventName": "end-session"
}
]
}

View file

@ -0,0 +1,20 @@
{
"sessionId": "1",
"messages": [
{
"role": "assistant",
"type": "message",
"text": "Hey, this is an assistant message",
"quickReplies": [
{
"text": "Sure, let's do it",
"type": "yes"
},
{
"text": "Nah, doesn't sound good",
"type": "no"
}
]
}
]
}

View file

@ -0,0 +1,10 @@
{
"sessionId": "1",
"messages": [
{
"role": "assistant",
"type": "message",
"text": "Hey, this is an assistant message"
}
]
}

View file

@ -0,0 +1,88 @@
{
"nodes": [
{
"parameters": {},
"id": "ebfced75-2ce1-4c41-a971-6c3b83522c4d",
"name": "When clicking Test workflow",
"type": "n8n-nodes-base.manualTrigger",
"typeVersion": 1,
"position": [
360,
220
]
},
{
"parameters": {
"errorMessage": "This is an error message"
},
"id": "f2e60459-401a-49d5-acfc-7b2b31cfdcf7",
"name": "Stop and Error",
"type": "n8n-nodes-base.stopAndError",
"typeVersion": 1,
"position": [
1020,
220
]
},
{
"parameters": {
"jsCode": "// Loop over input items and add a new field called 'myNewField' to the JSON of each one\nfor (const item of $input.all()) {\n item.json.myNewField = 1aaa;\n}\n\nreturn $input.all();"
},
"id": "b54d4db9-b257-41a8-862f-26d293115bad",
"name": "Code",
"type": "n8n-nodes-base.code",
"typeVersion": 2,
"position": [
840,
320
]
},
{
"parameters": {
"assignments": {
"assignments": [
{
"id": "053ada73-f7db-4e6a-8cc8-85756cc6ca4e",
"name": "age",
"value": "={{ 32sad }}",
"type": "number"
}
]
},
"options": {}
},
"id": "5fd89612-a871-4679-b7b0-d659e09c6a0e",
"name": "Edit Fields",
"type": "n8n-nodes-base.set",
"typeVersion": 3.4,
"position": [
600,
100
]
}
],
"connections": {
"When clicking Test workflow": {
"main": [
[
{
"node": "Stop and Error",
"type": "main",
"index": 0
},
{
"node": "Code",
"type": "main",
"index": 0
},
{
"node": "Edit Fields",
"type": "main",
"index": 0
}
]
]
}
},
"pinData": {}
}

View file

@ -0,0 +1,49 @@
import { overrideFeatureFlag } from '../../composables/featureFlags';
import { BasePage } from '../base';
const AI_ASSISTANT_FEATURE = {
name: 'aiAssistant',
experimentName: '021_ai_debug_helper',
enabledFor: 'variant',
disabledFor: 'control',
};
export class AIAssistant extends BasePage {
url = '/workflows/new';
getters = {
askAssistantFloatingButton: () => cy.getByTestId('ask-assistant-floating-button'),
askAssistantSidebar: () => cy.getByTestId('ask-assistant-sidebar'),
askAssistantSidebarResizer: () =>
this.getters.askAssistantSidebar().find('[class^=_resizer][data-dir=left]').first(),
askAssistantChat: () => cy.getByTestId('ask-assistant-chat'),
placeholderMessage: () => cy.getByTestId('placeholder-message'),
closeChatButton: () => cy.getByTestId('close-chat-button'),
chatInputWrapper: () => cy.getByTestId('chat-input-wrapper'),
chatInput: () => cy.getByTestId('chat-input'),
sendMessageButton: () => cy.getByTestId('send-message-button'),
chatMessagesAll: () => cy.get('[data-test-id^=chat-message]'),
chatMessagesAssistant: () => cy.getByTestId('chat-message-assistant'),
chatMessagesUser: () => cy.getByTestId('chat-message-user'),
chatMessagesSystem: () => cy.getByTestId('chat-message-system'),
quickReplies: () => cy.getByTestId('quick-replies').find('button'),
newAssistantSessionModal: () => cy.getByTestId('new-assistant-session-modal'),
codeDiffs: () => cy.getByTestId('code-diff-suggestion'),
applyCodeDiffButtons: () => cy.getByTestId('replace-code-button'),
undoReplaceCodeButtons: () => cy.getByTestId('undo-replace-button'),
codeReplacedMessage: () => cy.getByTestId('code-replaced-message'),
nodeErrorViewAssistantButton: () =>
cy.getByTestId('node-error-view-ask-assistant-button').find('button').first(),
};
actions = {
enableAssistant(): void {
overrideFeatureFlag(AI_ASSISTANT_FEATURE.experimentName, AI_ASSISTANT_FEATURE.enabledFor);
cy.enableFeature(AI_ASSISTANT_FEATURE.name);
},
disableAssistant(): void {
overrideFeatureFlag(AI_ASSISTANT_FEATURE.experimentName, AI_ASSISTANT_FEATURE.disabledFor);
cy.disableFeature(AI_ASSISTANT_FEATURE.name);
},
};
}

View file

@ -102,13 +102,20 @@ function growInput() {
</div>
<BetaTag />
</div>
<div :class="$style.back" @click="onClose">
<div :class="$style.back" data-test-id="close-chat-button" @click="onClose">
<n8n-icon icon="arrow-right" color="text-base" />
</div>
</div>
<div :class="$style.body">
<div v-if="messages?.length" :class="$style.messages">
<div v-for="(message, i) in messages" :key="i" :class="$style.message">
<div
v-for="(message, i) in messages"
:key="i"
:class="$style.message"
:data-test-id="
message.role === 'assistant' ? 'chat-message-assistant' : 'chat-message-user'
"
>
<div
v-if="
!isEndOfSessionEvent(message) && (i === 0 || message.role !== messages[i - 1].role)
@ -158,7 +165,11 @@ function growInput() {
v-if="streaming && i === messages?.length - 1 && message.role === 'assistant'"
/>
</div>
<div v-else-if="message.type === 'error'" :class="$style.error">
<div
v-else-if="message.type === 'error'"
:class="$style.error"
data-test-id="chat-message-system"
>
<span> {{ message.content }}</span>
</div>
<div v-else-if="message.type === 'code-diff'">
@ -173,7 +184,11 @@ function growInput() {
@undo="() => emit('codeUndo', i)"
/>
</div>
<div v-else-if="isEndOfSessionEvent(message)" :class="$style.endOfSessionText">
<div
v-else-if="isEndOfSessionEvent(message)"
:class="$style.endOfSessionText"
data-test-id="chat-message-system"
>
<span>
{{ t('assistantChat.sessionEndMessage.1') }}
</span>
@ -193,7 +208,7 @@ function growInput() {
:class="$style.quickReplies"
>
<div :class="$style.quickRepliesTitle">{{ t('assistantChat.quickRepliesTitle') }}</div>
<div v-for="opt in message.quickReplies" :key="opt.type">
<div v-for="opt in message.quickReplies" :key="opt.type" data-test-id="quick-replies">
<n8n-button
v-if="opt.text"
type="secondary"
@ -207,7 +222,7 @@ function growInput() {
</div>
</div>
<div v-else :class="$style.placeholder">
<div v-else :class="$style.placeholder" data-test-id="placeholder-message">
<div :class="$style.greeting">Hi {{ user?.firstName }} 👋</div>
<div :class="$style.info">
<p>
@ -232,6 +247,7 @@ function growInput() {
<div
v-if="messages?.length"
:class="{ [$style.inputWrapper]: true, [$style.disabledInput]: sessionEnded }"
data-test-id="chat-input-wrapper"
>
<textarea
ref="chatInput"
@ -240,6 +256,7 @@ function growInput() {
:placeholder="t('assistantChat.inputPlaceholder')"
rows="1"
wrap="hard"
data-test-id="chat-input"
@keydown.enter.exact.prevent="onSendMessage"
@input.prevent="growInput"
@keydown.stop
@ -249,6 +266,7 @@ function growInput() {
icon="paper-plane"
type="text"
size="large"
data-test-id="send-message-button"
:disabled="sendDisabled"
@click="onSendMessage"
/>

View file

@ -82,7 +82,7 @@ const diffs = computed(() => {
</script>
<template>
<div :class="$style.container">
<div :class="$style.container" data-test-id="code-diff-suggestion">
<div :class="$style.title">
{{ title }}
</div>
@ -109,17 +109,26 @@ const diffs = computed(() => {
<span :class="$style.infoText">{{ t('codeDiff.couldNotReplace') }}</span>
</div>
<div v-else-if="replaced">
<n8n-button type="secondary" size="mini" icon="undo" @click="() => emit('undo')">{{
t('codeDiff.undo')
}}</n8n-button>
<n8n-button
type="secondary"
size="mini"
icon="undo"
data-test-id="undo-replace-button"
@click="() => emit('undo')"
>
{{ t('codeDiff.undo') }}
</n8n-button>
<n8n-icon icon="check" color="success" class="ml-xs" />
<span :class="$style.infoText">{{ t('codeDiff.codeReplaced') }}</span>
<span :class="$style.infoText" data-test-id="code-replaced-message">
{{ t('codeDiff.codeReplaced') }}
</span>
</div>
<n8n-button
v-else
:type="replacing ? 'secondary' : 'primary'"
size="mini"
icon="refresh"
data-test-id="replace-code-button"
:disabled="!content || streaming"
:loading="replacing"
@click="() => emit('replace')"

View file

@ -71,6 +71,7 @@ function onClose() {
:supported-directions="['left']"
:width="assistantStore.chatWidth"
:class="$style.container"
data-test-id="ask-assistant-sidebar"
@resize="onResizeDebounced"
>
<div

View file

@ -45,6 +45,7 @@ const onClick = () => {
<div
v-if="assistantStore.canShowAssistantButtons && !assistantStore.isAssistantOpen"
:class="$style.container"
data-test-id="ask-assistant-floating-button"
>
<n8n-tooltip
:z-index="4000"

View file

@ -47,7 +47,13 @@ const startNewSession = async () => {
</script>
<template>
<Modal width="460px" height="250px" :name="NEW_ASSISTANT_SESSION_MODAL" :center="true">
<Modal
width="460px"
height="250px"
:name="NEW_ASSISTANT_SESSION_MODAL"
:center="true"
data-test-id="new-assistant-session-modal"
>
<template #header>
{{ i18n.baseText('aiAssistant.newSessionModal.title.part1') }}
<span :class="$style.assistantIcon"><AssistantIcon size="medium" /></span>

View file

@ -462,7 +462,11 @@ async function onAskAssistantClick() {
class="node-error-view__header-description"
v-html="getErrorDescription()"
></div>
<div v-if="isAskAssistantAvailable" class="node-error-view__assistant-button">
<div
v-if="isAskAssistantAvailable"
class="node-error-view__assistant-button"
data-test-id="node-error-view-ask-assistant-button"
>
<InlineAskAssistantButton :asked="assistantAlreadyAsked" @click="onAskAssistantClick" />
</div>
</div>