mirror of
https://github.com/n8n-io/n8n.git
synced 2024-12-25 04:34:06 -08:00
feat(Elasticsearch Node): Add bulk operations for Elasticsearch (#9940)
This commit is contained in:
parent
34334651e0
commit
bf8f848645
|
@ -4,11 +4,16 @@ import type {
|
||||||
INodeExecutionData,
|
INodeExecutionData,
|
||||||
INodeType,
|
INodeType,
|
||||||
INodeTypeDescription,
|
INodeTypeDescription,
|
||||||
|
JsonObject,
|
||||||
} from 'n8n-workflow';
|
} from 'n8n-workflow';
|
||||||
import { jsonParse } from 'n8n-workflow';
|
import { jsonParse, NodeApiError } from 'n8n-workflow';
|
||||||
|
|
||||||
import omit from 'lodash/omit';
|
import omit from 'lodash/omit';
|
||||||
import { elasticsearchApiRequest, elasticsearchApiRequestAllItems } from './GenericFunctions';
|
import {
|
||||||
|
elasticsearchApiRequest,
|
||||||
|
elasticsearchApiRequestAllItems,
|
||||||
|
elasticsearchBulkApiRequest,
|
||||||
|
} from './GenericFunctions';
|
||||||
|
|
||||||
import { documentFields, documentOperations, indexFields, indexOperations } from './descriptions';
|
import { documentFields, documentOperations, indexFields, indexOperations } from './descriptions';
|
||||||
|
|
||||||
|
@ -68,12 +73,14 @@ export class Elasticsearch implements INodeType {
|
||||||
|
|
||||||
let responseData;
|
let responseData;
|
||||||
|
|
||||||
|
let bulkBody: IDataObject = {};
|
||||||
|
|
||||||
for (let i = 0; i < items.length; i++) {
|
for (let i = 0; i < items.length; i++) {
|
||||||
|
const bulkOperation = this.getNodeParameter('options.bulkOperation', i, false);
|
||||||
if (resource === 'document') {
|
if (resource === 'document') {
|
||||||
// **********************************************************************
|
// **********************************************************************
|
||||||
// document
|
// document
|
||||||
// **********************************************************************
|
// **********************************************************************
|
||||||
|
|
||||||
if (operation === 'delete') {
|
if (operation === 'delete') {
|
||||||
// ----------------------------------------
|
// ----------------------------------------
|
||||||
// document: delete
|
// document: delete
|
||||||
|
@ -84,8 +91,17 @@ export class Elasticsearch implements INodeType {
|
||||||
const indexId = this.getNodeParameter('indexId', i);
|
const indexId = this.getNodeParameter('indexId', i);
|
||||||
const documentId = this.getNodeParameter('documentId', i);
|
const documentId = this.getNodeParameter('documentId', i);
|
||||||
|
|
||||||
|
if (bulkOperation) {
|
||||||
|
bulkBody[i] = JSON.stringify({
|
||||||
|
delete: {
|
||||||
|
_index: indexId,
|
||||||
|
_id: documentId,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
} else {
|
||||||
const endpoint = `/${indexId}/_doc/${documentId}`;
|
const endpoint = `/${indexId}/_doc/${documentId}`;
|
||||||
responseData = await elasticsearchApiRequest.call(this, 'DELETE', endpoint);
|
responseData = await elasticsearchApiRequest.call(this, 'DELETE', endpoint);
|
||||||
|
}
|
||||||
} else if (operation === 'get') {
|
} else if (operation === 'get') {
|
||||||
// ----------------------------------------
|
// ----------------------------------------
|
||||||
// document: get
|
// document: get
|
||||||
|
@ -223,6 +239,15 @@ export class Elasticsearch implements INodeType {
|
||||||
const indexId = this.getNodeParameter('indexId', i);
|
const indexId = this.getNodeParameter('indexId', i);
|
||||||
const { documentId } = additionalFields;
|
const { documentId } = additionalFields;
|
||||||
|
|
||||||
|
if (bulkOperation) {
|
||||||
|
bulkBody[i] = JSON.stringify({
|
||||||
|
index: {
|
||||||
|
_index: indexId,
|
||||||
|
_id: documentId,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
bulkBody[i] += `\n${JSON.stringify(body)}`;
|
||||||
|
} else {
|
||||||
if (documentId) {
|
if (documentId) {
|
||||||
const endpoint = `/${indexId}/_doc/${documentId}`;
|
const endpoint = `/${indexId}/_doc/${documentId}`;
|
||||||
responseData = await elasticsearchApiRequest.call(this, 'PUT', endpoint, body);
|
responseData = await elasticsearchApiRequest.call(this, 'PUT', endpoint, body);
|
||||||
|
@ -230,6 +255,7 @@ export class Elasticsearch implements INodeType {
|
||||||
const endpoint = `/${indexId}/_doc`;
|
const endpoint = `/${indexId}/_doc`;
|
||||||
responseData = await elasticsearchApiRequest.call(this, 'POST', endpoint, body);
|
responseData = await elasticsearchApiRequest.call(this, 'POST', endpoint, body);
|
||||||
}
|
}
|
||||||
|
}
|
||||||
} else if (operation === 'update') {
|
} else if (operation === 'update') {
|
||||||
// ----------------------------------------
|
// ----------------------------------------
|
||||||
// document: update
|
// document: update
|
||||||
|
@ -261,8 +287,18 @@ export class Elasticsearch implements INodeType {
|
||||||
const documentId = this.getNodeParameter('documentId', i);
|
const documentId = this.getNodeParameter('documentId', i);
|
||||||
|
|
||||||
const endpoint = `/${indexId}/_update/${documentId}`;
|
const endpoint = `/${indexId}/_update/${documentId}`;
|
||||||
|
if (bulkOperation) {
|
||||||
|
bulkBody[i] = JSON.stringify({
|
||||||
|
update: {
|
||||||
|
_index: indexId,
|
||||||
|
_id: documentId,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
bulkBody[i] += `\n${JSON.stringify(body)}`;
|
||||||
|
} else {
|
||||||
responseData = await elasticsearchApiRequest.call(this, 'POST', endpoint, body);
|
responseData = await elasticsearchApiRequest.call(this, 'POST', endpoint, body);
|
||||||
}
|
}
|
||||||
|
}
|
||||||
} else if (resource === 'index') {
|
} else if (resource === 'index') {
|
||||||
// **********************************************************************
|
// **********************************************************************
|
||||||
// index
|
// index
|
||||||
|
@ -341,13 +377,80 @@ export class Elasticsearch implements INodeType {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (!bulkOperation) {
|
||||||
const executionData = this.helpers.constructExecutionMetaData(
|
const executionData = this.helpers.constructExecutionMetaData(
|
||||||
this.helpers.returnJsonArray(responseData as IDataObject[]),
|
this.helpers.returnJsonArray(responseData as IDataObject[]),
|
||||||
{ itemData: { item: i } },
|
{ itemData: { item: i } },
|
||||||
);
|
);
|
||||||
returnData.push(...executionData);
|
returnData.push(...executionData);
|
||||||
}
|
}
|
||||||
|
if (Object.keys(bulkBody).length >= 50) {
|
||||||
|
responseData = (await elasticsearchBulkApiRequest.call(this, bulkBody)) as IDataObject[];
|
||||||
|
for (let j = 0; j < responseData.length; j++) {
|
||||||
|
const itemData = responseData[j];
|
||||||
|
if (itemData.error) {
|
||||||
|
const errorData = itemData.error as IDataObject;
|
||||||
|
const message = errorData.type as string;
|
||||||
|
const description = errorData.reason as string;
|
||||||
|
const itemIndex = parseInt(Object.keys(bulkBody)[j]);
|
||||||
|
if (this.continueOnFail()) {
|
||||||
|
returnData.push(
|
||||||
|
...this.helpers.constructExecutionMetaData(
|
||||||
|
this.helpers.returnJsonArray({ error: message, message: itemData.error }),
|
||||||
|
{ itemData: { item: itemIndex } },
|
||||||
|
),
|
||||||
|
);
|
||||||
|
continue;
|
||||||
|
} else {
|
||||||
|
throw new NodeApiError(this.getNode(), {
|
||||||
|
message,
|
||||||
|
description,
|
||||||
|
itemIndex,
|
||||||
|
} as JsonObject);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const executionData = this.helpers.constructExecutionMetaData(
|
||||||
|
this.helpers.returnJsonArray(itemData),
|
||||||
|
{ itemData: { item: parseInt(Object.keys(bulkBody)[j]) } },
|
||||||
|
);
|
||||||
|
returnData.push(...executionData);
|
||||||
|
}
|
||||||
|
bulkBody = {};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (Object.keys(bulkBody).length) {
|
||||||
|
responseData = (await elasticsearchBulkApiRequest.call(this, bulkBody)) as IDataObject[];
|
||||||
|
for (let j = 0; j < responseData.length; j++) {
|
||||||
|
const itemData = responseData[j];
|
||||||
|
if (itemData.error) {
|
||||||
|
const errorData = itemData.error as IDataObject;
|
||||||
|
const message = errorData.type as string;
|
||||||
|
const description = errorData.reason as string;
|
||||||
|
const itemIndex = parseInt(Object.keys(bulkBody)[j]);
|
||||||
|
if (this.continueOnFail()) {
|
||||||
|
returnData.push(
|
||||||
|
...this.helpers.constructExecutionMetaData(
|
||||||
|
this.helpers.returnJsonArray({ error: message, message: itemData.error }),
|
||||||
|
{ itemData: { item: itemIndex } },
|
||||||
|
),
|
||||||
|
);
|
||||||
|
continue;
|
||||||
|
} else {
|
||||||
|
throw new NodeApiError(this.getNode(), {
|
||||||
|
message,
|
||||||
|
description,
|
||||||
|
itemIndex,
|
||||||
|
} as JsonObject);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const executionData = this.helpers.constructExecutionMetaData(
|
||||||
|
this.helpers.returnJsonArray(itemData),
|
||||||
|
{ itemData: { item: parseInt(Object.keys(bulkBody)[j]) } },
|
||||||
|
);
|
||||||
|
returnData.push(...executionData);
|
||||||
|
}
|
||||||
|
}
|
||||||
return [returnData];
|
return [returnData];
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -2,13 +2,55 @@ import type {
|
||||||
IExecuteFunctions,
|
IExecuteFunctions,
|
||||||
IDataObject,
|
IDataObject,
|
||||||
JsonObject,
|
JsonObject,
|
||||||
IRequestOptions,
|
IHttpRequestOptions,
|
||||||
IHttpRequestMethods,
|
IHttpRequestMethods,
|
||||||
} from 'n8n-workflow';
|
} from 'n8n-workflow';
|
||||||
import { NodeApiError } from 'n8n-workflow';
|
import { NodeApiError } from 'n8n-workflow';
|
||||||
|
|
||||||
import type { ElasticsearchApiCredentials } from './types';
|
import type { ElasticsearchApiCredentials } from './types';
|
||||||
|
|
||||||
|
export async function elasticsearchBulkApiRequest(this: IExecuteFunctions, body: IDataObject) {
|
||||||
|
const { baseUrl, ignoreSSLIssues } = (await this.getCredentials(
|
||||||
|
'elasticsearchApi',
|
||||||
|
)) as ElasticsearchApiCredentials;
|
||||||
|
|
||||||
|
const bulkBody = Object.values(body).flat().join('\n') + '\n';
|
||||||
|
|
||||||
|
const options: IHttpRequestOptions = {
|
||||||
|
method: 'POST',
|
||||||
|
headers: { 'Content-Type': 'application/x-ndjson' },
|
||||||
|
body: bulkBody,
|
||||||
|
url: `${baseUrl}/_bulk`,
|
||||||
|
skipSslCertificateValidation: ignoreSSLIssues,
|
||||||
|
returnFullResponse: true,
|
||||||
|
ignoreHttpStatusErrors: true,
|
||||||
|
};
|
||||||
|
|
||||||
|
const response = await this.helpers.httpRequestWithAuthentication.call(
|
||||||
|
this,
|
||||||
|
'elasticsearchApi',
|
||||||
|
options,
|
||||||
|
);
|
||||||
|
|
||||||
|
if (response.statusCode > 299) {
|
||||||
|
if (this.continueOnFail()) {
|
||||||
|
return Object.values(body).map((_) => ({ error: response.body.error }));
|
||||||
|
} else {
|
||||||
|
throw new NodeApiError(this.getNode(), { error: response.body.error } as JsonObject);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return response.body.items.map((item: IDataObject) => {
|
||||||
|
return {
|
||||||
|
...(item.index as IDataObject),
|
||||||
|
...(item.update as IDataObject),
|
||||||
|
...(item.create as IDataObject),
|
||||||
|
...(item.delete as IDataObject),
|
||||||
|
...(item.error as IDataObject),
|
||||||
|
};
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
export async function elasticsearchApiRequest(
|
export async function elasticsearchApiRequest(
|
||||||
this: IExecuteFunctions,
|
this: IExecuteFunctions,
|
||||||
method: IHttpRequestMethods,
|
method: IHttpRequestMethods,
|
||||||
|
@ -20,13 +62,13 @@ export async function elasticsearchApiRequest(
|
||||||
'elasticsearchApi',
|
'elasticsearchApi',
|
||||||
)) as ElasticsearchApiCredentials;
|
)) as ElasticsearchApiCredentials;
|
||||||
|
|
||||||
const options: IRequestOptions = {
|
const options: IHttpRequestOptions = {
|
||||||
method,
|
method,
|
||||||
body,
|
body,
|
||||||
qs,
|
qs,
|
||||||
uri: `${baseUrl}${endpoint}`,
|
url: `${baseUrl}${endpoint}`,
|
||||||
json: true,
|
json: true,
|
||||||
rejectUnauthorized: !ignoreSSLIssues,
|
skipSslCertificateValidation: ignoreSSLIssues,
|
||||||
};
|
};
|
||||||
|
|
||||||
if (!Object.keys(body).length) {
|
if (!Object.keys(body).length) {
|
||||||
|
|
|
@ -81,6 +81,28 @@ export const documentFields: INodeProperties[] = [
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
displayName: 'Options',
|
||||||
|
name: 'options',
|
||||||
|
type: 'collection',
|
||||||
|
placeholder: 'Add Option',
|
||||||
|
default: {},
|
||||||
|
displayOptions: {
|
||||||
|
show: {
|
||||||
|
resource: ['document'],
|
||||||
|
operation: ['delete'],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
options: [
|
||||||
|
{
|
||||||
|
displayName: 'Bulk Delete',
|
||||||
|
name: 'bulkOperation',
|
||||||
|
type: 'boolean',
|
||||||
|
default: false,
|
||||||
|
description: 'Whether to use the bulk operation to delete the document/s',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
|
||||||
// ----------------------------------------
|
// ----------------------------------------
|
||||||
// document: get
|
// document: get
|
||||||
|
@ -644,6 +666,13 @@ export const documentFields: INodeProperties[] = [
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
options: [
|
options: [
|
||||||
|
{
|
||||||
|
displayName: 'Bulk Create',
|
||||||
|
name: 'bulkOperation',
|
||||||
|
type: 'boolean',
|
||||||
|
default: false,
|
||||||
|
description: 'Whether to use the bulk operation to create the document/s',
|
||||||
|
},
|
||||||
{
|
{
|
||||||
displayName: 'Pipeline ID',
|
displayName: 'Pipeline ID',
|
||||||
name: 'pipeline',
|
name: 'pipeline',
|
||||||
|
@ -802,6 +831,13 @@ export const documentFields: INodeProperties[] = [
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
options: [
|
options: [
|
||||||
|
{
|
||||||
|
displayName: 'Bulk Update',
|
||||||
|
name: 'bulkOperation',
|
||||||
|
type: 'boolean',
|
||||||
|
default: false,
|
||||||
|
description: 'Whether to use the bulk operation to update the document/s',
|
||||||
|
},
|
||||||
{
|
{
|
||||||
displayName: 'Refresh',
|
displayName: 'Refresh',
|
||||||
name: 'refresh',
|
name: 'refresh',
|
||||||
|
|
Loading…
Reference in a new issue