mirror of
https://github.com/n8n-io/n8n.git
synced 2025-01-11 21:07:28 -08:00
✨ Add Apache Kafka node (#1109)
* ✨ Add Apache Kafka node * ⚡ Improvements Co-authored-by: Tanay Pant <tanaypant@protonmail.com>
This commit is contained in:
Ricardo Espinoza
GitHub
parent
0f0dfa2191
commit
12bb00fcd6
30
packages/nodes-base/credentials/Kafka.credentials.ts
Normal file
30
packages/nodes-base/credentials/Kafka.credentials.ts
Normal file
|
|
@ -0,0 +1,30 @@
|
|||
import {
|
||||
ICredentialType,
|
||||
NodePropertyTypes,
|
||||
} from 'n8n-workflow';
|
||||
|
||||
export class Kafka implements ICredentialType {
|
||||
name = 'kafka';
|
||||
displayName = 'Kafka';
|
||||
documentationUrl = 'kafka';
|
||||
properties = [
|
||||
{
|
||||
displayName: 'Client ID',
|
||||
name: 'clientId',
|
||||
type: 'string' as NodePropertyTypes,
|
||||
default: '',
|
||||
},
|
||||
{
|
||||
displayName: 'Brokers',
|
||||
name: 'brokers',
|
||||
type: 'string' as NodePropertyTypes,
|
||||
default: '',
|
||||
},
|
||||
{
|
||||
displayName: 'SSL',
|
||||
name: 'ssl',
|
||||
type: 'boolean' as NodePropertyTypes,
|
||||
default: true,
|
||||
},
|
||||
];
|
||||
}
|
||||
29
packages/nodes-base/credentials/KafkaPlain.credentials.ts
Normal file
29
packages/nodes-base/credentials/KafkaPlain.credentials.ts
Normal file
|
|
@ -0,0 +1,29 @@
|
|||
import {
|
||||
ICredentialType,
|
||||
NodePropertyTypes,
|
||||
} from 'n8n-workflow';
|
||||
|
||||
export class KafkaPlain implements ICredentialType {
|
||||
extends = [
|
||||
'kafka',
|
||||
];
|
||||
name = 'kafkaPlain';
|
||||
displayName = 'Kafka';
|
||||
properties = [
|
||||
{
|
||||
displayName: 'Username',
|
||||
name: 'username',
|
||||
type: 'string' as NodePropertyTypes,
|
||||
default: '',
|
||||
},
|
||||
{
|
||||
displayName: 'Password',
|
||||
name: 'password',
|
||||
type: 'string' as NodePropertyTypes,
|
||||
typeOptions: {
|
||||
password: true,
|
||||
},
|
||||
default: '',
|
||||
},
|
||||
];
|
||||
}
|
||||
287
packages/nodes-base/nodes/Kafka/Kafka.node.ts
Normal file
287
packages/nodes-base/nodes/Kafka/Kafka.node.ts
Normal file
|
|
@ -0,0 +1,287 @@
|
|||
import {
|
||||
CompressionTypes,
|
||||
Kafka as apacheKafka,
|
||||
KafkaConfig,
|
||||
SASLOptions,
|
||||
TopicMessages,
|
||||
} from 'kafkajs';
|
||||
|
||||
import {
|
||||
IExecuteFunctions,
|
||||
} from 'n8n-core';
|
||||
|
||||
import {
|
||||
IDataObject,
|
||||
INodeExecutionData,
|
||||
INodeType,
|
||||
INodeTypeDescription,
|
||||
} from 'n8n-workflow';
|
||||
|
||||
export class Kafka implements INodeType {
|
||||
description: INodeTypeDescription = {
|
||||
displayName: 'Kafka',
|
||||
name: 'kafka',
|
||||
icon: 'file:kafka.svg',
|
||||
group: ['transform'],
|
||||
version: 1,
|
||||
description: 'Sends messages to a Kafka topic',
|
||||
defaults: {
|
||||
name: 'Kafka',
|
||||
color: '#000000',
|
||||
},
|
||||
inputs: ['main'],
|
||||
outputs: ['main'],
|
||||
credentials: [
|
||||
{
|
||||
name: 'kafka',
|
||||
displayOptions: {
|
||||
show: {
|
||||
authentication: [
|
||||
'none',
|
||||
],
|
||||
},
|
||||
},
|
||||
required: true,
|
||||
},
|
||||
{
|
||||
name: 'kafkaPlain',
|
||||
displayOptions: {
|
||||
show: {
|
||||
authentication: [
|
||||
'plain',
|
||||
],
|
||||
},
|
||||
},
|
||||
required: true,
|
||||
},
|
||||
],
|
||||
properties: [
|
||||
{
|
||||
displayName: 'Authentication',
|
||||
name: 'authentication',
|
||||
type: 'options',
|
||||
options: [
|
||||
{
|
||||
name: 'None',
|
||||
value: 'none',
|
||||
},
|
||||
{
|
||||
name: 'Plain',
|
||||
value: 'plain',
|
||||
},
|
||||
],
|
||||
default: 'none',
|
||||
},
|
||||
{
|
||||
displayName: 'Topic',
|
||||
name: 'topic',
|
||||
type: 'string',
|
||||
default: '',
|
||||
placeholder: 'topic-name',
|
||||
description: 'Name of the queue of topic to publish to',
|
||||
},
|
||||
{
|
||||
displayName: 'Message',
|
||||
name: 'message',
|
||||
type: 'string',
|
||||
default: '',
|
||||
description: 'The message to be sent',
|
||||
},
|
||||
{
|
||||
displayName: 'JSON Parameters',
|
||||
name: 'jsonParameters',
|
||||
type: 'boolean',
|
||||
default: false,
|
||||
},
|
||||
{
|
||||
displayName: 'Headers',
|
||||
name: 'headersUi',
|
||||
placeholder: 'Add Header',
|
||||
type: 'fixedCollection',
|
||||
displayOptions: {
|
||||
show: {
|
||||
jsonParameters: [
|
||||
false,
|
||||
],
|
||||
},
|
||||
},
|
||||
typeOptions: {
|
||||
multipleValues: true,
|
||||
},
|
||||
default: {},
|
||||
options: [
|
||||
{
|
||||
name: 'headerValues',
|
||||
displayName: 'Header',
|
||||
values: [
|
||||
{
|
||||
displayName: 'Key',
|
||||
name: 'key',
|
||||
type: 'string',
|
||||
default: '',
|
||||
},
|
||||
{
|
||||
displayName: 'Value',
|
||||
name: 'value',
|
||||
type: 'string',
|
||||
default: '',
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
displayName: 'Headers (JSON)',
|
||||
name: 'headerParametersJson',
|
||||
type: 'json',
|
||||
displayOptions: {
|
||||
show: {
|
||||
jsonParameters: [
|
||||
true,
|
||||
],
|
||||
},
|
||||
},
|
||||
default: '',
|
||||
description: 'Header parameters as JSON (flat object)',
|
||||
},
|
||||
{
|
||||
displayName: 'Options',
|
||||
name: 'options',
|
||||
type: 'collection',
|
||||
default: {},
|
||||
placeholder: 'Add Option',
|
||||
options: [
|
||||
{
|
||||
displayName: 'Acks',
|
||||
name: 'acks',
|
||||
type: 'boolean',
|
||||
default: false,
|
||||
description: 'Whether or not producer must wait for acknowledgement from all replicas',
|
||||
},
|
||||
{
|
||||
displayName: 'Compression',
|
||||
name: 'compression',
|
||||
type: 'boolean',
|
||||
default: false,
|
||||
description: 'Send the data in a compressed format using the GZIP codec',
|
||||
},
|
||||
{
|
||||
displayName: 'Timeout',
|
||||
name: 'timeout',
|
||||
type: 'number',
|
||||
default: 30000,
|
||||
description: 'The time to await a response in ms',
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
async execute(this: IExecuteFunctions): Promise<INodeExecutionData[][]> {
|
||||
const items = this.getInputData();
|
||||
|
||||
const length = items.length as unknown as number;
|
||||
|
||||
const authentication = this.getNodeParameter('authentication', 0) as string;
|
||||
|
||||
const topicMessages: TopicMessages[] = [];
|
||||
|
||||
let responseData;
|
||||
|
||||
const options = this.getNodeParameter('options', 0) as IDataObject;
|
||||
|
||||
const timeout = options.timeout as number;
|
||||
|
||||
let compression = CompressionTypes.None;
|
||||
|
||||
const acks = (options.acks === true) ? 1 : 0;
|
||||
|
||||
if (options.compression === true) {
|
||||
compression = CompressionTypes.GZIP;
|
||||
}
|
||||
|
||||
let credentials: IDataObject = {};
|
||||
|
||||
const sasl: SASLOptions | IDataObject = {};
|
||||
|
||||
const brokers = (credentials.brokers as string || '').split(',') as string[];
|
||||
|
||||
const clientId = credentials.clientId as string;
|
||||
|
||||
const ssl = credentials.ssl as boolean;
|
||||
|
||||
const config: KafkaConfig = {
|
||||
clientId,
|
||||
brokers,
|
||||
ssl,
|
||||
//@ts-ignore
|
||||
sasl,
|
||||
};
|
||||
|
||||
if (authentication === 'plain') {
|
||||
credentials = this.getCredentials('kafkaPlain') as IDataObject;
|
||||
sasl.username = credentials.username as string;
|
||||
sasl.password = credentials.password as string;
|
||||
sasl.mechanism = 'plain';
|
||||
} else {
|
||||
credentials = this.getCredentials('kafka') as IDataObject;
|
||||
delete config.sasl;
|
||||
}
|
||||
|
||||
const kafka = new apacheKafka(config);
|
||||
|
||||
const producer = kafka.producer();
|
||||
|
||||
await producer.connect();
|
||||
|
||||
for (let i = 0; i < length; i++) {
|
||||
|
||||
const message = this.getNodeParameter('message', i) as string;
|
||||
|
||||
const topic = this.getNodeParameter('topic', i) as string;
|
||||
|
||||
const jsonParameters = this.getNodeParameter('jsonParameters', i) as boolean;
|
||||
|
||||
let headers;
|
||||
|
||||
if (jsonParameters === true) {
|
||||
headers = this.getNodeParameter('headerParametersJson', i) as string;
|
||||
try {
|
||||
headers = JSON.parse(headers);
|
||||
} catch (exception) {
|
||||
throw new Error('Headers must be a valid json');
|
||||
}
|
||||
} else {
|
||||
const values = (this.getNodeParameter('headersUi', i) as IDataObject).headerValues as IDataObject[];
|
||||
headers = {};
|
||||
if (values !== undefined) {
|
||||
for (const value of values) {
|
||||
//@ts-ignore
|
||||
headers[value.key] = value.value;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
topicMessages.push(
|
||||
{
|
||||
topic,
|
||||
messages: [{
|
||||
value: message,
|
||||
headers,
|
||||
}],
|
||||
});
|
||||
}
|
||||
|
||||
responseData = await producer.sendBatch(
|
||||
{
|
||||
topicMessages,
|
||||
timeout,
|
||||
compression,
|
||||
acks,
|
||||
});
|
||||
|
||||
await producer.disconnect();
|
||||
|
||||
return [this.helpers.returnJsonArray(responseData)];
|
||||
}
|
||||
}
|
||||
1
packages/nodes-base/nodes/Kafka/kafka.svg
Normal file
1
packages/nodes-base/nodes/Kafka/kafka.svg
Normal file
|
|
@ -0,0 +1 @@
|
|||
<svg xmlns="http://www.w3.org/2000/svg" width="64" height="64" viewBox="0 0 32 32" preserveAspectRatio="xMidYMid"><path d="M21.538 17.724a4.16 4.16 0 0 0-3.128 1.42l-1.96-1.388c.208-.573.328-1.188.328-1.832a5.35 5.35 0 0 0-.317-1.802l1.956-1.373a4.16 4.16 0 0 0 3.122 1.414 4.18 4.18 0 0 0 4.172-4.172 4.18 4.18 0 0 0-4.172-4.172 4.18 4.18 0 0 0-4.172 4.172c0 .412.062.8.174 1.185l-1.957 1.374c-.818-1.014-1.995-1.723-3.336-1.94V8.25a4.18 4.18 0 0 0 3.313-4.082A4.18 4.18 0 0 0 11.388 0a4.18 4.18 0 0 0-4.172 4.172c0 1.98 1.387 3.637 3.24 4.063v2.4C7.928 11.067 6 13.273 6 15.925c0 2.665 1.947 4.88 4.493 5.308v2.523c-1.87.4-3.276 2.08-3.276 4.072A4.18 4.18 0 0 0 11.388 32a4.18 4.18 0 0 0 4.172-4.172c0-1.993-1.405-3.66-3.276-4.072v-2.523c1.315-.22 2.47-.916 3.28-1.907l1.973 1.397a4.15 4.15 0 0 0-.171 1.173 4.18 4.18 0 0 0 4.172 4.172 4.18 4.18 0 0 0 4.172-4.172 4.18 4.18 0 0 0-4.172-4.172zm0-9.754c1.115 0 2.022.908 2.022 2.023s-.907 2.022-2.022 2.022-2.022-.907-2.022-2.022.907-2.023 2.022-2.023zM9.366 4.172c0-1.115.907-2.022 2.023-2.022s2.022.907 2.022 2.022-.907 2.022-2.022 2.022-2.023-.907-2.023-2.022zM13.41 27.83c0 1.115-.907 2.022-2.022 2.022s-2.023-.907-2.023-2.022.907-2.022 2.023-2.022 2.022.907 2.022 2.022zm-2.023-9.082c-1.556 0-2.82-1.265-2.82-2.82s1.265-2.82 2.82-2.82 2.82 1.265 2.82 2.82-1.265 2.82-2.82 2.82zm10.15 5.172c-1.115 0-2.022-.908-2.022-2.023s.907-2.022 2.022-2.022 2.022.907 2.022 2.022-.907 2.023-2.022 2.023z"/></svg>
|
||||
|
After Width: | Height: | Size: 1.4 KiB |
|
|
@ -101,6 +101,8 @@
|
|||
"dist/credentials/JiraSoftwareCloudApi.credentials.js",
|
||||
"dist/credentials/JiraSoftwareServerApi.credentials.js",
|
||||
"dist/credentials/JotFormApi.credentials.js",
|
||||
"dist/credentials/Kafka.credentials.js",
|
||||
"dist/credentials/KafkaPlain.credentials.js",
|
||||
"dist/credentials/KeapOAuth2Api.credentials.js",
|
||||
"dist/credentials/LinkedInOAuth2Api.credentials.js",
|
||||
"dist/credentials/MailerLiteApi.credentials.js",
|
||||
|
|
@ -299,6 +301,7 @@
|
|||
"dist/nodes/Jira/Jira.node.js",
|
||||
"dist/nodes/Jira/JiraTrigger.node.js",
|
||||
"dist/nodes/JotForm/JotFormTrigger.node.js",
|
||||
"dist/nodes/Kafka/Kafka.node.js",
|
||||
"dist/nodes/Keap/Keap.node.js",
|
||||
"dist/nodes/Keap/KeapTrigger.node.js",
|
||||
"dist/nodes/LinkedIn/LinkedIn.node.js",
|
||||
|
|
@ -454,6 +457,7 @@
|
|||
"imap-simple": "^4.3.0",
|
||||
"iso-639-1": "^2.1.3",
|
||||
"jsonwebtoken": "^8.5.1",
|
||||
"kafkajs": "^1.14.0",
|
||||
"lodash.get": "^4.4.2",
|
||||
"lodash.set": "^4.3.2",
|
||||
"lodash.unset": "^4.5.2",
|
||||
|
|
|
|||
Loading…
Reference in a new issue