✨ Add Urlscan.io node (#2266)

* ✨ Create urlscan.io node

* ⚡ Change default visibility to private

Co-authored-by: ricardo <ricardoespinoza105@gmail.com>

packages/nodes-base/credentials/UrlScanIoApi.credentials.ts

@@ -0,0 +1,19 @@
+import {
+	ICredentialType,
+	INodeProperties,
+} from 'n8n-workflow';
+
+export class UrlScanIoApi implements ICredentialType {
+	name = 'urlScanIoApi';
+	displayName = 'urlscan.io API';
+	documentationUrl = 'urlScanIo';
+	properties: INodeProperties[] = [
+		{
+			displayName: 'API Key',
+			name: 'apiKey',
+			type: 'string',
+			default: '',
+			required: true,
+		},
+	];
+}

packages/nodes-base/nodes/UrlScanIo/GenericFunctions.ts

@@ -0,0 +1,85 @@
+import {
+	OptionsWithUri,
+} from 'request';
+
+import {
+	IExecuteFunctions,
+} from 'n8n-core';
+
+import {
+	IDataObject,
+	NodeApiError,
+} from 'n8n-workflow';
+
+export async function urlScanIoApiRequest(
+	this: IExecuteFunctions,
+	method: 'GET' | 'POST',
+	endpoint: string,
+	body: IDataObject = {},
+	qs: IDataObject = {},
+) {
+	const { apiKey } = await this.getCredentials('urlScanIoApi') as { apiKey: string };
+
+	const options: OptionsWithUri = {
+		headers: {
+			'API-KEY': apiKey,
+		},
+		method,
+		body,
+		qs,
+		uri: `https://urlscan.io/api/v1${endpoint}`,
+		json: true,
+	};
+
+	if (!Object.keys(body).length) {
+		delete options.body;
+	}
+
+	if (!Object.keys(qs).length) {
+		delete options.qs;
+	}
+
+	try {
+		return await this.helpers.request(options);
+	} catch (error) {
+		throw new NodeApiError(this.getNode(), error);
+	}
+}
+
+
+export async function handleListing(
+	this: IExecuteFunctions,
+	endpoint: string,
+	qs: IDataObject = {},
+): Promise<IDataObject[]> {
+	const returnData: IDataObject[] = [];
+	let responseData;
+
+	qs.size = 100;
+
+	const returnAll = this.getNodeParameter('returnAll', 0, false) as boolean;
+	const limit = this.getNodeParameter('limit', 0, 0) as number;
+
+	do {
+		responseData = await urlScanIoApiRequest.call(this, 'GET', endpoint, {}, qs);
+		returnData.push(...responseData.results);
+
+		if (!returnAll && returnData.length > limit) {
+			return returnData.slice(0, limit);
+		}
+
+		if (responseData.results.length) {
+			const lastResult = responseData.results[responseData.results.length -1];
+			qs.search_after = lastResult.sort;
+		}
+
+	} while (responseData.total > returnData.length);
+
+	return returnData;
+}
+
+export const normalizeId = ({ _id, uuid, ...rest }: IDataObject) => {
+	if (_id) return ({ scanId: _id, ...rest });
+	if (uuid) return ({ scanId: uuid, ...rest });
+	return rest;
+};

packages/nodes-base/nodes/UrlScanIo/UrlScanIo.node.ts

@@ -0,0 +1,212 @@
+import {
+	IExecuteFunctions,
+} from 'n8n-core';
+
+import {
+	ICredentialsDecrypted,
+	ICredentialTestFunctions,
+	IDataObject,
+	INodeExecutionData,
+	INodeType,
+	INodeTypeDescription,
+	NodeCredentialTestResult,
+	NodeOperationError,
+} from 'n8n-workflow';
+
+import {
+	OptionsWithUri,
+} from 'request';
+
+import {
+	scanFields,
+	scanOperations,
+} from './descriptions';
+
+import {
+	handleListing,
+	normalizeId,
+	urlScanIoApiRequest,
+} from './GenericFunctions';
+
+export class UrlScanIo implements INodeType {
+	description: INodeTypeDescription = {
+		displayName: 'urlscan.io',
+		name: 'urlScanIo',
+		icon: 'file:urlScanIo.svg',
+		group: ['transform'],
+		version: 1,
+		subtitle: '={{$parameter["operation"] + ": " + $parameter["resource"]}}',
+		description: 'Consume the urlscan.io API',
+		defaults: {
+			name: 'urlscan.io',
+			color: '#f3d337',
+		},
+		inputs: ['main'],
+		outputs: ['main'],
+		credentials: [
+			{
+				name: 'urlScanIoApi',
+				required: true,
+				testedBy: 'urlScanIoApiTest',
+			},
+		],
+		properties: [
+			{
+				displayName: 'Resource',
+				name: 'resource',
+				noDataExpression: true,
+				type: 'options',
+				options: [
+					{
+						name: 'Scan',
+						value: 'scan',
+					},
+				],
+				default: 'scan',
+			},
+			...scanOperations,
+			...scanFields,
+		],
+	};
+
+	methods = {
+		credentialTest: {
+			async urlScanIoApiTest(
+				this: ICredentialTestFunctions,
+				credentials: ICredentialsDecrypted,
+			): Promise<NodeCredentialTestResult> {
+				const { apiKey } = credentials.data as { apiKey: string };
+
+				const options: OptionsWithUri = {
+					headers: {
+						'API-KEY': apiKey,
+					},
+					method: 'GET',
+					uri: 'https://urlscan.io/user/quotas',
+					json: true,
+				};
+
+				try {
+					await this.helpers.request(options);
+					return {
+						status: 'OK',
+						message: 'Authentication successful',
+					};
+				} catch (error) {
+					return {
+						status: 'Error',
+						message: error.message,
+					};
+				}
+			},
+		},
+	};
+
+	async execute(this: IExecuteFunctions): Promise<INodeExecutionData[][]> {
+		const items = this.getInputData();
+		const returnData: IDataObject[] = [];
+
+		const resource = this.getNodeParameter('resource', 0) as 'scan';
+		const operation = this.getNodeParameter('operation', 0) as 'perform' | 'get' | 'getAll';
+
+		let responseData;
+
+		for (let i = 0; i < items.length; i++) {
+
+			try {
+
+				if (resource === 'scan') {
+
+					// **********************************************************************
+					//                               scan
+					// **********************************************************************
+
+					if (operation === 'get') {
+
+						// ----------------------------------------
+						//               scan: get
+						// ----------------------------------------
+
+						const scanId = this.getNodeParameter('scanId', i) as string;
+						responseData = await urlScanIoApiRequest.call(this, 'GET', `/result/${scanId}`);
+
+					} else if (operation === 'getAll') {
+
+						// ----------------------------------------
+						//             scan: getAll
+						// ----------------------------------------
+
+						// https://urlscan.io/docs/search
+
+						const filters = this.getNodeParameter('filters', i) as { query?: string };
+
+						const qs: IDataObject = {};
+
+						if (filters?.query) {
+							qs.q = filters.query;
+						}
+
+						responseData = await handleListing.call(this, '/search', qs);
+						responseData = responseData.map(normalizeId);
+
+					} else if (operation === 'perform') {
+
+						// ----------------------------------------
+						//             scan: perform
+						// ----------------------------------------
+
+						// https://urlscan.io/docs/search
+
+						const {
+							tags: rawTags,
+							...rest
+						} = this.getNodeParameter('additionalFields', i) as {
+							customAgent?: string;
+							visibility?: 'public' | 'private' | 'unlisted';
+							tags?: string;
+							referer?: string;
+							overrideSafety: string;
+						};
+
+						const body: IDataObject = {
+							url: this.getNodeParameter('url', i) as string,
+							...rest,
+						};
+
+						if (rawTags) {
+							const tags = rawTags.split(',').map(tag => tag.trim());
+
+							if (tags.length > 10) {
+								throw new NodeOperationError(
+									this.getNode(),
+									'Please enter at most 10 tags',
+								);
+							}
+
+							body.tags = tags;
+						}
+
+						responseData = await urlScanIoApiRequest.call(this, 'POST', '/scan', body);
+						responseData = normalizeId(responseData);
+
+					}
+
+				}
+
+				Array.isArray(responseData)
+					? returnData.push(...responseData)
+					: returnData.push(responseData);
+
+			} catch (error) {
+				if (this.continueOnFail()) {
+					returnData.push({ error: error.message });
+					continue;
+				}
+				throw error;
+			}
+
+		}
+
+		return [this.helpers.returnJsonArray(returnData)];
+	}
+}

packages/nodes-base/nodes/UrlScanIo/descriptions/ScanDescription.ts

@@ -0,0 +1,218 @@
+import {
+	INodeProperties,
+} from 'n8n-workflow';
+
+export const scanOperations: INodeProperties[] = [
+	{
+		displayName: 'Operation',
+		name: 'operation',
+		type: 'options',
+		noDataExpression: true,
+		displayOptions: {
+			show: {
+				resource: [
+					'scan',
+				],
+			},
+		},
+		options: [
+			{
+				name: 'Get',
+				value: 'get',
+			},
+			{
+				name: 'Get All',
+				value: 'getAll',
+			},
+			{
+				name: 'Perform',
+				value: 'perform',
+			},
+		],
+		default: 'perform',
+	},
+];
+
+export const scanFields: INodeProperties[] = [
+	// ----------------------------------------
+	//               scan: get
+	// ----------------------------------------
+	{
+		displayName: 'Scan ID',
+		name: 'scanId',
+		type: 'string',
+		default: '',
+		description: 'ID of the scan to retrieve',
+		displayOptions: {
+			show: {
+				resource: [
+					'scan',
+				],
+				operation: [
+					'get',
+				],
+			},
+		},
+	},
+
+	// ----------------------------------------
+	//             scan: getAll
+	// ----------------------------------------
+	{
+		displayName: 'Return All',
+		name: 'returnAll',
+		type: 'boolean',
+		default: false,
+		description: 'Whether to return all results or only up to a given limit',
+		displayOptions: {
+			show: {
+				resource: [
+					'scan',
+				],
+				operation: [
+					'getAll',
+				],
+			},
+		},
+	},
+	{
+		displayName: 'Limit',
+		name: 'limit',
+		type: 'number',
+		default: 50,
+		description: 'Max number of results to return',
+		typeOptions: {
+			minValue: 1,
+		},
+		displayOptions: {
+			show: {
+				resource: [
+					'scan',
+				],
+				operation: [
+					'getAll',
+				],
+				returnAll: [
+					false,
+				],
+			},
+		},
+	},
+	{
+		displayName: 'Filters',
+		name: 'filters',
+		type: 'collection',
+		placeholder: 'Add Filter',
+		default: {},
+		displayOptions: {
+			show: {
+				resource: [
+					'scan',
+				],
+				operation: [
+					'getAll',
+				],
+			},
+		},
+		options: [
+			{
+				displayName: 'Query',
+				name: 'query',
+				type: 'string',
+				description: 'Query using the <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-query-string-query.html#query-dsl-query-string-query">Elastic Search Query String syntax</a>. See <a href="https://urlscan.io/docs/search/">supported fields</a> in the documentation.',
+				default: '',
+				placeholder: 'domain:n8n.io',
+			},
+		],
+	},
+
+	// ----------------------------------------
+	//             scan: perform
+	// ----------------------------------------
+	{
+		displayName: 'URL',
+		name: 'url',
+		type: 'string',
+		default: '',
+		placeholder: 'https://n8n.io',
+		description: 'URL to scan',
+		displayOptions: {
+			show: {
+				resource: [
+					'scan',
+				],
+				operation: [
+					'perform',
+				],
+			},
+		},
+	},
+	{
+		displayName: 'Additional Fields',
+		name: 'additionalFields',
+		type: 'collection',
+		placeholder: 'Add Field',
+		default: {},
+		displayOptions: {
+			show: {
+				resource: [
+					'scan',
+				],
+				operation: [
+					'perform',
+				],
+			},
+		},
+		options: [
+			{
+				displayName: 'Custom Agent',
+				name: 'customAgent',
+				description: '<code>User-Agent</code> header to set for this scan. Defaults to <code>n8n</code>',
+				type: 'string',
+				default: '',
+			},
+			{
+				displayName: 'Override Safety',
+				name: 'overrideSafety',
+				description: 'Disable reclassification of URLs with potential PII in them',
+				type: 'string',
+				default: '',
+			},
+			{
+				displayName: 'Referer',
+				name: 'referer',
+				description: 'HTTP referer to set for this scan',
+				type: 'string',
+				default: '',
+			},
+			{
+				displayName: 'Tags',
+				name: 'tags',
+				description: 'Comma-separated list of user-defined tags to add to this scan. Limited to 10 tags.',
+				placeholder: 'phishing, malicious',
+				type: 'string',
+				default: '',
+			},
+			{
+				displayName: 'Visibility',
+				name: 'visibility',
+				type: 'options',
+				default: 'private',
+				options: [
+					{
+						name: 'Private',
+						value: 'private',
+					},
+					{
+						name: 'Public',
+						value: 'public',
+					},
+					{
+						name: 'Unlisted',
+						value: 'unlisted',
+					},
+				],
+			},
+		],
+	},
+];

packages/nodes-base/nodes/UrlScanIo/descriptions/index.ts

@@ -0,0 +1 @@
+export * from './ScanDescription';

packages/nodes-base/nodes/UrlScanIo/urlScanIo.svg

@@ -0,0 +1 @@
+<svg height="2500" width="2500" xmlns="http://www.w3.org/2000/svg" viewBox="70 70 884 884"><path d="M512 70c244 0 442 198 442 442S756 954 512 954 70 756 70 512 268 70 512 70z" fill="#e35946"/><path d="M772 730c10 9 16 22 16 37 0 29-24 53-53 53-15 0-28-6-37-16L548 655c-34 23-76 37-121 37-120 0-218-98-218-218s98-218 218-218 218 98 218 218c0 37-9 72-26 102z" fill="#b74837"/><path d="M789 721c0 29-24 53-53 53-15 0-28-6-37-16L504 564c32-18 57-46 70-80l199 200c10 9 16 22 16 37z" fill="#294658"/><path d="M428 272c86 0 156 70 156 156s-70 156-156 156-156-70-156-156 70-156 156-156z" fill="#26495d"/><path d="M428 606c-82 0-148-66-148-148s66-148 148-148 148 66 148 148-66 148-148 148z" fill="#3b637d"/><path d="M403 334c23 0 41 18 41 41s-18 41-41 41-41-18-41-41 18-41 41-41z" fill="#9db2c2"/><path d="M428 646c-120 0-218-98-218-218s98-218 218-218 218 98 218 218-98 218-218 218zm0-366c-82 0-148 66-148 148s66 148 148 148 148-66 148-148-66-148-148-148z" fill="#e5e9ec"/></svg>

packages/nodes-base/package.json

@@ -286,6 +286,7 @@
       "dist/credentials/UpleadApi.credentials.js",
       "dist/credentials/UProcApi.credentials.js",
       "dist/credentials/UptimeRobotApi.credentials.js",
+      "dist/credentials/UrlScanIoApi.credentials.js",
       "dist/credentials/VeroApi.credentials.js",
       "dist/credentials/VonageApi.credentials.js",
       "dist/credentials/WebflowApi.credentials.js",
@@ -606,6 +607,7 @@
       "dist/nodes/Uplead/Uplead.node.js",
       "dist/nodes/UProc/UProc.node.js",
       "dist/nodes/UptimeRobot/UptimeRobot.node.js",
+      "dist/nodes/UrlScanIo/UrlScanIo.node.js",
       "dist/nodes/Vero/Vero.node.js",
       "dist/nodes/Vonage/Vonage.node.js",
       "dist/nodes/Wait.node.js",