✨ Add Google BigQuery node (#1635)

* ✨ Google BigQuery node

* 🔨 Add comment dividers

* ⚡ Add whitespace for readability

* ✏️ Edit resource descriptions

* ⚡ Improvements

Co-authored-by: Iván Ovejero <ivov.src@gmail.com>

packages/nodes-base/credentials/GoogleBigQueryOAuth2Api.credentials.ts

@@ -0,0 +1,25 @@
+import {
+	ICredentialType,
+	NodePropertyTypes,
+} from 'n8n-workflow';
+
+const scopes = [
+	'https://www.googleapis.com/auth/bigquery',
+];
+
+export class GoogleBigQueryOAuth2Api implements ICredentialType {
+	name = 'googleBigQueryOAuth2Api';
+	extends = [
+		'googleOAuth2Api',
+	];
+	displayName = 'Google BigQuery OAuth2 API';
+	documentationUrl = 'google';
+	properties = [
+		{
+			displayName: 'Scope',
+			name: 'scope',
+			type: 'hidden' as NodePropertyTypes,
+			default: scopes.join(' '),
+		},
+	];
+}

packages/nodes-base/nodes/Google/BigQuery/GenericFunctions.ts

@@ -0,0 +1,80 @@
+import {
+	OptionsWithUri,
+} from 'request';
+
+import {
+	IExecuteFunctions,
+	IExecuteSingleFunctions,
+	ILoadOptionsFunctions,
+} from 'n8n-core';
+
+import {
+	IDataObject,
+} from 'n8n-workflow';
+
+export async function googleApiRequest(this: IExecuteFunctions | IExecuteSingleFunctions | ILoadOptionsFunctions, method: string, resource: string, body: any = {}, qs: IDataObject = {}, uri?: string, headers: IDataObject = {}): Promise<any> { // tslint:disable-line:no-any
+	const options: OptionsWithUri = {
+		headers: {
+			'Content-Type': 'application/json',
+		},
+		method,
+		body,
+		qs,
+		uri: uri || `https://bigquery.googleapis.com/bigquery${resource}`,
+		json: true,
+	};
+	try {
+		if (Object.keys(headers).length !== 0) {
+			options.headers = Object.assign({}, options.headers, headers);
+		}
+		if (Object.keys(body).length === 0) {
+			delete options.body;
+		}
+		console.log(options);
+		//@ts-ignore
+		return await this.helpers.requestOAuth2.call(this, 'googleBigQueryOAuth2Api', options);
+	} catch (error) {
+		if (error.response && error.response.body && error.response.body.error) {
+
+			let errors = error.response.body.error.errors;
+
+			errors = errors.map((e: IDataObject) => e.message);
+			// Try to return the error prettier
+			throw new Error(
+				`Google BigQuery error response [${error.statusCode}]: ${errors.join('|')}`,
+			);
+		}
+		throw error;
+	}
+}
+
+export async function googleApiRequestAllItems(this: IExecuteFunctions | ILoadOptionsFunctions, propertyName: string, method: string, endpoint: string, body: any = {}, query: IDataObject = {}): Promise<any> { // tslint:disable-line:no-any
+
+	const returnData: IDataObject[] = [];
+
+	let responseData;
+	query.maxResults = 100;
+
+	do {
+		responseData = await googleApiRequest.call(this, method, endpoint, body, query);
+		query.pageToken = responseData['nextPageToken'];
+		returnData.push.apply(returnData, responseData[propertyName]);
+	} while (
+		responseData['nextPageToken'] !== undefined &&
+		responseData['nextPageToken'] !== ''
+	);
+
+	return returnData;
+}
+
+export function simplify(rows: IDataObject[], fields: string[]) {
+	const results = [];
+	for (const row of rows) {
+		const record: IDataObject = {};
+		for (const [index, field] of fields.entries()) {
+			record[field] = (row.f as IDataObject[])[index].v;
+		}
+		results.push(record);
+	}
+	return results;
+}

packages/nodes-base/nodes/Google/BigQuery/GoogleBigQuery.node.ts

@@ -0,0 +1,250 @@
+import {
+	IExecuteFunctions,
+} from 'n8n-core';
+
+import {
+	IDataObject,
+	ILoadOptionsFunctions,
+	INodeExecutionData,
+	INodePropertyOptions,
+	INodeType,
+	INodeTypeDescription,
+} from 'n8n-workflow';
+
+import {
+	googleApiRequest,
+	googleApiRequestAllItems,
+	simplify,
+} from './GenericFunctions';
+
+import {
+	recordFields,
+	recordOperations,
+} from './RecordDescription';
+
+import * as uuid from 'uuid';
+
+export class GoogleBigQuery implements INodeType {
+	description: INodeTypeDescription = {
+		displayName: 'Google BigQuery',
+		name: 'googleBigQuery',
+		icon: 'file:googleBigQuery.svg',
+		group: ['input'],
+		version: 1,
+		subtitle: '={{$parameter["operation"] + ": " + $parameter["resource"]}}',
+		description: 'Consume Google BigQuery API.',
+		defaults: {
+			name: 'Google BigQuery',
+			color: '#3E87E4',
+		},
+		inputs: ['main'],
+		outputs: ['main'],
+		credentials: [
+			{
+				name: 'googleBigQueryOAuth2Api',
+				required: true,
+			},
+		],
+		properties: [
+			{
+				displayName: 'Resource',
+				name: 'resource',
+				type: 'options',
+				options: [
+					{
+						name: 'Record',
+						value: 'record',
+					},
+				],
+				default: 'record',
+				description: 'The resource to operate on.',
+			},
+			...recordOperations,
+			...recordFields,
+		],
+	};
+
+	methods = {
+		loadOptions: {
+			async getProjects(
+				this: ILoadOptionsFunctions,
+			): Promise<INodePropertyOptions[]> {
+				const returnData: INodePropertyOptions[] = [];
+				const { projects } = await googleApiRequest.call(
+					this,
+					'GET',
+					'/v2/projects',
+				);
+				for (const project of projects) {
+					returnData.push({
+						name: project.friendlyName as string,
+						value: project.id,
+					});
+				}
+				return returnData;
+			},
+			async getDatasets(
+				this: ILoadOptionsFunctions,
+			): Promise<INodePropertyOptions[]> {
+				const projectId = this.getCurrentNodeParameter('projectId');
+				const returnData: INodePropertyOptions[] = [];
+				const { datasets } = await googleApiRequest.call(
+					this,
+					'GET',
+					`/v2/projects/${projectId}/datasets`,
+				);
+				for (const dataset of datasets) {
+					returnData.push({
+						name: dataset.datasetReference.datasetId as string,
+						value: dataset.datasetReference.datasetId,
+					});
+				}
+				return returnData;
+			},
+			async getTables(
+				this: ILoadOptionsFunctions,
+			): Promise<INodePropertyOptions[]> {
+				const projectId = this.getCurrentNodeParameter('projectId');
+				const datasetId = this.getCurrentNodeParameter('datasetId');
+				const returnData: INodePropertyOptions[] = [];
+				const { tables } = await googleApiRequest.call(
+					this,
+					'GET',
+					`/v2/projects/${projectId}/datasets/${datasetId}/tables`,
+				);
+				for (const table of tables) {
+					returnData.push({
+						name: table.tableReference.tableId as string,
+						value: table.tableReference.tableId,
+					});
+				}
+				return returnData;
+			},
+		},
+	};
+
+	async execute(this: IExecuteFunctions): Promise<INodeExecutionData[][]> {
+		const items = this.getInputData();
+		const returnData: IDataObject[] = [];
+		const length = (items.length as unknown) as number;
+		const qs: IDataObject = {};
+		let responseData;
+		const resource = this.getNodeParameter('resource', 0) as string;
+		const operation = this.getNodeParameter('operation', 0) as string;
+
+		if (resource === 'record') {
+
+			// *********************************************************************
+			//                               record
+			// *********************************************************************
+
+			if (operation === 'create') {
+
+				// ----------------------------------
+				//         record: create
+				// ----------------------------------
+
+				// https://cloud.google.com/bigquery/docs/reference/rest/v2/tabledata/insertAll
+
+				const projectId = this.getNodeParameter('projectId', 0) as string;
+				const datasetId = this.getNodeParameter('datasetId', 0) as string;
+				const tableId = this.getNodeParameter('tableId', 0) as string;
+				const rows: IDataObject[] = [];
+				const body: IDataObject = {};
+
+				for (let i = 0; i < length; i++) {
+
+					const options = this.getNodeParameter('options', i) as IDataObject;
+					Object.assign(body, options);
+					if (body.traceId === undefined) {
+						body.traceId = uuid();
+					}
+					const columns = this.getNodeParameter('columns', i) as string;
+					const columnList = columns.split(',').map(column => column.trim());
+					const record: IDataObject = {};
+
+					for (const key of Object.keys(items[i].json)) {
+						if (columnList.includes(key)) {
+							record[`${key}`] = items[i].json[key];
+						}
+					}
+					rows.push({ json: record });
+				}
+
+				body.rows = rows;
+				responseData = await googleApiRequest.call(
+					this,
+					'POST',
+					`/v2/projects/${projectId}/datasets/${datasetId}/tables/${tableId}/insertAll`,
+					body,
+				);
+				returnData.push(responseData);
+
+			} else if (operation === 'getAll') {
+
+				// ----------------------------------
+				//         record: getAll
+				// ----------------------------------
+
+				// https://cloud.google.com/bigquery/docs/reference/rest/v2/tables/get
+
+				const returnAll = this.getNodeParameter('returnAll', 0) as boolean;
+				const projectId = this.getNodeParameter('projectId', 0) as string;
+				const datasetId = this.getNodeParameter('datasetId', 0) as string;
+				const tableId = this.getNodeParameter('tableId', 0) as string;
+				const simple = this.getNodeParameter('simple', 0) as boolean;
+				let fields;
+
+				if (simple === true) {
+					const { schema } = await googleApiRequest.call(
+						this,
+						'GET',
+						`/v2/projects/${projectId}/datasets/${datasetId}/tables/${tableId}`,
+						{},
+					);
+					fields = schema.fields.map((field: IDataObject) => field.name);
+				}
+
+				for (let i = 0; i < length; i++) {
+					const options = this.getNodeParameter('options', i) as IDataObject;
+					Object.assign(qs, options);
+
+					// if (qs.useInt64Timestamp !== undefined) {
+					// 	qs.formatOptions = {
+					// 		useInt64Timestamp: qs.useInt64Timestamp,
+					// 	};
+					// 	delete qs.useInt64Timestamp;
+					// }
+
+					if (qs.selectedFields) {
+						fields = (qs.selectedFields as string).split(',');
+					}
+
+					if (returnAll) {
+						responseData = await googleApiRequestAllItems.call(
+							this,
+							'rows',
+							'GET',
+							`/v2/projects/${projectId}/datasets/${datasetId}/tables/${tableId}/data`,
+							{},
+							qs,
+						);
+						returnData.push.apply(returnData, (simple) ? simplify(responseData, fields) : responseData);
+					} else {
+						qs.maxResults = this.getNodeParameter('limit', i) as number;
+						responseData = await googleApiRequest.call(
+							this,
+							'GET',
+							`/v2/projects/${projectId}/datasets/${datasetId}/tables/${tableId}/data`,
+							{},
+							qs,
+						);
+						returnData.push.apply(returnData, (simple) ? simplify(responseData.rows, fields) : responseData.rows);
+					}
+				}
+			}
+		}
+
+		return [this.helpers.returnJsonArray(returnData)];
+	}
+}

packages/nodes-base/nodes/Google/BigQuery/RecordDescription.ts

@@ -0,0 +1,339 @@
+import {
+	INodeProperties,
+} from 'n8n-workflow';
+
+export const recordOperations = [
+	{
+		displayName: 'Operation',
+		name: 'operation',
+		type: 'options',
+		displayOptions: {
+			show: {
+				resource: [
+					'record',
+				],
+			},
+		},
+		options: [
+			{
+				name: 'Create',
+				value: 'create',
+				description: 'Create a new record.',
+			},
+			{
+				name: 'Get All',
+				value: 'getAll',
+				description: 'Retrieve all records.',
+			},
+		],
+		default: 'create',
+		description: 'Operation to perform.',
+	},
+] as INodeProperties[];
+
+export const recordFields = [
+	// ----------------------------------
+	//         record: create
+	// ----------------------------------
+	{
+		displayName: 'Project ID',
+		name: 'projectId',
+		type: 'options',
+		typeOptions: {
+			loadOptionsMethod: 'getProjects',
+		},
+		required: true,
+		displayOptions: {
+			show: {
+				operation: [
+					'create',
+				],
+				resource: [
+					'record',
+				],
+			},
+		},
+		default: '',
+		description: 'ID of the project to create the record in.',
+	},
+	{
+		displayName: 'Dataset ID',
+		name: 'datasetId',
+		type: 'options',
+		typeOptions: {
+			loadOptionsMethod: 'getDatasets',
+			loadOptionsDependsOn: [
+				'projectId',
+			],
+		},
+		required: true,
+		displayOptions: {
+			show: {
+				operation: [
+					'create',
+				],
+				resource: [
+					'record',
+				],
+			},
+		},
+		default: '',
+		description: 'ID of the dataset to create the record in.',
+	},
+	{
+		displayName: 'Table ID',
+		name: 'tableId',
+		type: 'options',
+		typeOptions: {
+			loadOptionsMethod: 'getTables',
+			loadOptionsDependsOn: [
+				'projectId',
+				'datasetId',
+			],
+		},
+		required: true,
+		displayOptions: {
+			show: {
+				operation: [
+					'create',
+				],
+				resource: [
+					'record',
+				],
+			},
+		},
+		default: '',
+		description: 'ID of the table to create the record in.',
+	},
+	{
+		displayName: 'Columns',
+		name: 'columns',
+		type: 'string',
+		displayOptions: {
+			show: {
+				resource: [
+					'record',
+				],
+				operation: [
+					'create',
+				],
+			},
+		},
+		default: '',
+		required: true,
+		placeholder: 'id,name,description',
+		description: 'Comma-separated list of the item properties to use as columns.',
+	},
+	{
+		displayName: 'Options',
+		name: 'options',
+		type: 'collection',
+		placeholder: 'Add Options',
+		default: {},
+		displayOptions: {
+			show: {
+				operation: [
+					'create',
+				],
+				resource: [
+					'record',
+				],
+			},
+		},
+		options: [
+			{
+				displayName: 'Ignore Unknown Values',
+				name: 'ignoreUnknownValues',
+				type: 'boolean',
+				default: false,
+				description: 'Ignore row values that do not match the schema.',
+			},
+			{
+				displayName: 'Skip Invalid Rows',
+				name: 'skipInvalidRows',
+				type: 'boolean',
+				default: false,
+				description: 'Skip rows with values that do not match the schema.',
+			},
+			{
+				displayName: 'Template Suffix',
+				name: 'templateSuffix',
+				type: 'string',
+				default: '',
+				description: 'Create a new table based on the destination table and insert rows into the new table. The new table will be named <code>{destinationTable}{templateSuffix}</code>.',
+			},
+			{
+				displayName: 'Trace ID',
+				name: 'traceId',
+				type: 'string',
+				default: '',
+				description: 'Unique ID for the request, for debugging only. It is case-sensitive, limited to up to 36 ASCII characters. A UUID is recommended.',
+			},
+		],
+	},
+
+	// ----------------------------------
+	//         record: getAll
+	// ----------------------------------
+	{
+		displayName: 'Project ID',
+		name: 'projectId',
+		type: 'options',
+		typeOptions: {
+			loadOptionsMethod: 'getProjects',
+		},
+		required: true,
+		displayOptions: {
+			show: {
+				operation: [
+					'getAll',
+				],
+				resource: [
+					'record',
+				],
+			},
+		},
+		default: '',
+		description: 'ID of the project to retrieve all rows from.',
+	},
+	{
+		displayName: 'Dataset ID',
+		name: 'datasetId',
+		type: 'options',
+		typeOptions: {
+			loadOptionsMethod: 'getDatasets',
+			loadOptionsDependsOn: [
+				'projectId',
+			],
+		},
+		required: true,
+		displayOptions: {
+			show: {
+				operation: [
+					'getAll',
+				],
+				resource: [
+					'record',
+				],
+			},
+		},
+		default: '',
+		description: 'ID of the dataset to retrieve all rows from.',
+	},
+	{
+		displayName: 'Table ID',
+		name: 'tableId',
+		type: 'options',
+		typeOptions: {
+			loadOptionsMethod: 'getTables',
+			loadOptionsDependsOn: [
+				'projectId',
+				'datasetId',
+			],
+		},
+		required: true,
+		displayOptions: {
+			show: {
+				operation: [
+					'getAll',
+				],
+				resource: [
+					'record',
+				],
+			},
+		},
+		default: '',
+		description: 'ID of the table to retrieve all rows from.',
+	},
+	{
+		displayName: 'Return All',
+		name: 'returnAll',
+		type: 'boolean',
+		displayOptions: {
+			show: {
+				operation: [
+					'getAll',
+				],
+				resource: [
+					'record',
+				],
+			},
+		},
+		default: false,
+		description: 'If all results should be returned or only up to a given limit.',
+	},
+	{
+		displayName: 'Limit',
+		name: 'limit',
+		type: 'number',
+		displayOptions: {
+			show: {
+				operation: [
+					'getAll',
+				],
+				resource: [
+					'record',
+				],
+				returnAll: [
+					false,
+				],
+			},
+		},
+		typeOptions: {
+			minValue: 1,
+			maxValue: 500,
+		},
+		default: 100,
+		description: 'How many results to return.',
+	},
+	{
+		displayName: 'Simple',
+		name: 'simple',
+		type: 'boolean',
+		displayOptions: {
+			show: {
+				resource: [
+					'record',
+				],
+				operation: [
+					'getAll',
+				],
+			},
+		},
+		default: true,
+		description: 'Return a simplified version of the response instead of the raw data.',
+	},
+	{
+		displayName: 'Options',
+		name: 'options',
+		type: 'collection',
+		placeholder: 'Add Options',
+		default: {},
+		displayOptions: {
+			show: {
+				operation: [
+					'getAll',
+				],
+				resource: [
+					'record',
+				],
+			},
+		},
+		options: [
+			{
+				displayName: 'Fields',
+				name: 'selectedFields',
+				type: 'string',
+				default: '',
+				description: 'Subset of fields to return, supports select into sub fields. Example: <code>selectedFields = "a,e.d.f"</code>.',
+			},
+			// {
+			// 	displayName: 'Use Int64 Timestamp',
+			// 	name: 'useInt64Timestamp',
+			// 	type: 'boolean',
+			// 	default: false,
+			// 	description: 'Output timestamp as usec int64.',
+			// },
+		],
+	},
+] as INodeProperties[];

packages/nodes-base/nodes/Google/BigQuery/googleBigQuery.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 0 66 58" fill="#fff" fill-rule="evenodd" stroke="#000" stroke-linecap="round" stroke-linejoin="round"><use xlink:href="#A" x=".5" y=".5"/><symbol id="A" overflow="visible"><g stroke="none" fill-rule="nonzero"><path d="M14.48 54.473L.558 30.359c-.744-1.288-.744-2.876 0-4.164L14.48 2.082C15.223.795 16.595.002 18.08 0h27.857c1.48.007 2.845.8 3.585 2.082l13.92 24.113c.744 1.288.744 2.876 0 4.164L49.52 54.473c-.743 1.287-2.115 2.08-3.6 2.082H18.07c-1.483-.005-2.85-.798-3.593-2.082z" fill="#4386fa"/><path d="M40.697 20.512s3.87 9.283-1.406 14.545-14.883 1.894-14.883 1.894L43.95 56.547h1.984c1.486-.002 2.858-.796 3.6-2.082l9.216-15.958z" fill="#000" opacity=".1"/><path d="M45.266 39.507L41 35.23a.67.67 0 0 0-.158-.12c3.767-4.852 3.111-11.798-1.499-15.859a11.63 11.63 0 0 0-16.396 16.436 11.63 11.63 0 0 0 15.863 1.46c.031.055.069.106.113.15l4.277 4.277a.67.67 0 0 0 .947 0l1.12-1.12a.67.67 0 0 0 0-.947zM31.64 36.741a8.75 8.75 0 0 1-6.188-14.937 8.75 8.75 0 0 1 14.937 6.188 8.75 8.75 0 0 1-8.749 8.749zm-5.593-9.216v3.616c.557.983 1.363 1.803 2.338 2.375v-6.013zm4.375-2.998v9.772a6.45 6.45 0 0 0 2.338 0v-9.772zm6.764 6.606v-2.142H34.85v4.5a6.43 6.43 0 0 0 2.338-2.368z"/></g></symbol></svg>

packages/nodes-base/package.json

@@ -95,6 +95,7 @@
       "dist/credentials/GmailOAuth2Api.credentials.js",
       "dist/credentials/GoogleAnalyticsOAuth2Api.credentials.js",
       "dist/credentials/GoogleApi.credentials.js",
+      "dist/credentials/GoogleBigQueryOAuth2Api.credentials.js",
       "dist/credentials/GoogleBooksOAuth2Api.credentials.js",
       "dist/credentials/GoogleCalendarOAuth2Api.credentials.js",
       "dist/credentials/GoogleContactsOAuth2Api.credentials.js",
@@ -362,6 +363,7 @@
       "dist/nodes/Github/GithubTrigger.node.js",
       "dist/nodes/Gitlab/Gitlab.node.js",
       "dist/nodes/Gitlab/GitlabTrigger.node.js",
+      "dist/nodes/Google/BigQuery/GoogleBigQuery.node.js",
       "dist/nodes/Google/Books/GoogleBooks.node.js",
       "dist/nodes/Google/Analytics/GoogleAnalytics.node.js",
       "dist/nodes/Google/Calendar/GoogleCalendar.node.js",