feat(GoogleBigQuery Node): Add support for service account authentication (#3128)

* ⚡ Enable service account authentication with the BigQuery node

* 🔨 fixed auth issue with key, fixed nodelinter issues

* ⚡ added continue on fail

* ⚡ Improvements

Co-authored-by: Mark Steve Samson <marksteve@thinkingmachin.es>
Co-authored-by: ricardo <ricardoespinoza105@gmail.com>

packages/nodes-base/nodes/Google/BigQuery/GenericFunctions.ts

@@ -10,9 +10,18 @@ import {
 
 import {
 	IDataObject,
+	JsonObject,
+	NodeApiError,
+	NodeOperationError
 } from 'n8n-workflow';
 
+import moment from 'moment-timezone';
+
+import * as jwt from 'jsonwebtoken';
+
 export async function googleApiRequest(this: IExecuteFunctions | IExecuteSingleFunctions | ILoadOptionsFunctions, method: string, resource: string, body: any = {}, qs: IDataObject = {}, uri?: string, headers: IDataObject = {}): Promise<any> { // tslint:disable-line:no-any
+	const authenticationMethod = this.getNodeParameter('authentication', 0, 'serviceAccount') as string;
+
 	const options: OptionsWithUri = {
 		headers: {
 			'Content-Type': 'application/json',
@@ -30,20 +39,28 @@ export async function googleApiRequest(this: IExecuteFunctions | IExecuteSingleF
 		if (Object.keys(body).length === 0) {
 			delete options.body;
 		}
+
+		if (authenticationMethod === 'serviceAccount') {
+			const credentials = await this.getCredentials('googleApi');
+
+			if (credentials === undefined) {
+				throw new NodeOperationError(this.getNode(), 'No credentials got returned!');
+			}
+
+			const { access_token } = await getAccessToken.call(this, credentials as IDataObject);
+
+			options.headers!.Authorization = `Bearer ${access_token}`;
+			return await this.helpers.request!(options);
+		} else {
 			//@ts-ignore
 			return await this.helpers.requestOAuth2.call(this, 'googleBigQueryOAuth2Api', options);
-	} catch (error) {
-		if (error.response && error.response.body && error.response.body.error) {
-
-			let errors = error.response.body.error.errors;
-
-			errors = errors.map((e: IDataObject) => e.message);
-			// Try to return the error prettier
-			throw new Error(
-				`Google BigQuery error response [${error.statusCode}]: ${errors.join('|')}`,
-			);
 		}
-		throw error;
+	} catch (error) {
+		if (error.code === 'ERR_OSSL_PEM_NO_START_LINE') {
+			error.statusCode = '401';
+		}
+
+		throw new NodeApiError(this.getNode(), error as JsonObject);
 	}
 }
 
@@ -66,6 +83,53 @@ export async function googleApiRequestAllItems(this: IExecuteFunctions | ILoadOp
 	return returnData;
 }
 
+function getAccessToken(this: IExecuteFunctions | IExecuteSingleFunctions | ILoadOptionsFunctions, credentials: IDataObject): Promise<IDataObject> {
+	//https://developers.google.com/identity/protocols/oauth2/service-account#httprest
+
+	const privateKey = (credentials.privateKey as string).replace(/\\n/g, '\n').trim();
+
+	const scopes = [
+		'https://www.googleapis.com/auth/bigquery',
+	];
+
+	const now = moment().unix();
+
+	const signature = jwt.sign(
+		{
+			'iss': credentials.email as string,
+			'sub': credentials.delegatedEmail || credentials.email as string,
+			'scope': scopes.join(' '),
+			'aud': `https://oauth2.googleapis.com/token`,
+			'iat': now,
+			'exp': now + 3600,
+		},
+		privateKey,
+		{
+			algorithm: 'RS256',
+			header: {
+				'kid': privateKey,
+				'typ': 'JWT',
+				'alg': 'RS256',
+			},
+		},
+	);
+
+	const options: OptionsWithUri = {
+		headers: {
+			'Content-Type': 'application/x-www-form-urlencoded',
+		},
+		method: 'POST',
+		form: {
+			grant_type: 'urn:ietf:params:oauth:grant-type:jwt-bearer',
+			assertion: signature,
+		},
+		uri: 'https://oauth2.googleapis.com/token',
+		json: true,
+	};
+
+	return this.helpers.request!(options);
+}
+
 export function simplify(rows: IDataObject[], fields: string[]) {
 	const results = [];
 	for (const row of rows) {

packages/nodes-base/nodes/Google/BigQuery/GoogleBigQuery.node.ts

@@ -9,6 +9,7 @@ import {
 	INodePropertyOptions,
 	INodeType,
 	INodeTypeDescription,
+	NodeApiError,
 } from 'n8n-workflow';
 
 import {
@@ -39,16 +40,52 @@ export class GoogleBigQuery implements INodeType {
 		inputs: ['main'],
 		outputs: ['main'],
 		credentials: [
+			{
+				name: 'googleApi',
+				required: true,
+				displayOptions: {
+					show: {
+						authentication: [
+							'serviceAccount',
+						],
+					},
+				},
+			},
 			{
 				name: 'googleBigQueryOAuth2Api',
 				required: true,
+				displayOptions: {
+					show: {
+						authentication: [
+							'oAuth2',
+						],
+					},
+				},
 			},
 		],
 		properties: [
+			{
+				displayName: 'Authentication',
+				name: 'authentication',
+				type: 'options',
+				noDataExpression: true,
+				options: [
+					{
+						name: 'Service Account',
+						value: 'serviceAccount',
+					},
+					{
+						name: 'OAuth2',
+						value: 'oAuth2',
+					},
+				],
+				default: 'oAuth2',
+			},
 			{
 				displayName: 'Resource',
 				name: 'resource',
 				type: 'options',
+				noDataExpression: true,
 				options: [
 					{
 						name: 'Record',
@@ -56,7 +93,7 @@ export class GoogleBigQuery implements INodeType {
 					},
 				],
 				default: 'record',
-				description: 'The resource to operate on.',
+				description: 'The resource to operate on',
 			},
 			...recordOperations,
 			...recordFields,
@@ -171,6 +208,8 @@ export class GoogleBigQuery implements INodeType {
 				}
 
 				body.rows = rows;
+
+				try {
 					responseData = await googleApiRequest.call(
 						this,
 						'POST',
@@ -178,7 +217,13 @@ export class GoogleBigQuery implements INodeType {
 						body,
 					);
 					returnData.push(responseData);
-
+				} catch (error) {
+					if (this.continueOnFail()) {
+						returnData.push({ error: error.message });
+					} else {
+						throw new NodeApiError(this.getNode(), error);
+					}
+				}
 			} else if (operation === 'getAll') {
 
 				// ----------------------------------
@@ -205,6 +250,7 @@ export class GoogleBigQuery implements INodeType {
 				}
 
 				for (let i = 0; i < length; i++) {
+					try {
 						const options = this.getNodeParameter('options', i) as IDataObject;
 						Object.assign(qs, options);
 
@@ -240,6 +286,13 @@ export class GoogleBigQuery implements INodeType {
 							);
 							returnData.push.apply(returnData, (simple) ? simplify(responseData.rows, fields) : responseData.rows);
 						}
+					} catch (error) {
+						if (this.continueOnFail()) {
+							returnData.push({ error: error.message });
+							continue;
+						}
+						throw new NodeApiError(this.getNode(), error);
+					}
 				}
 			}
 		}

packages/nodes-base/nodes/Google/BigQuery/RecordDescription.ts

@@ -7,6 +7,7 @@ export const recordOperations: INodeProperties[] = [
 		displayName: 'Operation',
 		name: 'operation',
 		type: 'options',
+		noDataExpression: true,
 		displayOptions: {
 			show: {
 				resource: [
@@ -18,16 +19,16 @@ export const recordOperations: INodeProperties[] = [
 			{
 				name: 'Create',
 				value: 'create',
-				description: 'Create a new record.',
+				description: 'Create a new record',
 			},
 			{
 				name: 'Get All',
 				value: 'getAll',
-				description: 'Retrieve all records.',
+				description: 'Retrieve all records',
 			},
 		],
 		default: 'create',
-		description: 'Operation to perform.',
+		description: 'Operation to perform',
 	},
 ];
 
@@ -54,7 +55,7 @@ export const recordFields: INodeProperties[] = [
 			},
 		},
 		default: '',
-		description: 'ID of the project to create the record in.',
+		description: 'ID of the project to create the record in',
 	},
 	{
 		displayName: 'Dataset ID',
@@ -78,7 +79,7 @@ export const recordFields: INodeProperties[] = [
 			},
 		},
 		default: '',
-		description: 'ID of the dataset to create the record in.',
+		description: 'ID of the dataset to create the record in',
 	},
 	{
 		displayName: 'Table ID',
@@ -103,7 +104,7 @@ export const recordFields: INodeProperties[] = [
 			},
 		},
 		default: '',
-		description: 'ID of the table to create the record in.',
+		description: 'ID of the table to create the record in',
 	},
 	{
 		displayName: 'Columns',
@@ -122,7 +123,7 @@ export const recordFields: INodeProperties[] = [
 		default: '',
 		required: true,
 		placeholder: 'id,name,description',
-		description: 'Comma-separated list of the item properties to use as columns.',
+		description: 'Comma-separated list of the item properties to use as columns',
 	},
 	{
 		displayName: 'Options',
@@ -146,21 +147,21 @@ export const recordFields: INodeProperties[] = [
 				name: 'ignoreUnknownValues',
 				type: 'boolean',
 				default: false,
-				description: 'Ignore row values that do not match the schema.',
+				description: 'Whether to gnore row values that do not match the schema',
 			},
 			{
 				displayName: 'Skip Invalid Rows',
 				name: 'skipInvalidRows',
 				type: 'boolean',
 				default: false,
-				description: 'Skip rows with values that do not match the schema.',
+				description: 'Whether to skip rows with values that do not match the schema',
 			},
 			{
 				displayName: 'Template Suffix',
 				name: 'templateSuffix',
 				type: 'string',
 				default: '',
-				description: 'Create a new table based on the destination table and insert rows into the new table. The new table will be named <code>{destinationTable}{templateSuffix}</code>.',
+				description: 'Create a new table based on the destination table and insert rows into the new table. The new table will be named <code>{destinationTable}{templateSuffix}</code>',
 			},
 			{
 				displayName: 'Trace ID',
@@ -194,7 +195,7 @@ export const recordFields: INodeProperties[] = [
 			},
 		},
 		default: '',
-		description: 'ID of the project to retrieve all rows from.',
+		description: 'ID of the project to retrieve all rows from',
 	},
 	{
 		displayName: 'Dataset ID',
@@ -218,7 +219,7 @@ export const recordFields: INodeProperties[] = [
 			},
 		},
 		default: '',
-		description: 'ID of the dataset to retrieve all rows from.',
+		description: 'ID of the dataset to retrieve all rows from',
 	},
 	{
 		displayName: 'Table ID',
@@ -243,7 +244,7 @@ export const recordFields: INodeProperties[] = [
 			},
 		},
 		default: '',
-		description: 'ID of the table to retrieve all rows from.',
+		description: 'ID of the table to retrieve all rows from',
 	},
 	{
 		displayName: 'Return All',
@@ -260,7 +261,7 @@ export const recordFields: INodeProperties[] = [
 			},
 		},
 		default: false,
-		description: 'If all results should be returned or only up to a given limit.',
+		description: 'Whether to return all results or only up to a given limit',
 	},
 	{
 		displayName: 'Limit',
@@ -283,8 +284,8 @@ export const recordFields: INodeProperties[] = [
 			minValue: 1,
 			maxValue: 500,
 		},
-		default: 100,
-		description: 'How many results to return.',
+		default: 50,
+		description: 'Max number of results to return',
 	},
 	{
 		displayName: 'Simplify Response',
@@ -301,7 +302,7 @@ export const recordFields: INodeProperties[] = [
 			},
 		},
 		default: true,
-		description: 'Return a simplified version of the response instead of the raw data.',
+		description: 'Whether to return a simplified version of the response instead of the raw data',
 	},
 	{
 		displayName: 'Options',
@@ -325,7 +326,7 @@ export const recordFields: INodeProperties[] = [
 				name: 'selectedFields',
 				type: 'string',
 				default: '',
-				description: 'Subset of fields to return, supports select into sub fields. Example: <code>selectedFields = "a,e.d.f"</code>.',
+				description: 'Subset of fields to return, supports select into sub fields. Example: <code>selectedFields = "a,e.d.f"</code>',
 			},
 			// {
 			// 	displayName: 'Use Int64 Timestamp',