From ac5f357001b6887d649f65bc32a30e30aa75584b Mon Sep 17 00:00:00 2001 From: Michael Kret <88898367+michael-radency@users.noreply.github.com> Date: Mon, 18 Apr 2022 19:46:50 +0300 Subject: [PATCH] feat(GoogleBigQuery Node): Add support for service account authentication (#3128) * :zap: Enable service account authentication with the BigQuery node * :hammer: fixed auth issue with key, fixed nodelinter issues * :zap: added continue on fail * :zap: Improvements Co-authored-by: Mark Steve Samson Co-authored-by: ricardo --- .../nodes/Google/BigQuery/GenericFunctions.ts | 86 ++++++++++-- .../Google/BigQuery/GoogleBigQuery.node.ts | 131 ++++++++++++------ .../Google/BigQuery/RecordDescription.ts | 37 ++--- 3 files changed, 186 insertions(+), 68 deletions(-) diff --git a/packages/nodes-base/nodes/Google/BigQuery/GenericFunctions.ts b/packages/nodes-base/nodes/Google/BigQuery/GenericFunctions.ts index 4a1248130c..4761f74a07 100644 --- a/packages/nodes-base/nodes/Google/BigQuery/GenericFunctions.ts +++ b/packages/nodes-base/nodes/Google/BigQuery/GenericFunctions.ts @@ -10,9 +10,18 @@ import { import { IDataObject, + JsonObject, + NodeApiError, + NodeOperationError } from 'n8n-workflow'; +import moment from 'moment-timezone'; + +import * as jwt from 'jsonwebtoken'; + export async function googleApiRequest(this: IExecuteFunctions | IExecuteSingleFunctions | ILoadOptionsFunctions, method: string, resource: string, body: any = {}, qs: IDataObject = {}, uri?: string, headers: IDataObject = {}): Promise { // tslint:disable-line:no-any + const authenticationMethod = this.getNodeParameter('authentication', 0, 'serviceAccount') as string; + const options: OptionsWithUri = { headers: { 'Content-Type': 'application/json', @@ -30,20 +39,28 @@ export async function googleApiRequest(this: IExecuteFunctions | IExecuteSingleF if (Object.keys(body).length === 0) { delete options.body; } - //@ts-ignore - return await this.helpers.requestOAuth2.call(this, 'googleBigQueryOAuth2Api', options); - } catch (error) { - if (error.response && error.response.body && error.response.body.error) { - let errors = error.response.body.error.errors; + if (authenticationMethod === 'serviceAccount') { + const credentials = await this.getCredentials('googleApi'); - errors = errors.map((e: IDataObject) => e.message); - // Try to return the error prettier - throw new Error( - `Google BigQuery error response [${error.statusCode}]: ${errors.join('|')}`, - ); + if (credentials === undefined) { + throw new NodeOperationError(this.getNode(), 'No credentials got returned!'); + } + + const { access_token } = await getAccessToken.call(this, credentials as IDataObject); + + options.headers!.Authorization = `Bearer ${access_token}`; + return await this.helpers.request!(options); + } else { + //@ts-ignore + return await this.helpers.requestOAuth2.call(this, 'googleBigQueryOAuth2Api', options); } - throw error; + } catch (error) { + if (error.code === 'ERR_OSSL_PEM_NO_START_LINE') { + error.statusCode = '401'; + } + + throw new NodeApiError(this.getNode(), error as JsonObject); } } @@ -66,6 +83,53 @@ export async function googleApiRequestAllItems(this: IExecuteFunctions | ILoadOp return returnData; } +function getAccessToken(this: IExecuteFunctions | IExecuteSingleFunctions | ILoadOptionsFunctions, credentials: IDataObject): Promise { + //https://developers.google.com/identity/protocols/oauth2/service-account#httprest + + const privateKey = (credentials.privateKey as string).replace(/\\n/g, '\n').trim(); + + const scopes = [ + 'https://www.googleapis.com/auth/bigquery', + ]; + + const now = moment().unix(); + + const signature = jwt.sign( + { + 'iss': credentials.email as string, + 'sub': credentials.delegatedEmail || credentials.email as string, + 'scope': scopes.join(' '), + 'aud': `https://oauth2.googleapis.com/token`, + 'iat': now, + 'exp': now + 3600, + }, + privateKey, + { + algorithm: 'RS256', + header: { + 'kid': privateKey, + 'typ': 'JWT', + 'alg': 'RS256', + }, + }, + ); + + const options: OptionsWithUri = { + headers: { + 'Content-Type': 'application/x-www-form-urlencoded', + }, + method: 'POST', + form: { + grant_type: 'urn:ietf:params:oauth:grant-type:jwt-bearer', + assertion: signature, + }, + uri: 'https://oauth2.googleapis.com/token', + json: true, + }; + + return this.helpers.request!(options); +} + export function simplify(rows: IDataObject[], fields: string[]) { const results = []; for (const row of rows) { diff --git a/packages/nodes-base/nodes/Google/BigQuery/GoogleBigQuery.node.ts b/packages/nodes-base/nodes/Google/BigQuery/GoogleBigQuery.node.ts index 8a32332fb5..30c7e5fb77 100644 --- a/packages/nodes-base/nodes/Google/BigQuery/GoogleBigQuery.node.ts +++ b/packages/nodes-base/nodes/Google/BigQuery/GoogleBigQuery.node.ts @@ -9,6 +9,7 @@ import { INodePropertyOptions, INodeType, INodeTypeDescription, + NodeApiError, } from 'n8n-workflow'; import { @@ -39,16 +40,52 @@ export class GoogleBigQuery implements INodeType { inputs: ['main'], outputs: ['main'], credentials: [ + { + name: 'googleApi', + required: true, + displayOptions: { + show: { + authentication: [ + 'serviceAccount', + ], + }, + }, + }, { name: 'googleBigQueryOAuth2Api', required: true, + displayOptions: { + show: { + authentication: [ + 'oAuth2', + ], + }, + }, }, ], properties: [ + { + displayName: 'Authentication', + name: 'authentication', + type: 'options', + noDataExpression: true, + options: [ + { + name: 'Service Account', + value: 'serviceAccount', + }, + { + name: 'OAuth2', + value: 'oAuth2', + }, + ], + default: 'oAuth2', + }, { displayName: 'Resource', name: 'resource', type: 'options', + noDataExpression: true, options: [ { name: 'Record', @@ -56,7 +93,7 @@ export class GoogleBigQuery implements INodeType { }, ], default: 'record', - description: 'The resource to operate on.', + description: 'The resource to operate on', }, ...recordOperations, ...recordFields, @@ -171,14 +208,22 @@ export class GoogleBigQuery implements INodeType { } body.rows = rows; - responseData = await googleApiRequest.call( - this, - 'POST', - `/v2/projects/${projectId}/datasets/${datasetId}/tables/${tableId}/insertAll`, - body, - ); - returnData.push(responseData); + try { + responseData = await googleApiRequest.call( + this, + 'POST', + `/v2/projects/${projectId}/datasets/${datasetId}/tables/${tableId}/insertAll`, + body, + ); + returnData.push(responseData); + } catch (error) { + if (this.continueOnFail()) { + returnData.push({ error: error.message }); + } else { + throw new NodeApiError(this.getNode(), error); + } + } } else if (operation === 'getAll') { // ---------------------------------- @@ -205,40 +250,48 @@ export class GoogleBigQuery implements INodeType { } for (let i = 0; i < length; i++) { - const options = this.getNodeParameter('options', i) as IDataObject; - Object.assign(qs, options); + try { + const options = this.getNodeParameter('options', i) as IDataObject; + Object.assign(qs, options); - // if (qs.useInt64Timestamp !== undefined) { - // qs.formatOptions = { - // useInt64Timestamp: qs.useInt64Timestamp, - // }; - // delete qs.useInt64Timestamp; - // } + // if (qs.useInt64Timestamp !== undefined) { + // qs.formatOptions = { + // useInt64Timestamp: qs.useInt64Timestamp, + // }; + // delete qs.useInt64Timestamp; + // } - if (qs.selectedFields) { - fields = (qs.selectedFields as string).split(','); - } + if (qs.selectedFields) { + fields = (qs.selectedFields as string).split(','); + } - if (returnAll) { - responseData = await googleApiRequestAllItems.call( - this, - 'rows', - 'GET', - `/v2/projects/${projectId}/datasets/${datasetId}/tables/${tableId}/data`, - {}, - qs, - ); - returnData.push.apply(returnData, (simple) ? simplify(responseData, fields) : responseData); - } else { - qs.maxResults = this.getNodeParameter('limit', i) as number; - responseData = await googleApiRequest.call( - this, - 'GET', - `/v2/projects/${projectId}/datasets/${datasetId}/tables/${tableId}/data`, - {}, - qs, - ); - returnData.push.apply(returnData, (simple) ? simplify(responseData.rows, fields) : responseData.rows); + if (returnAll) { + responseData = await googleApiRequestAllItems.call( + this, + 'rows', + 'GET', + `/v2/projects/${projectId}/datasets/${datasetId}/tables/${tableId}/data`, + {}, + qs, + ); + returnData.push.apply(returnData, (simple) ? simplify(responseData, fields) : responseData); + } else { + qs.maxResults = this.getNodeParameter('limit', i) as number; + responseData = await googleApiRequest.call( + this, + 'GET', + `/v2/projects/${projectId}/datasets/${datasetId}/tables/${tableId}/data`, + {}, + qs, + ); + returnData.push.apply(returnData, (simple) ? simplify(responseData.rows, fields) : responseData.rows); + } + } catch (error) { + if (this.continueOnFail()) { + returnData.push({ error: error.message }); + continue; + } + throw new NodeApiError(this.getNode(), error); } } } diff --git a/packages/nodes-base/nodes/Google/BigQuery/RecordDescription.ts b/packages/nodes-base/nodes/Google/BigQuery/RecordDescription.ts index e671c65623..7ecd78e15b 100644 --- a/packages/nodes-base/nodes/Google/BigQuery/RecordDescription.ts +++ b/packages/nodes-base/nodes/Google/BigQuery/RecordDescription.ts @@ -7,6 +7,7 @@ export const recordOperations: INodeProperties[] = [ displayName: 'Operation', name: 'operation', type: 'options', + noDataExpression: true, displayOptions: { show: { resource: [ @@ -18,16 +19,16 @@ export const recordOperations: INodeProperties[] = [ { name: 'Create', value: 'create', - description: 'Create a new record.', + description: 'Create a new record', }, { name: 'Get All', value: 'getAll', - description: 'Retrieve all records.', + description: 'Retrieve all records', }, ], default: 'create', - description: 'Operation to perform.', + description: 'Operation to perform', }, ]; @@ -54,7 +55,7 @@ export const recordFields: INodeProperties[] = [ }, }, default: '', - description: 'ID of the project to create the record in.', + description: 'ID of the project to create the record in', }, { displayName: 'Dataset ID', @@ -78,7 +79,7 @@ export const recordFields: INodeProperties[] = [ }, }, default: '', - description: 'ID of the dataset to create the record in.', + description: 'ID of the dataset to create the record in', }, { displayName: 'Table ID', @@ -103,7 +104,7 @@ export const recordFields: INodeProperties[] = [ }, }, default: '', - description: 'ID of the table to create the record in.', + description: 'ID of the table to create the record in', }, { displayName: 'Columns', @@ -122,7 +123,7 @@ export const recordFields: INodeProperties[] = [ default: '', required: true, placeholder: 'id,name,description', - description: 'Comma-separated list of the item properties to use as columns.', + description: 'Comma-separated list of the item properties to use as columns', }, { displayName: 'Options', @@ -146,21 +147,21 @@ export const recordFields: INodeProperties[] = [ name: 'ignoreUnknownValues', type: 'boolean', default: false, - description: 'Ignore row values that do not match the schema.', + description: 'Whether to gnore row values that do not match the schema', }, { displayName: 'Skip Invalid Rows', name: 'skipInvalidRows', type: 'boolean', default: false, - description: 'Skip rows with values that do not match the schema.', + description: 'Whether to skip rows with values that do not match the schema', }, { displayName: 'Template Suffix', name: 'templateSuffix', type: 'string', default: '', - description: 'Create a new table based on the destination table and insert rows into the new table. The new table will be named {destinationTable}{templateSuffix}.', + description: 'Create a new table based on the destination table and insert rows into the new table. The new table will be named {destinationTable}{templateSuffix}', }, { displayName: 'Trace ID', @@ -194,7 +195,7 @@ export const recordFields: INodeProperties[] = [ }, }, default: '', - description: 'ID of the project to retrieve all rows from.', + description: 'ID of the project to retrieve all rows from', }, { displayName: 'Dataset ID', @@ -218,7 +219,7 @@ export const recordFields: INodeProperties[] = [ }, }, default: '', - description: 'ID of the dataset to retrieve all rows from.', + description: 'ID of the dataset to retrieve all rows from', }, { displayName: 'Table ID', @@ -243,7 +244,7 @@ export const recordFields: INodeProperties[] = [ }, }, default: '', - description: 'ID of the table to retrieve all rows from.', + description: 'ID of the table to retrieve all rows from', }, { displayName: 'Return All', @@ -260,7 +261,7 @@ export const recordFields: INodeProperties[] = [ }, }, default: false, - description: 'If all results should be returned or only up to a given limit.', + description: 'Whether to return all results or only up to a given limit', }, { displayName: 'Limit', @@ -283,8 +284,8 @@ export const recordFields: INodeProperties[] = [ minValue: 1, maxValue: 500, }, - default: 100, - description: 'How many results to return.', + default: 50, + description: 'Max number of results to return', }, { displayName: 'Simplify Response', @@ -301,7 +302,7 @@ export const recordFields: INodeProperties[] = [ }, }, default: true, - description: 'Return a simplified version of the response instead of the raw data.', + description: 'Whether to return a simplified version of the response instead of the raw data', }, { displayName: 'Options', @@ -325,7 +326,7 @@ export const recordFields: INodeProperties[] = [ name: 'selectedFields', type: 'string', default: '', - description: 'Subset of fields to return, supports select into sub fields. Example: selectedFields = "a,e.d.f".', + description: 'Subset of fields to return, supports select into sub fields. Example: selectedFields = "a,e.d.f"', }, // { // displayName: 'Use Int64 Timestamp',