mudhorn/n8n
1
0
Fork 0
mirror of https://github.com/n8n-io/n8n.git synced 2025-01-12 05:17:28 -08:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat(GoogleBigQuery Node): Add support for service account authentication (#3128)

Browse source 
* ⚡ Enable service account authentication with the BigQuery node

* 🔨 fixed auth issue with key, fixed nodelinter issues

* ⚡ added continue on fail

* ⚡ Improvements

Co-authored-by: Mark Steve Samson <marksteve@thinkingmachin.es>
Co-authored-by: ricardo <ricardoespinoza105@gmail.com>
This commit is contained in:
Michael Kret 2022-04-18 19:46:50 +03:00 committed by GitHub
parent 794ad7c756
commit ac5f357001
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 186 additions and 68 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

86
packages/nodes-base/nodes/Google/BigQuery/GenericFunctions.ts
View file

@ -10,9 +10,18 @@ import {
import { import {
IDataObject, IDataObject,
JsonObject,
NodeApiError,
NodeOperationError
} from 'n8n-workflow'; } from 'n8n-workflow';
import moment from 'moment-timezone';
import * as jwt from 'jsonwebtoken';
export async function googleApiRequest(this: IExecuteFunctions | IExecuteSingleFunctions | ILoadOptionsFunctions, method: string, resource: string, body: any = {}, qs: IDataObject = {}, uri?: string, headers: IDataObject = {}): Promise<any> { // tslint:disable-line:no-any export async function googleApiRequest(this: IExecuteFunctions | IExecuteSingleFunctions | ILoadOptionsFunctions, method: string, resource: string, body: any = {}, qs: IDataObject = {}, uri?: string, headers: IDataObject = {}): Promise<any> { // tslint:disable-line:no-any
const authenticationMethod = this.getNodeParameter('authentication', 0, 'serviceAccount') as string;
const options: OptionsWithUri = { const options: OptionsWithUri = {
headers: { headers: {
'Content-Type': 'application/json', 'Content-Type': 'application/json',
@ -30,20 +39,28 @@ export async function googleApiRequest(this: IExecuteFunctions | IExecuteSingleF
if (Object.keys(body).length === 0) { if (Object.keys(body).length === 0) {
delete options.body; delete options.body;
} }
//@ts-ignore
return await this.helpers.requestOAuth2.call(this, 'googleBigQueryOAuth2Api', options);
} catch (error) {
if (error.response && error.response.body && error.response.body.error) {
let errors = error.response.body.error.errors; if (authenticationMethod === 'serviceAccount') {
const credentials = await this.getCredentials('googleApi');
errors = errors.map((e: IDataObject) => e.message); if (credentials === undefined) {
// Try to return the error prettier throw new NodeOperationError(this.getNode(), 'No credentials got returned!');
throw new Error( }
`Google BigQuery error response [${error.statusCode}]: ${errors.join('|')}`,
); const { access_token } = await getAccessToken.call(this, credentials as IDataObject);
options.headers!.Authorization = `Bearer ${access_token}`;
return await this.helpers.request!(options);
} else {
//@ts-ignore
return await this.helpers.requestOAuth2.call(this, 'googleBigQueryOAuth2Api', options);
} }
throw error; } catch (error) {
if (error.code === 'ERR_OSSL_PEM_NO_START_LINE') {
error.statusCode = '401';
}
throw new NodeApiError(this.getNode(), error as JsonObject);
} }
} }
@ -66,6 +83,53 @@ export async function googleApiRequestAllItems(this: IExecuteFunctions | ILoadOp
return returnData; return returnData;
} }
function getAccessToken(this: IExecuteFunctions | IExecuteSingleFunctions | ILoadOptionsFunctions, credentials: IDataObject): Promise<IDataObject> {
//https://developers.google.com/identity/protocols/oauth2/service-account#httprest
const privateKey = (credentials.privateKey as string).replace(/\\n/g, '\n').trim();
const scopes = [
'https://www.googleapis.com/auth/bigquery',
];
const now = moment().unix();
const signature = jwt.sign(
{
'iss': credentials.email as string,
'sub': credentials.delegatedEmail || credentials.email as string,
'scope': scopes.join(' '),
'aud': `https://oauth2.googleapis.com/token`,
'iat': now,
'exp': now + 3600,
},
privateKey,
{
algorithm: 'RS256',
header: {
'kid': privateKey,
'typ': 'JWT',
'alg': 'RS256',
},
},
);
const options: OptionsWithUri = {
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
},
method: 'POST',
form: {
grant_type: 'urn:ietf:params:oauth:grant-type:jwt-bearer',
assertion: signature,
},
uri: 'https://oauth2.googleapis.com/token',
json: true,
};
return this.helpers.request!(options);
}
export function simplify(rows: IDataObject[], fields: string[]) { export function simplify(rows: IDataObject[], fields: string[]) {
const results = []; const results = [];
for (const row of rows) { for (const row of rows) {

131
packages/nodes-base/nodes/Google/BigQuery/GoogleBigQuery.node.ts
View file

@ -9,6 +9,7 @@ import {
INodePropertyOptions, INodePropertyOptions,
INodeType, INodeType,
INodeTypeDescription, INodeTypeDescription,
NodeApiError,
} from 'n8n-workflow'; } from 'n8n-workflow';
import { import {
@ -39,16 +40,52 @@ export class GoogleBigQuery implements INodeType {
inputs: ['main'], inputs: ['main'],
outputs: ['main'], outputs: ['main'],
credentials: [ credentials: [
{
name: 'googleApi',
required: true,
displayOptions: {
show: {
authentication: [
'serviceAccount',
],
},
},
},
{ {
name: 'googleBigQueryOAuth2Api', name: 'googleBigQueryOAuth2Api',
required: true, required: true,
displayOptions: {
show: {
authentication: [
'oAuth2',
],
},
},
}, },
], ],
properties: [ properties: [
{
displayName: 'Authentication',
name: 'authentication',
type: 'options',
noDataExpression: true,
options: [
{
name: 'Service Account',
value: 'serviceAccount',
},
{
name: 'OAuth2',
value: 'oAuth2',
},
],
default: 'oAuth2',
},
{ {
displayName: 'Resource', displayName: 'Resource',
name: 'resource', name: 'resource',
type: 'options', type: 'options',
noDataExpression: true,
options: [ options: [
{ {
name: 'Record', name: 'Record',
@ -56,7 +93,7 @@ export class GoogleBigQuery implements INodeType {
}, },
], ],
default: 'record', default: 'record',
description: 'The resource to operate on.', description: 'The resource to operate on',
}, },
...recordOperations, ...recordOperations,
...recordFields, ...recordFields,
@ -171,14 +208,22 @@ export class GoogleBigQuery implements INodeType {
} }
body.rows = rows; body.rows = rows;
responseData = await googleApiRequest.call(
this,
'POST',
`/v2/projects/${projectId}/datasets/${datasetId}/tables/${tableId}/insertAll`,
body,
);
returnData.push(responseData);
try {
responseData = await googleApiRequest.call(
this,
'POST',
`/v2/projects/${projectId}/datasets/${datasetId}/tables/${tableId}/insertAll`,
body,
);
returnData.push(responseData);
} catch (error) {
if (this.continueOnFail()) {
returnData.push({ error: error.message });
} else {
throw new NodeApiError(this.getNode(), error);
}
}
} else if (operation === 'getAll') { } else if (operation === 'getAll') {
// ---------------------------------- // ----------------------------------
@ -205,40 +250,48 @@ export class GoogleBigQuery implements INodeType {
} }
for (let i = 0; i < length; i++) { for (let i = 0; i < length; i++) {
const options = this.getNodeParameter('options', i) as IDataObject; try {
Object.assign(qs, options); const options = this.getNodeParameter('options', i) as IDataObject;
Object.assign(qs, options);
// if (qs.useInt64Timestamp !== undefined) { // if (qs.useInt64Timestamp !== undefined) {
// qs.formatOptions = { // qs.formatOptions = {
// useInt64Timestamp: qs.useInt64Timestamp, // useInt64Timestamp: qs.useInt64Timestamp,
// }; // };
// delete qs.useInt64Timestamp; // delete qs.useInt64Timestamp;
// } // }
if (qs.selectedFields) { if (qs.selectedFields) {
fields = (qs.selectedFields as string).split(','); fields = (qs.selectedFields as string).split(',');
} }
if (returnAll) { if (returnAll) {
responseData = await googleApiRequestAllItems.call( responseData = await googleApiRequestAllItems.call(
this, this,
'rows', 'rows',
'GET', 'GET',
`/v2/projects/${projectId}/datasets/${datasetId}/tables/${tableId}/data`, `/v2/projects/${projectId}/datasets/${datasetId}/tables/${tableId}/data`,
{}, {},
qs, qs,
); );
returnData.push.apply(returnData, (simple) ? simplify(responseData, fields) : responseData); returnData.push.apply(returnData, (simple) ? simplify(responseData, fields) : responseData);
} else { } else {
qs.maxResults = this.getNodeParameter('limit', i) as number; qs.maxResults = this.getNodeParameter('limit', i) as number;
responseData = await googleApiRequest.call( responseData = await googleApiRequest.call(
this, this,
'GET', 'GET',
`/v2/projects/${projectId}/datasets/${datasetId}/tables/${tableId}/data`, `/v2/projects/${projectId}/datasets/${datasetId}/tables/${tableId}/data`,
{}, {},
qs, qs,
); );
returnData.push.apply(returnData, (simple) ? simplify(responseData.rows, fields) : responseData.rows); returnData.push.apply(returnData, (simple) ? simplify(responseData.rows, fields) : responseData.rows);
}
} catch (error) {
if (this.continueOnFail()) {
returnData.push({ error: error.message });
continue;
}
throw new NodeApiError(this.getNode(), error);
} }
} }
} }

37
packages/nodes-base/nodes/Google/BigQuery/RecordDescription.ts
View file

@ -7,6 +7,7 @@ export const recordOperations: INodeProperties[] = [
displayName: 'Operation', displayName: 'Operation',
name: 'operation', name: 'operation',
type: 'options', type: 'options',
noDataExpression: true,
displayOptions: { displayOptions: {
show: { show: {
resource: [ resource: [
@ -18,16 +19,16 @@ export const recordOperations: INodeProperties[] = [
{ {
name: 'Create', name: 'Create',
value: 'create', value: 'create',
description: 'Create a new record.', description: 'Create a new record',
}, },
{ {
name: 'Get All', name: 'Get All',
value: 'getAll', value: 'getAll',
description: 'Retrieve all records.', description: 'Retrieve all records',
}, },
], ],
default: 'create', default: 'create',
description: 'Operation to perform.', description: 'Operation to perform',
}, },
]; ];
@ -54,7 +55,7 @@ export const recordFields: INodeProperties[] = [
}, },
}, },
default: '', default: '',
description: 'ID of the project to create the record in.', description: 'ID of the project to create the record in',
}, },
{ {
displayName: 'Dataset ID', displayName: 'Dataset ID',
@ -78,7 +79,7 @@ export const recordFields: INodeProperties[] = [
}, },
}, },
default: '', default: '',
description: 'ID of the dataset to create the record in.', description: 'ID of the dataset to create the record in',
}, },
{ {
displayName: 'Table ID', displayName: 'Table ID',
@ -103,7 +104,7 @@ export const recordFields: INodeProperties[] = [
}, },
}, },
default: '', default: '',
description: 'ID of the table to create the record in.', description: 'ID of the table to create the record in',
}, },
{ {
displayName: 'Columns', displayName: 'Columns',
@ -122,7 +123,7 @@ export const recordFields: INodeProperties[] = [
default: '', default: '',
required: true, required: true,
placeholder: 'id,name,description', placeholder: 'id,name,description',
description: 'Comma-separated list of the item properties to use as columns.', description: 'Comma-separated list of the item properties to use as columns',
}, },
{ {
displayName: 'Options', displayName: 'Options',
@ -146,21 +147,21 @@ export const recordFields: INodeProperties[] = [
name: 'ignoreUnknownValues', name: 'ignoreUnknownValues',
type: 'boolean', type: 'boolean',
default: false, default: false,
description: 'Ignore row values that do not match the schema.', description: 'Whether to gnore row values that do not match the schema',
}, },
{ {
displayName: 'Skip Invalid Rows', displayName: 'Skip Invalid Rows',
name: 'skipInvalidRows', name: 'skipInvalidRows',
type: 'boolean', type: 'boolean',
default: false, default: false,
description: 'Skip rows with values that do not match the schema.', description: 'Whether to skip rows with values that do not match the schema',
}, },
{ {
displayName: 'Template Suffix', displayName: 'Template Suffix',
name: 'templateSuffix', name: 'templateSuffix',
type: 'string', type: 'string',
default: '', default: '',
description: 'Create a new table based on the destination table and insert rows into the new table. The new table will be named <code>{destinationTable}{templateSuffix}</code>.', description: 'Create a new table based on the destination table and insert rows into the new table. The new table will be named <code>{destinationTable}{templateSuffix}</code>',
}, },
{ {
displayName: 'Trace ID', displayName: 'Trace ID',
@ -194,7 +195,7 @@ export const recordFields: INodeProperties[] = [
}, },
}, },
default: '', default: '',
description: 'ID of the project to retrieve all rows from.', description: 'ID of the project to retrieve all rows from',
}, },
{ {
displayName: 'Dataset ID', displayName: 'Dataset ID',
@ -218,7 +219,7 @@ export const recordFields: INodeProperties[] = [
}, },
}, },
default: '', default: '',
description: 'ID of the dataset to retrieve all rows from.', description: 'ID of the dataset to retrieve all rows from',
}, },
{ {
displayName: 'Table ID', displayName: 'Table ID',
@ -243,7 +244,7 @@ export const recordFields: INodeProperties[] = [
}, },
}, },
default: '', default: '',
description: 'ID of the table to retrieve all rows from.', description: 'ID of the table to retrieve all rows from',
}, },
{ {
displayName: 'Return All', displayName: 'Return All',
@ -260,7 +261,7 @@ export const recordFields: INodeProperties[] = [
}, },
}, },
default: false, default: false,
description: 'If all results should be returned or only up to a given limit.', description: 'Whether to return all results or only up to a given limit',
}, },
{ {
displayName: 'Limit', displayName: 'Limit',
@ -283,8 +284,8 @@ export const recordFields: INodeProperties[] = [
minValue: 1, minValue: 1,
maxValue: 500, maxValue: 500,
}, },
default: 100, default: 50,
description: 'How many results to return.', description: 'Max number of results to return',
}, },
{ {
displayName: 'Simplify Response', displayName: 'Simplify Response',
@ -301,7 +302,7 @@ export const recordFields: INodeProperties[] = [
}, },
}, },
default: true, default: true,
description: 'Return a simplified version of the response instead of the raw data.', description: 'Whether to return a simplified version of the response instead of the raw data',
}, },
{ {
displayName: 'Options', displayName: 'Options',
@ -325,7 +326,7 @@ export const recordFields: INodeProperties[] = [
name: 'selectedFields', name: 'selectedFields',
type: 'string', type: 'string',
default: '', default: '',
description: 'Subset of fields to return, supports select into sub fields. Example: <code>selectedFields = "a,e.d.f"</code>.', description: 'Subset of fields to return, supports select into sub fields. Example: <code>selectedFields = "a,e.d.f"</code>',
}, },
// { // {
// displayName: 'Use Int64 Timestamp', // displayName: 'Use Int64 Timestamp',