mudhorn/n8n
1
0
Fork 0
mirror of https://github.com/n8n-io/n8n.git synced 2024-12-24 20:24:05 -08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

feat(GoogleBigQuery Node): Add support for service account authentication (#3128)

Browse source 
* ⚡ Enable service account authentication with the BigQuery node

* 🔨 fixed auth issue with key, fixed nodelinter issues

* ⚡ added continue on fail

* ⚡ Improvements

Co-authored-by: Mark Steve Samson <marksteve@thinkingmachin.es>
Co-authored-by: ricardo <ricardoespinoza105@gmail.com>
This commit is contained in:
Michael Kret 2022-04-18 19:46:50 +03:00 committed by GitHub
parent 794ad7c756
commit ac5f357001
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 186 additions and 68 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

86
packages/nodes-base/nodes/Google/BigQuery/GenericFunctions.ts
View file

@ -10,9 +10,18 @@ import {
import {
IDataObject,
JsonObject,
NodeApiError,
NodeOperationError
} from 'n8n-workflow';
import moment from 'moment-timezone';
import * as jwt from 'jsonwebtoken';
export async function googleApiRequest(this: IExecuteFunctions | IExecuteSingleFunctions | ILoadOptionsFunctions, method: string, resource: string, body: any = {}, qs: IDataObject = {}, uri?: string, headers: IDataObject = {}): Promise<any> { // tslint:disable-line:no-any
const authenticationMethod = this.getNodeParameter('authentication', 0, 'serviceAccount') as string;
const options: OptionsWithUri = {
headers: {
'Content-Type': 'application/json',
@ -30,20 +39,28 @@ export async function googleApiRequest(this: IExecuteFunctions | IExecuteSingleF
if (Object.keys(body).length === 0) {
delete options.body;
}
//@ts-ignore
return await this.helpers.requestOAuth2.call(this, 'googleBigQueryOAuth2Api', options);
} catch (error) {
if (error.response && error.response.body && error.response.body.error) {
let errors = error.response.body.error.errors;
if (authenticationMethod === 'serviceAccount') {
const credentials = await this.getCredentials('googleApi');
errors = errors.map((e: IDataObject) => e.message);
// Try to return the error prettier
throw new Error(
`Google BigQuery error response [${error.statusCode}]: ${errors.join('|')}`,
);
if (credentials === undefined) {
throw new NodeOperationError(this.getNode(), 'No credentials got returned!');
}
const { access_token } = await getAccessToken.call(this, credentials as IDataObject);
options.headers!.Authorization = `Bearer ${access_token}`;
return await this.helpers.request!(options);
} else {
//@ts-ignore
return await this.helpers.requestOAuth2.call(this, 'googleBigQueryOAuth2Api', options);
}
throw error;
} catch (error) {
if (error.code === 'ERR_OSSL_PEM_NO_START_LINE') {
error.statusCode = '401';
}
throw new NodeApiError(this.getNode(), error as JsonObject);
}
}
@ -66,6 +83,53 @@ export async function googleApiRequestAllItems(this: IExecuteFunctions | ILoadOp
return returnData;
}
function getAccessToken(this: IExecuteFunctions | IExecuteSingleFunctions | ILoadOptionsFunctions, credentials: IDataObject): Promise<IDataObject> {
//https://developers.google.com/identity/protocols/oauth2/service-account#httprest
const privateKey = (credentials.privateKey as string).replace(/\\n/g, '\n').trim();
const scopes = [
'https://www.googleapis.com/auth/bigquery',
];
const now = moment().unix();
const signature = jwt.sign(
{
'iss': credentials.email as string,
'sub': credentials.delegatedEmail || credentials.email as string,
'scope': scopes.join(' '),
'aud': `https://oauth2.googleapis.com/token`,
'iat': now,
'exp': now + 3600,
},
privateKey,
{
algorithm: 'RS256',
header: {
'kid': privateKey,
'typ': 'JWT',
'alg': 'RS256',
},
},
);
const options: OptionsWithUri = {
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
},
method: 'POST',
form: {
grant_type: 'urn:ietf:params:oauth:grant-type:jwt-bearer',
assertion: signature,
},
uri: 'https://oauth2.googleapis.com/token',
json: true,
};
return this.helpers.request!(options);
}
export function simplify(rows: IDataObject[], fields: string[]) {
const results = [];
for (const row of rows) {

131
packages/nodes-base/nodes/Google/BigQuery/GoogleBigQuery.node.ts
View file

@ -9,6 +9,7 @@ import {
INodePropertyOptions,
INodeType,
INodeTypeDescription,
NodeApiError,
} from 'n8n-workflow';
import {
@ -39,16 +40,52 @@ export class GoogleBigQuery implements INodeType {
inputs: ['main'],
outputs: ['main'],
credentials: [
{
name: 'googleApi',
required: true,
displayOptions: {
show: {
authentication: [
'serviceAccount',
],
},
},
},
{
name: 'googleBigQueryOAuth2Api',
required: true,
displayOptions: {
show: {
authentication: [
'oAuth2',
],
},
},
},
],
properties: [
{
displayName: 'Authentication',
name: 'authentication',
type: 'options',
noDataExpression: true,
options: [
{
name: 'Service Account',
value: 'serviceAccount',
},
{
name: 'OAuth2',
value: 'oAuth2',
},
],
default: 'oAuth2',
},
{
displayName: 'Resource',
name: 'resource',
type: 'options',
noDataExpression: true,
options: [
{
name: 'Record',
@ -56,7 +93,7 @@ export class GoogleBigQuery implements INodeType {
},
],
default: 'record',
description: 'The resource to operate on.',
description: 'The resource to operate on',
},
...recordOperations,
...recordFields,
@ -171,14 +208,22 @@ export class GoogleBigQuery implements INodeType {
}
body.rows = rows;
responseData = await googleApiRequest.call(
this,
'POST',
`/v2/projects/${projectId}/datasets/${datasetId}/tables/${tableId}/insertAll`,
body,
);
returnData.push(responseData);
try {
responseData = await googleApiRequest.call(
this,
'POST',
`/v2/projects/${projectId}/datasets/${datasetId}/tables/${tableId}/insertAll`,
body,
);
returnData.push(responseData);
} catch (error) {
if (this.continueOnFail()) {
returnData.push({ error: error.message });
} else {
throw new NodeApiError(this.getNode(), error);
}
}
} else if (operation === 'getAll') {
// ----------------------------------
@ -205,40 +250,48 @@ export class GoogleBigQuery implements INodeType {
}
for (let i = 0; i < length; i++) {
const options = this.getNodeParameter('options', i) as IDataObject;
Object.assign(qs, options);
try {
const options = this.getNodeParameter('options', i) as IDataObject;
Object.assign(qs, options);
// if (qs.useInt64Timestamp !== undefined) {
// qs.formatOptions = {
// useInt64Timestamp: qs.useInt64Timestamp,
// };
// delete qs.useInt64Timestamp;
// }
// if (qs.useInt64Timestamp !== undefined) {
// qs.formatOptions = {
// useInt64Timestamp: qs.useInt64Timestamp,
// };
// delete qs.useInt64Timestamp;
// }
if (qs.selectedFields) {
fields = (qs.selectedFields as string).split(',');
}
if (qs.selectedFields) {
fields = (qs.selectedFields as string).split(',');
}
if (returnAll) {
responseData = await googleApiRequestAllItems.call(
this,
'rows',
'GET',
`/v2/projects/${projectId}/datasets/${datasetId}/tables/${tableId}/data`,
{},
qs,
);
returnData.push.apply(returnData, (simple) ? simplify(responseData, fields) : responseData);
} else {
qs.maxResults = this.getNodeParameter('limit', i) as number;
responseData = await googleApiRequest.call(
this,
'GET',
`/v2/projects/${projectId}/datasets/${datasetId}/tables/${tableId}/data`,
{},
qs,
);
returnData.push.apply(returnData, (simple) ? simplify(responseData.rows, fields) : responseData.rows);
if (returnAll) {
responseData = await googleApiRequestAllItems.call(
this,
'rows',
'GET',
`/v2/projects/${projectId}/datasets/${datasetId}/tables/${tableId}/data`,
{},
qs,
);
returnData.push.apply(returnData, (simple) ? simplify(responseData, fields) : responseData);
} else {
qs.maxResults = this.getNodeParameter('limit', i) as number;
responseData = await googleApiRequest.call(
this,
'GET',
`/v2/projects/${projectId}/datasets/${datasetId}/tables/${tableId}/data`,
{},
qs,
);
returnData.push.apply(returnData, (simple) ? simplify(responseData.rows, fields) : responseData.rows);
}
} catch (error) {
if (this.continueOnFail()) {
returnData.push({ error: error.message });
continue;
}
throw new NodeApiError(this.getNode(), error);
}
}
}

37
packages/nodes-base/nodes/Google/BigQuery/RecordDescription.ts
View file

@ -7,6 +7,7 @@ export const recordOperations: INodeProperties[] = [
displayName: 'Operation',
name: 'operation',
type: 'options',
noDataExpression: true,
displayOptions: {
show: {
resource: [
@ -18,16 +19,16 @@ export const recordOperations: INodeProperties[] = [
{
name: 'Create',
value: 'create',
description: 'Create a new record.',
description: 'Create a new record',
},
{
name: 'Get All',
value: 'getAll',
description: 'Retrieve all records.',
description: 'Retrieve all records',
},
],
default: 'create',
description: 'Operation to perform.',
description: 'Operation to perform',
},
];
@ -54,7 +55,7 @@ export const recordFields: INodeProperties[] = [
},
},
default: '',
description: 'ID of the project to create the record in.',
description: 'ID of the project to create the record in',
},
{
displayName: 'Dataset ID',
@ -78,7 +79,7 @@ export const recordFields: INodeProperties[] = [
},
},
default: '',
description: 'ID of the dataset to create the record in.',
description: 'ID of the dataset to create the record in',
},
{
displayName: 'Table ID',
@ -103,7 +104,7 @@ export const recordFields: INodeProperties[] = [
},
},
default: '',
description: 'ID of the table to create the record in.',
description: 'ID of the table to create the record in',
},
{
displayName: 'Columns',
@ -122,7 +123,7 @@ export const recordFields: INodeProperties[] = [
default: '',
required: true,
placeholder: 'id,name,description',
description: 'Comma-separated list of the item properties to use as columns.',
description: 'Comma-separated list of the item properties to use as columns',
},
{
displayName: 'Options',
@ -146,21 +147,21 @@ export const recordFields: INodeProperties[] = [
name: 'ignoreUnknownValues',
type: 'boolean',
default: false,
description: 'Ignore row values that do not match the schema.',
description: 'Whether to gnore row values that do not match the schema',
},
{
displayName: 'Skip Invalid Rows',
name: 'skipInvalidRows',
type: 'boolean',
default: false,
description: 'Skip rows with values that do not match the schema.',
description: 'Whether to skip rows with values that do not match the schema',
},
{
displayName: 'Template Suffix',
name: 'templateSuffix',
type: 'string',
default: '',
description: 'Create a new table based on the destination table and insert rows into the new table. The new table will be named <code>{destinationTable}{templateSuffix}</code>.',
description: 'Create a new table based on the destination table and insert rows into the new table. The new table will be named <code>{destinationTable}{templateSuffix}</code>',
},
{
displayName: 'Trace ID',
@ -194,7 +195,7 @@ export const recordFields: INodeProperties[] = [
},
},
default: '',
description: 'ID of the project to retrieve all rows from.',
description: 'ID of the project to retrieve all rows from',
},
{
displayName: 'Dataset ID',
@ -218,7 +219,7 @@ export const recordFields: INodeProperties[] = [
},
},
default: '',
description: 'ID of the dataset to retrieve all rows from.',
description: 'ID of the dataset to retrieve all rows from',
},
{
displayName: 'Table ID',
@ -243,7 +244,7 @@ export const recordFields: INodeProperties[] = [
},
},
default: '',
description: 'ID of the table to retrieve all rows from.',
description: 'ID of the table to retrieve all rows from',
},
{
displayName: 'Return All',
@ -260,7 +261,7 @@ export const recordFields: INodeProperties[] = [
},
},
default: false,
description: 'If all results should be returned or only up to a given limit.',
description: 'Whether to return all results or only up to a given limit',
},
{
displayName: 'Limit',
@ -283,8 +284,8 @@ export const recordFields: INodeProperties[] = [
minValue: 1,
maxValue: 500,
},
default: 100,
description: 'How many results to return.',
default: 50,
description: 'Max number of results to return',
},
{
displayName: 'Simplify Response',
@ -301,7 +302,7 @@ export const recordFields: INodeProperties[] = [
},
},
default: true,
description: 'Return a simplified version of the response instead of the raw data.',
description: 'Whether to return a simplified version of the response instead of the raw data',
},
{
displayName: 'Options',
@ -325,7 +326,7 @@ export const recordFields: INodeProperties[] = [
name: 'selectedFields',
type: 'string',
default: '',
description: 'Subset of fields to return, supports select into sub fields. Example: <code>selectedFields = "a,e.d.f"</code>.',
description: 'Subset of fields to return, supports select into sub fields. Example: <code>selectedFields = "a,e.d.f"</code>',
},
// {
// displayName: 'Use Int64 Timestamp',