Add Google BigQuery node (#1635)

*  Google BigQuery node

* 🔨 Add comment dividers

*  Add whitespace for readability

* ✏️ Edit resource descriptions

*  Improvements

Co-authored-by: Iván Ovejero <ivov.src@gmail.com>
This commit is contained in:
Ricardo Espinoza 2021-04-17 09:04:25 -04:00 committed by GitHub
parent a0f42708d4
commit b0d3554ced
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 697 additions and 0 deletions

View file

@ -0,0 +1,25 @@
import {
ICredentialType,
NodePropertyTypes,
} from 'n8n-workflow';
const scopes = [
'https://www.googleapis.com/auth/bigquery',
];
export class GoogleBigQueryOAuth2Api implements ICredentialType {
name = 'googleBigQueryOAuth2Api';
extends = [
'googleOAuth2Api',
];
displayName = 'Google BigQuery OAuth2 API';
documentationUrl = 'google';
properties = [
{
displayName: 'Scope',
name: 'scope',
type: 'hidden' as NodePropertyTypes,
default: scopes.join(' '),
},
];
}

View file

@ -0,0 +1,80 @@
import {
OptionsWithUri,
} from 'request';
import {
IExecuteFunctions,
IExecuteSingleFunctions,
ILoadOptionsFunctions,
} from 'n8n-core';
import {
IDataObject,
} from 'n8n-workflow';
export async function googleApiRequest(this: IExecuteFunctions | IExecuteSingleFunctions | ILoadOptionsFunctions, method: string, resource: string, body: any = {}, qs: IDataObject = {}, uri?: string, headers: IDataObject = {}): Promise<any> { // tslint:disable-line:no-any
const options: OptionsWithUri = {
headers: {
'Content-Type': 'application/json',
},
method,
body,
qs,
uri: uri || `https://bigquery.googleapis.com/bigquery${resource}`,
json: true,
};
try {
if (Object.keys(headers).length !== 0) {
options.headers = Object.assign({}, options.headers, headers);
}
if (Object.keys(body).length === 0) {
delete options.body;
}
console.log(options);
//@ts-ignore
return await this.helpers.requestOAuth2.call(this, 'googleBigQueryOAuth2Api', options);
} catch (error) {
if (error.response && error.response.body && error.response.body.error) {
let errors = error.response.body.error.errors;
errors = errors.map((e: IDataObject) => e.message);
// Try to return the error prettier
throw new Error(
`Google BigQuery error response [${error.statusCode}]: ${errors.join('|')}`,
);
}
throw error;
}
}
export async function googleApiRequestAllItems(this: IExecuteFunctions | ILoadOptionsFunctions, propertyName: string, method: string, endpoint: string, body: any = {}, query: IDataObject = {}): Promise<any> { // tslint:disable-line:no-any
const returnData: IDataObject[] = [];
let responseData;
query.maxResults = 100;
do {
responseData = await googleApiRequest.call(this, method, endpoint, body, query);
query.pageToken = responseData['nextPageToken'];
returnData.push.apply(returnData, responseData[propertyName]);
} while (
responseData['nextPageToken'] !== undefined &&
responseData['nextPageToken'] !== ''
);
return returnData;
}
export function simplify(rows: IDataObject[], fields: string[]) {
const results = [];
for (const row of rows) {
const record: IDataObject = {};
for (const [index, field] of fields.entries()) {
record[field] = (row.f as IDataObject[])[index].v;
}
results.push(record);
}
return results;
}

View file

@ -0,0 +1,250 @@
import {
IExecuteFunctions,
} from 'n8n-core';
import {
IDataObject,
ILoadOptionsFunctions,
INodeExecutionData,
INodePropertyOptions,
INodeType,
INodeTypeDescription,
} from 'n8n-workflow';
import {
googleApiRequest,
googleApiRequestAllItems,
simplify,
} from './GenericFunctions';
import {
recordFields,
recordOperations,
} from './RecordDescription';
import * as uuid from 'uuid';
export class GoogleBigQuery implements INodeType {
description: INodeTypeDescription = {
displayName: 'Google BigQuery',
name: 'googleBigQuery',
icon: 'file:googleBigQuery.svg',
group: ['input'],
version: 1,
subtitle: '={{$parameter["operation"] + ": " + $parameter["resource"]}}',
description: 'Consume Google BigQuery API.',
defaults: {
name: 'Google BigQuery',
color: '#3E87E4',
},
inputs: ['main'],
outputs: ['main'],
credentials: [
{
name: 'googleBigQueryOAuth2Api',
required: true,
},
],
properties: [
{
displayName: 'Resource',
name: 'resource',
type: 'options',
options: [
{
name: 'Record',
value: 'record',
},
],
default: 'record',
description: 'The resource to operate on.',
},
...recordOperations,
...recordFields,
],
};
methods = {
loadOptions: {
async getProjects(
this: ILoadOptionsFunctions,
): Promise<INodePropertyOptions[]> {
const returnData: INodePropertyOptions[] = [];
const { projects } = await googleApiRequest.call(
this,
'GET',
'/v2/projects',
);
for (const project of projects) {
returnData.push({
name: project.friendlyName as string,
value: project.id,
});
}
return returnData;
},
async getDatasets(
this: ILoadOptionsFunctions,
): Promise<INodePropertyOptions[]> {
const projectId = this.getCurrentNodeParameter('projectId');
const returnData: INodePropertyOptions[] = [];
const { datasets } = await googleApiRequest.call(
this,
'GET',
`/v2/projects/${projectId}/datasets`,
);
for (const dataset of datasets) {
returnData.push({
name: dataset.datasetReference.datasetId as string,
value: dataset.datasetReference.datasetId,
});
}
return returnData;
},
async getTables(
this: ILoadOptionsFunctions,
): Promise<INodePropertyOptions[]> {
const projectId = this.getCurrentNodeParameter('projectId');
const datasetId = this.getCurrentNodeParameter('datasetId');
const returnData: INodePropertyOptions[] = [];
const { tables } = await googleApiRequest.call(
this,
'GET',
`/v2/projects/${projectId}/datasets/${datasetId}/tables`,
);
for (const table of tables) {
returnData.push({
name: table.tableReference.tableId as string,
value: table.tableReference.tableId,
});
}
return returnData;
},
},
};
async execute(this: IExecuteFunctions): Promise<INodeExecutionData[][]> {
const items = this.getInputData();
const returnData: IDataObject[] = [];
const length = (items.length as unknown) as number;
const qs: IDataObject = {};
let responseData;
const resource = this.getNodeParameter('resource', 0) as string;
const operation = this.getNodeParameter('operation', 0) as string;
if (resource === 'record') {
// *********************************************************************
// record
// *********************************************************************
if (operation === 'create') {
// ----------------------------------
// record: create
// ----------------------------------
// https://cloud.google.com/bigquery/docs/reference/rest/v2/tabledata/insertAll
const projectId = this.getNodeParameter('projectId', 0) as string;
const datasetId = this.getNodeParameter('datasetId', 0) as string;
const tableId = this.getNodeParameter('tableId', 0) as string;
const rows: IDataObject[] = [];
const body: IDataObject = {};
for (let i = 0; i < length; i++) {
const options = this.getNodeParameter('options', i) as IDataObject;
Object.assign(body, options);
if (body.traceId === undefined) {
body.traceId = uuid();
}
const columns = this.getNodeParameter('columns', i) as string;
const columnList = columns.split(',').map(column => column.trim());
const record: IDataObject = {};
for (const key of Object.keys(items[i].json)) {
if (columnList.includes(key)) {
record[`${key}`] = items[i].json[key];
}
}
rows.push({ json: record });
}
body.rows = rows;
responseData = await googleApiRequest.call(
this,
'POST',
`/v2/projects/${projectId}/datasets/${datasetId}/tables/${tableId}/insertAll`,
body,
);
returnData.push(responseData);
} else if (operation === 'getAll') {
// ----------------------------------
// record: getAll
// ----------------------------------
// https://cloud.google.com/bigquery/docs/reference/rest/v2/tables/get
const returnAll = this.getNodeParameter('returnAll', 0) as boolean;
const projectId = this.getNodeParameter('projectId', 0) as string;
const datasetId = this.getNodeParameter('datasetId', 0) as string;
const tableId = this.getNodeParameter('tableId', 0) as string;
const simple = this.getNodeParameter('simple', 0) as boolean;
let fields;
if (simple === true) {
const { schema } = await googleApiRequest.call(
this,
'GET',
`/v2/projects/${projectId}/datasets/${datasetId}/tables/${tableId}`,
{},
);
fields = schema.fields.map((field: IDataObject) => field.name);
}
for (let i = 0; i < length; i++) {
const options = this.getNodeParameter('options', i) as IDataObject;
Object.assign(qs, options);
// if (qs.useInt64Timestamp !== undefined) {
// qs.formatOptions = {
// useInt64Timestamp: qs.useInt64Timestamp,
// };
// delete qs.useInt64Timestamp;
// }
if (qs.selectedFields) {
fields = (qs.selectedFields as string).split(',');
}
if (returnAll) {
responseData = await googleApiRequestAllItems.call(
this,
'rows',
'GET',
`/v2/projects/${projectId}/datasets/${datasetId}/tables/${tableId}/data`,
{},
qs,
);
returnData.push.apply(returnData, (simple) ? simplify(responseData, fields) : responseData);
} else {
qs.maxResults = this.getNodeParameter('limit', i) as number;
responseData = await googleApiRequest.call(
this,
'GET',
`/v2/projects/${projectId}/datasets/${datasetId}/tables/${tableId}/data`,
{},
qs,
);
returnData.push.apply(returnData, (simple) ? simplify(responseData.rows, fields) : responseData.rows);
}
}
}
}
return [this.helpers.returnJsonArray(returnData)];
}
}

View file

@ -0,0 +1,339 @@
import {
INodeProperties,
} from 'n8n-workflow';
export const recordOperations = [
{
displayName: 'Operation',
name: 'operation',
type: 'options',
displayOptions: {
show: {
resource: [
'record',
],
},
},
options: [
{
name: 'Create',
value: 'create',
description: 'Create a new record.',
},
{
name: 'Get All',
value: 'getAll',
description: 'Retrieve all records.',
},
],
default: 'create',
description: 'Operation to perform.',
},
] as INodeProperties[];
export const recordFields = [
// ----------------------------------
// record: create
// ----------------------------------
{
displayName: 'Project ID',
name: 'projectId',
type: 'options',
typeOptions: {
loadOptionsMethod: 'getProjects',
},
required: true,
displayOptions: {
show: {
operation: [
'create',
],
resource: [
'record',
],
},
},
default: '',
description: 'ID of the project to create the record in.',
},
{
displayName: 'Dataset ID',
name: 'datasetId',
type: 'options',
typeOptions: {
loadOptionsMethod: 'getDatasets',
loadOptionsDependsOn: [
'projectId',
],
},
required: true,
displayOptions: {
show: {
operation: [
'create',
],
resource: [
'record',
],
},
},
default: '',
description: 'ID of the dataset to create the record in.',
},
{
displayName: 'Table ID',
name: 'tableId',
type: 'options',
typeOptions: {
loadOptionsMethod: 'getTables',
loadOptionsDependsOn: [
'projectId',
'datasetId',
],
},
required: true,
displayOptions: {
show: {
operation: [
'create',
],
resource: [
'record',
],
},
},
default: '',
description: 'ID of the table to create the record in.',
},
{
displayName: 'Columns',
name: 'columns',
type: 'string',
displayOptions: {
show: {
resource: [
'record',
],
operation: [
'create',
],
},
},
default: '',
required: true,
placeholder: 'id,name,description',
description: 'Comma-separated list of the item properties to use as columns.',
},
{
displayName: 'Options',
name: 'options',
type: 'collection',
placeholder: 'Add Options',
default: {},
displayOptions: {
show: {
operation: [
'create',
],
resource: [
'record',
],
},
},
options: [
{
displayName: 'Ignore Unknown Values',
name: 'ignoreUnknownValues',
type: 'boolean',
default: false,
description: 'Ignore row values that do not match the schema.',
},
{
displayName: 'Skip Invalid Rows',
name: 'skipInvalidRows',
type: 'boolean',
default: false,
description: 'Skip rows with values that do not match the schema.',
},
{
displayName: 'Template Suffix',
name: 'templateSuffix',
type: 'string',
default: '',
description: 'Create a new table based on the destination table and insert rows into the new table. The new table will be named <code>{destinationTable}{templateSuffix}</code>.',
},
{
displayName: 'Trace ID',
name: 'traceId',
type: 'string',
default: '',
description: 'Unique ID for the request, for debugging only. It is case-sensitive, limited to up to 36 ASCII characters. A UUID is recommended.',
},
],
},
// ----------------------------------
// record: getAll
// ----------------------------------
{
displayName: 'Project ID',
name: 'projectId',
type: 'options',
typeOptions: {
loadOptionsMethod: 'getProjects',
},
required: true,
displayOptions: {
show: {
operation: [
'getAll',
],
resource: [
'record',
],
},
},
default: '',
description: 'ID of the project to retrieve all rows from.',
},
{
displayName: 'Dataset ID',
name: 'datasetId',
type: 'options',
typeOptions: {
loadOptionsMethod: 'getDatasets',
loadOptionsDependsOn: [
'projectId',
],
},
required: true,
displayOptions: {
show: {
operation: [
'getAll',
],
resource: [
'record',
],
},
},
default: '',
description: 'ID of the dataset to retrieve all rows from.',
},
{
displayName: 'Table ID',
name: 'tableId',
type: 'options',
typeOptions: {
loadOptionsMethod: 'getTables',
loadOptionsDependsOn: [
'projectId',
'datasetId',
],
},
required: true,
displayOptions: {
show: {
operation: [
'getAll',
],
resource: [
'record',
],
},
},
default: '',
description: 'ID of the table to retrieve all rows from.',
},
{
displayName: 'Return All',
name: 'returnAll',
type: 'boolean',
displayOptions: {
show: {
operation: [
'getAll',
],
resource: [
'record',
],
},
},
default: false,
description: 'If all results should be returned or only up to a given limit.',
},
{
displayName: 'Limit',
name: 'limit',
type: 'number',
displayOptions: {
show: {
operation: [
'getAll',
],
resource: [
'record',
],
returnAll: [
false,
],
},
},
typeOptions: {
minValue: 1,
maxValue: 500,
},
default: 100,
description: 'How many results to return.',
},
{
displayName: 'Simple',
name: 'simple',
type: 'boolean',
displayOptions: {
show: {
resource: [
'record',
],
operation: [
'getAll',
],
},
},
default: true,
description: 'Return a simplified version of the response instead of the raw data.',
},
{
displayName: 'Options',
name: 'options',
type: 'collection',
placeholder: 'Add Options',
default: {},
displayOptions: {
show: {
operation: [
'getAll',
],
resource: [
'record',
],
},
},
options: [
{
displayName: 'Fields',
name: 'selectedFields',
type: 'string',
default: '',
description: 'Subset of fields to return, supports select into sub fields. Example: <code>selectedFields = "a,e.d.f"</code>.',
},
// {
// displayName: 'Use Int64 Timestamp',
// name: 'useInt64Timestamp',
// type: 'boolean',
// default: false,
// description: 'Output timestamp as usec int64.',
// },
],
},
] as INodeProperties[];

View file

@ -0,0 +1 @@
<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 0 66 58" fill="#fff" fill-rule="evenodd" stroke="#000" stroke-linecap="round" stroke-linejoin="round"><use xlink:href="#A" x=".5" y=".5"/><symbol id="A" overflow="visible"><g stroke="none" fill-rule="nonzero"><path d="M14.48 54.473L.558 30.359c-.744-1.288-.744-2.876 0-4.164L14.48 2.082C15.223.795 16.595.002 18.08 0h27.857c1.48.007 2.845.8 3.585 2.082l13.92 24.113c.744 1.288.744 2.876 0 4.164L49.52 54.473c-.743 1.287-2.115 2.08-3.6 2.082H18.07c-1.483-.005-2.85-.798-3.593-2.082z" fill="#4386fa"/><path d="M40.697 20.512s3.87 9.283-1.406 14.545-14.883 1.894-14.883 1.894L43.95 56.547h1.984c1.486-.002 2.858-.796 3.6-2.082l9.216-15.958z" fill="#000" opacity=".1"/><path d="M45.266 39.507L41 35.23a.67.67 0 0 0-.158-.12c3.767-4.852 3.111-11.798-1.499-15.859a11.63 11.63 0 0 0-16.396 16.436 11.63 11.63 0 0 0 15.863 1.46c.031.055.069.106.113.15l4.277 4.277a.67.67 0 0 0 .947 0l1.12-1.12a.67.67 0 0 0 0-.947zM31.64 36.741a8.75 8.75 0 0 1-6.188-14.937 8.75 8.75 0 0 1 14.937 6.188 8.75 8.75 0 0 1-8.749 8.749zm-5.593-9.216v3.616c.557.983 1.363 1.803 2.338 2.375v-6.013zm4.375-2.998v9.772a6.45 6.45 0 0 0 2.338 0v-9.772zm6.764 6.606v-2.142H34.85v4.5a6.43 6.43 0 0 0 2.338-2.368z"/></g></symbol></svg>

After

Width:  |  Height:  |  Size: 1.3 KiB

View file

@ -95,6 +95,7 @@
"dist/credentials/GmailOAuth2Api.credentials.js",
"dist/credentials/GoogleAnalyticsOAuth2Api.credentials.js",
"dist/credentials/GoogleApi.credentials.js",
"dist/credentials/GoogleBigQueryOAuth2Api.credentials.js",
"dist/credentials/GoogleBooksOAuth2Api.credentials.js",
"dist/credentials/GoogleCalendarOAuth2Api.credentials.js",
"dist/credentials/GoogleContactsOAuth2Api.credentials.js",
@ -362,6 +363,7 @@
"dist/nodes/Github/GithubTrigger.node.js",
"dist/nodes/Gitlab/Gitlab.node.js",
"dist/nodes/Gitlab/GitlabTrigger.node.js",
"dist/nodes/Google/BigQuery/GoogleBigQuery.node.js",
"dist/nodes/Google/Books/GoogleBooks.node.js",
"dist/nodes/Google/Analytics/GoogleAnalytics.node.js",
"dist/nodes/Google/Calendar/GoogleCalendar.node.js",