diff --git a/packages/nodes-base/credentials/GoogleBigQueryOAuth2Api.credentials.ts b/packages/nodes-base/credentials/GoogleBigQueryOAuth2Api.credentials.ts new file mode 100644 index 0000000000..5349ceebba --- /dev/null +++ b/packages/nodes-base/credentials/GoogleBigQueryOAuth2Api.credentials.ts @@ -0,0 +1,25 @@ +import { + ICredentialType, + NodePropertyTypes, +} from 'n8n-workflow'; + +const scopes = [ + 'https://www.googleapis.com/auth/bigquery', +]; + +export class GoogleBigQueryOAuth2Api implements ICredentialType { + name = 'googleBigQueryOAuth2Api'; + extends = [ + 'googleOAuth2Api', + ]; + displayName = 'Google BigQuery OAuth2 API'; + documentationUrl = 'google'; + properties = [ + { + displayName: 'Scope', + name: 'scope', + type: 'hidden' as NodePropertyTypes, + default: scopes.join(' '), + }, + ]; +} diff --git a/packages/nodes-base/nodes/Google/BigQuery/GenericFunctions.ts b/packages/nodes-base/nodes/Google/BigQuery/GenericFunctions.ts new file mode 100644 index 0000000000..b212c0c981 --- /dev/null +++ b/packages/nodes-base/nodes/Google/BigQuery/GenericFunctions.ts @@ -0,0 +1,80 @@ +import { + OptionsWithUri, +} from 'request'; + +import { + IExecuteFunctions, + IExecuteSingleFunctions, + ILoadOptionsFunctions, +} from 'n8n-core'; + +import { + IDataObject, +} from 'n8n-workflow'; + +export async function googleApiRequest(this: IExecuteFunctions | IExecuteSingleFunctions | ILoadOptionsFunctions, method: string, resource: string, body: any = {}, qs: IDataObject = {}, uri?: string, headers: IDataObject = {}): Promise { // tslint:disable-line:no-any + const options: OptionsWithUri = { + headers: { + 'Content-Type': 'application/json', + }, + method, + body, + qs, + uri: uri || `https://bigquery.googleapis.com/bigquery${resource}`, + json: true, + }; + try { + if (Object.keys(headers).length !== 0) { + options.headers = Object.assign({}, options.headers, headers); + } + if (Object.keys(body).length === 0) { + delete options.body; + } + console.log(options); + //@ts-ignore + return await this.helpers.requestOAuth2.call(this, 'googleBigQueryOAuth2Api', options); + } catch (error) { + if (error.response && error.response.body && error.response.body.error) { + + let errors = error.response.body.error.errors; + + errors = errors.map((e: IDataObject) => e.message); + // Try to return the error prettier + throw new Error( + `Google BigQuery error response [${error.statusCode}]: ${errors.join('|')}`, + ); + } + throw error; + } +} + +export async function googleApiRequestAllItems(this: IExecuteFunctions | ILoadOptionsFunctions, propertyName: string, method: string, endpoint: string, body: any = {}, query: IDataObject = {}): Promise { // tslint:disable-line:no-any + + const returnData: IDataObject[] = []; + + let responseData; + query.maxResults = 100; + + do { + responseData = await googleApiRequest.call(this, method, endpoint, body, query); + query.pageToken = responseData['nextPageToken']; + returnData.push.apply(returnData, responseData[propertyName]); + } while ( + responseData['nextPageToken'] !== undefined && + responseData['nextPageToken'] !== '' + ); + + return returnData; +} + +export function simplify(rows: IDataObject[], fields: string[]) { + const results = []; + for (const row of rows) { + const record: IDataObject = {}; + for (const [index, field] of fields.entries()) { + record[field] = (row.f as IDataObject[])[index].v; + } + results.push(record); + } + return results; +} diff --git a/packages/nodes-base/nodes/Google/BigQuery/GoogleBigQuery.node.ts b/packages/nodes-base/nodes/Google/BigQuery/GoogleBigQuery.node.ts new file mode 100644 index 0000000000..9a848cd58f --- /dev/null +++ b/packages/nodes-base/nodes/Google/BigQuery/GoogleBigQuery.node.ts @@ -0,0 +1,250 @@ +import { + IExecuteFunctions, +} from 'n8n-core'; + +import { + IDataObject, + ILoadOptionsFunctions, + INodeExecutionData, + INodePropertyOptions, + INodeType, + INodeTypeDescription, +} from 'n8n-workflow'; + +import { + googleApiRequest, + googleApiRequestAllItems, + simplify, +} from './GenericFunctions'; + +import { + recordFields, + recordOperations, +} from './RecordDescription'; + +import * as uuid from 'uuid'; + +export class GoogleBigQuery implements INodeType { + description: INodeTypeDescription = { + displayName: 'Google BigQuery', + name: 'googleBigQuery', + icon: 'file:googleBigQuery.svg', + group: ['input'], + version: 1, + subtitle: '={{$parameter["operation"] + ": " + $parameter["resource"]}}', + description: 'Consume Google BigQuery API.', + defaults: { + name: 'Google BigQuery', + color: '#3E87E4', + }, + inputs: ['main'], + outputs: ['main'], + credentials: [ + { + name: 'googleBigQueryOAuth2Api', + required: true, + }, + ], + properties: [ + { + displayName: 'Resource', + name: 'resource', + type: 'options', + options: [ + { + name: 'Record', + value: 'record', + }, + ], + default: 'record', + description: 'The resource to operate on.', + }, + ...recordOperations, + ...recordFields, + ], + }; + + methods = { + loadOptions: { + async getProjects( + this: ILoadOptionsFunctions, + ): Promise { + const returnData: INodePropertyOptions[] = []; + const { projects } = await googleApiRequest.call( + this, + 'GET', + '/v2/projects', + ); + for (const project of projects) { + returnData.push({ + name: project.friendlyName as string, + value: project.id, + }); + } + return returnData; + }, + async getDatasets( + this: ILoadOptionsFunctions, + ): Promise { + const projectId = this.getCurrentNodeParameter('projectId'); + const returnData: INodePropertyOptions[] = []; + const { datasets } = await googleApiRequest.call( + this, + 'GET', + `/v2/projects/${projectId}/datasets`, + ); + for (const dataset of datasets) { + returnData.push({ + name: dataset.datasetReference.datasetId as string, + value: dataset.datasetReference.datasetId, + }); + } + return returnData; + }, + async getTables( + this: ILoadOptionsFunctions, + ): Promise { + const projectId = this.getCurrentNodeParameter('projectId'); + const datasetId = this.getCurrentNodeParameter('datasetId'); + const returnData: INodePropertyOptions[] = []; + const { tables } = await googleApiRequest.call( + this, + 'GET', + `/v2/projects/${projectId}/datasets/${datasetId}/tables`, + ); + for (const table of tables) { + returnData.push({ + name: table.tableReference.tableId as string, + value: table.tableReference.tableId, + }); + } + return returnData; + }, + }, + }; + + async execute(this: IExecuteFunctions): Promise { + const items = this.getInputData(); + const returnData: IDataObject[] = []; + const length = (items.length as unknown) as number; + const qs: IDataObject = {}; + let responseData; + const resource = this.getNodeParameter('resource', 0) as string; + const operation = this.getNodeParameter('operation', 0) as string; + + if (resource === 'record') { + + // ********************************************************************* + // record + // ********************************************************************* + + if (operation === 'create') { + + // ---------------------------------- + // record: create + // ---------------------------------- + + // https://cloud.google.com/bigquery/docs/reference/rest/v2/tabledata/insertAll + + const projectId = this.getNodeParameter('projectId', 0) as string; + const datasetId = this.getNodeParameter('datasetId', 0) as string; + const tableId = this.getNodeParameter('tableId', 0) as string; + const rows: IDataObject[] = []; + const body: IDataObject = {}; + + for (let i = 0; i < length; i++) { + + const options = this.getNodeParameter('options', i) as IDataObject; + Object.assign(body, options); + if (body.traceId === undefined) { + body.traceId = uuid(); + } + const columns = this.getNodeParameter('columns', i) as string; + const columnList = columns.split(',').map(column => column.trim()); + const record: IDataObject = {}; + + for (const key of Object.keys(items[i].json)) { + if (columnList.includes(key)) { + record[`${key}`] = items[i].json[key]; + } + } + rows.push({ json: record }); + } + + body.rows = rows; + responseData = await googleApiRequest.call( + this, + 'POST', + `/v2/projects/${projectId}/datasets/${datasetId}/tables/${tableId}/insertAll`, + body, + ); + returnData.push(responseData); + + } else if (operation === 'getAll') { + + // ---------------------------------- + // record: getAll + // ---------------------------------- + + // https://cloud.google.com/bigquery/docs/reference/rest/v2/tables/get + + const returnAll = this.getNodeParameter('returnAll', 0) as boolean; + const projectId = this.getNodeParameter('projectId', 0) as string; + const datasetId = this.getNodeParameter('datasetId', 0) as string; + const tableId = this.getNodeParameter('tableId', 0) as string; + const simple = this.getNodeParameter('simple', 0) as boolean; + let fields; + + if (simple === true) { + const { schema } = await googleApiRequest.call( + this, + 'GET', + `/v2/projects/${projectId}/datasets/${datasetId}/tables/${tableId}`, + {}, + ); + fields = schema.fields.map((field: IDataObject) => field.name); + } + + for (let i = 0; i < length; i++) { + const options = this.getNodeParameter('options', i) as IDataObject; + Object.assign(qs, options); + + // if (qs.useInt64Timestamp !== undefined) { + // qs.formatOptions = { + // useInt64Timestamp: qs.useInt64Timestamp, + // }; + // delete qs.useInt64Timestamp; + // } + + if (qs.selectedFields) { + fields = (qs.selectedFields as string).split(','); + } + + if (returnAll) { + responseData = await googleApiRequestAllItems.call( + this, + 'rows', + 'GET', + `/v2/projects/${projectId}/datasets/${datasetId}/tables/${tableId}/data`, + {}, + qs, + ); + returnData.push.apply(returnData, (simple) ? simplify(responseData, fields) : responseData); + } else { + qs.maxResults = this.getNodeParameter('limit', i) as number; + responseData = await googleApiRequest.call( + this, + 'GET', + `/v2/projects/${projectId}/datasets/${datasetId}/tables/${tableId}/data`, + {}, + qs, + ); + returnData.push.apply(returnData, (simple) ? simplify(responseData.rows, fields) : responseData.rows); + } + } + } + } + + return [this.helpers.returnJsonArray(returnData)]; + } +} diff --git a/packages/nodes-base/nodes/Google/BigQuery/RecordDescription.ts b/packages/nodes-base/nodes/Google/BigQuery/RecordDescription.ts new file mode 100644 index 0000000000..3f490ee648 --- /dev/null +++ b/packages/nodes-base/nodes/Google/BigQuery/RecordDescription.ts @@ -0,0 +1,339 @@ +import { + INodeProperties, +} from 'n8n-workflow'; + +export const recordOperations = [ + { + displayName: 'Operation', + name: 'operation', + type: 'options', + displayOptions: { + show: { + resource: [ + 'record', + ], + }, + }, + options: [ + { + name: 'Create', + value: 'create', + description: 'Create a new record.', + }, + { + name: 'Get All', + value: 'getAll', + description: 'Retrieve all records.', + }, + ], + default: 'create', + description: 'Operation to perform.', + }, +] as INodeProperties[]; + +export const recordFields = [ + // ---------------------------------- + // record: create + // ---------------------------------- + { + displayName: 'Project ID', + name: 'projectId', + type: 'options', + typeOptions: { + loadOptionsMethod: 'getProjects', + }, + required: true, + displayOptions: { + show: { + operation: [ + 'create', + ], + resource: [ + 'record', + ], + }, + }, + default: '', + description: 'ID of the project to create the record in.', + }, + { + displayName: 'Dataset ID', + name: 'datasetId', + type: 'options', + typeOptions: { + loadOptionsMethod: 'getDatasets', + loadOptionsDependsOn: [ + 'projectId', + ], + }, + required: true, + displayOptions: { + show: { + operation: [ + 'create', + ], + resource: [ + 'record', + ], + }, + }, + default: '', + description: 'ID of the dataset to create the record in.', + }, + { + displayName: 'Table ID', + name: 'tableId', + type: 'options', + typeOptions: { + loadOptionsMethod: 'getTables', + loadOptionsDependsOn: [ + 'projectId', + 'datasetId', + ], + }, + required: true, + displayOptions: { + show: { + operation: [ + 'create', + ], + resource: [ + 'record', + ], + }, + }, + default: '', + description: 'ID of the table to create the record in.', + }, + { + displayName: 'Columns', + name: 'columns', + type: 'string', + displayOptions: { + show: { + resource: [ + 'record', + ], + operation: [ + 'create', + ], + }, + }, + default: '', + required: true, + placeholder: 'id,name,description', + description: 'Comma-separated list of the item properties to use as columns.', + }, + { + displayName: 'Options', + name: 'options', + type: 'collection', + placeholder: 'Add Options', + default: {}, + displayOptions: { + show: { + operation: [ + 'create', + ], + resource: [ + 'record', + ], + }, + }, + options: [ + { + displayName: 'Ignore Unknown Values', + name: 'ignoreUnknownValues', + type: 'boolean', + default: false, + description: 'Ignore row values that do not match the schema.', + }, + { + displayName: 'Skip Invalid Rows', + name: 'skipInvalidRows', + type: 'boolean', + default: false, + description: 'Skip rows with values that do not match the schema.', + }, + { + displayName: 'Template Suffix', + name: 'templateSuffix', + type: 'string', + default: '', + description: 'Create a new table based on the destination table and insert rows into the new table. The new table will be named {destinationTable}{templateSuffix}.', + }, + { + displayName: 'Trace ID', + name: 'traceId', + type: 'string', + default: '', + description: 'Unique ID for the request, for debugging only. It is case-sensitive, limited to up to 36 ASCII characters. A UUID is recommended.', + }, + ], + }, + + // ---------------------------------- + // record: getAll + // ---------------------------------- + { + displayName: 'Project ID', + name: 'projectId', + type: 'options', + typeOptions: { + loadOptionsMethod: 'getProjects', + }, + required: true, + displayOptions: { + show: { + operation: [ + 'getAll', + ], + resource: [ + 'record', + ], + }, + }, + default: '', + description: 'ID of the project to retrieve all rows from.', + }, + { + displayName: 'Dataset ID', + name: 'datasetId', + type: 'options', + typeOptions: { + loadOptionsMethod: 'getDatasets', + loadOptionsDependsOn: [ + 'projectId', + ], + }, + required: true, + displayOptions: { + show: { + operation: [ + 'getAll', + ], + resource: [ + 'record', + ], + }, + }, + default: '', + description: 'ID of the dataset to retrieve all rows from.', + }, + { + displayName: 'Table ID', + name: 'tableId', + type: 'options', + typeOptions: { + loadOptionsMethod: 'getTables', + loadOptionsDependsOn: [ + 'projectId', + 'datasetId', + ], + }, + required: true, + displayOptions: { + show: { + operation: [ + 'getAll', + ], + resource: [ + 'record', + ], + }, + }, + default: '', + description: 'ID of the table to retrieve all rows from.', + }, + { + displayName: 'Return All', + name: 'returnAll', + type: 'boolean', + displayOptions: { + show: { + operation: [ + 'getAll', + ], + resource: [ + 'record', + ], + }, + }, + default: false, + description: 'If all results should be returned or only up to a given limit.', + }, + { + displayName: 'Limit', + name: 'limit', + type: 'number', + displayOptions: { + show: { + operation: [ + 'getAll', + ], + resource: [ + 'record', + ], + returnAll: [ + false, + ], + }, + }, + typeOptions: { + minValue: 1, + maxValue: 500, + }, + default: 100, + description: 'How many results to return.', + }, + { + displayName: 'Simple', + name: 'simple', + type: 'boolean', + displayOptions: { + show: { + resource: [ + 'record', + ], + operation: [ + 'getAll', + ], + }, + }, + default: true, + description: 'Return a simplified version of the response instead of the raw data.', + }, + { + displayName: 'Options', + name: 'options', + type: 'collection', + placeholder: 'Add Options', + default: {}, + displayOptions: { + show: { + operation: [ + 'getAll', + ], + resource: [ + 'record', + ], + }, + }, + options: [ + { + displayName: 'Fields', + name: 'selectedFields', + type: 'string', + default: '', + description: 'Subset of fields to return, supports select into sub fields. Example: selectedFields = "a,e.d.f".', + }, + // { + // displayName: 'Use Int64 Timestamp', + // name: 'useInt64Timestamp', + // type: 'boolean', + // default: false, + // description: 'Output timestamp as usec int64.', + // }, + ], + }, +] as INodeProperties[]; diff --git a/packages/nodes-base/nodes/Google/BigQuery/googleBigQuery.svg b/packages/nodes-base/nodes/Google/BigQuery/googleBigQuery.svg new file mode 100644 index 0000000000..b006a05d6a --- /dev/null +++ b/packages/nodes-base/nodes/Google/BigQuery/googleBigQuery.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/packages/nodes-base/package.json b/packages/nodes-base/package.json index 3025284dd7..ac3416f4ca 100644 --- a/packages/nodes-base/package.json +++ b/packages/nodes-base/package.json @@ -95,6 +95,7 @@ "dist/credentials/GmailOAuth2Api.credentials.js", "dist/credentials/GoogleAnalyticsOAuth2Api.credentials.js", "dist/credentials/GoogleApi.credentials.js", + "dist/credentials/GoogleBigQueryOAuth2Api.credentials.js", "dist/credentials/GoogleBooksOAuth2Api.credentials.js", "dist/credentials/GoogleCalendarOAuth2Api.credentials.js", "dist/credentials/GoogleContactsOAuth2Api.credentials.js", @@ -362,6 +363,7 @@ "dist/nodes/Github/GithubTrigger.node.js", "dist/nodes/Gitlab/Gitlab.node.js", "dist/nodes/Gitlab/GitlabTrigger.node.js", + "dist/nodes/Google/BigQuery/GoogleBigQuery.node.js", "dist/nodes/Google/Books/GoogleBooks.node.js", "dist/nodes/Google/Analytics/GoogleAnalytics.node.js", "dist/nodes/Google/Calendar/GoogleCalendar.node.js",