mirror of
https://github.com/n8n-io/n8n.git
synced 2024-12-25 04:34:06 -08:00
fix(Google BigQuery Node): Send timeoutMs in query, pagination support (#10205)
This commit is contained in:
parent
cf70b06545
commit
f5722e8823
|
@ -10,7 +10,7 @@ jest.mock('../../../v2/transport', () => {
|
||||||
const originalModule = jest.requireActual('../../../v2/transport');
|
const originalModule = jest.requireActual('../../../v2/transport');
|
||||||
return {
|
return {
|
||||||
...originalModule,
|
...originalModule,
|
||||||
googleApiRequest: jest.fn(async (method: IHttpRequestMethods, resource: string) => {
|
googleBigQueryApiRequest: jest.fn(async (method: IHttpRequestMethods, resource: string) => {
|
||||||
if (resource === '/v2/projects/test-project/jobs' && method === 'POST') {
|
if (resource === '/v2/projects/test-project/jobs' && method === 'POST') {
|
||||||
return {
|
return {
|
||||||
jobReference: {
|
jobReference: {
|
||||||
|
@ -25,7 +25,7 @@ jest.mock('../../../v2/transport', () => {
|
||||||
return {};
|
return {};
|
||||||
}
|
}
|
||||||
}),
|
}),
|
||||||
// googleApiRequestAllItems: jest.fn(async () => {}),
|
googleBigQueryApiRequestAllItems: jest.fn(async () => ({ rows: [], schema: {} })),
|
||||||
};
|
};
|
||||||
});
|
});
|
||||||
|
|
||||||
|
@ -47,8 +47,9 @@ describe('Test Google BigQuery V2, executeQuery', () => {
|
||||||
const testNode = async (testData: WorkflowTestData, types: INodeTypes) => {
|
const testNode = async (testData: WorkflowTestData, types: INodeTypes) => {
|
||||||
const { result } = await executeWorkflow(testData, types);
|
const { result } = await executeWorkflow(testData, types);
|
||||||
|
|
||||||
expect(transport.googleApiRequest).toHaveBeenCalledTimes(2);
|
expect(transport.googleBigQueryApiRequest).toHaveBeenCalledTimes(1);
|
||||||
expect(transport.googleApiRequest).toHaveBeenCalledWith(
|
expect(transport.googleBigQueryApiRequestAllItems).toHaveBeenCalledTimes(1);
|
||||||
|
expect(transport.googleBigQueryApiRequest).toHaveBeenCalledWith(
|
||||||
'POST',
|
'POST',
|
||||||
'/v2/projects/test-project/jobs',
|
'/v2/projects/test-project/jobs',
|
||||||
{
|
{
|
||||||
|
@ -60,11 +61,11 @@ describe('Test Google BigQuery V2, executeQuery', () => {
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
expect(transport.googleApiRequest).toHaveBeenCalledWith(
|
expect(transport.googleBigQueryApiRequestAllItems).toHaveBeenCalledWith(
|
||||||
'GET',
|
'GET',
|
||||||
'/v2/projects/test-project/queries/job_123',
|
'/v2/projects/test-project/queries/job_123',
|
||||||
undefined,
|
undefined,
|
||||||
{},
|
{ location: undefined, maxResults: 1000, timeoutMs: 10000 },
|
||||||
);
|
);
|
||||||
|
|
||||||
expect(result.finished).toEqual(true);
|
expect(result.finished).toEqual(true);
|
||||||
|
|
|
@ -10,7 +10,7 @@ jest.mock('../../../v2/transport', () => {
|
||||||
const originalModule = jest.requireActual('../../../v2/transport');
|
const originalModule = jest.requireActual('../../../v2/transport');
|
||||||
return {
|
return {
|
||||||
...originalModule,
|
...originalModule,
|
||||||
googleApiRequest: jest.fn(async (method: IHttpRequestMethods, resource: string) => {
|
googleBigQueryApiRequest: jest.fn(async (method: IHttpRequestMethods, resource: string) => {
|
||||||
if (
|
if (
|
||||||
resource ===
|
resource ===
|
||||||
'/v2/projects/test-project/datasets/bigquery_node_dev_test_dataset/tables/num_text' &&
|
'/v2/projects/test-project/datasets/bigquery_node_dev_test_dataset/tables/num_text' &&
|
||||||
|
@ -55,13 +55,13 @@ describe('Test Google BigQuery V2, insert auto map', () => {
|
||||||
const testNode = async (testData: WorkflowTestData, types: INodeTypes) => {
|
const testNode = async (testData: WorkflowTestData, types: INodeTypes) => {
|
||||||
const { result } = await executeWorkflow(testData, types);
|
const { result } = await executeWorkflow(testData, types);
|
||||||
|
|
||||||
expect(transport.googleApiRequest).toHaveBeenCalledTimes(2);
|
expect(transport.googleBigQueryApiRequest).toHaveBeenCalledTimes(2);
|
||||||
expect(transport.googleApiRequest).toHaveBeenCalledWith(
|
expect(transport.googleBigQueryApiRequest).toHaveBeenCalledWith(
|
||||||
'GET',
|
'GET',
|
||||||
'/v2/projects/test-project/datasets/bigquery_node_dev_test_dataset/tables/num_text',
|
'/v2/projects/test-project/datasets/bigquery_node_dev_test_dataset/tables/num_text',
|
||||||
{},
|
{},
|
||||||
);
|
);
|
||||||
expect(transport.googleApiRequest).toHaveBeenCalledWith(
|
expect(transport.googleBigQueryApiRequest).toHaveBeenCalledWith(
|
||||||
'POST',
|
'POST',
|
||||||
'/v2/projects/test-project/datasets/bigquery_node_dev_test_dataset/tables/num_text/insertAll',
|
'/v2/projects/test-project/datasets/bigquery_node_dev_test_dataset/tables/num_text/insertAll',
|
||||||
{
|
{
|
||||||
|
|
|
@ -10,7 +10,7 @@ jest.mock('../../../v2/transport', () => {
|
||||||
const originalModule = jest.requireActual('../../../v2/transport');
|
const originalModule = jest.requireActual('../../../v2/transport');
|
||||||
return {
|
return {
|
||||||
...originalModule,
|
...originalModule,
|
||||||
googleApiRequest: jest.fn(async (method: IHttpRequestMethods, resource: string) => {
|
googleBigQueryApiRequest: jest.fn(async (method: IHttpRequestMethods, resource: string) => {
|
||||||
if (
|
if (
|
||||||
resource ===
|
resource ===
|
||||||
'/v2/projects/test-project/datasets/bigquery_node_dev_test_dataset/tables/test_json' &&
|
'/v2/projects/test-project/datasets/bigquery_node_dev_test_dataset/tables/test_json' &&
|
||||||
|
@ -56,13 +56,13 @@ describe('Test Google BigQuery V2, insert define manually', () => {
|
||||||
const testNode = async (testData: WorkflowTestData, types: INodeTypes) => {
|
const testNode = async (testData: WorkflowTestData, types: INodeTypes) => {
|
||||||
const { result } = await executeWorkflow(testData, types);
|
const { result } = await executeWorkflow(testData, types);
|
||||||
|
|
||||||
expect(transport.googleApiRequest).toHaveBeenCalledTimes(2);
|
expect(transport.googleBigQueryApiRequest).toHaveBeenCalledTimes(2);
|
||||||
expect(transport.googleApiRequest).toHaveBeenCalledWith(
|
expect(transport.googleBigQueryApiRequest).toHaveBeenCalledWith(
|
||||||
'GET',
|
'GET',
|
||||||
'/v2/projects/test-project/datasets/bigquery_node_dev_test_dataset/tables/test_json',
|
'/v2/projects/test-project/datasets/bigquery_node_dev_test_dataset/tables/test_json',
|
||||||
{},
|
{},
|
||||||
);
|
);
|
||||||
expect(transport.googleApiRequest).toHaveBeenCalledWith(
|
expect(transport.googleBigQueryApiRequest).toHaveBeenCalledWith(
|
||||||
'POST',
|
'POST',
|
||||||
'/v2/projects/test-project/datasets/bigquery_node_dev_test_dataset/tables/test_json/insertAll',
|
'/v2/projects/test-project/datasets/bigquery_node_dev_test_dataset/tables/test_json/insertAll',
|
||||||
{
|
{
|
||||||
|
|
|
@ -9,7 +9,7 @@ import { ApplicationError, NodeOperationError, sleep } from 'n8n-workflow';
|
||||||
import type { ResponseWithJobReference } from '../../helpers/interfaces';
|
import type { ResponseWithJobReference } from '../../helpers/interfaces';
|
||||||
|
|
||||||
import { prepareOutput } from '../../helpers/utils';
|
import { prepareOutput } from '../../helpers/utils';
|
||||||
import { googleApiRequest } from '../../transport';
|
import { googleBigQueryApiRequestAllItems, googleBigQueryApiRequest } from '../../transport';
|
||||||
import { getResolvables, updateDisplayOptions } from '@utils/utilities';
|
import { getResolvables, updateDisplayOptions } from '@utils/utilities';
|
||||||
|
|
||||||
const properties: INodeProperties[] = [
|
const properties: INodeProperties[] = [
|
||||||
|
@ -108,18 +108,21 @@ const properties: INodeProperties[] = [
|
||||||
'Limits the bytes billed for this query. Queries with bytes billed above this limit will fail (without incurring a charge). String in <a href="https://developers.google.com/discovery/v1/type-format?utm_source=cloud.google.com&utm_medium=referral" target="_blank">Int64Value</a> format',
|
'Limits the bytes billed for this query. Queries with bytes billed above this limit will fail (without incurring a charge). String in <a href="https://developers.google.com/discovery/v1/type-format?utm_source=cloud.google.com&utm_medium=referral" target="_blank">Int64Value</a> format',
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
displayName: 'Max Results',
|
displayName: 'Max Results Per Page',
|
||||||
name: 'maxResults',
|
name: 'maxResults',
|
||||||
type: 'number',
|
type: 'number',
|
||||||
default: 1000,
|
default: 1000,
|
||||||
description: 'The maximum number of rows of data to return',
|
description:
|
||||||
|
'Maximum number of results to return per page of results. This is particularly useful when dealing with large datasets. It will not affect the total number of results returned, e.g. rows in a table. You can use LIMIT in your SQL query to limit the number of rows returned.',
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
displayName: 'Timeout',
|
displayName: 'Timeout',
|
||||||
name: 'timeoutMs',
|
name: 'timeoutMs',
|
||||||
type: 'number',
|
type: 'number',
|
||||||
default: 10000,
|
default: 10000,
|
||||||
description: 'How long to wait for the query to complete, in milliseconds',
|
hint: 'How long to wait for the query to complete, in milliseconds',
|
||||||
|
description:
|
||||||
|
'Specifies the maximum amount of time, in milliseconds, that the client is willing to wait for the query to complete. Be aware that the call is not guaranteed to wait for the specified timeout; it typically returns after around 200 seconds (200,000 milliseconds), even if the query is not complete.',
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
displayName: 'Raw Output',
|
displayName: 'Raw Output',
|
||||||
|
@ -154,19 +157,31 @@ const displayOptions = {
|
||||||
export const description = updateDisplayOptions(displayOptions, properties);
|
export const description = updateDisplayOptions(displayOptions, properties);
|
||||||
|
|
||||||
export async function execute(this: IExecuteFunctions): Promise<INodeExecutionData[]> {
|
export async function execute(this: IExecuteFunctions): Promise<INodeExecutionData[]> {
|
||||||
// https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs/query
|
|
||||||
|
|
||||||
const items = this.getInputData();
|
const items = this.getInputData();
|
||||||
const length = items.length;
|
const length = items.length;
|
||||||
|
|
||||||
const returnData: INodeExecutionData[] = [];
|
const returnData: INodeExecutionData[] = [];
|
||||||
|
|
||||||
let jobs = [];
|
let jobs = [];
|
||||||
|
let maxResults = 1000;
|
||||||
|
let timeoutMs = 10000;
|
||||||
|
|
||||||
for (let i = 0; i < length; i++) {
|
for (let i = 0; i < length; i++) {
|
||||||
try {
|
try {
|
||||||
let sqlQuery = this.getNodeParameter('sqlQuery', i) as string;
|
let sqlQuery = this.getNodeParameter('sqlQuery', i) as string;
|
||||||
const options = this.getNodeParameter('options', i);
|
|
||||||
|
const options = this.getNodeParameter('options', i) as {
|
||||||
|
defaultDataset?: string;
|
||||||
|
dryRun?: boolean;
|
||||||
|
includeSchema?: boolean;
|
||||||
|
location?: string;
|
||||||
|
maximumBytesBilled?: string;
|
||||||
|
maxResults?: number;
|
||||||
|
timeoutMs?: number;
|
||||||
|
rawOutput?: boolean;
|
||||||
|
useLegacySql?: boolean;
|
||||||
|
};
|
||||||
|
|
||||||
const projectId = this.getNodeParameter('projectId', i, undefined, {
|
const projectId = this.getNodeParameter('projectId', i, undefined, {
|
||||||
extractValue: true,
|
extractValue: true,
|
||||||
});
|
});
|
||||||
|
@ -179,15 +194,25 @@ export async function execute(this: IExecuteFunctions): Promise<INodeExecutionDa
|
||||||
let includeSchema = false;
|
let includeSchema = false;
|
||||||
|
|
||||||
if (options.rawOutput !== undefined) {
|
if (options.rawOutput !== undefined) {
|
||||||
rawOutput = options.rawOutput as boolean;
|
rawOutput = options.rawOutput;
|
||||||
delete options.rawOutput;
|
delete options.rawOutput;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (options.includeSchema !== undefined) {
|
if (options.includeSchema !== undefined) {
|
||||||
includeSchema = options.includeSchema as boolean;
|
includeSchema = options.includeSchema;
|
||||||
delete options.includeSchema;
|
delete options.includeSchema;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (options.maxResults) {
|
||||||
|
maxResults = options.maxResults;
|
||||||
|
delete options.maxResults;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (options.timeoutMs) {
|
||||||
|
timeoutMs = options.timeoutMs;
|
||||||
|
delete options.timeoutMs;
|
||||||
|
}
|
||||||
|
|
||||||
const body: IDataObject = { ...options };
|
const body: IDataObject = { ...options };
|
||||||
|
|
||||||
body.query = sqlQuery;
|
body.query = sqlQuery;
|
||||||
|
@ -203,7 +228,8 @@ export async function execute(this: IExecuteFunctions): Promise<INodeExecutionDa
|
||||||
body.useLegacySql = false;
|
body.useLegacySql = false;
|
||||||
}
|
}
|
||||||
|
|
||||||
const response: ResponseWithJobReference = await googleApiRequest.call(
|
//https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs/insert
|
||||||
|
const response: ResponseWithJobReference = await googleBigQueryApiRequest.call(
|
||||||
this,
|
this,
|
||||||
'POST',
|
'POST',
|
||||||
`/v2/projects/${projectId}/jobs`,
|
`/v2/projects/${projectId}/jobs`,
|
||||||
|
@ -222,13 +248,14 @@ export async function execute(this: IExecuteFunctions): Promise<INodeExecutionDa
|
||||||
}
|
}
|
||||||
|
|
||||||
const jobId = response?.jobReference?.jobId;
|
const jobId = response?.jobReference?.jobId;
|
||||||
const raw = rawOutput || (options.dryRun as boolean) || false;
|
const raw = rawOutput || options.dryRun || false;
|
||||||
const location = options.location || response.jobReference.location;
|
const location = options.location || response.jobReference.location;
|
||||||
|
|
||||||
if (response.status?.state === 'DONE') {
|
if (response.status?.state === 'DONE') {
|
||||||
const qs = { location };
|
const qs = { location, maxResults, timeoutMs };
|
||||||
|
|
||||||
const queryResponse: IDataObject = await googleApiRequest.call(
|
//https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs/getQueryResults
|
||||||
|
const queryResponse: IDataObject = await googleBigQueryApiRequestAllItems.call(
|
||||||
this,
|
this,
|
||||||
'GET',
|
'GET',
|
||||||
`/v2/projects/${projectId}/queries/${jobId}`,
|
`/v2/projects/${projectId}/queries/${jobId}`,
|
||||||
|
@ -272,9 +299,13 @@ export async function execute(this: IExecuteFunctions): Promise<INodeExecutionDa
|
||||||
|
|
||||||
for (const job of jobs) {
|
for (const job of jobs) {
|
||||||
try {
|
try {
|
||||||
const qs = job.location ? { location: job.location } : {};
|
const qs: IDataObject = job.location ? { location: job.location } : {};
|
||||||
|
|
||||||
const response: IDataObject = await googleApiRequest.call(
|
qs.maxResults = maxResults;
|
||||||
|
qs.timeoutMs = timeoutMs;
|
||||||
|
|
||||||
|
//https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs/getQueryResults
|
||||||
|
const response: IDataObject = await googleBigQueryApiRequestAllItems.call(
|
||||||
this,
|
this,
|
||||||
'GET',
|
'GET',
|
||||||
`/v2/projects/${job.projectId}/queries/${job.jobId}`,
|
`/v2/projects/${job.projectId}/queries/${job.jobId}`,
|
||||||
|
|
|
@ -8,7 +8,7 @@ import { NodeOperationError } from 'n8n-workflow';
|
||||||
import { v4 as uuid } from 'uuid';
|
import { v4 as uuid } from 'uuid';
|
||||||
import type { TableSchema } from '../../helpers/interfaces';
|
import type { TableSchema } from '../../helpers/interfaces';
|
||||||
import { checkSchema, wrapData } from '../../helpers/utils';
|
import { checkSchema, wrapData } from '../../helpers/utils';
|
||||||
import { googleApiRequest } from '../../transport';
|
import { googleBigQueryApiRequest } from '../../transport';
|
||||||
import { generatePairedItemData, updateDisplayOptions } from '@utils/utilities';
|
import { generatePairedItemData, updateDisplayOptions } from '@utils/utilities';
|
||||||
|
|
||||||
const properties: INodeProperties[] = [
|
const properties: INodeProperties[] = [
|
||||||
|
@ -178,7 +178,7 @@ export async function execute(this: IExecuteFunctions): Promise<INodeExecutionDa
|
||||||
}
|
}
|
||||||
|
|
||||||
const schema = (
|
const schema = (
|
||||||
await googleApiRequest.call(
|
await googleBigQueryApiRequest.call(
|
||||||
this,
|
this,
|
||||||
'GET',
|
'GET',
|
||||||
`/v2/projects/${projectId}/datasets/${datasetId}/tables/${tableId}`,
|
`/v2/projects/${projectId}/datasets/${datasetId}/tables/${tableId}`,
|
||||||
|
@ -230,7 +230,7 @@ export async function execute(this: IExecuteFunctions): Promise<INodeExecutionDa
|
||||||
const batch = rows.slice(i, i + batchSize);
|
const batch = rows.slice(i, i + batchSize);
|
||||||
body.rows = batch;
|
body.rows = batch;
|
||||||
|
|
||||||
const responseData = await googleApiRequest.call(
|
const responseData = await googleBigQueryApiRequest.call(
|
||||||
this,
|
this,
|
||||||
'POST',
|
'POST',
|
||||||
`/v2/projects/${projectId}/datasets/${datasetId}/tables/${tableId}/insertAll`,
|
`/v2/projects/${projectId}/datasets/${datasetId}/tables/${tableId}/insertAll`,
|
||||||
|
|
|
@ -1,5 +1,5 @@
|
||||||
import type { IDataObject, ILoadOptionsFunctions, INodeListSearchResult } from 'n8n-workflow';
|
import type { IDataObject, ILoadOptionsFunctions, INodeListSearchResult } from 'n8n-workflow';
|
||||||
import { googleApiRequest } from '../transport';
|
import { googleBigQueryApiRequest } from '../transport';
|
||||||
|
|
||||||
export async function searchProjects(
|
export async function searchProjects(
|
||||||
this: ILoadOptionsFunctions,
|
this: ILoadOptionsFunctions,
|
||||||
|
@ -10,7 +10,7 @@ export async function searchProjects(
|
||||||
pageToken: (paginationToken as string) || undefined,
|
pageToken: (paginationToken as string) || undefined,
|
||||||
};
|
};
|
||||||
|
|
||||||
const response = await googleApiRequest.call(this, 'GET', '/v2/projects', undefined, qs);
|
const response = await googleBigQueryApiRequest.call(this, 'GET', '/v2/projects', undefined, qs);
|
||||||
|
|
||||||
let { projects } = response;
|
let { projects } = response;
|
||||||
|
|
||||||
|
@ -45,7 +45,7 @@ export async function searchDatasets(
|
||||||
pageToken: (paginationToken as string) || undefined,
|
pageToken: (paginationToken as string) || undefined,
|
||||||
};
|
};
|
||||||
|
|
||||||
const response = await googleApiRequest.call(
|
const response = await googleBigQueryApiRequest.call(
|
||||||
this,
|
this,
|
||||||
'GET',
|
'GET',
|
||||||
`/v2/projects/${projectId}/datasets`,
|
`/v2/projects/${projectId}/datasets`,
|
||||||
|
@ -87,7 +87,7 @@ export async function searchTables(
|
||||||
pageToken: (paginationToken as string) || undefined,
|
pageToken: (paginationToken as string) || undefined,
|
||||||
};
|
};
|
||||||
|
|
||||||
const response = await googleApiRequest.call(
|
const response = await googleBigQueryApiRequest.call(
|
||||||
this,
|
this,
|
||||||
'GET',
|
'GET',
|
||||||
`/v2/projects/${projectId}/datasets/${datasetId}/tables`,
|
`/v2/projects/${projectId}/datasets/${datasetId}/tables`,
|
||||||
|
|
|
@ -1,12 +1,12 @@
|
||||||
import type { IDataObject, ILoadOptionsFunctions, INodePropertyOptions } from 'n8n-workflow';
|
import type { IDataObject, ILoadOptionsFunctions, INodePropertyOptions } from 'n8n-workflow';
|
||||||
import { googleApiRequest } from '../transport';
|
import { googleBigQueryApiRequest } from '../transport';
|
||||||
|
|
||||||
export async function getDatasets(this: ILoadOptionsFunctions): Promise<INodePropertyOptions[]> {
|
export async function getDatasets(this: ILoadOptionsFunctions): Promise<INodePropertyOptions[]> {
|
||||||
const projectId = this.getNodeParameter('projectId', undefined, {
|
const projectId = this.getNodeParameter('projectId', undefined, {
|
||||||
extractValue: true,
|
extractValue: true,
|
||||||
});
|
});
|
||||||
const returnData: INodePropertyOptions[] = [];
|
const returnData: INodePropertyOptions[] = [];
|
||||||
const { datasets } = await googleApiRequest.call(
|
const { datasets } = await googleBigQueryApiRequest.call(
|
||||||
this,
|
this,
|
||||||
'GET',
|
'GET',
|
||||||
`/v2/projects/${projectId}/datasets`,
|
`/v2/projects/${projectId}/datasets`,
|
||||||
|
@ -33,7 +33,7 @@ export async function getSchema(this: ILoadOptionsFunctions): Promise<INodePrope
|
||||||
|
|
||||||
const returnData: INodePropertyOptions[] = [];
|
const returnData: INodePropertyOptions[] = [];
|
||||||
|
|
||||||
const { schema } = await googleApiRequest.call(
|
const { schema } = await googleBigQueryApiRequest.call(
|
||||||
this,
|
this,
|
||||||
'GET',
|
'GET',
|
||||||
`/v2/projects/${projectId}/datasets/${datasetId}/tables/${tableId}`,
|
`/v2/projects/${projectId}/datasets/${datasetId}/tables/${tableId}`,
|
||||||
|
|
|
@ -9,7 +9,7 @@ import type {
|
||||||
import { NodeApiError, NodeOperationError } from 'n8n-workflow';
|
import { NodeApiError, NodeOperationError } from 'n8n-workflow';
|
||||||
import { getGoogleAccessToken } from '../../../GenericFunctions';
|
import { getGoogleAccessToken } from '../../../GenericFunctions';
|
||||||
|
|
||||||
export async function googleApiRequest(
|
export async function googleBigQueryApiRequest(
|
||||||
this: IExecuteFunctions | ILoadOptionsFunctions,
|
this: IExecuteFunctions | ILoadOptionsFunctions,
|
||||||
method: IHttpRequestMethods,
|
method: IHttpRequestMethods,
|
||||||
resource: string,
|
resource: string,
|
||||||
|
@ -67,25 +67,26 @@ export async function googleApiRequest(
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function googleApiRequestAllItems(
|
export async function googleBigQueryApiRequestAllItems(
|
||||||
this: IExecuteFunctions | ILoadOptionsFunctions,
|
this: IExecuteFunctions | ILoadOptionsFunctions,
|
||||||
propertyName: string,
|
|
||||||
method: IHttpRequestMethods,
|
method: IHttpRequestMethods,
|
||||||
endpoint: string,
|
endpoint: string,
|
||||||
body: IDataObject = {},
|
body: IDataObject = {},
|
||||||
query: IDataObject = {},
|
query: IDataObject = {},
|
||||||
) {
|
) {
|
||||||
const returnData: IDataObject[] = [];
|
let rows: IDataObject[] = [];
|
||||||
|
|
||||||
let responseData;
|
let responseData;
|
||||||
query.maxResults = 10000;
|
if (query.maxResults === undefined) {
|
||||||
|
query.maxResults = 1000;
|
||||||
|
}
|
||||||
|
|
||||||
do {
|
do {
|
||||||
responseData = await googleApiRequest.call(this, method, endpoint, body, query);
|
responseData = await googleBigQueryApiRequest.call(this, method, endpoint, body, query);
|
||||||
|
|
||||||
query.pageToken = responseData.pageToken;
|
query.pageToken = responseData.pageToken;
|
||||||
returnData.push.apply(returnData, responseData[propertyName] as IDataObject[]);
|
rows = rows.concat((responseData.rows as IDataObject[]) ?? []);
|
||||||
} while (responseData.pageToken !== undefined && responseData.pageToken !== '');
|
} while (responseData.pageToken !== undefined && responseData.pageToken !== '');
|
||||||
|
|
||||||
return returnData;
|
return { ...(responseData || {}), rows };
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in a new issue