From 87ceb6f4b8ed1838b874639f176b421e0292b576 Mon Sep 17 00:00:00 2001 From: Jan Oberhauser Date: Wed, 1 Feb 2023 08:26:13 -0600 Subject: [PATCH] feat(core): Simplify pagination in declarative node design (#5161) * feat(core): Add pagination to declarative node design * :zap: Actually make it work * :zap: Remove rootProperty * :zap: Fix typo * :zap: Add support to overwrite url --------- Co-authored-by: Omar Ajoue --- packages/workflow/src/Interfaces.ts | 12 ++- packages/workflow/src/RoutingNode.ts | 131 ++++++++++++++++++++++----- 2 files changed, 117 insertions(+), 26 deletions(-) diff --git a/packages/workflow/src/Interfaces.ts b/packages/workflow/src/Interfaces.ts index eb74598da8..ed7346d8a2 100644 --- a/packages/workflow/src/Interfaces.ts +++ b/packages/workflow/src/Interfaces.ts @@ -185,6 +185,7 @@ export interface IRequestOptionsSimplifiedAuth { body?: IDataObject; headers?: IDataObject; qs?: IDataObject; + url?: string; skipSslCertificateValidation?: boolean | string; } @@ -529,6 +530,7 @@ export interface IN8nHttpFullResponse { export interface IN8nRequestOperations { pagination?: + | IN8nRequestOperationPaginationGeneric | IN8nRequestOperationPaginationOffset | (( this: IExecutePaginationFunctions, @@ -539,7 +541,15 @@ export interface IN8nRequestOperations { export interface IN8nRequestOperationPaginationBase { type: string; properties: { - [key: string]: string | number; + [key: string]: unknown; + }; +} + +export interface IN8nRequestOperationPaginationGeneric extends IN8nRequestOperationPaginationBase { + type: 'generic'; + properties: { + continue: boolean | string; + request: IRequestOptionsSimplifiedAuth; }; } diff --git a/packages/workflow/src/RoutingNode.ts b/packages/workflow/src/RoutingNode.ts index cef5391e6b..727fe5efca 100644 --- a/packages/workflow/src/RoutingNode.ts +++ b/packages/workflow/src/RoutingNode.ts @@ -422,28 +422,13 @@ export class RoutingNode { return []; } - async rawRoutingRequest( + async postProcessResponseData( executeSingleFunctions: IExecuteSingleFunctions, + responseData: IN8nHttpFullResponse, requestData: DeclarativeRestApiSettings.ResultOptions, itemIndex: number, runIndex: number, - credentialType?: string, - credentialsDecrypted?: ICredentialsDecrypted, ): Promise { - let responseData: IN8nHttpFullResponse; - requestData.options.returnFullResponse = true; - if (credentialType) { - responseData = (await executeSingleFunctions.helpers.httpRequestWithAuthentication.call( - executeSingleFunctions, - credentialType, - requestData.options as IHttpRequestOptions, - { credentialsDecrypted }, - )) as IN8nHttpFullResponse; - } else { - responseData = (await executeSingleFunctions.helpers.httpRequest( - requestData.options as IHttpRequestOptions, - )) as IN8nHttpFullResponse; - } let returnData: INodeExecutionData[] = [ { json: responseData.body as IDataObject, @@ -482,6 +467,30 @@ export class RoutingNode { return returnData; } + async rawRoutingRequest( + executeSingleFunctions: IExecuteSingleFunctions, + requestData: DeclarativeRestApiSettings.ResultOptions, + credentialType?: string, + credentialsDecrypted?: ICredentialsDecrypted, + ): Promise { + let responseData: IN8nHttpFullResponse; + requestData.options.returnFullResponse = true; + if (credentialType) { + responseData = (await executeSingleFunctions.helpers.httpRequestWithAuthentication.call( + executeSingleFunctions, + credentialType, + requestData.options as IHttpRequestOptions, + { credentialsDecrypted }, + )) as IN8nHttpFullResponse; + } else { + responseData = (await executeSingleFunctions.helpers.httpRequest( + requestData.options as IHttpRequestOptions, + )) as IN8nHttpFullResponse; + } + + return responseData; + } + async makeRoutingRequest( requestData: DeclarativeRestApiSettings.ResultOptions, executeSingleFunctions: IExecuteSingleFunctions, @@ -505,10 +514,16 @@ export class RoutingNode { return this.rawRoutingRequest( executeSingleFunctions, requestOptions, - itemIndex, - runIndex, credentialType, credentialsDecrypted, + ).then(async (data) => + this.postProcessResponseData( + executeSingleFunctions, + data, + requestData, + itemIndex, + runIndex, + ), ); }, }; @@ -524,14 +539,68 @@ export class RoutingNode { ); } else { // Pagination via JSON properties - const { properties } = requestOperations.pagination; responseData = []; if (!requestData.options.qs) { requestData.options.qs = {}; } // Different predefined pagination types - if (requestOperations.pagination.type === 'offset') { + if (requestOperations.pagination.type === 'generic') { + let tempResponseData: IN8nHttpFullResponse; + let tempResponseItems: INodeExecutionData[]; + let makeAdditionalRequest: boolean; + let paginateRequestData: IHttpRequestOptions; + + const additionalKeys = { + $request: requestData.options, + $response: {} as IN8nHttpFullResponse, + $version: this.node.typeVersion, + }; + + do { + additionalKeys.$request = requestData.options; + + paginateRequestData = this.getParameterValue( + requestOperations.pagination.properties.request as unknown as NodeParameterValueType, + itemIndex, + runIndex, + executeSingleFunctions.getExecuteData(), + additionalKeys, + false, + ) as object as IHttpRequestOptions; + + // Make the HTTP request + tempResponseData = await this.rawRoutingRequest( + executeSingleFunctions, + { ...requestData, options: { ...requestData.options, ...paginateRequestData } }, + credentialType, + credentialsDecrypted, + ); + + additionalKeys.$response = tempResponseData; + + tempResponseItems = await this.postProcessResponseData( + executeSingleFunctions, + tempResponseData, + requestData, + itemIndex, + runIndex, + ); + + responseData.push(...tempResponseItems); + + makeAdditionalRequest = this.getParameterValue( + requestOperations.pagination.properties.continue, + itemIndex, + runIndex, + executeSingleFunctions.getExecuteData(), + additionalKeys, + false, + ) as boolean; + } while (makeAdditionalRequest); + } else if (requestOperations.pagination.type === 'offset') { + const { properties } = requestOperations.pagination; + const optionsType = properties.type === 'body' ? 'body' : 'qs'; if (properties.type === 'body' && !requestData.options.body) { requestData.options.body = {}; @@ -555,10 +624,16 @@ export class RoutingNode { tempResponseData = await this.rawRoutingRequest( executeSingleFunctions, requestData, - itemIndex, - runIndex, credentialType, credentialsDecrypted, + ).then(async (data) => + this.postProcessResponseData( + executeSingleFunctions, + data, + requestData, + itemIndex, + runIndex, + ), ); (requestData.options[optionsType] as IDataObject)[properties.offsetParameter] = @@ -594,10 +669,16 @@ export class RoutingNode { responseData = await this.rawRoutingRequest( executeSingleFunctions, requestData, - itemIndex, - runIndex, credentialType, credentialsDecrypted, + ).then(async (data) => + this.postProcessResponseData( + executeSingleFunctions, + data, + requestData, + itemIndex, + runIndex, + ), ); } return responseData;