feat(core): Simplify pagination in declarative node design (#5161)

* feat(core): Add pagination to declarative node design

*  Actually make it work

*  Remove rootProperty

*  Fix typo

*  Add support to overwrite url

---------

Co-authored-by: Omar Ajoue <krynble@gmail.com>
This commit is contained in:
Jan Oberhauser 2023-02-01 08:26:13 -06:00 committed by GitHub
parent b27a60b665
commit 87ceb6f4b8
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 117 additions and 26 deletions

View file

@ -185,6 +185,7 @@ export interface IRequestOptionsSimplifiedAuth {
body?: IDataObject; body?: IDataObject;
headers?: IDataObject; headers?: IDataObject;
qs?: IDataObject; qs?: IDataObject;
url?: string;
skipSslCertificateValidation?: boolean | string; skipSslCertificateValidation?: boolean | string;
} }
@ -529,6 +530,7 @@ export interface IN8nHttpFullResponse {
export interface IN8nRequestOperations { export interface IN8nRequestOperations {
pagination?: pagination?:
| IN8nRequestOperationPaginationGeneric
| IN8nRequestOperationPaginationOffset | IN8nRequestOperationPaginationOffset
| (( | ((
this: IExecutePaginationFunctions, this: IExecutePaginationFunctions,
@ -539,7 +541,15 @@ export interface IN8nRequestOperations {
export interface IN8nRequestOperationPaginationBase { export interface IN8nRequestOperationPaginationBase {
type: string; type: string;
properties: { properties: {
[key: string]: string | number; [key: string]: unknown;
};
}
export interface IN8nRequestOperationPaginationGeneric extends IN8nRequestOperationPaginationBase {
type: 'generic';
properties: {
continue: boolean | string;
request: IRequestOptionsSimplifiedAuth;
}; };
} }

View file

@ -422,28 +422,13 @@ export class RoutingNode {
return []; return [];
} }
async rawRoutingRequest( async postProcessResponseData(
executeSingleFunctions: IExecuteSingleFunctions, executeSingleFunctions: IExecuteSingleFunctions,
responseData: IN8nHttpFullResponse,
requestData: DeclarativeRestApiSettings.ResultOptions, requestData: DeclarativeRestApiSettings.ResultOptions,
itemIndex: number, itemIndex: number,
runIndex: number, runIndex: number,
credentialType?: string,
credentialsDecrypted?: ICredentialsDecrypted,
): Promise<INodeExecutionData[]> { ): Promise<INodeExecutionData[]> {
let responseData: IN8nHttpFullResponse;
requestData.options.returnFullResponse = true;
if (credentialType) {
responseData = (await executeSingleFunctions.helpers.httpRequestWithAuthentication.call(
executeSingleFunctions,
credentialType,
requestData.options as IHttpRequestOptions,
{ credentialsDecrypted },
)) as IN8nHttpFullResponse;
} else {
responseData = (await executeSingleFunctions.helpers.httpRequest(
requestData.options as IHttpRequestOptions,
)) as IN8nHttpFullResponse;
}
let returnData: INodeExecutionData[] = [ let returnData: INodeExecutionData[] = [
{ {
json: responseData.body as IDataObject, json: responseData.body as IDataObject,
@ -482,6 +467,30 @@ export class RoutingNode {
return returnData; return returnData;
} }
async rawRoutingRequest(
executeSingleFunctions: IExecuteSingleFunctions,
requestData: DeclarativeRestApiSettings.ResultOptions,
credentialType?: string,
credentialsDecrypted?: ICredentialsDecrypted,
): Promise<IN8nHttpFullResponse> {
let responseData: IN8nHttpFullResponse;
requestData.options.returnFullResponse = true;
if (credentialType) {
responseData = (await executeSingleFunctions.helpers.httpRequestWithAuthentication.call(
executeSingleFunctions,
credentialType,
requestData.options as IHttpRequestOptions,
{ credentialsDecrypted },
)) as IN8nHttpFullResponse;
} else {
responseData = (await executeSingleFunctions.helpers.httpRequest(
requestData.options as IHttpRequestOptions,
)) as IN8nHttpFullResponse;
}
return responseData;
}
async makeRoutingRequest( async makeRoutingRequest(
requestData: DeclarativeRestApiSettings.ResultOptions, requestData: DeclarativeRestApiSettings.ResultOptions,
executeSingleFunctions: IExecuteSingleFunctions, executeSingleFunctions: IExecuteSingleFunctions,
@ -505,10 +514,16 @@ export class RoutingNode {
return this.rawRoutingRequest( return this.rawRoutingRequest(
executeSingleFunctions, executeSingleFunctions,
requestOptions, requestOptions,
itemIndex,
runIndex,
credentialType, credentialType,
credentialsDecrypted, credentialsDecrypted,
).then(async (data) =>
this.postProcessResponseData(
executeSingleFunctions,
data,
requestData,
itemIndex,
runIndex,
),
); );
}, },
}; };
@ -524,14 +539,68 @@ export class RoutingNode {
); );
} else { } else {
// Pagination via JSON properties // Pagination via JSON properties
const { properties } = requestOperations.pagination;
responseData = []; responseData = [];
if (!requestData.options.qs) { if (!requestData.options.qs) {
requestData.options.qs = {}; requestData.options.qs = {};
} }
// Different predefined pagination types // Different predefined pagination types
if (requestOperations.pagination.type === 'offset') { if (requestOperations.pagination.type === 'generic') {
let tempResponseData: IN8nHttpFullResponse;
let tempResponseItems: INodeExecutionData[];
let makeAdditionalRequest: boolean;
let paginateRequestData: IHttpRequestOptions;
const additionalKeys = {
$request: requestData.options,
$response: {} as IN8nHttpFullResponse,
$version: this.node.typeVersion,
};
do {
additionalKeys.$request = requestData.options;
paginateRequestData = this.getParameterValue(
requestOperations.pagination.properties.request as unknown as NodeParameterValueType,
itemIndex,
runIndex,
executeSingleFunctions.getExecuteData(),
additionalKeys,
false,
) as object as IHttpRequestOptions;
// Make the HTTP request
tempResponseData = await this.rawRoutingRequest(
executeSingleFunctions,
{ ...requestData, options: { ...requestData.options, ...paginateRequestData } },
credentialType,
credentialsDecrypted,
);
additionalKeys.$response = tempResponseData;
tempResponseItems = await this.postProcessResponseData(
executeSingleFunctions,
tempResponseData,
requestData,
itemIndex,
runIndex,
);
responseData.push(...tempResponseItems);
makeAdditionalRequest = this.getParameterValue(
requestOperations.pagination.properties.continue,
itemIndex,
runIndex,
executeSingleFunctions.getExecuteData(),
additionalKeys,
false,
) as boolean;
} while (makeAdditionalRequest);
} else if (requestOperations.pagination.type === 'offset') {
const { properties } = requestOperations.pagination;
const optionsType = properties.type === 'body' ? 'body' : 'qs'; const optionsType = properties.type === 'body' ? 'body' : 'qs';
if (properties.type === 'body' && !requestData.options.body) { if (properties.type === 'body' && !requestData.options.body) {
requestData.options.body = {}; requestData.options.body = {};
@ -555,10 +624,16 @@ export class RoutingNode {
tempResponseData = await this.rawRoutingRequest( tempResponseData = await this.rawRoutingRequest(
executeSingleFunctions, executeSingleFunctions,
requestData, requestData,
itemIndex,
runIndex,
credentialType, credentialType,
credentialsDecrypted, credentialsDecrypted,
).then(async (data) =>
this.postProcessResponseData(
executeSingleFunctions,
data,
requestData,
itemIndex,
runIndex,
),
); );
(requestData.options[optionsType] as IDataObject)[properties.offsetParameter] = (requestData.options[optionsType] as IDataObject)[properties.offsetParameter] =
@ -594,10 +669,16 @@ export class RoutingNode {
responseData = await this.rawRoutingRequest( responseData = await this.rawRoutingRequest(
executeSingleFunctions, executeSingleFunctions,
requestData, requestData,
itemIndex,
runIndex,
credentialType, credentialType,
credentialsDecrypted, credentialsDecrypted,
).then(async (data) =>
this.postProcessResponseData(
executeSingleFunctions,
data,
requestData,
itemIndex,
runIndex,
),
); );
} }
return responseData; return responseData;