From 3b3405089d9194515192f10ab4085e1493bb86b5 Mon Sep 17 00:00:00 2001 From: Ugo Bataillard Date: Wed, 9 Nov 2022 19:26:13 +0900 Subject: [PATCH] fix(Notion Trigger (Beta) Node): Fix Notion trigger polling strategy --- .../credentials/NotionApi.credentials.ts | 33 ++++++--- .../nodes/Notion/BlockDescription.ts | 4 +- .../nodes/Notion/DatabaseDescription.ts | 13 ++++ .../nodes/Notion/DatabasePageDescription.ts | 2 +- .../nodes/Notion/GenericFunctions.ts | 9 +-- .../nodes/Notion/NotionTrigger.node.ts | 68 ++++++++++++++----- .../nodes/Notion/PageDescription.ts | 4 +- 7 files changed, 97 insertions(+), 36 deletions(-) diff --git a/packages/nodes-base/credentials/NotionApi.credentials.ts b/packages/nodes-base/credentials/NotionApi.credentials.ts index f1d4336309..97d0e5016c 100644 --- a/packages/nodes-base/credentials/NotionApi.credentials.ts +++ b/packages/nodes-base/credentials/NotionApi.credentials.ts @@ -1,7 +1,10 @@ +import { request } from 'http'; import { IAuthenticateGeneric, + ICredentialDataDecryptedObject, ICredentialTestRequest, ICredentialType, + IHttpRequestOptions, INodeProperties, } from 'n8n-workflow'; @@ -24,13 +27,25 @@ export class NotionApi implements ICredentialType { url: '/users', }, }; - authenticate: IAuthenticateGeneric = { - type: 'generic', - properties: { - headers: { - Authorization: '=Bearer {{$credentials.apiKey}}', - 'Notion-Version': '2021-05-13', - }, - }, - }; + async authenticate( + credentials: ICredentialDataDecryptedObject, + requestOptions: IHttpRequestOptions, + ): Promise { + requestOptions.headers = { + ...requestOptions.headers, + Authorization: `Bearer ${credentials.apiKey} `, + }; + + // if version it's not set, set it to last one + // version is only set when the request is made from + // the notion node, or was set explicitly in the http node + if (!requestOptions.headers['Notion-Version']) { + requestOptions.headers = { + ...requestOptions.headers, + 'Notion-Version': '2022-02-22', + }; + } + + return requestOptions; + } } diff --git a/packages/nodes-base/nodes/Notion/BlockDescription.ts b/packages/nodes-base/nodes/Notion/BlockDescription.ts index b76234f135..932676b9c8 100644 --- a/packages/nodes-base/nodes/Notion/BlockDescription.ts +++ b/packages/nodes-base/nodes/Notion/BlockDescription.ts @@ -32,7 +32,7 @@ export const blockOperations: INodeProperties[] = [ }, ]; -export const blockFields = [ +export const blockFields: INodeProperties[] = [ /* -------------------------------------------------------------------------- */ /* block:append */ /* -------------------------------------------------------------------------- */ @@ -101,4 +101,4 @@ export const blockFields = [ default: 50, description: 'Max number of results to return', }, -] as INodeProperties[]; +]; diff --git a/packages/nodes-base/nodes/Notion/DatabaseDescription.ts b/packages/nodes-base/nodes/Notion/DatabaseDescription.ts index a773bf78d7..8d0be72bd5 100644 --- a/packages/nodes-base/nodes/Notion/DatabaseDescription.ts +++ b/packages/nodes-base/nodes/Notion/DatabaseDescription.ts @@ -85,6 +85,19 @@ export const databaseFields: INodeProperties[] = [ /* -------------------------------------------------------------------------- */ /* database:getAll */ /* -------------------------------------------------------------------------- */ + { + displayName: + 'In Notion, make sure you share your database with your integration . Otherwise it won\'t be accessible, or listed here.', + name: 'notionNotice', + type: 'notice', + default: '', + displayOptions: { + show: { + resource: ['database'], + operation: ['getAll'], + }, + }, + }, { displayName: 'Return All', name: 'returnAll', diff --git a/packages/nodes-base/nodes/Notion/DatabasePageDescription.ts b/packages/nodes-base/nodes/Notion/DatabasePageDescription.ts index 716acb8f48..960f4c53f5 100644 --- a/packages/nodes-base/nodes/Notion/DatabasePageDescription.ts +++ b/packages/nodes-base/nodes/Notion/DatabasePageDescription.ts @@ -81,7 +81,7 @@ export const databasePageOperations: INodeProperties[] = [ }, ]; -export const databasePageFields = [ +export const databasePageFields: INodeProperties[] = [ /* -------------------------------------------------------------------------- */ /* databasePage:create */ /* -------------------------------------------------------------------------- */ diff --git a/packages/nodes-base/nodes/Notion/GenericFunctions.ts b/packages/nodes-base/nodes/Notion/GenericFunctions.ts index 74c7af01e8..9d39594bb9 100644 --- a/packages/nodes-base/nodes/Notion/GenericFunctions.ts +++ b/packages/nodes-base/nodes/Notion/GenericFunctions.ts @@ -219,12 +219,11 @@ function getTexts( annotations: text.annotationUi, }); } else { - //@ts-ignore results.push({ type: 'mention', mention: { type: text.mentionType, - //@ts-ignore + //@ts-expect-error any [text.mentionType]: { id: text[text.mentionType] as string }, }, annotations: text.annotationUi, @@ -252,9 +251,8 @@ export function formatBlocks(blocks: IDataObject[]) { [block.type as string]: { ...(block.type === 'to_do' ? { checked: block.checked } : {}), // prettier-ignore - //@ts-expect-error // tslint:disable-next-line: no-any - text: (block.richText === false) ? formatText(block.textContent).text : getTexts(block.text.text as any || []), + text: (block.richText === false) ? formatText(block.textContent as string).text : getTexts((block.text as IDataObject).text as any || []), }, }); } @@ -266,7 +264,7 @@ function getPropertyKeyValue(value: any, type: string, timezone: string, version // tslint:disable-next-line: no-any const ignoreIfEmpty = (v: T, cb: (v: T) => any) => !v && value.ignoreIfEmpty ? undefined : cb(v); - let result = {}; + let result: IDataObject = {}; switch (type) { case 'rich_text': @@ -362,7 +360,6 @@ function getPropertyKeyValue(value: any, type: string, timezone: string, version //if the date was left empty, set it to null so it resets the value in notion if (value.date === '' || (value.dateStart === '' && value.dateEnd === '')) { - //@ts-ignore result.date = null; } diff --git a/packages/nodes-base/nodes/Notion/NotionTrigger.node.ts b/packages/nodes-base/nodes/Notion/NotionTrigger.node.ts index 885715152f..779b0d67b0 100644 --- a/packages/nodes-base/nodes/Notion/NotionTrigger.node.ts +++ b/packages/nodes-base/nodes/Notion/NotionTrigger.node.ts @@ -51,11 +51,11 @@ export class NotionTrigger implements INodeType { }, ], required: true, - default: '', + default: 'pageAddedToDatabase', }, { displayName: - "In Notion, make sure you share your database with your integration. Otherwise it won't be accessible, or listed here.", + 'In Notion, make sure you share your database with your integration . Otherwise it won\'t be accessible, or listed here.', name: 'notionNotice', type: 'notice', default: '', @@ -127,16 +127,24 @@ export class NotionTrigger implements INodeType { const event = this.getNodeParameter('event') as string; const simple = this.getNodeParameter('simple') as boolean; - const now = moment().utc().format(); + const lastTimeChecked = webhookData.lastTimeChecked + ? moment(webhookData.lastTimeChecked as string) + : moment().set({ second: 0, millisecond: 0 }); // Notion timestamp accuracy is only down to the minute - const startDate = (webhookData.lastTimeChecked as string) || now; + // update lastTimeChecked to now + webhookData.lastTimeChecked = moment().set({ second: 0, millisecond: 0 }); - const endDate = now; - - webhookData.lastTimeChecked = endDate; + // because Notion timestamp accuracy is only down to the minute some duplicates can be fetch + const possibleDuplicates = (webhookData.possibleDuplicates as string[]) ?? []; const sortProperty = event === 'pageAddedToDatabase' ? 'created_time' : 'last_edited_time'; + const option: IDataObject = { + headers: { + 'Notion-Version': '2022-02-22', + }, + }; + const body: IDataObject = { page_size: 1, sorts: [ @@ -145,6 +153,14 @@ export class NotionTrigger implements INodeType { direction: 'descending', }, ], + ...(this.getMode() !== 'manual' && { + filter: { + timestamp: sortProperty, + [sortProperty]: { + on_or_after: lastTimeChecked.utc().format(), + }, + }, + }), }; let records: IDataObject[] = []; @@ -157,6 +173,9 @@ export class NotionTrigger implements INodeType { 'POST', `/databases/${databaseId}/query`, body, + {}, + '', + option, ); if (this.getMode() === 'manual') { @@ -169,7 +188,7 @@ export class NotionTrigger implements INodeType { } // if something changed after the last check - if (Object.keys(data[0]).length !== 0 && webhookData.lastRecordProccesed !== data[0].id) { + if (Array.isArray(data) && data.length && Object.keys(data[0]).length !== 0) { do { body.page_size = 10; const { results, has_more, next_cursor } = await notionApiRequest.call( @@ -177,29 +196,46 @@ export class NotionTrigger implements INodeType { 'POST', `/databases/${databaseId}/query`, body, + {}, + '', + option, ); - records.push.apply(records, results); + records.push(...results); hasMore = has_more; if (next_cursor !== null) { body['start_cursor'] = next_cursor; } + // Only stop when we reach records strictly before last recorded time to be sure we catch records from the same minute } while ( - !moment(records[records.length - 1][sortProperty] as string).isSameOrBefore(startDate) && + !moment(records[records.length - 1][sortProperty] as string).isBefore(lastTimeChecked) && hasMore === true ); - if (this.getMode() !== 'manual') { - records = records.filter((record: IDataObject) => - moment(record[sortProperty] as string).isBetween(startDate, endDate), - ); + // Filter out already processed left over records: + // with a time strictly before the last record processed + // or from the same minute not present in the list of processed records + records = records.filter( + (record: IDataObject) => !possibleDuplicates.includes(record.id as string), + ); + + // Save the time of the most recent record processed + if (records[0]) { + const latestTimestamp = moment(records[0][sortProperty] as string); + + // Save record ids with the same timestamp as the latest processed records + webhookData.possibleDuplicates = records + .filter((record: IDataObject) => + moment(record[sortProperty] as string).isSame(latestTimestamp), + ) + .map((record: IDataObject) => record.id); + } else { + webhookData.possibleDuplicates = undefined; } if (simple === true) { records = simplifyObjects(records, false, 1); } - webhookData.lastRecordProccesed = data[0].id; - if (Array.isArray(records) && records.length) { return [this.helpers.returnJsonArray(records)]; } diff --git a/packages/nodes-base/nodes/Notion/PageDescription.ts b/packages/nodes-base/nodes/Notion/PageDescription.ts index 064c7bb37d..d9d5ea562f 100644 --- a/packages/nodes-base/nodes/Notion/PageDescription.ts +++ b/packages/nodes-base/nodes/Notion/PageDescription.ts @@ -71,7 +71,7 @@ export const pageOperations: INodeProperties[] = [ }, ]; -export const pageFields = [ +export const pageFields: INodeProperties[] = [ /* -------------------------------------------------------------------------- */ /* page:archive */ /* -------------------------------------------------------------------------- */ @@ -355,4 +355,4 @@ export const pageFields = [ }, ], }, -] as INodeProperties[]; +];