import type { IDataObject, IExecuteFunctions, ILoadOptionsFunctions, IHookFunctions, IHttpRequestOptions, INodeExecutionData, INodePropertyOptions, IWebhookFunctions, } from 'n8n-workflow'; import set from 'lodash/set'; import concat from 'lodash/concat'; import split from 'lodash/split'; import every from 'lodash/every'; import toString from 'lodash/toString'; import toNumber from 'lodash/toNumber'; import isString from 'lodash/isString'; import compact from 'lodash/compact'; import first from 'lodash/first'; import last from 'lodash/last'; import clone from 'lodash/clone'; import some from 'lodash/some'; import isArray from 'lodash/isArray'; import trim from 'lodash/trim'; import escapeRegExp from 'lodash/escapeRegExp'; export async function koBoToolboxApiRequest( this: IExecuteFunctions | IWebhookFunctions | IHookFunctions | ILoadOptionsFunctions, option: IDataObject = {}, ): Promise { const credentials = await this.getCredentials('koBoToolboxApi'); // Set up pagination / scrolling const returnAll = !!option.returnAll; if (returnAll) { // Override manual pagination options set(option, 'qs.limit', 3000); // Don't pass this custom param to helpers.httpRequest delete option.returnAll; } const options: IHttpRequestOptions = { url: '', headers: { Accept: 'application/json', }, json: true, }; if (Object.keys(option)) { Object.assign(options, option); } if (options.url && !/^http(s)?:/.test(options.url)) { options.url = (credentials.URL as string) + options.url; } let results = null; let keepLooking = true; while (keepLooking) { const response = await this.helpers.httpRequestWithAuthentication.call( this, 'koBoToolboxApi', options, ); // Append or set results results = response.results ? concat(results || [], response.results) : response; if (returnAll && response.next) { options.url = response.next; } else { keepLooking = false; } } return results; } export async function koBoToolboxRawRequest( this: IExecuteFunctions | IWebhookFunctions | IHookFunctions | ILoadOptionsFunctions, option: IHttpRequestOptions, ): Promise { const credentials = await this.getCredentials('koBoToolboxApi'); if (option.url && !/^http(s)?:/.test(option.url)) { option.url = (credentials.URL as string) + option.url; } return this.helpers.httpRequestWithAuthentication.call(this, 'koBoToolboxApi', option); } function parseGeoPoint(geoPoint: string): null | number[] { // Check if it looks like a "lat lon z precision" flat string e.g. "-1.931161 30.079811 0 0" (lat, lon, elevation, precision) // NOTE: we are discarding the elevation and precision values since they're not (well) supported in GeoJSON const coordinates = split(geoPoint, ' '); if ( coordinates.length >= 2 && every(coordinates, (coord) => coord && /^-?\d+(?:\.\d+)?$/.test(toString(coord))) ) { // NOTE: GeoJSON uses lon, lat, while most common systems use lat, lon order! return [toNumber(coordinates[1]), toNumber(coordinates[0])]; } return null; } export function parseStringList(value: string): string[] { return split(toString(value), /[\s,]+/); } const matchWildcard = (value: string, pattern: string): boolean => { const regex = new RegExp(`^${escapeRegExp(pattern).replace('\\*', '.*')}$`); return regex.test(value); }; const formatValue = (value: any, format: string): any => { if (isString(value)) { // Sanitize value value = toString(value); // Parse geoPoints const geoPoint = parseGeoPoint(value as string); if (geoPoint) { return { type: 'Point', coordinates: geoPoint, }; } // Check if it's a closed polygon geo-shape: -1.954117 30.085159 0 0;-1.955005 30.084622 0 0;-1.956057 30.08506 0 0;-1.956393 30.086229 0 0;-1.955853 30.087143 0 0;-1.954609 30.08725 0 0;-1.953966 30.086735 0 0;-1.953805 30.085897 0 0;-1.954117 30.085159 0 0 const points = value.split(';'); if (points.length >= 2 && /^[-\d\.\s;]+$/.test(value as string)) { // Using the GeoJSON format as per https://geojson.org/ const coordinates = compact(points.map(parseGeoPoint) as number[]); // Only return if all values are properly parsed if (coordinates.length === points.length) { // If the shape is closed, declare it as Polygon, otherwise as LineString if (first(points) === last(points)) { return { type: 'Polygon', coordinates: [coordinates], }; } return { type: 'LineString', coordinates }; } } // Parse numbers if ('number' === format) { return toNumber(value); } // Split multi-select if ('multiSelect' === format) { return split(toString(value), ' '); } } return value; }; export function formatSubmission( submission: IDataObject, selectMasks: string[] = [], numberMasks: string[] = [], ): IDataObject { // Create a shallow copy of the submission const response = {} as IDataObject; for (const key of Object.keys(submission)) { let value = clone(submission[key]); // Sanitize key names: split by group, trim _ const sanitizedKey = key .split('/') .map((k) => trim(k, ' _')) .join('.'); const leafKey = sanitizedKey.split('.').pop() || ''; let format = 'string'; if (some(numberMasks, (mask) => matchWildcard(leafKey, mask))) { format = 'number'; } if (some(selectMasks, (mask) => matchWildcard(leafKey, mask))) { format = 'multiSelect'; } value = formatValue(value, format); set(response, sanitizedKey, value); } // Reformat _geolocation if ( isArray(response.geolocation) && response.geolocation.length === 2 && response.geolocation[0] && response.geolocation[1] ) { response.geolocation = { type: 'Point', coordinates: [response.geolocation[1], response.geolocation[0]], }; } return response; } export async function downloadAttachments( this: IExecuteFunctions | IWebhookFunctions, submission: IDataObject, options: IDataObject, ): Promise { // Initialize return object with the original submission JSON content const binaryItem: INodeExecutionData = { json: { ...submission, }, binary: {}, }; const credentials = await this.getCredentials('koBoToolboxApi'); // Look for attachment links - there can be more than one const attachmentList = (submission._attachments || submission.attachments) as any[]; if (attachmentList?.length) { for (const [index, attachment] of attachmentList.entries()) { // look for the question name linked to this attachment const fileName = attachment.filename; const sanitizedFileName = toString(fileName).replace(/_[^_]+(?=\.\w+)/, ''); // strip suffix let relatedQuestion = null; if ('question' === options.binaryNamingScheme) { for (const question of Object.keys(submission)) { // The logic to map attachments to question is sometimes ambiguous: // - If the attachment is linked to a question, the question's value is the same as the attachment's filename (with spaces replaced by underscores) // - BUT sometimes the attachment's filename has an extra suffix, e.g. "My_Picture_0OdlaKJ.jpg", would map to the question "picture": "My Picture.jpg" const sanitizedQuestionValue = toString(submission[question]).replace(/\s/g, '_'); // replace spaces with underscores if (sanitizedFileName === sanitizedQuestionValue) { relatedQuestion = question; // Just use the first match... break; } } } // Download attachment // NOTE: this needs to follow redirects (possibly across domains), while keeping Authorization headers // The Axios client will not propagate the Authorization header on redirects (see https://github.com/axios/axios/issues/3607), so we need to follow ourselves... let response = null; const attachmentUrl = attachment[options.version as string] || (attachment.download_url as string); let final = false, redir = 0; const axiosOptions: IHttpRequestOptions = { url: attachmentUrl, method: 'GET', headers: { Authorization: `Token ${credentials.token}`, }, ignoreHttpStatusErrors: true, returnFullResponse: true, disableFollowRedirect: true, encoding: 'arraybuffer', }; while (!final && redir < 5) { response = await this.helpers.httpRequest(axiosOptions); if (response?.headers.location) { // Follow redirect axiosOptions.url = response.headers.location; redir++; } else { final = true; } } if (response?.body) { // Use the provided prefix if any, otherwise try to use the original question name let binaryName; if ('question' === options.binaryNamingScheme && relatedQuestion) { binaryName = relatedQuestion; } else { binaryName = `${options.dataPropertyAttachmentsPrefixName || 'attachment_'}${index}`; } binaryItem.binary![binaryName] = await this.helpers.prepareBinaryData( response.body as Buffer, fileName as string, ); } } } else { delete binaryItem.binary; } // Add item to final output - even if there's no attachment retrieved return binaryItem; } export async function loadForms(this: ILoadOptionsFunctions): Promise { const responseData = await koBoToolboxApiRequest.call(this, { url: '/api/v2/assets/', qs: { q: 'asset_type:survey', ordering: 'name', }, scroll: true, }); return responseData?.map((survey: any) => ({ name: survey.name, value: survey.uid })) || []; }