mirror of
https://github.com/n8n-io/n8n.git
synced 2024-12-28 22:19:41 -08:00
04cfa548af
* ⚡ scaffolding * ⚡ finished scaffolding * ⚡ renamed types * ⚡ updated subtitle * ⚡ renamed functions file, UI updates * ⚡ query parameters fixes, ui updates, refactoring * ⚡ fixes for credentials test, setup for error parsing * ⚡ rlc for schema and table, error handling tweaks * ⚡ delete operation, new options * ⚡ columns loader * ⚡ linter fixes * ⚡ where clauses setup * ⚡ logic for processing where clauses * ⚡ select operation * ⚡ refactoring * ⚡ data mode for insert and update, wip * ⚡ data mapping, insert update, skip on conflict option * ⚡ select columns with spaces fix * ⚡ update operation update, wip * ⚡ finished update operation * ⚡ upsert operation * ⚡ ui fixes * Copy updates. * Copy updates. * ⚡ option to convert empty strings to nulls, schema checks * ⚡ UI requested updates * ⚡ ssh setup WIP * ⚡ fixes, ssh WIP * ⚡ ssh fixes, credentials * ⚡ credentials testing update * ⚡ uncaught error fix * ⚡ clean up * ⚡ address in use fix * ⚡ improved error message * ⚡ tests setup * ⚡ unit tests wip * ⚡ config files clean up * ⚡ utils unit tests * ⚡ refactoring * ⚡ setup for testing operations, tests for deleteTable operation * ⚡ executeQuery and insert operations tests * ⚡ select, update, upsert operations tests * ⚡ runQueries tests setup * ⚡ hint to query * Copy updates. * ⚡ ui fixes * ⚡ clean up * ⚡ error message update * ⚡ ui update * Minor tweaks to query params decription. * feat(Google Sheets Node): Implement Resource mapper in Google Sheets node (#5752) * ✨ Added initial resource mapping support in google sheets node * ✨ Wired mapping API endpoint with node-specific logic for fetching mapping fields * ✨ Implementing mapping fields logic for google sheets * ✨ Updating Google Sheets execute methods to support resource mapper fields * 🚧 Added initial version of `ResourceLocator` component * 👌 Added `update` mode to resource mapper modes * 👌 Addressing PR feedback * 👌 Removing leftover const reference * 👕 Fixing lint errors * ⚡ singlton for conections * ⚡ credentials test fix, clean up * feat(Postgres Node): Add resource mapper to new version of Postgres node (#5814) * ⚡ scaffolding * ⚡ finished scaffolding * ⚡ renamed types * ⚡ updated subtitle * ⚡ renamed functions file, UI updates * ⚡ query parameters fixes, ui updates, refactoring * ⚡ fixes for credentials test, setup for error parsing * ⚡ rlc for schema and table, error handling tweaks * ⚡ delete operation, new options * ⚡ columns loader * ⚡ linter fixes * ⚡ where clauses setup * ⚡ logic for processing where clauses * ⚡ select operation * ⚡ refactoring * ⚡ data mode for insert and update, wip * ⚡ data mapping, insert update, skip on conflict option * ⚡ select columns with spaces fix * ⚡ update operation update, wip * ⚡ finished update operation * ⚡ upsert operation * ⚡ ui fixes * Copy updates. * Copy updates. * ⚡ option to convert empty strings to nulls, schema checks * ⚡ UI requested updates * ⚡ ssh setup WIP * ⚡ fixes, ssh WIP * ⚡ ssh fixes, credentials * ⚡ credentials testing update * ⚡ uncaught error fix * ⚡ clean up * ⚡ address in use fix * ⚡ improved error message * ⚡ tests setup * ⚡ unit tests wip * ⚡ config files clean up * ⚡ utils unit tests * ⚡ refactoring * ⚡ setup for testing operations, tests for deleteTable operation * ⚡ executeQuery and insert operations tests * ⚡ select, update, upsert operations tests * ⚡ runQueries tests setup * ⚡ hint to query * Copy updates. * ⚡ ui fixes * ⚡ clean up * ⚡ error message update * ⚡ ui update * Minor tweaks to query params decription. * ✨ Updated Postgres node to use resource mapper component * ✨ Implemented postgres <-> resource mapper type mapping * ✨ Updated Postgres node execution to use resource mapper fields in v3 * 🔥 Removing unused import --------- Co-authored-by: Michael Kret <michael.k@radency.com> Co-authored-by: Giulio Andreini <g.andreini@gmail.com> * feat(core): Resource editor componend P0 (#5970) * ✨ Added inital value of mapping mode dropdown * ✨ Finished mapping mode selector * ✨ Finished implementing mapping mode selector * ✨ Implemented 'Columns to match on' dropdown * ✨ Implemented `loadOptionsDependOn` support in resource mapper * ✨ Implemented initial version of mapping fields * ✨ Implementing dependant fields watcher in new component setup * ✨ Generating correct resource mapper field types. Added `supportAutoMap` to node specification and UI. Not showing fields with `display=false`. Pre-selecting matching columns if it's the only one * ✨ Handling matching columns correctly in UI * ✨ Saving and loading resourceMapper values in component * ✨ Implemented proper data saving and loading * ✨ ResourceMapper component refactor, fixing value save/load * ✨ Refactoring MatchingColumnSelect component. Updating Sheets node to use single key match and Postgres to use multi key * ✨ Updated Google Sheets node to work with the new UI * ✨ Updating Postgres Node to work with new UI * ✨ Additional loading indicator that shown if there is no mapping mode selector * ✨ Removing hard-coded values, fixing matching columns ordering, refactoring * ✨ Updating field names in nodes * ✨ Fixing minor UI issues * ✨ Implemented matching fields filter logic * ✨ Moving loading label outside of fields list * ✅ Added initial unit tests for resource mapper * ✅ Finished default rendering test * ✅ Test refactoring * ✅ Finished unit tests * 🔨 Updating the way i18n is used in resource mapper components * ✔️ Fixing value to match on logic for postgres node * ✨ Hiding mapping fields when auto-map mode is selected * ✨ Syncing selected mapping mode between components * ✨ Fixing dateTime input rendering and adding update check to Postgres node * ✨ Properly handling database connections. Sending null for empty string values. * 💄 Updated wording in the error message for non-existing rows * ✨ Fixing issues with selected matching values * ✔️ Updating unit tests after matching logic update * ✨ Updating matching columns when new fields are loaded * ✨ Defaulting to null for empty parameter values * ✨ Allowing zero as valid value for number imputs * ✨ Updated list of types that use datepicker as widger * ✨ Using text inputs for time types * ✨ Initial mapping field rework * ✨ Added new component for mapping fields, moved bit of logic from root component to matching selector, fixing some lint errors * ✨ Added tooltip for columns that cannot be deleted * ✨ Saving deleted values in parameter value * ✨ Implemented control to add/remove mapping fields * ✨ Syncing field list with add field dropdown when changing dependent values * ✨ Not showing removed fields in matching columns selector. Updating wording in matching columns selector description * ✨ Implementing disabled states for add/remove all fields options * ✨ Saving removed columns separately, updating copy * ✨ Implemented resource mapper values validation * ✨ Updated validation logic and error input styling * ✨ Validating resource mapper fields when new nodes are added * ✨ Using node field words in validation, refactoring resource mapper component * ✨ Implemented schema syncing and add/remove all fields * ✨ Implemented custom parameter actions * ✨ Implemented loading indicator in parameter options * 🔨 Removing unnecessary constants and vue props * ✨ Handling default values properly * ✨ Fixing validation logic * 👕 Fixing lint errors * ⚡ Fixing type issues * ⚡ Not showing fields by default if `addAllFields` is set to `false` * ✨ Implemented field type validation in resource mapper * ✨ Updated casing in copy, removed all/remove all option from bottom menu * ✨ Added auto mapping mode notice * ✨ Added support for more types in validation * ✨ Added support for enumerated values * ✨ Fixing imports after merging * ✨ Not showing removed fields in matching columns selector. Refactoring validation logic. * 👕 Fixing imports * ✔️ Updating unit tests * ✅ Added resource mapper schema tests * ⚡ Removing `match` from resource mapper field definition, fixing matching columns loading * ⚡ Fixed schema merging * ⚡ update operation return data fix * ⚡ review * 🐛 Added missing import * 💄 Updating parameter actions icon based on the ui review * 💄 Updating word capitalisation in tooltips * 💄 Added empty state to mapping fields list * 💄 Removing asterisk from fields, updating tooltips for matching fields * ⚡ Preventing matching fields from being removed by 'Remove All option' * ⚡ Not showing hidden fields in the `Add field` dropdown * ⚡ Added support for custom matching columns labels * ⚡ query optimization * ⚡ fix * ⚡ Optimizing Postgres node enumeration logic * ⚡ Added empty state for matching columns * ⚡ Only fully loading fields if there is no schema fetched * ⚡ Hiding mapping fields if there is no matching columns available in the schema * ✔️ Fixing minor issues * ✨ Implemented runtime type validation * 🔨 Refactoring validation logic * ✨ Implemented required check, added more custom messages * ✨ Skipping boolean type in required check * Type check improvements * ✨ Only reloading fields if dependent values actually change * ✨ Adding item index to validation error title * ✨ Updating Postgres fetching logic, using resource mapper mode to determine if a field can be deleted * ✨ Resetting field values when adding them via the addAll option * ⚡ Using minor version (2.2) for new Postgres node * ⚡ Implemented proper date validation and type casting * 👕 Consolidating typing * ✅ Added unit tests for type validations * 👌 Addressing front-end review comments * ⚡ More refactoring to address review changes * ⚡ Updating leftover props * ⚡ Added fallback for ISO dates with invalid timezones * Added timestamp to datetime test cases * ⚡ Reseting matching columns if operation changes * ⚡ Not forcing auto-increment fields to be filled in in Postgres node. Handling null values * 💄 Added a custom message for invalid dates * ⚡ Better handling of JSON values * ⚡ Updating codemirror readonly stauts based on component property, handling objects in json validation * Deleting leftover console.log * ⚡ Better time validation * ⚡ Fixing build error after merging * 👕 Fixing lint error * ⚡ Updating node configuration values * ⚡ Handling postgres arrays better * ⚡ Handling SQL array syntax * ⚡ Updating time validation rules to include timezone * ⚡ Sending expressions that resolve to `null` or `undefined` by the resource mapper to delete cell content in Google Sheets * ⚡ Allowing removed fields to be selected for match * ⚡ Updated the query for fetching unique columns and primary keys * ⚡ Optimizing the unique query * ⚡ Setting timezone to all parsed dates * ⚡ Addressing PR review feedback * ⚡ Configuring Sheets node for production, minor vue component update * New cases added to the TypeValidation test. * ✅ Tweaking validation rules for arrays/objects and updating test cases --------- Co-authored-by: Michael Kret <michael.k@radency.com> Co-authored-by: Giulio Andreini <g.andreini@gmail.com>
315 lines
8.8 KiB
TypeScript
315 lines
8.8 KiB
TypeScript
import type {
|
|
IExecuteFunctions,
|
|
IDataObject,
|
|
INodeExecutionData,
|
|
INodeListSearchItems,
|
|
INodePropertyOptions,
|
|
} from 'n8n-workflow';
|
|
import { NodeOperationError } from 'n8n-workflow';
|
|
import type { GoogleSheet } from './GoogleSheet';
|
|
import type {
|
|
RangeDetectionOptions,
|
|
ResourceLocator,
|
|
SheetRangeData,
|
|
ValueInputOption,
|
|
} from './GoogleSheets.types';
|
|
import { ResourceLocatorUiNames, ROW_NUMBER } from './GoogleSheets.types';
|
|
|
|
export const untilSheetSelected = { sheetName: [''] };
|
|
|
|
// Used to extract the ID from the URL
|
|
export function getSpreadsheetId(documentIdType: ResourceLocator, value: string): string {
|
|
if (!value) {
|
|
throw new Error(
|
|
`Can not get sheet '${ResourceLocatorUiNames[documentIdType]}' with a value of '${value}'`,
|
|
);
|
|
}
|
|
if (documentIdType === 'url') {
|
|
const regex = /([-\w]{25,})/;
|
|
const parts = value.match(regex);
|
|
|
|
if (parts == null || parts.length < 2) {
|
|
return '';
|
|
} else {
|
|
return parts[0];
|
|
}
|
|
}
|
|
// If it is byID or byList we can just return
|
|
return value;
|
|
}
|
|
|
|
// Convert number to Sheets / Excel column name
|
|
export function getColumnName(colNumber: number): string {
|
|
const baseChar = 'A'.charCodeAt(0);
|
|
let letters = '';
|
|
do {
|
|
colNumber -= 1;
|
|
letters = String.fromCharCode(baseChar + (colNumber % 26)) + letters;
|
|
colNumber = (colNumber / 26) >> 0;
|
|
} while (colNumber > 0);
|
|
|
|
return letters;
|
|
}
|
|
|
|
// Convert Column Name to Number (A = 1, B = 2, AA = 27)
|
|
export function getColumnNumber(colPosition: string): number {
|
|
let colNum = 0;
|
|
for (let i = 0; i < colPosition.length; i++) {
|
|
colNum *= 26;
|
|
colNum += colPosition[i].charCodeAt(0) - 'A'.charCodeAt(0) + 1;
|
|
}
|
|
return colNum;
|
|
}
|
|
|
|
// Hex to RGB
|
|
export function hexToRgb(hex: string) {
|
|
// Expand shorthand form (e.g. "03F") to full form (e.g. "0033FF")
|
|
const shorthandRegex = /^#?([a-f\d])([a-f\d])([a-f\d])$/i;
|
|
hex = hex.replace(shorthandRegex, (m, r, g, b) => {
|
|
// eslint-disable-next-line @typescript-eslint/restrict-plus-operands
|
|
return r + r + g + g + b + b;
|
|
});
|
|
|
|
const result = /^#?([a-f\d]{2})([a-f\d]{2})([a-f\d]{2})$/i.exec(hex);
|
|
|
|
if (result) {
|
|
return {
|
|
red: parseInt(result[1], 16),
|
|
green: parseInt(result[2], 16),
|
|
blue: parseInt(result[3], 16),
|
|
};
|
|
} else {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
export function addRowNumber(data: SheetRangeData, headerRow: number) {
|
|
if (data.length === 0) return data;
|
|
const sheetData = data.map((row, i) => [i + 1, ...row]);
|
|
sheetData[headerRow][0] = ROW_NUMBER;
|
|
return sheetData;
|
|
}
|
|
|
|
export function trimToFirstEmptyRow(data: SheetRangeData, includesRowNumber = true) {
|
|
const baseLength = includesRowNumber ? 1 : 0;
|
|
const emtyRowIndex = data.findIndex((row) => row.slice(baseLength).every((cell) => cell === ''));
|
|
if (emtyRowIndex === -1) {
|
|
return data;
|
|
}
|
|
return data.slice(0, emtyRowIndex);
|
|
}
|
|
|
|
export function removeEmptyRows(data: SheetRangeData, includesRowNumber = true) {
|
|
const baseLength = includesRowNumber ? 1 : 0;
|
|
const notEmptyRows = data.filter((row) =>
|
|
row.slice(baseLength).some((cell) => cell || typeof cell === 'number'),
|
|
);
|
|
if (includesRowNumber) {
|
|
notEmptyRows[0][0] = ROW_NUMBER;
|
|
}
|
|
return notEmptyRows;
|
|
}
|
|
|
|
export function trimLeadingEmptyRows(
|
|
data: SheetRangeData,
|
|
includesRowNumber = true,
|
|
rowNumbersColumnName = ROW_NUMBER,
|
|
) {
|
|
const baseLength = includesRowNumber ? 1 : 0;
|
|
const firstNotEmptyRowIndex = data.findIndex((row) =>
|
|
row.slice(baseLength).some((cell) => cell || typeof cell === 'number'),
|
|
);
|
|
|
|
const returnData = data.slice(firstNotEmptyRowIndex);
|
|
if (includesRowNumber) {
|
|
returnData[0][0] = rowNumbersColumnName;
|
|
}
|
|
|
|
return returnData;
|
|
}
|
|
|
|
export function removeEmptyColumns(data: SheetRangeData) {
|
|
if (!data || data.length === 0) return [];
|
|
const returnData: SheetRangeData = [];
|
|
const longestRow = data.reduce((a, b) => (a.length > b.length ? a : b), []).length;
|
|
for (let col = 0; col < longestRow; col++) {
|
|
const column = data.map((row) => row[col]);
|
|
if (column[0] !== '') {
|
|
returnData.push(column);
|
|
continue;
|
|
}
|
|
const hasData = column.slice(1).some((cell) => cell || typeof cell === 'number');
|
|
if (hasData) {
|
|
returnData.push(column);
|
|
}
|
|
}
|
|
return (returnData[0] || []).map((_, i) => returnData.map((row) => row[i] || ''));
|
|
}
|
|
|
|
export function prepareSheetData(
|
|
data: SheetRangeData,
|
|
options: RangeDetectionOptions,
|
|
addRowNumbersToData = true,
|
|
) {
|
|
let returnData: SheetRangeData = [...(data || [])];
|
|
|
|
let headerRow = 0;
|
|
let firstDataRow = 1;
|
|
|
|
if (options.rangeDefinition === 'specifyRange') {
|
|
headerRow = parseInt(options.headerRow as string, 10) - 1;
|
|
firstDataRow = parseInt(options.firstDataRow as string, 10) - 1;
|
|
}
|
|
|
|
if (addRowNumbersToData) {
|
|
returnData = addRowNumber(returnData, headerRow);
|
|
}
|
|
|
|
if (options.rangeDefinition === 'detectAutomatically') {
|
|
returnData = removeEmptyColumns(returnData);
|
|
returnData = trimLeadingEmptyRows(returnData, addRowNumbersToData);
|
|
|
|
if (options.readRowsUntil === 'firstEmptyRow') {
|
|
returnData = trimToFirstEmptyRow(returnData, addRowNumbersToData);
|
|
} else {
|
|
returnData = removeEmptyRows(returnData, addRowNumbersToData);
|
|
}
|
|
}
|
|
|
|
return { data: returnData, headerRow, firstDataRow };
|
|
}
|
|
|
|
export function getRangeString(sheetName: string, options: RangeDetectionOptions) {
|
|
if (options.rangeDefinition === 'specifyRangeA1') {
|
|
return options.range ? `${sheetName}!${options.range}` : sheetName;
|
|
}
|
|
return sheetName;
|
|
}
|
|
|
|
export async function getExistingSheetNames(sheet: GoogleSheet) {
|
|
const { sheets } = await sheet.spreadsheetGetSheets();
|
|
return ((sheets as IDataObject[]) || []).map((entry) => (entry.properties as IDataObject)?.title);
|
|
}
|
|
|
|
export function mapFields(this: IExecuteFunctions, inputSize: number) {
|
|
const returnData: IDataObject[] = [];
|
|
|
|
for (let i = 0; i < inputSize; i++) {
|
|
const nodeVersion = this.getNode().typeVersion;
|
|
if (nodeVersion < 4) {
|
|
const fields = this.getNodeParameter('fieldsUi.fieldValues', i, []) as IDataObject[];
|
|
let dataToSend: IDataObject = {};
|
|
for (const field of fields) {
|
|
dataToSend = { ...dataToSend, [field.fieldId as string]: field.fieldValue };
|
|
}
|
|
returnData.push(dataToSend);
|
|
} else {
|
|
const mappingValues = this.getNodeParameter('columns.value', i) as IDataObject;
|
|
if (Object.keys(mappingValues).length === 0) {
|
|
throw new NodeOperationError(
|
|
this.getNode(),
|
|
"At least one value has to be added under 'Values to Send'",
|
|
);
|
|
}
|
|
returnData.push(mappingValues);
|
|
}
|
|
}
|
|
|
|
return returnData;
|
|
}
|
|
|
|
export async function autoMapInputData(
|
|
this: IExecuteFunctions,
|
|
sheetNameWithRange: string,
|
|
sheet: GoogleSheet,
|
|
items: INodeExecutionData[],
|
|
options: IDataObject,
|
|
) {
|
|
const returnData: IDataObject[] = [];
|
|
const [sheetName, _sheetRange] = sheetNameWithRange.split('!');
|
|
const locationDefine = (options.locationDefine as IDataObject)?.values as IDataObject;
|
|
const handlingExtraData = (options.handlingExtraData as string) || 'insertInNewColumn';
|
|
|
|
let headerRow = 1;
|
|
|
|
if (locationDefine) {
|
|
headerRow = parseInt(locationDefine.headerRow as string, 10);
|
|
}
|
|
|
|
let columnNames: string[] = [];
|
|
const response = await sheet.getData(`${sheetName}!${headerRow}:${headerRow}`, 'FORMATTED_VALUE');
|
|
|
|
columnNames = response ? response[0] : [];
|
|
|
|
if (handlingExtraData === 'insertInNewColumn') {
|
|
if (!columnNames.length) {
|
|
await sheet.updateRows(
|
|
sheetName,
|
|
[Object.keys(items[0].json).filter((key) => key !== ROW_NUMBER)],
|
|
(options.cellFormat as ValueInputOption) || 'RAW',
|
|
headerRow,
|
|
);
|
|
columnNames = Object.keys(items[0].json);
|
|
}
|
|
|
|
const newColumns = new Set<string>();
|
|
|
|
items.forEach((item) => {
|
|
Object.keys(item.json).forEach((key) => {
|
|
if (key !== ROW_NUMBER && !columnNames.includes(key)) {
|
|
newColumns.add(key);
|
|
}
|
|
});
|
|
if (item.json[ROW_NUMBER]) {
|
|
delete item.json[ROW_NUMBER];
|
|
}
|
|
returnData.push(item.json);
|
|
});
|
|
if (newColumns.size) {
|
|
await sheet.updateRows(
|
|
sheetName,
|
|
[columnNames.concat([...newColumns])],
|
|
(options.cellFormat as ValueInputOption) || 'RAW',
|
|
headerRow,
|
|
);
|
|
}
|
|
}
|
|
if (handlingExtraData === 'ignoreIt') {
|
|
items.forEach((item) => {
|
|
returnData.push(item.json);
|
|
});
|
|
}
|
|
if (handlingExtraData === 'error') {
|
|
items.forEach((item, itemIndex) => {
|
|
Object.keys(item.json).forEach((key) => {
|
|
if (!columnNames.includes(key)) {
|
|
throw new NodeOperationError(this.getNode(), 'Unexpected fields in node input', {
|
|
itemIndex,
|
|
description: `The input field '${key}' doesn't match any column in the Sheet. You can ignore this by changing the 'Handling extra data' field, which you can find under 'Options'.`,
|
|
});
|
|
}
|
|
});
|
|
returnData.push(item.json);
|
|
});
|
|
}
|
|
|
|
return returnData;
|
|
}
|
|
|
|
export function sortLoadOptions(data: INodePropertyOptions[] | INodeListSearchItems[]) {
|
|
const returnData = [...data];
|
|
returnData.sort((a, b) => {
|
|
const aName = a.name.toLowerCase();
|
|
const bName = b.name.toLowerCase();
|
|
if (aName < bName) {
|
|
return -1;
|
|
}
|
|
if (aName > bName) {
|
|
return 1;
|
|
}
|
|
return 0;
|
|
});
|
|
|
|
return returnData;
|
|
}
|