mudhorn/n8n
1
0
Fork 0
mirror of https://github.com/n8n-io/n8n.git synced 2025-02-21 02:56:40 -08:00
Code Issues Actions Packages Projects Releases Wiki Activity

fix(Google Sheets Node): Append fails if cells have some default values added by data validation rules (#9950)

Browse source
This commit is contained in:
Michael Kret 2024-07-06 01:05:08 +03:00 committed by GitHub
parent b910ed6847
commit d1821eba92
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 216 additions and 161 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

28
packages/nodes-base/nodes/Google/Sheet/test/v2/utils/utils.test.ts
View file

@ -304,11 +304,11 @@ describe('Test Google Sheets, lookupValues', () => {
const googleSheet = new GoogleSheet('spreadsheetId', fakeExecuteFunction);
const result = await googleSheet.lookupValues(
const result = await googleSheet.lookupValues({
inputData,
0,
1,
[
keyRowIndex: 0,
dataStartRowIndex: 1,
lookupValues: [
{
lookupColumn: 'num',
lookupValue: '1',
@ -318,9 +318,9 @@ describe('Test Google Sheets, lookupValues', () => {
lookupValue: 'foo',
},
],
true,
'OR',
);
returnAllMatches: true,
combineFilters: 'OR',
});
expect(result).toBeDefined();
expect(result).toEqual([
@ -366,11 +366,11 @@ describe('Test Google Sheets, lookupValues', () => {
const googleSheet = new GoogleSheet('spreadsheetId', fakeExecuteFunction);
const result = await googleSheet.lookupValues(
const result = await googleSheet.lookupValues({
inputData,
0,
1,
[
keyRowIndex: 0,
dataStartRowIndex: 1,
lookupValues: [
{
lookupColumn: 'num',
lookupValue: '1',
@ -380,9 +380,9 @@ describe('Test Google Sheets, lookupValues', () => {
lookupValue: 'baz',
},
],
true,
'AND',
);
returnAllMatches: true,
combineFilters: 'AND',
});
expect(result).toBeDefined();
expect(result).toEqual([

67
packages/nodes-base/nodes/Google/Sheet/v2/actions/sheet/append.operation.ts
View file

@ -213,7 +213,7 @@ export const description: SheetProperties = [
export async function execute(
this: IExecuteFunctions,
sheet: GoogleSheet,
sheetName: string,
range: string,
sheetId: string,
): Promise<INodeExecutionData[]> {
const items = this.getInputData();
@ -228,57 +228,62 @@ export async function execute(
const options = this.getNodeParameter('options', 0, {});
const locationDefine = (options.locationDefine as IDataObject)?.values as IDataObject;
let headerRow = 1;
let keyRowIndex = 1;
if (locationDefine?.headerRow) {
headerRow = locationDefine.headerRow as number;
keyRowIndex = locationDefine.headerRow as number;
}
const sheetData = await sheet.getData(range, 'FORMATTED_VALUE');
if (nodeVersion >= 4.4 && dataMode !== 'autoMapInputData') {
//not possible to refresh columns when mode is autoMapInputData
const sheetData = await sheet.getData(sheetName, 'FORMATTED_VALUE');
if (sheetData?.[headerRow - 1] === undefined) {
if (sheetData?.[keyRowIndex - 1] === undefined) {
throw new NodeOperationError(
this.getNode(),
`Could not retrieve the column names from row ${headerRow}`,
`Could not retrieve the column names from row ${keyRowIndex}`,
);
}
const schema = this.getNodeParameter('columns.schema', 0) as ResourceMapperField[];
checkForSchemaChanges(this.getNode(), sheetData[headerRow - 1], schema);
checkForSchemaChanges(this.getNode(), sheetData[keyRowIndex - 1], schema);
}
let setData: IDataObject[] = [];
let inputData: IDataObject[] = [];
if (dataMode === 'autoMapInputData') {
setData = await autoMapInputData.call(this, sheetName, sheet, items, options);
inputData = await autoMapInputData.call(this, range, sheet, items, options);
} else {
setData = mapFields.call(this, items.length);
inputData = mapFields.call(this, items.length);
}
if (setData.length === 0) {
if (inputData.length === 0) {
return [];
} else if (options.useAppend) {
await sheet.appendSheetData(
setData,
sheetName,
headerRow,
(options.cellFormat as ValueInputOption) || cellFormatDefault(nodeVersion),
false,
undefined,
undefined,
options.useAppend as boolean,
);
}
const valueInputMode = (options.cellFormat as ValueInputOption) || cellFormatDefault(nodeVersion);
const useAppend = options.useAppend as boolean;
if (options.useAppend) {
await sheet.appendSheetData({
inputData,
range,
keyRowIndex,
valueInputMode,
useAppend,
});
} else {
//if no trailing empty row exists in the sheet update operation will fail
await sheet.appendEmptyRowsOrColumns(sheetId, 1, 0);
await sheet.appendSheetData(
setData,
sheetName,
headerRow,
(options.cellFormat as ValueInputOption) || cellFormatDefault(nodeVersion),
false,
);
const lastRow = (sheetData ?? []).length + 1;
await sheet.appendSheetData({
inputData,
range,
keyRowIndex,
valueInputMode,
lastRow,
});
}
if (nodeVersion < 4 || dataMode === 'autoMapInputData') {
@ -288,7 +293,7 @@ export async function execute(
});
} else {
const returnData: INodeExecutionData[] = [];
for (const [index, entry] of setData.entries()) {
for (const [index, entry] of inputData.entries()) {
returnData.push({
json: entry,
pairedItem: { item: index },

84
packages/nodes-base/nodes/Google/Sheet/v2/actions/sheet/appendOrUpdate.operation.ts
View file

@ -246,15 +246,15 @@ export async function execute(
const locationDefineOption = (options.locationDefine as IDataObject)?.values as IDataObject;
let headerRow = 0;
let firstDataRow = 1;
let keyRowIndex = 0;
let dataStartRowIndex = 1;
if (locationDefineOption) {
if (locationDefineOption.headerRow) {
headerRow = parseInt(locationDefineOption.headerRow as string, 10) - 1;
keyRowIndex = parseInt(locationDefineOption.headerRow as string, 10) - 1;
}
if (locationDefineOption.firstDataRow) {
firstDataRow = parseInt(locationDefineOption.firstDataRow as string, 10) - 1;
dataStartRowIndex = parseInt(locationDefineOption.firstDataRow as string, 10) - 1;
}
}
@ -267,14 +267,14 @@ export async function execute(
const sheetData = (await sheet.getData(sheetName, 'FORMATTED_VALUE')) ?? [];
if (!sheetData[headerRow] && dataMode !== 'autoMapInputData') {
if (!sheetData[keyRowIndex] && dataMode !== 'autoMapInputData') {
throw new NodeOperationError(
this.getNode(),
`Could not retrieve the column names from row ${headerRow + 1}`,
`Could not retrieve the column names from row ${keyRowIndex + 1}`,
);
}
columnNames = sheetData[headerRow] ?? [];
columnNames = sheetData[keyRowIndex] ?? [];
if (nodeVersion >= 4.4) {
const schema = this.getNodeParameter('columns.schema', 0) as ResourceMapperField[];
@ -291,13 +291,13 @@ export async function execute(
// TODO: Add support for multiple columns to match on in the next overhaul
const keyIndex = columnNames.indexOf(columnsToMatchOn[0]);
const columnValues = await sheet.getColumnValues(
const columnValuesList = await sheet.getColumnValues({
range,
keyIndex,
firstDataRow,
dataStartRowIndex,
valueRenderMode,
sheetData,
);
});
const updateData: ISheetUpdateData[] = [];
const appendData: IDataObject[] = [];
@ -321,20 +321,20 @@ export async function execute(
for (let i = 0; i < items.length; i++) {
if (dataMode === 'nothing') continue;
const data: IDataObject[] = [];
const inputData: IDataObject[] = [];
if (dataMode === 'autoMapInputData') {
const handlingExtraDataOption = (options.handlingExtraData as string) || 'insertInNewColumn';
if (handlingExtraDataOption === 'ignoreIt') {
data.push(items[i].json);
inputData.push(items[i].json);
}
if (handlingExtraDataOption === 'error') {
Object.keys(items[i].json).forEach((key) => errorOnUnexpectedColumn(key, i));
data.push(items[i].json);
inputData.push(items[i].json);
}
if (handlingExtraDataOption === 'insertInNewColumn') {
Object.keys(items[i].json).forEach(addNewColumn);
data.push(items[i].json);
inputData.push(items[i].json);
}
} else {
const valueToMatchOn =
@ -364,7 +364,7 @@ export async function execute(
return acc;
}, {} as IDataObject);
fields[columnsToMatchOn[0]] = valueToMatchOn;
data.push(fields);
inputData.push(fields);
} else {
const mappingValues = this.getNodeParameter('columns.value', i) as IDataObject;
if (Object.keys(mappingValues).length === 0) {
@ -379,7 +379,7 @@ export async function execute(
mappingValues[key] = '';
}
});
data.push(mappingValues);
inputData.push(mappingValues);
mappedValues.push(mappingValues);
}
}
@ -390,56 +390,60 @@ export async function execute(
sheetName,
[newColumnNames],
(options.cellFormat as ValueInputOption) || cellFormatDefault(nodeVersion),
headerRow + 1,
keyRowIndex + 1,
);
columnNames = newColumnNames;
sheetData[headerRow] = newColumnNames;
sheetData[keyRowIndex] = newColumnNames;
newColumns.clear();
}
const preparedData = await sheet.prepareDataForUpdateOrUpsert(
data,
columnsToMatchOn[0],
const indexKey = columnsToMatchOn[0];
const preparedData = await sheet.prepareDataForUpdateOrUpsert({
inputData,
indexKey,
range,
headerRow,
firstDataRow,
keyRowIndex,
dataStartRowIndex,
valueRenderMode,
true,
[columnNames.concat([...newColumns])],
columnValues,
);
upsert: true,
columnNamesList: [columnNames.concat([...newColumns])],
columnValuesList,
});
updateData.push(...preparedData.updateData);
appendData.push(...preparedData.appendData);
}
const columnNamesList = [columnNames.concat([...newColumns])];
if (updateData.length) {
await sheet.batchUpdate(updateData, valueInputMode);
}
if (appendData.length) {
const lastRow = sheetData.length + 1;
const useAppend = options.useAppend as boolean;
if (options.useAppend) {
await sheet.appendSheetData(
appendData,
await sheet.appendSheetData({
inputData: appendData,
range,
headerRow + 1,
keyRowIndex: keyRowIndex + 1,
valueInputMode,
false,
[columnNames.concat([...newColumns])],
columnNamesList,
lastRow,
options.useAppend as boolean,
);
useAppend,
});
} else {
await sheet.appendEmptyRowsOrColumns(sheetId, 1, 0);
await sheet.appendSheetData(
appendData,
await sheet.appendSheetData({
inputData: appendData,
range,
headerRow + 1,
keyRowIndex: keyRowIndex + 1,
valueInputMode,
false,
[columnNames.concat([...newColumns])],
columnNamesList,
lastRow,
);
});
}
}

23
packages/nodes-base/nodes/Google/Sheet/v2/actions/sheet/read.operation.ts
View file

@ -202,10 +202,11 @@ export async function execute(
return [];
}
const { data, headerRow, firstDataRow } = prepareSheetData(
sheetData,
dataLocationOnSheetOptions,
);
const {
data,
headerRow: keyRowIndex,
firstDataRow: dataStartRowIndex,
} = prepareSheetData(sheetData, dataLocationOnSheetOptions);
let responseData = [];
@ -215,6 +216,8 @@ export async function execute(
[],
) as ILookupValues[];
const inputData = data as string[][];
if (lookupValues.length) {
const returnAllMatches = options.returnAllMatches === 'returnAllMatches' ? true : false;
@ -235,16 +238,16 @@ export async function execute(
| 'AND'
| 'OR';
responseData = await sheet.lookupValues(
data as string[][],
headerRow,
firstDataRow,
responseData = await sheet.lookupValues({
inputData,
keyRowIndex,
dataStartRowIndex,
lookupValues,
returnAllMatches,
combineFilters,
);
});
} else {
responseData = sheet.structureArrayDataByColumn(data as string[][], headerRow, firstDataRow);
responseData = sheet.structureArrayDataByColumn(inputData, keyRowIndex, dataStartRowIndex);
}
returnData.push(

59
packages/nodes-base/nodes/Google/Sheet/v2/actions/sheet/update.operation.ts
View file

@ -237,15 +237,15 @@ export async function execute(
const locationDefineOptions = (options.locationDefine as IDataObject)?.values as IDataObject;
let headerRow = 0;
let firstDataRow = 1;
let keyRowIndex = 0;
let dataStartRowIndex = 1;
if (locationDefineOptions) {
if (locationDefineOptions.headerRow) {
headerRow = parseInt(locationDefineOptions.headerRow as string, 10) - 1;
keyRowIndex = parseInt(locationDefineOptions.headerRow as string, 10) - 1;
}
if (locationDefineOptions.firstDataRow) {
firstDataRow = parseInt(locationDefineOptions.firstDataRow as string, 10) - 1;
dataStartRowIndex = parseInt(locationDefineOptions.firstDataRow as string, 10) - 1;
}
}
@ -253,14 +253,14 @@ export async function execute(
const sheetData = await sheet.getData(sheetName, 'FORMATTED_VALUE');
if (sheetData?.[headerRow] === undefined) {
if (sheetData?.[keyRowIndex] === undefined) {
throw new NodeOperationError(
this.getNode(),
`Could not retrieve the column names from row ${headerRow + 1}`,
`Could not retrieve the column names from row ${keyRowIndex + 1}`,
);
}
columnNames = sheetData[headerRow];
columnNames = sheetData[keyRowIndex];
if (nodeVersion >= 4.4) {
const schema = this.getNodeParameter('columns.schema', 0) as ResourceMapperField[];
@ -283,13 +283,13 @@ export async function execute(
const keyIndex = columnNames.indexOf(columnsToMatchOn[0]);
//not used when updating row
const columnValues = await sheet.getColumnValues(
const columnValuesList = await sheet.getColumnValues({
range,
keyIndex,
firstDataRow,
dataStartRowIndex,
valueRenderMode,
sheetData,
);
});
const updateData: ISheetUpdateData[] = [];
@ -313,20 +313,20 @@ export async function execute(
for (let i = 0; i < items.length; i++) {
if (dataMode === 'nothing') continue;
const data: IDataObject[] = [];
const inputData: IDataObject[] = [];
if (dataMode === 'autoMapInputData') {
const handlingExtraDataOption = (options.handlingExtraData as string) || 'insertInNewColumn';
if (handlingExtraDataOption === 'ignoreIt') {
data.push(items[i].json);
inputData.push(items[i].json);
}
if (handlingExtraDataOption === 'error' && columnsToMatchOn[0] !== 'row_number') {
Object.keys(items[i].json).forEach((key) => errorOnUnexpectedColumn(key, i));
data.push(items[i].json);
inputData.push(items[i].json);
}
if (handlingExtraDataOption === 'insertInNewColumn' && columnsToMatchOn[0] !== 'row_number') {
Object.keys(items[i].json).forEach(addNewColumn);
data.push(items[i].json);
inputData.push(items[i].json);
}
} else {
const valueToMatchOn =
@ -360,7 +360,7 @@ export async function execute(
fields[columnsToMatchOn[0]] = valueToMatchOn;
data.push(fields);
inputData.push(fields);
} else {
const mappingValues = this.getNodeParameter('columns.value', i) as IDataObject;
if (Object.keys(mappingValues).length === 0) {
@ -375,7 +375,7 @@ export async function execute(
mappingValues[key] = '';
}
});
data.push(mappingValues);
inputData.push(mappingValues);
mappedValues.push(mappingValues);
}
}
@ -386,29 +386,30 @@ export async function execute(
sheetName,
[newColumnNames],
(options.cellFormat as ValueInputOption) || cellFormatDefault(nodeVersion),
headerRow + 1,
keyRowIndex + 1,
);
columnNames = newColumnNames;
newColumns.clear();
}
let preparedData;
const columnNamesList = [columnNames.concat([...newColumns])];
if (columnsToMatchOn[0] === 'row_number') {
preparedData = sheet.prepareDataForUpdatingByRowNumber(data, range, [
columnNames.concat([...newColumns]),
]);
preparedData = sheet.prepareDataForUpdatingByRowNumber(inputData, range, columnNamesList);
} else {
preparedData = await sheet.prepareDataForUpdateOrUpsert(
data,
columnsToMatchOn[0],
const indexKey = columnsToMatchOn[0];
preparedData = await sheet.prepareDataForUpdateOrUpsert({
inputData,
indexKey,
range,
headerRow,
firstDataRow,
keyRowIndex,
dataStartRowIndex,
valueRenderMode,
false,
[columnNames.concat([...newColumns])],
columnValues,
);
columnNamesList,
columnValuesList,
});
}
updateData.push(...preparedData.updateData);

10
packages/nodes-base/nodes/Google/Sheet/v2/actions/versionDescription.ts
View file

@ -17,6 +17,16 @@ export const versionDescription: INodeTypeDescription = {
},
inputs: ['main'],
outputs: ['main'],
hints: [
{
message:
"Use the 'Use Append' option for greater efficiency if your sheet is uniformly formatted without gaps between columns or rows",
displayCondition:
'={{$parameter["operation"] === "append" && !$parameter["options"]["useAppend"]}}',
whenToDisplay: 'beforeExecution',
location: 'outputPane',
},
],
credentials: [
{
name: 'googleApi',

106
packages/nodes-base/nodes/Google/Sheet/v2/helpers/GoogleSheet.ts
View file

@ -380,16 +380,25 @@ export class GoogleSheet {
return keys;
}
async appendSheetData(
inputData: IDataObject[],
range: string,
keyRowIndex: number,
valueInputMode: ValueInputOption,
usePathForKeyRow: boolean,
columnNamesList?: string[][],
lastRow?: number,
useAppend?: boolean,
): Promise<string[][]> {
async appendSheetData({
inputData,
range,
keyRowIndex,
valueInputMode,
usePathForKeyRow,
columnNamesList,
lastRow,
useAppend,
}: {
inputData: IDataObject[];
range: string;
keyRowIndex: number;
valueInputMode: ValueInputOption;
usePathForKeyRow?: boolean;
columnNamesList?: string[][];
lastRow?: number;
useAppend?: boolean;
}): Promise<string[][]> {
const data = await this.convertObjectArrayToSheetDataArray(
inputData,
range,
@ -406,13 +415,19 @@ export class GoogleSheet {
return xlsxUtils.encode_col(columnIndex);
}
async getColumnValues(
range: string,
keyIndex: number,
dataStartRowIndex: number,
valueRenderMode: ValueRenderOption,
sheetData?: string[][],
): Promise<string[]> {
async getColumnValues({
range,
keyIndex,
dataStartRowIndex,
valueRenderMode,
sheetData,
}: {
range: string;
keyIndex: number;
dataStartRowIndex: number;
valueRenderMode: ValueRenderOption;
sheetData?: string[][];
}): Promise<string[]> {
let columnValuesList;
if (sheetData) {
columnValuesList = sheetData.slice(dataStartRowIndex - 1).map((row) => row[keyIndex]);
@ -448,17 +463,27 @@ export class GoogleSheet {
* @returns {Promise<string[][]>}
* @memberof GoogleSheet
*/
async prepareDataForUpdateOrUpsert(
inputData: IDataObject[],
indexKey: string,
range: string,
keyRowIndex: number,
dataStartRowIndex: number,
valueRenderMode: ValueRenderOption,
async prepareDataForUpdateOrUpsert({
inputData,
indexKey,
range,
keyRowIndex,
dataStartRowIndex,
valueRenderMode,
upsert = false,
columnNamesList?: string[][],
columnValuesList?: string[],
) {
columnNamesList,
columnValuesList,
}: {
inputData: IDataObject[];
indexKey: string;
range: string;
keyRowIndex: number;
dataStartRowIndex: number;
valueRenderMode: ValueRenderOption;
upsert?: boolean;
columnNamesList?: string[][];
columnValuesList?: string[];
}) {
const decodedRange = this.getDecodedSheetRange(range);
// prettier-ignore
const keyRowRange = `${decodedRange.name}!${decodedRange.start?.column || ''}${keyRowIndex + 1}:${decodedRange.end?.column || ''}${keyRowIndex + 1}`;
@ -485,7 +510,7 @@ export class GoogleSheet {
const columnValues: Array<string | number> =
columnValuesList ||
(await this.getColumnValues(range, keyIndex, dataStartRowIndex, valueRenderMode));
(await this.getColumnValues({ range, keyIndex, dataStartRowIndex, valueRenderMode }));
const updateData: ISheetUpdateData[] = [];
const appendData: IDataObject[] = [];
@ -620,14 +645,21 @@ export class GoogleSheet {
* @returns {Promise<IDataObject[]>}
* @memberof GoogleSheet
*/
async lookupValues(
inputData: string[][],
keyRowIndex: number,
dataStartRowIndex: number,
lookupValues: ILookupValues[],
returnAllMatches?: boolean,
combineFilters: 'AND' | 'OR' = 'OR',
): Promise<IDataObject[]> {
async lookupValues({
inputData,
keyRowIndex,
dataStartRowIndex,
lookupValues,
returnAllMatches,
combineFilters = 'OR',
}: {
inputData: string[][];
keyRowIndex: number;
dataStartRowIndex: number;
lookupValues: ILookupValues[];
returnAllMatches?: boolean;
combineFilters?: 'AND' | 'OR';
}): Promise<IDataObject[]> {
const keys: string[] = [];
if (keyRowIndex < 0 || dataStartRowIndex < keyRowIndex || keyRowIndex >= inputData.length) {
@ -740,7 +772,7 @@ export class GoogleSheet {
inputData: IDataObject[],
range: string,
keyRowIndex: number,
usePathForKeyRow: boolean,
usePathForKeyRow?: boolean,
columnNamesList?: string[][],
emptyValue: string | null = '',
): Promise<string[][]> {