n8n/packages/nodes-base/nodes/Google/Sheet/v1/GoogleSheet.ts

601 lines
15 KiB
TypeScript
Raw Normal View History

import type { IExecuteFunctions, ILoadOptionsFunctions, IDataObject } from 'n8n-workflow';
import { NodeOperationError } from 'n8n-workflow';
2020-05-30 12:09:04 -07:00
import { googleApiRequest } from './GenericFunctions';
2019-06-23 03:35:23 -07:00
import { utils as xlsxUtils } from 'xlsx';
import get from 'lodash/get';
2019-06-23 03:35:23 -07:00
export interface ISheetOptions {
scope: string[];
}
export interface IGoogleAuthCredentials {
email: string;
privateKey: string;
}
export interface ISheetUpdateData {
range: string;
values: string[][];
}
export interface ILookupValues {
lookupColumn: string;
lookupValue: string;
}
2019-06-23 03:35:23 -07:00
export interface IToDeleteRange {
amount: number;
startIndex: number;
sheetId: number;
}
export interface IToDelete {
[key: string]: IToDeleteRange[] | undefined;
columns?: IToDeleteRange[];
rows?: IToDeleteRange[];
}
export type ValueInputOption = 'RAW' | 'USER_ENTERED';
export type ValueRenderOption = 'FORMATTED_VALUE' | 'FORMULA' | 'UNFORMATTED_VALUE';
2019-06-23 03:35:23 -07:00
export class GoogleSheet {
id: string;
2020-05-30 12:09:04 -07:00
executeFunctions: IExecuteFunctions | ILoadOptionsFunctions;
2019-06-23 03:35:23 -07:00
constructor(
spreadsheetId: string,
executeFunctions: IExecuteFunctions | ILoadOptionsFunctions,
options?: ISheetOptions | undefined,
) {
2019-06-23 03:35:23 -07:00
// options = <SheetOptions>options || {};
if (!options) {
options = {} as ISheetOptions;
}
2020-05-30 12:09:04 -07:00
this.executeFunctions = executeFunctions;
2019-06-23 03:35:23 -07:00
this.id = spreadsheetId;
}
/**
* Encodes the range that also none latin character work
*
*/
encodeRange(range: string): string {
if (range.includes('!')) {
const [sheet, ranges] = range.split('!');
range = `${encodeURIComponent(sheet)}!${ranges}`;
}
return range;
}
/**
* Clears values from a sheet
*
*/
async clearData(range: string): Promise<object> {
2020-05-30 12:09:04 -07:00
const body = {
spreadsheetId: this.id,
range,
};
const response = await googleApiRequest.call(
this.executeFunctions,
'POST',
`/v4/spreadsheets/${this.id}/values/${range}:clear`,
body,
);
2020-05-30 12:09:04 -07:00
return response;
}
/**
* Returns the cell values
*/
async getData(
range: string,
valueRenderMode: ValueRenderOption,
): Promise<string[][] | undefined> {
2020-05-30 12:09:04 -07:00
const query = {
valueRenderOption: valueRenderMode,
};
const response = await googleApiRequest.call(
this.executeFunctions,
'GET',
`/v4/spreadsheets/${this.id}/values/${range}`,
{},
query,
);
2020-05-30 12:09:04 -07:00
return response.values as string[][] | undefined;
2019-06-23 03:35:23 -07:00
}
/**
* Returns the sheets in a Spreadsheet
*/
async spreadsheetGetSheets() {
2020-05-30 12:09:04 -07:00
const query = {
fields: 'sheets.properties',
};
const response = await googleApiRequest.call(
this.executeFunctions,
'GET',
`/v4/spreadsheets/${this.id}`,
{},
query,
);
2020-05-30 12:09:04 -07:00
return response;
}
/**
* Sets values in one or more ranges of a spreadsheet.
*/
async spreadsheetBatchUpdate(requests: IDataObject[]) {
2020-05-30 12:09:04 -07:00
const body = {
2020-10-22 06:46:03 -07:00
requests,
2020-05-30 12:09:04 -07:00
};
const response = await googleApiRequest.call(
this.executeFunctions,
'POST',
`/v4/spreadsheets/${this.id}:batchUpdate`,
body,
);
2020-05-30 12:09:04 -07:00
return response;
}
/**
* Sets the cell values
*/
async batchUpdate(updateData: ISheetUpdateData[], valueInputMode: ValueInputOption) {
2020-05-30 12:09:04 -07:00
const body = {
data: updateData,
valueInputOption: valueInputMode,
};
const response = await googleApiRequest.call(
this.executeFunctions,
'POST',
`/v4/spreadsheets/${this.id}/values:batchUpdate`,
body,
);
2020-05-30 12:09:04 -07:00
return response;
2019-06-23 03:35:23 -07:00
}
/**
* Sets the cell values
*/
async setData(range: string, data: string[][], valueInputMode: ValueInputOption) {
2020-05-30 12:09:04 -07:00
const body = {
valueInputOption: valueInputMode,
values: data,
};
const response = await googleApiRequest.call(
this.executeFunctions,
'POST',
`/v4/spreadsheets/${this.id}/values/${range}`,
body,
);
2020-05-30 12:09:04 -07:00
return response;
2019-06-23 03:35:23 -07:00
}
/**
* Appends the cell values
*/
async appendData(range: string, data: string[][], valueInputMode: ValueInputOption) {
2020-05-30 12:09:04 -07:00
const body = {
range: decodeURIComponent(range),
2020-05-30 12:09:04 -07:00
values: data,
};
2019-06-23 03:35:23 -07:00
2020-05-30 12:09:04 -07:00
const query = {
valueInputOption: valueInputMode,
};
2019-06-23 03:35:23 -07:00
const response = await googleApiRequest.call(
this.executeFunctions,
'POST',
`/v4/spreadsheets/${this.id}/values/${range}:append`,
body,
query,
);
2019-06-23 03:35:23 -07:00
2020-05-30 12:09:04 -07:00
return response;
}
2019-06-23 03:35:23 -07:00
/**
* Returns the given sheet data in a structured way
*/
structureData(
inputData: string[][],
startRow: number,
keys: string[],
addEmpty?: boolean,
): IDataObject[] {
2019-06-23 03:35:23 -07:00
const returnData = [];
let tempEntry: IDataObject, rowIndex: number, columnIndex: number, key: string;
for (rowIndex = startRow; rowIndex < inputData.length; rowIndex++) {
tempEntry = {};
for (columnIndex = 0; columnIndex < inputData[rowIndex].length; columnIndex++) {
key = keys[columnIndex];
if (key) {
// Only add the data for which a key was given and ignore all others
tempEntry[key] = inputData[rowIndex][columnIndex];
}
}
if (Object.keys(tempEntry).length || addEmpty === true) {
2019-06-23 03:35:23 -07:00
// Only add the entry if data got found to not have empty ones
returnData.push(tempEntry);
}
}
return returnData;
}
/**
* Returns the given sheet data in a structured way using
* the startRow as the one with the name of the key
*/
structureArrayDataByColumn(
inputData: string[][],
keyRow: number,
dataStartRow: number,
): IDataObject[] {
2019-06-23 03:35:23 -07:00
const keys: string[] = [];
if (keyRow < 0 || dataStartRow < keyRow || keyRow >= inputData.length) {
// The key row does not exist so it is not possible to structure data
2019-06-23 03:35:23 -07:00
return [];
}
// Create the keys array
for (let columnIndex = 0; columnIndex < inputData[keyRow].length; columnIndex++) {
keys.push(inputData[keyRow][columnIndex]);
}
return this.structureData(inputData, dataStartRow, keys);
}
async appendSheetData(
inputData: IDataObject[],
range: string,
keyRowIndex: number,
valueInputMode: ValueInputOption,
usePathForKeyRow: boolean,
): Promise<string[][]> {
const data = await this.convertStructuredDataToArray(
inputData,
range,
keyRowIndex,
usePathForKeyRow,
);
return this.appendData(range, data, valueInputMode);
2019-06-23 03:35:23 -07:00
}
getColumnWithOffset(startColumn: string, offset: number): string {
const columnIndex = xlsxUtils.decode_col(startColumn) + offset;
return xlsxUtils.encode_col(columnIndex);
}
2019-06-23 03:35:23 -07:00
/**
* Updates data in a sheet
*
* @param {IDataObject[]} inputData Data to update Sheet with
* @param {string} indexKey The name of the key which gets used to know which rows to update
* @param {string} range The range to look for data
* @param {number} keyRowIndex Index of the row which contains the keys
* @param {number} dataStartRowIndex Index of the first row which contains data
*/
async updateSheetData(
inputData: IDataObject[],
indexKey: string,
range: string,
keyRowIndex: number,
dataStartRowIndex: number,
valueInputMode: ValueInputOption,
valueRenderMode: ValueRenderOption,
upsert = false,
): Promise<string[][]> {
2019-06-23 03:35:23 -07:00
// Get current data in Google Sheet
let rangeFull: string;
2019-06-23 03:35:23 -07:00
let sheet: string | undefined = undefined;
if (range.includes('!')) {
[sheet, rangeFull] = range.split('!');
} else {
rangeFull = range;
2019-06-23 03:35:23 -07:00
}
const [rangeStart, rangeEnd] = rangeFull.split(':');
2019-06-23 03:35:23 -07:00
const rangeStartSplit = rangeStart.match(/([a-zA-Z]{1,10})([0-9]{0,10})/);
const rangeEndSplit = rangeEnd.match(/([a-zA-Z]{1,10})([0-9]{0,10})/);
if (
rangeStartSplit === null ||
rangeStartSplit.length !== 3 ||
rangeEndSplit === null ||
rangeEndSplit.length !== 3
) {
throw new NodeOperationError(
this.executeFunctions.getNode(),
`The range "${range}" is not valid.`,
);
}
const keyRowRange = `${sheet ? sheet + '!' : ''}${rangeStartSplit[1]}${keyRowIndex + 1}:${
rangeEndSplit[1]
}${keyRowIndex + 1}`;
2019-06-23 03:35:23 -07:00
const sheetDatakeyRow = await this.getData(this.encodeRange(keyRowRange), valueRenderMode);
2019-06-23 03:35:23 -07:00
if (sheetDatakeyRow === undefined) {
throw new NodeOperationError(
this.executeFunctions.getNode(),
'Could not retrieve the key row!',
);
2019-06-23 03:35:23 -07:00
}
const keyColumnOrder = sheetDatakeyRow[0];
const keyIndex = keyColumnOrder.indexOf(indexKey);
if (keyIndex === -1) {
throw new NodeOperationError(
this.executeFunctions.getNode(),
`Could not find column for key "${indexKey}"!`,
);
2019-06-23 03:35:23 -07:00
}
const startRowIndex = rangeStartSplit[2] || dataStartRowIndex;
const endRowIndex = rangeEndSplit[2] || '';
const keyColumn = this.getColumnWithOffset(rangeStartSplit[1], keyIndex);
const keyColumnRange = `${
sheet ? sheet + '!' : ''
}${keyColumn}${startRowIndex}:${keyColumn}${endRowIndex}`;
2019-06-23 03:35:23 -07:00
const sheetDataKeyColumn = await this.getData(
this.encodeRange(keyColumnRange),
valueRenderMode,
);
2019-06-23 03:35:23 -07:00
if (sheetDataKeyColumn === undefined) {
throw new NodeOperationError(
this.executeFunctions.getNode(),
'Could not retrieve the key column!',
);
2019-06-23 03:35:23 -07:00
}
// TODO: The data till here can be cached optionally. Maybe add an option which can
// can be activated if it is used in a loop and nothing else updates the data.
// Remove the first row which contains the key
sheetDataKeyColumn.shift();
// Create an Array which all the key-values of the Google Sheet
const keyColumnIndexLookup = sheetDataKeyColumn.map((rowContent) => rowContent[0]);
2019-06-23 03:35:23 -07:00
const updateData: ISheetUpdateData[] = [];
let itemKey: string | number | undefined | null;
let propertyName: string;
let itemKeyIndex: number;
let updateRowIndex: number;
let updateColumnName: string;
for (const inputItem of inputData) {
itemKey = inputItem[indexKey] as string;
// if ([undefined, null].includes(inputItem[indexKey] as string | undefined | null)) {
if (itemKey === undefined || itemKey === null) {
// Item does not have the indexKey so we can ignore it or append it if upsert true
if (upsert) {
const _data = await this.appendSheetData(
[inputItem],
this.encodeRange(range),
keyRowIndex,
valueInputMode,
false,
);
}
2019-06-23 03:35:23 -07:00
continue;
}
// Item does have the key so check if it exists in Sheet
itemKeyIndex = keyColumnIndexLookup.indexOf(itemKey);
2019-06-23 03:35:23 -07:00
if (itemKeyIndex === -1) {
// Key does not exist in the Sheet so it can not be updated so skip it or append it if upsert true
if (upsert) {
const _data = await this.appendSheetData(
[inputItem],
this.encodeRange(range),
keyRowIndex,
valueInputMode,
false,
);
}
2019-06-23 03:35:23 -07:00
continue;
}
// Get the row index in which the data should be updated
updateRowIndex = keyColumnIndexLookup.indexOf(itemKey) + dataStartRowIndex + 1;
// Check all the properties in the sheet and check which ones exist on the
// item and should be updated
for (propertyName of keyColumnOrder) {
if (propertyName === indexKey) {
// Ignore the key itself as that does not get changed it gets
// only used to find the correct row to update
continue;
}
if (inputItem[propertyName] === undefined || inputItem[propertyName] === null) {
// Property does not exist so skip it
continue;
}
// Property exists so add it to the data to update
// Get the column name in which the property data can be found
updateColumnName = this.getColumnWithOffset(
rangeStartSplit[1],
keyColumnOrder.indexOf(propertyName),
);
2019-06-23 03:35:23 -07:00
updateData.push({
range: `${sheet ? sheet + '!' : ''}${updateColumnName}${updateRowIndex}`,
values: [[inputItem[propertyName] as string]],
2019-06-23 03:35:23 -07:00
});
}
}
return this.batchUpdate(updateData, valueInputMode);
2019-06-23 03:35:23 -07:00
}
/**
* Looks for a specific value in a column and if it gets found it returns the whole row
*
* @param {string[][]} inputData Data to to check for lookup value in
* @param {number} keyRowIndex Index of the row which contains the keys
* @param {number} dataStartRowIndex Index of the first row which contains data
* @param {ILookupValues[]} lookupValues The lookup values which decide what data to return
* @param {boolean} [returnAllMatches] Returns all the found matches instead of only the first one
*/
async lookupValues(
inputData: string[][],
keyRowIndex: number,
dataStartRowIndex: number,
lookupValues: ILookupValues[],
returnAllMatches?: boolean,
): Promise<IDataObject[]> {
const keys: string[] = [];
if (keyRowIndex < 0 || dataStartRowIndex < keyRowIndex || keyRowIndex >= inputData.length) {
// The key row does not exist so it is not possible to look up the data
throw new NodeOperationError(this.executeFunctions.getNode(), 'The key row does not exist!');
}
// Create the keys array
for (let columnIndex = 0; columnIndex < inputData[keyRowIndex].length; columnIndex++) {
keys.push(inputData[keyRowIndex][columnIndex]);
}
const returnData = [inputData[keyRowIndex]];
// Standardise values array, if rows is [[]], map it to [['']] (Keep the columns into consideration)
for (let rowIndex = 0; rowIndex < inputData?.length; rowIndex++) {
if (inputData[rowIndex].length === 0) {
for (let i = 0; i < keys.length; i++) {
inputData[rowIndex][i] = '';
}
} else if (inputData[rowIndex].length < keys.length) {
for (let i = 0; i < keys.length; i++) {
if (inputData[rowIndex][i] === undefined) {
inputData[rowIndex].push('');
}
}
}
}
// Loop over all the lookup values and try to find a row to return
let rowIndex: number;
let returnColumnIndex: number;
lookupLoop: for (const lookupValue of lookupValues) {
returnColumnIndex = keys.indexOf(lookupValue.lookupColumn);
if (returnColumnIndex === -1) {
throw new NodeOperationError(
this.executeFunctions.getNode(),
`The column "${lookupValue.lookupColumn}" could not be found!`,
);
}
// Loop over all the items and find the one with the matching value
for (rowIndex = dataStartRowIndex; rowIndex < inputData.length; rowIndex++) {
if (
inputData[rowIndex][returnColumnIndex]?.toString() === lookupValue.lookupValue.toString()
) {
returnData.push(inputData[rowIndex]);
if (returnAllMatches !== true) {
continue lookupLoop;
}
}
}
// If value could not be found add an empty one that the order of
// the returned items stays the same
if (returnAllMatches !== true) {
returnData.push([]);
}
}
return this.structureData(returnData, 1, keys, true);
}
async convertStructuredDataToArray(
inputData: IDataObject[],
range: string,
keyRowIndex: number,
usePathForKeyRow: boolean,
): Promise<string[][]> {
2019-06-23 03:35:23 -07:00
let sheet: string | undefined = undefined;
if (range.includes('!')) {
[sheet, range] = range.split('!');
}
const [startColumn, endColumn] = range.split(':');
2019-06-23 03:35:23 -07:00
let getRange = `${startColumn}${keyRowIndex + 1}:${endColumn}${keyRowIndex + 1}`;
if (sheet !== undefined) {
getRange = `${sheet}!${getRange}`;
}
const keyColumnData = await this.getData(getRange, 'UNFORMATTED_VALUE');
2019-06-23 03:35:23 -07:00
if (keyColumnData === undefined) {
throw new NodeOperationError(
this.executeFunctions.getNode(),
'Could not retrieve the column data!',
);
2019-06-23 03:35:23 -07:00
}
const keyColumnOrder = keyColumnData[0];
const setData: string[][] = [];
let rowData: string[] = [];
inputData.forEach((item) => {
rowData = [];
keyColumnOrder.forEach((key) => {
fix(Google Sheet Node): Fix issue with null values and "Use Header Names as JSON Paths" option (#3395) * fixed Google Sheet 'Use Header Names as JSON Paths' issue when input data contains null values. https://community.n8n.io/t/error-cannot-read-properties-of-null-reading-tostring/14385/2 * :zap: Update URLs to n8n.io API (#3391) * fix(core): Fix migrations on non-public Postgres schema (#3356) * :bug: Fix UM migration * :zap: Account for schema in `search_path` * :fire: Remove unneeded schema refs * :test_tube: Account for alt schema in DB testing * :zap: Add schema to `IncreaseTypeVarcharLimit` * :zap: Set `search_path` in every migration * :zap: Set `search_path` in down migrations * refactor(core): Improve error message * feat(RabbitMQ Trigger Node): Make message acknowledgement and parallel processing configurable (#3385) * feat(RabbitMQ Trigger Node): Make message acknowledgement and concurrent processing configurable * :zap: Make sure that messages do not get executed multiple times * :shirt: Fix lint issue * :bug: Fix issue that for manual executions in "own" mode messages got know acknowledged * :zap: Increment count now that console.log got removed * :zap: Improvements * :zap: Fix default value * :zap: Improve display name * fix(Gmail Node): Fix sending attachments when filesystem mode is used (#3396) * :zap: Minor improvement Co-authored-by: Taha Sönmez <mtssonmez@gmail.com> Co-authored-by: Ben Hesseldieck <1849459+BHesseldieck@users.noreply.github.com> Co-authored-by: Iván Ovejero <ivov.src@gmail.com> Co-authored-by: Jan Oberhauser <jan.oberhauser@gmail.com> Co-authored-by: Jan Oberhauser <janober@users.noreply.github.com> Co-authored-by: Michael Kret <88898367+michael-radency@users.noreply.github.com>
2022-05-30 04:37:34 -07:00
const value = get(item, key) as string;
if (usePathForKeyRow && value !== undefined && value !== null) {
//match by key path
rowData.push(value.toString());
} else if (
!usePathForKeyRow &&
item.hasOwnProperty(key) &&
item[key] !== null &&
item[key] !== undefined
) {
//match by exact key name
rowData.push(item[key]!.toString());
2019-06-23 03:35:23 -07:00
} else {
rowData.push('');
}
});
setData.push(rowData);
});
return setData;
}
}