fix(Google Sheets Node): Append fails if cells have some default values added by data validation rules (#9950)

packages/nodes-base/nodes/Google/Sheet/test/v2/utils/utils.test.ts

@@ -304,11 +304,11 @@ describe('Test Google Sheets, lookupValues', () => {
 
 		const googleSheet = new GoogleSheet('spreadsheetId', fakeExecuteFunction);
 
-		const result = await googleSheet.lookupValues(
+		const result = await googleSheet.lookupValues({
 			inputData,
-			0,
+			keyRowIndex: 0,
-			1,
+			dataStartRowIndex: 1,
-			[
+			lookupValues: [
 				{
 					lookupColumn: 'num',
 					lookupValue: '1',
@@ -318,9 +318,9 @@ describe('Test Google Sheets, lookupValues', () => {
 					lookupValue: 'foo',
 				},
 			],
-			true,
+			returnAllMatches: true,
-			'OR',
+			combineFilters: 'OR',
-		);
+		});
 
 		expect(result).toBeDefined();
 		expect(result).toEqual([
@@ -366,11 +366,11 @@ describe('Test Google Sheets, lookupValues', () => {
 
 		const googleSheet = new GoogleSheet('spreadsheetId', fakeExecuteFunction);
 
-		const result = await googleSheet.lookupValues(
+		const result = await googleSheet.lookupValues({
 			inputData,
-			0,
+			keyRowIndex: 0,
-			1,
+			dataStartRowIndex: 1,
-			[
+			lookupValues: [
 				{
 					lookupColumn: 'num',
 					lookupValue: '1',
@@ -380,9 +380,9 @@ describe('Test Google Sheets, lookupValues', () => {
 					lookupValue: 'baz',
 				},
 			],
-			true,
+			returnAllMatches: true,
-			'AND',
+			combineFilters: 'AND',
-		);
+		});
 
 		expect(result).toBeDefined();
 		expect(result).toEqual([

packages/nodes-base/nodes/Google/Sheet/v2/actions/sheet/append.operation.ts

@@ -213,7 +213,7 @@ export const description: SheetProperties = [
 export async function execute(
 	this: IExecuteFunctions,
 	sheet: GoogleSheet,
-	sheetName: string,
+	range: string,
 	sheetId: string,
 ): Promise<INodeExecutionData[]> {
 	const items = this.getInputData();
@@ -228,57 +228,62 @@ export async function execute(
 	const options = this.getNodeParameter('options', 0, {});
 	const locationDefine = (options.locationDefine as IDataObject)?.values as IDataObject;
 
-	let headerRow = 1;
+	let keyRowIndex = 1;
 	if (locationDefine?.headerRow) {
-		headerRow = locationDefine.headerRow as number;
+		keyRowIndex = locationDefine.headerRow as number;
 	}
 
+	const sheetData = await sheet.getData(range, 'FORMATTED_VALUE');
+
 	if (nodeVersion >= 4.4 && dataMode !== 'autoMapInputData') {
 		//not possible to refresh columns when mode is autoMapInputData
-		const sheetData = await sheet.getData(sheetName, 'FORMATTED_VALUE');
+		if (sheetData?.[keyRowIndex - 1] === undefined) {
-
-		if (sheetData?.[headerRow - 1] === undefined) {
 			throw new NodeOperationError(
 				this.getNode(),
-				`Could not retrieve the column names from row ${headerRow}`,
+				`Could not retrieve the column names from row ${keyRowIndex}`,
 			);
 		}
 
 		const schema = this.getNodeParameter('columns.schema', 0) as ResourceMapperField[];
-		checkForSchemaChanges(this.getNode(), sheetData[headerRow - 1], schema);
+		checkForSchemaChanges(this.getNode(), sheetData[keyRowIndex - 1], schema);
 	}
 
-	let setData: IDataObject[] = [];
+	let inputData: IDataObject[] = [];
 
 	if (dataMode === 'autoMapInputData') {
-		setData = await autoMapInputData.call(this, sheetName, sheet, items, options);
+		inputData = await autoMapInputData.call(this, range, sheet, items, options);
 	} else {
-		setData = mapFields.call(this, items.length);
+		inputData = mapFields.call(this, items.length);
 	}
 
-	if (setData.length === 0) {
+	if (inputData.length === 0) {
 		return [];
-	} else if (options.useAppend) {
+	}
-		await sheet.appendSheetData(
+
-			setData,
+	const valueInputMode = (options.cellFormat as ValueInputOption) || cellFormatDefault(nodeVersion);
-			sheetName,
+	const useAppend = options.useAppend as boolean;
-			headerRow,
+
-			(options.cellFormat as ValueInputOption) || cellFormatDefault(nodeVersion),
+	if (options.useAppend) {
-			false,
+		await sheet.appendSheetData({
-			undefined,
+			inputData,
-			undefined,
+			range,
-			options.useAppend as boolean,
+			keyRowIndex,
-		);
+			valueInputMode,
+			useAppend,
+		});
 	} else {
+		//if no trailing empty row exists in the sheet update operation will fail
 		await sheet.appendEmptyRowsOrColumns(sheetId, 1, 0);
 
-		await sheet.appendSheetData(
+		const lastRow = (sheetData ?? []).length + 1;
-			setData,
+
-			sheetName,
+		await sheet.appendSheetData({
-			headerRow,
+			inputData,
-			(options.cellFormat as ValueInputOption) || cellFormatDefault(nodeVersion),
+			range,
-			false,
+			keyRowIndex,
-		);
+			valueInputMode,
+			lastRow,
+		});
 	}
 
 	if (nodeVersion < 4 || dataMode === 'autoMapInputData') {
@@ -288,7 +293,7 @@ export async function execute(
 		});
 	} else {
 		const returnData: INodeExecutionData[] = [];
-		for (const [index, entry] of setData.entries()) {
+		for (const [index, entry] of inputData.entries()) {
 			returnData.push({
 				json: entry,
 				pairedItem: { item: index },

packages/nodes-base/nodes/Google/Sheet/v2/actions/sheet/appendOrUpdate.operation.ts

@@ -246,15 +246,15 @@ export async function execute(
 
 	const locationDefineOption = (options.locationDefine as IDataObject)?.values as IDataObject;
 
-	let headerRow = 0;
+	let keyRowIndex = 0;
-	let firstDataRow = 1;
+	let dataStartRowIndex = 1;
 
 	if (locationDefineOption) {
 		if (locationDefineOption.headerRow) {
-			headerRow = parseInt(locationDefineOption.headerRow as string, 10) - 1;
+			keyRowIndex = parseInt(locationDefineOption.headerRow as string, 10) - 1;
 		}
 		if (locationDefineOption.firstDataRow) {
-			firstDataRow = parseInt(locationDefineOption.firstDataRow as string, 10) - 1;
+			dataStartRowIndex = parseInt(locationDefineOption.firstDataRow as string, 10) - 1;
 		}
 	}
 
@@ -267,14 +267,14 @@ export async function execute(
 
 	const sheetData = (await sheet.getData(sheetName, 'FORMATTED_VALUE')) ?? [];
 
-	if (!sheetData[headerRow] && dataMode !== 'autoMapInputData') {
+	if (!sheetData[keyRowIndex] && dataMode !== 'autoMapInputData') {
 		throw new NodeOperationError(
 			this.getNode(),
-			`Could not retrieve the column names from row ${headerRow + 1}`,
+			`Could not retrieve the column names from row ${keyRowIndex + 1}`,
 		);
 	}
 
-	columnNames = sheetData[headerRow] ?? [];
+	columnNames = sheetData[keyRowIndex] ?? [];
 
 	if (nodeVersion >= 4.4) {
 		const schema = this.getNodeParameter('columns.schema', 0) as ResourceMapperField[];
@@ -291,13 +291,13 @@ export async function execute(
 	// TODO: Add support for multiple columns to match on in the next overhaul
 	const keyIndex = columnNames.indexOf(columnsToMatchOn[0]);
 
-	const columnValues = await sheet.getColumnValues(
+	const columnValuesList = await sheet.getColumnValues({
 		range,
 		keyIndex,
-		firstDataRow,
+		dataStartRowIndex,
 		valueRenderMode,
 		sheetData,
-	);
+	});
 
 	const updateData: ISheetUpdateData[] = [];
 	const appendData: IDataObject[] = [];
@@ -321,20 +321,20 @@ export async function execute(
 	for (let i = 0; i < items.length; i++) {
 		if (dataMode === 'nothing') continue;
 
-		const data: IDataObject[] = [];
+		const inputData: IDataObject[] = [];
 
 		if (dataMode === 'autoMapInputData') {
 			const handlingExtraDataOption = (options.handlingExtraData as string) || 'insertInNewColumn';
 			if (handlingExtraDataOption === 'ignoreIt') {
-				data.push(items[i].json);
+				inputData.push(items[i].json);
 			}
 			if (handlingExtraDataOption === 'error') {
 				Object.keys(items[i].json).forEach((key) => errorOnUnexpectedColumn(key, i));
-				data.push(items[i].json);
+				inputData.push(items[i].json);
 			}
 			if (handlingExtraDataOption === 'insertInNewColumn') {
 				Object.keys(items[i].json).forEach(addNewColumn);
-				data.push(items[i].json);
+				inputData.push(items[i].json);
 			}
 		} else {
 			const valueToMatchOn =
@@ -364,7 +364,7 @@ export async function execute(
 					return acc;
 				}, {} as IDataObject);
 				fields[columnsToMatchOn[0]] = valueToMatchOn;
-				data.push(fields);
+				inputData.push(fields);
 			} else {
 				const mappingValues = this.getNodeParameter('columns.value', i) as IDataObject;
 				if (Object.keys(mappingValues).length === 0) {
@@ -379,7 +379,7 @@ export async function execute(
 						mappingValues[key] = '';
 					}
 				});
-				data.push(mappingValues);
+				inputData.push(mappingValues);
 				mappedValues.push(mappingValues);
 			}
 		}
@@ -390,56 +390,60 @@ export async function execute(
 				sheetName,
 				[newColumnNames],
 				(options.cellFormat as ValueInputOption) || cellFormatDefault(nodeVersion),
-				headerRow + 1,
+				keyRowIndex + 1,
 			);
 			columnNames = newColumnNames;
-			sheetData[headerRow] = newColumnNames;
+			sheetData[keyRowIndex] = newColumnNames;
 			newColumns.clear();
 		}
 
-		const preparedData = await sheet.prepareDataForUpdateOrUpsert(
+		const indexKey = columnsToMatchOn[0];
-			data,
+
-			columnsToMatchOn[0],
+		const preparedData = await sheet.prepareDataForUpdateOrUpsert({
+			inputData,
+			indexKey,
 			range,
-			headerRow,
+			keyRowIndex,
-			firstDataRow,
+			dataStartRowIndex,
 			valueRenderMode,
-			true,
+			upsert: true,
-			[columnNames.concat([...newColumns])],
+			columnNamesList: [columnNames.concat([...newColumns])],
-			columnValues,
+			columnValuesList,
-		);
+		});
 
 		updateData.push(...preparedData.updateData);
 		appendData.push(...preparedData.appendData);
 	}
 
+	const columnNamesList = [columnNames.concat([...newColumns])];
+
 	if (updateData.length) {
 		await sheet.batchUpdate(updateData, valueInputMode);
 	}
 	if (appendData.length) {
 		const lastRow = sheetData.length + 1;
+		const useAppend = options.useAppend as boolean;
+
 		if (options.useAppend) {
-			await sheet.appendSheetData(
+			await sheet.appendSheetData({
-				appendData,
+				inputData: appendData,
 				range,
-				headerRow + 1,
+				keyRowIndex: keyRowIndex + 1,
 				valueInputMode,
-				false,
+				columnNamesList,
-				[columnNames.concat([...newColumns])],
 				lastRow,
-				options.useAppend as boolean,
+				useAppend,
-			);
+			});
 		} else {
 			await sheet.appendEmptyRowsOrColumns(sheetId, 1, 0);
-			await sheet.appendSheetData(
+			await sheet.appendSheetData({
-				appendData,
+				inputData: appendData,
 				range,
-				headerRow + 1,
+				keyRowIndex: keyRowIndex + 1,
 				valueInputMode,
-				false,
+				columnNamesList,
-				[columnNames.concat([...newColumns])],
 				lastRow,
-			);
+			});
 		}
 	}
 

packages/nodes-base/nodes/Google/Sheet/v2/actions/sheet/read.operation.ts

@@ -202,10 +202,11 @@ export async function execute(
 			return [];
 		}
 
-		const { data, headerRow, firstDataRow } = prepareSheetData(
+		const {
-			sheetData,
+			data,
-			dataLocationOnSheetOptions,
+			headerRow: keyRowIndex,
-		);
+			firstDataRow: dataStartRowIndex,
+		} = prepareSheetData(sheetData, dataLocationOnSheetOptions);
 
 		let responseData = [];
 
@@ -215,6 +216,8 @@ export async function execute(
 			[],
 		) as ILookupValues[];
 
+		const inputData = data as string[][];
+
 		if (lookupValues.length) {
 			const returnAllMatches = options.returnAllMatches === 'returnAllMatches' ? true : false;
 
@@ -235,16 +238,16 @@ export async function execute(
 				| 'AND'
 				| 'OR';
 
-			responseData = await sheet.lookupValues(
+			responseData = await sheet.lookupValues({
-				data as string[][],
+				inputData,
-				headerRow,
+				keyRowIndex,
-				firstDataRow,
+				dataStartRowIndex,
 				lookupValues,
 				returnAllMatches,
 				combineFilters,
-			);
+			});
 		} else {
-			responseData = sheet.structureArrayDataByColumn(data as string[][], headerRow, firstDataRow);
+			responseData = sheet.structureArrayDataByColumn(inputData, keyRowIndex, dataStartRowIndex);
 		}
 
 		returnData.push(

packages/nodes-base/nodes/Google/Sheet/v2/actions/sheet/update.operation.ts

@@ -237,15 +237,15 @@ export async function execute(
 
 	const locationDefineOptions = (options.locationDefine as IDataObject)?.values as IDataObject;
 
-	let headerRow = 0;
+	let keyRowIndex = 0;
-	let firstDataRow = 1;
+	let dataStartRowIndex = 1;
 
 	if (locationDefineOptions) {
 		if (locationDefineOptions.headerRow) {
-			headerRow = parseInt(locationDefineOptions.headerRow as string, 10) - 1;
+			keyRowIndex = parseInt(locationDefineOptions.headerRow as string, 10) - 1;
 		}
 		if (locationDefineOptions.firstDataRow) {
-			firstDataRow = parseInt(locationDefineOptions.firstDataRow as string, 10) - 1;
+			dataStartRowIndex = parseInt(locationDefineOptions.firstDataRow as string, 10) - 1;
 		}
 	}
 
@@ -253,14 +253,14 @@ export async function execute(
 
 	const sheetData = await sheet.getData(sheetName, 'FORMATTED_VALUE');
 
-	if (sheetData?.[headerRow] === undefined) {
+	if (sheetData?.[keyRowIndex] === undefined) {
 		throw new NodeOperationError(
 			this.getNode(),
-			`Could not retrieve the column names from row ${headerRow + 1}`,
+			`Could not retrieve the column names from row ${keyRowIndex + 1}`,
 		);
 	}
 
-	columnNames = sheetData[headerRow];
+	columnNames = sheetData[keyRowIndex];
 
 	if (nodeVersion >= 4.4) {
 		const schema = this.getNodeParameter('columns.schema', 0) as ResourceMapperField[];
@@ -283,13 +283,13 @@ export async function execute(
 	const keyIndex = columnNames.indexOf(columnsToMatchOn[0]);
 
 	//not used when updating row
-	const columnValues = await sheet.getColumnValues(
+	const columnValuesList = await sheet.getColumnValues({
 		range,
 		keyIndex,
-		firstDataRow,
+		dataStartRowIndex,
 		valueRenderMode,
 		sheetData,
-	);
+	});
 
 	const updateData: ISheetUpdateData[] = [];
 
@@ -313,20 +313,20 @@ export async function execute(
 	for (let i = 0; i < items.length; i++) {
 		if (dataMode === 'nothing') continue;
 
-		const data: IDataObject[] = [];
+		const inputData: IDataObject[] = [];
 
 		if (dataMode === 'autoMapInputData') {
 			const handlingExtraDataOption = (options.handlingExtraData as string) || 'insertInNewColumn';
 			if (handlingExtraDataOption === 'ignoreIt') {
-				data.push(items[i].json);
+				inputData.push(items[i].json);
 			}
 			if (handlingExtraDataOption === 'error' && columnsToMatchOn[0] !== 'row_number') {
 				Object.keys(items[i].json).forEach((key) => errorOnUnexpectedColumn(key, i));
-				data.push(items[i].json);
+				inputData.push(items[i].json);
 			}
 			if (handlingExtraDataOption === 'insertInNewColumn' && columnsToMatchOn[0] !== 'row_number') {
 				Object.keys(items[i].json).forEach(addNewColumn);
-				data.push(items[i].json);
+				inputData.push(items[i].json);
 			}
 		} else {
 			const valueToMatchOn =
@@ -360,7 +360,7 @@ export async function execute(
 
 				fields[columnsToMatchOn[0]] = valueToMatchOn;
 
-				data.push(fields);
+				inputData.push(fields);
 			} else {
 				const mappingValues = this.getNodeParameter('columns.value', i) as IDataObject;
 				if (Object.keys(mappingValues).length === 0) {
@@ -375,7 +375,7 @@ export async function execute(
 						mappingValues[key] = '';
 					}
 				});
-				data.push(mappingValues);
+				inputData.push(mappingValues);
 				mappedValues.push(mappingValues);
 			}
 		}
@@ -386,29 +386,30 @@ export async function execute(
 				sheetName,
 				[newColumnNames],
 				(options.cellFormat as ValueInputOption) || cellFormatDefault(nodeVersion),
-				headerRow + 1,
+				keyRowIndex + 1,
 			);
 			columnNames = newColumnNames;
 			newColumns.clear();
 		}
 
 		let preparedData;
+		const columnNamesList = [columnNames.concat([...newColumns])];
+
 		if (columnsToMatchOn[0] === 'row_number') {
-			preparedData = sheet.prepareDataForUpdatingByRowNumber(data, range, [
+			preparedData = sheet.prepareDataForUpdatingByRowNumber(inputData, range, columnNamesList);
-				columnNames.concat([...newColumns]),
-			]);
 		} else {
-			preparedData = await sheet.prepareDataForUpdateOrUpsert(
+			const indexKey = columnsToMatchOn[0];
-				data,
+
-				columnsToMatchOn[0],
+			preparedData = await sheet.prepareDataForUpdateOrUpsert({
+				inputData,
+				indexKey,
 				range,
-				headerRow,
+				keyRowIndex,
-				firstDataRow,
+				dataStartRowIndex,
 				valueRenderMode,
-				false,
+				columnNamesList,
-				[columnNames.concat([...newColumns])],
+				columnValuesList,
-				columnValues,
+			});
-			);
 		}
 
 		updateData.push(...preparedData.updateData);

packages/nodes-base/nodes/Google/Sheet/v2/actions/versionDescription.ts

@@ -17,6 +17,16 @@ export const versionDescription: INodeTypeDescription = {
 	},
 	inputs: ['main'],
 	outputs: ['main'],
+	hints: [
+		{
+			message:
+				"Use the 'Use Append' option for greater efficiency if your sheet is uniformly formatted without gaps between columns or rows",
+			displayCondition:
+				'={{$parameter["operation"] === "append" && !$parameter["options"]["useAppend"]}}',
+			whenToDisplay: 'beforeExecution',
+			location: 'outputPane',
+		},
+	],
 	credentials: [
 		{
 			name: 'googleApi',

packages/nodes-base/nodes/Google/Sheet/v2/helpers/GoogleSheet.ts

@@ -380,16 +380,25 @@ export class GoogleSheet {
 		return keys;
 	}
 
-	async appendSheetData(
+	async appendSheetData({
-		inputData: IDataObject[],
+		inputData,
-		range: string,
+		range,
-		keyRowIndex: number,
+		keyRowIndex,
-		valueInputMode: ValueInputOption,
+		valueInputMode,
-		usePathForKeyRow: boolean,
+		usePathForKeyRow,
-		columnNamesList?: string[][],
+		columnNamesList,
-		lastRow?: number,
+		lastRow,
-		useAppend?: boolean,
+		useAppend,
-	): Promise<string[][]> {
+	}: {
+		inputData: IDataObject[];
+		range: string;
+		keyRowIndex: number;
+		valueInputMode: ValueInputOption;
+		usePathForKeyRow?: boolean;
+		columnNamesList?: string[][];
+		lastRow?: number;
+		useAppend?: boolean;
+	}): Promise<string[][]> {
 		const data = await this.convertObjectArrayToSheetDataArray(
 			inputData,
 			range,
@@ -406,13 +415,19 @@ export class GoogleSheet {
 		return xlsxUtils.encode_col(columnIndex);
 	}
 
-	async getColumnValues(
+	async getColumnValues({
-		range: string,
+		range,
-		keyIndex: number,
+		keyIndex,
-		dataStartRowIndex: number,
+		dataStartRowIndex,
-		valueRenderMode: ValueRenderOption,
+		valueRenderMode,
-		sheetData?: string[][],
+		sheetData,
-	): Promise<string[]> {
+	}: {
+		range: string;
+		keyIndex: number;
+		dataStartRowIndex: number;
+		valueRenderMode: ValueRenderOption;
+		sheetData?: string[][];
+	}): Promise<string[]> {
 		let columnValuesList;
 		if (sheetData) {
 			columnValuesList = sheetData.slice(dataStartRowIndex - 1).map((row) => row[keyIndex]);
@@ -448,17 +463,27 @@ export class GoogleSheet {
 	 * @returns {Promise<string[][]>}
 	 * @memberof GoogleSheet
 	 */
-	async prepareDataForUpdateOrUpsert(
+	async prepareDataForUpdateOrUpsert({
-		inputData: IDataObject[],
+		inputData,
-		indexKey: string,
+		indexKey,
-		range: string,
+		range,
-		keyRowIndex: number,
+		keyRowIndex,
-		dataStartRowIndex: number,
+		dataStartRowIndex,
-		valueRenderMode: ValueRenderOption,
+		valueRenderMode,
 		upsert = false,
-		columnNamesList?: string[][],
+		columnNamesList,
-		columnValuesList?: string[],
+		columnValuesList,
-	) {
+	}: {
+		inputData: IDataObject[];
+		indexKey: string;
+		range: string;
+		keyRowIndex: number;
+		dataStartRowIndex: number;
+		valueRenderMode: ValueRenderOption;
+		upsert?: boolean;
+		columnNamesList?: string[][];
+		columnValuesList?: string[];
+	}) {
 		const decodedRange = this.getDecodedSheetRange(range);
 		// prettier-ignore
 		const	keyRowRange = `${decodedRange.name}!${decodedRange.start?.column || ''}${keyRowIndex + 1}:${decodedRange.end?.column || ''}${keyRowIndex + 1}`;
@@ -485,7 +510,7 @@ export class GoogleSheet {
 
 		const columnValues: Array<string | number> =
 			columnValuesList ||
-			(await this.getColumnValues(range, keyIndex, dataStartRowIndex, valueRenderMode));
+			(await this.getColumnValues({ range, keyIndex, dataStartRowIndex, valueRenderMode }));
 
 		const updateData: ISheetUpdateData[] = [];
 		const appendData: IDataObject[] = [];
@@ -620,14 +645,21 @@ export class GoogleSheet {
 	 * @returns {Promise<IDataObject[]>}
 	 * @memberof GoogleSheet
 	 */
-	async lookupValues(
+	async lookupValues({
-		inputData: string[][],
+		inputData,
-		keyRowIndex: number,
+		keyRowIndex,
-		dataStartRowIndex: number,
+		dataStartRowIndex,
-		lookupValues: ILookupValues[],
+		lookupValues,
-		returnAllMatches?: boolean,
+		returnAllMatches,
-		combineFilters: 'AND' | 'OR' = 'OR',
+		combineFilters = 'OR',
-	): Promise<IDataObject[]> {
+	}: {
+		inputData: string[][];
+		keyRowIndex: number;
+		dataStartRowIndex: number;
+		lookupValues: ILookupValues[];
+		returnAllMatches?: boolean;
+		combineFilters?: 'AND' | 'OR';
+	}): Promise<IDataObject[]> {
 		const keys: string[] = [];
 
 		if (keyRowIndex < 0 || dataStartRowIndex < keyRowIndex || keyRowIndex >= inputData.length) {
@@ -740,7 +772,7 @@ export class GoogleSheet {
 		inputData: IDataObject[],
 		range: string,
 		keyRowIndex: number,
-		usePathForKeyRow: boolean,
+		usePathForKeyRow?: boolean,
 		columnNamesList?: string[][],
 		emptyValue: string | null = '',
 	): Promise<string[][]> {