diff --git a/packages/nodes-base/nodes/Google/Sheet/test/v2/utils/utils.test.ts b/packages/nodes-base/nodes/Google/Sheet/test/v2/utils/utils.test.ts index 52ec8ae4c3..033f7249f8 100644 --- a/packages/nodes-base/nodes/Google/Sheet/test/v2/utils/utils.test.ts +++ b/packages/nodes-base/nodes/Google/Sheet/test/v2/utils/utils.test.ts @@ -304,11 +304,11 @@ describe('Test Google Sheets, lookupValues', () => { const googleSheet = new GoogleSheet('spreadsheetId', fakeExecuteFunction); - const result = await googleSheet.lookupValues( + const result = await googleSheet.lookupValues({ inputData, - 0, - 1, - [ + keyRowIndex: 0, + dataStartRowIndex: 1, + lookupValues: [ { lookupColumn: 'num', lookupValue: '1', @@ -318,9 +318,9 @@ describe('Test Google Sheets, lookupValues', () => { lookupValue: 'foo', }, ], - true, - 'OR', - ); + returnAllMatches: true, + combineFilters: 'OR', + }); expect(result).toBeDefined(); expect(result).toEqual([ @@ -366,11 +366,11 @@ describe('Test Google Sheets, lookupValues', () => { const googleSheet = new GoogleSheet('spreadsheetId', fakeExecuteFunction); - const result = await googleSheet.lookupValues( + const result = await googleSheet.lookupValues({ inputData, - 0, - 1, - [ + keyRowIndex: 0, + dataStartRowIndex: 1, + lookupValues: [ { lookupColumn: 'num', lookupValue: '1', @@ -380,9 +380,9 @@ describe('Test Google Sheets, lookupValues', () => { lookupValue: 'baz', }, ], - true, - 'AND', - ); + returnAllMatches: true, + combineFilters: 'AND', + }); expect(result).toBeDefined(); expect(result).toEqual([ diff --git a/packages/nodes-base/nodes/Google/Sheet/v2/actions/sheet/append.operation.ts b/packages/nodes-base/nodes/Google/Sheet/v2/actions/sheet/append.operation.ts index 14851bf7fa..8b479d4ed6 100644 --- a/packages/nodes-base/nodes/Google/Sheet/v2/actions/sheet/append.operation.ts +++ b/packages/nodes-base/nodes/Google/Sheet/v2/actions/sheet/append.operation.ts @@ -213,7 +213,7 @@ export const description: SheetProperties = [ export async function execute( this: IExecuteFunctions, sheet: GoogleSheet, - sheetName: string, + range: string, sheetId: string, ): Promise { const items = this.getInputData(); @@ -228,57 +228,62 @@ export async function execute( const options = this.getNodeParameter('options', 0, {}); const locationDefine = (options.locationDefine as IDataObject)?.values as IDataObject; - let headerRow = 1; + let keyRowIndex = 1; if (locationDefine?.headerRow) { - headerRow = locationDefine.headerRow as number; + keyRowIndex = locationDefine.headerRow as number; } + const sheetData = await sheet.getData(range, 'FORMATTED_VALUE'); + if (nodeVersion >= 4.4 && dataMode !== 'autoMapInputData') { //not possible to refresh columns when mode is autoMapInputData - const sheetData = await sheet.getData(sheetName, 'FORMATTED_VALUE'); - - if (sheetData?.[headerRow - 1] === undefined) { + if (sheetData?.[keyRowIndex - 1] === undefined) { throw new NodeOperationError( this.getNode(), - `Could not retrieve the column names from row ${headerRow}`, + `Could not retrieve the column names from row ${keyRowIndex}`, ); } const schema = this.getNodeParameter('columns.schema', 0) as ResourceMapperField[]; - checkForSchemaChanges(this.getNode(), sheetData[headerRow - 1], schema); + checkForSchemaChanges(this.getNode(), sheetData[keyRowIndex - 1], schema); } - let setData: IDataObject[] = []; + let inputData: IDataObject[] = []; if (dataMode === 'autoMapInputData') { - setData = await autoMapInputData.call(this, sheetName, sheet, items, options); + inputData = await autoMapInputData.call(this, range, sheet, items, options); } else { - setData = mapFields.call(this, items.length); + inputData = mapFields.call(this, items.length); } - if (setData.length === 0) { + if (inputData.length === 0) { return []; - } else if (options.useAppend) { - await sheet.appendSheetData( - setData, - sheetName, - headerRow, - (options.cellFormat as ValueInputOption) || cellFormatDefault(nodeVersion), - false, - undefined, - undefined, - options.useAppend as boolean, - ); + } + + const valueInputMode = (options.cellFormat as ValueInputOption) || cellFormatDefault(nodeVersion); + const useAppend = options.useAppend as boolean; + + if (options.useAppend) { + await sheet.appendSheetData({ + inputData, + range, + keyRowIndex, + valueInputMode, + useAppend, + }); } else { + //if no trailing empty row exists in the sheet update operation will fail await sheet.appendEmptyRowsOrColumns(sheetId, 1, 0); - await sheet.appendSheetData( - setData, - sheetName, - headerRow, - (options.cellFormat as ValueInputOption) || cellFormatDefault(nodeVersion), - false, - ); + const lastRow = (sheetData ?? []).length + 1; + + await sheet.appendSheetData({ + inputData, + range, + keyRowIndex, + valueInputMode, + lastRow, + }); } if (nodeVersion < 4 || dataMode === 'autoMapInputData') { @@ -288,7 +293,7 @@ export async function execute( }); } else { const returnData: INodeExecutionData[] = []; - for (const [index, entry] of setData.entries()) { + for (const [index, entry] of inputData.entries()) { returnData.push({ json: entry, pairedItem: { item: index }, diff --git a/packages/nodes-base/nodes/Google/Sheet/v2/actions/sheet/appendOrUpdate.operation.ts b/packages/nodes-base/nodes/Google/Sheet/v2/actions/sheet/appendOrUpdate.operation.ts index b6b5b6a4cc..b8782c8432 100644 --- a/packages/nodes-base/nodes/Google/Sheet/v2/actions/sheet/appendOrUpdate.operation.ts +++ b/packages/nodes-base/nodes/Google/Sheet/v2/actions/sheet/appendOrUpdate.operation.ts @@ -246,15 +246,15 @@ export async function execute( const locationDefineOption = (options.locationDefine as IDataObject)?.values as IDataObject; - let headerRow = 0; - let firstDataRow = 1; + let keyRowIndex = 0; + let dataStartRowIndex = 1; if (locationDefineOption) { if (locationDefineOption.headerRow) { - headerRow = parseInt(locationDefineOption.headerRow as string, 10) - 1; + keyRowIndex = parseInt(locationDefineOption.headerRow as string, 10) - 1; } if (locationDefineOption.firstDataRow) { - firstDataRow = parseInt(locationDefineOption.firstDataRow as string, 10) - 1; + dataStartRowIndex = parseInt(locationDefineOption.firstDataRow as string, 10) - 1; } } @@ -267,14 +267,14 @@ export async function execute( const sheetData = (await sheet.getData(sheetName, 'FORMATTED_VALUE')) ?? []; - if (!sheetData[headerRow] && dataMode !== 'autoMapInputData') { + if (!sheetData[keyRowIndex] && dataMode !== 'autoMapInputData') { throw new NodeOperationError( this.getNode(), - `Could not retrieve the column names from row ${headerRow + 1}`, + `Could not retrieve the column names from row ${keyRowIndex + 1}`, ); } - columnNames = sheetData[headerRow] ?? []; + columnNames = sheetData[keyRowIndex] ?? []; if (nodeVersion >= 4.4) { const schema = this.getNodeParameter('columns.schema', 0) as ResourceMapperField[]; @@ -291,13 +291,13 @@ export async function execute( // TODO: Add support for multiple columns to match on in the next overhaul const keyIndex = columnNames.indexOf(columnsToMatchOn[0]); - const columnValues = await sheet.getColumnValues( + const columnValuesList = await sheet.getColumnValues({ range, keyIndex, - firstDataRow, + dataStartRowIndex, valueRenderMode, sheetData, - ); + }); const updateData: ISheetUpdateData[] = []; const appendData: IDataObject[] = []; @@ -321,20 +321,20 @@ export async function execute( for (let i = 0; i < items.length; i++) { if (dataMode === 'nothing') continue; - const data: IDataObject[] = []; + const inputData: IDataObject[] = []; if (dataMode === 'autoMapInputData') { const handlingExtraDataOption = (options.handlingExtraData as string) || 'insertInNewColumn'; if (handlingExtraDataOption === 'ignoreIt') { - data.push(items[i].json); + inputData.push(items[i].json); } if (handlingExtraDataOption === 'error') { Object.keys(items[i].json).forEach((key) => errorOnUnexpectedColumn(key, i)); - data.push(items[i].json); + inputData.push(items[i].json); } if (handlingExtraDataOption === 'insertInNewColumn') { Object.keys(items[i].json).forEach(addNewColumn); - data.push(items[i].json); + inputData.push(items[i].json); } } else { const valueToMatchOn = @@ -364,7 +364,7 @@ export async function execute( return acc; }, {} as IDataObject); fields[columnsToMatchOn[0]] = valueToMatchOn; - data.push(fields); + inputData.push(fields); } else { const mappingValues = this.getNodeParameter('columns.value', i) as IDataObject; if (Object.keys(mappingValues).length === 0) { @@ -379,7 +379,7 @@ export async function execute( mappingValues[key] = ''; } }); - data.push(mappingValues); + inputData.push(mappingValues); mappedValues.push(mappingValues); } } @@ -390,56 +390,60 @@ export async function execute( sheetName, [newColumnNames], (options.cellFormat as ValueInputOption) || cellFormatDefault(nodeVersion), - headerRow + 1, + keyRowIndex + 1, ); columnNames = newColumnNames; - sheetData[headerRow] = newColumnNames; + sheetData[keyRowIndex] = newColumnNames; newColumns.clear(); } - const preparedData = await sheet.prepareDataForUpdateOrUpsert( - data, - columnsToMatchOn[0], + const indexKey = columnsToMatchOn[0]; + + const preparedData = await sheet.prepareDataForUpdateOrUpsert({ + inputData, + indexKey, range, - headerRow, - firstDataRow, + keyRowIndex, + dataStartRowIndex, valueRenderMode, - true, - [columnNames.concat([...newColumns])], - columnValues, - ); + upsert: true, + columnNamesList: [columnNames.concat([...newColumns])], + columnValuesList, + }); updateData.push(...preparedData.updateData); appendData.push(...preparedData.appendData); } + const columnNamesList = [columnNames.concat([...newColumns])]; + if (updateData.length) { await sheet.batchUpdate(updateData, valueInputMode); } if (appendData.length) { const lastRow = sheetData.length + 1; + const useAppend = options.useAppend as boolean; + if (options.useAppend) { - await sheet.appendSheetData( - appendData, + await sheet.appendSheetData({ + inputData: appendData, range, - headerRow + 1, + keyRowIndex: keyRowIndex + 1, valueInputMode, - false, - [columnNames.concat([...newColumns])], + columnNamesList, lastRow, - options.useAppend as boolean, - ); + useAppend, + }); } else { await sheet.appendEmptyRowsOrColumns(sheetId, 1, 0); - await sheet.appendSheetData( - appendData, + await sheet.appendSheetData({ + inputData: appendData, range, - headerRow + 1, + keyRowIndex: keyRowIndex + 1, valueInputMode, - false, - [columnNames.concat([...newColumns])], + columnNamesList, lastRow, - ); + }); } } diff --git a/packages/nodes-base/nodes/Google/Sheet/v2/actions/sheet/read.operation.ts b/packages/nodes-base/nodes/Google/Sheet/v2/actions/sheet/read.operation.ts index 9650e73a4a..63e5a6989f 100644 --- a/packages/nodes-base/nodes/Google/Sheet/v2/actions/sheet/read.operation.ts +++ b/packages/nodes-base/nodes/Google/Sheet/v2/actions/sheet/read.operation.ts @@ -202,10 +202,11 @@ export async function execute( return []; } - const { data, headerRow, firstDataRow } = prepareSheetData( - sheetData, - dataLocationOnSheetOptions, - ); + const { + data, + headerRow: keyRowIndex, + firstDataRow: dataStartRowIndex, + } = prepareSheetData(sheetData, dataLocationOnSheetOptions); let responseData = []; @@ -215,6 +216,8 @@ export async function execute( [], ) as ILookupValues[]; + const inputData = data as string[][]; + if (lookupValues.length) { const returnAllMatches = options.returnAllMatches === 'returnAllMatches' ? true : false; @@ -235,16 +238,16 @@ export async function execute( | 'AND' | 'OR'; - responseData = await sheet.lookupValues( - data as string[][], - headerRow, - firstDataRow, + responseData = await sheet.lookupValues({ + inputData, + keyRowIndex, + dataStartRowIndex, lookupValues, returnAllMatches, combineFilters, - ); + }); } else { - responseData = sheet.structureArrayDataByColumn(data as string[][], headerRow, firstDataRow); + responseData = sheet.structureArrayDataByColumn(inputData, keyRowIndex, dataStartRowIndex); } returnData.push( diff --git a/packages/nodes-base/nodes/Google/Sheet/v2/actions/sheet/update.operation.ts b/packages/nodes-base/nodes/Google/Sheet/v2/actions/sheet/update.operation.ts index bf5fb0c142..2746ef5b90 100644 --- a/packages/nodes-base/nodes/Google/Sheet/v2/actions/sheet/update.operation.ts +++ b/packages/nodes-base/nodes/Google/Sheet/v2/actions/sheet/update.operation.ts @@ -237,15 +237,15 @@ export async function execute( const locationDefineOptions = (options.locationDefine as IDataObject)?.values as IDataObject; - let headerRow = 0; - let firstDataRow = 1; + let keyRowIndex = 0; + let dataStartRowIndex = 1; if (locationDefineOptions) { if (locationDefineOptions.headerRow) { - headerRow = parseInt(locationDefineOptions.headerRow as string, 10) - 1; + keyRowIndex = parseInt(locationDefineOptions.headerRow as string, 10) - 1; } if (locationDefineOptions.firstDataRow) { - firstDataRow = parseInt(locationDefineOptions.firstDataRow as string, 10) - 1; + dataStartRowIndex = parseInt(locationDefineOptions.firstDataRow as string, 10) - 1; } } @@ -253,14 +253,14 @@ export async function execute( const sheetData = await sheet.getData(sheetName, 'FORMATTED_VALUE'); - if (sheetData?.[headerRow] === undefined) { + if (sheetData?.[keyRowIndex] === undefined) { throw new NodeOperationError( this.getNode(), - `Could not retrieve the column names from row ${headerRow + 1}`, + `Could not retrieve the column names from row ${keyRowIndex + 1}`, ); } - columnNames = sheetData[headerRow]; + columnNames = sheetData[keyRowIndex]; if (nodeVersion >= 4.4) { const schema = this.getNodeParameter('columns.schema', 0) as ResourceMapperField[]; @@ -283,13 +283,13 @@ export async function execute( const keyIndex = columnNames.indexOf(columnsToMatchOn[0]); //not used when updating row - const columnValues = await sheet.getColumnValues( + const columnValuesList = await sheet.getColumnValues({ range, keyIndex, - firstDataRow, + dataStartRowIndex, valueRenderMode, sheetData, - ); + }); const updateData: ISheetUpdateData[] = []; @@ -313,20 +313,20 @@ export async function execute( for (let i = 0; i < items.length; i++) { if (dataMode === 'nothing') continue; - const data: IDataObject[] = []; + const inputData: IDataObject[] = []; if (dataMode === 'autoMapInputData') { const handlingExtraDataOption = (options.handlingExtraData as string) || 'insertInNewColumn'; if (handlingExtraDataOption === 'ignoreIt') { - data.push(items[i].json); + inputData.push(items[i].json); } if (handlingExtraDataOption === 'error' && columnsToMatchOn[0] !== 'row_number') { Object.keys(items[i].json).forEach((key) => errorOnUnexpectedColumn(key, i)); - data.push(items[i].json); + inputData.push(items[i].json); } if (handlingExtraDataOption === 'insertInNewColumn' && columnsToMatchOn[0] !== 'row_number') { Object.keys(items[i].json).forEach(addNewColumn); - data.push(items[i].json); + inputData.push(items[i].json); } } else { const valueToMatchOn = @@ -360,7 +360,7 @@ export async function execute( fields[columnsToMatchOn[0]] = valueToMatchOn; - data.push(fields); + inputData.push(fields); } else { const mappingValues = this.getNodeParameter('columns.value', i) as IDataObject; if (Object.keys(mappingValues).length === 0) { @@ -375,7 +375,7 @@ export async function execute( mappingValues[key] = ''; } }); - data.push(mappingValues); + inputData.push(mappingValues); mappedValues.push(mappingValues); } } @@ -386,29 +386,30 @@ export async function execute( sheetName, [newColumnNames], (options.cellFormat as ValueInputOption) || cellFormatDefault(nodeVersion), - headerRow + 1, + keyRowIndex + 1, ); columnNames = newColumnNames; newColumns.clear(); } let preparedData; + const columnNamesList = [columnNames.concat([...newColumns])]; + if (columnsToMatchOn[0] === 'row_number') { - preparedData = sheet.prepareDataForUpdatingByRowNumber(data, range, [ - columnNames.concat([...newColumns]), - ]); + preparedData = sheet.prepareDataForUpdatingByRowNumber(inputData, range, columnNamesList); } else { - preparedData = await sheet.prepareDataForUpdateOrUpsert( - data, - columnsToMatchOn[0], + const indexKey = columnsToMatchOn[0]; + + preparedData = await sheet.prepareDataForUpdateOrUpsert({ + inputData, + indexKey, range, - headerRow, - firstDataRow, + keyRowIndex, + dataStartRowIndex, valueRenderMode, - false, - [columnNames.concat([...newColumns])], - columnValues, - ); + columnNamesList, + columnValuesList, + }); } updateData.push(...preparedData.updateData); diff --git a/packages/nodes-base/nodes/Google/Sheet/v2/actions/versionDescription.ts b/packages/nodes-base/nodes/Google/Sheet/v2/actions/versionDescription.ts index c18a6c75b8..013b4bd702 100644 --- a/packages/nodes-base/nodes/Google/Sheet/v2/actions/versionDescription.ts +++ b/packages/nodes-base/nodes/Google/Sheet/v2/actions/versionDescription.ts @@ -17,6 +17,16 @@ export const versionDescription: INodeTypeDescription = { }, inputs: ['main'], outputs: ['main'], + hints: [ + { + message: + "Use the 'Use Append' option for greater efficiency if your sheet is uniformly formatted without gaps between columns or rows", + displayCondition: + '={{$parameter["operation"] === "append" && !$parameter["options"]["useAppend"]}}', + whenToDisplay: 'beforeExecution', + location: 'outputPane', + }, + ], credentials: [ { name: 'googleApi', diff --git a/packages/nodes-base/nodes/Google/Sheet/v2/helpers/GoogleSheet.ts b/packages/nodes-base/nodes/Google/Sheet/v2/helpers/GoogleSheet.ts index 993c49a0ff..fa96649464 100644 --- a/packages/nodes-base/nodes/Google/Sheet/v2/helpers/GoogleSheet.ts +++ b/packages/nodes-base/nodes/Google/Sheet/v2/helpers/GoogleSheet.ts @@ -380,16 +380,25 @@ export class GoogleSheet { return keys; } - async appendSheetData( - inputData: IDataObject[], - range: string, - keyRowIndex: number, - valueInputMode: ValueInputOption, - usePathForKeyRow: boolean, - columnNamesList?: string[][], - lastRow?: number, - useAppend?: boolean, - ): Promise { + async appendSheetData({ + inputData, + range, + keyRowIndex, + valueInputMode, + usePathForKeyRow, + columnNamesList, + lastRow, + useAppend, + }: { + inputData: IDataObject[]; + range: string; + keyRowIndex: number; + valueInputMode: ValueInputOption; + usePathForKeyRow?: boolean; + columnNamesList?: string[][]; + lastRow?: number; + useAppend?: boolean; + }): Promise { const data = await this.convertObjectArrayToSheetDataArray( inputData, range, @@ -406,13 +415,19 @@ export class GoogleSheet { return xlsxUtils.encode_col(columnIndex); } - async getColumnValues( - range: string, - keyIndex: number, - dataStartRowIndex: number, - valueRenderMode: ValueRenderOption, - sheetData?: string[][], - ): Promise { + async getColumnValues({ + range, + keyIndex, + dataStartRowIndex, + valueRenderMode, + sheetData, + }: { + range: string; + keyIndex: number; + dataStartRowIndex: number; + valueRenderMode: ValueRenderOption; + sheetData?: string[][]; + }): Promise { let columnValuesList; if (sheetData) { columnValuesList = sheetData.slice(dataStartRowIndex - 1).map((row) => row[keyIndex]); @@ -448,17 +463,27 @@ export class GoogleSheet { * @returns {Promise} * @memberof GoogleSheet */ - async prepareDataForUpdateOrUpsert( - inputData: IDataObject[], - indexKey: string, - range: string, - keyRowIndex: number, - dataStartRowIndex: number, - valueRenderMode: ValueRenderOption, + async prepareDataForUpdateOrUpsert({ + inputData, + indexKey, + range, + keyRowIndex, + dataStartRowIndex, + valueRenderMode, upsert = false, - columnNamesList?: string[][], - columnValuesList?: string[], - ) { + columnNamesList, + columnValuesList, + }: { + inputData: IDataObject[]; + indexKey: string; + range: string; + keyRowIndex: number; + dataStartRowIndex: number; + valueRenderMode: ValueRenderOption; + upsert?: boolean; + columnNamesList?: string[][]; + columnValuesList?: string[]; + }) { const decodedRange = this.getDecodedSheetRange(range); // prettier-ignore const keyRowRange = `${decodedRange.name}!${decodedRange.start?.column || ''}${keyRowIndex + 1}:${decodedRange.end?.column || ''}${keyRowIndex + 1}`; @@ -485,7 +510,7 @@ export class GoogleSheet { const columnValues: Array = columnValuesList || - (await this.getColumnValues(range, keyIndex, dataStartRowIndex, valueRenderMode)); + (await this.getColumnValues({ range, keyIndex, dataStartRowIndex, valueRenderMode })); const updateData: ISheetUpdateData[] = []; const appendData: IDataObject[] = []; @@ -620,14 +645,21 @@ export class GoogleSheet { * @returns {Promise} * @memberof GoogleSheet */ - async lookupValues( - inputData: string[][], - keyRowIndex: number, - dataStartRowIndex: number, - lookupValues: ILookupValues[], - returnAllMatches?: boolean, - combineFilters: 'AND' | 'OR' = 'OR', - ): Promise { + async lookupValues({ + inputData, + keyRowIndex, + dataStartRowIndex, + lookupValues, + returnAllMatches, + combineFilters = 'OR', + }: { + inputData: string[][]; + keyRowIndex: number; + dataStartRowIndex: number; + lookupValues: ILookupValues[]; + returnAllMatches?: boolean; + combineFilters?: 'AND' | 'OR'; + }): Promise { const keys: string[] = []; if (keyRowIndex < 0 || dataStartRowIndex < keyRowIndex || keyRowIndex >= inputData.length) { @@ -740,7 +772,7 @@ export class GoogleSheet { inputData: IDataObject[], range: string, keyRowIndex: number, - usePathForKeyRow: boolean, + usePathForKeyRow?: boolean, columnNamesList?: string[][], emptyValue: string | null = '', ): Promise {