From 8625a71f15de0f0c3f9f167d8a2a7bcbd46227d8 Mon Sep 17 00:00:00 2001 From: Tate Thurston Date: Thu, 29 Feb 2024 08:45:44 -0800 Subject: [PATCH] Add export as csv (#4034) * Add export as csv Resolves 2183. * collect over paginated data * refactor * add tests * parameterize pageSize (limit) * use pageInfo for onCompleted callback * json column variable naming * omit relations from csv exports --- package.json | 1 + .../object-record/hooks/useFindManyRecords.ts | 52 +++--- .../hooks/useLoadRecordIndexTable.ts | 30 ++-- .../components/RecordIndexOptionsDropdown.tsx | 36 ++-- .../RecordIndexOptionsDropdownContent.tsx | 15 ++ .../__tests__/useExportTableData.test.ts | 108 ++++++++++++ .../options/hooks/useExportTableData.ts | 165 ++++++++++++++++++ .../src/modules/ui/display/icon/index.ts | 1 + yarn.lock | 25 +++ 9 files changed, 378 insertions(+), 55 deletions(-) create mode 100644 packages/twenty-front/src/modules/object-record/record-index/options/hooks/__tests__/useExportTableData.test.ts create mode 100644 packages/twenty-front/src/modules/object-record/record-index/options/hooks/useExportTableData.ts diff --git a/package.json b/package.json index 022b741ae7..d0c8e0c6ce 100644 --- a/package.json +++ b/package.json @@ -101,6 +101,7 @@ "jest-mock-extended": "^3.0.4", "js-cookie": "^3.0.5", "js-levenshtein": "^1.1.6", + "json-2-csv": "^5.4.0", "jsonwebtoken": "^9.0.0", "libphonenumber-js": "^1.10.26", "lodash.camelcase": "^4.3.0", diff --git a/packages/twenty-front/src/modules/object-record/hooks/useFindManyRecords.ts b/packages/twenty-front/src/modules/object-record/hooks/useFindManyRecords.ts index 78aafcf682..d3651828b8 100644 --- a/packages/twenty-front/src/modules/object-record/hooks/useFindManyRecords.ts +++ b/packages/twenty-front/src/modules/object-record/hooks/useFindManyRecords.ts @@ -34,7 +34,10 @@ export const useFindManyRecords = ({ depth, }: ObjectMetadataItemIdentifier & ObjectRecordQueryVariables & { - onCompleted?: (data: ObjectRecordConnection) => void; + onCompleted?: ( + data: ObjectRecordConnection, + pageInfo: ObjectRecordConnection['pageInfo'], + ) => void; skip?: boolean; useRecordsWithoutConnection?: boolean; depth?: number; @@ -77,15 +80,13 @@ export const useFindManyRecords = ({ orderBy, }, onCompleted: (data) => { - onCompleted?.(data[objectMetadataItem.namePlural]); + const pageInfo = data?.[objectMetadataItem.namePlural]?.pageInfo; + + onCompleted?.(data[objectMetadataItem.namePlural], pageInfo); if (data?.[objectMetadataItem.namePlural]) { - setLastCursor( - data?.[objectMetadataItem.namePlural]?.pageInfo.endCursor ?? '', - ); - setHasNextPage( - data?.[objectMetadataItem.namePlural]?.pageInfo.hasNextPage ?? false, - ); + setLastCursor(pageInfo.endCursor ?? ''); + setHasNextPage(pageInfo.hasNextPage ?? false); } }, onError: (error) => { @@ -128,27 +129,26 @@ export const useFindManyRecords = ({ ]); } + const pageInfo = + fetchMoreResult?.[objectMetadataItem.namePlural]?.pageInfo; if (data?.[objectMetadataItem.namePlural]) { - setLastCursor( - fetchMoreResult?.[objectMetadataItem.namePlural]?.pageInfo - .endCursor ?? '', - ); - setHasNextPage( - fetchMoreResult?.[objectMetadataItem.namePlural]?.pageInfo - .hasNextPage ?? false, - ); + setLastCursor(pageInfo.endCursor ?? ''); + setHasNextPage(pageInfo.hasNextPage ?? false); } - onCompleted?.({ - __typename: `${capitalize( - objectMetadataItem.nameSingular, - )}Connection`, - edges: newEdges, - pageInfo: - fetchMoreResult?.[objectMetadataItem.namePlural].pageInfo, - totalCount: - fetchMoreResult?.[objectMetadataItem.namePlural].totalCount, - }); + onCompleted?.( + { + __typename: `${capitalize( + objectMetadataItem.nameSingular, + )}Connection`, + edges: newEdges, + pageInfo: + fetchMoreResult?.[objectMetadataItem.namePlural].pageInfo, + totalCount: + fetchMoreResult?.[objectMetadataItem.namePlural].totalCount, + }, + pageInfo, + ); return Object.assign({}, prev, { [objectMetadataItem.namePlural]: { diff --git a/packages/twenty-front/src/modules/object-record/record-index/hooks/useLoadRecordIndexTable.ts b/packages/twenty-front/src/modules/object-record/record-index/hooks/useLoadRecordIndexTable.ts index e447084e7a..48d9f64ef8 100644 --- a/packages/twenty-front/src/modules/object-record/record-index/hooks/useLoadRecordIndexTable.ts +++ b/packages/twenty-front/src/modules/object-record/record-index/hooks/useLoadRecordIndexTable.ts @@ -10,26 +10,17 @@ import { SIGN_IN_BACKGROUND_MOCK_COMPANIES } from '@/sign-in-background-mock/con import { useFindManyRecords } from '../../hooks/useFindManyRecords'; -export const useLoadRecordIndexTable = (objectNameSingular: string) => { - const { setRecordTableData, setIsRecordTableInitialLoading } = - useRecordTable(); - - const currentWorkspace = useRecoilValue(currentWorkspaceState); +export const useFindManyParams = (objectNameSingular: string) => { const { objectMetadataItem } = useObjectMetadataItem({ objectNameSingular, }); - const { - getTableFiltersState, - getTableSortsState, - getTableLastRowVisibleState, - } = useRecordTableStates(); + const { getTableFiltersState, getTableSortsState } = useRecordTableStates(); const tableFilters = useRecoilValue(getTableFiltersState()); const tableSorts = useRecoilValue(getTableSortsState()); - const setLastRowVisible = useSetRecoilState(getTableLastRowVisibleState()); - const requestFilters = turnObjectDropdownFilterIntoQueryFilter( + const filter = turnObjectDropdownFilterIntoQueryFilter( tableFilters, objectMetadataItem?.fields ?? [], ); @@ -39,6 +30,17 @@ export const useLoadRecordIndexTable = (objectNameSingular: string) => { objectMetadataItem?.fields ?? [], ); + return { objectNameSingular, filter, orderBy }; +}; + +export const useLoadRecordIndexTable = (objectNameSingular: string) => { + const { setRecordTableData, setIsRecordTableInitialLoading } = + useRecordTable(); + const { getTableLastRowVisibleState } = useRecordTableStates(); + const setLastRowVisible = useSetRecoilState(getTableLastRowVisibleState()); + const currentWorkspace = useRecoilValue(currentWorkspaceState); + const params = useFindManyParams(objectNameSingular); + const { records, loading, @@ -46,9 +48,7 @@ export const useLoadRecordIndexTable = (objectNameSingular: string) => { fetchMoreRecords, queryStateIdentifier, } = useFindManyRecords({ - objectNameSingular, - filter: requestFilters, - orderBy, + ...params, onCompleted: () => { setLastRowVisible(false); setIsRecordTableInitialLoading(false); diff --git a/packages/twenty-front/src/modules/object-record/record-index/options/components/RecordIndexOptionsDropdown.tsx b/packages/twenty-front/src/modules/object-record/record-index/options/components/RecordIndexOptionsDropdown.tsx index 8d420f42cd..d79427c0fe 100644 --- a/packages/twenty-front/src/modules/object-record/record-index/options/components/RecordIndexOptionsDropdown.tsx +++ b/packages/twenty-front/src/modules/object-record/record-index/options/components/RecordIndexOptionsDropdown.tsx @@ -1,6 +1,8 @@ import { RecordIndexOptionsDropdownButton } from '@/object-record/record-index/options/components/RecordIndexOptionsDropdownButton'; import { RecordIndexOptionsDropdownContent } from '@/object-record/record-index/options/components/RecordIndexOptionsDropdownContent'; import { RECORD_INDEX_OPTIONS_DROPDOWN_ID } from '@/object-record/record-index/options/constants/RecordIndexOptionsDropdownId'; +import { useRecordTableStates } from '@/object-record/record-table/hooks/internal/useRecordTableStates'; +import { RecordTableScope } from '@/object-record/record-table/scopes/RecordTableScope'; import { TableOptionsHotkeyScope } from '@/object-record/record-table/types/TableOptionsHotkeyScope'; import { Dropdown } from '@/ui/layout/dropdown/components/Dropdown'; import { useViewBar } from '@/views/hooks/useViewBar'; @@ -18,21 +20,27 @@ export const RecordIndexOptionsDropdown = ({ viewType, }: RecordIndexOptionsDropdownProps) => { const { setViewEditMode } = useViewBar(); + const { scopeId } = useRecordTableStates(recordIndexId); return ( - } - dropdownHotkeyScope={{ scope: TableOptionsHotkeyScope.Dropdown }} - dropdownOffset={{ y: 8 }} - dropdownComponents={ - - } - onClickOutside={() => setViewEditMode('none')} - /> + false} + > + } + dropdownHotkeyScope={{ scope: TableOptionsHotkeyScope.Dropdown }} + dropdownOffset={{ y: 8 }} + dropdownComponents={ + + } + onClickOutside={() => setViewEditMode('none')} + /> + ); }; diff --git a/packages/twenty-front/src/modules/object-record/record-index/options/components/RecordIndexOptionsDropdownContent.tsx b/packages/twenty-front/src/modules/object-record/record-index/options/components/RecordIndexOptionsDropdownContent.tsx index 59831da293..d341b88b46 100644 --- a/packages/twenty-front/src/modules/object-record/record-index/options/components/RecordIndexOptionsDropdownContent.tsx +++ b/packages/twenty-front/src/modules/object-record/record-index/options/components/RecordIndexOptionsDropdownContent.tsx @@ -10,6 +10,7 @@ import { useSpreadsheetRecordImport } from '@/object-record/spreadsheet-import/u import { IconBaselineDensitySmall, IconChevronLeft, + IconFileExport, IconFileImport, IconTag, } from '@/ui/display/icon'; @@ -26,6 +27,8 @@ import { useViewScopedStates } from '@/views/hooks/internal/useViewScopedStates' import { useViewBar } from '@/views/hooks/useViewBar'; import { ViewType } from '@/views/types/ViewType'; +import { useExportTableData } from '../hooks/useExportTableData'; + type RecordIndexOptionsMenu = 'fields'; type RecordIndexOptionsDropdownContentProps = { @@ -119,6 +122,13 @@ export const RecordIndexOptionsDropdownContent = ({ const { openRecordSpreadsheetImport } = useSpreadsheetRecordImport(objectNameSingular); + const { progress, download } = useExportTableData({ + delayMs: 100, + filename: `${objectNameSingular}.csv`, + objectNameSingular, + recordIndexId, + }); + return ( <> {!currentMenu && ( @@ -147,6 +157,11 @@ export const RecordIndexOptionsDropdownContent = ({ LeftIcon={IconFileImport} text="Import" /> + )} diff --git a/packages/twenty-front/src/modules/object-record/record-index/options/hooks/__tests__/useExportTableData.test.ts b/packages/twenty-front/src/modules/object-record/record-index/options/hooks/__tests__/useExportTableData.test.ts new file mode 100644 index 0000000000..c811a9d2d8 --- /dev/null +++ b/packages/twenty-front/src/modules/object-record/record-index/options/hooks/__tests__/useExportTableData.test.ts @@ -0,0 +1,108 @@ +import { FieldMetadata } from '@/object-record/record-field/types/FieldMetadata'; +import { ColumnDefinition } from '@/object-record/record-table/types/ColumnDefinition'; + +import { + csvDownloader, + download, + generateCsv, + percentage, + sleep, +} from '../useExportTableData'; + +jest.useFakeTimers(); + +describe('sleep', () => { + it('waits the provided number of milliseconds', async () => { + const spy = jest.fn(); + sleep(1000).then(spy); + + jest.advanceTimersByTime(999); + expect(spy).not.toHaveBeenCalled(); + jest.advanceTimersByTime(1); + await Promise.resolve(); // let queued promises execute + expect(spy).toHaveBeenCalledTimes(1); + }); +}); + +describe('download', () => { + it('creates a download link and clicks it', () => { + const link = document.createElement('a'); + document.createElement = jest.fn().mockReturnValue(link); + const appendChild = jest.spyOn(document.body, 'appendChild'); + const click = jest.spyOn(link, 'click'); + + URL.createObjectURL = jest.fn().mockReturnValue('fake-url'); + download(new Blob(['test'], { type: 'text/plain' }), 'test.txt'); + + expect(appendChild).toHaveBeenCalledWith(link); + expect(link.href).toEqual('http://localhost/fake-url'); + expect(link.getAttribute('download')).toEqual('test.txt'); + expect(click).toHaveBeenCalledTimes(1); + }); +}); + +describe('generateCsv', () => { + it('generates a csv with formatted headers', async () => { + const columns = [ + { label: 'Foo', metadata: { fieldName: 'foo' } }, + { label: 'Empty', metadata: { fieldName: 'empty' } }, + { label: 'Nested', metadata: { fieldName: 'nested' } }, + { + label: 'Relation', + metadata: { fieldName: 'relation', relationType: 'TO_ONE_OBJECT' }, + }, + ] as ColumnDefinition[]; + const rows = [ + { + bar: 'another field', + empty: null, + foo: 'some field', + nested: { __typename: 'type', foo: 'foo', nested: 'nested' }, + relation: 'a relation', + }, + ]; + const csv = generateCsv({ columns, rows }); + expect(csv).toEqual(`Foo,Empty,Nested Foo,Nested Nested +some field,,foo,nested`); + }); +}); + +describe('csvDownloader', () => { + it('downloads a csv', () => { + const filename = 'test.csv'; + const data = { + rows: [ + { id: 1, name: 'John' }, + { id: 2, name: 'Alice' }, + ], + columns: [], + }; + + const link = document.createElement('a'); + document.createElement = jest.fn().mockReturnValue(link); + const createObjectURL = jest.spyOn(URL, 'createObjectURL'); + + csvDownloader(filename, data); + + expect(link.getAttribute('download')).toEqual('test.csv'); + expect(createObjectURL).toHaveBeenCalledWith(expect.any(Blob)); + expect(createObjectURL).toHaveBeenCalledWith( + expect.objectContaining({ type: 'text/csv' }), + ); + }); +}); + +describe('percentage', () => { + it.each([ + [20, 50, 40], + [0, 100, 0], + [10, 10, 100], + [10, 10, 100], + [7, 9, 78], + ])( + 'calculates the percentage %p/%p = %p', + (part, whole, expectedPercentage) => { + expect(percentage(part, whole)).toEqual(expectedPercentage); + }, + ); +}); diff --git a/packages/twenty-front/src/modules/object-record/record-index/options/hooks/useExportTableData.ts b/packages/twenty-front/src/modules/object-record/record-index/options/hooks/useExportTableData.ts new file mode 100644 index 0000000000..6d91fe33b5 --- /dev/null +++ b/packages/twenty-front/src/modules/object-record/record-index/options/hooks/useExportTableData.ts @@ -0,0 +1,165 @@ +import { useEffect, useState } from 'react'; +import { json2csv } from 'json-2-csv'; +import { useRecoilValue } from 'recoil'; + +import { useFindManyRecords } from '@/object-record/hooks/useFindManyRecords'; +import { FieldMetadata } from '@/object-record/record-field/types/FieldMetadata'; +import { useRecordTableStates } from '@/object-record/record-table/hooks/internal/useRecordTableStates'; +import { ColumnDefinition } from '@/object-record/record-table/types/ColumnDefinition'; + +import { useFindManyParams } from '../../hooks/useLoadRecordIndexTable'; + +export const sleep = (ms: number) => + new Promise((resolve) => setTimeout(resolve, ms)); + +export const download = (blob: Blob, filename: string) => { + const url = URL.createObjectURL(blob); + const link = document.createElement('a'); + link.href = url; + link.setAttribute('download', filename); + document.body.appendChild(link); + link.click(); + link.parentNode?.removeChild(link); +}; + +type GenerateExportOptions = { + columns: ColumnDefinition[]; + rows: object[]; +}; + +type GenerateExport = (data: GenerateExportOptions) => string; + +export const generateCsv: GenerateExport = ({ + columns, + rows, +}: GenerateExportOptions): string => { + const columnsWithoutRelations = columns.filter( + (col) => !('relationType' in col.metadata && col.metadata.relationType), + ); + + const keys = columnsWithoutRelations.flatMap((col) => { + const column = { + field: col.metadata.fieldName, + title: col.label, + }; + + const fieldsWithSubFields = rows.find((row) => { + const fieldValue = (row as any)[column.field]; + const hasSubFields = + fieldValue && + typeof fieldValue === 'object' && + !Array.isArray(fieldValue); + return hasSubFields; + }); + + if (fieldsWithSubFields) { + const nestedFieldsWithoutTypename = Object.keys( + (fieldsWithSubFields as any)[column.field], + ) + .filter((key) => key !== '__typename') + .map((key) => ({ + field: `${column.field}.${key}`, + title: `${column.title} ${key[0].toUpperCase() + key.slice(1)}`, + })); + return nestedFieldsWithoutTypename; + } + return [column]; + }); + + return json2csv(rows, { + keys, + emptyFieldValue: '', + }); +}; + +export const percentage = (part: number, whole: number): number => { + return Math.round((part / whole) * 100); +}; + +const downloader = (mimeType: string, generator: GenerateExport) => { + return (filename: string, data: GenerateExportOptions) => { + const blob = new Blob([generator(data)], { type: mimeType }); + download(blob, filename); + }; +}; + +export const csvDownloader = downloader('text/csv', generateCsv); + +type UseExportTableDataOptions = { + delayMs: number; + filename: string; + maximumRequests?: number; + objectNameSingular: string; + pageSize?: number; + recordIndexId: string; +}; + +export const useExportTableData = ({ + delayMs, + filename, + maximumRequests = 100, + objectNameSingular, + pageSize = 30, + recordIndexId, +}: UseExportTableDataOptions) => { + const [isDownloading, setIsDownloading] = useState(false); + const [inflight, setInflight] = useState(false); + const [pageCount, setPageCount] = useState(0); + const [progress, setProgress] = useState(undefined); + const [hasNextPage, setHasNextPage] = useState(true); + const { getVisibleTableColumnsSelector } = + useRecordTableStates(recordIndexId); + const columns = useRecoilValue(getVisibleTableColumnsSelector()); + const params = useFindManyParams(objectNameSingular); + const { totalCount, records, fetchMoreRecords } = useFindManyRecords({ + ...params, + limit: pageSize, + onCompleted: (_data, { hasNextPage }) => { + setHasNextPage(hasNextPage ?? false); + }, + }); + + useEffect(() => { + const MAXIMUM_REQUESTS = Math.min(maximumRequests, totalCount / pageSize); + + const downloadCsv = (rows: object[]) => { + csvDownloader(filename, { rows, columns }); + setIsDownloading(false); + setProgress(undefined); + }; + + const fetchNextPage = async () => { + setInflight(true); + await fetchMoreRecords(); + setPageCount((state) => state + 1); + setProgress(percentage(pageCount, MAXIMUM_REQUESTS)); + await sleep(delayMs); + setInflight(false); + }; + + if (!isDownloading || inflight) { + return; + } + + if (!hasNextPage || pageCount >= MAXIMUM_REQUESTS) { + downloadCsv(records); + } else { + fetchNextPage(); + } + }, [ + delayMs, + fetchMoreRecords, + filename, + hasNextPage, + inflight, + isDownloading, + pageCount, + records, + totalCount, + columns, + maximumRequests, + pageSize, + ]); + + return { progress, download: () => setIsDownloading(true) }; +}; diff --git a/packages/twenty-front/src/modules/ui/display/icon/index.ts b/packages/twenty-front/src/modules/ui/display/icon/index.ts index d003540ff2..cc798c7716 100644 --- a/packages/twenty-front/src/modules/ui/display/icon/index.ts +++ b/packages/twenty-front/src/modules/ui/display/icon/index.ts @@ -56,6 +56,7 @@ export { IconEyeOff, IconFile, IconFileCheck, + IconFileExport, IconFileImport, IconFileText, IconFileUpload, diff --git a/yarn.lock b/yarn.lock index 29dbeb00a8..38f19702dc 100644 --- a/yarn.lock +++ b/yarn.lock @@ -22905,6 +22905,13 @@ __metadata: languageName: node linkType: hard +"deeks@npm:3.1.0": + version: 3.1.0 + resolution: "deeks@npm:3.1.0" + checksum: 3173ca28466cf31d550248c034c5466d93c5aecb8ee8ca547a2c9f471e62af4ebed7456c3310503be901d982867071b4411030a6b724528739895aee1dc2b482 + languageName: node + linkType: hard + "deep-eql@npm:^4.1.3": version: 4.1.3 resolution: "deep-eql@npm:4.1.3" @@ -23362,6 +23369,13 @@ __metadata: languageName: node linkType: hard +"doc-path@npm:4.1.0": + version: 4.1.0 + resolution: "doc-path@npm:4.1.0" + checksum: 134a272a0c41c5f03083fbe7d12fba88615734f15d75e5c1e59e6d9d41e6ad2c5d2ec442282a55ceab45695c4b31aaf6194de1db57ea840e10e77a5c8dd65fa4 + languageName: node + linkType: hard + "doctrine@npm:^2.1.0": version: 2.1.0 resolution: "doctrine@npm:2.1.0" @@ -31211,6 +31225,16 @@ __metadata: languageName: node linkType: hard +"json-2-csv@npm:^5.4.0": + version: 5.4.0 + resolution: "json-2-csv@npm:5.4.0" + dependencies: + deeks: "npm:3.1.0" + doc-path: "npm:4.1.0" + checksum: e33a646315aca132a535aa77e7aa3702452a7d535fece17f8247dd938ee97b93ff44a621a193efcd2e6b73251026044b6dfddb222086941f030fd7cf309e3aa1 + languageName: node + linkType: hard + "json-bigint@npm:^1.0.0": version: 1.0.0 resolution: "json-bigint@npm:1.0.0" @@ -44489,6 +44513,7 @@ __metadata: js-cookie: "npm:^3.0.5" js-levenshtein: "npm:^1.1.6" jsdom: "npm:~22.1.0" + json-2-csv: "npm:^5.4.0" jsonwebtoken: "npm:^9.0.0" libphonenumber-js: "npm:^1.10.26" lodash.camelcase: "npm:^4.3.0"