Add export as csv (#4034)

* Add export as csv

Resolves 2183.

* collect over paginated data

* refactor

* add tests

* parameterize pageSize (limit)

* use pageInfo for onCompleted callback

* json column variable naming

* omit relations from csv exports

package.json

@@ -101,6 +101,7 @@
     "jest-mock-extended": "^3.0.4",
     "js-cookie": "^3.0.5",
     "js-levenshtein": "^1.1.6",
+    "json-2-csv": "^5.4.0",
     "jsonwebtoken": "^9.0.0",
     "libphonenumber-js": "^1.10.26",
     "lodash.camelcase": "^4.3.0",

packages/twenty-front/src/modules/object-record/hooks/useFindManyRecords.ts

@@ -34,7 +34,10 @@ export const useFindManyRecords = <T extends ObjectRecord = ObjectRecord>({
   depth,
 }: ObjectMetadataItemIdentifier &
   ObjectRecordQueryVariables & {
-    onCompleted?: (data: ObjectRecordConnection<T>) => void;
+    onCompleted?: (
+      data: ObjectRecordConnection<T>,
+      pageInfo: ObjectRecordConnection<T>['pageInfo'],
+    ) => void;
     skip?: boolean;
     useRecordsWithoutConnection?: boolean;
     depth?: number;
@@ -77,15 +80,13 @@ export const useFindManyRecords = <T extends ObjectRecord = ObjectRecord>({
       orderBy,
     },
     onCompleted: (data) => {
-      onCompleted?.(data[objectMetadataItem.namePlural]);
+      const pageInfo = data?.[objectMetadataItem.namePlural]?.pageInfo;
+
+      onCompleted?.(data[objectMetadataItem.namePlural], pageInfo);
 
       if (data?.[objectMetadataItem.namePlural]) {
-        setLastCursor(
-          data?.[objectMetadataItem.namePlural]?.pageInfo.endCursor ?? '',
-        );
-        setHasNextPage(
-          data?.[objectMetadataItem.namePlural]?.pageInfo.hasNextPage ?? false,
-        );
+        setLastCursor(pageInfo.endCursor ?? '');
+        setHasNextPage(pageInfo.hasNextPage ?? false);
       }
     },
     onError: (error) => {
@@ -128,18 +129,15 @@ export const useFindManyRecords = <T extends ObjectRecord = ObjectRecord>({
               ]);
             }
 
+            const pageInfo =
+              fetchMoreResult?.[objectMetadataItem.namePlural]?.pageInfo;
             if (data?.[objectMetadataItem.namePlural]) {
-              setLastCursor(
-                fetchMoreResult?.[objectMetadataItem.namePlural]?.pageInfo
-                  .endCursor ?? '',
-              );
-              setHasNextPage(
-                fetchMoreResult?.[objectMetadataItem.namePlural]?.pageInfo
-                  .hasNextPage ?? false,
-              );
+              setLastCursor(pageInfo.endCursor ?? '');
+              setHasNextPage(pageInfo.hasNextPage ?? false);
             }
 
-            onCompleted?.({
+            onCompleted?.(
+              {
                 __typename: `${capitalize(
                   objectMetadataItem.nameSingular,
                 )}Connection`,
@@ -148,7 +146,9 @@ export const useFindManyRecords = <T extends ObjectRecord = ObjectRecord>({
                   fetchMoreResult?.[objectMetadataItem.namePlural].pageInfo,
                 totalCount:
                   fetchMoreResult?.[objectMetadataItem.namePlural].totalCount,
-            });
+              },
+              pageInfo,
+            );
 
             return Object.assign({}, prev, {
               [objectMetadataItem.namePlural]: {

packages/twenty-front/src/modules/object-record/record-index/hooks/useLoadRecordIndexTable.ts

@@ -10,26 +10,17 @@ import { SIGN_IN_BACKGROUND_MOCK_COMPANIES } from '@/sign-in-background-mock/con
 
 import { useFindManyRecords } from '../../hooks/useFindManyRecords';
 
-export const useLoadRecordIndexTable = (objectNameSingular: string) => {
-  const { setRecordTableData, setIsRecordTableInitialLoading } =
-    useRecordTable();
-
-  const currentWorkspace = useRecoilValue(currentWorkspaceState);
+export const useFindManyParams = (objectNameSingular: string) => {
   const { objectMetadataItem } = useObjectMetadataItem({
     objectNameSingular,
   });
 
-  const {
-    getTableFiltersState,
-    getTableSortsState,
-    getTableLastRowVisibleState,
-  } = useRecordTableStates();
+  const { getTableFiltersState, getTableSortsState } = useRecordTableStates();
 
   const tableFilters = useRecoilValue(getTableFiltersState());
   const tableSorts = useRecoilValue(getTableSortsState());
-  const setLastRowVisible = useSetRecoilState(getTableLastRowVisibleState());
 
-  const requestFilters = turnObjectDropdownFilterIntoQueryFilter(
+  const filter = turnObjectDropdownFilterIntoQueryFilter(
     tableFilters,
     objectMetadataItem?.fields ?? [],
   );
@@ -39,6 +30,17 @@ export const useLoadRecordIndexTable = (objectNameSingular: string) => {
     objectMetadataItem?.fields ?? [],
   );
 
+  return { objectNameSingular, filter, orderBy };
+};
+
+export const useLoadRecordIndexTable = (objectNameSingular: string) => {
+  const { setRecordTableData, setIsRecordTableInitialLoading } =
+    useRecordTable();
+  const { getTableLastRowVisibleState } = useRecordTableStates();
+  const setLastRowVisible = useSetRecoilState(getTableLastRowVisibleState());
+  const currentWorkspace = useRecoilValue(currentWorkspaceState);
+  const params = useFindManyParams(objectNameSingular);
+
   const {
     records,
     loading,
@@ -46,9 +48,7 @@ export const useLoadRecordIndexTable = (objectNameSingular: string) => {
     fetchMoreRecords,
     queryStateIdentifier,
   } = useFindManyRecords({
-    objectNameSingular,
-    filter: requestFilters,
-    orderBy,
+    ...params,
     onCompleted: () => {
       setLastRowVisible(false);
       setIsRecordTableInitialLoading(false);

packages/twenty-front/src/modules/object-record/record-index/options/components/RecordIndexOptionsDropdown.tsx

@@ -1,6 +1,8 @@
 import { RecordIndexOptionsDropdownButton } from '@/object-record/record-index/options/components/RecordIndexOptionsDropdownButton';
 import { RecordIndexOptionsDropdownContent } from '@/object-record/record-index/options/components/RecordIndexOptionsDropdownContent';
 import { RECORD_INDEX_OPTIONS_DROPDOWN_ID } from '@/object-record/record-index/options/constants/RecordIndexOptionsDropdownId';
+import { useRecordTableStates } from '@/object-record/record-table/hooks/internal/useRecordTableStates';
+import { RecordTableScope } from '@/object-record/record-table/scopes/RecordTableScope';
 import { TableOptionsHotkeyScope } from '@/object-record/record-table/types/TableOptionsHotkeyScope';
 import { Dropdown } from '@/ui/layout/dropdown/components/Dropdown';
 import { useViewBar } from '@/views/hooks/useViewBar';
@@ -18,8 +20,13 @@ export const RecordIndexOptionsDropdown = ({
   viewType,
 }: RecordIndexOptionsDropdownProps) => {
   const { setViewEditMode } = useViewBar();
+  const { scopeId } = useRecordTableStates(recordIndexId);
 
   return (
+    <RecordTableScope
+      recordTableScopeId={scopeId}
+      onColumnsChange={() => false}
+    >
       <Dropdown
         dropdownId={RECORD_INDEX_OPTIONS_DROPDOWN_ID}
         clickableComponent={<RecordIndexOptionsDropdownButton />}
@@ -34,5 +41,6 @@ export const RecordIndexOptionsDropdown = ({
         }
         onClickOutside={() => setViewEditMode('none')}
       />
+    </RecordTableScope>
   );
 };

packages/twenty-front/src/modules/object-record/record-index/options/components/RecordIndexOptionsDropdownContent.tsx

@@ -10,6 +10,7 @@ import { useSpreadsheetRecordImport } from '@/object-record/spreadsheet-import/u
 import {
   IconBaselineDensitySmall,
   IconChevronLeft,
+  IconFileExport,
   IconFileImport,
   IconTag,
 } from '@/ui/display/icon';
@@ -26,6 +27,8 @@ import { useViewScopedStates } from '@/views/hooks/internal/useViewScopedStates'
 import { useViewBar } from '@/views/hooks/useViewBar';
 import { ViewType } from '@/views/types/ViewType';
 
+import { useExportTableData } from '../hooks/useExportTableData';
+
 type RecordIndexOptionsMenu = 'fields';
 
 type RecordIndexOptionsDropdownContentProps = {
@@ -119,6 +122,13 @@ export const RecordIndexOptionsDropdownContent = ({
   const { openRecordSpreadsheetImport } =
     useSpreadsheetRecordImport(objectNameSingular);
 
+  const { progress, download } = useExportTableData({
+    delayMs: 100,
+    filename: `${objectNameSingular}.csv`,
+    objectNameSingular,
+    recordIndexId,
+  });
+
   return (
     <>
       {!currentMenu && (
@@ -147,6 +157,11 @@ export const RecordIndexOptionsDropdownContent = ({
               LeftIcon={IconFileImport}
               text="Import"
             />
+            <MenuItem
+              onClick={download}
+              LeftIcon={IconFileExport}
+              text={progress === undefined ? `Export` : `Export (${progress}%)`}
+            />
           </DropdownMenuItemsContainer>
         </>
       )}

packages/twenty-front/src/modules/object-record/record-index/options/hooks/__tests__/useExportTableData.test.ts

@@ -0,0 +1,108 @@
+import { FieldMetadata } from '@/object-record/record-field/types/FieldMetadata';
+import { ColumnDefinition } from '@/object-record/record-table/types/ColumnDefinition';
+
+import {
+  csvDownloader,
+  download,
+  generateCsv,
+  percentage,
+  sleep,
+} from '../useExportTableData';
+
+jest.useFakeTimers();
+
+describe('sleep', () => {
+  it('waits the provided number of milliseconds', async () => {
+    const spy = jest.fn();
+    sleep(1000).then(spy);
+
+    jest.advanceTimersByTime(999);
+    expect(spy).not.toHaveBeenCalled();
+    jest.advanceTimersByTime(1);
+    await Promise.resolve(); // let queued promises execute
+    expect(spy).toHaveBeenCalledTimes(1);
+  });
+});
+
+describe('download', () => {
+  it('creates a download link and clicks it', () => {
+    const link = document.createElement('a');
+    document.createElement = jest.fn().mockReturnValue(link);
+    const appendChild = jest.spyOn(document.body, 'appendChild');
+    const click = jest.spyOn(link, 'click');
+
+    URL.createObjectURL = jest.fn().mockReturnValue('fake-url');
+    download(new Blob(['test'], { type: 'text/plain' }), 'test.txt');
+
+    expect(appendChild).toHaveBeenCalledWith(link);
+    expect(link.href).toEqual('http://localhost/fake-url');
+    expect(link.getAttribute('download')).toEqual('test.txt');
+    expect(click).toHaveBeenCalledTimes(1);
+  });
+});
+
+describe('generateCsv', () => {
+  it('generates a csv with formatted headers', async () => {
+    const columns = [
+      { label: 'Foo', metadata: { fieldName: 'foo' } },
+      { label: 'Empty', metadata: { fieldName: 'empty' } },
+      { label: 'Nested', metadata: { fieldName: 'nested' } },
+      {
+        label: 'Relation',
+        metadata: { fieldName: 'relation', relationType: 'TO_ONE_OBJECT' },
+      },
+    ] as ColumnDefinition<FieldMetadata>[];
+    const rows = [
+      {
+        bar: 'another field',
+        empty: null,
+        foo: 'some field',
+        nested: { __typename: 'type', foo: 'foo', nested: 'nested' },
+        relation: 'a relation',
+      },
+    ];
+    const csv = generateCsv({ columns, rows });
+    expect(csv).toEqual(`Foo,Empty,Nested Foo,Nested Nested
+some field,,foo,nested`);
+  });
+});
+
+describe('csvDownloader', () => {
+  it('downloads a csv', () => {
+    const filename = 'test.csv';
+    const data = {
+      rows: [
+        { id: 1, name: 'John' },
+        { id: 2, name: 'Alice' },
+      ],
+      columns: [],
+    };
+
+    const link = document.createElement('a');
+    document.createElement = jest.fn().mockReturnValue(link);
+    const createObjectURL = jest.spyOn(URL, 'createObjectURL');
+
+    csvDownloader(filename, data);
+
+    expect(link.getAttribute('download')).toEqual('test.csv');
+    expect(createObjectURL).toHaveBeenCalledWith(expect.any(Blob));
+    expect(createObjectURL).toHaveBeenCalledWith(
+      expect.objectContaining({ type: 'text/csv' }),
+    );
+  });
+});
+
+describe('percentage', () => {
+  it.each([
+    [20, 50, 40],
+    [0, 100, 0],
+    [10, 10, 100],
+    [10, 10, 100],
+    [7, 9, 78],
+  ])(
+    'calculates the percentage %p/%p = %p',
+    (part, whole, expectedPercentage) => {
+      expect(percentage(part, whole)).toEqual(expectedPercentage);
+    },
+  );
+});

packages/twenty-front/src/modules/object-record/record-index/options/hooks/useExportTableData.ts

@@ -0,0 +1,165 @@
+import { useEffect, useState } from 'react';
+import { json2csv } from 'json-2-csv';
+import { useRecoilValue } from 'recoil';
+
+import { useFindManyRecords } from '@/object-record/hooks/useFindManyRecords';
+import { FieldMetadata } from '@/object-record/record-field/types/FieldMetadata';
+import { useRecordTableStates } from '@/object-record/record-table/hooks/internal/useRecordTableStates';
+import { ColumnDefinition } from '@/object-record/record-table/types/ColumnDefinition';
+
+import { useFindManyParams } from '../../hooks/useLoadRecordIndexTable';
+
+export const sleep = (ms: number) =>
+  new Promise((resolve) => setTimeout(resolve, ms));
+
+export const download = (blob: Blob, filename: string) => {
+  const url = URL.createObjectURL(blob);
+  const link = document.createElement('a');
+  link.href = url;
+  link.setAttribute('download', filename);
+  document.body.appendChild(link);
+  link.click();
+  link.parentNode?.removeChild(link);
+};
+
+type GenerateExportOptions = {
+  columns: ColumnDefinition<FieldMetadata>[];
+  rows: object[];
+};
+
+type GenerateExport = (data: GenerateExportOptions) => string;
+
+export const generateCsv: GenerateExport = ({
+  columns,
+  rows,
+}: GenerateExportOptions): string => {
+  const columnsWithoutRelations = columns.filter(
+    (col) => !('relationType' in col.metadata && col.metadata.relationType),
+  );
+
+  const keys = columnsWithoutRelations.flatMap((col) => {
+    const column = {
+      field: col.metadata.fieldName,
+      title: col.label,
+    };
+
+    const fieldsWithSubFields = rows.find((row) => {
+      const fieldValue = (row as any)[column.field];
+      const hasSubFields =
+        fieldValue &&
+        typeof fieldValue === 'object' &&
+        !Array.isArray(fieldValue);
+      return hasSubFields;
+    });
+
+    if (fieldsWithSubFields) {
+      const nestedFieldsWithoutTypename = Object.keys(
+        (fieldsWithSubFields as any)[column.field],
+      )
+        .filter((key) => key !== '__typename')
+        .map((key) => ({
+          field: `${column.field}.${key}`,
+          title: `${column.title} ${key[0].toUpperCase() + key.slice(1)}`,
+        }));
+      return nestedFieldsWithoutTypename;
+    }
+    return [column];
+  });
+
+  return json2csv(rows, {
+    keys,
+    emptyFieldValue: '',
+  });
+};
+
+export const percentage = (part: number, whole: number): number => {
+  return Math.round((part / whole) * 100);
+};
+
+const downloader = (mimeType: string, generator: GenerateExport) => {
+  return (filename: string, data: GenerateExportOptions) => {
+    const blob = new Blob([generator(data)], { type: mimeType });
+    download(blob, filename);
+  };
+};
+
+export const csvDownloader = downloader('text/csv', generateCsv);
+
+type UseExportTableDataOptions = {
+  delayMs: number;
+  filename: string;
+  maximumRequests?: number;
+  objectNameSingular: string;
+  pageSize?: number;
+  recordIndexId: string;
+};
+
+export const useExportTableData = ({
+  delayMs,
+  filename,
+  maximumRequests = 100,
+  objectNameSingular,
+  pageSize = 30,
+  recordIndexId,
+}: UseExportTableDataOptions) => {
+  const [isDownloading, setIsDownloading] = useState(false);
+  const [inflight, setInflight] = useState(false);
+  const [pageCount, setPageCount] = useState(0);
+  const [progress, setProgress] = useState<number | undefined>(undefined);
+  const [hasNextPage, setHasNextPage] = useState(true);
+  const { getVisibleTableColumnsSelector } =
+    useRecordTableStates(recordIndexId);
+  const columns = useRecoilValue(getVisibleTableColumnsSelector());
+  const params = useFindManyParams(objectNameSingular);
+  const { totalCount, records, fetchMoreRecords } = useFindManyRecords({
+    ...params,
+    limit: pageSize,
+    onCompleted: (_data, { hasNextPage }) => {
+      setHasNextPage(hasNextPage ?? false);
+    },
+  });
+
+  useEffect(() => {
+    const MAXIMUM_REQUESTS = Math.min(maximumRequests, totalCount / pageSize);
+
+    const downloadCsv = (rows: object[]) => {
+      csvDownloader(filename, { rows, columns });
+      setIsDownloading(false);
+      setProgress(undefined);
+    };
+
+    const fetchNextPage = async () => {
+      setInflight(true);
+      await fetchMoreRecords();
+      setPageCount((state) => state + 1);
+      setProgress(percentage(pageCount, MAXIMUM_REQUESTS));
+      await sleep(delayMs);
+      setInflight(false);
+    };
+
+    if (!isDownloading || inflight) {
+      return;
+    }
+
+    if (!hasNextPage || pageCount >= MAXIMUM_REQUESTS) {
+      downloadCsv(records);
+    } else {
+      fetchNextPage();
+    }
+  }, [
+    delayMs,
+    fetchMoreRecords,
+    filename,
+    hasNextPage,
+    inflight,
+    isDownloading,
+    pageCount,
+    records,
+    totalCount,
+    columns,
+    maximumRequests,
+    pageSize,
+  ]);
+
+  return { progress, download: () => setIsDownloading(true) };
+};

packages/twenty-front/src/modules/ui/display/icon/index.ts

@@ -56,6 +56,7 @@ export {
   IconEyeOff,
   IconFile,
   IconFileCheck,
+  IconFileExport,
   IconFileImport,
   IconFileText,
   IconFileUpload,

yarn.lock

@@ -22905,6 +22905,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"deeks@npm:3.1.0":
+  version: 3.1.0
+  resolution: "deeks@npm:3.1.0"
+  checksum: 3173ca28466cf31d550248c034c5466d93c5aecb8ee8ca547a2c9f471e62af4ebed7456c3310503be901d982867071b4411030a6b724528739895aee1dc2b482
+  languageName: node
+  linkType: hard
+
 "deep-eql@npm:^4.1.3":
   version: 4.1.3
   resolution: "deep-eql@npm:4.1.3"
@@ -23362,6 +23369,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"doc-path@npm:4.1.0":
+  version: 4.1.0
+  resolution: "doc-path@npm:4.1.0"
+  checksum: 134a272a0c41c5f03083fbe7d12fba88615734f15d75e5c1e59e6d9d41e6ad2c5d2ec442282a55ceab45695c4b31aaf6194de1db57ea840e10e77a5c8dd65fa4
+  languageName: node
+  linkType: hard
+
 "doctrine@npm:^2.1.0":
   version: 2.1.0
   resolution: "doctrine@npm:2.1.0"
@@ -31211,6 +31225,16 @@ __metadata:
   languageName: node
   linkType: hard
 
+"json-2-csv@npm:^5.4.0":
+  version: 5.4.0
+  resolution: "json-2-csv@npm:5.4.0"
+  dependencies:
+    deeks: "npm:3.1.0"
+    doc-path: "npm:4.1.0"
+  checksum: e33a646315aca132a535aa77e7aa3702452a7d535fece17f8247dd938ee97b93ff44a621a193efcd2e6b73251026044b6dfddb222086941f030fd7cf309e3aa1
+  languageName: node
+  linkType: hard
+
 "json-bigint@npm:^1.0.0":
   version: 1.0.0
   resolution: "json-bigint@npm:1.0.0"
@@ -44489,6 +44513,7 @@ __metadata:
     js-cookie: "npm:^3.0.5"
     js-levenshtein: "npm:^1.1.6"
     jsdom: "npm:~22.1.0"
+    json-2-csv: "npm:^5.4.0"
     jsonwebtoken: "npm:^9.0.0"
     libphonenumber-js: "npm:^1.10.26"
     lodash.camelcase: "npm:^4.3.0"