5425 - Introducing support for all Composite Fields Import (#5470)

Adding support for all Composite Fields while using the "import"
functionality. This includes:
- Currency
- Address

Edit : 
- Refactored a lot of types in the spreadsheet import module
- Renamed a lot of functions, hooks and types that were not
self-explanatory enough

---------

Co-authored-by: Charles Bochet <charles@twenty.com>
Co-authored-by: Félix Malfait <felix.malfait@gmail.com>
Co-authored-by: Lucas Bordeau <bordeau.lucas@gmail.com>
This commit is contained in:
Aryan Singh 2024-07-23 21:32:23 +05:30 committed by GitHub
parent 2cc0597ee4
commit 5c8fe027f9
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
46 changed files with 888 additions and 535 deletions

View File

@ -4,7 +4,7 @@ import { useRecoilState, useRecoilValue } from 'recoil';
import { useRecordFieldInput } from '@/object-record/record-field/hooks/useRecordFieldInput'; import { useRecordFieldInput } from '@/object-record/record-field/hooks/useRecordFieldInput';
import { recordStoreFamilySelector } from '@/object-record/record-store/states/selectors/recordStoreFamilySelector'; import { recordStoreFamilySelector } from '@/object-record/record-store/states/selectors/recordStoreFamilySelector';
import { FieldMetadataType } from '~/generated-metadata/graphql'; import { FieldMetadataType } from '~/generated-metadata/graphql';
import { convertCurrencyToCurrencyMicros } from '~/utils/convert-currency-amount'; import { convertCurrencyAmountToCurrencyMicros } from '~/utils/convertCurrencyToCurrencyMicros';
import { FieldContext } from '../../contexts/FieldContext'; import { FieldContext } from '../../contexts/FieldContext';
import { usePersistField } from '../../hooks/usePersistField'; import { usePersistField } from '../../hooks/usePersistField';
@ -45,7 +45,7 @@ export const useCurrencyField = () => {
const newCurrencyValue = { const newCurrencyValue = {
amountMicros: isNaN(amount) amountMicros: isNaN(amount)
? null ? null
: convertCurrencyToCurrencyMicros(amount), : convertCurrencyAmountToCurrencyMicros(amount),
currencyCode, currencyCode,
}; };

View File

@ -19,7 +19,7 @@ import {
import { useRecordIndexOptionsForBoard } from '@/object-record/record-index/options/hooks/useRecordIndexOptionsForBoard'; import { useRecordIndexOptionsForBoard } from '@/object-record/record-index/options/hooks/useRecordIndexOptionsForBoard';
import { useRecordIndexOptionsForTable } from '@/object-record/record-index/options/hooks/useRecordIndexOptionsForTable'; import { useRecordIndexOptionsForTable } from '@/object-record/record-index/options/hooks/useRecordIndexOptionsForTable';
import { TableOptionsHotkeyScope } from '@/object-record/record-table/types/TableOptionsHotkeyScope'; import { TableOptionsHotkeyScope } from '@/object-record/record-table/types/TableOptionsHotkeyScope';
import { useSpreadsheetRecordImport } from '@/object-record/spreadsheet-import/useSpreadsheetRecordImport'; import { useOpenObjectRecordsSpreasheetImportDialog } from '@/object-record/spreadsheet-import/hooks/useOpenObjectRecordsSpreasheetImportDialog';
import { getSettingsPagePath } from '@/settings/utils/getSettingsPagePath'; import { getSettingsPagePath } from '@/settings/utils/getSettingsPagePath';
import { SettingsPath } from '@/types/SettingsPath'; import { SettingsPath } from '@/types/SettingsPath';
import { DropdownMenuHeader } from '@/ui/layout/dropdown/components/DropdownMenuHeader'; import { DropdownMenuHeader } from '@/ui/layout/dropdown/components/DropdownMenuHeader';
@ -114,8 +114,8 @@ export const RecordIndexOptionsDropdownContent = ({
? handleBoardFieldVisibilityChange ? handleBoardFieldVisibilityChange
: handleColumnVisibilityChange; : handleColumnVisibilityChange;
const { openRecordSpreadsheetImport } = const { openObjectRecordsSpreasheetImportDialog } =
useSpreadsheetRecordImport(objectNameSingular); useOpenObjectRecordsSpreasheetImportDialog(objectNameSingular);
const { progress, download } = useExportTableData({ const { progress, download } = useExportTableData({
delayMs: 100, delayMs: 100,
@ -135,7 +135,7 @@ export const RecordIndexOptionsDropdownContent = ({
hasSubMenu hasSubMenu
/> />
<MenuItem <MenuItem
onClick={() => openRecordSpreadsheetImport()} onClick={() => openObjectRecordsSpreasheetImportDialog()}
LeftIcon={IconFileImport} LeftIcon={IconFileImport}
text="Import" text="Import"
/> />

View File

@ -1,7 +1,8 @@
import { useMemo } from 'react';
import { json2csv } from 'json-2-csv'; import { json2csv } from 'json-2-csv';
import { useMemo } from 'react';
import { FieldMetadata } from '@/object-record/record-field/types/FieldMetadata'; import { FieldMetadata } from '@/object-record/record-field/types/FieldMetadata';
import { useProcessRecordsForCSVExport } from '@/object-record/record-index/options/hooks/useProcessRecordsForCSVExport';
import { import {
useTableData, useTableData,
UseTableDataOptions, UseTableDataOptions,
@ -66,12 +67,15 @@ export const generateCsv: GenerateExport = ({
.filter(isDefined) .filter(isDefined)
.join(' '), .join(' '),
}; };
const fieldsWithSubFields = rows.find((row) => { const fieldsWithSubFields = rows.find((row) => {
const fieldValue = (row as any)[column.field]; const fieldValue = (row as any)[column.field];
const hasSubFields = const hasSubFields =
fieldValue && fieldValue &&
typeof fieldValue === 'object' && typeof fieldValue === 'object' &&
!Array.isArray(fieldValue); !Array.isArray(fieldValue);
return hasSubFields; return hasSubFields;
}); });
@ -84,8 +88,10 @@ export const generateCsv: GenerateExport = ({
field: `${column.field}.${key}`, field: `${column.field}.${key}`,
title: `${column.title} ${key[0].toUpperCase() + key.slice(1)}`, title: `${column.title} ${key[0].toUpperCase() + key.slice(1)}`,
})); }));
return nestedFieldsWithoutTypename; return nestedFieldsWithoutTypename;
} }
return [column]; return [column];
}); });
@ -138,12 +144,17 @@ export const useExportTableData = ({
pageSize = 30, pageSize = 30,
recordIndexId, recordIndexId,
}: UseExportTableDataOptions) => { }: UseExportTableDataOptions) => {
const { processRecordsForCSVExport } =
useProcessRecordsForCSVExport(objectNameSingular);
const downloadCsv = useMemo( const downloadCsv = useMemo(
() => () =>
(rows: ObjectRecord[], columns: ColumnDefinition<FieldMetadata>[]) => { (records: ObjectRecord[], columns: ColumnDefinition<FieldMetadata>[]) => {
csvDownloader(filename, { rows, columns }); const recordsProcessedForExport = processRecordsForCSVExport(records);
csvDownloader(filename, { rows: recordsProcessedForExport, columns });
}, },
[filename], [filename, processRecordsForCSVExport],
); );
const { getTableData: download, progress } = useTableData({ const { getTableData: download, progress } = useTableData({

View File

@ -0,0 +1,39 @@
import { useObjectMetadataItem } from '@/object-metadata/hooks/useObjectMetadataItem';
import { FieldCurrencyValue } from '@/object-record/record-field/types/FieldMetadata';
import { ObjectRecord } from '@/object-record/types/ObjectRecord';
import { isDefined } from 'twenty-ui';
import { FieldMetadataType } from '~/generated-metadata/graphql';
import { convertCurrencyMicrosToCurrencyAmount } from '~/utils/convertCurrencyToCurrencyMicros';
export const useProcessRecordsForCSVExport = (objectNameSingular: string) => {
const { objectMetadataItem } = useObjectMetadataItem({
objectNameSingular,
});
const processRecordsForCSVExport = (records: ObjectRecord[]) => {
return records.map((record) => {
const currencyFields = objectMetadataItem.fields.filter(
(field) => field.type === FieldMetadataType.Currency,
);
const processedRecord = {
...record,
};
for (const currencyField of currencyFields) {
if (isDefined(record[currencyField.name])) {
processedRecord[currencyField.name] = {
amountMicros: convertCurrencyMicrosToCurrencyAmount(
record[currencyField.name].amountMicros,
),
currencyCode: record[currencyField.name].currencyCode,
} satisfies FieldCurrencyValue;
}
}
return processedRecord;
});
};
return { processRecordsForCSVExport };
};

View File

@ -1,14 +1,14 @@
import { ReactNode } from 'react';
import { gql } from '@apollo/client'; import { gql } from '@apollo/client';
import { MockedProvider } from '@apollo/client/testing'; import { MockedProvider } from '@apollo/client/testing';
import { act, renderHook, waitFor } from '@testing-library/react'; import { act, renderHook, waitFor } from '@testing-library/react';
import { ReactNode } from 'react';
import { RecoilRoot, useRecoilValue } from 'recoil'; import { RecoilRoot, useRecoilValue } from 'recoil';
import { CoreObjectNameSingular } from '@/object-metadata/types/CoreObjectNameSingular'; import { CoreObjectNameSingular } from '@/object-metadata/types/CoreObjectNameSingular';
import { spreadsheetImportState } from '@/spreadsheet-import/states/spreadsheetImportState'; import { spreadsheetImportDialogState } from '@/spreadsheet-import/states/spreadsheetImportDialogState';
import { SnackBarProviderScope } from '@/ui/feedback/snack-bar-manager/scopes/SnackBarProviderScope';
import { useSpreadsheetRecordImport } from '../useSpreadsheetRecordImport'; import { SnackBarManagerScopeInternalContext } from '@/ui/feedback/snack-bar-manager/scopes/scope-internal-context/SnackBarManagerScopeInternalContext';
import { useOpenObjectRecordsSpreasheetImportDialog } from '../hooks/useOpenObjectRecordsSpreasheetImportDialog';
const companyId = 'cb2e9f4b-20c3-4759-9315-4ffeecfaf71a'; const companyId = 'cb2e9f4b-20c3-4759-9315-4ffeecfaf71a';
@ -62,7 +62,6 @@ const companyMocks = [
variables: { variables: {
data: [ data: [
{ {
address: 'test',
domainName: 'example.com', domainName: 'example.com',
employees: 0, employees: 0,
idealCustomerProfile: true, idealCustomerProfile: true,
@ -94,67 +93,81 @@ const fakeCsv = () => {
const Wrapper = ({ children }: { children: ReactNode }) => ( const Wrapper = ({ children }: { children: ReactNode }) => (
<RecoilRoot> <RecoilRoot>
<MockedProvider mocks={companyMocks} addTypename={false}> <MockedProvider mocks={companyMocks} addTypename={false}>
<SnackBarProviderScope snackBarManagerScopeId="snack-bar-manager"> <SnackBarManagerScopeInternalContext.Provider
value={{ scopeId: 'snack-bar-manager' }}
>
{children} {children}
</SnackBarProviderScope> </SnackBarManagerScopeInternalContext.Provider>
</MockedProvider> </MockedProvider>
</RecoilRoot> </RecoilRoot>
); );
// TODO: improve object metadata item seeds to have more field types to add tests on composite fields here
describe('useSpreadsheetCompanyImport', () => { describe('useSpreadsheetCompanyImport', () => {
it('should work as expected', async () => { it('should work as expected', async () => {
const { result } = renderHook( const { result } = renderHook(
() => { () => {
const spreadsheetImport = useRecoilValue(spreadsheetImportState); const spreadsheetImportDialog = useRecoilValue(
const { openRecordSpreadsheetImport } = useSpreadsheetRecordImport( spreadsheetImportDialogState,
);
const {
openObjectRecordsSpreasheetImportDialog: openRecordSpreadsheetImport,
} = useOpenObjectRecordsSpreasheetImportDialog(
CoreObjectNameSingular.Company, CoreObjectNameSingular.Company,
); );
return { openRecordSpreadsheetImport, spreadsheetImport }; return {
openRecordSpreadsheetImport,
spreadsheetImportDialog,
};
}, },
{ {
wrapper: Wrapper, wrapper: Wrapper,
}, },
); );
const { spreadsheetImport, openRecordSpreadsheetImport } = result.current; const { spreadsheetImportDialog, openRecordSpreadsheetImport } =
result.current;
expect(spreadsheetImport.isOpen).toBe(false); expect(spreadsheetImportDialog.isOpen).toBe(false);
expect(spreadsheetImport.options).toBeNull(); expect(spreadsheetImportDialog.options).toBeNull();
await act(async () => { await act(async () => {
openRecordSpreadsheetImport(); openRecordSpreadsheetImport();
}); });
const { spreadsheetImport: updatedImport } = result.current; const { spreadsheetImportDialog: spreadsheetImportDialogAfterOpen } =
result.current;
expect(updatedImport.isOpen).toBe(true); expect(spreadsheetImportDialogAfterOpen.isOpen).toBe(true);
expect(updatedImport.options).toHaveProperty('onSubmit'); expect(spreadsheetImportDialogAfterOpen.options).toHaveProperty('onSubmit');
expect(updatedImport.options?.onSubmit).toBeInstanceOf(Function); expect(spreadsheetImportDialogAfterOpen.options?.onSubmit).toBeInstanceOf(
expect(updatedImport.options).toHaveProperty('fields'); Function,
expect(Array.isArray(updatedImport.options?.fields)).toBe(true); );
expect(spreadsheetImportDialogAfterOpen.options).toHaveProperty('fields');
expect(
Array.isArray(spreadsheetImportDialogAfterOpen.options?.fields),
).toBe(true);
act(() => { act(() => {
updatedImport.options?.onSubmit( spreadsheetImportDialogAfterOpen.options?.onSubmit(
{ {
validData: [ validStructuredRows: [
{ {
id: companyId, id: companyId,
name: 'Example Company', name: 'Example Company',
domainName: 'example.com', domainName: 'example.com',
idealCustomerProfile: true, idealCustomerProfile: true,
address: 'test',
employees: '0', employees: '0',
}, },
], ],
invalidData: [], invalidStructuredRows: [],
all: [ allStructuredRows: [
{ {
id: companyId, id: companyId,
name: 'Example Company', name: 'Example Company',
domainName: 'example.com', domainName: 'example.com',
__index: 'cbc3985f-dde9-46d1-bae2-c124141700ac', __index: 'cbc3985f-dde9-46d1-bae2-c124141700ac',
idealCustomerProfile: true, idealCustomerProfile: true,
address: 'test',
employees: '0', employees: '0',
}, },
], ],

View File

@ -0,0 +1,28 @@
import {
FieldAddressValue,
FieldCurrencyValue,
FieldFullNameValue,
} from '@/object-record/record-field/types/FieldMetadata';
import { CompositeFieldLabels } from '@/object-record/spreadsheet-import/types/CompositeFieldLabels';
import { FieldMetadataType } from '~/generated-metadata/graphql';
export const COMPOSITE_FIELD_IMPORT_LABELS = {
[FieldMetadataType.FullName]: {
firstNameLabel: 'First Name',
lastNameLabel: 'Last Name',
} satisfies CompositeFieldLabels<FieldFullNameValue>,
[FieldMetadataType.Currency]: {
currencyCodeLabel: 'Currency Code',
amountMicrosLabel: 'Amount',
} satisfies CompositeFieldLabels<FieldCurrencyValue>,
[FieldMetadataType.Address]: {
addressStreet1Label: 'Address 1',
addressStreet2Label: 'Address 2',
addressCityLabel: 'City',
addressPostcodeLabel: 'Post Code',
addressStateLabel: 'State',
addressCountryLabel: 'Country',
addressLatLabel: 'Latitude',
addressLngLabel: 'Longitude',
} satisfies CompositeFieldLabels<FieldAddressValue>,
};

View File

@ -0,0 +1,128 @@
import { useIcons } from 'twenty-ui';
import { FieldMetadataItem } from '@/object-metadata/types/FieldMetadataItem';
import { COMPOSITE_FIELD_IMPORT_LABELS } from '@/object-record/spreadsheet-import/constants/CompositeFieldImportLabels';
import { AvailableFieldForImport } from '@/object-record/spreadsheet-import/types/AvailableFieldForImport';
import { getSpreadSheetFieldValidationDefinitions } from '@/object-record/spreadsheet-import/util/getSpreadSheetFieldValidationDefinitions';
import { FieldMetadataType } from '~/generated-metadata/graphql';
export const useBuildAvailableFieldsForImport = () => {
const { getIcon } = useIcons();
const buildAvailableFieldsForImport = (
fieldMetadataItems: FieldMetadataItem[],
) => {
const availableFieldsForImport: AvailableFieldForImport[] = [];
for (const fieldMetadataItem of fieldMetadataItems) {
if (fieldMetadataItem.type === FieldMetadataType.FullName) {
const { firstNameLabel, lastNameLabel } =
COMPOSITE_FIELD_IMPORT_LABELS[FieldMetadataType.FullName];
availableFieldsForImport.push({
icon: getIcon(fieldMetadataItem.icon),
label: `${firstNameLabel} (${fieldMetadataItem.label})`,
key: `${firstNameLabel} (${fieldMetadataItem.name})`,
fieldType: {
type: 'input',
},
fieldValidationDefinitions: getSpreadSheetFieldValidationDefinitions(
fieldMetadataItem.type,
`${firstNameLabel} (${fieldMetadataItem.label})`,
),
});
availableFieldsForImport.push({
icon: getIcon(fieldMetadataItem.icon),
label: `${lastNameLabel} (${fieldMetadataItem.label})`,
key: `${lastNameLabel} (${fieldMetadataItem.name})`,
fieldType: {
type: 'input',
},
fieldValidationDefinitions: getSpreadSheetFieldValidationDefinitions(
fieldMetadataItem.type,
`${lastNameLabel} (${fieldMetadataItem.label})`,
),
});
} else if (fieldMetadataItem.type === FieldMetadataType.Relation) {
availableFieldsForImport.push({
icon: getIcon(fieldMetadataItem.icon),
label: fieldMetadataItem.label + ' (ID)',
key: fieldMetadataItem.name,
fieldType: {
type: 'input',
},
fieldValidationDefinitions: getSpreadSheetFieldValidationDefinitions(
fieldMetadataItem.type,
fieldMetadataItem.label + ' (ID)',
),
});
} else if (fieldMetadataItem.type === FieldMetadataType.Currency) {
const { currencyCodeLabel, amountMicrosLabel } =
COMPOSITE_FIELD_IMPORT_LABELS[FieldMetadataType.Currency];
availableFieldsForImport.push({
icon: getIcon(fieldMetadataItem.icon),
label: `${currencyCodeLabel} (${fieldMetadataItem.label})`,
key: `${currencyCodeLabel} (${fieldMetadataItem.name})`,
fieldType: {
type: 'input',
},
fieldValidationDefinitions: getSpreadSheetFieldValidationDefinitions(
fieldMetadataItem.type,
`${currencyCodeLabel} (${fieldMetadataItem.label})`,
),
});
availableFieldsForImport.push({
icon: getIcon(fieldMetadataItem.icon),
label: `${amountMicrosLabel} (${fieldMetadataItem.label})`,
key: `${amountMicrosLabel} (${fieldMetadataItem.name})`,
fieldType: {
type: 'input',
},
fieldValidationDefinitions: getSpreadSheetFieldValidationDefinitions(
FieldMetadataType.Number,
`${amountMicrosLabel} (${fieldMetadataItem.label})`,
),
});
} else if (fieldMetadataItem.type === FieldMetadataType.Address) {
Object.entries(
COMPOSITE_FIELD_IMPORT_LABELS[FieldMetadataType.Address],
).forEach(([_, fieldLabel]) => {
availableFieldsForImport.push({
icon: getIcon(fieldMetadataItem.icon),
label: `${fieldLabel} (${fieldMetadataItem.label})`,
key: `${fieldLabel} (${fieldMetadataItem.name})`,
fieldType: {
type: 'input',
},
fieldValidationDefinitions:
getSpreadSheetFieldValidationDefinitions(
fieldMetadataItem.type,
`${fieldLabel} (${fieldMetadataItem.label})`,
),
});
});
} else {
availableFieldsForImport.push({
icon: getIcon(fieldMetadataItem.icon),
label: fieldMetadataItem.label,
key: fieldMetadataItem.name,
fieldType: {
type: 'input',
},
fieldValidationDefinitions: getSpreadSheetFieldValidationDefinitions(
fieldMetadataItem.type,
fieldMetadataItem.label,
),
});
}
}
return availableFieldsForImport;
};
return { buildAvailableFieldsForImport };
};

View File

@ -0,0 +1,78 @@
import { useObjectMetadataItem } from '@/object-metadata/hooks/useObjectMetadataItem';
import { useCreateManyRecords } from '@/object-record/hooks/useCreateManyRecords';
import { useBuildAvailableFieldsForImport } from '@/object-record/spreadsheet-import/hooks/useBuildAvailableFieldsForImport';
import { buildRecordFromImportedStructuredRow } from '@/object-record/spreadsheet-import/util/buildRecordFromImportedStructuredRow';
import { useOpenSpreadsheetImportDialog } from '@/spreadsheet-import/hooks/useOpenSpreadsheetImportDialog';
import { SpreadsheetImportDialogOptions } from '@/spreadsheet-import/types';
import { SnackBarVariant } from '@/ui/feedback/snack-bar-manager/components/SnackBar';
import { useSnackBar } from '@/ui/feedback/snack-bar-manager/hooks/useSnackBar';
import { FieldMetadataType } from '~/generated-metadata/graphql';
export const useOpenObjectRecordsSpreasheetImportDialog = (
objectNameSingular: string,
) => {
const { openSpreadsheetImportDialog } = useOpenSpreadsheetImportDialog<any>();
const { enqueueSnackBar } = useSnackBar();
const { objectMetadataItem } = useObjectMetadataItem({
objectNameSingular,
});
const { createManyRecords } = useCreateManyRecords({
objectNameSingular,
});
const { buildAvailableFieldsForImport } = useBuildAvailableFieldsForImport();
const openObjectRecordsSpreasheetImportDialog = (
options?: Omit<
SpreadsheetImportDialogOptions<any>,
'fields' | 'isOpen' | 'onClose'
>,
) => {
const availableFieldMetadataItems = objectMetadataItem.fields
.filter(
(fieldMetadataItem) =>
fieldMetadataItem.isActive &&
(!fieldMetadataItem.isSystem || fieldMetadataItem.name === 'id') &&
fieldMetadataItem.name !== 'createdAt' &&
(fieldMetadataItem.type !== FieldMetadataType.Relation ||
fieldMetadataItem.toRelationMetadata),
)
.sort((fieldMetadataItemA, fieldMetadataItemB) =>
fieldMetadataItemA.name.localeCompare(fieldMetadataItemB.name),
);
const availableFields = buildAvailableFieldsForImport(
availableFieldMetadataItems,
);
openSpreadsheetImportDialog({
...options,
onSubmit: async (data) => {
const createInputs = data.validStructuredRows.map((record) => {
const fieldMapping: Record<string, any> =
buildRecordFromImportedStructuredRow(
record,
availableFieldMetadataItems,
);
return fieldMapping;
});
try {
await createManyRecords(createInputs, true);
} catch (error: any) {
enqueueSnackBar(error?.message || 'Something went wrong', {
variant: SnackBarVariant.Error,
});
}
},
fields: availableFields,
});
};
return {
openObjectRecordsSpreasheetImportDialog,
};
};

View File

@ -0,0 +1,12 @@
import { FieldValidationDefinition } from '@/spreadsheet-import/types';
import { IconComponent } from 'twenty-ui';
export type AvailableFieldForImport = {
icon: IconComponent;
label: string;
key: string;
fieldType: {
type: 'input' | 'checkbox';
};
fieldValidationDefinitions?: FieldValidationDefinition[];
};

View File

@ -0,0 +1,5 @@
import { KeyOfCompositeField } from '@/object-record/spreadsheet-import/types/KeyOfCompositeField';
export type CompositeFieldLabels<T> = {
[key in `${KeyOfCompositeField<T>}Label`]: string;
};

View File

@ -0,0 +1,3 @@
export type KeyOfCompositeField<T> = keyof Omit<T, '__typename'> extends string
? keyof Omit<T, '__typename'>
: never;

View File

@ -1,202 +0,0 @@
import { isNonEmptyString } from '@sniptt/guards';
import { useIcons } from 'twenty-ui';
import { useObjectMetadataItem } from '@/object-metadata/hooks/useObjectMetadataItem';
import { useCreateManyRecords } from '@/object-record/hooks/useCreateManyRecords';
import { getSpreadSheetValidation } from '@/object-record/spreadsheet-import/util/getSpreadSheetValidation';
import { useSpreadsheetImport } from '@/spreadsheet-import/hooks/useSpreadsheetImport';
import { Field, SpreadsheetOptions } from '@/spreadsheet-import/types';
import { SnackBarVariant } from '@/ui/feedback/snack-bar-manager/components/SnackBar';
import { useSnackBar } from '@/ui/feedback/snack-bar-manager/hooks/useSnackBar';
import { FieldMetadataType } from '~/generated-metadata/graphql';
import { isDefined } from '~/utils/isDefined';
const firstName = 'Firstname';
const lastName = 'Lastname';
export const useSpreadsheetRecordImport = (objectNameSingular: string) => {
const { openSpreadsheetImport } = useSpreadsheetImport<any>();
const { enqueueSnackBar } = useSnackBar();
const { getIcon } = useIcons();
const { objectMetadataItem } = useObjectMetadataItem({
objectNameSingular,
});
const fields = objectMetadataItem.fields
.filter(
(x) =>
x.isActive &&
(!x.isSystem || x.name === 'id') &&
x.name !== 'createdAt' &&
(x.type !== FieldMetadataType.Relation || x.toRelationMetadata),
)
.sort((a, b) => a.name.localeCompare(b.name));
const templateFields: Field<string>[] = [];
for (const field of fields) {
if (field.type === FieldMetadataType.FullName) {
templateFields.push({
icon: getIcon(field.icon),
label: `${firstName} (${field.label})`,
key: `${firstName} (${field.name})`,
fieldType: {
type: 'input',
},
validations: getSpreadSheetValidation(
field.type,
`${firstName} (${field.label})`,
),
});
templateFields.push({
icon: getIcon(field.icon),
label: `${lastName} (${field.label})`,
key: `${lastName} (${field.name})`,
fieldType: {
type: 'input',
},
validations: getSpreadSheetValidation(
field.type,
`${lastName} (${field.label})`,
),
});
} else if (field.type === FieldMetadataType.Relation) {
templateFields.push({
icon: getIcon(field.icon),
label: field.label + ' (ID)',
key: field.name,
fieldType: {
type: 'input',
},
validations: getSpreadSheetValidation(
field.type,
field.label + ' (ID)',
),
});
} else if (field.type === FieldMetadataType.Select) {
templateFields.push({
icon: getIcon(field.icon),
label: field.label,
key: field.name,
fieldType: {
type: 'select',
options:
field.options?.map((option) => ({
label: option.label,
value: option.value,
})) || [],
},
validations: getSpreadSheetValidation(
field.type,
field.label + ' (ID)',
),
});
} else if (field.type === FieldMetadataType.Boolean) {
templateFields.push({
icon: getIcon(field.icon),
label: field.label,
key: field.name,
fieldType: {
type: 'checkbox',
},
validations: getSpreadSheetValidation(field.type, field.label),
});
} else {
templateFields.push({
icon: getIcon(field.icon),
label: field.label,
key: field.name,
fieldType: {
type: 'input',
},
validations: getSpreadSheetValidation(field.type, field.label),
});
}
}
const { createManyRecords } = useCreateManyRecords({
objectNameSingular,
});
const openRecordSpreadsheetImport = (
options?: Omit<SpreadsheetOptions<any>, 'fields' | 'isOpen' | 'onClose'>,
) => {
openSpreadsheetImport({
...options,
onSubmit: async (data) => {
const createInputs = data.validData.map((record) => {
const fieldMapping: Record<string, any> = {};
for (const field of fields) {
const value = record[field.name];
switch (field.type) {
case FieldMetadataType.Boolean:
if (value !== undefined) {
fieldMapping[field.name] = value === 'true' || value === true;
}
break;
case FieldMetadataType.Number:
case FieldMetadataType.Numeric:
if (value !== undefined) {
fieldMapping[field.name] = Number(value);
}
break;
case FieldMetadataType.Currency:
if (value !== undefined) {
fieldMapping[field.name] = {
amountMicros: Number(value),
currencyCode: 'USD',
};
}
break;
case FieldMetadataType.Link:
if (value !== undefined) {
fieldMapping[field.name] = {
label: field.name,
url: value || null,
};
}
break;
case FieldMetadataType.Relation:
if (
isDefined(value) &&
(isNonEmptyString(value) || value !== false)
) {
fieldMapping[field.name + 'Id'] = value;
}
break;
case FieldMetadataType.FullName:
if (
isDefined(
record[`${firstName} (${field.name})`] ||
record[`${lastName} (${field.name})`],
)
) {
fieldMapping[field.name] = {
firstName: record[`${firstName} (${field.name})`] || '',
lastName: record[`${lastName} (${field.name})`] || '',
};
}
break;
default:
if (value !== undefined) {
fieldMapping[field.name] = value;
}
break;
}
}
return fieldMapping;
});
try {
await createManyRecords(createInputs, true);
} catch (error: any) {
enqueueSnackBar(error?.message || 'Something went wrong', {
variant: SnackBarVariant.Error,
});
}
},
fields: templateFields,
});
};
return { openRecordSpreadsheetImport };
};

View File

@ -0,0 +1,147 @@
import { FieldMetadataItem } from '@/object-metadata/types/FieldMetadataItem';
import { FieldAddressValue } from '@/object-record/record-field/types/FieldMetadata';
import { COMPOSITE_FIELD_IMPORT_LABELS } from '@/object-record/spreadsheet-import/constants/CompositeFieldImportLabels';
import { ImportedStructuredRow } from '@/spreadsheet-import/types';
import { isNonEmptyString } from '@sniptt/guards';
import { isDefined } from 'twenty-ui';
import { FieldMetadataType } from '~/generated-metadata/graphql';
import { castToString } from '~/utils/castToString';
import { convertCurrencyAmountToCurrencyMicros } from '~/utils/convertCurrencyToCurrencyMicros';
export const buildRecordFromImportedStructuredRow = (
importedStructuredRow: ImportedStructuredRow<any>,
fields: FieldMetadataItem[],
) => {
const recordToBuild: Record<string, any> = {};
const {
ADDRESS: {
addressCityLabel,
addressCountryLabel,
addressLatLabel,
addressLngLabel,
addressPostcodeLabel,
addressStateLabel,
addressStreet1Label,
addressStreet2Label,
},
CURRENCY: { amountMicrosLabel, currencyCodeLabel },
FULL_NAME: { firstNameLabel, lastNameLabel },
} = COMPOSITE_FIELD_IMPORT_LABELS;
for (const field of fields) {
const importedFieldValue = importedStructuredRow[field.name];
switch (field.type) {
case FieldMetadataType.Boolean:
recordToBuild[field.name] =
importedFieldValue === 'true' || importedFieldValue === true;
break;
case FieldMetadataType.Number:
case FieldMetadataType.Numeric:
recordToBuild[field.name] = Number(importedFieldValue);
break;
case FieldMetadataType.Currency:
if (
isDefined(
importedStructuredRow[`${amountMicrosLabel} (${field.name})`],
) ||
isDefined(
importedStructuredRow[`${currencyCodeLabel} (${field.name})`],
)
) {
recordToBuild[field.name] = {
amountMicros: convertCurrencyAmountToCurrencyMicros(
Number(
importedStructuredRow[`${amountMicrosLabel} (${field.name})`],
),
),
currencyCode:
importedStructuredRow[`${currencyCodeLabel} (${field.name})`] ||
'USD',
};
}
break;
case FieldMetadataType.Address: {
if (
isDefined(
importedStructuredRow[`${addressStreet1Label} (${field.name})`] ||
importedStructuredRow[`${addressStreet2Label} (${field.name})`] ||
importedStructuredRow[`${addressCityLabel} (${field.name})`] ||
importedStructuredRow[
`${addressPostcodeLabel} (${field.name})`
] ||
importedStructuredRow[`${addressStateLabel} (${field.name})`] ||
importedStructuredRow[`${addressCountryLabel} (${field.name})`] ||
importedStructuredRow[`${addressLatLabel} (${field.name})`] ||
importedStructuredRow[`${addressLngLabel} (${field.name})`],
)
) {
recordToBuild[field.name] = {
addressStreet1: castToString(
importedStructuredRow[`${addressStreet1Label} (${field.name})`],
),
addressStreet2: castToString(
importedStructuredRow[`${addressStreet2Label} (${field.name})`],
),
addressCity: castToString(
importedStructuredRow[`${addressCityLabel} (${field.name})`],
),
addressPostcode: castToString(
importedStructuredRow[`${addressPostcodeLabel} (${field.name})`],
),
addressState: castToString(
importedStructuredRow[`${addressStateLabel} (${field.name})`],
),
addressCountry: castToString(
importedStructuredRow[`${addressCountryLabel} (${field.name})`],
),
addressLat: Number(
importedStructuredRow[`${addressLatLabel} (${field.name})`],
),
addressLng: Number(
importedStructuredRow[`${addressLngLabel} (${field.name})`],
),
} satisfies FieldAddressValue;
}
break;
}
case FieldMetadataType.Link:
if (importedFieldValue !== undefined) {
recordToBuild[field.name] = {
label: field.name,
url: importedFieldValue || null,
};
}
break;
case FieldMetadataType.Relation:
if (
isDefined(importedFieldValue) &&
(isNonEmptyString(importedFieldValue) || importedFieldValue !== false)
) {
recordToBuild[field.name + 'Id'] = importedFieldValue;
}
break;
case FieldMetadataType.FullName:
if (
isDefined(
importedStructuredRow[`${firstNameLabel} (${field.name})`] ??
importedStructuredRow[`${lastNameLabel} (${field.name})`],
)
) {
recordToBuild[field.name] = {
firstName:
importedStructuredRow[`${firstNameLabel} (${field.name})`] ?? '',
lastName:
importedStructuredRow[`${lastNameLabel} (${field.name})`] ?? '',
};
}
break;
default:
recordToBuild[field.name] = importedFieldValue;
break;
}
}
return recordToBuild;
};

View File

@ -1,14 +1,37 @@
import { isValidPhoneNumber } from 'libphonenumber-js'; import { isValidPhoneNumber } from 'libphonenumber-js';
import { isValidUuid } from '@/object-record/spreadsheet-import/util/isValidUuid'; import { FieldValidationDefinition } from '@/spreadsheet-import/types';
import { Validation } from '@/spreadsheet-import/types'; import { isDefined } from 'twenty-ui';
import { FieldMetadataType } from '~/generated-metadata/graphql'; import { FieldMetadataType } from '~/generated-metadata/graphql';
import { isValidUuid } from '~/utils/isValidUuid';
export const getSpreadSheetValidation = ( export const getSpreadSheetFieldValidationDefinitions = (
type: FieldMetadataType, type: FieldMetadataType,
fieldName: string, fieldName: string,
): Validation[] => { ): FieldValidationDefinition[] => {
switch (type) { switch (type) {
case FieldMetadataType.FullName:
return [
{
rule: 'object',
isValid: ({
firstName,
lastName,
}: {
firstName: string;
lastName: string;
}) => {
return (
isDefined(firstName) &&
isDefined(lastName) &&
typeof firstName === 'string' &&
typeof lastName === 'string'
);
},
errorMessage: fieldName + ' must be a full name',
level: 'error',
},
];
case FieldMetadataType.Number: case FieldMetadataType.Number:
return [ return [
{ {

View File

@ -1,13 +1,13 @@
import { createContext } from 'react'; import { createContext } from 'react';
import { SpreadsheetOptions } from '@/spreadsheet-import/types'; import { SpreadsheetImportDialogOptions } from '@/spreadsheet-import/types';
import { isUndefinedOrNull } from '~/utils/isUndefinedOrNull'; import { isUndefinedOrNull } from '~/utils/isUndefinedOrNull';
export const RsiContext = createContext({} as any); export const RsiContext = createContext({} as any);
type ProvidersProps<T extends string> = { type ProvidersProps<T extends string> = {
children: React.ReactNode; children: React.ReactNode;
values: SpreadsheetOptions<T>; values: SpreadsheetImportDialogOptions<T>;
}; };
export const Providers = <T extends string>({ export const Providers = <T extends string>({

View File

@ -1,50 +1,57 @@
import { act, renderHook } from '@testing-library/react'; import { act, renderHook } from '@testing-library/react';
import { RecoilRoot, useRecoilState } from 'recoil'; import { RecoilRoot, useRecoilState } from 'recoil';
import { useSpreadsheetImport } from '@/spreadsheet-import/hooks/useSpreadsheetImport'; import { useOpenSpreadsheetImportDialog } from '@/spreadsheet-import/hooks/useOpenSpreadsheetImportDialog';
import { spreadsheetImportState } from '@/spreadsheet-import/states/spreadsheetImportState'; import { spreadsheetImportDialogState } from '@/spreadsheet-import/states/spreadsheetImportDialogState';
import { StepType } from '@/spreadsheet-import/steps/components/UploadFlow'; import { StepType } from '@/spreadsheet-import/steps/components/UploadFlow';
import { RawData, SpreadsheetOptions } from '@/spreadsheet-import/types'; import {
ImportedRow,
SpreadsheetImportDialogOptions,
} from '@/spreadsheet-import/types';
const Wrapper = ({ children }: { children: React.ReactNode }) => ( const Wrapper = ({ children }: { children: React.ReactNode }) => (
<RecoilRoot>{children}</RecoilRoot> <RecoilRoot>{children}</RecoilRoot>
); );
type SpreadsheetKey = 'spreadsheet_key'; type SpreadsheetKey = 'spreadsheet_key';
export const mockedSpreadsheetOptions: SpreadsheetOptions<SpreadsheetKey> = { export const mockedSpreadsheetOptions: SpreadsheetImportDialogOptions<SpreadsheetKey> =
isOpen: true, {
onClose: () => {}, isOpen: true,
fields: [], onClose: () => {},
uploadStepHook: async () => [], fields: [],
selectHeaderStepHook: async (headerValues: RawData, data: RawData[]) => ({ uploadStepHook: async () => [],
headerValues, selectHeaderStepHook: async (
data, headerValues: ImportedRow,
}), data: ImportedRow[],
matchColumnsStepHook: async () => [], ) => ({
rowHook: () => ({ spreadsheet_key: 'rowHook' }), headerRow: headerValues,
tableHook: () => [{ spreadsheet_key: 'tableHook' }], importedRows: data,
onSubmit: async () => {}, }),
allowInvalidSubmit: false, matchColumnsStepHook: async () => [],
customTheme: {}, rowHook: () => ({ spreadsheet_key: 'rowHook' }),
maxRecords: 10, tableHook: () => [{ spreadsheet_key: 'tableHook' }],
maxFileSize: 50, onSubmit: async () => {},
autoMapHeaders: true, allowInvalidSubmit: false,
autoMapDistance: 1, customTheme: {},
initialStepState: { maxRecords: 10,
type: StepType.upload, maxFileSize: 50,
}, autoMapHeaders: true,
dateFormat: 'MM/DD/YY', autoMapDistance: 1,
parseRaw: true, initialStepState: {
rtl: false, type: StepType.upload,
selectHeader: true, },
}; dateFormat: 'MM/DD/YY',
parseRaw: true,
rtl: false,
selectHeader: true,
};
describe('useSpreadsheetImport', () => { describe('useSpreadsheetImport', () => {
it('should set isOpen to true, and update the options in the Recoil state', async () => { it('should set isOpen to true, and update the options in the Recoil state', async () => {
const { result } = renderHook( const { result } = renderHook(
() => ({ () => ({
useSpreadsheetImport: useSpreadsheetImport<SpreadsheetKey>(), useSpreadsheetImport: useOpenSpreadsheetImportDialog<SpreadsheetKey>(),
spreadsheetImportState: useRecoilState(spreadsheetImportState)[0], spreadsheetImportState: useRecoilState(spreadsheetImportDialogState)[0],
}), }),
{ {
wrapper: Wrapper, wrapper: Wrapper,
@ -55,7 +62,7 @@ describe('useSpreadsheetImport', () => {
options: null, options: null,
}); });
act(() => { act(() => {
result.current.useSpreadsheetImport.openSpreadsheetImport( result.current.useSpreadsheetImport.openSpreadsheetImportDialog(
mockedSpreadsheetOptions, mockedSpreadsheetOptions,
); );
}); });

View File

@ -0,0 +1,19 @@
import { useSetRecoilState } from 'recoil';
import { spreadsheetImportDialogState } from '@/spreadsheet-import/states/spreadsheetImportDialogState';
import { SpreadsheetImportDialogOptions } from '@/spreadsheet-import/types';
export const useOpenSpreadsheetImportDialog = <T extends string>() => {
const setSpreadSheetImport = useSetRecoilState(spreadsheetImportDialogState);
const openSpreadsheetImportDialog = (
options: Omit<SpreadsheetImportDialogOptions<T>, 'isOpen' | 'onClose'>,
) => {
setSpreadSheetImport({
isOpen: true,
options,
});
};
return { openSpreadsheetImportDialog };
};

View File

@ -1,19 +0,0 @@
import { useSetRecoilState } from 'recoil';
import { spreadsheetImportState } from '@/spreadsheet-import/states/spreadsheetImportState';
import { SpreadsheetOptions } from '@/spreadsheet-import/types';
export const useSpreadsheetImport = <T extends string>() => {
const setSpreadSheetImport = useSetRecoilState(spreadsheetImportState);
const openSpreadsheetImport = (
options: Omit<SpreadsheetOptions<T>, 'isOpen' | 'onClose'>,
) => {
setSpreadSheetImport({
isOpen: true,
options,
});
};
return { openSpreadsheetImport };
};

View File

@ -3,12 +3,12 @@ import { SetRequired } from 'type-fest';
import { RsiContext } from '@/spreadsheet-import/components/Providers'; import { RsiContext } from '@/spreadsheet-import/components/Providers';
import { defaultSpreadsheetImportProps } from '@/spreadsheet-import/provider/components/SpreadsheetImport'; import { defaultSpreadsheetImportProps } from '@/spreadsheet-import/provider/components/SpreadsheetImport';
import { SpreadsheetOptions } from '@/spreadsheet-import/types'; import { SpreadsheetImportDialogOptions } from '@/spreadsheet-import/types';
export const useSpreadsheetImportInternal = <T extends string>() => export const useSpreadsheetImportInternal = <T extends string>() =>
useContext< useContext<
SetRequired< SetRequired<
SpreadsheetOptions<T>, SpreadsheetImportDialogOptions<T>,
keyof typeof defaultSpreadsheetImportProps keyof typeof defaultSpreadsheetImportProps
> >
>(RsiContext); >(RsiContext);

View File

@ -1,7 +1,7 @@
import { ModalWrapper } from '@/spreadsheet-import/components/ModalWrapper'; import { ModalWrapper } from '@/spreadsheet-import/components/ModalWrapper';
import { Providers } from '@/spreadsheet-import/components/Providers'; import { Providers } from '@/spreadsheet-import/components/Providers';
import { Steps } from '@/spreadsheet-import/steps/components/Steps'; import { Steps } from '@/spreadsheet-import/steps/components/Steps';
import { SpreadsheetOptions as SpreadsheetImportProps } from '@/spreadsheet-import/types'; import { SpreadsheetImportDialogOptions as SpreadsheetImportProps } from '@/spreadsheet-import/types';
export const defaultSpreadsheetImportProps: Partial< export const defaultSpreadsheetImportProps: Partial<
SpreadsheetImportProps<any> SpreadsheetImportProps<any>
@ -10,7 +10,10 @@ export const defaultSpreadsheetImportProps: Partial<
allowInvalidSubmit: true, allowInvalidSubmit: true,
autoMapDistance: 2, autoMapDistance: 2,
uploadStepHook: async (value) => value, uploadStepHook: async (value) => value,
selectHeaderStepHook: async (headerValues, data) => ({ headerValues, data }), selectHeaderStepHook: async (headerValues, data) => ({
headerRow: headerValues,
importedRows: data,
}),
matchColumnsStepHook: async (table) => table, matchColumnsStepHook: async (table) => table,
dateFormat: 'yyyy-mm-dd', // ISO 8601, dateFormat: 'yyyy-mm-dd', // ISO 8601,
parseRaw: true, parseRaw: true,

View File

@ -1,7 +1,7 @@
import React from 'react'; import React from 'react';
import { useRecoilState } from 'recoil'; import { useRecoilState } from 'recoil';
import { spreadsheetImportState } from '@/spreadsheet-import/states/spreadsheetImportState'; import { spreadsheetImportDialogState } from '@/spreadsheet-import/states/spreadsheetImportDialogState';
import { SpreadsheetImport } from './SpreadsheetImport'; import { SpreadsheetImport } from './SpreadsheetImport';
@ -10,12 +10,12 @@ type SpreadsheetImportProviderProps = React.PropsWithChildren;
export const SpreadsheetImportProvider = ( export const SpreadsheetImportProvider = (
props: SpreadsheetImportProviderProps, props: SpreadsheetImportProviderProps,
) => { ) => {
const [spreadsheetImport, setSpreadsheetImport] = useRecoilState( const [spreadsheetImportDialog, setSpreadsheetImportDialog] = useRecoilState(
spreadsheetImportState, spreadsheetImportDialogState,
); );
const handleClose = () => { const handleClose = () => {
setSpreadsheetImport({ setSpreadsheetImportDialog({
isOpen: false, isOpen: false,
options: null, options: null,
}); });
@ -24,12 +24,12 @@ export const SpreadsheetImportProvider = (
return ( return (
<> <>
{props.children} {props.children}
{spreadsheetImport.isOpen && spreadsheetImport.options && ( {spreadsheetImportDialog.isOpen && spreadsheetImportDialog.options && (
<SpreadsheetImport <SpreadsheetImport
isOpen={true} isOpen={true}
onClose={handleClose} onClose={handleClose}
// eslint-disable-next-line react/jsx-props-no-spreading // eslint-disable-next-line react/jsx-props-no-spreading
{...spreadsheetImport.options} {...spreadsheetImportDialog.options}
/> />
)} )}
</> </>

View File

@ -0,0 +1,18 @@
import { createState } from 'twenty-ui';
import { SpreadsheetImportDialogOptions } from '../types';
export type SpreadsheetImportDialogState<T extends string> = {
isOpen: boolean;
options: Omit<SpreadsheetImportDialogOptions<T>, 'isOpen' | 'onClose'> | null;
};
export const spreadsheetImportDialogState = createState<
SpreadsheetImportDialogState<any>
>({
key: 'spreadsheetImportDialogState',
defaultValue: {
isOpen: false,
options: null,
},
});

View File

@ -1,16 +0,0 @@
import { createState } from 'twenty-ui';
import { SpreadsheetOptions } from '../types';
export type SpreadsheetImportState<T extends string> = {
isOpen: boolean;
options: Omit<SpreadsheetOptions<T>, 'isOpen' | 'onClose'> | null;
};
export const spreadsheetImportState = createState<SpreadsheetImportState<any>>({
key: 'spreadsheetImportState',
defaultValue: {
isOpen: false,
options: null,
},
});

View File

@ -1,10 +1,10 @@
import { useCallback, useEffect, useMemo, useState } from 'react';
import styled from '@emotion/styled'; import styled from '@emotion/styled';
import { useCallback, useEffect, useMemo, useState } from 'react';
import { Heading } from '@/spreadsheet-import/components/Heading'; import { Heading } from '@/spreadsheet-import/components/Heading';
import { StepNavigationButton } from '@/spreadsheet-import/components/StepNavigationButton'; import { StepNavigationButton } from '@/spreadsheet-import/components/StepNavigationButton';
import { useSpreadsheetImportInternal } from '@/spreadsheet-import/hooks/useSpreadsheetImportInternal'; import { useSpreadsheetImportInternal } from '@/spreadsheet-import/hooks/useSpreadsheetImportInternal';
import { Field, RawData } from '@/spreadsheet-import/types'; import { Field, ImportedRow } from '@/spreadsheet-import/types';
import { findUnmatchedRequiredFields } from '@/spreadsheet-import/utils/findUnmatchedRequiredFields'; import { findUnmatchedRequiredFields } from '@/spreadsheet-import/utils/findUnmatchedRequiredFields';
import { getMatchedColumns } from '@/spreadsheet-import/utils/getMatchedColumns'; import { getMatchedColumns } from '@/spreadsheet-import/utils/getMatchedColumns';
import { normalizeTableData } from '@/spreadsheet-import/utils/normalizeTableData'; import { normalizeTableData } from '@/spreadsheet-import/utils/normalizeTableData';
@ -46,9 +46,13 @@ const StyledColumn = styled.span`
`; `;
export type MatchColumnsStepProps<T extends string> = { export type MatchColumnsStepProps<T extends string> = {
data: RawData[]; data: ImportedRow[];
headerValues: RawData; headerValues: ImportedRow;
onContinue: (data: any[], rawData: RawData[], columns: Columns<T>) => void; onContinue: (
data: any[],
rawData: ImportedRow[],
columns: Columns<T>,
) => void;
onBack: () => void; onBack: () => void;
}; };
@ -67,23 +71,27 @@ export type MatchedOptions<T> = {
}; };
type EmptyColumn = { type: ColumnType.empty; index: number; header: string }; type EmptyColumn = { type: ColumnType.empty; index: number; header: string };
type IgnoredColumn = { type IgnoredColumn = {
type: ColumnType.ignored; type: ColumnType.ignored;
index: number; index: number;
header: string; header: string;
}; };
type MatchedColumn<T> = { type MatchedColumn<T> = {
type: ColumnType.matched; type: ColumnType.matched;
index: number; index: number;
header: string; header: string;
value: T; value: T;
}; };
type MatchedSwitchColumn<T> = { type MatchedSwitchColumn<T> = {
type: ColumnType.matchedCheckbox; type: ColumnType.matchedCheckbox;
index: number; index: number;
header: string; header: string;
value: T; value: T;
}; };
export type MatchedSelectColumn<T> = { export type MatchedSelectColumn<T> = {
type: ColumnType.matchedSelect; type: ColumnType.matchedSelect;
index: number; index: number;
@ -91,6 +99,7 @@ export type MatchedSelectColumn<T> = {
value: T; value: T;
matchedOptions: Partial<MatchedOptions<T>>[]; matchedOptions: Partial<MatchedOptions<T>>[];
}; };
export type MatchedSelectOptionsColumn<T> = { export type MatchedSelectOptionsColumn<T> = {
type: ColumnType.matchedSelectOptions; type: ColumnType.matchedSelectOptions;
index: number; index: number;
@ -271,7 +280,7 @@ export const MatchColumnsStep = <T extends string>({
renderUserColumn={(columns, columnIndex) => ( renderUserColumn={(columns, columnIndex) => (
<UserTableColumn <UserTableColumn
column={columns[columnIndex]} column={columns[columnIndex]}
entries={dataExample.map( importedRow={dataExample.map(
(row) => row[columns[columnIndex].index], (row) => row[columns[columnIndex].index],
)} )}
/> />

View File

@ -1,6 +1,6 @@
import styled from '@emotion/styled'; import styled from '@emotion/styled';
import { RawData } from '@/spreadsheet-import/types'; import { ImportedRow } from '@/spreadsheet-import/types';
import { isDefined } from '~/utils/isDefined'; import { isDefined } from '~/utils/isDefined';
import { Column } from '../MatchColumnsStep'; import { Column } from '../MatchColumnsStep';
@ -31,20 +31,22 @@ const StyledExample = styled.span`
type UserTableColumnProps<T extends string> = { type UserTableColumnProps<T extends string> = {
column: Column<T>; column: Column<T>;
entries: RawData; importedRow: ImportedRow;
}; };
export const UserTableColumn = <T extends string>({ export const UserTableColumn = <T extends string>({
column, column,
entries, importedRow,
}: UserTableColumnProps<T>) => { }: UserTableColumnProps<T>) => {
const { header } = column; const { header } = column;
const entry = entries.find(isDefined); const firstDefinedValue = importedRow.find(isDefined);
return ( return (
<StyledContainer> <StyledContainer>
<StyledValue>{header}</StyledValue> <StyledValue>{header}</StyledValue>
{entry && <StyledExample>{`ex: ${entry}`}</StyledExample>} {firstDefinedValue && (
<StyledExample>{`ex: ${firstDefinedValue}`}</StyledExample>
)}
</StyledContainer> </StyledContainer>
); );
}; };

View File

@ -1,9 +1,9 @@
import { useCallback, useState } from 'react';
import styled from '@emotion/styled'; import styled from '@emotion/styled';
import { useCallback, useState } from 'react';
import { Heading } from '@/spreadsheet-import/components/Heading'; import { Heading } from '@/spreadsheet-import/components/Heading';
import { StepNavigationButton } from '@/spreadsheet-import/components/StepNavigationButton'; import { StepNavigationButton } from '@/spreadsheet-import/components/StepNavigationButton';
import { RawData } from '@/spreadsheet-import/types'; import { ImportedRow } from '@/spreadsheet-import/types';
import { Modal } from '@/ui/layout/modal/components/Modal'; import { Modal } from '@/ui/layout/modal/components/Modal';
import { SelectHeaderTable } from './components/SelectHeaderTable'; import { SelectHeaderTable } from './components/SelectHeaderTable';
@ -19,29 +19,36 @@ const StyledTableContainer = styled.div`
`; `;
type SelectHeaderStepProps = { type SelectHeaderStepProps = {
data: RawData[]; importedRows: ImportedRow[];
onContinue: (headerValues: RawData, data: RawData[]) => Promise<void>; onContinue: (
headerValues: ImportedRow,
importedRows: ImportedRow[],
) => Promise<void>;
onBack: () => void; onBack: () => void;
}; };
export const SelectHeaderStep = ({ export const SelectHeaderStep = ({
data, importedRows,
onContinue, onContinue,
onBack, onBack,
}: SelectHeaderStepProps) => { }: SelectHeaderStepProps) => {
const [selectedRows, setSelectedRows] = useState<ReadonlySet<number>>( const [selectedRowIndexes, setSelectedRowIndexes] = useState<
new Set([0]), ReadonlySet<number>
); >(new Set([0]));
const [isLoading, setIsLoading] = useState(false); const [isLoading, setIsLoading] = useState(false);
const handleContinue = useCallback(async () => { const handleContinue = useCallback(async () => {
const [selectedRowIndex] = Array.from(new Set(selectedRows)); const [selectedRowIndex] = Array.from(new Set(selectedRowIndexes));
// We consider data above header to be redundant // We consider data above header to be redundant
const trimmedData = data.slice(selectedRowIndex + 1); const trimmedData = importedRows.slice(selectedRowIndex + 1);
setIsLoading(true); setIsLoading(true);
await onContinue(data[selectedRowIndex], trimmedData);
await onContinue(importedRows[selectedRowIndex], trimmedData);
setIsLoading(false); setIsLoading(false);
}, [onContinue, data, selectedRows]); }, [onContinue, importedRows, selectedRowIndexes]);
return ( return (
<> <>
@ -49,9 +56,9 @@ export const SelectHeaderStep = ({
<StyledHeading title="Select header row" /> <StyledHeading title="Select header row" />
<StyledTableContainer> <StyledTableContainer>
<SelectHeaderTable <SelectHeaderTable
data={data} importedRows={importedRows}
selectedRows={selectedRows} selectedRowIndexes={selectedRowIndexes}
setSelectedRows={setSelectedRows} setSelectedRowIndexes={setSelectedRowIndexes}
/> />
</StyledTableContainer> </StyledTableContainer>
</Modal.Content> </Modal.Content>

View File

@ -1,7 +1,7 @@
// @ts-expect-error // Todo: remove usage of react-data-grid // @ts-expect-error // Todo: remove usage of react-data-grid
import { Column, FormatterProps, useRowSelection } from 'react-data-grid'; import { Column, FormatterProps, useRowSelection } from 'react-data-grid';
import { RawData } from '@/spreadsheet-import/types'; import { ImportedRow } from '@/spreadsheet-import/types';
import { Radio } from '@/ui/input/components/Radio'; import { Radio } from '@/ui/input/components/Radio';
const SELECT_COLUMN_KEY = 'select-row'; const SELECT_COLUMN_KEY = 'select-row';
@ -39,7 +39,7 @@ export const SelectColumn: Column<any, any> = {
formatter: SelectFormatter, formatter: SelectFormatter,
}; };
export const generateSelectionColumns = (data: RawData[]) => { export const generateSelectionColumns = (data: ImportedRow[]) => {
const longestRowLength = data.reduce( const longestRowLength = data.reduce(
(acc, curr) => (acc > curr.length ? acc : curr.length), (acc, curr) => (acc > curr.length ? acc : curr.length),
0, 0,

View File

@ -1,41 +1,44 @@
import { useMemo } from 'react'; import { useMemo } from 'react';
import { Table } from '@/spreadsheet-import/components/Table'; import { Table } from '@/spreadsheet-import/components/Table';
import { RawData } from '@/spreadsheet-import/types'; import { ImportedRow } from '@/spreadsheet-import/types';
import { generateSelectionColumns } from './SelectColumn'; import { generateSelectionColumns } from './SelectColumn';
interface SelectHeaderTableProps { type SelectHeaderTableProps = {
data: RawData[]; importedRows: ImportedRow[];
selectedRows: ReadonlySet<number>; selectedRowIndexes: ReadonlySet<number>;
setSelectedRows: (rows: ReadonlySet<number>) => void; setSelectedRowIndexes: (rowIndexes: ReadonlySet<number>) => void;
} };
export const SelectHeaderTable = ({ export const SelectHeaderTable = ({
data, importedRows,
selectedRows, selectedRowIndexes,
setSelectedRows, setSelectedRowIndexes,
}: SelectHeaderTableProps) => { }: SelectHeaderTableProps) => {
const columns = useMemo(() => generateSelectionColumns(data), [data]); const columns = useMemo(
() => generateSelectionColumns(importedRows),
[importedRows],
);
return ( return (
<Table <Table
// Todo: remove usage of react-data-grid // Todo: remove usage of react-data-grid
rowKeyGetter={(row: any) => data.indexOf(row)} rowKeyGetter={(row: any) => importedRows.indexOf(row)}
rows={data} rows={importedRows}
columns={columns} columns={columns}
selectedRows={selectedRows} selectedRowIndexes={selectedRowIndexes}
onSelectedRowsChange={(newRows: any) => { onSelectedRowIndexesChange={(newRowIndexes: number[]) => {
// allow selecting only one row // allow selecting only one row
newRows.forEach((value: any) => { newRowIndexes.forEach((value: any) => {
if (!selectedRows.has(value as number)) { if (!selectedRowIndexes.has(value as number)) {
setSelectedRows(new Set([value as number])); setSelectedRowIndexes(new Set([value as number]));
return; return;
} }
}); });
}} }}
onRowClick={(row: any) => { onRowClick={(row: any) => {
setSelectedRows(new Set([data.indexOf(row)])); setSelectedRowIndexes(new Set([importedRows.indexOf(row)]));
}} }}
headerRowHeight={0} headerRowHeight={0}
/> />

View File

@ -1,10 +1,10 @@
import { useCallback, useState } from 'react';
import { useTheme } from '@emotion/react'; import { useTheme } from '@emotion/react';
import styled from '@emotion/styled'; import styled from '@emotion/styled';
import { useCallback, useState } from 'react';
import { WorkBook } from 'xlsx-ugnis'; import { WorkBook } from 'xlsx-ugnis';
import { useSpreadsheetImportInternal } from '@/spreadsheet-import/hooks/useSpreadsheetImportInternal'; import { useSpreadsheetImportInternal } from '@/spreadsheet-import/hooks/useSpreadsheetImportInternal';
import { RawData } from '@/spreadsheet-import/types'; import { ImportedRow } from '@/spreadsheet-import/types';
import { exceedsMaxRecords } from '@/spreadsheet-import/utils/exceedsMaxRecords'; import { exceedsMaxRecords } from '@/spreadsheet-import/utils/exceedsMaxRecords';
import { mapWorkbook } from '@/spreadsheet-import/utils/mapWorkbook'; import { mapWorkbook } from '@/spreadsheet-import/utils/mapWorkbook';
import { CircularProgressBar } from '@/ui/feedback/progress-bar/components/CircularProgressBar'; import { CircularProgressBar } from '@/ui/feedback/progress-bar/components/CircularProgressBar';
@ -42,12 +42,12 @@ export type StepState =
} }
| { | {
type: StepType.selectHeader; type: StepType.selectHeader;
data: RawData[]; data: ImportedRow[];
} }
| { | {
type: StepType.matchColumns; type: StepType.matchColumns;
data: RawData[]; data: ImportedRow[];
headerValues: RawData; headerValues: ImportedRow;
} }
| { | {
type: StepType.validateData; type: StepType.validateData;
@ -131,10 +131,8 @@ export const UploadFlow = ({ nextStep, prevStep }: UploadFlowProps) => {
// Automatically select first row as header // Automatically select first row as header
const trimmedData = mappedWorkbook.slice(1); const trimmedData = mappedWorkbook.slice(1);
const { data, headerValues } = await selectHeaderStepHook( const { importedRows: data, headerRow: headerValues } =
mappedWorkbook[0], await selectHeaderStepHook(mappedWorkbook[0], trimmedData);
trimmedData,
);
setState({ setState({
type: StepType.matchColumns, type: StepType.matchColumns,
@ -186,12 +184,11 @@ export const UploadFlow = ({ nextStep, prevStep }: UploadFlowProps) => {
case StepType.selectHeader: case StepType.selectHeader:
return ( return (
<SelectHeaderStep <SelectHeaderStep
data={state.data} importedRows={state.data}
onContinue={async (...args) => { onContinue={async (...args) => {
try { try {
const { data, headerValues } = await selectHeaderStepHook( const { importedRows: data, headerRow: headerValues } =
...args, await selectHeaderStepHook(...args);
);
setState({ setState({
type: StepType.matchColumns, type: StepType.matchColumns,
data, data,

View File

@ -1,6 +1,6 @@
import styled from '@emotion/styled';
import { useState } from 'react'; import { useState } from 'react';
import { useDropzone } from 'react-dropzone'; import { useDropzone } from 'react-dropzone';
import styled from '@emotion/styled';
import * as XLSX from 'xlsx-ugnis'; import * as XLSX from 'xlsx-ugnis';
import { useSpreadsheetImportInternal } from '@/spreadsheet-import/hooks/useSpreadsheetImportInternal'; import { useSpreadsheetImportInternal } from '@/spreadsheet-import/hooks/useSpreadsheetImportInternal';
@ -79,11 +79,7 @@ const StyledText = styled.span`
font-size: ${({ theme }) => theme.font.size.sm}; font-size: ${({ theme }) => theme.font.size.sm};
font-weight: ${({ theme }) => theme.font.weight.medium}; font-weight: ${({ theme }) => theme.font.weight.medium};
text-align: center; text-align: center;
`; padding: 15px;
const StyledButton = styled(MainButton)`
margin-top: ${({ theme }) => theme.spacing(2)};
width: 200px;
`; `;
type DropZoneProps = { type DropZoneProps = {
@ -151,7 +147,7 @@ export const DropZone = ({ onContinue, isLoading }: DropZoneProps) => {
) : ( ) : (
<> <>
<StyledText>Upload .xlsx, .xls or .csv file</StyledText> <StyledText>Upload .xlsx, .xls or .csv file</StyledText>
<StyledButton onClick={open} title="Select file" /> <MainButton onClick={open} title="Select file" />
</> </>
)} )}
</StyledContainer> </StyledContainer>

View File

@ -1,7 +1,7 @@
import styled from '@emotion/styled';
import { useCallback, useMemo, useState } from 'react'; import { useCallback, useMemo, useState } from 'react';
// @ts-expect-error Todo: remove usage of react-data-grid // @ts-expect-error Todo: remove usage of react-data-grid
import { RowsChangeData } from 'react-data-grid'; import { RowsChangeData } from 'react-data-grid';
import styled from '@emotion/styled';
import { IconTrash } from 'twenty-ui'; import { IconTrash } from 'twenty-ui';
import { Heading } from '@/spreadsheet-import/components/Heading'; import { Heading } from '@/spreadsheet-import/components/Heading';
@ -12,7 +12,10 @@ import {
Columns, Columns,
ColumnType, ColumnType,
} from '@/spreadsheet-import/steps/components/MatchColumnsStep/MatchColumnsStep'; } from '@/spreadsheet-import/steps/components/MatchColumnsStep/MatchColumnsStep';
import { Data } from '@/spreadsheet-import/types'; import {
ImportedStructuredRow,
ImportValidationResult,
} from '@/spreadsheet-import/types';
import { addErrorsAndRunHooks } from '@/spreadsheet-import/utils/dataMutations'; import { addErrorsAndRunHooks } from '@/spreadsheet-import/utils/dataMutations';
import { useDialogManager } from '@/ui/feedback/dialog-manager/hooks/useDialogManager'; import { useDialogManager } from '@/ui/feedback/dialog-manager/hooks/useDialogManager';
import { Button } from '@/ui/input/button/components/Button'; import { Button } from '@/ui/input/button/components/Button';
@ -21,7 +24,7 @@ import { Modal } from '@/ui/layout/modal/components/Modal';
import { isDefined } from '~/utils/isDefined'; import { isDefined } from '~/utils/isDefined';
import { generateColumns } from './components/columns'; import { generateColumns } from './components/columns';
import { Meta } from './types'; import { ImportedStructuredRowMetadata } from './types';
const StyledContent = styled(Modal.Content)` const StyledContent = styled(Modal.Content)`
padding-left: ${({ theme }) => theme.spacing(6)}; padding-left: ${({ theme }) => theme.spacing(6)};
@ -65,7 +68,7 @@ const StyledNoRowsContainer = styled.div`
`; `;
type ValidationStepProps<T extends string> = { type ValidationStepProps<T extends string> = {
initialData: Data<T>[]; initialData: ImportedStructuredRow<T>[];
importedColumns: Columns<string>; importedColumns: Columns<string>;
file: File; file: File;
onSubmitStart?: () => void; onSubmitStart?: () => void;
@ -83,7 +86,9 @@ export const ValidationStep = <T extends string>({
const { fields, onClose, onSubmit, rowHook, tableHook } = const { fields, onClose, onSubmit, rowHook, tableHook } =
useSpreadsheetImportInternal<T>(); useSpreadsheetImportInternal<T>();
const [data, setData] = useState<(Data<T> & Meta)[]>( const [data, setData] = useState<
(ImportedStructuredRow<T> & ImportedStructuredRowMetadata)[]
>(
useMemo( useMemo(
() => addErrorsAndRunHooks<T>(initialData, fields, rowHook, tableHook), () => addErrorsAndRunHooks<T>(initialData, fields, rowHook, tableHook),
// eslint-disable-next-line react-hooks/exhaustive-deps // eslint-disable-next-line react-hooks/exhaustive-deps
@ -173,7 +178,11 @@ export const ValidationStep = <T extends string>({
return data; return data;
}, [data, filterByErrors]); }, [data, filterByErrors]);
const rowKeyGetter = useCallback((row: Data<T> & Meta) => row.__index, []); const rowKeyGetter = useCallback(
(row: ImportedStructuredRow<T> & ImportedStructuredRowMetadata) =>
row.__index,
[],
);
const submitData = async () => { const submitData = async () => {
const calculatedData = data.reduce( const calculatedData = data.reduce(
@ -182,15 +191,23 @@ export const ValidationStep = <T extends string>({
if (isDefined(__errors)) { if (isDefined(__errors)) {
for (const key in __errors) { for (const key in __errors) {
if (__errors[key].level === 'error') { if (__errors[key].level === 'error') {
acc.invalidData.push(values as unknown as Data<T>); acc.invalidStructuredRows.push(
values as unknown as ImportedStructuredRow<T>,
);
return acc; return acc;
} }
} }
} }
acc.validData.push(values as unknown as Data<T>); acc.validStructuredRows.push(
values as unknown as ImportedStructuredRow<T>,
);
return acc; return acc;
}, },
{ validData: [] as Data<T>[], invalidData: [] as Data<T>[], all: data }, {
validStructuredRows: [] as ImportedStructuredRow<T>[],
invalidStructuredRows: [] as ImportedStructuredRow<T>[],
allStructuredRows: data,
} satisfies ImportValidationResult<T>,
); );
onSubmitStart?.(); onSubmitStart?.();
await onSubmit(calculatedData, file); await onSubmit(calculatedData, file);

View File

@ -1,17 +1,17 @@
import styled from '@emotion/styled';
// @ts-expect-error // Todo: remove usage of react-data-grid // @ts-expect-error // Todo: remove usage of react-data-grid
import { Column, useRowSelection } from 'react-data-grid'; import { Column, useRowSelection } from 'react-data-grid';
import { createPortal } from 'react-dom'; import { createPortal } from 'react-dom';
import styled from '@emotion/styled';
import { AppTooltip } from 'twenty-ui'; import { AppTooltip } from 'twenty-ui';
import { MatchColumnSelect } from '@/spreadsheet-import/components/MatchColumnSelect'; import { MatchColumnSelect } from '@/spreadsheet-import/components/MatchColumnSelect';
import { Data, Fields } from '@/spreadsheet-import/types'; import { Fields, ImportedStructuredRow } from '@/spreadsheet-import/types';
import { Checkbox, CheckboxVariant } from '@/ui/input/components/Checkbox'; import { Checkbox, CheckboxVariant } from '@/ui/input/components/Checkbox';
import { TextInput } from '@/ui/input/components/TextInput'; import { TextInput } from '@/ui/input/components/TextInput';
import { Toggle } from '@/ui/input/components/Toggle'; import { Toggle } from '@/ui/input/components/Toggle';
import { isDefined } from '~/utils/isDefined'; import { isDefined } from '~/utils/isDefined';
import { Meta } from '../types'; import { ImportedStructuredRowMetadata } from '../types';
const StyledHeaderContainer = styled.div` const StyledHeaderContainer = styled.div`
align-items: center; align-items: center;
@ -63,7 +63,7 @@ const SELECT_COLUMN_KEY = 'select-row';
export const generateColumns = <T extends string>( export const generateColumns = <T extends string>(
fields: Fields<T>, fields: Fields<T>,
): Column<Data<T> & Meta>[] => [ ): Column<ImportedStructuredRow<T> & ImportedStructuredRowMetadata>[] => [
{ {
key: SELECT_COLUMN_KEY, key: SELECT_COLUMN_KEY,
name: '', name: '',
@ -96,7 +96,9 @@ export const generateColumns = <T extends string>(
}, },
}, },
...fields.map( ...fields.map(
(column): Column<Data<T> & Meta> => ({ (
column,
): Column<ImportedStructuredRow<T> & ImportedStructuredRowMetadata> => ({
key: column.key, key: column.key,
name: column.label, name: column.label,
minWidth: 150, minWidth: 150,
@ -120,7 +122,8 @@ export const generateColumns = <T extends string>(
editable: column.fieldType.type !== 'checkbox', editable: column.fieldType.type !== 'checkbox',
// Todo: remove usage of react-data-grid // Todo: remove usage of react-data-grid
editor: ({ row, onRowChange, onClose }: any) => { editor: ({ row, onRowChange, onClose }: any) => {
const columnKey = column.key as keyof (Data<T> & Meta); const columnKey = column.key as keyof (ImportedStructuredRow<T> &
ImportedStructuredRowMetadata);
let component; let component;
switch (column.fieldType.type) { switch (column.fieldType.type) {
@ -167,7 +170,8 @@ export const generateColumns = <T extends string>(
}, },
// Todo: remove usage of react-data-grid // Todo: remove usage of react-data-grid
formatter: ({ row, onRowChange }: { row: any; onRowChange: any }) => { formatter: ({ row, onRowChange }: { row: any; onRowChange: any }) => {
const columnKey = column.key as keyof (Data<T> & Meta); const columnKey = column.key as keyof (ImportedStructuredRow<T> &
ImportedStructuredRowMetadata);
let component; let component;
switch (column.fieldType.type) { switch (column.fieldType.type) {
@ -226,7 +230,7 @@ export const generateColumns = <T extends string>(
return component; return component;
}, },
cellClass: (row: Meta) => { cellClass: (row: ImportedStructuredRowMetadata) => {
switch (row.__errors?.[column.key]?.level) { switch (row.__errors?.[column.key]?.level) {
case 'error': case 'error':
return 'rdg-cell-error'; return 'rdg-cell-error';

View File

@ -1,5 +1,8 @@
import { Info } from '@/spreadsheet-import/types'; import { Info } from '@/spreadsheet-import/types';
export type Meta = { __index: string; __errors?: Error | null }; export type ImportedStructuredRowMetadata = {
__index: string;
__errors?: Error | null;
};
export type Error = { [key: string]: Info }; export type Error = { [key: string]: Info };
export type Errors = { [id: string]: Error }; export type Errors = { [id: string]: Error };

View File

@ -24,7 +24,7 @@ export const Default = () => (
<Providers values={mockRsiValues}> <Providers values={mockRsiValues}>
<ModalWrapper isOpen={true} onClose={() => null}> <ModalWrapper isOpen={true} onClose={() => null}>
<SelectHeaderStep <SelectHeaderStep
data={headerSelectionTableFields} importedRows={headerSelectionTableFields}
onContinue={() => Promise.resolve()} onContinue={() => Promise.resolve()}
onBack={() => Promise.resolve()} onBack={() => Promise.resolve()}
/> />

View File

@ -1,6 +1,9 @@
import { defaultSpreadsheetImportProps } from '@/spreadsheet-import/provider/components/SpreadsheetImport'; import { defaultSpreadsheetImportProps } from '@/spreadsheet-import/provider/components/SpreadsheetImport';
import { Columns } from '@/spreadsheet-import/steps/components/MatchColumnsStep/MatchColumnsStep'; import { Columns } from '@/spreadsheet-import/steps/components/MatchColumnsStep/MatchColumnsStep';
import { Fields, SpreadsheetOptions } from '@/spreadsheet-import/types'; import {
Fields,
SpreadsheetImportDialogOptions,
} from '@/spreadsheet-import/types';
import { sleep } from '~/utils/sleep'; import { sleep } from '~/utils/sleep';
const fields = [ const fields = [
@ -13,7 +16,7 @@ const fields = [
type: 'input', type: 'input',
}, },
example: 'Stephanie', example: 'Stephanie',
validations: [ fieldValidationDefinitions: [
{ {
rule: 'required', rule: 'required',
errorMessage: 'Name is required', errorMessage: 'Name is required',
@ -29,7 +32,7 @@ const fields = [
type: 'input', type: 'input',
}, },
example: 'McDonald', example: 'McDonald',
validations: [ fieldValidationDefinitions: [
{ {
rule: 'unique', rule: 'unique',
errorMessage: 'Last name must be unique', errorMessage: 'Last name must be unique',
@ -47,7 +50,7 @@ const fields = [
type: 'input', type: 'input',
}, },
example: '23', example: '23',
validations: [ fieldValidationDefinitions: [
{ {
rule: 'regex', rule: 'regex',
value: '^\\d+$', value: '^\\d+$',
@ -69,7 +72,7 @@ const fields = [
], ],
}, },
example: 'Team one', example: 'Team one',
validations: [ fieldValidationDefinitions: [
{ {
rule: 'required', rule: 'required',
errorMessage: 'Team is required', errorMessage: 'Team is required',
@ -117,7 +120,7 @@ export const importedColums: Columns<string> = [
]; ];
const mockComponentBehaviourForTypes = <T extends string>( const mockComponentBehaviourForTypes = <T extends string>(
props: SpreadsheetOptions<T>, props: SpreadsheetImportDialogOptions<T>,
) => props; ) => props;
export const mockRsiValues = mockComponentBehaviourForTypes({ export const mockRsiValues = mockComponentBehaviourForTypes({
@ -142,8 +145,8 @@ export const mockRsiValues = mockComponentBehaviourForTypes({
}), }),
); );
return { return {
headerValues: hData, headerRow: hData,
data, importedRows: data,
}; };
}, },
// Runs after column matching and on entry change, more performant // Runs after column matching and on entry change, more performant

View File

@ -3,34 +3,37 @@ import { ReadonlyDeep } from 'type-fest';
import { Columns } from '@/spreadsheet-import/steps/components/MatchColumnsStep/MatchColumnsStep'; import { Columns } from '@/spreadsheet-import/steps/components/MatchColumnsStep/MatchColumnsStep';
import { StepState } from '@/spreadsheet-import/steps/components/UploadFlow'; import { StepState } from '@/spreadsheet-import/steps/components/UploadFlow';
import { Meta } from '@/spreadsheet-import/steps/components/ValidationStep/types'; import { ImportedStructuredRowMetadata } from '@/spreadsheet-import/steps/components/ValidationStep/types';
export type SpreadsheetOptions<Keys extends string> = { export type SpreadsheetImportDialogOptions<FieldNames extends string> = {
// Is modal visible. // Is modal visible.
isOpen: boolean; isOpen: boolean;
// callback when RSI is closed before final submit // callback when RSI is closed before final submit
onClose: () => void; onClose: () => void;
// Field description for requested data // Field description for requested data
fields: Fields<Keys>; fields: Fields<FieldNames>;
// Runs after file upload step, receives and returns raw sheet data // Runs after file upload step, receives and returns raw sheet data
uploadStepHook?: (data: RawData[]) => Promise<RawData[]>; uploadStepHook?: (importedRows: ImportedRow[]) => Promise<ImportedRow[]>;
// Runs after header selection step, receives and returns raw sheet data // Runs after header selection step, receives and returns raw sheet data
selectHeaderStepHook?: ( selectHeaderStepHook?: (
headerValues: RawData, headerRow: ImportedRow,
data: RawData[], importedRows: ImportedRow[],
) => Promise<{ headerValues: RawData; data: RawData[] }>; ) => Promise<{ headerRow: ImportedRow; importedRows: ImportedRow[] }>;
// Runs once before validation step, used for data mutations and if you want to change how columns were matched // Runs once before validation step, used for data mutations and if you want to change how columns were matched
matchColumnsStepHook?: ( matchColumnsStepHook?: (
table: Data<Keys>[], importedStructuredRows: ImportedStructuredRow<FieldNames>[],
rawData: RawData[], importedRows: ImportedRow[],
columns: Columns<Keys>, columns: Columns<FieldNames>,
) => Promise<Data<Keys>[]>; ) => Promise<ImportedStructuredRow<FieldNames>[]>;
// Runs after column matching and on entry change // Runs after column matching and on entry change
rowHook?: RowHook<Keys>; rowHook?: RowHook<FieldNames>;
// Runs after column matching and on entry change // Runs after column matching and on entry change
tableHook?: TableHook<Keys>; tableHook?: TableHook<FieldNames>;
// Function called after user finishes the flow // Function called after user finishes the flow
onSubmit: (data: Result<Keys>, file: File) => Promise<void>; onSubmit: (
validationResult: ImportValidationResult<FieldNames>,
file: File,
) => Promise<void>;
// Allows submitting with errors. Default: true // Allows submitting with errors. Default: true
allowInvalidSubmit?: boolean; allowInvalidSubmit?: boolean;
// Theme configuration passed to underlying Chakra-UI // Theme configuration passed to underlying Chakra-UI
@ -55,9 +58,9 @@ export type SpreadsheetOptions<Keys extends string> = {
selectHeader?: boolean; selectHeader?: boolean;
}; };
export type RawData = Array<string | undefined>; export type ImportedRow = Array<string | undefined>;
export type Data<T extends string> = { export type ImportedStructuredRow<T extends string> = {
[key in T]: string | boolean | undefined; [key in T]: string | boolean | undefined;
}; };
@ -76,7 +79,7 @@ export type Field<T extends string> = {
// Alternate labels used for fields' auto-matching, e.g. "fname" -> "firstName" // Alternate labels used for fields' auto-matching, e.g. "fname" -> "firstName"
alternateMatches?: string[]; alternateMatches?: string[];
// Validations used for field entries // Validations used for field entries
validations?: Validation[]; fieldValidationDefinitions?: FieldValidationDefinition[];
// Field entry component, default: Input // Field entry component, default: Input
fieldType: Checkbox | Select | Input; fieldType: Checkbox | Select | Input;
// UI-facing values shown to user as field examples pre-upload phase // UI-facing values shown to user as field examples pre-upload phase
@ -110,11 +113,19 @@ export type Input = {
type: 'input'; type: 'input';
}; };
export type Validation = export type FieldValidationDefinition =
| RequiredValidation | RequiredValidation
| UniqueValidation | UniqueValidation
| RegexValidation | RegexValidation
| FunctionValidation; | FunctionValidation
| ObjectValidation;
export type ObjectValidation = {
rule: 'object';
isValid: (objectValue: any) => boolean;
errorMessage: string;
level?: ErrorLevel;
};
export type RequiredValidation = { export type RequiredValidation = {
rule: 'required'; rule: 'required';
@ -145,14 +156,15 @@ export type FunctionValidation = {
}; };
export type RowHook<T extends string> = ( export type RowHook<T extends string> = (
row: Data<T>, row: ImportedStructuredRow<T>,
addError: (fieldKey: T, error: Info) => void, addError: (fieldKey: T, error: Info) => void,
table: Data<T>[], table: ImportedStructuredRow<T>[],
) => Data<T>; ) => ImportedStructuredRow<T>;
export type TableHook<T extends string> = ( export type TableHook<T extends string> = (
table: Data<T>[], table: ImportedStructuredRow<T>[],
addError: (rowIndex: number, fieldKey: T, error: Info) => void, addError: (rowIndex: number, fieldKey: T, error: Info) => void,
) => Data<T>[]; ) => ImportedStructuredRow<T>[];
export type ErrorLevel = 'info' | 'warning' | 'error'; export type ErrorLevel = 'info' | 'warning' | 'error';
@ -161,8 +173,9 @@ export type Info = {
level: ErrorLevel; level: ErrorLevel;
}; };
export type Result<T extends string> = { export type ImportValidationResult<T extends string> = {
validData: Data<T>[]; validStructuredRows: ImportedStructuredRow<T>[];
invalidData: Data<T>[]; invalidStructuredRows: ImportedStructuredRow<T>[];
all: (Data<T> & Meta)[]; allStructuredRows: (ImportedStructuredRow<T> &
ImportedStructuredRowMetadata)[];
}; };

View File

@ -1,6 +1,6 @@
import { import {
Data,
Field, Field,
ImportedStructuredRow,
Info, Info,
RowHook, RowHook,
TableHook, TableHook,
@ -8,11 +8,11 @@ import {
import { addErrorsAndRunHooks } from '@/spreadsheet-import/utils/dataMutations'; import { addErrorsAndRunHooks } from '@/spreadsheet-import/utils/dataMutations';
describe('addErrorsAndRunHooks', () => { describe('addErrorsAndRunHooks', () => {
type FullData = Data<'name' | 'age' | 'country'>; type FullData = ImportedStructuredRow<'name' | 'age' | 'country'>;
const requiredField: Field<'name'> = { const requiredField: Field<'name'> = {
key: 'name', key: 'name',
label: 'Name', label: 'Name',
validations: [{ rule: 'required' }], fieldValidationDefinitions: [{ rule: 'required' }],
icon: null, icon: null,
fieldType: { type: 'input' }, fieldType: { type: 'input' },
}; };
@ -20,7 +20,7 @@ describe('addErrorsAndRunHooks', () => {
const regexField: Field<'age'> = { const regexField: Field<'age'> = {
key: 'age', key: 'age',
label: 'Age', label: 'Age',
validations: [ fieldValidationDefinitions: [
{ rule: 'regex', value: '\\d+', errorMessage: 'Regex error' }, { rule: 'regex', value: '\\d+', errorMessage: 'Regex error' },
], ],
icon: null, icon: null,
@ -30,7 +30,7 @@ describe('addErrorsAndRunHooks', () => {
const uniqueField: Field<'country'> = { const uniqueField: Field<'country'> = {
key: 'country', key: 'country',
label: 'Country', label: 'Country',
validations: [{ rule: 'unique' }], fieldValidationDefinitions: [{ rule: 'unique' }],
icon: null, icon: null,
fieldType: { type: 'input' }, fieldType: { type: 'input' },
}; };
@ -38,7 +38,7 @@ describe('addErrorsAndRunHooks', () => {
const functionValidationFieldTrue: Field<'email'> = { const functionValidationFieldTrue: Field<'email'> = {
key: 'email', key: 'email',
label: 'Email', label: 'Email',
validations: [ fieldValidationDefinitions: [
{ {
rule: 'function', rule: 'function',
isValid: () => true, isValid: () => true,
@ -52,7 +52,7 @@ describe('addErrorsAndRunHooks', () => {
const functionValidationFieldFalse: Field<'email'> = { const functionValidationFieldFalse: Field<'email'> = {
key: 'email', key: 'email',
label: 'Email', label: 'Email',
validations: [ fieldValidationDefinitions: [
{ {
rule: 'function', rule: 'function',
isValid: () => false, isValid: () => false,
@ -63,8 +63,11 @@ describe('addErrorsAndRunHooks', () => {
fieldType: { type: 'input' }, fieldType: { type: 'input' },
}; };
const validData: Data<'name' | 'age'> = { name: 'John', age: '30' }; const validData: ImportedStructuredRow<'name' | 'age'> = {
const dataWithoutNameAndInvalidAge: Data<'name' | 'age'> = { name: 'John',
age: '30',
};
const dataWithoutNameAndInvalidAge: ImportedStructuredRow<'name' | 'age'> = {
name: '', name: '',
age: 'Invalid', age: 'Invalid',
}; };
@ -74,7 +77,7 @@ describe('addErrorsAndRunHooks', () => {
country: 'Brazil', country: 'Brazil',
}; };
const data: Data<'name' | 'age'>[] = [ const data: ImportedStructuredRow<'name' | 'age'>[] = [
validData, validData,
dataWithoutNameAndInvalidAge, dataWithoutNameAndInvalidAge,
]; ];
@ -180,7 +183,12 @@ describe('addErrorsAndRunHooks', () => {
it('should not add errors for unique field with empty values if allowEmpty is true', () => { it('should not add errors for unique field with empty values if allowEmpty is true', () => {
const result = addErrorsAndRunHooks( const result = addErrorsAndRunHooks(
[{ country: '' }, { country: '' }], [{ country: '' }, { country: '' }],
[{ ...uniqueField, validations: [{ rule: 'unique', allowEmpty: true }] }], [
{
...uniqueField,
fieldValidationDefinitions: [{ rule: 'unique', allowEmpty: true }],
},
],
); );
expect(result[0].__errors).toBeUndefined(); expect(result[0].__errors).toBeUndefined();

View File

@ -2,7 +2,7 @@ import {
Column, Column,
ColumnType, ColumnType,
} from '@/spreadsheet-import/steps/components/MatchColumnsStep/MatchColumnsStep'; } from '@/spreadsheet-import/steps/components/MatchColumnsStep/MatchColumnsStep';
import { Field, Validation } from '@/spreadsheet-import/types'; import { Field, FieldValidationDefinition } from '@/spreadsheet-import/types';
import { findUnmatchedRequiredFields } from '@/spreadsheet-import/utils/findUnmatchedRequiredFields'; import { findUnmatchedRequiredFields } from '@/spreadsheet-import/utils/findUnmatchedRequiredFields';
const nameField: Field<'Name'> = { const nameField: Field<'Name'> = {
@ -22,9 +22,15 @@ const ageField: Field<'Age'> = {
type: 'input', type: 'input',
}, },
}; };
const validations: Validation[] = [{ rule: 'required' }]; const validations: FieldValidationDefinition[] = [{ rule: 'required' }];
const nameFieldWithValidations: Field<'Name'> = { ...nameField, validations }; const nameFieldWithValidations: Field<'Name'> = {
const ageFieldWithValidations: Field<'Age'> = { ...ageField, validations }; ...nameField,
fieldValidationDefinitions: validations,
};
const ageFieldWithValidations: Field<'Age'> = {
...ageField,
fieldValidationDefinitions: validations,
};
type ColumnValues = 'Name' | 'Age'; type ColumnValues = 'Name' | 'Age';

View File

@ -3,11 +3,11 @@ import { v4 } from 'uuid';
import { import {
Errors, Errors,
Meta, ImportedStructuredRowMetadata,
} from '@/spreadsheet-import/steps/components/ValidationStep/types'; } from '@/spreadsheet-import/steps/components/ValidationStep/types';
import { import {
Data,
Fields, Fields,
ImportedStructuredRow,
Info, Info,
RowHook, RowHook,
TableHook, TableHook,
@ -16,11 +16,11 @@ import { isDefined } from '~/utils/isDefined';
import { isUndefinedOrNull } from '~/utils/isUndefinedOrNull'; import { isUndefinedOrNull } from '~/utils/isUndefinedOrNull';
export const addErrorsAndRunHooks = <T extends string>( export const addErrorsAndRunHooks = <T extends string>(
data: (Data<T> & Partial<Meta>)[], data: (ImportedStructuredRow<T> & Partial<ImportedStructuredRowMetadata>)[],
fields: Fields<T>, fields: Fields<T>,
rowHook?: RowHook<T>, rowHook?: RowHook<T>,
tableHook?: TableHook<T>, tableHook?: TableHook<T>,
): (Data<T> & Meta)[] => { ): (ImportedStructuredRow<T> & ImportedStructuredRowMetadata)[] => {
const errors: Errors = {}; const errors: Errors = {};
const addHookError = (rowIndex: number, fieldKey: T, error: Info) => { const addHookError = (rowIndex: number, fieldKey: T, error: Info) => {
@ -41,8 +41,8 @@ export const addErrorsAndRunHooks = <T extends string>(
} }
fields.forEach((field) => { fields.forEach((field) => {
field.validations?.forEach((validation) => { field.fieldValidationDefinitions?.forEach((fieldValidationDefinition) => {
switch (validation.rule) { switch (fieldValidationDefinition.rule) {
case 'unique': { case 'unique': {
const values = data.map((entry) => entry[field.key as T]); const values = data.map((entry) => entry[field.key as T]);
@ -51,7 +51,7 @@ export const addErrorsAndRunHooks = <T extends string>(
values.forEach((value) => { values.forEach((value) => {
if ( if (
validation.allowEmpty === true && fieldValidationDefinition.allowEmpty === true &&
(isUndefinedOrNull(value) || value === '' || !value) (isUndefinedOrNull(value) || value === '' || !value)
) { ) {
// If allowEmpty is set, we will not validate falsy fields such as undefined or empty string. // If allowEmpty is set, we will not validate falsy fields such as undefined or empty string.
@ -70,8 +70,10 @@ export const addErrorsAndRunHooks = <T extends string>(
errors[index] = { errors[index] = {
...errors[index], ...errors[index],
[field.key]: { [field.key]: {
level: validation.level || 'error', level: fieldValidationDefinition.level || 'error',
message: validation.errorMessage || 'Field must be unique', message:
fieldValidationDefinition.errorMessage ||
'Field must be unique',
}, },
}; };
} }
@ -88,8 +90,10 @@ export const addErrorsAndRunHooks = <T extends string>(
errors[index] = { errors[index] = {
...errors[index], ...errors[index],
[field.key]: { [field.key]: {
level: validation.level || 'error', level: fieldValidationDefinition.level || 'error',
message: validation.errorMessage || 'Field is required', message:
fieldValidationDefinition.errorMessage ||
'Field is required',
}, },
}; };
} }
@ -97,7 +101,10 @@ export const addErrorsAndRunHooks = <T extends string>(
break; break;
} }
case 'regex': { case 'regex': {
const regex = new RegExp(validation.value, validation.flags); const regex = new RegExp(
fieldValidationDefinition.value,
fieldValidationDefinition.flags,
);
data.forEach((entry, index) => { data.forEach((entry, index) => {
const value = entry[field.key]?.toString(); const value = entry[field.key]?.toString();
@ -105,10 +112,10 @@ export const addErrorsAndRunHooks = <T extends string>(
errors[index] = { errors[index] = {
...errors[index], ...errors[index],
[field.key]: { [field.key]: {
level: validation.level || 'error', level: fieldValidationDefinition.level || 'error',
message: message:
validation.errorMessage || fieldValidationDefinition.errorMessage ||
`Field did not match the regex /${validation.value}/${validation.flags} `, `Field did not match the regex /${fieldValidationDefinition.value}/${fieldValidationDefinition.flags} `,
}, },
}; };
} }
@ -119,12 +126,17 @@ export const addErrorsAndRunHooks = <T extends string>(
data.forEach((entry, index) => { data.forEach((entry, index) => {
const value = entry[field.key]?.toString(); const value = entry[field.key]?.toString();
if (isNonEmptyString(value) && !validation.isValid(value)) { if (
isNonEmptyString(value) &&
!fieldValidationDefinition.isValid(value)
) {
errors[index] = { errors[index] = {
...errors[index], ...errors[index],
[field.key]: { [field.key]: {
level: validation.level || 'error', level: fieldValidationDefinition.level || 'error',
message: validation.errorMessage || 'Field is invalid', message:
fieldValidationDefinition.errorMessage ||
'Field is invalid',
}, },
}; };
} }
@ -140,7 +152,8 @@ export const addErrorsAndRunHooks = <T extends string>(
if (!('__index' in value)) { if (!('__index' in value)) {
value.__index = v4(); value.__index = v4();
} }
const newValue = value as Data<T> & Meta; const newValue = value as ImportedStructuredRow<T> &
ImportedStructuredRowMetadata;
if (isDefined(errors[index])) { if (isDefined(errors[index])) {
return { ...newValue, __errors: errors[index] }; return { ...newValue, __errors: errors[index] };

View File

@ -8,7 +8,9 @@ export const findUnmatchedRequiredFields = <T extends string>(
fields fields
.filter( .filter(
(field) => (field) =>
field.validations?.some((validation) => validation.rule === 'required'), field.fieldValidationDefinitions?.some(
(validation) => validation.rule === 'required',
),
) )
.filter( .filter(
(field) => (field) =>

View File

@ -2,13 +2,17 @@ import {
Columns, Columns,
ColumnType, ColumnType,
} from '@/spreadsheet-import/steps/components/MatchColumnsStep/MatchColumnsStep'; } from '@/spreadsheet-import/steps/components/MatchColumnsStep/MatchColumnsStep';
import { Data, Fields, RawData } from '@/spreadsheet-import/types'; import {
Fields,
ImportedRow,
ImportedStructuredRow,
} from '@/spreadsheet-import/types';
import { normalizeCheckboxValue } from './normalizeCheckboxValue'; import { normalizeCheckboxValue } from './normalizeCheckboxValue';
export const normalizeTableData = <T extends string>( export const normalizeTableData = <T extends string>(
columns: Columns<T>, columns: Columns<T>,
data: RawData[], data: ImportedRow[],
fields: Fields<T>, fields: Fields<T>,
) => ) =>
data.map((row) => data.map((row) =>
@ -63,5 +67,5 @@ export const normalizeTableData = <T extends string>(
default: default:
return acc; return acc;
} }
}, {} as Data<T>), }, {} as ImportedStructuredRow<T>),
); );

View File

@ -1,35 +1,17 @@
import { import {
convertCurrencyMicrosToCurrency, convertCurrencyAmountToCurrencyMicros,
convertCurrencyToCurrencyMicros, convertCurrencyMicrosToCurrencyAmount,
} from '~/utils/convert-currency-amount'; } from '~/utils/convertCurrencyToCurrencyMicros';
describe('convertCurrencyToCurrencyMicros', () => {
it('should return null if currencyAmount is null', () => {
expect(convertCurrencyToCurrencyMicros(null)).toBeNull();
});
it('should throw an error if currencyAmount converted to micros is not a whole number', () => {
expect(() => convertCurrencyToCurrencyMicros(1.023)).toThrow(
'Cannot convert 1.023 to micros',
);
});
describe('convertCurrencyAmountToCurrencyMicros', () => {
it('should convert currencyAmount to micros', () => { it('should convert currencyAmount to micros', () => {
expect(convertCurrencyToCurrencyMicros(1)).toBe(1000000); expect(convertCurrencyAmountToCurrencyMicros(1)).toBe(1000000);
expect(convertCurrencyToCurrencyMicros(1.5)).toBe(1500000); expect(convertCurrencyAmountToCurrencyMicros(1.5)).toBe(1500000);
}); });
}); });
describe('convertCurrencyMicrosToCurrency', () => { describe('convertCurrencyMicrosToCurrencyAmount', () => {
it('should return null if currencyAmountMicros is null', () => {
expect(convertCurrencyMicrosToCurrency(null)).toBeNull();
});
it('should return null if currencyAmountMicros is undefined', () => {
expect(convertCurrencyMicrosToCurrency(undefined)).toBeNull();
});
it('should convert currency micros to currency', () => { it('should convert currency micros to currency', () => {
expect(convertCurrencyMicrosToCurrency(24000000)).toBe(24); expect(convertCurrencyMicrosToCurrencyAmount(24000000)).toBe(24);
}); });
}); });

View File

@ -0,0 +1,3 @@
export const castToString = (value: any) => {
return String(value ?? '');
};

View File

@ -1,29 +0,0 @@
export const convertCurrencyToCurrencyMicros = (
currencyAmount: number | null | undefined,
) => {
if (currencyAmount == null) {
return null;
}
const currencyAmountAsNumber = +currencyAmount;
if (isNaN(currencyAmountAsNumber)) {
throw new Error(`Cannot convert ${currencyAmount} to micros`);
}
const currencyAmountAsMicros = currencyAmountAsNumber * 1000000;
if (currencyAmountAsMicros % 1 !== 0) {
throw new Error(`Cannot convert ${currencyAmount} to micros`);
}
return currencyAmountAsMicros;
};
export const convertCurrencyMicrosToCurrency = (
currencyAmountMicros: number | null | undefined,
) => {
if (currencyAmountMicros == null) {
return null;
}
const currencyAmountMicrosAsNumber = +currencyAmountMicros;
if (isNaN(currencyAmountMicrosAsNumber)) {
throw new Error(`Cannot convert ${currencyAmountMicros} to currency`);
}
return currencyAmountMicrosAsNumber / 1000000;
};

View File

@ -0,0 +1,13 @@
export const convertCurrencyAmountToCurrencyMicros = (
currencyAmount: number,
) => {
const currencyAmountAsMicros = currencyAmount * 1000000;
return currencyAmountAsMicros;
};
export const convertCurrencyMicrosToCurrencyAmount = (
currencyAmountMicros: number,
) => {
return currencyAmountMicros / 1000000;
};