diff --git a/frontend/libs/console/legacy-ce/src/lib/features/Data/MongoTrackCollection/utils/inferLogicalModel.test.ts b/frontend/libs/console/legacy-ce/src/lib/features/Data/MongoTrackCollection/utils/inferLogicalModel.test.ts index dabd2900ac5..cda85e9b969 100644 --- a/frontend/libs/console/legacy-ce/src/lib/features/Data/MongoTrackCollection/utils/inferLogicalModel.test.ts +++ b/frontend/libs/console/legacy-ce/src/lib/features/Data/MongoTrackCollection/utils/inferLogicalModel.test.ts @@ -3,7 +3,9 @@ import { inferLogicalModels } from './inferLogicalModel'; describe('inferLogicalModels', () => { it('returns a logical model', () => { const document = { - _id: '11123-123-123', + _id: { + $oid: '11123-123-123', + }, name: 'John', age: 30, isActive: true, @@ -28,7 +30,9 @@ describe('inferLogicalModels', () => { it('returns multiple logical models with array', () => { const document = { - _id: '11123-123-123', + _id: { + $oid: '11123-123-123', + }, name: 'John', age: 30, isActive: true, @@ -108,7 +112,9 @@ describe('inferLogicalModels', () => { it('returns multiple logical models with object', () => { const document = { - _id: 'asd', + _id: { + $oid: 'asd', + }, name: 'Stu', year: 2018, gpa: 3.5, @@ -264,4 +270,43 @@ describe('inferLogicalModels', () => { }, ]); }); + + it('handles documents with object ids with names other than _id', () => { + const document = { + _id: { + $oid: '5a9427648b0beebeb69579cc', + }, + name: 'Andrea Le', + email: 'andrea_le@fakegmail.com', + movie_id: { + $oid: '573a1390f29313caabcd418c', + }, + text: 'Rem officiis eaque repellendus amet eos doloribus. Porro dolor voluptatum voluptates neque culpa molestias. Voluptate unde nulla temporibus ullam.', + date: { + $date: '2012-03-26T23:20:16.000Z', + }, + }; + + const logicalModels = inferLogicalModels( + 'new-documents', + JSON.stringify(document) + ); + + expect(logicalModels).toEqual([ + { + fields: [ + { name: '_id', type: { nullable: false, scalar: 'objectId' } }, + { name: 'name', type: { nullable: false, scalar: 'string' } }, + { name: 'email', type: { nullable: false, scalar: 'string' } }, + { + name: 'movie_id', + type: { nullable: false, scalar: 'objectId' }, + }, + { name: 'text', type: { nullable: false, scalar: 'string' } }, + { name: 'date', type: { nullable: false, scalar: 'date' } }, + ], + name: 'newdocuments', + }, + ]); + }); }); diff --git a/frontend/libs/console/legacy-ce/src/lib/features/Data/MongoTrackCollection/utils/inferLogicalModel.ts b/frontend/libs/console/legacy-ce/src/lib/features/Data/MongoTrackCollection/utils/inferLogicalModel.ts index 46030a2fdc4..98a766c6c44 100644 --- a/frontend/libs/console/legacy-ce/src/lib/features/Data/MongoTrackCollection/utils/inferLogicalModel.ts +++ b/frontend/libs/console/legacy-ce/src/lib/features/Data/MongoTrackCollection/utils/inferLogicalModel.ts @@ -27,79 +27,173 @@ const getLogicalModelsFromProperties = ( collectionName: string, name: string, properties: ObjectSchema['properties'], - requiredProperties: string[] + requiredProperties: string[] = [], + parentName = '' ): LogicalModel[] => { const logicalModels: LogicalModel[] = []; const fields: LogicalModelField[] = []; - for (const [rawFieldName, fieldSchema] of Object.entries(properties)) { const fieldName = sanitizeGraphQLFieldNames(rawFieldName); - if (fieldName === '_id') { - fields.push({ - name: fieldName, - type: { - scalar: 'objectId', - nullable: false, - }, - }); - continue; - } - const nullable = !requiredProperties.includes(fieldName); + const logicalModelPath = parentName + ? `${parentName}_${fieldName}` + : fieldName; + + // Get scalars from MongoDB objectid and date objects + const handleMongoDBFieldTypes = ( + properties: ObjectSchema['properties'] + ): { type: 'objectId' | 'date' | 'string' | 'none'; name?: string } => { + if (!properties) { + return { type: 'string' }; + } + if (Object.prototype.hasOwnProperty.call(properties, '$oid')) { + return { type: 'objectId' }; + } + if (Object.prototype.hasOwnProperty.call(properties, '$date')) { + return { type: 'date' }; + } + return { type: 'none' }; + }; + if (fieldSchema.type === 'object') { + // Seperate MongoDB objectid and date scalars from logical model objects + const mongoDBFieldType = handleMongoDBFieldTypes(fieldSchema.properties); + if (mongoDBFieldType.type !== 'none') { + fields.push({ + name: fieldName, + type: { + scalar: mongoDBFieldType.type, + nullable: false, + }, + }); + continue; + } + // Make new logical model const newLogicalModels = getLogicalModelsFromProperties( collectionName, - `${collectionName}_${fieldName}`, + `${collectionName}_${logicalModelPath}`, fieldSchema.properties, - fieldSchema.required + fieldSchema.required, + logicalModelPath ); - logicalModels.push(...newLogicalModels); - fields.push({ name: fieldName, type: { - logical_model: `${collectionName}_${fieldName}`, + logical_model: `${collectionName}_${logicalModelPath}`, nullable, }, }); } if (fieldSchema.type === 'array') { - if (fieldSchema.items.type === 'object') { - // new logical model needed - fields.push({ - name: fieldName, - type: { - array: { - logical_model: `${collectionName}_${fieldName}`, - nullable, - }, - }, - }); - - const newLogicalModels = getLogicalModelsFromProperties( - collectionName, - `${collectionName}_${fieldName}`, - fieldSchema.items.properties, - fieldSchema.items?.required || [] + // Throw error for mixed object / scalar array + // Schema inferer returns anyOf if there are any type conflicts + const hasNestedAnyOf = (function checkNestedAnyOf(obj: any): boolean { + if (typeof obj !== 'object' || obj === null) return false; + if ('anyOf' in obj) return true; + return Object.values(obj).some( + val => typeof val === 'object' && checkNestedAnyOf(val) ); + })(fieldSchema.items); + if (hasNestedAnyOf) { + throw new Error( + `The array for field "${fieldName}" contains both multiple types (objects, string, int, etc.). Please check and ensure it only contains one for inference. \n Exact key with issue: "${logicalModelPath}"` + ); + } - logicalModels.push(...newLogicalModels); - } else { - // scalar array - fields.push({ - name: fieldName, - type: { - array: { - scalar: fieldSchema.items.type, - nullable, + // Array of objects + if (fieldSchema.items.type === 'object') { + // Check for special mongo scalars + const mongoDBFieldType = handleMongoDBFieldTypes( + fieldSchema.items.properties + ); + if (mongoDBFieldType.type !== 'none') { + fields.push({ + name: fieldName, + type: { + array: { + scalar: mongoDBFieldType.type, + nullable, + }, }, - }, - }); + }); + } else { + // Make new logical model for array + const newLogicalModels = getLogicalModelsFromProperties( + collectionName, + `${collectionName}_${logicalModelPath}`, + fieldSchema.items.properties, + fieldSchema.items?.required || [], + logicalModelPath + ); + + logicalModels.push(...newLogicalModels); + + fields.push({ + name: fieldName, + type: { + array: { + logical_model: `${collectionName}_${logicalModelPath}`, + nullable, + }, + }, + }); + } + continue; + } + // Array of scalars + if (fieldSchema.items.type !== 'object') { + // Process scalar types in array + // TODO: DRY this out with scalar processing below + if (fieldSchema.items.type === 'string') { + fields.push({ + name: fieldName, + type: { + array: { + scalar: 'string', + nullable, + }, + }, + }); + } + if (fieldSchema.items.type === 'integer') { + fields.push({ + name: fieldName, + type: { + array: { + scalar: 'int', + nullable, + }, + }, + }); + } + if (fieldSchema.items.type === 'number') { + fields.push({ + name: fieldName, + type: { + array: { + scalar: 'double', + nullable, + }, + }, + }); + } + if (fieldSchema.items.type === 'boolean') { + fields.push({ + name: fieldName, + type: { + array: { + scalar: 'bool', + nullable, + }, + }, + }); + } } } + // Process scalars if (fieldSchema.type === 'string') { fields.push({ name: fieldName, @@ -140,7 +234,6 @@ const getLogicalModelsFromProperties = ( }); } } - return [ { name,