1
1
mirror of https://github.com/leon-ai/leon.git synced 2024-11-27 16:16:48 +03:00

feat: validate JSON Schemas with AJV

This commit is contained in:
Divlo 2022-11-13 13:00:15 +01:00
parent d973ab4318
commit fb75b40fb6
No known key found for this signature in database
GPG Key ID: 8F9478F220CE65E9
4 changed files with 225 additions and 189 deletions

264
package-lock.json generated
View File

@ -20,7 +20,10 @@
"@nlpjs/core-loader": "^4.22.7",
"@nlpjs/lang-all": "^4.22.12",
"@nlpjs/nlp": "^4.22.17",
"@sinclair/typebox": "0.24.49",
"@segment/ajv-human-errors": "2.1.2",
"@sinclair/typebox": "0.25.8",
"ajv": "8.11.0",
"ajv-formats": "2.1.1",
"archiver": "^5.3.1",
"async": "^3.2.0",
"axios": "1.1.2",
@ -1644,6 +1647,28 @@
"url": "https://opencollective.com/eslint"
}
},
"node_modules/@eslint/eslintrc/node_modules/ajv": {
"version": "6.12.6",
"resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz",
"integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==",
"dev": true,
"dependencies": {
"fast-deep-equal": "^3.1.1",
"fast-json-stable-stringify": "^2.0.0",
"json-schema-traverse": "^0.4.1",
"uri-js": "^4.2.2"
},
"funding": {
"type": "github",
"url": "https://github.com/sponsors/epoberezkin"
}
},
"node_modules/@eslint/eslintrc/node_modules/json-schema-traverse": {
"version": "0.4.1",
"resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz",
"integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==",
"dev": true
},
"node_modules/@fast-csv/format": {
"version": "4.3.5",
"dev": true,
@ -1700,24 +1725,6 @@
"fast-uri": "^2.0.0"
}
},
"node_modules/@fastify/ajv-compiler/node_modules/ajv": {
"version": "8.11.0",
"license": "MIT",
"dependencies": {
"fast-deep-equal": "^3.1.1",
"json-schema-traverse": "^1.0.0",
"require-from-string": "^2.0.2",
"uri-js": "^4.2.2"
},
"funding": {
"type": "github",
"url": "https://github.com/sponsors/epoberezkin"
}
},
"node_modules/@fastify/ajv-compiler/node_modules/json-schema-traverse": {
"version": "1.0.0",
"license": "MIT"
},
"node_modules/@fastify/deepmerge": {
"version": "1.1.0",
"license": "MIT"
@ -3311,10 +3318,18 @@
"version": "1.1.0",
"license": "BSD-3-Clause"
},
"node_modules/@segment/ajv-human-errors": {
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/@segment/ajv-human-errors/-/ajv-human-errors-2.1.2.tgz",
"integrity": "sha512-d1uQndRFLRO01+xW1y5m+joxDgHf5SLJ70YCY2ArLoo2FJ50o6AoX2mEbuGvnKz/IdwnvDugm9Ti3mZQkW1OoA==",
"peerDependencies": {
"ajv": "^8.0.0"
}
},
"node_modules/@sinclair/typebox": {
"version": "0.24.49",
"resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.24.49.tgz",
"integrity": "sha512-qWgJVeCThMWTJSAZHyHDlHBkJY0LARFoq/96RH4oIFAwJptMwB3Isq62c4zRVRIAF2r4RMOc2WOhtOLj5C4InA=="
"version": "0.25.8",
"resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.25.8.tgz",
"integrity": "sha512-/7GXgMOCfnxpIov52cPetqQ7bxRBaTBJAgp04Se2UQB1J0vUfEOIMpn63cLc3S5JXDUflCWxELKDV8eiPpmUTQ=="
},
"node_modules/@sinonjs/commons": {
"version": "1.8.5",
@ -4107,38 +4122,9 @@
}
},
"node_modules/ajv": {
"version": "6.12.6",
"dev": true,
"license": "MIT",
"dependencies": {
"fast-deep-equal": "^3.1.1",
"fast-json-stable-stringify": "^2.0.0",
"json-schema-traverse": "^0.4.1",
"uri-js": "^4.2.2"
},
"funding": {
"type": "github",
"url": "https://github.com/sponsors/epoberezkin"
}
},
"node_modules/ajv-formats": {
"version": "2.1.1",
"license": "MIT",
"dependencies": {
"ajv": "^8.0.0"
},
"peerDependencies": {
"ajv": "^8.0.0"
},
"peerDependenciesMeta": {
"ajv": {
"optional": true
}
}
},
"node_modules/ajv-formats/node_modules/ajv": {
"version": "8.11.0",
"license": "MIT",
"resolved": "https://registry.npmjs.org/ajv/-/ajv-8.11.0.tgz",
"integrity": "sha512-wGgprdCvMalC0BztXvitD2hC04YffAvtsUn93JbGXYLAtCUO4xd17mCCZQxUOItiBwZvJScWo8NIvQMQ71rdpg==",
"dependencies": {
"fast-deep-equal": "^3.1.1",
"json-schema-traverse": "^1.0.0",
@ -4150,9 +4136,21 @@
"url": "https://github.com/sponsors/epoberezkin"
}
},
"node_modules/ajv-formats/node_modules/json-schema-traverse": {
"version": "1.0.0",
"license": "MIT"
"node_modules/ajv-formats": {
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-2.1.1.tgz",
"integrity": "sha512-Wx0Kx52hxE7C18hkMEggYlEifqWZtYaRgouJor+WMdPnQyEK13vgEWyVNup7SoeeoLMsr4kf5h6dOW11I15MUA==",
"dependencies": {
"ajv": "^8.0.0"
},
"peerDependencies": {
"ajv": "^8.0.0"
},
"peerDependenciesMeta": {
"ajv": {
"optional": true
}
}
},
"node_modules/ansi-colors": {
"version": "4.1.3",
@ -6607,6 +6605,22 @@
"node": "^12.22.0 || ^14.17.0 || >=16.0.0"
}
},
"node_modules/eslint/node_modules/ajv": {
"version": "6.12.6",
"resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz",
"integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==",
"dev": true,
"dependencies": {
"fast-deep-equal": "^3.1.1",
"fast-json-stable-stringify": "^2.0.0",
"json-schema-traverse": "^0.4.1",
"uri-js": "^4.2.2"
},
"funding": {
"type": "github",
"url": "https://github.com/sponsors/epoberezkin"
}
},
"node_modules/eslint/node_modules/eslint-utils": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/eslint-utils/-/eslint-utils-3.0.0.tgz",
@ -6634,6 +6648,12 @@
"node": ">=10"
}
},
"node_modules/eslint/node_modules/json-schema-traverse": {
"version": "0.4.1",
"resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz",
"integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==",
"dev": true
},
"node_modules/espree": {
"version": "9.4.1",
"resolved": "https://registry.npmjs.org/espree/-/espree-9.4.1.tgz",
@ -6952,24 +6972,6 @@
"rfdc": "^1.2.0"
}
},
"node_modules/fast-json-stringify/node_modules/ajv": {
"version": "8.11.0",
"license": "MIT",
"dependencies": {
"fast-deep-equal": "^3.1.1",
"json-schema-traverse": "^1.0.0",
"require-from-string": "^2.0.2",
"uri-js": "^4.2.2"
},
"funding": {
"type": "github",
"url": "https://github.com/sponsors/epoberezkin"
}
},
"node_modules/fast-json-stringify/node_modules/json-schema-traverse": {
"version": "1.0.0",
"license": "MIT"
},
"node_modules/fast-levenshtein": {
"version": "2.0.6",
"dev": true,
@ -10080,9 +10082,9 @@
"license": "MIT"
},
"node_modules/json-schema-traverse": {
"version": "0.4.1",
"dev": true,
"license": "MIT"
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz",
"integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug=="
},
"node_modules/json-stable-stringify-without-jsonify": {
"version": "1.0.1",
@ -15544,6 +15546,26 @@
"js-yaml": "^4.1.0",
"minimatch": "^3.1.2",
"strip-json-comments": "^3.1.1"
},
"dependencies": {
"ajv": {
"version": "6.12.6",
"resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz",
"integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==",
"dev": true,
"requires": {
"fast-deep-equal": "^3.1.1",
"fast-json-stable-stringify": "^2.0.0",
"json-schema-traverse": "^0.4.1",
"uri-js": "^4.2.2"
}
},
"json-schema-traverse": {
"version": "0.4.1",
"resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz",
"integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==",
"dev": true
}
}
},
"@fast-csv/format": {
@ -15598,20 +15620,6 @@
"ajv": "^8.11.0",
"ajv-formats": "^2.1.1",
"fast-uri": "^2.0.0"
},
"dependencies": {
"ajv": {
"version": "8.11.0",
"requires": {
"fast-deep-equal": "^3.1.1",
"json-schema-traverse": "^1.0.0",
"require-from-string": "^2.0.2",
"uri-js": "^4.2.2"
}
},
"json-schema-traverse": {
"version": "1.0.0"
}
}
},
"@fastify/deepmerge": {
@ -16839,10 +16847,16 @@
"@protobufjs/utf8": {
"version": "1.1.0"
},
"@segment/ajv-human-errors": {
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/@segment/ajv-human-errors/-/ajv-human-errors-2.1.2.tgz",
"integrity": "sha512-d1uQndRFLRO01+xW1y5m+joxDgHf5SLJ70YCY2ArLoo2FJ50o6AoX2mEbuGvnKz/IdwnvDugm9Ti3mZQkW1OoA==",
"requires": {}
},
"@sinclair/typebox": {
"version": "0.24.49",
"resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.24.49.tgz",
"integrity": "sha512-qWgJVeCThMWTJSAZHyHDlHBkJY0LARFoq/96RH4oIFAwJptMwB3Isq62c4zRVRIAF2r4RMOc2WOhtOLj5C4InA=="
"version": "0.25.8",
"resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.25.8.tgz",
"integrity": "sha512-/7GXgMOCfnxpIov52cPetqQ7bxRBaTBJAgp04Se2UQB1J0vUfEOIMpn63cLc3S5JXDUflCWxELKDV8eiPpmUTQ=="
},
"@sinonjs/commons": {
"version": "1.8.5",
@ -17338,33 +17352,22 @@
}
},
"ajv": {
"version": "6.12.6",
"dev": true,
"version": "8.11.0",
"resolved": "https://registry.npmjs.org/ajv/-/ajv-8.11.0.tgz",
"integrity": "sha512-wGgprdCvMalC0BztXvitD2hC04YffAvtsUn93JbGXYLAtCUO4xd17mCCZQxUOItiBwZvJScWo8NIvQMQ71rdpg==",
"requires": {
"fast-deep-equal": "^3.1.1",
"fast-json-stable-stringify": "^2.0.0",
"json-schema-traverse": "^0.4.1",
"json-schema-traverse": "^1.0.0",
"require-from-string": "^2.0.2",
"uri-js": "^4.2.2"
}
},
"ajv-formats": {
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-2.1.1.tgz",
"integrity": "sha512-Wx0Kx52hxE7C18hkMEggYlEifqWZtYaRgouJor+WMdPnQyEK13vgEWyVNup7SoeeoLMsr4kf5h6dOW11I15MUA==",
"requires": {
"ajv": "^8.0.0"
},
"dependencies": {
"ajv": {
"version": "8.11.0",
"requires": {
"fast-deep-equal": "^3.1.1",
"json-schema-traverse": "^1.0.0",
"require-from-string": "^2.0.2",
"uri-js": "^4.2.2"
}
},
"json-schema-traverse": {
"version": "1.0.0"
}
}
},
"ansi-colors": {
@ -18715,6 +18718,18 @@
"v8-compile-cache": "^2.0.3"
},
"dependencies": {
"ajv": {
"version": "6.12.6",
"resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz",
"integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==",
"dev": true,
"requires": {
"fast-deep-equal": "^3.1.1",
"fast-json-stable-stringify": "^2.0.0",
"json-schema-traverse": "^0.4.1",
"uri-js": "^4.2.2"
}
},
"eslint-utils": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/eslint-utils/-/eslint-utils-3.0.0.tgz",
@ -18731,6 +18746,12 @@
"dev": true
}
}
},
"json-schema-traverse": {
"version": "0.4.1",
"resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz",
"integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==",
"dev": true
}
}
},
@ -19139,20 +19160,6 @@
"fast-deep-equal": "^3.1.3",
"fast-uri": "^2.1.0",
"rfdc": "^1.2.0"
},
"dependencies": {
"ajv": {
"version": "8.11.0",
"requires": {
"fast-deep-equal": "^3.1.1",
"json-schema-traverse": "^1.0.0",
"require-from-string": "^2.0.2",
"uri-js": "^4.2.2"
}
},
"json-schema-traverse": {
"version": "1.0.0"
}
}
},
"fast-levenshtein": {
@ -21282,8 +21289,9 @@
"dev": true
},
"json-schema-traverse": {
"version": "0.4.1",
"dev": true
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz",
"integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug=="
},
"json-stable-stringify-without-jsonify": {
"version": "1.0.1",

View File

@ -75,7 +75,10 @@
"@nlpjs/core-loader": "^4.22.7",
"@nlpjs/lang-all": "^4.22.12",
"@nlpjs/nlp": "^4.22.17",
"@sinclair/typebox": "0.24.49",
"@segment/ajv-human-errors": "2.1.2",
"@sinclair/typebox": "0.25.8",
"ajv": "8.11.0",
"ajv-formats": "2.1.1",
"archiver": "^5.3.1",
"async": "^3.2.0",
"axios": "1.1.2",

25
server/src/ajv.ts Normal file
View File

@ -0,0 +1,25 @@
import addFormats from 'ajv-formats'
import Ajv from 'ajv'
export const ajv = addFormats(
new Ajv({
allErrors: true,
verbose: true
}),
[
'date-time',
'time',
'date',
'email',
'hostname',
'ipv4',
'ipv6',
'uri',
'uri-reference',
'uuid',
'uri-template',
'json-pointer',
'relative-json-pointer',
'regex'
]
)

View File

@ -1,8 +1,9 @@
import fs from 'node:fs'
import path from 'node:path'
import { TypeCompiler } from '@sinclair/typebox/compiler'
import { AggregateAjvError } from '@segment/ajv-human-errors'
import { ajv } from '@/ajv'
import {
amazonVoiceConfiguration,
googleCloudVoiceConfiguration,
@ -31,6 +32,29 @@ import { SkillDomainHelper } from '@/helpers/skill-domain-helper'
import { VOICE_CONFIG_PATH, GLOBAL_DATA_PATH } from '@/constants'
import { getGlobalEntitiesPath, getGlobalResolversPath } from '@/utilities'
interface ObjectUnknown {
[key: string]: unknown
}
const validateSchema = (
schema: ObjectUnknown,
contentToValidate: ObjectUnknown,
customErrorMesage: string
): void => {
const validate = ajv.compile(schema)
const isValid = validate(contentToValidate)
if (!isValid) {
const errors = new AggregateAjvError(validate.errors ?? [])
const messages: string[] = []
for (const error of errors) {
messages.push(error.message)
}
LogHelper.error(customErrorMesage)
LogHelper.error(messages.join('\n'))
process.exit(1)
}
}
/**
* Pre-checking
* Ensure JSON files are correctly formatted
@ -64,14 +88,11 @@ const GLOBAL_DATA_SCHEMAS = {
await fs.promises.readFile(path.join(VOICE_CONFIG_PATH, file), 'utf8')
)
const [configName] = file.split('.') as [keyof typeof VOICE_CONFIG_SCHEMAS]
const result = TypeCompiler.Compile(VOICE_CONFIG_SCHEMAS[configName])
const errors = [...result.Errors(config)]
if (errors.length > 0) {
LogHelper.error(`The voice configuration schema "${file}" is not valid:`)
LogHelper.error(JSON.stringify(errors, null, 2))
process.exit(1)
}
validateSchema(
VOICE_CONFIG_SCHEMAS[configName],
config,
`The voice configuration schema "${file}" is not valid:`
)
}
LogHelper.success('Voice configuration schemas checked')
@ -94,14 +115,11 @@ const GLOBAL_DATA_SCHEMAS = {
const globalEntity: GlobalEntity = JSON.parse(
await fs.promises.readFile(path.join(globalEntitiesPath, file), 'utf8')
)
const result = TypeCompiler.Compile(globalEntitySchemaObject)
const errors = [...result.Errors(globalEntity)]
if (errors.length > 0) {
LogHelper.error(`The global entity schema "${file}" is not valid:`)
LogHelper.error(JSON.stringify(errors, null, 2))
process.exit(1)
}
validateSchema(
globalEntitySchemaObject,
globalEntity,
`The global entity schema "${file}" is not valid:`
)
}
/**
@ -116,14 +134,11 @@ const GLOBAL_DATA_SCHEMAS = {
const globalResolver: GlobalResolver = JSON.parse(
await fs.promises.readFile(path.join(globalResolversPath, file), 'utf8')
)
const result = TypeCompiler.Compile(globalResolverSchemaObject)
const errors = [...result.Errors(globalResolver)]
if (errors.length > 0) {
LogHelper.error(`The global resolver schema "${file}" is not valid:`)
LogHelper.error(JSON.stringify(errors, null, 2))
process.exit(1)
}
validateSchema(
globalResolverSchemaObject,
globalResolver,
`The global resolver schema "${file}" is not valid:`
)
}
/**
@ -135,15 +150,11 @@ const GLOBAL_DATA_SCHEMAS = {
'utf8'
)
)
const result = TypeCompiler.Compile(GLOBAL_DATA_SCHEMAS.answers)
const errors = [...result.Errors(answers)]
if (errors.length > 0) {
LogHelper.error(`The global answers schema "answers.json" is not valid:`)
LogHelper.error(JSON.stringify(errors, null, 2))
process.exit(1)
}
validateSchema(
GLOBAL_DATA_SCHEMAS.answers,
answers,
`The global answers schema "answers.json" is not valid:`
)
}
LogHelper.success('Global data schemas checked')
@ -162,14 +173,11 @@ const GLOBAL_DATA_SCHEMAS = {
const domainObject: Domain = JSON.parse(
await fs.promises.readFile(pathToDomain, 'utf8')
)
const domainResult = TypeCompiler.Compile(domainSchemaObject)
const domainErrors = [...domainResult.Errors(domainObject)]
if (domainErrors.length > 0) {
LogHelper.error(`The domain schema "${pathToDomain}" is not valid:`)
LogHelper.error(JSON.stringify(domainErrors, null, 2))
process.exit(1)
}
validateSchema(
domainSchemaObject,
domainObject,
`The domain schema "${pathToDomain}" is not valid:`
)
const skillKeys = Object.keys(currentDomain.skills)
@ -184,14 +192,11 @@ const GLOBAL_DATA_SCHEMAS = {
const skillObject: Skill = JSON.parse(
await fs.promises.readFile(pathToSkill, 'utf8')
)
const skillResult = TypeCompiler.Compile(skillSchemaObject)
const skillErrors = [...skillResult.Errors(skillObject)]
if (skillErrors.length > 0) {
LogHelper.error(`The skill schema "${pathToSkill}" is not valid:`)
LogHelper.error(JSON.stringify(skillErrors, null, 2))
process.exit(1)
}
validateSchema(
skillSchemaObject,
skillObject,
`The skill schema "${pathToSkill}" is not valid:`
)
/**
* Skills config checking
@ -206,16 +211,11 @@ const GLOBAL_DATA_SCHEMAS = {
const skillConfig: SkillConfig = JSON.parse(
await fs.promises.readFile(skillConfigPath, 'utf8')
)
const result = TypeCompiler.Compile(skillConfigSchemaObject)
const errors = [...result.Errors(skillConfig)]
if (errors.length > 0) {
LogHelper.error(
`The skill config schema "${skillConfigPath}" is not valid:`
)
LogHelper.error(JSON.stringify(errors, null, 2))
process.exit(1)
}
validateSchema(
skillConfigSchemaObject,
skillConfig,
`The skill config schema "${skillConfigPath}" is not valid:`
)
}
}
}