feat: add pre-checking for all JSON configs

package.json

@@ -52,7 +52,7 @@
     "build:python-bridge": "ts-node scripts/build-binaries.js python-bridge",
     "build:tcp-server": "ts-node scripts/build-binaries.js tcp-server",
     "start:tcp-server": "cross-env PIPENV_PIPFILE=tcp_server/src/Pipfile pipenv run python tcp_server/src/main.py",
-    "start": "cross-env LEON_NODE_ENV=production node ./server/dist/index.js",
+    "start": "cross-env LEON_NODE_ENV=production node server/dist/pre-check.js && node server/dist/index.js",
     "python-bridge": "cross-env PIPENV_PIPFILE=bridges/python/src/Pipfile pipenv run python bridges/python/src/main.py server/src/intent-object.sample.json",
     "train": "ts-node scripts/train/run-train.js",
     "prepare-release": "ts-node scripts/release/prepare-release.js",
@@ -75,8 +75,8 @@
     "@nlpjs/core-loader": "^4.22.7",
     "@nlpjs/lang-all": "^4.22.12",
     "@nlpjs/nlp": "^4.22.17",
-    "archiver": "^5.3.1",
     "@sinclair/typebox": "0.24.49",
+    "archiver": "^5.3.1",
     "async": "^3.2.0",
     "axios": "1.1.2",
     "cross-env": "^7.0.3",
@@ -100,7 +100,7 @@
   },
   "devDependencies": {
     "@nlpjs/utils": "^4.24.1",
-    "@swc/core": "^1.2.245",
+    "@swc/core": "^1.3.14",
     "@tsconfig/node16-strictest": "^1.0.3",
     "@types/cli-spinner": "0.2.1",
     "@types/node": "^18.7.13",

server/src/constants.ts

@@ -95,3 +95,9 @@ export const HAS_LOGGER = process.env['LEON_LOGGER'] === 'true'
 
 export const TCP_SERVER_HOST = process.env['LEON_PY_TCP_SERVER_HOST']
 export const TCP_SERVER_PORT = process.env['LEON_PY_TCP_SERVER_PORT']
+
+/**
+ * Paths
+ */
+export const GLOBAL_DATA_PATH = path.join('core', 'data')
+export const VOICE_CONFIG_PATH = path.join('core', 'config', 'voice')

server/src/helpers/skill-domain-helper.ts

@@ -2,8 +2,13 @@ import fs from 'node:fs'
 import path from 'node:path'
 
 import type { ShortLanguageCode } from '@/helpers/lang-helper'
-import type { GlobalEntity } from '@@/core/data/schemas'
-import type { Domain, Skill, SkillConfig, SkillBridge } from '@@/skills/schemas'
+import type { GlobalEntity } from '@/schemas/global-data-schemas'
+import type {
+  Domain,
+  Skill,
+  SkillConfig,
+  SkillBridge
+} from '@/schemas/skill-schemas'
 
 interface SkillDomain {
   name: string
@@ -132,8 +137,8 @@ export class SkillDomainHelper {
           const entityRawData = fs.readFileSync(entityFilePath, {
             encoding: 'utf8'
           })
-          const entityData = JSON.parse(entityRawData) as GlobalEntity
-          result.entities[entity] = entityData
+
+          result.entities[entity] = JSON.parse(entityRawData) as GlobalEntity
         }
       })
 

server/src/index.ts

@@ -13,6 +13,7 @@ import server from '@/core/http-server/server'
 ;(async (): Promise<void> => {
   process.title = 'leon'
 
+  // Start the TCP server
   global.tcpServerProcess = spawn(
     `${TCP_SERVER_BIN_PATH} ${LangHelper.getShortCode(LEON_LANG)}`,
     {
@@ -21,7 +22,9 @@ import server from '@/core/http-server/server'
     }
   )
 
+  // Start the TCP client
   global.tcpClient = new TcpClient(TCP_SERVER_HOST, TCP_SERVER_PORT)
 
+  // Start the core server
   await server.init()
 })()

server/src/pre-check.ts

@@ -0,0 +1,219 @@
+import fs from 'node:fs'
+import path from 'node:path'
+
+import { TypeCompiler } from '@sinclair/typebox/compiler'
+
+import {
+  amazonVoiceConfiguration,
+  googleCloudVoiceConfiguration,
+  watsonVoiceConfiguration
+} from '@/schemas/voice-config-schemas'
+import {
+  answersSchemaObject,
+  globalEntitySchemaObject,
+  globalResolverSchemaObject
+} from '@/schemas/global-data-schemas'
+import {
+  domainSchemaObject,
+  skillSchemaObject,
+  skillConfigSchemaObject
+} from '@/schemas/skill-schemas'
+import { LogHelper } from '@/helpers/log-helper'
+import { LangHelper } from '@/helpers/lang-helper'
+import { SkillDomainHelper } from '@/helpers/skill-domain-helper'
+import { VOICE_CONFIG_PATH, GLOBAL_DATA_PATH } from '@/constants'
+import { getGlobalEntitiesPath, getGlobalResolversPath } from '@/utilities'
+
+/**
+ * Pre-checking
+ * Ensure JSON files are correctly formatted
+ */
+const VOICE_CONFIG_SCHEMAS = {
+  amazon: amazonVoiceConfiguration,
+  'google-cloud': googleCloudVoiceConfiguration,
+  'watson-stt': watsonVoiceConfiguration,
+  'watson-tts': watsonVoiceConfiguration
+}
+const GLOBAL_DATA_SCHEMAS = {
+  answers: answersSchemaObject,
+  globalEntities: globalEntitySchemaObject,
+  globalResolvers: globalResolverSchemaObject
+}
+
+;(async (): Promise<void> => {
+  LogHelper.title('Pre-checking')
+
+  /**
+   * Voice configuration checking
+   */
+  LogHelper.info('Checking voice configuration schemas...')
+
+  const voiceConfigFiles = (
+    await fs.promises.readdir(VOICE_CONFIG_PATH)
+  ).filter((file) => file.endsWith('.json') && !file.includes('.sample.'))
+
+  for (const file of voiceConfigFiles) {
+    const config = JSON.parse(
+      await fs.promises.readFile(path.join(VOICE_CONFIG_PATH, file), 'utf8')
+    )
+    const [configName] = file.split('.') as [keyof typeof VOICE_CONFIG_SCHEMAS]
+    const result = TypeCompiler.Compile(VOICE_CONFIG_SCHEMAS[configName])
+    const errors = [...result.Errors(config)]
+
+    if (errors.length > 0) {
+      LogHelper.error(`The voice configuration schema "${file}" is not valid:`)
+      LogHelper.error(JSON.stringify(errors, null, 2))
+      process.exit(1)
+    }
+  }
+  LogHelper.success('Voice configuration schemas checked')
+
+  /**
+   * Global data checking
+   */
+  LogHelper.info('Checking global data schemas...')
+
+  const supportedLangs = LangHelper.getShortCodes()
+  for (const lang of supportedLangs) {
+    /**
+     * Global entities checking
+     */
+    const globalEntitiesPath = getGlobalEntitiesPath(lang)
+    const globalEntityFiles = (
+      await fs.promises.readdir(globalEntitiesPath)
+    ).filter((file) => file.endsWith('.json'))
+
+    for (const file of globalEntityFiles) {
+      const globalEntity = JSON.parse(
+        await fs.promises.readFile(path.join(globalEntitiesPath, file), 'utf8')
+      )
+      const result = TypeCompiler.Compile(globalEntitySchemaObject)
+      const errors = [...result.Errors(globalEntity)]
+
+      if (errors.length > 0) {
+        LogHelper.error(`The global entity schema "${file}" is not valid:`)
+        LogHelper.error(JSON.stringify(errors, null, 2))
+        process.exit(1)
+      }
+    }
+
+    /**
+     * Global resolvers checking
+     */
+    const globalResolversPath = getGlobalResolversPath(lang)
+    const globalResolverFiles = (
+      await fs.promises.readdir(globalResolversPath)
+    ).filter((file) => file.endsWith('.json'))
+
+    for (const file of globalResolverFiles) {
+      const globalResolver = JSON.parse(
+        await fs.promises.readFile(path.join(globalResolversPath, file), 'utf8')
+      )
+      const result = TypeCompiler.Compile(globalResolverSchemaObject)
+      const errors = [...result.Errors(globalResolver)]
+
+      if (errors.length > 0) {
+        LogHelper.error(`The global resolver schema "${file}" is not valid:`)
+        LogHelper.error(JSON.stringify(errors, null, 2))
+        process.exit(1)
+      }
+    }
+
+    /**
+     * Global answers checking
+     */
+    const answers = JSON.parse(
+      await fs.promises.readFile(
+        path.join(GLOBAL_DATA_PATH, lang, 'answers.json'),
+        'utf8'
+      )
+    )
+    const result = TypeCompiler.Compile(GLOBAL_DATA_SCHEMAS.answers)
+
+    const errors = [...result.Errors(answers)]
+
+    if (errors.length > 0) {
+      LogHelper.error(`The global answers schema "answers.json" is not valid:`)
+      LogHelper.error(JSON.stringify(errors, null, 2))
+      process.exit(1)
+    }
+  }
+  LogHelper.success('Global data schemas checked')
+
+  /**
+   * Skills data checking
+   */
+  LogHelper.info('Checking skills data schemas...')
+
+  const skillDomains = await SkillDomainHelper.getSkillDomains()
+
+  for (const [, currentDomain] of skillDomains) {
+    /**
+     * Domain checking
+     */
+    const pathToDomain = path.join(currentDomain.path, 'domain.json')
+    const domainObject = JSON.parse(
+      await fs.promises.readFile(pathToDomain, 'utf8')
+    )
+    const domainResult = TypeCompiler.Compile(domainSchemaObject)
+    const domainErrors = [...domainResult.Errors(domainObject)]
+
+    if (domainErrors.length > 0) {
+      LogHelper.error(`The domain schema "${pathToDomain}" is not valid:`)
+      LogHelper.error(JSON.stringify(domainErrors, null, 2))
+      process.exit(1)
+    }
+
+    const skillKeys = Object.keys(currentDomain.skills)
+
+    for (const skillKey of skillKeys) {
+      const currentSkill = currentDomain.skills[skillKey]
+
+      /**
+       * Skills checking
+       */
+      if (currentSkill) {
+        const pathToSkill = path.join(currentSkill.path, 'skill.json')
+        const skillObject = JSON.parse(
+          await fs.promises.readFile(pathToSkill, 'utf8')
+        )
+        const skillResult = TypeCompiler.Compile(skillSchemaObject)
+        const skillErrors = [...skillResult.Errors(skillObject)]
+
+        if (skillErrors.length > 0) {
+          LogHelper.error(`The skill schema "${pathToSkill}" is not valid:`)
+          LogHelper.error(JSON.stringify(skillErrors, null, 2))
+          process.exit(1)
+        }
+
+        /**
+         * Skills config checking
+         */
+        const pathToSkillConfig = path.join(currentSkill.path, 'config')
+        const skillConfigFiles = (
+          await fs.promises.readdir(pathToSkillConfig)
+        ).filter((file) => file.endsWith('.json'))
+
+        for (const file of skillConfigFiles) {
+          const skillConfigPath = path.join(pathToSkillConfig, file)
+          const skillConfig = JSON.parse(
+            await fs.promises.readFile(skillConfigPath, 'utf8')
+          )
+          const result = TypeCompiler.Compile(skillConfigSchemaObject)
+          const errors = [...result.Errors(skillConfig)]
+
+          if (errors.length > 0) {
+            LogHelper.error(
+              `The skill config schema "${skillConfigPath}" is not valid:`
+            )
+            LogHelper.error(JSON.stringify(errors, null, 2))
+            process.exit(1)
+          }
+        }
+      }
+    }
+  }
+  LogHelper.success('Skills data schemas checked')
+
+  process.exit(0)
+})()

server/src/schemas/global-data-schemas.ts

@@ -1,16 +1,16 @@
 import type { Static } from '@sinclair/typebox'
 import { Type } from '@sinclair/typebox'
 
-const globalEntity = {
+export const globalEntitySchemaObject = Type.Object({
   options: Type.Record(
     Type.String(),
     Type.Object({
       synonyms: Type.Array(Type.String()),
-      data: Type.Record(Type.String(), Type.Array(Type.String()))
+      data: Type.Optional(Type.Record(Type.String(), Type.Array(Type.String())))
     })
   )
-}
-const globalResolver = {
+})
+export const globalResolverSchemaObject = Type.Object({
   name: Type.String(),
   intents: Type.Record(
     Type.String(),
@@ -19,8 +19,8 @@ const globalResolver = {
       value: Type.Unknown()
     })
   )
-}
-const answers = {
+})
+export const answersSchemaObject = Type.Object({
   answers: Type.Record(
     Type.String(),
     Type.Union([
@@ -28,17 +28,7 @@ const answers = {
       Type.Array(Type.String())
     ])
   )
-}
-
-const globalEntitySchemaObject = Type.Strict(
-  Type.Object(globalEntity, { additionalProperties: false })
-)
-const globalResolverSchemaObject = Type.Strict(
-  Type.Object(globalResolver, { additionalProperties: false })
-)
-const answersSchemaObject = Type.Strict(
-  Type.Object(answers, { additionalProperties: false })
-)
+})
 
 export type GlobalEntity = Static<typeof globalEntitySchemaObject>
 export type GlobalResolver = Static<typeof globalResolverSchemaObject>

server/src/schemas/skill-schemas.ts

@@ -1,28 +1,81 @@
 import type { Static } from '@sinclair/typebox'
 import { Type } from '@sinclair/typebox'
 
-const domainSchema = {
-  name: Type.String({ minLength: 1 })
-}
-const skillBridges = [Type.Literal('python')]
-const skillSchema = {
-  name: Type.String({ minLength: 1 }),
-  bridge: Type.Union(skillBridges),
-  version: Type.String({ minLength: 1 }),
-  description: Type.String({ minLength: 1 }),
-  author: Type.Object({
-    name: Type.String({ minLength: 1 }),
-    email: Type.String({ minLength: 1, maxLength: 254, format: 'email' }),
-    url: Type.String({ minLength: 1, maxLength: 255, format: 'uri' })
-  })
-}
+const skillBridges = [Type.Literal('python'), Type.Null()]
 const skillActionTypes = [Type.Literal('logic'), Type.Literal('dialog')]
 const skillDataTypes = [
   Type.Literal('skill_resolver'),
   Type.Literal('global_resolver'),
   Type.Literal('entity')
 ]
-const skillConfigSchema = {
+const skillCustomEntityTypes = [
+  Type.Array(
+    Type.Object({
+      type: Type.Literal('trim'),
+      name: Type.String({ minLength: 1 }),
+      conditions: Type.Array(
+        Type.Object({
+          type: Type.Union([
+            Type.Literal('between'),
+            Type.Literal('after'),
+            Type.Literal('after_first'),
+            Type.Literal('after_last'),
+            Type.Literal('before'),
+            Type.Literal('before_first'),
+            Type.Literal('before_last')
+          ]),
+          from: Type.Optional(
+            Type.Union([
+              Type.Array(Type.String({ minLength: 1 })),
+              Type.String({ minLength: 1 })
+            ])
+          ),
+          to: Type.Optional(
+            Type.Union([
+              Type.Array(Type.String({ minLength: 1 })),
+              Type.String({ minLength: 1 })
+            ])
+          )
+        })
+      )
+    })
+  ),
+  Type.Array(
+    Type.Object({
+      type: Type.Literal('regex'),
+      name: Type.String({ minLength: 1 }),
+      regex: Type.String({ minLength: 1 })
+    })
+  ),
+  Type.Array(
+    Type.Object({
+      type: Type.Literal('enum'),
+      name: Type.String(),
+      options: Type.Record(
+        Type.String({ minLength: 1 }),
+        Type.Object({
+          synonyms: Type.Array(Type.String({ minLength: 1 }))
+        })
+      )
+    })
+  )
+]
+
+export const domainSchemaObject = Type.Object({
+  name: Type.String({ minLength: 1 })
+})
+export const skillSchemaObject = Type.Object({
+  name: Type.String({ minLength: 1 }),
+  bridge: Type.Union(skillBridges),
+  version: Type.String({ minLength: 1 }),
+  description: Type.String({ minLength: 1 }),
+  author: Type.Object({
+    name: Type.String({ minLength: 1 }),
+    email: Type.Optional(Type.String({ minLength: 1, maxLength: 254 })),
+    url: Type.Optional(Type.String({ minLength: 1, maxLength: 255 }))
+  })
+})
+export const skillConfigSchemaObject = Type.Object({
   variables: Type.Optional(Type.Record(Type.String(), Type.String())),
   actions: Type.Record(
     Type.String(),
@@ -53,38 +106,15 @@ const skillConfigSchema = {
           })
         )
       ),
-      entities: Type.Optional(
-        Type.Array(
-          Type.Object({
-            type: Type.Literal('enum'),
-            name: Type.String(),
-            options: Type.Record(
-              Type.String(),
-              Type.Object({
-                synonyms: Type.Array(Type.String())
-              })
-            )
-          })
-        )
-      ),
+      entities: Type.Optional(Type.Union(skillCustomEntityTypes)),
       next_action: Type.Optional(Type.String())
     })
   ),
   answers: Type.Optional(Type.Record(Type.String(), Type.Array(Type.String()))),
   entities: Type.Optional(Type.Record(Type.String(), Type.String()))
-}
-
-const domainSchemaObject = Type.Strict(
-  Type.Object(domainSchema, { additionalProperties: false })
-)
-const skillSchemaObject = Type.Strict(
-  Type.Object(skillSchema, { additionalProperties: false })
-)
-const skillConfigSchemaObject = Type.Strict(
-  Type.Object(skillConfigSchema, { additionalProperties: false })
-)
+})
 
 export type Domain = Static<typeof domainSchemaObject>
 export type Skill = Static<typeof skillSchemaObject>
 export type SkillConfig = Static<typeof skillConfigSchemaObject>
-export type SkillBridge = Static<typeof skillSchema.bridge>
+export type SkillBridge = Static<typeof skillSchemaObject.bridge>

server/src/schemas/voice-config-schemas.ts

@@ -1,28 +1,28 @@
 import type { Static } from '@sinclair/typebox'
 import { Type } from '@sinclair/typebox'
 
-const amazonVoiceConfiguration = Type.Object({
+export const amazonVoiceConfiguration = Type.Object({
   credentials: Type.Object({
     accessKeyId: Type.String(),
     secretAccessKey: Type.String()
   }),
   region: Type.String()
 })
-const googleCloudVoiceConfiguration = Type.Object({
+export const googleCloudVoiceConfiguration = Type.Object({
   type: Type.Literal('service_account'),
   project_id: Type.String(),
   private_key_id: Type.String(),
   private_key: Type.String(),
-  client_email: Type.String({ format: 'email' }),
+  client_email: Type.String(),
   client_id: Type.String(),
-  auth_uri: Type.String({ format: 'uri' }),
-  token_uri: Type.String({ format: 'uri' }),
-  auth_provider_x509_cert_url: Type.String({ format: 'uri' }),
-  client_x509_cert_url: Type.String({ format: 'uri' })
+  auth_uri: Type.String(),
+  token_uri: Type.String(),
+  auth_provider_x509_cert_url: Type.String(),
+  client_x509_cert_url: Type.String()
 })
-const watsonVoiceConfiguration = Type.Object({
+export const watsonVoiceConfiguration = Type.Object({
   apikey: Type.String(),
-  url: Type.String({ format: 'uri' })
+  url: Type.String()
 })
 
 export type AmazonVoiceConfiguration = Static<typeof amazonVoiceConfiguration>

server/src/utilities.ts

@@ -0,0 +1,14 @@
+import path from 'node:path'
+
+import type { ShortLanguageCode } from '@/helpers/lang-helper'
+import { GLOBAL_DATA_PATH } from '@/constants'
+
+/**
+ * Paths
+ */
+export function getGlobalEntitiesPath(lang: ShortLanguageCode): string {
+  return path.join(GLOBAL_DATA_PATH, lang, 'global-entities')
+}
+export function getGlobalResolversPath(lang: ShortLanguageCode): string {
+  return path.join(GLOBAL_DATA_PATH, lang, 'global-resolvers')
+}

skills/utilities/speed_test/skill.json

@@ -5,7 +5,6 @@
   "description": "Gives info about your network speed.",
   "author": {
     "name": "Florian Bouché",
-    "email": "",
     "url": "https://github.com/fkeloks"
   }
 }