Merge branch 'feat/add-typebox-ajv' into develop

package.json

@@ -52,7 +52,7 @@
     "build:python-bridge": "ts-node scripts/build-binaries.js python-bridge",
     "build:tcp-server": "ts-node scripts/build-binaries.js tcp-server",
     "start:tcp-server": "cross-env PIPENV_PIPFILE=tcp_server/src/Pipfile pipenv run python tcp_server/src/main.py",
-    "start": "cross-env LEON_NODE_ENV=production node ./server/dist/index.js",
+    "start": "cross-env LEON_NODE_ENV=production node server/dist/pre-check.js && node server/dist/index.js",
     "python-bridge": "cross-env PIPENV_PIPFILE=bridges/python/src/Pipfile pipenv run python bridges/python/src/main.py server/src/intent-object.sample.json",
     "train": "ts-node scripts/train/run-train.js",
     "prepare-release": "ts-node scripts/release/prepare-release.js",
@@ -75,6 +75,7 @@
     "@nlpjs/core-loader": "^4.22.7",
     "@nlpjs/lang-all": "^4.22.12",
     "@nlpjs/nlp": "^4.22.17",
+    "@sinclair/typebox": "0.24.49",
     "archiver": "^5.3.1",
     "async": "^3.2.0",
     "axios": "1.1.2",
@@ -99,7 +100,7 @@
   },
   "devDependencies": {
     "@nlpjs/utils": "^4.24.1",
-    "@swc/core": "^1.2.245",
+    "@swc/core": "^1.3.14",
     "@tsconfig/node16-strictest": "^1.0.3",
     "@types/cli-spinner": "0.2.1",
     "@types/node": "^18.7.13",

server/src/constants.ts

@@ -95,3 +95,9 @@ export const HAS_LOGGER = process.env['LEON_LOGGER'] === 'true'
 
 export const TCP_SERVER_HOST = process.env['LEON_PY_TCP_SERVER_HOST']
 export const TCP_SERVER_PORT = process.env['LEON_PY_TCP_SERVER_PORT']
+
+/**
+ * Paths
+ */
+export const GLOBAL_DATA_PATH = path.join('core', 'data')
+export const VOICE_CONFIG_PATH = path.join('core', 'config', 'voice')

server/src/helpers/skill-domain-helper.ts

@@ -2,25 +2,30 @@ import fs from 'node:fs'
 import path from 'node:path'
 
 import type { ShortLanguageCode } from '@/helpers/lang-helper'
-
-enum SkillBridges {
-  Python = 'python'
-}
-
-interface Skill {
-  name: string
-  path: string
-  bridge: `${SkillBridges}`
-}
+import type { GlobalEntity } from '@/schemas/global-data-schemas'
+import type {
+  Domain,
+  Skill,
+  SkillConfig,
+  SkillBridge
+} from '@/schemas/skill-schemas'
 
 interface SkillDomain {
   name: string
   path: string
   skills: {
-    [key: string]: Skill
+    [key: string]: {
+      name: string
+      path: string
+      bridge: SkillBridge
+    }
   }
 }
 
+interface SkillConfigWithGlobalEntities extends Omit<SkillConfig, 'entities'> {
+  entities: Record<string, GlobalEntity>
+}
+
 const DOMAINS_DIR = path.join(process.cwd(), 'skills')
 
 export class SkillDomainHelper {
@@ -38,7 +43,7 @@ export class SkillDomainHelper {
           const skills: SkillDomain['skills'] = {}
           const { name: domainName } = (await import(
             path.join(domainPath, 'domain.json')
-          )) as SkillDomain
+          )) as Domain
           const skillFolders = fs.readdirSync(domainPath)
 
           for (let i = 0; i < skillFolders.length; i += 1) {
@@ -77,7 +82,7 @@ export class SkillDomainHelper {
    * Get information of a specific domain
    * @param domain Domain to get info from
    */
-  public static getSkillDomainInfo(domain: SkillDomain['name']): unknown {
+  public static getSkillDomainInfo(domain: SkillDomain['name']): Domain {
     return JSON.parse(
       fs.readFileSync(path.join(DOMAINS_DIR, domain, 'domain.json'), 'utf8')
     )
@@ -91,7 +96,7 @@ export class SkillDomainHelper {
   public static getSkillInfo(
     domain: SkillDomain['name'],
     skill: Skill['name']
-  ): unknown {
+  ): Skill {
     return JSON.parse(
       fs.readFileSync(
         path.join(DOMAINS_DIR, domain, skill, 'skill.json'),
@@ -108,9 +113,15 @@ export class SkillDomainHelper {
   public static getSkillConfig(
     configFilePath: string,
     lang: ShortLanguageCode
-  ): unknown {
-    const sharedDataPath = path.join(process.cwd(), 'core/data', lang)
-    const configData = JSON.parse(fs.readFileSync(configFilePath, 'utf8'))
+  ): SkillConfigWithGlobalEntities {
+    const sharedDataPath = path.join(process.cwd(), 'core', 'data', lang)
+    const configData = JSON.parse(
+      fs.readFileSync(configFilePath, 'utf8')
+    ) as SkillConfig
+    const result: SkillConfigWithGlobalEntities = {
+      ...configData,
+      entities: {}
+    }
     const { entities } = configData
 
     // Load shared data entities if entity = 'xxx.json'
@@ -119,15 +130,21 @@ export class SkillDomainHelper {
 
       entitiesKeys.forEach((entity) => {
         if (typeof entities[entity] === 'string') {
-          entities[entity] = JSON.parse(
-            fs.readFileSync(path.join(sharedDataPath, entities[entity]), 'utf8')
+          const entityFilePath = path.join(
+            sharedDataPath,
+            entities[entity] as string
           )
+          const entityRawData = fs.readFileSync(entityFilePath, {
+            encoding: 'utf8'
+          })
+
+          result.entities[entity] = JSON.parse(entityRawData) as GlobalEntity
         }
       })
 
       configData.entities = entities
     }
 
-    return configData
+    return result
   }
 }

server/src/index.ts

@@ -13,6 +13,7 @@ import server from '@/core/http-server/server'
 ;(async (): Promise<void> => {
   process.title = 'leon'
 
+  // Start the TCP server
   global.tcpServerProcess = spawn(
     `${TCP_SERVER_BIN_PATH} ${LangHelper.getShortCode(LEON_LANG)}`,
     {
@@ -21,7 +22,9 @@ import server from '@/core/http-server/server'
     }
   )
 
+  // Start the TCP client
   global.tcpClient = new TcpClient(TCP_SERVER_HOST, TCP_SERVER_PORT)
 
+  // Start the core server
   await server.init()
 })()

server/src/pre-check.ts

@@ -0,0 +1,219 @@
+import fs from 'node:fs'
+import path from 'node:path'
+
+import { TypeCompiler } from '@sinclair/typebox/compiler'
+
+import {
+  amazonVoiceConfiguration,
+  googleCloudVoiceConfiguration,
+  watsonVoiceConfiguration
+} from '@/schemas/voice-config-schemas'
+import {
+  answersSchemaObject,
+  globalEntitySchemaObject,
+  globalResolverSchemaObject
+} from '@/schemas/global-data-schemas'
+import {
+  domainSchemaObject,
+  skillSchemaObject,
+  skillConfigSchemaObject
+} from '@/schemas/skill-schemas'
+import { LogHelper } from '@/helpers/log-helper'
+import { LangHelper } from '@/helpers/lang-helper'
+import { SkillDomainHelper } from '@/helpers/skill-domain-helper'
+import { VOICE_CONFIG_PATH, GLOBAL_DATA_PATH } from '@/constants'
+import { getGlobalEntitiesPath, getGlobalResolversPath } from '@/utilities'
+
+/**
+ * Pre-checking
+ * Ensure JSON files are correctly formatted
+ */
+const VOICE_CONFIG_SCHEMAS = {
+  amazon: amazonVoiceConfiguration,
+  'google-cloud': googleCloudVoiceConfiguration,
+  'watson-stt': watsonVoiceConfiguration,
+  'watson-tts': watsonVoiceConfiguration
+}
+const GLOBAL_DATA_SCHEMAS = {
+  answers: answersSchemaObject,
+  globalEntities: globalEntitySchemaObject,
+  globalResolvers: globalResolverSchemaObject
+}
+
+;(async (): Promise<void> => {
+  LogHelper.title('Pre-checking')
+
+  /**
+   * Voice configuration checking
+   */
+  LogHelper.info('Checking voice configuration schemas...')
+
+  const voiceConfigFiles = (
+    await fs.promises.readdir(VOICE_CONFIG_PATH)
+  ).filter((file) => file.endsWith('.json') && !file.includes('.sample.'))
+
+  for (const file of voiceConfigFiles) {
+    const config = JSON.parse(
+      await fs.promises.readFile(path.join(VOICE_CONFIG_PATH, file), 'utf8')
+    )
+    const [configName] = file.split('.') as [keyof typeof VOICE_CONFIG_SCHEMAS]
+    const result = TypeCompiler.Compile(VOICE_CONFIG_SCHEMAS[configName])
+    const errors = [...result.Errors(config)]
+
+    if (errors.length > 0) {
+      LogHelper.error(`The voice configuration schema "${file}" is not valid:`)
+      LogHelper.error(JSON.stringify(errors, null, 2))
+      process.exit(1)
+    }
+  }
+  LogHelper.success('Voice configuration schemas checked')
+
+  /**
+   * Global data checking
+   */
+  LogHelper.info('Checking global data schemas...')
+
+  const supportedLangs = LangHelper.getShortCodes()
+  for (const lang of supportedLangs) {
+    /**
+     * Global entities checking
+     */
+    const globalEntitiesPath = getGlobalEntitiesPath(lang)
+    const globalEntityFiles = (
+      await fs.promises.readdir(globalEntitiesPath)
+    ).filter((file) => file.endsWith('.json'))
+
+    for (const file of globalEntityFiles) {
+      const globalEntity = JSON.parse(
+        await fs.promises.readFile(path.join(globalEntitiesPath, file), 'utf8')
+      )
+      const result = TypeCompiler.Compile(globalEntitySchemaObject)
+      const errors = [...result.Errors(globalEntity)]
+
+      if (errors.length > 0) {
+        LogHelper.error(`The global entity schema "${file}" is not valid:`)
+        LogHelper.error(JSON.stringify(errors, null, 2))
+        process.exit(1)
+      }
+    }
+
+    /**
+     * Global resolvers checking
+     */
+    const globalResolversPath = getGlobalResolversPath(lang)
+    const globalResolverFiles = (
+      await fs.promises.readdir(globalResolversPath)
+    ).filter((file) => file.endsWith('.json'))
+
+    for (const file of globalResolverFiles) {
+      const globalResolver = JSON.parse(
+        await fs.promises.readFile(path.join(globalResolversPath, file), 'utf8')
+      )
+      const result = TypeCompiler.Compile(globalResolverSchemaObject)
+      const errors = [...result.Errors(globalResolver)]
+
+      if (errors.length > 0) {
+        LogHelper.error(`The global resolver schema "${file}" is not valid:`)
+        LogHelper.error(JSON.stringify(errors, null, 2))
+        process.exit(1)
+      }
+    }
+
+    /**
+     * Global answers checking
+     */
+    const answers = JSON.parse(
+      await fs.promises.readFile(
+        path.join(GLOBAL_DATA_PATH, lang, 'answers.json'),
+        'utf8'
+      )
+    )
+    const result = TypeCompiler.Compile(GLOBAL_DATA_SCHEMAS.answers)
+
+    const errors = [...result.Errors(answers)]
+
+    if (errors.length > 0) {
+      LogHelper.error(`The global answers schema "answers.json" is not valid:`)
+      LogHelper.error(JSON.stringify(errors, null, 2))
+      process.exit(1)
+    }
+  }
+  LogHelper.success('Global data schemas checked')
+
+  /**
+   * Skills data checking
+   */
+  LogHelper.info('Checking skills data schemas...')
+
+  const skillDomains = await SkillDomainHelper.getSkillDomains()
+
+  for (const [, currentDomain] of skillDomains) {
+    /**
+     * Domain checking
+     */
+    const pathToDomain = path.join(currentDomain.path, 'domain.json')
+    const domainObject = JSON.parse(
+      await fs.promises.readFile(pathToDomain, 'utf8')
+    )
+    const domainResult = TypeCompiler.Compile(domainSchemaObject)
+    const domainErrors = [...domainResult.Errors(domainObject)]
+
+    if (domainErrors.length > 0) {
+      LogHelper.error(`The domain schema "${pathToDomain}" is not valid:`)
+      LogHelper.error(JSON.stringify(domainErrors, null, 2))
+      process.exit(1)
+    }
+
+    const skillKeys = Object.keys(currentDomain.skills)
+
+    for (const skillKey of skillKeys) {
+      const currentSkill = currentDomain.skills[skillKey]
+
+      /**
+       * Skills checking
+       */
+      if (currentSkill) {
+        const pathToSkill = path.join(currentSkill.path, 'skill.json')
+        const skillObject = JSON.parse(
+          await fs.promises.readFile(pathToSkill, 'utf8')
+        )
+        const skillResult = TypeCompiler.Compile(skillSchemaObject)
+        const skillErrors = [...skillResult.Errors(skillObject)]
+
+        if (skillErrors.length > 0) {
+          LogHelper.error(`The skill schema "${pathToSkill}" is not valid:`)
+          LogHelper.error(JSON.stringify(skillErrors, null, 2))
+          process.exit(1)
+        }
+
+        /**
+         * Skills config checking
+         */
+        const pathToSkillConfig = path.join(currentSkill.path, 'config')
+        const skillConfigFiles = (
+          await fs.promises.readdir(pathToSkillConfig)
+        ).filter((file) => file.endsWith('.json'))
+
+        for (const file of skillConfigFiles) {
+          const skillConfigPath = path.join(pathToSkillConfig, file)
+          const skillConfig = JSON.parse(
+            await fs.promises.readFile(skillConfigPath, 'utf8')
+          )
+          const result = TypeCompiler.Compile(skillConfigSchemaObject)
+          const errors = [...result.Errors(skillConfig)]
+
+          if (errors.length > 0) {
+            LogHelper.error(
+              `The skill config schema "${skillConfigPath}" is not valid:`
+            )
+            LogHelper.error(JSON.stringify(errors, null, 2))
+            process.exit(1)
+          }
+        }
+      }
+    }
+  }
+  LogHelper.success('Skills data schemas checked')
+
+  process.exit(0)
+})()

server/src/schemas/global-data-schemas.ts

@@ -0,0 +1,35 @@
+import type { Static } from '@sinclair/typebox'
+import { Type } from '@sinclair/typebox'
+
+export const globalEntitySchemaObject = Type.Object({
+  options: Type.Record(
+    Type.String(),
+    Type.Object({
+      synonyms: Type.Array(Type.String()),
+      data: Type.Optional(Type.Record(Type.String(), Type.Array(Type.String())))
+    })
+  )
+})
+export const globalResolverSchemaObject = Type.Object({
+  name: Type.String(),
+  intents: Type.Record(
+    Type.String(),
+    Type.Object({
+      utterance_samples: Type.Array(Type.String()),
+      value: Type.Unknown()
+    })
+  )
+})
+export const answersSchemaObject = Type.Object({
+  answers: Type.Record(
+    Type.String(),
+    Type.Union([
+      Type.Record(Type.String(), Type.String()),
+      Type.Array(Type.String())
+    ])
+  )
+})
+
+export type GlobalEntity = Static<typeof globalEntitySchemaObject>
+export type GlobalResolver = Static<typeof globalResolverSchemaObject>
+export type Answers = Static<typeof answersSchemaObject>

server/src/schemas/skill-schemas.ts

@@ -0,0 +1,120 @@
+import type { Static } from '@sinclair/typebox'
+import { Type } from '@sinclair/typebox'
+
+const skillBridges = [Type.Literal('python'), Type.Null()]
+const skillActionTypes = [Type.Literal('logic'), Type.Literal('dialog')]
+const skillDataTypes = [
+  Type.Literal('skill_resolver'),
+  Type.Literal('global_resolver'),
+  Type.Literal('entity')
+]
+const skillCustomEntityTypes = [
+  Type.Array(
+    Type.Object({
+      type: Type.Literal('trim'),
+      name: Type.String({ minLength: 1 }),
+      conditions: Type.Array(
+        Type.Object({
+          type: Type.Union([
+            Type.Literal('between'),
+            Type.Literal('after'),
+            Type.Literal('after_first'),
+            Type.Literal('after_last'),
+            Type.Literal('before'),
+            Type.Literal('before_first'),
+            Type.Literal('before_last')
+          ]),
+          from: Type.Optional(
+            Type.Union([
+              Type.Array(Type.String({ minLength: 1 })),
+              Type.String({ minLength: 1 })
+            ])
+          ),
+          to: Type.Optional(
+            Type.Union([
+              Type.Array(Type.String({ minLength: 1 })),
+              Type.String({ minLength: 1 })
+            ])
+          )
+        })
+      )
+    })
+  ),
+  Type.Array(
+    Type.Object({
+      type: Type.Literal('regex'),
+      name: Type.String({ minLength: 1 }),
+      regex: Type.String({ minLength: 1 })
+    })
+  ),
+  Type.Array(
+    Type.Object({
+      type: Type.Literal('enum'),
+      name: Type.String(),
+      options: Type.Record(
+        Type.String({ minLength: 1 }),
+        Type.Object({
+          synonyms: Type.Array(Type.String({ minLength: 1 }))
+        })
+      )
+    })
+  )
+]
+
+export const domainSchemaObject = Type.Object({
+  name: Type.String({ minLength: 1 })
+})
+export const skillSchemaObject = Type.Object({
+  name: Type.String({ minLength: 1 }),
+  bridge: Type.Union(skillBridges),
+  version: Type.String({ minLength: 1 }),
+  description: Type.String({ minLength: 1 }),
+  author: Type.Object({
+    name: Type.String({ minLength: 1 }),
+    email: Type.Optional(Type.String({ minLength: 1, maxLength: 254 })),
+    url: Type.Optional(Type.String({ minLength: 1, maxLength: 255 }))
+  })
+})
+export const skillConfigSchemaObject = Type.Object({
+  variables: Type.Optional(Type.Record(Type.String(), Type.String())),
+  actions: Type.Record(
+    Type.String(),
+    Type.Object({
+      type: Type.Union(skillActionTypes),
+      loop: Type.Optional(
+        Type.Object({
+          expected_item: Type.Object({
+            type: Type.Union(skillDataTypes),
+            name: Type.String()
+          })
+        })
+      ),
+      utterance_samples: Type.Optional(Type.Array(Type.String())),
+      answers: Type.Optional(Type.Array(Type.String())),
+      unknown_answers: Type.Optional(Type.Array(Type.String())),
+      suggestions: Type.Optional(Type.Array(Type.String())),
+      slots: Type.Optional(
+        Type.Array(
+          Type.Object({
+            name: Type.String(),
+            item: Type.Object({
+              type: Type.Union(skillDataTypes),
+              name: Type.String()
+            }),
+            questions: Type.Array(Type.String()),
+            suggestions: Type.Optional(Type.Array(Type.String()))
+          })
+        )
+      ),
+      entities: Type.Optional(Type.Union(skillCustomEntityTypes)),
+      next_action: Type.Optional(Type.String())
+    })
+  ),
+  answers: Type.Optional(Type.Record(Type.String(), Type.Array(Type.String()))),
+  entities: Type.Optional(Type.Record(Type.String(), Type.String()))
+})
+
+export type Domain = Static<typeof domainSchemaObject>
+export type Skill = Static<typeof skillSchemaObject>
+export type SkillConfig = Static<typeof skillConfigSchemaObject>
+export type SkillBridge = Static<typeof skillSchemaObject.bridge>

server/src/schemas/voice-config-schemas.ts

@@ -0,0 +1,32 @@
+import type { Static } from '@sinclair/typebox'
+import { Type } from '@sinclair/typebox'
+
+export const amazonVoiceConfiguration = Type.Object({
+  credentials: Type.Object({
+    accessKeyId: Type.String(),
+    secretAccessKey: Type.String()
+  }),
+  region: Type.String()
+})
+export const googleCloudVoiceConfiguration = Type.Object({
+  type: Type.Literal('service_account'),
+  project_id: Type.String(),
+  private_key_id: Type.String(),
+  private_key: Type.String(),
+  client_email: Type.String(),
+  client_id: Type.String(),
+  auth_uri: Type.String(),
+  token_uri: Type.String(),
+  auth_provider_x509_cert_url: Type.String(),
+  client_x509_cert_url: Type.String()
+})
+export const watsonVoiceConfiguration = Type.Object({
+  apikey: Type.String(),
+  url: Type.String()
+})
+
+export type AmazonVoiceConfiguration = Static<typeof amazonVoiceConfiguration>
+export type GoogleCloudVoiceConfiguration = Static<
+  typeof googleCloudVoiceConfiguration
+>
+export type WatsonVoiceConfiguration = Static<typeof watsonVoiceConfiguration>

server/src/utilities.ts

@@ -0,0 +1,14 @@
+import path from 'node:path'
+
+import type { ShortLanguageCode } from '@/helpers/lang-helper'
+import { GLOBAL_DATA_PATH } from '@/constants'
+
+/**
+ * Paths
+ */
+export function getGlobalEntitiesPath(lang: ShortLanguageCode): string {
+  return path.join(GLOBAL_DATA_PATH, lang, 'global-entities')
+}
+export function getGlobalResolversPath(lang: ShortLanguageCode): string {
+  return path.join(GLOBAL_DATA_PATH, lang, 'global-resolvers')
+}

skills/utilities/speed_test/skill.json

@@ -5,7 +5,6 @@
   "description": "Gives info about your network speed.",
   "author": {
     "name": "Florian Bouché",
-    "email": "",
     "url": "https://github.com/fkeloks"
   }
 }