refactor(server): implement all NER entity types

core/skills-endpoints.json

@@ -3,7 +3,9 @@
     {
       "method": "POST",
       "route": "/api/action/games/akinator/choose_thematic",
-      "params": ["thematic"],
+      "params": [
+        "thematic"
+      ],
       "entitiesType": "trim"
     },
     {
@@ -44,7 +46,9 @@
     {
       "method": "POST",
       "route": "/api/action/games/rochambeau/play",
-      "params": ["handsign"],
+      "params": [
+        "handsign"
+      ],
       "entitiesType": "trim"
     },
     {
@@ -73,9 +77,12 @@
       "params": []
     },
     {
-      "method": "GET",
+      "method": "POST",
       "route": "/api/action/leon/greeting/run",
-      "params": []
+      "params": [
+        "pet"
+      ],
+      "entitiesType": "trim"
     },
     {
       "method": "GET",
@@ -120,7 +127,10 @@
     {
       "method": "POST",
       "route": "/api/action/news/github_trends/run",
-      "params": ["number", "daterange"],
+      "params": [
+        "number",
+        "daterange"
+      ],
       "entitiesType": "builtIn"
     },
     {
@@ -131,7 +141,9 @@
     {
       "method": "POST",
       "route": "/api/action/productivity/todo_list/create_list",
-      "params": ["list"],
+      "params": [
+        "list"
+      ],
       "entitiesType": "trim"
     },
     {
@@ -142,37 +154,53 @@
     {
       "method": "POST",
       "route": "/api/action/productivity/todo_list/view_list",
-      "params": ["list"],
+      "params": [
+        "list"
+      ],
       "entitiesType": "trim"
     },
     {
       "method": "POST",
       "route": "/api/action/productivity/todo_list/rename_list",
-      "params": ["old_list", "new_list"],
+      "params": [
+        "old_list",
+        "new_list"
+      ],
       "entitiesType": "trim"
     },
     {
       "method": "POST",
       "route": "/api/action/productivity/todo_list/delete_list",
-      "params": ["list"],
+      "params": [
+        "list"
+      ],
       "entitiesType": "trim"
     },
     {
       "method": "POST",
       "route": "/api/action/productivity/todo_list/add_todos",
-      "params": ["todos", "list"],
+      "params": [
+        "todos",
+        "list"
+      ],
       "entitiesType": "trim"
     },
     {
       "method": "POST",
       "route": "/api/action/productivity/todo_list/complete_todos",
-      "params": ["todos", "list"],
+      "params": [
+        "todos",
+        "list"
+      ],
       "entitiesType": "trim"
     },
     {
       "method": "POST",
       "route": "/api/action/productivity/todo_list/uncheck_todos",
-      "params": ["todos", "list"],
+      "params": [
+        "todos",
+        "list"
+      ],
       "entitiesType": "trim"
     },
     {
@@ -193,7 +221,9 @@
     {
       "method": "POST",
       "route": "/api/action/utilities/is_it_down/run",
-      "params": ["url"],
+      "params": [
+        "url"
+      ],
       "entitiesType": "builtIn"
     },
     {
@@ -207,4 +237,4 @@
       "params": []
     }
   ]
-}
+}

server/src/core/brain/brain.ts

@@ -4,7 +4,9 @@ import { spawn, ChildProcessWithoutNullStreams } from 'node:child_process'
 
 import type { ShortLanguageCode } from '@/types'
 import type { GlobalAnswers } from '@/schemas/global-data-schemas'
-import type { NLUResult } from '@/core/nlu/types'
+import type { NLUResult } from '@/core/nlp/nlu/types'
+import type { SkillConfig } from '@/schemas/skill-schemas'
+import type { NEREntity } from '@/core/nlp/ner/types'
 import { langs } from '@@/core/langs.json'
 import { HAS_TTS, PYTHON_BRIDGE_BIN_PATH, TMP_PATH } from '@/constants'
 import { SOCKET_SERVER, TTS } from '@/core'
@@ -13,15 +15,14 @@ import { LogHelper } from '@/helpers/log-helper'
 import { SkillDomainHelper } from '@/helpers/skill-domain-helper'
 import { StringHelper } from '@/helpers/string-helper'
 import Synchronizer from '@/core/synchronizer'
-import type { SkillConfig } from '@/schemas/skill-schemas'
 
-interface BrainSkillOutput {
+interface SkillResult {
   domain: string // leon
   skill: string // greeting
   action: string // run
-  lang: string // en
+  lang: ShortLanguageCode // en
   utterance: string // hi
-  entities: [] // [] // TODO
+  entities: NEREntity[]
   slots: object // {} // TODO
   output: {
     type: 'inter' | 'end'
@@ -348,8 +349,6 @@ export default class Brain {
 
             this.skillFinalOutput = output
 
-            console.log('output', output)
-
             // Check if there is an output (no skill error)
             if (this.skillFinalOutput !== '') {
               this.skillFinalOutput = JSON.parse(this.skillFinalOutput).output

server/src/core/index.ts

@@ -5,7 +5,7 @@ import SocketServer from '@/core/socket-server'
 import SpeechToText from '@/core/stt/stt'
 import TextToSpeech from '@/core/tts/tts'
 import AutomaticSpeechRecognition from '@/core/asr/asr'
-import NaturalLanguageUnderstanding from '@/core/nlu/nlu'
+import NaturalLanguageUnderstanding from '@/core/nlp/nlu/nlu'
 import Brain from '@/core/brain/brain'
 
 /**

server/src/core/nlp/ner/types.ts

@@ -0,0 +1,250 @@
+/* eslint-disable @typescript-eslint/no-empty-interface */
+
+interface Entity {
+  start: number
+  end: number
+  len: number
+  accuracy: number
+  sourceText: string
+  utteranceText: string
+  entity: unknown
+  resolution: unknown
+}
+
+/**
+ * Built-in entity types
+ */
+
+interface BuiltInEntity extends Entity {}
+
+interface BuiltInNumberEntity extends BuiltInEntity {
+  resolution: {
+    strValue: string
+    value: number
+    subtype: string
+  }
+}
+interface BuiltInIPEntity extends BuiltInEntity {
+  resolution: {
+    value: string
+    type: string
+  }
+}
+interface BuiltInHashtagEntity extends BuiltInEntity {
+  resolution: {
+    value: string
+  }
+}
+interface BuiltInPhoneNumberEntity extends BuiltInEntity {
+  resolution: {
+    value: string
+    score: string
+  }
+}
+interface BuiltInCurrencyEntity extends BuiltInEntity {
+  resolution: {
+    strValue: string
+    value: number
+    unit: string
+    localeUnit: string
+  }
+}
+interface BuiltInPercentageEntity extends BuiltInEntity {
+  resolution: {
+    strValue: string
+    value: number
+    subtype: string
+  }
+}
+interface BuiltInDateEntity extends BuiltInEntity {
+  resolution: {
+    type: string
+    timex: string
+    strPastValue: string
+    pastDate: Date
+    strFutureValue: string
+    futureDate: Date
+  }
+}
+interface BuiltInTimeEntity extends BuiltInEntity {
+  resolution: {
+    values: {
+      timex: string
+      type: string
+      value: string
+    }[]
+  }
+}
+interface BuiltInTimeRangeEntity extends BuiltInEntity {
+  resolution: {
+    values: {
+      timex: string
+      type: string
+      start: string
+      end: string
+    }[]
+  }
+}
+interface BuiltInDateRangeEntity extends BuiltInEntity {
+  resolution: {
+    type: string
+    timex: string
+    strPastStartValue: string
+    pastStartDate: Date
+    strPastEndValue: string
+    pastEndDate: Date
+    strFutureStartValue: string
+    futureStartDate: Date
+    strFutureEndValue: string
+    futureEndDate: Date
+  }
+}
+interface BuiltInDateTimeRangeEntity extends BuiltInEntity {
+  resolution: {
+    type: string
+    timex: string
+    strPastStartValue: string
+    pastStartDate: Date
+    strPastEndValue: string
+    pastEndDate: Date
+    strFutureStartValue: string
+    futureStartDate: Date
+    strFutureEndValue: string
+    futureEndDate: Date
+  }
+}
+interface BuiltInDurationEntity extends BuiltInEntity {
+  resolution: {
+    values: {
+      timex: string
+      type: string
+      value: string
+    }[]
+  }
+}
+interface BuiltInDimensionEntity extends BuiltInEntity {
+  resolution: {
+    strValue: string
+    value: number
+    unit: string
+    localeUnit: string
+  }
+}
+interface BuiltInEmailEntity extends BuiltInEntity {
+  resolution: {
+    value: string
+  }
+}
+interface BuiltInOrdinalEntity extends BuiltInEntity {
+  resolution: {
+    strValue: string
+    value: number
+    subtype: string
+  }
+}
+interface BuiltInAgeEntity extends BuiltInEntity {
+  resolution: {
+    strValue: string
+    value: number
+    unit: string
+    localeUnit: string
+  }
+}
+interface BuiltInURLEntity extends BuiltInEntity {
+  resolution: {
+    value: string
+  }
+}
+interface BuiltInTemperatureEntity extends BuiltInEntity {
+  resolution: {
+    strValue: string
+    value: number
+    unit: string
+    localeUnit: string
+  }
+}
+
+/**
+ * Custom entity types
+ */
+
+interface CustomEntity<T> extends Entity {
+  type: T
+}
+
+interface CustomEnumEntity extends CustomEntity<'enum'> {
+  levenshtein: number
+  option: string
+  resolution: {
+    value: string
+  }
+}
+type GlobalEntity = CustomEnumEntity
+interface CustomRegexEntity extends CustomEntity<'regex'> {
+  resolution: {
+    value: string
+  }
+}
+interface CustomTrimEntity extends CustomEntity<'trim'> {
+  subtype: 'between' | 'after' | 'afterFirst' | 'afterLast'
+    | 'before' | 'beforeFirst' | 'beforeLast'
+  resolution: {
+    value: string
+  }
+}
+
+/**
+ * spaCy entity types
+ */
+
+interface SpacyEntity<T> extends CustomEnumEntity {
+  entity: T
+}
+
+interface SpacyLocationCountryEntity extends SpacyEntity<'location:country'> {}
+interface SpacyLocationCityEntity extends SpacyEntity<'location:city'> {}
+interface SpacyPersonEntity extends SpacyEntity<'person'> {}
+interface SpacyOrganizationEntity extends SpacyEntity<'organization'> {}
+
+/**
+ * Exported entity types
+ */
+
+export type NERBuiltInEntity =
+  BuiltInNumberEntity
+  | BuiltInIPEntity
+  | BuiltInHashtagEntity
+  | BuiltInPhoneNumberEntity
+  | BuiltInCurrencyEntity
+  | BuiltInPercentageEntity
+  | BuiltInDateEntity
+  | BuiltInTimeEntity
+  | BuiltInTimeRangeEntity
+  | BuiltInDateRangeEntity
+  | BuiltInDateTimeRangeEntity
+  | BuiltInDurationEntity
+  | BuiltInDimensionEntity
+  | BuiltInEmailEntity
+  | BuiltInOrdinalEntity
+  | BuiltInAgeEntity
+  | BuiltInURLEntity
+  | BuiltInTemperatureEntity
+
+export type NERCustomEntity =
+  CustomEnumEntity
+  | CustomRegexEntity
+  | CustomTrimEntity
+
+export type NERGlobalEntity = GlobalEntity
+
+export type NERSpacyEntity =
+  SpacyLocationCountryEntity
+  | SpacyLocationCityEntity
+  | SpacyPersonEntity
+  | SpacyOrganizationEntity
+
+export type NEREntity =
+  NERBuiltInEntity
+  | NERCustomEntity
+  | NERGlobalEntity
+  | NERSpacyEntity