feat: auto update mood on client; fix NER on slot filling

app/src/js/client.js

@@ -34,6 +34,15 @@ export default class Client {
     return this._recorder
   }
 
+  updateMood(mood) {
+    if (this.info.llm.enabled) {
+      const moodContainer = document.querySelector('#mood')
+
+      moodContainer.innerHTML = `Leon's mood: ${mood.emoji}`
+      moodContainer.setAttribute('title', mood.type)
+    }
+  }
+
   async sendInitMessages() {
     for (let i = 0; i < INIT_MESSAGES.length; i++) {
       const messages = INIT_MESSAGES[i]
@@ -109,6 +118,10 @@ export default class Client {
       this.chatbot.createBubble('leon', data)
     })
 
+    this.socket.on('new-mood', (mood) => {
+      this.updateMood(mood)
+    })
+
     this.socket.on('llm-token', (data) => {
       const previousGenerationId = this.answerGenerationId
       const newGenerationId = data.generationId

app/src/js/main.js

@@ -28,18 +28,13 @@ document.addEventListener('DOMContentLoaded', async () => {
     const input = document.querySelector('#utterance')
     const mic = document.querySelector('#mic-button')
     const v = document.querySelector('#version small')
-    const moodContainer = document.querySelector('#mood')
     const client = new Client(config.app, serverUrl, input, response.data)
     let rec = {}
     let chunks = []
 
     v.innerHTML += client.info.version
 
-    if (client.info.llm.enabled) {
-      moodContainer.innerHTML = `Leon's mood: ${client.info.mood.emoji}`
-      moodContainer.setAttribute('title', client.info.mood.type)
-    }
-
+    client.updateMood(client.info.mood)
     client.init(loader)
 
     if (navigator.mediaDevices && navigator.mediaDevices.getUserMedia) {

server/src/core/index.ts

@@ -49,8 +49,6 @@ export const LLM_PROVIDER = new LLMProvider()
 
 export const LLM_MANAGER = new LLMManager()
 
-export const PERSONA = new Persona()
-
 export const CONVERSATION_LOGGER = new ConversationLogger({
   loggerName: 'Conversation Logger',
   fileName: 'conversation_log.json',
@@ -68,6 +66,8 @@ export const HTTP_SERVER = new HTTPServer(String(HOST), PORT)
 
 export const SOCKET_SERVER = new SocketServer()
 
+export const PERSONA = new Persona()
+
 export const STT = new SpeechToText()
 
 export const TTS = new TextToSpeech()

server/src/core/llm-manager/persona.ts

@@ -1,3 +1,4 @@
+import { SOCKET_SERVER } from '@/core'
 import { LogHelper } from '@/helpers/log-helper'
 
 /**
@@ -19,8 +20,10 @@ enum Moods {
 }
 
 /**
- * TODO: add environment context such as time of the day, day of the week, weather, etc.
- * TODO: make sure the new system prompt is then being updated for long-lived duty such as chit-chat
+ * TODO:
+ * Add environment context such as time of the day, day of the week, weather, etc.
+ * Make sure the new system prompt is then being updated for long-lived duty such as chit-chat.
+ * Provide more user context to the persona (habits, preferences, etc.)
  */
 const WHO_YOU_ARE = `WHO YOU ARE:
 Your name is Leon, you are a helpful AI assistant.
@@ -132,7 +135,12 @@ export default class Persona {
       this._mood = pickedMood
     }
 
-    // TODO: send socket message to the client to display the new mood represented by an emoji
+    if (SOCKET_SERVER) {
+      SOCKET_SERVER.socket?.emit('new-mood', {
+        type: this._mood.type,
+        emoji: this._mood.emoji
+      })
+    }
 
     LogHelper.info(`Mood set to: ${this._mood.type}`)
   }

server/src/core/nlp/conversation.ts

@@ -197,9 +197,11 @@ export default class Conversation {
       }
 
       // Match the slot with the submitted entity and ensure the slot hasn't been filled yet
-      const [foundEntity] = entities.filter(
+      const foundEntities = entities.filter(
         ({ entity }) => entity === slotEntity && !slotObj.isFilled
       )
+      // get the last found entity
+      const foundEntity = foundEntities[foundEntities.length - 1]
       const pickedQuestion = questions[
         Math.floor(Math.random() * questions.length)
       ] as string

tcp_server/src/lib/asr/api.py

@@ -55,7 +55,7 @@ class ASR:
         self.channels = 1
         self.rate = 16000
         self.chunk = 4096
-        self.threshold = 200
+        self.threshold = 128
         self.silence_duration = 1.5  # duration of silence in seconds
         self.buffer_size = 64  # Size of the circular buffer
 

tcp_server/src/main.py

@@ -20,10 +20,10 @@ tcp_server_port = os.environ.get('LEON_PY_TCP_SERVER_PORT', 1342)
 
 tcp_server = TCPServer(tcp_server_host, tcp_server_port)
 
+# Use thread as ASR starts recording audio and it blocks the main thread
 asr_thread = threading.Thread(target=tcp_server.init_asr)
 asr_thread.start()
 
-tts_thread = threading.Thread(target=tcp_server.init_tts)
-tts_thread.start()
+tcp_server.init_tts()
 
 tcp_server.init()