From 91fb441dcd0f9715db8c370544a4f6158ba28b2f Mon Sep 17 00:00:00 2001 From: louistiti Date: Thu, 23 May 2024 17:49:55 +0800 Subject: [PATCH] feat: auto update mood on client; fix NER on slot filling --- app/src/js/client.js | 13 +++++++++++++ app/src/js/main.js | 7 +------ server/src/core/index.ts | 4 ++-- server/src/core/llm-manager/persona.ts | 14 +++++++++++--- server/src/core/nlp/conversation.ts | 4 +++- tcp_server/src/lib/asr/api.py | 2 +- tcp_server/src/main.py | 4 ++-- 7 files changed, 33 insertions(+), 15 deletions(-) diff --git a/app/src/js/client.js b/app/src/js/client.js index 5cc780ed..86e3b0d5 100644 --- a/app/src/js/client.js +++ b/app/src/js/client.js @@ -34,6 +34,15 @@ export default class Client { return this._recorder } + updateMood(mood) { + if (this.info.llm.enabled) { + const moodContainer = document.querySelector('#mood') + + moodContainer.innerHTML = `Leon's mood: ${mood.emoji}` + moodContainer.setAttribute('title', mood.type) + } + } + async sendInitMessages() { for (let i = 0; i < INIT_MESSAGES.length; i++) { const messages = INIT_MESSAGES[i] @@ -109,6 +118,10 @@ export default class Client { this.chatbot.createBubble('leon', data) }) + this.socket.on('new-mood', (mood) => { + this.updateMood(mood) + }) + this.socket.on('llm-token', (data) => { const previousGenerationId = this.answerGenerationId const newGenerationId = data.generationId diff --git a/app/src/js/main.js b/app/src/js/main.js index b0202b4a..2f3039d2 100644 --- a/app/src/js/main.js +++ b/app/src/js/main.js @@ -28,18 +28,13 @@ document.addEventListener('DOMContentLoaded', async () => { const input = document.querySelector('#utterance') const mic = document.querySelector('#mic-button') const v = document.querySelector('#version small') - const moodContainer = document.querySelector('#mood') const client = new Client(config.app, serverUrl, input, response.data) let rec = {} let chunks = [] v.innerHTML += client.info.version - if (client.info.llm.enabled) { - moodContainer.innerHTML = `Leon's mood: ${client.info.mood.emoji}` - moodContainer.setAttribute('title', client.info.mood.type) - } - + client.updateMood(client.info.mood) client.init(loader) if (navigator.mediaDevices && navigator.mediaDevices.getUserMedia) { diff --git a/server/src/core/index.ts b/server/src/core/index.ts index 19faaf8d..a21bf21e 100644 --- a/server/src/core/index.ts +++ b/server/src/core/index.ts @@ -49,8 +49,6 @@ export const LLM_PROVIDER = new LLMProvider() export const LLM_MANAGER = new LLMManager() -export const PERSONA = new Persona() - export const CONVERSATION_LOGGER = new ConversationLogger({ loggerName: 'Conversation Logger', fileName: 'conversation_log.json', @@ -68,6 +66,8 @@ export const HTTP_SERVER = new HTTPServer(String(HOST), PORT) export const SOCKET_SERVER = new SocketServer() +export const PERSONA = new Persona() + export const STT = new SpeechToText() export const TTS = new TextToSpeech() diff --git a/server/src/core/llm-manager/persona.ts b/server/src/core/llm-manager/persona.ts index 0909cf41..88c38c17 100644 --- a/server/src/core/llm-manager/persona.ts +++ b/server/src/core/llm-manager/persona.ts @@ -1,3 +1,4 @@ +import { SOCKET_SERVER } from '@/core' import { LogHelper } from '@/helpers/log-helper' /** @@ -19,8 +20,10 @@ enum Moods { } /** - * TODO: add environment context such as time of the day, day of the week, weather, etc. - * TODO: make sure the new system prompt is then being updated for long-lived duty such as chit-chat + * TODO: + * Add environment context such as time of the day, day of the week, weather, etc. + * Make sure the new system prompt is then being updated for long-lived duty such as chit-chat. + * Provide more user context to the persona (habits, preferences, etc.) */ const WHO_YOU_ARE = `WHO YOU ARE: Your name is Leon, you are a helpful AI assistant. @@ -132,7 +135,12 @@ export default class Persona { this._mood = pickedMood } - // TODO: send socket message to the client to display the new mood represented by an emoji + if (SOCKET_SERVER) { + SOCKET_SERVER.socket?.emit('new-mood', { + type: this._mood.type, + emoji: this._mood.emoji + }) + } LogHelper.info(`Mood set to: ${this._mood.type}`) } diff --git a/server/src/core/nlp/conversation.ts b/server/src/core/nlp/conversation.ts index a107ddc5..5e79e638 100644 --- a/server/src/core/nlp/conversation.ts +++ b/server/src/core/nlp/conversation.ts @@ -197,9 +197,11 @@ export default class Conversation { } // Match the slot with the submitted entity and ensure the slot hasn't been filled yet - const [foundEntity] = entities.filter( + const foundEntities = entities.filter( ({ entity }) => entity === slotEntity && !slotObj.isFilled ) + // get the last found entity + const foundEntity = foundEntities[foundEntities.length - 1] const pickedQuestion = questions[ Math.floor(Math.random() * questions.length) ] as string diff --git a/tcp_server/src/lib/asr/api.py b/tcp_server/src/lib/asr/api.py index 67291ba3..2f0fc877 100644 --- a/tcp_server/src/lib/asr/api.py +++ b/tcp_server/src/lib/asr/api.py @@ -55,7 +55,7 @@ class ASR: self.channels = 1 self.rate = 16000 self.chunk = 4096 - self.threshold = 200 + self.threshold = 128 self.silence_duration = 1.5 # duration of silence in seconds self.buffer_size = 64 # Size of the circular buffer diff --git a/tcp_server/src/main.py b/tcp_server/src/main.py index 3fa1ca2b..4bc22fd6 100644 --- a/tcp_server/src/main.py +++ b/tcp_server/src/main.py @@ -20,10 +20,10 @@ tcp_server_port = os.environ.get('LEON_PY_TCP_SERVER_PORT', 1342) tcp_server = TCPServer(tcp_server_host, tcp_server_port) +# Use thread as ASR starts recording audio and it blocks the main thread asr_thread = threading.Thread(target=tcp_server.init_asr) asr_thread.start() -tts_thread = threading.Thread(target=tcp_server.init_tts) -tts_thread.start() +tcp_server.init_tts() tcp_server.init()