mirror of
https://github.com/leon-ai/leon.git
synced 2024-08-17 06:00:33 +03:00
feat: new ASR engine ready
This commit is contained in:
parent
f051c1d2cd
commit
596b7552dd
@ -91,8 +91,8 @@ export default class Client {
|
||||
this.chatbot.createBubble('leon', data)
|
||||
})
|
||||
|
||||
this.socket.on('asr-speech', (data) => {
|
||||
console.log('Wake word detected', data)
|
||||
this.socket.on('asr-speech', (text) => {
|
||||
console.log('Wake word detected', text)
|
||||
})
|
||||
|
||||
this.socket.on('asr-end-of-owner-speech', () => {
|
||||
|
@ -63,7 +63,8 @@
|
||||
"pre-release:nodejs-bridge": "tsx scripts/release/pre-release-binaries.js nodejs-bridge",
|
||||
"pre-release:python-bridge": "tsx scripts/release/pre-release-binaries.js python-bridge",
|
||||
"pre-release:tcp-server": "tsx scripts/release/pre-release-binaries.js tcp-server",
|
||||
"check": "tsx scripts/check.js"
|
||||
"check": "tsx scripts/check.js",
|
||||
"kill": "pkill -f node && pkill -f leon-tcp-server && pkill -f pt_main_thread"
|
||||
},
|
||||
"dependencies": {
|
||||
"@aws-sdk/client-polly": "3.18.0",
|
||||
|
@ -160,6 +160,32 @@ SPACY_MODELS.set('fr', {
|
||||
stdio: 'inherit'
|
||||
})
|
||||
LogHelper.success('PyTorch with CUDA support installed')
|
||||
|
||||
if (osType === OSTypes.Linux) {
|
||||
LogHelper.info(
|
||||
'Exporting LD_LIBRARY_PATH to map NVIDIA libs as it is needed by Whisper Faster. Cf. https://github.com/SYSTRAN/faster-whisper/issues/153...'
|
||||
)
|
||||
|
||||
try {
|
||||
await command(
|
||||
// eslint-disable-next-line no-useless-escape
|
||||
'export LD_LIBRARY_PATH=`pipenv run python -c "import os; import nvidia.cublas.lib; import nvidia.cudnn.lib; print(os.path.dirname(nvidia.cublas.lib.__file__) + ":" + os.path.dirname(nvidia.cudnn.lib.__file__))"`',
|
||||
{
|
||||
shell: true,
|
||||
stdio: 'inherit'
|
||||
}
|
||||
)
|
||||
await command('echo $LD_LIBRARY_PATH', {
|
||||
shell: true,
|
||||
stdio: 'inherit'
|
||||
})
|
||||
|
||||
LogHelper.success('LD_LIBRARY_PATH exported')
|
||||
} catch (e) {
|
||||
LogHelper.error(`Failed to export LD_LIBRARY_PATH: ${e}`)
|
||||
process.exit(1)
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
LogHelper.error(`Failed to install PyTorch with CUDA support: ${e}`)
|
||||
process.exit(1)
|
||||
|
@ -72,6 +72,28 @@ export const NODEJS_BRIDGE_BIN_NAME = 'leon-nodejs-bridge.js'
|
||||
export const PYTHON_BRIDGE_BIN_NAME = 'leon-python-bridge'
|
||||
export const PYTHON_TCP_SERVER_BIN_NAME = 'leon-tcp-server'
|
||||
|
||||
/**
|
||||
* NVIDIA libraries paths for CUDA. Needed by Whisper Faster
|
||||
* @see https://github.com/SYSTRAN/faster-whisper/issues/153
|
||||
*/
|
||||
export const PYTHON_TCP_SERVER_NVIDIA_CUBLAS_LIB_PATH = path.join(
|
||||
PYTHON_TCP_SERVER_DIST_PATH,
|
||||
BINARIES_FOLDER_NAME,
|
||||
'lib',
|
||||
'nvidia',
|
||||
'cublas',
|
||||
'lib'
|
||||
)
|
||||
export const PYTHON_TCP_SERVER_NVIDIA_CUDNN_LIB_PATH = path.join(
|
||||
PYTHON_TCP_SERVER_DIST_PATH,
|
||||
BINARIES_FOLDER_NAME,
|
||||
'lib',
|
||||
'nvidia',
|
||||
'cudnn',
|
||||
'lib'
|
||||
)
|
||||
export const PYTHON_TCP_SERVER_LD_LIBRARY_PATH = `${PYTHON_TCP_SERVER_NVIDIA_CUBLAS_LIB_PATH}:${PYTHON_TCP_SERVER_NVIDIA_CUDNN_LIB_PATH}`
|
||||
|
||||
export const PYTHON_TCP_SERVER_BIN_PATH = path.join(
|
||||
PYTHON_TCP_SERVER_DIST_PATH,
|
||||
BINARIES_FOLDER_NAME,
|
||||
|
@ -2,7 +2,8 @@ import {
|
||||
HOST,
|
||||
PORT,
|
||||
PYTHON_TCP_SERVER_HOST,
|
||||
PYTHON_TCP_SERVER_PORT
|
||||
PYTHON_TCP_SERVER_PORT,
|
||||
PYTHON_TCP_SERVER_LD_LIBRARY_PATH
|
||||
} from '@/constants'
|
||||
import TCPClient from '@/core/tcp-client'
|
||||
import HTTPServer from '@/core/http-server/http-server'
|
||||
@ -18,6 +19,17 @@ import LLMManager from '@/core/llm-manager/llm-manager'
|
||||
import LLMProvider from '@/core/llm-manager/llm-provider'
|
||||
import Persona from '@/core/llm-manager/persona'
|
||||
import { ConversationLogger } from '@/conversation-logger'
|
||||
import { SystemHelper } from '@/helpers/system-helper'
|
||||
import { LogHelper } from '@/helpers/log-helper'
|
||||
|
||||
/**
|
||||
* Set environment variables
|
||||
*/
|
||||
|
||||
if (SystemHelper.isLinux()) {
|
||||
process.env['LD_LIBRARY_PATH'] = PYTHON_TCP_SERVER_LD_LIBRARY_PATH
|
||||
LogHelper.info(`LD_LIBRARY_PATH set to: ${process.env['LD_LIBRARY_PATH']}`)
|
||||
}
|
||||
|
||||
/**
|
||||
* Register core nodes
|
||||
|
@ -155,4 +155,34 @@ export class SystemHelper {
|
||||
|
||||
return str.replace(new RegExp(username, 'g'), '{username}')
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if the current OS is Windows
|
||||
* @example isWindows() // false
|
||||
*/
|
||||
public static isWindows(): boolean {
|
||||
const { type } = this.getInformation()
|
||||
|
||||
return type === OSTypes.Windows
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if the current OS is macOS
|
||||
* @example isMacOS() // false
|
||||
*/
|
||||
public static isMacOS(): boolean {
|
||||
const { type } = this.getInformation()
|
||||
|
||||
return type === OSTypes.MacOS
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if the current OS is Linux
|
||||
* @example isLinux() // true
|
||||
*/
|
||||
public static isLinux(): boolean {
|
||||
const { type } = this.getInformation()
|
||||
|
||||
return type === OSTypes.Linux
|
||||
}
|
||||
}
|
||||
|
@ -47,7 +47,8 @@ import { LogHelper } from '@/helpers/log-helper'
|
||||
'<00:00',
|
||||
'00:00<',
|
||||
'CUDNN_STATUS_NOT_SUPPORTED',
|
||||
'cls.seq_relationship.weight'
|
||||
'cls.seq_relationship.weight',
|
||||
'ALSA lib'
|
||||
]
|
||||
|
||||
if (skipError.some((error) => formattedData.includes(error))) {
|
||||
|
@ -25,6 +25,13 @@ class ASR:
|
||||
|
||||
self.log(f'Device: {device}')
|
||||
|
||||
compute_type = "float16"
|
||||
|
||||
if device == 'cpu':
|
||||
compute_type = "int8_float32"
|
||||
|
||||
self.compute_type = compute_type
|
||||
|
||||
self.transcription_callback = transcription_callback
|
||||
self.wake_word_callback = wake_word_callback
|
||||
self.end_of_owner_speech_callback = end_of_owner_speech_callback
|
||||
@ -51,8 +58,22 @@ class ASR:
|
||||
self.buffer_size = 64 # Size of the circular buffer
|
||||
|
||||
self.audio = pyaudio.PyAudio()
|
||||
self.model = WhisperModel(self.model_size, device=self.device, compute_type="float16")
|
||||
self.stream = None
|
||||
self.model = None
|
||||
|
||||
if self.device == 'cpu':
|
||||
self.model = WhisperModel(
|
||||
self.model_size,
|
||||
device=self.device,
|
||||
compute_type=self.compute_type,
|
||||
cpu_threads=4
|
||||
)
|
||||
else:
|
||||
self.model = WhisperModel(
|
||||
self.model_size,
|
||||
device=self.device,
|
||||
compute_type=self.compute_type
|
||||
)
|
||||
|
||||
self.log('Model loaded')
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user