leon-ai/leon
1
1
Fork 0
mirror of https://github.com/leon-ai/leon.git synced 2024-08-17 06:00:33 +03:00
Code Issues Projects Releases Wiki Activity

feat(server): enable FlashAttention for faster inference

Browse Source
This commit is contained in:
louistiti 2024-07-14 21:19:42 +08:00
parent 626c77d340
commit fd5e952695
1 changed files with 2 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
server/src/core/llm-manager/llm-manager.ts
View File

@ -230,7 +230,8 @@ export default class LLMManager {
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-expect-error
this._model = await this._llama.loadModel({
modelPath: LLM_PATH
modelPath: LLM_PATH,
defaultContextFlashAttention: true
})
if (HAS_LLM_NLG) {