2024-01-29 18:57:23 +03:00
import fs from 'node:fs'
import path from 'node:path'
import stream from 'node:stream'
2024-01-28 19:20:38 +03:00
2024-02-08 16:05:06 +03:00
import { command } from 'execa'
2024-01-28 19:20:38 +03:00
import {
2024-01-29 18:57:23 +03:00
LLM _NAME ,
LLM _NAME _WITH _VERSION ,
2024-07-03 18:41:23 +03:00
LLM _MINIMUM _TOTAL _VRAM ,
2024-01-29 18:57:23 +03:00
LLM _DIR _PATH ,
LLM _PATH ,
LLM _VERSION ,
LLM _HF _DOWNLOAD _URL ,
2024-02-08 16:05:06 +03:00
LLM _LLAMA _CPP _RELEASE _TAG
2024-01-29 18:57:23 +03:00
} from '@/constants'
2024-02-08 16:05:06 +03:00
import { OSTypes , CPUArchitectures } from '@/types'
2024-01-29 18:57:23 +03:00
import { SystemHelper } from '@/helpers/system-helper'
import { LogHelper } from '@/helpers/log-helper'
import { FileHelper } from '@/helpers/file-helper'
2024-08-05 18:00:36 +03:00
import { NetworkHelper } from '@/helpers/network-helper'
2024-01-28 19:20:38 +03:00
/ * *
* Download and set up LLM
2024-01-29 19:09:20 +03:00
* 1. Check minimum hardware requirements
* 2. Check if Hugging Face is accessible
* 3. Download the latest LLM from Hugging Face or mirror
2024-02-08 16:05:06 +03:00
* 4. Download and compile the latest llama . cpp release
* 5. Create manifest file
2024-01-28 19:20:38 +03:00
* /
2024-02-08 16:05:06 +03:00
const LLM _MANIFEST _PATH = path . join ( LLM _DIR _PATH , 'manifest.json' )
let manifest = null
2024-07-03 18:41:23 +03:00
async function checkMinimumHardwareRequirements ( ) {
2024-08-05 18:00:36 +03:00
LogHelper . info (
'Checking minimum hardware requirements can take a few minutes...'
)
2024-07-03 18:41:23 +03:00
const { getLlama , LlamaLogLevel } = await Function (
'return import("node-llama-cpp")'
) ( )
const llama = await getLlama ( {
logLevel : LlamaLogLevel . disabled
} )
if ( ! ( await SystemHelper . hasGPU ( llama ) ) ) {
return false
}
LogHelper . info (
` GPU detected: ${ ( await SystemHelper . getGPUDeviceNames ( llama ) ) [ 0 ] } `
)
LogHelper . info (
` Graphics compute API: ${ await SystemHelper . getGraphicsComputeAPI ( llama ) } `
)
LogHelper . info ( ` Total VRAM: ${ await SystemHelper . getTotalVRAM ( llama ) } GB ` )
return ( await SystemHelper . getTotalVRAM ( llama ) ) >= LLM _MINIMUM _TOTAL _VRAM
2024-01-29 18:57:23 +03:00
}
2024-05-22 10:22:30 +03:00
async function downloadLLM ( ) {
2024-01-29 18:57:23 +03:00
try {
2024-02-08 16:05:06 +03:00
LogHelper . info ( 'Downloading LLM...' )
2024-01-29 18:57:23 +03:00
2024-02-08 16:05:06 +03:00
if ( fs . existsSync ( LLM _MANIFEST _PATH ) ) {
manifest = JSON . parse (
await fs . promises . readFile ( LLM _MANIFEST _PATH , 'utf8' )
)
2024-01-29 18:57:23 +03:00
LogHelper . info ( ` Found ${ LLM _NAME } ${ manifest . version } ` )
LogHelper . info ( ` Latest version is ${ LLM _VERSION } ` )
}
if ( ! manifest || manifest . version !== LLM _VERSION ) {
// Just in case the LLM file already exists, delete it first
if ( fs . existsSync ( LLM _PATH ) ) {
await fs . promises . unlink ( LLM _PATH )
}
2024-08-05 18:00:36 +03:00
const llmDownloadURL =
await NetworkHelper . setHuggingFaceURL ( LLM _HF _DOWNLOAD _URL )
2024-02-08 16:05:06 +03:00
LogHelper . info (
2024-08-05 18:00:36 +03:00
` Downloading ${ LLM _NAME _WITH _VERSION } from ${ llmDownloadURL } ... `
2024-02-08 16:05:06 +03:00
)
2024-01-29 18:57:23 +03:00
const llmWriter = fs . createWriteStream ( LLM _PATH )
2024-08-05 18:00:36 +03:00
const response = await FileHelper . downloadFile ( llmDownloadURL , 'stream' )
2024-01-29 18:57:23 +03:00
response . data . pipe ( llmWriter )
await stream . promises . finished ( llmWriter )
2024-04-28 05:10:57 +03:00
await FileHelper . createManifestFile (
LLM _MANIFEST _PATH ,
LLM _NAME ,
LLM _VERSION ,
{
2024-06-01 16:33:07 +03:00
llamaCPPVersion : manifest ? . llamaCPPVersion
2024-04-28 05:10:57 +03:00
? manifest . llamaCPPVersion
2024-06-01 16:33:07 +03:00
: null
2024-04-28 05:10:57 +03:00
}
)
LogHelper . success ( 'Manifest file updated' )
2024-01-29 18:57:23 +03:00
LogHelper . success ( ` ${ LLM _NAME _WITH _VERSION } downloaded ` )
LogHelper . success ( ` ${ LLM _NAME _WITH _VERSION } ready ` )
2024-01-29 19:09:20 +03:00
} else {
2024-02-08 16:05:06 +03:00
LogHelper . success (
` ${ LLM _NAME _WITH _VERSION } is already set up and use the latest version `
)
}
} catch ( e ) {
LogHelper . error ( ` Failed to download LLM: ${ e } ` )
2024-06-01 16:33:07 +03:00
process . exit ( 1 )
2024-02-08 16:05:06 +03:00
}
}
async function downloadAndCompileLlamaCPP ( ) {
2024-06-01 16:57:20 +03:00
const { type : osType , cpuArchitecture } = SystemHelper . getInformation ( )
2024-02-08 16:05:06 +03:00
try {
LogHelper . info (
` Downloading and compiling " ${ LLM _LLAMA _CPP _RELEASE _TAG } " llama.cpp release... `
)
2024-06-01 16:33:07 +03:00
if ( manifest ? . llamaCPPVersion ) {
2024-02-08 16:05:06 +03:00
LogHelper . info ( ` Found llama.cpp ${ manifest . llamaCPPVersion } ` )
LogHelper . info ( ` Latest version is ${ LLM _LLAMA _CPP _RELEASE _TAG } ` )
2024-01-29 18:57:23 +03:00
}
2024-06-01 16:33:07 +03:00
if ( ! manifest || manifest ? . llamaCPPVersion !== LLM _LLAMA _CPP _RELEASE _TAG ) {
if ( manifest ? . llamaCPPVersion !== LLM _LLAMA _CPP _RELEASE _TAG ) {
2024-02-08 16:05:06 +03:00
LogHelper . info ( ` Updating llama.cpp to ${ LLM _LLAMA _CPP _RELEASE _TAG } ... ` )
}
let llamaCPPDownloadCommand = ` npx --no node-llama-cpp download --release " ${ LLM _LLAMA _CPP _RELEASE _TAG } " `
if (
osType === OSTypes . MacOS &&
cpuArchitecture === CPUArchitectures . X64
) {
2024-04-16 15:18:18 +03:00
// llamaCPPDownloadCommand = `${llamaCPPDownloadCommand} --no-metal`
2024-02-08 16:05:06 +03:00
LogHelper . info ( ` macOS Intel chipset detected, Metal support disabled ` )
}
await command ( llamaCPPDownloadCommand , {
shell : true ,
stdio : 'inherit'
} )
await FileHelper . createManifestFile (
LLM _MANIFEST _PATH ,
LLM _NAME ,
LLM _VERSION ,
{
llamaCPPVersion : LLM _LLAMA _CPP _RELEASE _TAG
}
)
2024-04-28 05:10:57 +03:00
LogHelper . success ( 'Manifest file updated' )
2024-02-08 16:05:06 +03:00
LogHelper . success ( ` llama.cpp downloaded and compiled ` )
LogHelper . success ( 'The LLM is ready to go' )
} else {
LogHelper . success (
` llama.cpp is already set up and use the latest version ( ${ LLM _LLAMA _CPP _RELEASE _TAG } ) `
)
}
2024-01-29 18:57:23 +03:00
} catch ( e ) {
2024-02-08 16:05:06 +03:00
LogHelper . error ( ` Failed to set up llama.cpp: ${ e } ` )
2024-06-01 16:57:20 +03:00
if (
osType === OSTypes . MacOS &&
cpuArchitecture === CPUArchitectures . ARM64
) {
LogHelper . error (
` Please verify that the Metal developer tools for macOS are installed. \n
You can verify by running the following command in your terminal : xcode - select -- install \ n
Otherwise download them here and retry : https : //developer.apple.com/xcode/`
)
}
2024-06-01 16:33:07 +03:00
process . exit ( 1 )
2024-01-29 18:57:23 +03:00
}
}
export default async ( ) => {
2024-07-03 18:41:23 +03:00
const canSetupLLM = await checkMinimumHardwareRequirements ( )
2024-01-28 19:20:38 +03:00
2024-01-29 18:57:23 +03:00
if ( ! canSetupLLM ) {
2024-07-03 18:41:23 +03:00
const { getLlama , LlamaLogLevel } = await Function (
'return import("node-llama-cpp")'
) ( )
const llama = await getLlama ( {
logLevel : LlamaLogLevel . disabled
} )
const totalVRAM = await SystemHelper . getTotalVRAM ( llama )
2024-01-28 19:20:38 +03:00
2024-02-08 16:05:06 +03:00
LogHelper . warning (
2024-07-03 18:41:23 +03:00
` LLM requires at least ${ LLM _MINIMUM _TOTAL _VRAM } GB of total VRAM. Current total VRAM is ${ totalVRAM } GB. No worries though, Leon can still run without LLM. `
2024-02-08 16:05:06 +03:00
)
2024-01-28 19:20:38 +03:00
} else {
2024-02-08 16:05:06 +03:00
await downloadLLM ( )
await downloadAndCompileLlamaCPP ( )
2024-01-29 18:57:23 +03:00
}
2024-01-28 19:20:38 +03:00
}