2024-01-29 18:57:23 +03:00
|
|
|
import fs from 'node:fs'
|
|
|
|
import path from 'node:path'
|
|
|
|
import stream from 'node:stream'
|
2024-01-28 19:20:38 +03:00
|
|
|
|
2024-02-08 16:05:06 +03:00
|
|
|
import { command } from 'execa'
|
|
|
|
|
2024-01-28 19:20:38 +03:00
|
|
|
import {
|
2024-01-29 18:57:23 +03:00
|
|
|
LLM_NAME,
|
|
|
|
LLM_NAME_WITH_VERSION,
|
|
|
|
LLM_MINIMUM_TOTAL_RAM,
|
|
|
|
LLM_DIR_PATH,
|
|
|
|
LLM_PATH,
|
|
|
|
LLM_VERSION,
|
|
|
|
LLM_HF_DOWNLOAD_URL,
|
2024-02-08 16:05:06 +03:00
|
|
|
LLM_LLAMA_CPP_RELEASE_TAG
|
2024-01-29 18:57:23 +03:00
|
|
|
} from '@/constants'
|
2024-02-08 16:05:06 +03:00
|
|
|
import { OSTypes, CPUArchitectures } from '@/types'
|
2024-01-29 18:57:23 +03:00
|
|
|
import { SystemHelper } from '@/helpers/system-helper'
|
|
|
|
import { LogHelper } from '@/helpers/log-helper'
|
|
|
|
import { FileHelper } from '@/helpers/file-helper'
|
2024-01-28 19:20:38 +03:00
|
|
|
|
|
|
|
/**
|
|
|
|
* Download and set up LLM
|
2024-01-29 19:09:20 +03:00
|
|
|
* 1. Check minimum hardware requirements
|
|
|
|
* 2. Check if Hugging Face is accessible
|
|
|
|
* 3. Download the latest LLM from Hugging Face or mirror
|
2024-02-08 16:05:06 +03:00
|
|
|
* 4. Download and compile the latest llama.cpp release
|
|
|
|
* 5. Create manifest file
|
2024-01-28 19:20:38 +03:00
|
|
|
*/
|
|
|
|
|
2024-02-08 16:05:06 +03:00
|
|
|
const LLM_MANIFEST_PATH = path.join(LLM_DIR_PATH, 'manifest.json')
|
|
|
|
let manifest = null
|
|
|
|
|
2024-01-29 18:57:23 +03:00
|
|
|
function checkMinimumHardwareRequirements() {
|
|
|
|
return SystemHelper.getTotalRAM() >= LLM_MINIMUM_TOTAL_RAM
|
|
|
|
}
|
|
|
|
|
2024-05-22 10:22:30 +03:00
|
|
|
async function downloadLLM() {
|
2024-01-29 18:57:23 +03:00
|
|
|
try {
|
2024-02-08 16:05:06 +03:00
|
|
|
LogHelper.info('Downloading LLM...')
|
2024-01-29 18:57:23 +03:00
|
|
|
|
2024-02-08 16:05:06 +03:00
|
|
|
if (fs.existsSync(LLM_MANIFEST_PATH)) {
|
|
|
|
manifest = JSON.parse(
|
|
|
|
await fs.promises.readFile(LLM_MANIFEST_PATH, 'utf8')
|
|
|
|
)
|
2024-01-29 18:57:23 +03:00
|
|
|
|
|
|
|
LogHelper.info(`Found ${LLM_NAME} ${manifest.version}`)
|
|
|
|
LogHelper.info(`Latest version is ${LLM_VERSION}`)
|
|
|
|
}
|
|
|
|
|
|
|
|
if (!manifest || manifest.version !== LLM_VERSION) {
|
|
|
|
// Just in case the LLM file already exists, delete it first
|
|
|
|
if (fs.existsSync(LLM_PATH)) {
|
|
|
|
await fs.promises.unlink(LLM_PATH)
|
|
|
|
}
|
|
|
|
|
2024-02-08 16:05:06 +03:00
|
|
|
LogHelper.info(
|
2024-05-22 10:22:30 +03:00
|
|
|
`Downloading ${LLM_NAME_WITH_VERSION} from ${LLM_HF_DOWNLOAD_URL}...`
|
2024-02-08 16:05:06 +03:00
|
|
|
)
|
2024-01-29 18:57:23 +03:00
|
|
|
|
|
|
|
const llmWriter = fs.createWriteStream(LLM_PATH)
|
2024-05-22 10:22:30 +03:00
|
|
|
const response = await FileHelper.downloadFile(
|
|
|
|
LLM_HF_DOWNLOAD_URL,
|
|
|
|
'stream'
|
|
|
|
)
|
2024-01-29 18:57:23 +03:00
|
|
|
|
|
|
|
response.data.pipe(llmWriter)
|
|
|
|
await stream.promises.finished(llmWriter)
|
|
|
|
|
2024-04-28 05:10:57 +03:00
|
|
|
await FileHelper.createManifestFile(
|
|
|
|
LLM_MANIFEST_PATH,
|
|
|
|
LLM_NAME,
|
|
|
|
LLM_VERSION,
|
|
|
|
{
|
2024-06-01 16:33:07 +03:00
|
|
|
llamaCPPVersion: manifest?.llamaCPPVersion
|
2024-04-28 05:10:57 +03:00
|
|
|
? manifest.llamaCPPVersion
|
2024-06-01 16:33:07 +03:00
|
|
|
: null
|
2024-04-28 05:10:57 +03:00
|
|
|
}
|
|
|
|
)
|
|
|
|
LogHelper.success('Manifest file updated')
|
|
|
|
|
2024-01-29 18:57:23 +03:00
|
|
|
LogHelper.success(`${LLM_NAME_WITH_VERSION} downloaded`)
|
|
|
|
LogHelper.success(`${LLM_NAME_WITH_VERSION} ready`)
|
2024-01-29 19:09:20 +03:00
|
|
|
} else {
|
2024-02-08 16:05:06 +03:00
|
|
|
LogHelper.success(
|
|
|
|
`${LLM_NAME_WITH_VERSION} is already set up and use the latest version`
|
|
|
|
)
|
|
|
|
}
|
|
|
|
} catch (e) {
|
|
|
|
LogHelper.error(`Failed to download LLM: ${e}`)
|
2024-06-01 16:33:07 +03:00
|
|
|
process.exit(1)
|
2024-02-08 16:05:06 +03:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
async function downloadAndCompileLlamaCPP() {
|
2024-06-01 16:57:20 +03:00
|
|
|
const { type: osType, cpuArchitecture } = SystemHelper.getInformation()
|
|
|
|
|
2024-02-08 16:05:06 +03:00
|
|
|
try {
|
|
|
|
LogHelper.info(
|
|
|
|
`Downloading and compiling "${LLM_LLAMA_CPP_RELEASE_TAG}" llama.cpp release...`
|
|
|
|
)
|
|
|
|
|
2024-06-01 16:33:07 +03:00
|
|
|
if (manifest?.llamaCPPVersion) {
|
2024-02-08 16:05:06 +03:00
|
|
|
LogHelper.info(`Found llama.cpp ${manifest.llamaCPPVersion}`)
|
|
|
|
LogHelper.info(`Latest version is ${LLM_LLAMA_CPP_RELEASE_TAG}`)
|
2024-01-29 18:57:23 +03:00
|
|
|
}
|
|
|
|
|
2024-06-01 16:33:07 +03:00
|
|
|
if (!manifest || manifest?.llamaCPPVersion !== LLM_LLAMA_CPP_RELEASE_TAG) {
|
|
|
|
if (manifest?.llamaCPPVersion !== LLM_LLAMA_CPP_RELEASE_TAG) {
|
2024-02-08 16:05:06 +03:00
|
|
|
LogHelper.info(`Updating llama.cpp to ${LLM_LLAMA_CPP_RELEASE_TAG}...`)
|
|
|
|
}
|
|
|
|
|
|
|
|
let llamaCPPDownloadCommand = `npx --no node-llama-cpp download --release "${LLM_LLAMA_CPP_RELEASE_TAG}"`
|
|
|
|
|
|
|
|
if (
|
|
|
|
osType === OSTypes.MacOS &&
|
|
|
|
cpuArchitecture === CPUArchitectures.X64
|
|
|
|
) {
|
2024-04-16 15:18:18 +03:00
|
|
|
// llamaCPPDownloadCommand = `${llamaCPPDownloadCommand} --no-metal`
|
2024-02-08 16:05:06 +03:00
|
|
|
|
|
|
|
LogHelper.info(`macOS Intel chipset detected, Metal support disabled`)
|
|
|
|
}
|
|
|
|
|
|
|
|
await command(llamaCPPDownloadCommand, {
|
|
|
|
shell: true,
|
|
|
|
stdio: 'inherit'
|
|
|
|
})
|
|
|
|
|
|
|
|
await FileHelper.createManifestFile(
|
|
|
|
LLM_MANIFEST_PATH,
|
|
|
|
LLM_NAME,
|
|
|
|
LLM_VERSION,
|
|
|
|
{
|
|
|
|
llamaCPPVersion: LLM_LLAMA_CPP_RELEASE_TAG
|
|
|
|
}
|
|
|
|
)
|
2024-04-28 05:10:57 +03:00
|
|
|
LogHelper.success('Manifest file updated')
|
2024-02-08 16:05:06 +03:00
|
|
|
|
|
|
|
LogHelper.success(`llama.cpp downloaded and compiled`)
|
|
|
|
LogHelper.success('The LLM is ready to go')
|
|
|
|
} else {
|
|
|
|
LogHelper.success(
|
|
|
|
`llama.cpp is already set up and use the latest version (${LLM_LLAMA_CPP_RELEASE_TAG})`
|
|
|
|
)
|
|
|
|
}
|
2024-01-29 18:57:23 +03:00
|
|
|
} catch (e) {
|
2024-02-08 16:05:06 +03:00
|
|
|
LogHelper.error(`Failed to set up llama.cpp: ${e}`)
|
2024-06-01 16:57:20 +03:00
|
|
|
|
|
|
|
if (
|
|
|
|
osType === OSTypes.MacOS &&
|
|
|
|
cpuArchitecture === CPUArchitectures.ARM64
|
|
|
|
) {
|
|
|
|
LogHelper.error(
|
|
|
|
`Please verify that the Metal developer tools for macOS are installed.\n
|
|
|
|
You can verify by running the following command in your terminal: xcode-select --install\n
|
|
|
|
Otherwise download them here and retry: https://developer.apple.com/xcode/`
|
|
|
|
)
|
|
|
|
}
|
|
|
|
|
2024-06-01 16:33:07 +03:00
|
|
|
process.exit(1)
|
2024-01-29 18:57:23 +03:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
export default async () => {
|
|
|
|
const canSetupLLM = checkMinimumHardwareRequirements()
|
2024-01-28 19:20:38 +03:00
|
|
|
|
2024-01-29 18:57:23 +03:00
|
|
|
if (!canSetupLLM) {
|
|
|
|
const totalRAM = SystemHelper.getTotalRAM()
|
2024-01-28 19:20:38 +03:00
|
|
|
|
2024-02-08 16:05:06 +03:00
|
|
|
LogHelper.warning(
|
|
|
|
`LLM requires at least ${LLM_MINIMUM_TOTAL_RAM} of total RAM. Current total RAM is ${totalRAM} GB. No worries though, Leon can still run without LLM.`
|
|
|
|
)
|
2024-01-28 19:20:38 +03:00
|
|
|
} else {
|
2024-02-08 16:05:06 +03:00
|
|
|
await downloadLLM()
|
|
|
|
await downloadAndCompileLlamaCPP()
|
2024-01-29 18:57:23 +03:00
|
|
|
}
|
2024-01-28 19:20:38 +03:00
|
|
|
}
|