mirror of
https://github.com/leon-ai/leon.git
synced 2024-10-26 18:18:46 +03:00
scripts(setup-llm): set up LLM
This commit is contained in:
parent
c4d2524922
commit
93eb2d22b3
@ -1,11 +1,21 @@
|
||||
/*import fs from 'node:fs'
|
||||
import fs from 'node:fs'
|
||||
import path from 'node:path'
|
||||
import dns from 'node:dns'
|
||||
import stream from 'node:stream'
|
||||
|
||||
import {
|
||||
LLM_FILE_NAME,
|
||||
LLM_PATH
|
||||
} from '@/constants'*/
|
||||
// import { LogHelper } from '@/helpers/log-helper'
|
||||
// import { FileHelper } from '@/helpers/file-helper'
|
||||
LLM_NAME,
|
||||
LLM_NAME_WITH_VERSION,
|
||||
LLM_MINIMUM_TOTAL_RAM,
|
||||
LLM_DIR_PATH,
|
||||
LLM_PATH,
|
||||
LLM_VERSION,
|
||||
LLM_HF_DOWNLOAD_URL,
|
||||
LLM_MIRROR_DOWNLOAD_URL
|
||||
} from '@/constants'
|
||||
import { SystemHelper } from '@/helpers/system-helper'
|
||||
import { LogHelper } from '@/helpers/log-helper'
|
||||
import { FileHelper } from '@/helpers/file-helper'
|
||||
|
||||
/**
|
||||
* Download and set up LLM
|
||||
@ -14,14 +24,72 @@ import {
|
||||
* Create manifest via Helper method...
|
||||
*/
|
||||
|
||||
export default async () => {
|
||||
/*const canSetupLLM = await checkMinimumHardwareRequirements()
|
||||
|
||||
if (canSetupLLM) {
|
||||
|
||||
|
||||
// await setupLLM()
|
||||
} else {
|
||||
LogHelper.error('Minimum hardware requirements not met')
|
||||
}*/
|
||||
function checkMinimumHardwareRequirements() {
|
||||
return SystemHelper.getTotalRAM() >= LLM_MINIMUM_TOTAL_RAM
|
||||
}
|
||||
|
||||
async function canAccessHuggingFace() {
|
||||
try {
|
||||
await dns.promises.resolve('huggingface.co')
|
||||
|
||||
return true
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
async function setupLLM() {
|
||||
try {
|
||||
LogHelper.info('Setting up LLM...')
|
||||
|
||||
const llmManifestPath = path.join(LLM_DIR_PATH, 'manifest.json')
|
||||
let manifest = null
|
||||
|
||||
if (fs.existsSync(llmManifestPath)) {
|
||||
manifest = JSON.parse(await fs.promises.readFile(llmManifestPath, 'utf8'))
|
||||
|
||||
LogHelper.info(`Found ${LLM_NAME} ${manifest.version}`)
|
||||
LogHelper.info(`Latest version is ${LLM_VERSION}`)
|
||||
}
|
||||
|
||||
if (!manifest || manifest.version !== LLM_VERSION) {
|
||||
const downloadURL = await canAccessHuggingFace() ? LLM_HF_DOWNLOAD_URL : LLM_MIRROR_DOWNLOAD_URL
|
||||
|
||||
// Just in case the LLM file already exists, delete it first
|
||||
if (fs.existsSync(LLM_PATH)) {
|
||||
await fs.promises.unlink(LLM_PATH)
|
||||
}
|
||||
|
||||
LogHelper.info(`Downloading ${LLM_NAME_WITH_VERSION} from ${downloadURL}...`)
|
||||
|
||||
const llmWriter = fs.createWriteStream(LLM_PATH)
|
||||
const response = await FileHelper.downloadFile(downloadURL, 'stream')
|
||||
|
||||
response.data.pipe(llmWriter)
|
||||
await stream.promises.finished(llmWriter)
|
||||
|
||||
LogHelper.success(`${LLM_NAME_WITH_VERSION} downloaded`)
|
||||
|
||||
await FileHelper.createManifestFile(llmManifestPath, LLM_NAME, LLM_VERSION)
|
||||
|
||||
LogHelper.success('Manifest file created')
|
||||
LogHelper.success(`${LLM_NAME_WITH_VERSION} ready`)
|
||||
}
|
||||
|
||||
LogHelper.success('LLM is set up')
|
||||
} catch (e) {
|
||||
LogHelper.error(`Failed to set up LLM: ${e}`)
|
||||
}
|
||||
}
|
||||
|
||||
export default async () => {
|
||||
const canSetupLLM = checkMinimumHardwareRequirements()
|
||||
|
||||
if (!canSetupLLM) {
|
||||
const totalRAM = SystemHelper.getTotalRAM()
|
||||
|
||||
LogHelper.warning(`LLM requires at least ${LLM_MINIMUM_TOTAL_RAM} of total RAM. Current total RAM is ${totalRAM} GB. No worries though, Leon can still run without LLM.`)
|
||||
} else {
|
||||
await setupLLM()
|
||||
}
|
||||
}
|
||||
|
@ -151,13 +151,15 @@ export const LEON_FILE_PATH = path.join(process.cwd(), 'leon.json')
|
||||
* LLMs
|
||||
*/
|
||||
export const LLM_VERSION = 'v0.2.Q4_K_S'
|
||||
export const LLM_NAME = `Mistral 7B Instruct ${LLM_VERSION}`
|
||||
export const LLM_NAME = 'Mistral 7B Instruct'
|
||||
export const LLM_NAME_WITH_VERSION = `${LLM_NAME} ${LLM_VERSION}`
|
||||
export const LLM_FILE_NAME = `mistral-7b-instruct-${LLM_VERSION}.gguf`
|
||||
export const LLM_PATH = path.join(
|
||||
MODELS_PATH,
|
||||
'llm',
|
||||
LLM_FILE_NAME
|
||||
)
|
||||
export const LLM_DIR_PATH = path.join(MODELS_PATH, 'llm')
|
||||
export const LLM_PATH = path.join(LLM_DIR_PATH, LLM_FILE_NAME)
|
||||
export const LLM_MINIMUM_TOTAL_RAM = 8
|
||||
export const LLM_MINIMUM_FREE_RAM = 8
|
||||
export const LLM_HF_DOWNLOAD_URL = 'https://huggingface.co/TheBloke/Mistral-7B-Instruct-v0.2-GGUF/resolve/main/mistral-7b-instruct-v0.2.Q4_K_S.gguf?download=true'
|
||||
export const LLM_MIRROR_DOWNLOAD_URL = 'https://hf-mirror.com/TheBloke/Mistral-7B-Instruct-v0.2-GGUF/resolve/main/mistral-7b-instruct-v0.2.Q4_K_S.gguf?download=true'
|
||||
|
||||
/**
|
||||
* Misc
|
||||
|
@ -19,25 +19,16 @@ export class FileHelper {
|
||||
return axios.get(fileURL, {
|
||||
responseType,
|
||||
onDownloadProgress: ({ loaded, total, progress, estimated, rate }) => {
|
||||
// TODO: remove
|
||||
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
|
||||
// @ts-expect-error
|
||||
const percentage = Math.floor(progress * 100)
|
||||
const percentage = Math.floor(Number(progress) * 100)
|
||||
const downloadedSize = prettyBytes(loaded)
|
||||
// TODO: remove
|
||||
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
|
||||
// @ts-expect-error
|
||||
const totalSize = prettyBytes(total)
|
||||
const totalSize = prettyBytes(Number(total))
|
||||
const estimatedTime = !estimated
|
||||
? 0
|
||||
: prettyMilliseconds(estimated * 1_000, { secondsDecimalDigits: 0 })
|
||||
const downloadRate = !rate ? 0 : prettyBytes(rate)
|
||||
|
||||
readline.clearLine(process.stdout, 0)
|
||||
// TODO: remove
|
||||
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
|
||||
// @ts-expect-error
|
||||
readline.cursorTo(process.stdout, 0, null)
|
||||
readline.cursorTo(process.stdout, 0)
|
||||
process.stdout.write(
|
||||
`Download progress: ${percentage}% (${downloadedSize}/${totalSize} | ${downloadRate}/s | ${estimatedTime} ETA)`
|
||||
)
|
||||
|
Loading…
Reference in New Issue
Block a user