leon-ai/leon
1
1
Fork 0
mirror of https://github.com/leon-ai/leon.git synced 2024-08-17 06:00:33 +03:00
Code Issues Projects Releases Wiki Activity

feat(scripts): download and compile llama.cpp

Browse Source
This commit is contained in:
louistiti 2024-02-08 21:05:06 +08:00
parent 46f6c09739
commit 1c0c8080bf
No known key found for this signature in database
GPG Key ID: 334C1BA3550BBDE5
5 changed files with 112 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
package.json
View File

@ -95,6 +95,7 @@
"getos": "3.2.1",
"googleapis": "67.1.1",
"ibm-watson": "6.1.1",
"node-llama-cpp": "2.8.6",
"node-wav": "0.0.2",
"os-name": "4.0.1",
"pretty-bytes": "5.6.0",

5
scripts/setup/setup-binaries.js
View File

@ -107,7 +107,10 @@ const setupBinaries = async (key) => {
const archiveWriter = fs.createWriteStream(archivePath)
const latestReleaseAssetURL = `${GITHUB_URL}/releases/download/${key}_v${version}/${archiveName}`
const { data } = await FileHelper.downloadFile(latestReleaseAssetURL, 'stream')
const { data } = await FileHelper.downloadFile(
latestReleaseAssetURL,
'stream'
)
data.pipe(archiveWriter)
await stream.promises.finished(archiveWriter)

104
scripts/setup/setup-llm.js
View File

@ -3,6 +3,8 @@ import path from 'node:path'
import dns from 'node:dns'
import stream from 'node:stream'
import { command } from 'execa'
import {
LLM_NAME,
LLM_NAME_WITH_VERSION,
@ -11,8 +13,10 @@ import {
LLM_PATH,
LLM_VERSION,
LLM_HF_DOWNLOAD_URL,
LLM_MIRROR_DOWNLOAD_URL
LLM_MIRROR_DOWNLOAD_URL,
LLM_LLAMA_CPP_RELEASE_TAG
} from '@/constants'
import { OSTypes, CPUArchitectures } from '@/types'
import { SystemHelper } from '@/helpers/system-helper'
import { LogHelper } from '@/helpers/log-helper'
import { FileHelper } from '@/helpers/file-helper'
@ -22,9 +26,13 @@ import { FileHelper } from '@/helpers/file-helper'
* 1. Check minimum hardware requirements
* 2. Check if Hugging Face is accessible
* 3. Download the latest LLM from Hugging Face or mirror
* 4. Create manifest file
* 4. Download and compile the latest llama.cpp release
* 5. Create manifest file
*/
const LLM_MANIFEST_PATH = path.join(LLM_DIR_PATH, 'manifest.json')
let manifest = null
function checkMinimumHardwareRequirements() {
return SystemHelper.getTotalRAM() >= LLM_MINIMUM_TOTAL_RAM
}
@ -39,29 +47,32 @@ async function canAccessHuggingFace() {
}
}
async function setupLLM() {
async function downloadLLM() {
try {
LogHelper.info('Setting up LLM...')
LogHelper.info('Downloading LLM...')
const llmManifestPath = path.join(LLM_DIR_PATH, 'manifest.json')
let manifest = null
if (fs.existsSync(llmManifestPath)) {
manifest = JSON.parse(await fs.promises.readFile(llmManifestPath, 'utf8'))
if (fs.existsSync(LLM_MANIFEST_PATH)) {
manifest = JSON.parse(
await fs.promises.readFile(LLM_MANIFEST_PATH, 'utf8')
)
LogHelper.info(`Found ${LLM_NAME} ${manifest.version}`)
LogHelper.info(`Latest version is ${LLM_VERSION}`)
}
if (!manifest || manifest.version !== LLM_VERSION) {
const downloadURL = await canAccessHuggingFace() ? LLM_HF_DOWNLOAD_URL : LLM_MIRROR_DOWNLOAD_URL
const downloadURL = (await canAccessHuggingFace())
? LLM_HF_DOWNLOAD_URL
: LLM_MIRROR_DOWNLOAD_URL
// Just in case the LLM file already exists, delete it first
if (fs.existsSync(LLM_PATH)) {
await fs.promises.unlink(LLM_PATH)
}
LogHelper.info(`Downloading ${LLM_NAME_WITH_VERSION} from ${downloadURL}...`)
LogHelper.info(
`Downloading ${LLM_NAME_WITH_VERSION} from ${downloadURL}...`
)
const llmWriter = fs.createWriteStream(LLM_PATH)
const response = await FileHelper.downloadFile(downloadURL, 'stream')
@ -71,17 +82,69 @@ async function setupLLM() {
LogHelper.success(`${LLM_NAME_WITH_VERSION} downloaded`)
await FileHelper.createManifestFile(llmManifestPath, LLM_NAME, LLM_VERSION)
LogHelper.success('Manifest file created')
LogHelper.success(`${LLM_NAME_WITH_VERSION} ready`)
} else {
LogHelper.info(`${LLM_NAME_WITH_VERSION} is already set up and use the latest version`)
LogHelper.success(
`${LLM_NAME_WITH_VERSION} is already set up and use the latest version`
)
}
} catch (e) {
LogHelper.error(`Failed to download LLM: ${e}`)
}
}
async function downloadAndCompileLlamaCPP() {
try {
LogHelper.info(
`Downloading and compiling "${LLM_LLAMA_CPP_RELEASE_TAG}" llama.cpp release...`
)
if (manifest.llamaCPPVersion) {
LogHelper.info(`Found llama.cpp ${manifest.llamaCPPVersion}`)
LogHelper.info(`Latest version is ${LLM_LLAMA_CPP_RELEASE_TAG}`)
}
LogHelper.success('LLM is set up')
if (!manifest || manifest.llamaCPPVersion !== LLM_LLAMA_CPP_RELEASE_TAG) {
if (manifest.llamaCPPVersion !== LLM_LLAMA_CPP_RELEASE_TAG) {
LogHelper.info(`Updating llama.cpp to ${LLM_LLAMA_CPP_RELEASE_TAG}...`)
}
const { type: osType, cpuArchitecture } = SystemHelper.getInformation()
let llamaCPPDownloadCommand = `npx --no node-llama-cpp download --release "${LLM_LLAMA_CPP_RELEASE_TAG}"`
if (
osType === OSTypes.MacOS &&
cpuArchitecture === CPUArchitectures.X64
) {
llamaCPPDownloadCommand = `${llamaCPPDownloadCommand} --no-metal`
LogHelper.info(`macOS Intel chipset detected, Metal support disabled`)
}
await command(llamaCPPDownloadCommand, {
shell: true,
stdio: 'inherit'
})
await FileHelper.createManifestFile(
LLM_MANIFEST_PATH,
LLM_NAME,
LLM_VERSION,
{
llamaCPPVersion: LLM_LLAMA_CPP_RELEASE_TAG
}
)
LogHelper.success('Manifest file created')
LogHelper.success(`llama.cpp downloaded and compiled`)
LogHelper.success('The LLM is ready to go')
} else {
LogHelper.success(
`llama.cpp is already set up and use the latest version (${LLM_LLAMA_CPP_RELEASE_TAG})`
)
}
} catch (e) {
LogHelper.error(`Failed to set up LLM: ${e}`)
LogHelper.error(`Failed to set up llama.cpp: ${e}`)
}
}
@ -91,8 +154,11 @@ export default async () => {
if (!canSetupLLM) {
const totalRAM = SystemHelper.getTotalRAM()
LogHelper.warning(`LLM requires at least ${LLM_MINIMUM_TOTAL_RAM} of total RAM. Current total RAM is ${totalRAM} GB. No worries though, Leon can still run without LLM.`)
LogHelper.warning(
`LLM requires at least ${LLM_MINIMUM_TOTAL_RAM} of total RAM. Current total RAM is ${totalRAM} GB. No worries though, Leon can still run without LLM.`
)
} else {
await setupLLM()
await downloadLLM()
await downloadAndCompileLlamaCPP()
}
}

12
server/src/constants.ts
View File

@ -85,7 +85,7 @@ export const LEON_VERSION = process.env['npm_package_version']
/**
* spaCy models
* Find new spaCy models: https://github.com/explosion/spacy-models/releases
* @see Find new spaCy models: https://github.com/explosion/spacy-models/releases
*/
export const EN_SPACY_MODEL_NAME = 'en_core_web_trf'
export const EN_SPACY_MODEL_VERSION = '3.4.0'
@ -158,8 +158,14 @@ export const LLM_DIR_PATH = path.join(MODELS_PATH, 'llm')
export const LLM_PATH = path.join(LLM_DIR_PATH, LLM_FILE_NAME)
export const LLM_MINIMUM_TOTAL_RAM = 8
export const LLM_MINIMUM_FREE_RAM = 8
export const LLM_HF_DOWNLOAD_URL = 'https://huggingface.co/TheBloke/Mistral-7B-Instruct-v0.2-GGUF/resolve/main/mistral-7b-instruct-v0.2.Q4_K_S.gguf?download=true'
export const LLM_MIRROR_DOWNLOAD_URL = 'https://hf-mirror.com/TheBloke/Mistral-7B-Instruct-v0.2-GGUF/resolve/main/mistral-7b-instruct-v0.2.Q4_K_S.gguf?download=true'
export const LLM_HF_DOWNLOAD_URL =
'https://huggingface.co/TheBloke/Mistral-7B-Instruct-v0.2-GGUF/resolve/main/mistral-7b-instruct-v0.2.Q4_K_S.gguf?download=true'
export const LLM_MIRROR_DOWNLOAD_URL =
'https://hf-mirror.com/TheBloke/Mistral-7B-Instruct-v0.2-GGUF/resolve/main/mistral-7b-instruct-v0.2.Q4_K_S.gguf?download=true'
/**
* @see llama.cpp releases: https://github.com/ggerganov/llama.cpp/releases
*/
export const LLM_LLAMA_CPP_RELEASE_TAG = 'b2096'
/**
* Misc

16
server/src/helpers/file-helper.ts
View File

@ -15,7 +15,10 @@ export class FileHelper {
* @param responseType The Axios request response type
* @example downloadFile('https://example.com/file.zip', 'arraybuffer') // ArrayBuffer
*/
public static downloadFile(fileURL: string, responseType: AxiosResponseType): Promise<AxiosResponse> {
public static downloadFile(
fileURL: string,
responseType: AxiosResponseType
): Promise<AxiosResponse> {
return axios.get(fileURL, {
responseType,
onDownloadProgress: ({ loaded, total, progress, estimated, rate }) => {
@ -45,12 +48,19 @@ export class FileHelper {
* @param manifestPath The manifest file path
* @param manifestName The manifest name
* @param manifestVersion The manifest version
* @param extraData Extra data to add to the manifest
*/
public static async createManifestFile(manifestPath: string, manifestName: string, manifestVersion: string): Promise<void> {
public static async createManifestFile(
manifestPath: string,
manifestName: string,
manifestVersion: string,
extraData?: Record<string, unknown>
): Promise<void> {
const manifest = {
name: manifestName,
version: manifestVersion,
setupDate: Date.now()
setupDate: Date.now(),
...extraData
}
await fs.promises.writeFile(manifestPath, JSON.stringify(manifest, null, 2))