scripts(setup-llm): set up LLM

scripts/setup/setup-llm.js

@@ -1,11 +1,21 @@
-/*import fs from 'node:fs'
+import fs from 'node:fs'
+import path from 'node:path'
+import dns from 'node:dns'
+import stream from 'node:stream'
 
 import {
-  LLM_FILE_NAME,
-  LLM_PATH
-} from '@/constants'*/
-// import { LogHelper } from '@/helpers/log-helper'
-// import { FileHelper } from '@/helpers/file-helper'
+  LLM_NAME,
+  LLM_NAME_WITH_VERSION,
+  LLM_MINIMUM_TOTAL_RAM,
+  LLM_DIR_PATH,
+  LLM_PATH,
+  LLM_VERSION,
+  LLM_HF_DOWNLOAD_URL,
+  LLM_MIRROR_DOWNLOAD_URL
+} from '@/constants'
+import { SystemHelper } from '@/helpers/system-helper'
+import { LogHelper } from '@/helpers/log-helper'
+import { FileHelper } from '@/helpers/file-helper'
 
 /**
  * Download and set up LLM
@@ -14,14 +24,72 @@ import {
  * Create manifest via Helper method...
  */
 
-export default async () => {
-  /*const canSetupLLM = await checkMinimumHardwareRequirements()
-
-  if (canSetupLLM) {
-
-
-    // await setupLLM()
-  } else {
-    LogHelper.error('Minimum hardware requirements not met')
-  }*/
+function checkMinimumHardwareRequirements() {
+  return SystemHelper.getTotalRAM() >= LLM_MINIMUM_TOTAL_RAM
+}
+
+async function canAccessHuggingFace() {
+  try {
+    await dns.promises.resolve('huggingface.co')
+
+    return true
+  } catch {
+    return false
+  }
+}
+
+async function setupLLM() {
+  try {
+    LogHelper.info('Setting up LLM...')
+
+    const llmManifestPath = path.join(LLM_DIR_PATH, 'manifest.json')
+    let manifest = null
+
+    if (fs.existsSync(llmManifestPath)) {
+      manifest = JSON.parse(await fs.promises.readFile(llmManifestPath, 'utf8'))
+
+      LogHelper.info(`Found ${LLM_NAME} ${manifest.version}`)
+      LogHelper.info(`Latest version is ${LLM_VERSION}`)
+    }
+
+    if (!manifest || manifest.version !== LLM_VERSION) {
+      const downloadURL = await canAccessHuggingFace() ? LLM_HF_DOWNLOAD_URL : LLM_MIRROR_DOWNLOAD_URL
+
+      // Just in case the LLM file already exists, delete it first
+      if (fs.existsSync(LLM_PATH)) {
+        await fs.promises.unlink(LLM_PATH)
+      }
+
+      LogHelper.info(`Downloading ${LLM_NAME_WITH_VERSION} from ${downloadURL}...`)
+
+      const llmWriter = fs.createWriteStream(LLM_PATH)
+      const response = await FileHelper.downloadFile(downloadURL, 'stream')
+
+      response.data.pipe(llmWriter)
+      await stream.promises.finished(llmWriter)
+
+      LogHelper.success(`${LLM_NAME_WITH_VERSION} downloaded`)
+
+      await FileHelper.createManifestFile(llmManifestPath, LLM_NAME, LLM_VERSION)
+
+      LogHelper.success('Manifest file created')
+      LogHelper.success(`${LLM_NAME_WITH_VERSION} ready`)
+    }
+
+    LogHelper.success('LLM is set up')
+  } catch (e) {
+    LogHelper.error(`Failed to set up LLM: ${e}`)
+  }
+}
+
+export default async () => {
+  const canSetupLLM = checkMinimumHardwareRequirements()
+
+  if (!canSetupLLM) {
+    const totalRAM = SystemHelper.getTotalRAM()
+
+    LogHelper.warning(`LLM requires at least ${LLM_MINIMUM_TOTAL_RAM} of total RAM. Current total RAM is ${totalRAM} GB. No worries though, Leon can still run without LLM.`)
+  } else {
+    await setupLLM()
+  }
 }

server/src/constants.ts

@@ -151,13 +151,15 @@ export const LEON_FILE_PATH = path.join(process.cwd(), 'leon.json')
  * LLMs
  */
 export const LLM_VERSION = 'v0.2.Q4_K_S'
-export const LLM_NAME = `Mistral 7B Instruct ${LLM_VERSION}`
+export const LLM_NAME = 'Mistral 7B Instruct'
+export const LLM_NAME_WITH_VERSION = `${LLM_NAME} ${LLM_VERSION}`
 export const LLM_FILE_NAME = `mistral-7b-instruct-${LLM_VERSION}.gguf`
-export const LLM_PATH = path.join(
-  MODELS_PATH,
-  'llm',
-  LLM_FILE_NAME
-)
+export const LLM_DIR_PATH = path.join(MODELS_PATH, 'llm')
+export const LLM_PATH = path.join(LLM_DIR_PATH, LLM_FILE_NAME)
+export const LLM_MINIMUM_TOTAL_RAM = 8
+export const LLM_MINIMUM_FREE_RAM = 8
+export const LLM_HF_DOWNLOAD_URL = 'https://huggingface.co/TheBloke/Mistral-7B-Instruct-v0.2-GGUF/resolve/main/mistral-7b-instruct-v0.2.Q4_K_S.gguf?download=true'
+export const LLM_MIRROR_DOWNLOAD_URL = 'https://hf-mirror.com/TheBloke/Mistral-7B-Instruct-v0.2-GGUF/resolve/main/mistral-7b-instruct-v0.2.Q4_K_S.gguf?download=true'
 
 /**
  * Misc

server/src/helpers/file-helper.ts

@@ -19,25 +19,16 @@ export class FileHelper {
     return axios.get(fileURL, {
       responseType,
       onDownloadProgress: ({ loaded, total, progress, estimated, rate }) => {
-        // TODO: remove
-        // eslint-disable-next-line @typescript-eslint/ban-ts-comment
-        // @ts-expect-error
-        const percentage = Math.floor(progress * 100)
+        const percentage = Math.floor(Number(progress) * 100)
         const downloadedSize = prettyBytes(loaded)
-        // TODO: remove
-        // eslint-disable-next-line @typescript-eslint/ban-ts-comment
-        // @ts-expect-error
-        const totalSize = prettyBytes(total)
+        const totalSize = prettyBytes(Number(total))
         const estimatedTime = !estimated
           ? 0
           : prettyMilliseconds(estimated * 1_000, { secondsDecimalDigits: 0 })
         const downloadRate = !rate ? 0 : prettyBytes(rate)
 
         readline.clearLine(process.stdout, 0)
-        // TODO: remove
-        // eslint-disable-next-line @typescript-eslint/ban-ts-comment
-        // @ts-expect-error
-        readline.cursorTo(process.stdout, 0, null)
+        readline.cursorTo(process.stdout, 0)
         process.stdout.write(
           `Download progress: ${percentage}% (${downloadedSize}/${totalSize} | ${downloadRate}/s | ${estimatedTime} ETA)`
         )