2022-09-26 18:28:25 +03:00
import fs from 'node:fs'
import path from 'node:path'
2019-02-10 15:26:50 +03:00
2022-09-26 19:01:03 +03:00
import { command } from 'execa'
2022-10-07 10:31:24 +03:00
import {
EN _SPACY _MODEL _NAME ,
EN _SPACY _MODEL _VERSION ,
FR _SPACY _MODEL _NAME ,
FR _SPACY _MODEL _VERSION ,
PYTHON _BRIDGE _SRC _PATH ,
2024-06-18 04:02:09 +03:00
PYTHON _TCP _SERVER _SRC _PATH
2022-10-07 10:31:24 +03:00
} from '@/constants'
2022-11-20 18:21:06 +03:00
import { CPUArchitectures , OSTypes } from '@/types'
2022-09-26 16:29:56 +03:00
import { LogHelper } from '@/helpers/log-helper'
2022-10-06 21:37:43 +03:00
import { LoaderHelper } from '@/helpers/loader-helper'
2022-11-20 18:21:06 +03:00
import { SystemHelper } from '@/helpers/system-helper'
2019-02-10 15:26:50 +03:00
/ * *
2022-10-06 21:37:43 +03:00
* Set up development environment according to the given setup target
* 1. Verify Python environment
* 2. Verify if the targeted development environment is up - to - date
* 3. If up - to - date , exit
* 4. If not up - to - date , delete the outdated development environment and install the new one
* 5. Install spaCy models if the targeted development environment is the TCP server
2019-02-10 15:26:50 +03:00
* /
2022-10-07 10:31:24 +03:00
// Define mirror to download models installation file
function getModelInstallationFileUrl ( model , mirror = undefined ) {
const { name , version } = SPACY _MODELS . get ( model )
const suffix = 'py3-none-any.whl'
let urlPrefix = 'https://github.com/explosion/spacy-models/releases/download'
if ( mirror === 'cn' ) {
LogHelper . info (
'Using Chinese mirror to download model installation file...'
)
urlPrefix =
'https://download.fastgit.org/explosion/spacy-models/releases/download'
}
return ` ${ urlPrefix } / ${ name } - ${ version } / ${ name } - ${ version } - ${ suffix } `
}
2022-10-06 21:37:43 +03:00
const SETUP _TARGETS = new Map ( )
2022-10-07 10:31:24 +03:00
const SPACY _MODELS = new Map ( )
2022-10-06 21:37:43 +03:00
SETUP _TARGETS . set ( 'python-bridge' , {
name : 'Python bridge' ,
pipfilePath : path . join ( PYTHON _BRIDGE _SRC _PATH , 'Pipfile' ) ,
dotVenvPath : path . join ( PYTHON _BRIDGE _SRC _PATH , '.venv' ) ,
dotProjectPath : path . join ( PYTHON _BRIDGE _SRC _PATH , '.venv' , '.project' )
} )
SETUP _TARGETS . set ( 'tcp-server' , {
name : 'TCP server' ,
2024-05-08 14:04:15 +03:00
pipfilePath : path . join ( PYTHON _TCP _SERVER _SRC _PATH , 'Pipfile' ) ,
dotVenvPath : path . join ( PYTHON _TCP _SERVER _SRC _PATH , '.venv' ) ,
dotProjectPath : path . join ( PYTHON _TCP _SERVER _SRC _PATH , '.venv' , '.project' )
2022-10-06 21:37:43 +03:00
} )
2022-10-07 10:31:24 +03:00
SPACY _MODELS . set ( 'en' , {
name : EN _SPACY _MODEL _NAME ,
version : EN _SPACY _MODEL _VERSION
} )
SPACY _MODELS . set ( 'fr' , {
name : FR _SPACY _MODEL _NAME ,
version : FR _SPACY _MODEL _VERSION
} )
2022-10-06 21:37:43 +03:00
; ( async ( ) => {
LoaderHelper . start ( )
const { argv } = process
const givenSetupTarget = argv [ 2 ] . toLowerCase ( )
2022-10-07 10:31:24 +03:00
// cn
const givenMirror = argv [ 3 ] ? . toLowerCase ( )
2022-10-06 21:37:43 +03:00
if ( ! SETUP _TARGETS . has ( givenSetupTarget ) ) {
LogHelper . error (
` Invalid setup target: ${ givenSetupTarget } . Valid targets are: ${ Array . from (
SETUP _TARGETS . keys ( )
) . join ( ', ' ) } `
)
process . exit ( 1 )
}
const {
name : setupTarget ,
pipfilePath ,
dotVenvPath ,
dotProjectPath
} = SETUP _TARGETS . get ( givenSetupTarget )
LogHelper . info ( 'Checking Python environment...' )
/ * *
* Verify Python environment
* /
// Check if the Pipfile exists
if ( fs . existsSync ( pipfilePath ) ) {
LogHelper . success ( ` ${ pipfilePath } found ` )
try {
// Check if Pipenv is installed
const pipenvVersionChild = await command ( 'pipenv --version' , {
shell : true
} )
let pipenvVersion = String ( pipenvVersionChild . stdout )
if ( pipenvVersion . includes ( 'version' ) ) {
pipenvVersion = pipenvVersion . split ( 'version' ) [ 1 ] . trim ( )
pipenvVersion = ` ${ pipenvVersion } version `
}
LogHelper . success ( ` Pipenv ${ pipenvVersion } found ` )
} catch ( e ) {
LogHelper . error (
` ${ e } \n Please install Pipenv: "pip install pipenv" or read the documentation https://docs.pipenv.org `
)
process . exit ( 1 )
}
}
/ * *
* Install Python packages
* /
LogHelper . info ( ` Setting up ${ setupTarget } development environment... ` )
const pipfileMtime = fs . statSync ( pipfilePath ) . mtime
const hasDotVenv = fs . existsSync ( dotVenvPath )
2022-11-20 18:21:06 +03:00
const { type : osType , cpuArchitecture } = SystemHelper . getInformation ( )
2024-05-17 07:20:34 +03:00
/ * *
2024-05-17 20:11:12 +03:00
* Install PyTorch with CUDA support
* as it is required by the latest NVIDIA drivers for CUDA runtime APIs .
* PyTorch will automatically download nvidia - * packages and bundle them .
2024-05-17 07:20:34 +03:00
*
2024-05-17 20:11:12 +03:00
* It is important to specify the "--ignore-installed" flag to make sure the
2024-06-23 18:29:09 +03:00
* "~/.pyenv/versions/3.11.9/lib/python3.11/site-packages" is not used in case
2024-05-17 20:11:12 +03:00
* NVIDIA deps are already installed . Otherwise , it won ' t install it in our
* TCP server . venv as it is already installed ( satisfied ) in
* the path mentioned above
*
* @ see https : //github.com/pytorch/pytorch/blob/main/RELEASE.md#release-compatibility-matrix
* @ see https : //pytorch.org/get-started/locally/
2024-05-17 07:20:34 +03:00
* @ see https : //stackoverflow.com/a/76972265/1768162
2024-05-17 20:11:12 +03:00
* @ see https : //docs.nvidia.com/deeplearning/cudnn/latest/reference/support-matrix.html
2024-05-17 07:20:34 +03:00
* /
const installPytorch = async ( ) => {
2024-06-24 03:28:32 +03:00
const logInfo =
osType === OSTypes . MacOS
? 'Installing PyTorch...'
: 'Installing PyTorch with CUDA support...'
LogHelper . info ( logInfo )
2024-05-17 07:20:34 +03:00
try {
2024-05-17 20:11:12 +03:00
// There is no CUDA support on macOS
const commandToExecute =
osType === OSTypes . MacOS
2024-06-20 04:54:16 +03:00
? 'pipenv run pip install --ignore-installed --force-reinstall torch==2.3.1'
: 'pipenv run pip install --ignore-installed --force-reinstall torch==2.3.1 --index-url https://download.pytorch.org/whl/cu121'
2024-05-17 20:11:12 +03:00
await command ( commandToExecute , {
shell : true ,
stdio : 'inherit'
} )
LogHelper . success ( 'PyTorch with CUDA support installed' )
2024-05-17 07:20:34 +03:00
} catch ( e ) {
2024-05-17 20:11:12 +03:00
LogHelper . error ( ` Failed to install PyTorch with CUDA support: ${ e } ` )
2024-05-17 07:20:34 +03:00
process . exit ( 1 )
}
}
2022-10-06 21:37:43 +03:00
const installPythonPackages = async ( ) => {
2024-06-11 19:31:47 +03:00
LogHelper . info ( ` Installing Python packages from ${ pipfilePath } ... ` )
2022-10-06 21:37:43 +03:00
// Delete .venv directory to reset the development environment
if ( hasDotVenv ) {
LogHelper . info ( ` Deleting ${ dotVenvPath } ... ` )
2023-04-02 21:34:50 +03:00
await fs . promises . rm ( dotVenvPath , { recursive : true , force : true } )
2022-10-06 21:37:43 +03:00
LogHelper . success ( ` ${ dotVenvPath } deleted ` )
}
2022-10-12 19:10:45 +03:00
try {
2024-06-11 19:31:47 +03:00
await command ( 'pipenv install --verbose --skip-lock' , {
2022-10-12 19:10:45 +03:00
shell : true ,
stdio : 'inherit'
} )
2022-10-15 16:03:44 +03:00
if (
osType === OSTypes . MacOS &&
cpuArchitecture === CPUArchitectures . ARM64
) {
LogHelper . info ( 'macOS ARM64 detected' )
LogHelper . info (
'Installing Rust installer as it is needed for the "tokenizers" package for macOS ARM64 architecture...'
)
2023-04-02 20:18:15 +03:00
await command ( 'curl https://sh.rustup.rs -sSf | sh -s -- -y' , {
2022-10-15 16:03:44 +03:00
shell : true ,
stdio : 'inherit'
} )
LogHelper . success ( 'Rust installer installed' )
LogHelper . info ( 'Reloading configuration from "$HOME/.cargo/env"...' )
2023-04-02 20:18:15 +03:00
await command ( 'source "$HOME/.cargo/env"' , {
2022-10-15 16:03:44 +03:00
shell : true ,
stdio : 'inherit'
} )
LogHelper . success ( 'Configuration reloaded' )
LogHelper . info ( 'Checking Rust compiler version...' )
2023-04-02 20:18:15 +03:00
await command ( 'rustc --version' , {
2022-10-15 16:03:44 +03:00
shell : true ,
stdio : 'inherit'
} )
2022-10-15 16:11:29 +03:00
LogHelper . success ( 'Rust compiler OK' )
2022-10-15 16:03:44 +03:00
}
2022-10-12 19:10:45 +03:00
LogHelper . success ( 'Python packages installed' )
2024-05-17 07:20:34 +03:00
2024-06-23 17:39:39 +03:00
if ( givenSetupTarget === 'tcp-server' ) {
await installPytorch ( )
}
2022-10-12 19:10:45 +03:00
} catch ( e ) {
2024-06-01 18:23:00 +03:00
if ( hasDotVenv ) {
await fs . promises . rm ( dotVenvPath , { recursive : true , force : true } )
LogHelper . info ( ` Error occurred, so " ${ dotVenvPath } " was deleted ` )
}
2022-10-12 19:10:45 +03:00
LogHelper . error ( ` Failed to install Python packages: ${ e } ` )
2022-10-07 08:24:58 +03:00
2024-05-19 19:39:33 +03:00
if ( osType === OSTypes . Linux || osType === OSTypes . MacOS ) {
LogHelper . error (
'If the error is related to "PortAudio" not installed or found, you can install it by running: "sudo apt install portaudio19-dev" or "brew install portaudio". Then retry. PortAudio is required for the "pyaudio" package used to record audio'
)
}
2022-10-12 19:10:45 +03:00
if ( osType === OSTypes . Windows ) {
LogHelper . error (
'Please check the error above. It might be related to Microsoft C++ Build Tools. If it is, you can check here: "https://stackoverflow.com/a/64262038/1768162" then restart your machine and retry'
)
2022-10-12 19:53:58 +03:00
LogHelper . error (
2024-06-23 18:29:09 +03:00
'If it is related to some hash mismatch, you can try by installing Pipenv 2024.0.1: pip install pipenv==2024.0.1'
2022-10-12 19:53:58 +03:00
)
2022-10-12 19:10:45 +03:00
}
process . exit ( 1 )
}
2022-10-06 21:37:43 +03:00
}
/ * *
* Verify if a fresh development environment installation is necessary
* /
2022-10-07 06:26:39 +03:00
// Required environment variables to set up
process . env . PIPENV _PIPFILE = pipfilePath
process . env . PIPENV _VENV _IN _PROJECT = true
2022-10-06 21:37:43 +03:00
2022-10-07 06:26:39 +03:00
if ( givenSetupTarget === 'python-bridge' ) {
// As per: https://github.com/marcelotduarte/cx_Freeze/issues/1548
process . env . PIP _NO _BINARY = 'cx_Freeze'
}
2019-02-10 15:26:50 +03:00
2022-10-07 06:26:39 +03:00
try {
2022-10-06 21:37:43 +03:00
if ( ! hasDotVenv ) {
await installPythonPackages ( )
2022-10-07 06:26:39 +03:00
} else {
if ( fs . existsSync ( dotProjectPath ) ) {
2023-04-02 21:34:50 +03:00
const dotProjectMtime = ( await fs . promises . stat ( dotProjectPath ) ) . mtime
2022-02-04 08:44:40 +03:00
2022-10-07 06:26:39 +03:00
// Check if Python deps tree has been modified since the initial setup
if ( pipfileMtime > dotProjectMtime ) {
LogHelper . info ( 'The development environment is not up-to-date' )
2022-09-03 14:12:41 +03:00
await installPythonPackages ( )
} else {
2022-10-07 06:26:39 +03:00
LogHelper . success ( 'Python packages are up-to-date' )
2022-02-04 08:44:40 +03:00
}
2022-10-06 21:37:43 +03:00
} else {
2022-10-07 06:26:39 +03:00
await installPythonPackages ( )
2022-09-03 14:12:41 +03:00
}
2019-02-10 15:26:50 +03:00
}
2022-10-06 21:37:43 +03:00
} catch ( e ) {
LogHelper . error (
` Failed to set up the ${ setupTarget } development environment: ${ e } `
)
} finally {
LoaderHelper . stop ( )
}
2022-10-07 06:26:39 +03:00
if ( givenSetupTarget === 'tcp-server' ) {
2022-10-07 08:44:48 +03:00
const installSpacyModels = async ( ) => {
try {
LogHelper . info ( 'Installing spaCy models...' )
// Install models one by one to avoid network throttling
2022-10-07 10:31:24 +03:00
for ( const modelLanguage of SPACY _MODELS . keys ( ) ) {
const modelInstallationFileUrl = getModelInstallationFileUrl (
modelLanguage ,
givenMirror
)
await command ( ` pipenv run pip install ${ modelInstallationFileUrl } ` , {
2022-10-07 08:44:48 +03:00
shell : true ,
stdio : 'inherit'
} )
}
LogHelper . success ( 'spaCy models installed' )
} catch ( e ) {
LogHelper . error ( ` Failed to install spaCy models: ${ e } ` )
process . exit ( 1 )
}
}
2022-10-07 06:26:39 +03:00
LogHelper . info ( 'Checking whether all spaCy models are installed...' )
try {
2022-10-07 10:31:24 +03:00
for ( const { name : modelName } of SPACY _MODELS . values ( ) ) {
2022-10-07 09:10:32 +03:00
const { stderr } = await command (
2022-10-07 10:31:24 +03:00
` pipenv run python -c "import ${ modelName } " ` ,
2022-10-07 09:10:32 +03:00
{ shell : true }
)
// Check stderr output for Windows as no exception is thrown
if ( osType === OSTypes . Windows ) {
if ( String ( stderr ) . length > 0 ) {
await installSpacyModels ( )
break
}
}
2022-10-07 06:26:39 +03:00
}
LogHelper . success ( 'All spaCy models are already installed' )
} catch ( e ) {
LogHelper . info ( 'Not all spaCy models are installed' )
2022-10-07 08:44:48 +03:00
await installSpacyModels ( )
2022-10-07 06:26:39 +03:00
}
}
LogHelper . success ( ` ${ setupTarget } development environment ready ` )
2022-10-06 21:37:43 +03:00
} ) ( )