mirror of
https://github.com/marian-nmt/marian.git
synced 2024-10-03 18:17:12 +03:00
Merged PR 19904: Update instructions for building on Windows
Changes proposed in this pull request: 1. Clarified instructions how to build Marian on Windows in `vs/README.md`. 2. `vs/CheckOrInstallDeps.bat` does not stop if CUDA or MKL are not installed.
This commit is contained in:
Roman Grundkiewicz
parent
b83b06fb73
commit
6b568f4afa
@ -2,7 +2,7 @@
|
||||
:: Usage: CheckOrInstallDeps.bat
|
||||
::
|
||||
:: This script is used to verify that all the dependencies required to build Marian are available.
|
||||
:: The Cuda SDK and the Intel MKL must be installed beforehand by the user.
|
||||
:: The CUDA SDK and the Intel MKL must be installed beforehand by the user.
|
||||
:: The rest of libraries (see README.md), if not found, will be installed by this script using
|
||||
:: vcpkg.
|
||||
::
|
||||
@ -96,6 +96,9 @@ echo.
|
||||
echo --- Checking dependencies...
|
||||
|
||||
set CMAKE_OPT=
|
||||
set FOUND_CUDA=
|
||||
set FOUND_MKL=
|
||||
set FOUND_BOOST=
|
||||
|
||||
|
||||
:: -------------------------
|
||||
@ -105,8 +108,9 @@ echo.
|
||||
echo ... CUDA
|
||||
if "%CUDA_PATH%"=="" (
|
||||
echo The CUDA_PATH environment variable is not defined: this will compile only the CPU version.
|
||||
set "FOUND_CUDA=false"
|
||||
) else (
|
||||
echo Found Cuda SDK in %CUDA_PATH%
|
||||
echo Found Cuda SDK in "%CUDA_PATH%"
|
||||
)
|
||||
|
||||
:: -------------------------
|
||||
@ -119,26 +123,30 @@ echo ... Intel MKL
|
||||
if "%MKLROOT%" == "" (
|
||||
set "MKLROOT=C:\Program Files (x86)\IntelSWTools\compilers_and_libraries\windows\mkl"
|
||||
)
|
||||
|
||||
if not exist "%MKLROOT%" (
|
||||
echo MKLROOT is set to a non existing path:
|
||||
echo "%MKLROOT%"
|
||||
echo "%MKLROOT%"
|
||||
echo Please make sure the Intel MKL libraries are installed and set MKLROOT to the installation path.
|
||||
exit /b 1
|
||||
)
|
||||
if not exist "%MKLROOT%\include\mkl_version.h" (
|
||||
set "FOUND_MKL=false"
|
||||
) else if not exist "%MKLROOT%\include\mkl_version.h" (
|
||||
echo MKL header files were not found in this folder:
|
||||
echo "%MKLROOT%"
|
||||
echo "%MKLROOT%\include"
|
||||
echo Please make sure Intel MKL is properly installed.
|
||||
exit /b 1
|
||||
)
|
||||
if not exist "%MKLROOT%\lib\intel64\mkl_core.lib" (
|
||||
set "FOUND_MKL=false"
|
||||
) else if not exist "%MKLROOT%\lib\intel64\mkl_core.lib" (
|
||||
echo MKL library files were not found in this folder:
|
||||
echo "%MKLROOT%"
|
||||
echo "%MKLROOT%\lib\intel64"
|
||||
echo Please make sure Intel MKL is properly installed.
|
||||
exit /b 1
|
||||
set "FOUND_MKL=false"
|
||||
) else (
|
||||
echo Found Intel MKL library in "%MKLROOT%"
|
||||
)
|
||||
|
||||
echo Found Intel MKL library in %MKLROOT%
|
||||
if "%FOUND_MKL%" == "false" if "%FOUND_CUDA%" == "false" (
|
||||
echo.
|
||||
echo Error: neither CUDA SDK nor Intel MKL were found, but at least one of them must be installed.
|
||||
)
|
||||
|
||||
:: -------------------------
|
||||
:: BOOST_INCLUDEDIR and BOOST_LIBRARYDIR can be both set to an existing Boost installation.
|
||||
@ -156,29 +164,31 @@ if not exist "%BOOST_INCLUDEDIR%" (
|
||||
echo BOOST_INCLUDEDIR is set to a non existing path:
|
||||
echo "%BOOST_INCLUDEDIR%"
|
||||
echo Please set BOOST_INCLUDEDIR and BOOST_LIBRARYDIR to the installation path of the Boost library.
|
||||
exit /b 1
|
||||
)
|
||||
if not exist "%BOOST_INCLUDEDIR%\boost\version.hpp" (
|
||||
set "FOUND_BOOST=false"
|
||||
) else if not exist "%BOOST_INCLUDEDIR%\boost\version.hpp" (
|
||||
echo Boost header files were not found in this folder:
|
||||
echo "%BOOST_INCLUDEDIR%"
|
||||
echo "%BOOST_INCLUDEDIR%\boost"
|
||||
echo Please make sure Boost is correctly installed.
|
||||
exit /b 1
|
||||
)
|
||||
|
||||
if not exist "%BOOST_LIBRARYDIR%" (
|
||||
set "FOUND_BOOST=false"
|
||||
) else if not exist "%BOOST_LIBRARYDIR%" (
|
||||
echo BOOST_LIBRARYDIR is set to a non existing path:
|
||||
echo "%BOOST_LIBRARYDIR%"
|
||||
echo Please set BOOST_INCLUDEDIR and BOOST_LIBRARYDIR to the installation path of the Boost library.
|
||||
exit /b 1
|
||||
)
|
||||
if not exist "%BOOST_LIBRARYDIR%\boost_*.lib" (
|
||||
set "FOUND_BOOST=false"
|
||||
) else if not exist "%BOOST_LIBRARYDIR%\boost_*.lib" (
|
||||
echo Boost library files were not found in this folder:
|
||||
echo "%BOOST_LIBRARYDIR%"
|
||||
echo Please make sure Boost is correctly installed.
|
||||
exit /b 1
|
||||
set "FOUND_BOOST=false"
|
||||
) else (
|
||||
echo Found Boost headers in "%BOOST_INCLUDEDIR%" and libs in "%BOOST_LIBRARYDIR%"
|
||||
)
|
||||
|
||||
if "%FOUND_BOOST%" == "false" (
|
||||
echo.
|
||||
echo Warning: Boost was not found. marian-server will not be compiled.
|
||||
)
|
||||
|
||||
echo Found Boost headers in "%BOOST_INCLUDEDIR%" and libs in "%BOOST_LIBRARYDIR%"
|
||||
|
||||
:: -------------------------
|
||||
:: OPENSSL_ROOT_DIR can be set to an existing OpenSSL installation.
|
||||
|
||||
31
vs/NOTES.md
31
vs/NOTES.md
@ -1,6 +1,35 @@
|
||||
# How to build Marian on Windows with GPU support
|
||||
|
||||
This is interesting for developers, exctracted from README.
|
||||
This is interesting mostly for developers. Warning: it has been extracted from
|
||||
an old `vs/README.md` and some information might be outdated.
|
||||
|
||||
---
|
||||
## Known issues
|
||||
|
||||
1. __Patch for CUDA 9.2 error: Unsupported Visual Studio Version Error__
|
||||
|
||||
When using CUDA 9.2, the latest versions of Visual Studio 2017 are not
|
||||
officially supported by CUDA. Two fixes are proposed:
|
||||
- Downgrade Visual Studio to a supported version
|
||||
- Edit the file `<CUDA install path>\include\crt\host_config.h` and change the line 131:
|
||||
|
||||
131 #if _MSC_VER < 1600 || _MSC_VER > 1914
|
||||
|
||||
into:
|
||||
|
||||
131 #if _MSC_VER < 1600 || _MSC_VER > 1915
|
||||
|
||||
For more information, read this [nVidia forum](https://devtalk.nvidia.com/default/topic/1022648/cuda-setup-and-installation/cuda-9-unsupported-visual-studio-version-error/4)
|
||||
|
||||
2. __It does not compile with Boost 1.73 or newer__
|
||||
|
||||
It may happen that SimpleWebSocketServer, a 3rd party library that Marian uses for
|
||||
marian-server, does not support the version of Boost available in vcpkg. In such case install a
|
||||
supported version of Boost; if you use vcpkg, an option is to checkout to #5970385, which has
|
||||
Boost 1.72.
|
||||
|
||||
Note that Boost is required only if you compile with marian-server, for compilation using CMake,
|
||||
it is if you set `COMPILE_SERVER` to `TRUE` in CMakeSettings.json.
|
||||
|
||||
---
|
||||
## Changes from the master branch
|
||||
|
||||
166
vs/README.md
166
vs/README.md
@ -1,34 +1,34 @@
|
||||
# How to build Marian on Windows with GPU support
|
||||
# Building Marian on Windows
|
||||
|
||||
|
||||
## Install prerequisites
|
||||
|
||||
The following SDK are required to build Marian with GPU support. At least one of them needs to be
|
||||
installed. If only CUDA is installed but not MKL, a GPU-only version will be build. If only MKL is
|
||||
installed and not CUDA, only the CPU version will be built. So if you are interested in only one
|
||||
functionality, you can omit one of them. Install both for full functionality.
|
||||
At least one of the following SDK is required to build Marian on Windows:
|
||||
|
||||
- [CUDA](https://developer.nvidia.com/cuda-downloads?target_os=Windows&target_arch=x86_64&target_version=10&target_type=exelocal),
|
||||
Base installer, CUDA 10.0+ is recommended, there might be issues with CUDA 9.2, see below
|
||||
Base installer, CUDA 10.0+ is recommended, there might be issues with CUDA
|
||||
9.2, see below
|
||||
- [Intel MKL](https://software.intel.com/en-us/mkl)
|
||||
|
||||
CUDA is required for running Marian on GPU, and Intel MKL for CPU. If only one
|
||||
of them is installed, a GPU-only or CPU-only version can be built.
|
||||
|
||||
---
|
||||
## Check dependencies : `CheckOrInstallDeps.bat`
|
||||
|
||||
In addition to the 2 previous prerequisites, Marian may need the following libraries that you may
|
||||
already have on your system:
|
||||
The script `CheckOrInstallDeps.bat` can be used to verify that all dependencies
|
||||
are found on your system. If not, it will use the `vcpkg` library manager to
|
||||
download and manage your dependencies for CMake, including the following
|
||||
optional libraries needed only if you want to compile Marian server:
|
||||
|
||||
- Boost (1.58-1.72), optional for marian-server (`COMPILE_SERVER=TRUE` in CMake)
|
||||
- Boost (1.58-1.72), optional for marian-server (`COMPILE_SERVER=TRUE` in
|
||||
`CMakeSettings.json`)
|
||||
- OpenSSL, optional for marian-server
|
||||
|
||||
The script `CheckOrInstallDeps.bat` can be used to verify that all dependencies are found on your
|
||||
system. If not, it will use the `vcpkg` library manager to download and manage your dependencies for
|
||||
CMake.
|
||||
|
||||
If you already have a working `vcpkg` installation, this script can use it.
|
||||
If vcpkg is in your `PATH` environment variable, the script will find it and use it automatically.
|
||||
Otherwise, you need to edit the script and set the `VCPKG_ROOT` variable.
|
||||
Please see the script for more details.
|
||||
If you already have a working `vcpkg` installation, this script can use it. If
|
||||
vcpkg is in your `PATH` environment variable, the script will find it and use
|
||||
it automatically. Otherwise, you need to edit the script and set the
|
||||
`VCPKG_ROOT` variable. Please see the script for more details.
|
||||
|
||||
---
|
||||
## Build the project
|
||||
@ -41,119 +41,59 @@ There are 3 alternatives to build the project:
|
||||
|
||||
### 1. Use VS2017+ with built-in support for CMake
|
||||
|
||||
VS2017 or newer now allows to develop projects built with CMake without the need to generate VS
|
||||
projects and solutions. For more information, please read [this article](https://blogs.msdn.microsoft.com/vcblog/2016/10/05/cmake-support-in-visual-studio/)
|
||||
VS2017 or newer now allows to develop projects built with CMake without the
|
||||
need to generate VS projects and solutions. For more information, please read
|
||||
[this article](https://blogs.msdn.microsoft.com/vcblog/2016/10/05/cmake-support-in-visual-studio/)
|
||||
from the Visual C++ Team.
|
||||
|
||||
You just need to open the root folder of the git repository in VS (which contains the file
|
||||
`CMakeSettings.json`):
|
||||
1. Open the root folder of the git repository in VS (which contains the file
|
||||
`CMakeSettings.json`) using `Open local folder` on the welcome page or `File
|
||||
> Open > Folder...` in a VS instance.
|
||||
2. Edit the file `CMakeSettings.json` to set the environment variable for the
|
||||
dependencies. Set `COMPILE_CUDA` or `COMPILE_CPU` to `FALSE` if you wish to
|
||||
compile a CPU-only or a GPU-only version respectively.
|
||||
3. VS2017 should automatically detect `CMakeSettings.json` and generate CMake
|
||||
Cache.
|
||||
4. Build the project with `F7`. If build is successful, the executables will be
|
||||
in the `build` folder.
|
||||
|
||||
- In an Explorer window, right-click then `Open in Visual Studio`
|
||||
- In a VS2017 instance, `File > Open > Folder...`
|
||||
|
||||
You may need to edit the file `CMakeSettings.json` to set the environment variable for the
|
||||
dependencies.
|
||||
#### Development
|
||||
|
||||
The developing experience is very similar that when using a solution file (Intellisense, build
|
||||
project with `F7`, debug, set breakpoints and watch variables, ...), except that the project
|
||||
configuration is done in 3 different files:
|
||||
The developing experience is very similar that when using a solution file
|
||||
(Intellisense, build project with `F7`, debug, set breakpoints and watch
|
||||
variables, ...), except that the project configuration is done in 3 different
|
||||
files:
|
||||
|
||||
- `CMakeList.txt`: this is the CMake source file from the original project.
|
||||
It is used to configure the build targets, add/remove files to compile and configure the
|
||||
compiler flags.
|
||||
- `CMakeList.txt`: this is the CMake source file from the original project.
|
||||
It is used to configure the build targets, add/remove files to compile and configure the
|
||||
compiler flags.
|
||||
|
||||
- `CMakeSettings.json`: this file is required to enable CMake integration in VS2017.
|
||||
Use this file to configure the environment variables and the parameters passed to CMake to
|
||||
generate the project.
|
||||
- `CMakeSettings.json`: this file is required to enable CMake integration in VS2017.
|
||||
Use this file to configure the environment variables and the parameters passed to CMake to
|
||||
generate the project.
|
||||
|
||||
- `.vs\launch.vs.json`: this is an optional user specific file and it is not commited in the Git
|
||||
repo. Use this file to configure the debugging targets. For example:
|
||||
|
||||
{
|
||||
"version": "0.2.1",
|
||||
"defaults": {},
|
||||
"configurations": [
|
||||
{
|
||||
"type": "default",
|
||||
"name": "Training Basics",
|
||||
"project": "CMakeLists.txt",
|
||||
"projectTarget": "marian.exe",
|
||||
"currentDir": "D:\\Perso\\github\\marian\\marian-examples\\training-basics",
|
||||
"args": [
|
||||
"--devices 0",
|
||||
"--type amun",
|
||||
"--model model/model.npz",
|
||||
"--train-sets data/corpus.bpe.ro data/corpus.bpe.en",
|
||||
"--vocabs model/vocab.ro.yml model/vocab.en.yml",
|
||||
"--dim-vocabs 66000 50000",
|
||||
"--mini-batch-fit",
|
||||
"-w 3000",
|
||||
"--layer-normalization",
|
||||
"--dropout-rnn 0.2",
|
||||
"--dropout-src 0.1",
|
||||
"--dropout-trg 0.1",
|
||||
"--early-stopping 5",
|
||||
"--valid-freq 100",
|
||||
"--save-freq 10000",
|
||||
"--disp-freq 100",
|
||||
"--valid-metrics cross-entropy translation",
|
||||
"--valid-sets data/newsdev2016.bpe.ro data/newsdev2016.bpe.en",
|
||||
"--valid-script-path .\\scripts\\validate.bat",
|
||||
"--log model/train.log",
|
||||
"--valid-log model/valid.log",
|
||||
"--overwrite",
|
||||
"--keep-best",
|
||||
"--seed 1111",
|
||||
"--exponential-smoothing",
|
||||
"--normalize 1",
|
||||
"--beam-size 12",
|
||||
"--quiet-translation"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
- `.vs\launch.vs.json`: this is an optional user specific file and it is not commited in the Git
|
||||
repo. Use this file to configure the debugging targets.
|
||||
|
||||
|
||||
### 2. Create solution and projects files for Visual Studio : `CreateVSProjects.bat`
|
||||
|
||||
If you have a previous version of Visual Studio, you will need to use CMake to generate the projects
|
||||
If you have a previous version of Visual Studio, you will need to use CMake to
|
||||
generate the projects
|
||||
files.
|
||||
|
||||
The provided script `CreateVSProjects.bat` runs the dependency checks then invokes CMake with the
|
||||
right parameters to create the solutions for Visual Studio.
|
||||
The provided script `CreateVSProjects.bat` runs the dependency checks then
|
||||
invokes CMake with the right parameters to create the solutions for Visual
|
||||
Studio.
|
||||
|
||||
Warning: the Visual Studio Solution file included in the `vs/` folder might not
|
||||
work out of the box with your environment and require customization.
|
||||
|
||||
|
||||
### 3. Use MSBuild : `BuildRelease.bat`
|
||||
|
||||
The last alternative is to use the script `BuildRelease.bat` that will:
|
||||
- Check the dependencies
|
||||
- Create the VS project files
|
||||
- Check the dependencies.
|
||||
- Create the VS project files.
|
||||
- Invoke MSBuild on these projects to build the targets in Release.
|
||||
|
||||
---
|
||||
## Known issues
|
||||
|
||||
1. __Patch for CUDA 9.2 error: Unsupported Visual Studio Version Error__
|
||||
|
||||
When using CUDA 9.2, the latest versions of Visual Studio 2017 are not officially supported by
|
||||
CUDA. Two fixes are proposed:
|
||||
- Downgrade Visual Studio to a supported version
|
||||
- Edit the file `<CUDA install path>\include\crt\host_config.h` and change the line 131:
|
||||
|
||||
131 #if _MSC_VER < 1600 || _MSC_VER > 1914
|
||||
|
||||
into:
|
||||
|
||||
131 #if _MSC_VER < 1600 || _MSC_VER > 1915
|
||||
|
||||
For more information, read this [nVidia forum](https://devtalk.nvidia.com/default/topic/1022648/cuda-setup-and-installation/cuda-9-unsupported-visual-studio-version-error/4)
|
||||
|
||||
2. __It does not compile with Boost 1.73 or newer__
|
||||
|
||||
It may happen that SimpleWebSocketServer, a 3rd party library that Marian uses for
|
||||
marian-server, does not support the version of Boost available in vcpkg. In such case install a
|
||||
supported version of Boost; if you use vcpkg, an option is to checkout to #5970385, which has
|
||||
Boost 1.72.
|
||||
|
||||
Note that Boost is required only if you compile with marian-server, for compilation using CMake,
|
||||
it is if you set `COMPILE_SERVER` to `TRUE` in CMakeSettings.json.
|
||||
|
||||
Loading…
Reference in New Issue
Block a user