mirror of
https://github.com/marian-nmt/marian.git
synced 2024-11-05 01:31:46 +03:00
Merged PR 14262: Update MSVC CMake build and instructions
This PR updates Windows build via CMake and build instructions. With https://github.com/marian-nmt/marian-dev/pull/676, this should be fully workable, including CUDA, FBGEMM, SentencePiece, unit tests, marian-server. List of changes: - Fixing compilation of marian-server on Windows via CMake - Updating vs/CheckDeps.bat - zlib no longer needs to be installed as it is included in 3rd_party - Installing Boost 1.72 since newer is not supported - Installing minimal required Boost components in CheckDeps.bat - Installing protobuf in CheckDeps.bat - Updating CMakeSettings.json - Updating vs/README.md - Development notes extracted to vs/NOTES.md I did not update and test with CUDA, because I do not have a machine for that, but AFAIK it works properly.
This commit is contained in:
Roman Grundkiewicz
Martin Junczys-Dowmunt
parent
38bd181937
commit
71dccf343e
@ -391,6 +391,13 @@ if(COMPILE_SERVER)
|
||||
message(STATUS "Found OpenSSL")
|
||||
include_directories(${OPENSSL_INCLUDE_DIR})
|
||||
set(EXT_LIBS ${EXT_LIBS} ${OPENSSL_CRYPTO_LIBRARY})
|
||||
if(MSVC AND USE_STATIC_LIBS)
|
||||
# "If you link with static OpenSSL libraries then you're expected to additionally link your
|
||||
# application with WS2_32.LIB, GDI32.LIB, ADVAPI32.LIB, CRYPT32.LIB and USER32.LIB"
|
||||
# See https://github.com/openssl/openssl/blob/OpenSSL_1_1_1d/NOTES.WIN#L127
|
||||
# Linking with crypt32.lib seem to be enough.
|
||||
set(EXT_LIBS ${EXT_LIBS} crypt32.lib)
|
||||
endif()
|
||||
set(BOOST_COMPONENTS ${BOOST_COMPONENTS} system)
|
||||
else(OpenSSL_FOUND)
|
||||
message(WARNING "Cannot find OpenSSL library. Not compiling server.")
|
||||
|
||||
@ -2,20 +2,25 @@
|
||||
"environments": [
|
||||
{
|
||||
// Dependencies can be found automatically if you use vcpkg as your library manager.
|
||||
// In this case, please set the VCPKG_ROOT variable to the directory that contains the vcpkg.exe
|
||||
// In this case, please set the VCPKG_ROOT variable to the directory that contains the
|
||||
// vcpkg.exe. If you used CheckDeps.bat to install dependencies, by default this should be an
|
||||
// absolute path to marian-dev\vs\deps\vcpkg.
|
||||
// If you prefer managing yourself the dependencies, please fill in the other variables.
|
||||
"VCPKG_ROOT": "D:\\Perso\\Dev\\vcpkg",
|
||||
"VCPKG_ROOT": "C:\\path\\to\\marian-dev\\vs\\deps\\vcpkg",
|
||||
|
||||
// The MKL library can be automatically found by CMake. However, if you installed it in a
|
||||
// custom directory, please set the MKLROOT to this directory path.
|
||||
// Default is c:\Program Files (x86)\IntelSWTools\compilers_and_libraries\windows\mkl on
|
||||
// Windows, or /opt/intel/mkl on Linux
|
||||
"MKLROOT": "",
|
||||
|
||||
// Boost and OpenSSL are required if you compile with COMPILE_SERVER=TRUE
|
||||
"BOOST_INCLUDEDIR": "",
|
||||
"BOOST_LIBRARYDIR": "",
|
||||
"ZLIB_ROOT": "",
|
||||
"OPENSSL_ROOT_DIR": "",
|
||||
|
||||
// The MKL library can be automatically found by CMake. However, if you installed it in a custom
|
||||
// directory, please set the MKLROOT to this directory path.
|
||||
// Default is c:\Program Files (x86)\IntelSWTools\compilers_and_libraries\windows\mkl on Windows, or
|
||||
// /opt/intel/mkl on Linux
|
||||
"MKLROOT": ""
|
||||
// Protobuf is required if you compile with USE_SENTENCEPIECE=TRUE
|
||||
"Protobuf_SRC_ROOT_FOLDER": ""
|
||||
}
|
||||
],
|
||||
"configurations": [
|
||||
@ -32,18 +37,21 @@
|
||||
"variables": [
|
||||
{ "name": "CMAKE_TOOLCHAIN_FILE", "value": "${env.VCPKG_ROOT}\\scripts\\buildsystems\\vcpkg.cmake" },
|
||||
{ "name": "VCPKG_TARGET_TRIPLET", "value": "x64-windows-static" },
|
||||
|
||||
|
||||
{ "name": "OPENSSL_USE_STATIC_LIBS:BOOL", "value": "TRUE" },
|
||||
{ "name": "OPENSSL_MSVC_STATIC_RT:BOOL", "value": "TRUE" },
|
||||
|
||||
{ "name": "COMPILE_SERVER:BOOL", "value": "TRUE" },
|
||||
{ "name": "COMPILE_EXAMPLES:BOOL", "value": "FALSE" },
|
||||
{ "name": "COMPILE_TESTS:BOOL", "value": "FALSE" },
|
||||
{ "name": "COMPILE_CPU:BOOL", "value": "TRUE" },
|
||||
{ "name": "COMPILE_CUDA:BOOL", "value": "TRUE" },
|
||||
|
||||
{ "name": "USE_CUDNN:BOOL", "value": "TRUE" },
|
||||
{ "name": "USE_MPI:BOOL", "value": "FALSE" }
|
||||
|
||||
{ "name": "COMPILE_CUDA:BOOL", "value": "TRUE" },
|
||||
{ "name": "COMPILE_CPU:BOOL", "value": "TRUE" },
|
||||
{ "name": "COMPILE_EXAMPLES:BOOL", "value": "FALSE" },
|
||||
{ "name": "COMPILE_SERVER:BOOL", "value": "TRUE" },
|
||||
{ "name": "COMPILE_TESTS:BOOL", "value": "FALSE" },
|
||||
|
||||
{ "name": "USE_CUDNN:BOOL", "value": "FALSE" },
|
||||
{ "name": "USE_FBGEMM:BOOL", "value": "TRUE" },
|
||||
{ "name": "USE_MPI:BOOL", "value": "FALSE" },
|
||||
{ "name": "USE_SENTENCEPIECE:BOOL", "value": "TRUE" },
|
||||
{ "name": "USE_STATIC_LIBS:BOOL", "value": "TRUE" }
|
||||
]
|
||||
},
|
||||
{
|
||||
@ -59,19 +67,22 @@
|
||||
"variables": [
|
||||
{ "name": "CMAKE_TOOLCHAIN_FILE", "value": "${env.VCPKG_ROOT}\\scripts\\buildsystems\\vcpkg.cmake" },
|
||||
{ "name": "VCPKG_TARGET_TRIPLET", "value": "x64-windows-static" },
|
||||
|
||||
|
||||
{ "name": "OPENSSL_USE_STATIC_LIBS:BOOL", "value": "TRUE" },
|
||||
{ "name": "OPENSSL_MSVC_STATIC_RT:BOOL", "value": "TRUE" },
|
||||
|
||||
{ "name": "COMPILE_SERVER:BOOL", "value": "TRUE" },
|
||||
{ "name": "COMPILE_EXAMPLES:BOOL", "value": "FALSE" },
|
||||
{ "name": "COMPILE_TESTS:BOOL", "value": "FALSE" },
|
||||
{ "name": "COMPILE_CPU:BOOL", "value": "TRUE" },
|
||||
{ "name": "COMPILE_CUDA:BOOL", "value": "TRUE" },
|
||||
|
||||
{ "name": "USE_CUDNN:BOOL", "value": "TRUE" },
|
||||
{ "name": "USE_MPI:BOOL", "value": "FALSE" }
|
||||
|
||||
{ "name": "COMPILE_CUDA:BOOL", "value": "TRUE" },
|
||||
{ "name": "COMPILE_CPU:BOOL", "value": "TRUE" },
|
||||
{ "name": "COMPILE_EXAMPLES:BOOL", "value": "FALSE" },
|
||||
{ "name": "COMPILE_SERVER:BOOL", "value": "TRUE" },
|
||||
{ "name": "COMPILE_TESTS:BOOL", "value": "TRUE" },
|
||||
|
||||
{ "name": "USE_CUDNN:BOOL", "value": "FALSE" },
|
||||
{ "name": "USE_FBGEMM:BOOL", "value": "TRUE" },
|
||||
{ "name": "USE_MPI:BOOL", "value": "FALSE" },
|
||||
{ "name": "USE_SENTENCEPIECE:BOOL", "value": "TRUE" },
|
||||
{ "name": "USE_STATIC_LIBS:BOOL", "value": "TRUE" }
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
@ -219,7 +219,13 @@ endif(USE_STATIC_LIBS)
|
||||
if(COMPILE_SERVER)
|
||||
add_executable(marian_server command/marian_server.cpp)
|
||||
set_target_properties(marian_server PROPERTIES OUTPUT_NAME marian-server)
|
||||
target_compile_options(marian_server PUBLIC ${ALL_WARNINGS})
|
||||
if(MSVC)
|
||||
# Disable warnings from the SimpleWebSocketServer library
|
||||
target_compile_options(marian_server PUBLIC ${ALL_WARNINGS} /wd4267 /wd4244 /wd4456 /wd4458)
|
||||
else(MSVC)
|
||||
# -Wno-suggest-override disables warnings from Boost 1.69+
|
||||
target_compile_options(marian_server PUBLIC ${ALL_WARNINGS} -Wno-suggest-override)
|
||||
endif(MSVC)
|
||||
set(EXECUTABLES ${EXECUTABLES} marian_server)
|
||||
endif(COMPILE_SERVER)
|
||||
|
||||
|
||||
@ -3,7 +3,8 @@
|
||||
::
|
||||
:: This script is used to verify that all the dependencies required to build Marian are available.
|
||||
:: The Cuda SDK and the Intel MKL must be installed beforehand by the user.
|
||||
:: The Boost and OpenSSH libraries, if not found, will be installed by this script using vcpkg
|
||||
:: The rest of libraries (see README.md), if not found, will be installed by this script using
|
||||
:: vcpkg.
|
||||
::
|
||||
::
|
||||
@echo off
|
||||
@ -28,6 +29,7 @@ set ROOT=%~dp0
|
||||
::set BOOST_INCLUDEDIR=
|
||||
::set BOOST_LIBRARYDIR=
|
||||
::set OPENSSL_ROOT_DIR=
|
||||
::set Protobuf_SRC_ROOT_FOLDER=
|
||||
|
||||
|
||||
:: If all the variables are empty and vcpkg is found in a known path, the script will download and
|
||||
@ -41,9 +43,9 @@ set ROOT=%~dp0
|
||||
::set MKLROOT=
|
||||
|
||||
|
||||
|
||||
if "%BOOST_INCLUDEDIR%" == "" goto :needVcPkg
|
||||
if "%OPENSSL_ROOT_DIR%" == "" goto :needVcPkg
|
||||
if "%Protobuf_SRC_ROOT_FOLDER%"=="" goto :needVcPkg
|
||||
|
||||
goto :checkDeps
|
||||
|
||||
@ -64,6 +66,7 @@ if not exist %VCPKG_ROOT% (
|
||||
echo --- Cloning vcpkg...
|
||||
git clone https://github.com/Microsoft/vcpkg.git %VCPKG_ROOT%
|
||||
|
||||
set USE_BOOST_172=1
|
||||
set BOOTSTRAP_VCPKG=1
|
||||
|
||||
) else (
|
||||
@ -73,6 +76,7 @@ if not exist %VCPKG_ROOT% (
|
||||
echo --- Updating vcpkg...
|
||||
for /f "delims=" %%p in ('git pull') do (
|
||||
if not "%%p" == "Already up to date." (
|
||||
set USE_BOOST_172=1
|
||||
set BOOTSTRAP_VCPKG=1
|
||||
)
|
||||
)
|
||||
@ -80,6 +84,17 @@ if not exist %VCPKG_ROOT% (
|
||||
popd
|
||||
)
|
||||
|
||||
:: Checkout to the last version of Boost that is supported by the 3rd party library
|
||||
:: SimpleWebSocketServer. Vcpkg does not allow installing a specific version of library yet, but
|
||||
:: the feature has a PR and should be available soon.
|
||||
:: For more details, see https://github.com/microsoft/vcpkg/issues/1681
|
||||
if "%USE_BOOST_172%"=="1" (
|
||||
pushd %VCPKG_ROOT%
|
||||
echo --- Checkout to Boost version 1.72...
|
||||
git checkout 597038559647776ee39d02dcf159da05d9342f1d --pathspec-from-file=../../pathspec-boost-1.72.txt
|
||||
popd
|
||||
)
|
||||
|
||||
if "%BOOTSTRAP_VCPKG%"=="1" (
|
||||
pushd %VCPKG_ROOT%
|
||||
call bootstrap-vcpkg.bat
|
||||
@ -91,7 +106,6 @@ set VCPKG_INSTALL=%VCPKG_ROOT%\installed\%VCPKG_DEFAULT_TRIPLET%
|
||||
set VCPKG=%VCPKG_ROOT%\vcpkg
|
||||
|
||||
|
||||
|
||||
:: -------------------------------------------------------
|
||||
:: Check dependencies and configure CMake
|
||||
:checkDeps
|
||||
@ -117,12 +131,12 @@ else (
|
||||
|
||||
:: -------------------------
|
||||
:: The MKL setup does not set any environment variable to the installation path.
|
||||
:: The script look into the standard default installation dir
|
||||
:: The script look into the standard default installation directory.
|
||||
:: If you installed MKL in a custom directory, please set the variable MKLROOT at the top of this file.
|
||||
::
|
||||
echo.
|
||||
echo ... Intel MKL
|
||||
if "%MKLROOT%" == "" (
|
||||
if "%MKLROOT%" == "" (
|
||||
set "MKLROOT=C:\Program Files (x86)\IntelSWTools\compilers_and_libraries\windows\mkl"
|
||||
)
|
||||
if not exist "%MKLROOT%" (
|
||||
@ -153,7 +167,7 @@ echo Found Intel MKL library in %MKLROOT%
|
||||
echo.
|
||||
echo ... Boost (1.58+)
|
||||
if "%BOOST_INCLUDEDIR%" == "" (
|
||||
"%VCPKG%" install boost-chrono boost-filesystem boost-iostreams boost-program-options boost-regex boost-system boost-thread boost-timer boost-asio
|
||||
"%VCPKG%" install boost-system boost-asio
|
||||
set BOOST_INCLUDEDIR=%VCPKG_INSTALL%\include
|
||||
set BOOST_LIBRARYDIR=%VCPKG_INSTALL%\lib
|
||||
)
|
||||
@ -197,17 +211,15 @@ if "%OPENSSL_ROOT_DIR%"=="" (
|
||||
set OPENSSL_ROOT_DIR=%VCPKG_INSTALL%
|
||||
)
|
||||
|
||||
if not exist "%VCPKG_INSTALL%/bin/protoc.exe" (
|
||||
mkdir build
|
||||
cd build
|
||||
git clone https://github.com/protocolbuffers/protobuf
|
||||
cd protobuf
|
||||
git checkout v3.6.1
|
||||
cd cmake
|
||||
cmake . -A x64 -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_INSTALL_PREFIX=%VCPKG_INSTALL%
|
||||
cmake --build . --config Release --target install
|
||||
cd ..\..\..
|
||||
|
||||
:: -------------------------
|
||||
:: Protobuf_SRC_ROOT_FOLDER can be set to an existing Protobuf installation.
|
||||
:: If not, we use vcpkg to install the library
|
||||
::
|
||||
echo.
|
||||
echo ... Protobuf
|
||||
if "%Protobuf_SRC_ROOT_FOLDER%"=="" (
|
||||
%VCPKG% install protobuf
|
||||
set Protobuf_SRC_ROOT_FOLDER=%VCPKG_INSTALL%
|
||||
)
|
||||
|
||||
set CMAKE_PREFIX_PATH=%VCPKG_INSTALL%
|
||||
@ -215,13 +227,14 @@ set CMAKE_PREFIX_PATH=%VCPKG_INSTALL%
|
||||
echo.
|
||||
echo.
|
||||
echo --------------------------------------------------
|
||||
echo CUDA_PATH ^| %CUDA_PATH%
|
||||
echo MKLROOT ^| %MKLROOT%
|
||||
echo VCPKG_ROOT ^| %VCPKG_ROOT%
|
||||
echo BOOST_INCLUDEDIR ^| %BOOST_INCLUDEDIR%
|
||||
echo BOOST_LIBRARYDIR ^| %BOOST_LIBRARYDIR%
|
||||
echo OPENSSL_ROOT_DIR ^| %OPENSSL_ROOT_DIR%
|
||||
echo CMAKE_PREFIX_PATH ^| %CMAKE_PREFIX_PATH%
|
||||
echo BOOST_INCLUDEDIR ^| %BOOST_INCLUDEDIR%
|
||||
echo BOOST_LIBRARYDIR ^| %BOOST_LIBRARYDIR%
|
||||
echo CMAKE_PREFIX_PATH ^| %CMAKE_PREFIX_PATH%
|
||||
echo CUDA_PATH ^| %CUDA_PATH%
|
||||
echo MKLROOT ^| %MKLROOT%
|
||||
echo OPENSSL_ROOT_DIR ^| %OPENSSL_ROOT_DIR%
|
||||
echo Protobuf_SRC_ROOT_FOLDER ^| %Protobuf_SRC_ROOT_FOLDER%
|
||||
echo VCPKG_ROOT ^| %VCPKG_ROOT%
|
||||
echo --------------------------------------------------
|
||||
echo.
|
||||
echo.
|
||||
|
||||
58
vs/NOTES.md
Normal file
58
vs/NOTES.md
Normal file
@ -0,0 +1,58 @@
|
||||
# How to build Marian on Windows with GPU support
|
||||
|
||||
This is interesting for developers, exctracted from README.
|
||||
|
||||
---
|
||||
## Changes from the master branch
|
||||
This part gives more information on all changes done in this PR. Refer to [this page](https://github.com/cedrou/marian-dev/commits/build_on_win) for commits.
|
||||
|
||||
1. __Fix Cuda error : Unsupported Visual Studio Version Error__
|
||||
See above for justification and fixes
|
||||
|
||||
2. __Fix VS compiler flags / Build in Release, with improved debug info__
|
||||
Added VS specific compile and link flags
|
||||
|
||||
3. __Fix Warning: D9002: ignoring unknown option '-m64'__
|
||||
This one is related to a compiler flag added while finding the package MKL that does not exists for MS compiler.
|
||||
|
||||
4. __Fix marian::Backend, marian::cpu::Backend and marian::gpu::Backend conflicts__
|
||||
There were name conflicts between the 3 `Backend` classes that confused the compiler:
|
||||
|
||||
> template instantiation resulted in unexpected function type of "void(Ptr\<marian::gpu::Backend\> backend, [...])" (the meaning of a name may have changed since the template declaration -- the type of the template is "void(Ptr\<marian::Backend\> backend, [...]").
|
||||
|
||||
To solve this, I changed the declaration of 3 methods to specify the full name with namespace (`marian::Backend`, instead of `Backend`).
|
||||
|
||||
5. __Fix error : identifier "CUDA_FLT_MAX" is undefined in device code__
|
||||
`CUDA_FLT_MAX` is not seen by CUDA from the device code and I had to declare it as `__constant__`.
|
||||
|
||||
From [StackOverflow](https://stackoverflow.com/questions/20111409/how-to-pass-structures-into-cuda-device#comment29972423_20112013):
|
||||
> Undecorated constants get compiled into both host and device code with gcc based toolchains, but not with the Microsoft compiler.
|
||||
|
||||
6. __Fix fatal error C1019: unexpected #else__
|
||||
There was preprocessor instructions (`#ifdef ... #else ... #endif`) in the middle of a call of a macro function (`CUDNN_CALL`), which is not allowed with MS compiler.
|
||||
|
||||
7. __Fix mismatched class/struct forward declarations__
|
||||
Microsoft's C++ name mangling makes a distinction between `class` and `struct` objects, so definitions and forward declaration must match.
|
||||
See [this pdf](https://www.agner.org/optimize/calling_conventions.pdf), page 27, for more information.
|
||||
|
||||
_Note_: This fix was invalidated by commit # from @frankseide
|
||||
|
||||
8. __Fix unresolved external due to a removed #include directive__
|
||||
There was an include directive removed from MSVC compilation, but this prevented the build of the project.
|
||||
I'm not sure why this was removed; the comment is:
|
||||
|
||||
#ifndef _WIN32 // TODO: remove this once I updated the Linux-side makefile
|
||||
|
||||
9. __Fix CUDA+MSVC incompatibility with Boost.Preprocessor__
|
||||
The toolchain nvcc+msvc is not correctly handled in Boost.Preprocessor module. See [this issue](https://github.com/boostorg/preprocessor/issues/15). In the meantime, the recommended workaround is to disable Variadic Macro support in Boost.
|
||||
I created a [PR](https://github.com/boostorg/preprocessor/pull/18) in the Boost repo on GitHub to fix this.
|
||||
|
||||
_Note_: The library sources have been fixed, but this fix is still needed until the next release of Boost.Preprocessor
|
||||
|
||||
10. __Provide implementation for mkstemp / Fix temporary file creation__
|
||||
The code explicitely disabled the creation of temporary files because "mkstemp not available in Windows". In fact, `mktemp` and `unlink` are both implemented, but they don't work as expected. I used `tempnam` to replace `mkstemp`, and added the flag `_O_TEMPORARY` to the parameters of `open` to automatically delete the file when it is closed. If `unlinkEarly` is not set, I added a call to `remove` in the destructor to delete the file after its closure.
|
||||
I also handled the case of the default value for the `base` parameter: the path `\tmp` doesnot exist on Windows, so it is replaced by the value of the `%TMP%` environment variable in `NormalizeTempPrefix`.
|
||||
|
||||
11. __Revert commit #2f8b093 + Fix copy/paste error while fixing #301 + restrict fix to MSVC compiler.__
|
||||
cf [Issue #301](https://github.com/marian-nmt/marian-dev/issues/301) -->
|
||||
|
||||
168
vs/README.md
168
vs/README.md
@ -3,81 +3,72 @@
|
||||
|
||||
## Install prerequisites
|
||||
|
||||
The following SDK are required to build Marian with GPU support. At least one of them needs to be installed. If only CUDA is installed but not MKL,
|
||||
a GPU-only version will be build. If only MKL is installed and not CUDA, only the CPU version will be built. So if you are interested in only one
|
||||
functionality, you can ommit one of them. Install both for full functionality.
|
||||
The following SDK are required to build Marian with GPU support. At least one of them needs to be
|
||||
installed. If only CUDA is installed but not MKL, a GPU-only version will be build. If only MKL is
|
||||
installed and not CUDA, only the CPU version will be built. So if you are interested in only one
|
||||
functionality, you can omit one of them. Install both for full functionality.
|
||||
|
||||
- [Cuda 10](https://developer.nvidia.com/cuda-downloads?target_os=Windows&target_arch=x86_64&target_version=10&target_type=exelocal)
|
||||
- Base installer
|
||||
|
||||
- [MKL](https://software.intel.com/en-us/mkl)
|
||||
|
||||
|
||||
__Note: Patch for CUDA 9.2 error: Unsupported Visual Studio Version Error__
|
||||
|
||||
This seems to work fine with CUDA 10.0.
|
||||
|
||||
When using CUDA 9.2, the latest versions of Visual Studio 2017 are not officially supported by CUDA. Two fixes are proposed:
|
||||
|
||||
- Downgrade Visual Studio to a supported version
|
||||
|
||||
- Edit the file `<CUDA install path>\include\crt\host_config.h` and change the line 131:
|
||||
|
||||
131 #if _MSC_VER < 1600 || _MSC_VER > 1914
|
||||
|
||||
into:
|
||||
|
||||
131 #if _MSC_VER < 1600 || _MSC_VER > 1915
|
||||
|
||||
For more information, read this [nVidia forum](https://devtalk.nvidia.com/default/topic/1022648/cuda-setup-and-installation/cuda-9-unsupported-visual-studio-version-error/4)
|
||||
- [Cuda 10](https://developer.nvidia.com/cuda-downloads?target_os=Windows&target_arch=x86_64&target_version=10&target_type=exelocal),
|
||||
Base installer, CUDA 10.0+ is recommended, there might be issues with CUDA 9.2, see below
|
||||
- [Intel MKL](https://software.intel.com/en-us/mkl)
|
||||
|
||||
---
|
||||
## Check dependencies : `CheckDeps.bat`
|
||||
|
||||
In addition to the 2 previous prerequisites, Marian needs 2 libraries that you may already have on your system:
|
||||
In addition to the 2 previous prerequisites, Marian may need the following libraries that you may
|
||||
already have on your system:
|
||||
|
||||
- Boost (1.58+)
|
||||
- OpenSSL (optional for server)
|
||||
- Boost (1.58-1.72), optional for marian-server (`COMPILE_SERVER=TRUE` in CMake)
|
||||
- OpenSSL, optional for marian-server
|
||||
- Protobuf, optional for compiling with SentencePiece (`USE_SENTENCEPIECE=TRUE` in CMake),
|
||||
recommended
|
||||
|
||||
The script `CheckDeps.bat` can be used to verify that all dependencies are found on your system. If not, it will use the `vcpkg` library manager to download and manage your dependencies for CMake.
|
||||
The script `CheckDeps.bat` can be used to verify that all dependencies are found on your system. If
|
||||
not, it will use the `vcpkg` library manager to download and manage your dependencies for CMake.
|
||||
|
||||
If you already have a working `vcpkg` installation, this script can use it:
|
||||
- If vcpkg is in your `PATH` environment variable, the script will find it and use it automatically.
|
||||
- Otherwise, you need to edit the script and set the `VCPKG_ROOT` variable to the directory that contains the vcpkg.exe
|
||||
|
||||
|
||||
If you prefer to manage yourself the dependencies, you can edit the script file to set the following variables to the respective installation paths. These variable can also be already set in your environment.
|
||||
- `BOOST_INCLUDE_PATH` and `BOOST_LIB_PATH`
|
||||
- `OPENSSL_PATH`
|
||||
If you already have a working `vcpkg` installation, this script can use it.
|
||||
If vcpkg is in your `PATH` environment variable, the script will find it and use it automatically.
|
||||
Otherwise, you need to edit the script and set the `VCPKG_ROOT` variable.
|
||||
Please see the script for more details.
|
||||
|
||||
---
|
||||
## Build the project
|
||||
|
||||
There are 3 alternatives to build the project:
|
||||
1. Use Visual Studio 2017 built-in support for CMake
|
||||
1. Use Visual Studio 2017+ built-in support for CMake
|
||||
2. Create a Solution file for Visual Studio
|
||||
3. Use a script (MSBuild)
|
||||
|
||||
### 1. Use VS2017 with built-in support for CMake
|
||||
|
||||
VS2017 now allows to develop projects built with CMake without the need to generate VS projects and solutions. For more information, please read [this article](https://blogs.msdn.microsoft.com/vcblog/2016/10/05/cmake-support-in-visual-studio/) from the Visual C++ Team.
|
||||
### 1. Use VS2017+ with built-in support for CMake
|
||||
|
||||
VS2017 or newer now allows to develop projects built with CMake without the need to generate VS
|
||||
projects and solutions. For more information, please read [this article](https://blogs.msdn.microsoft.com/vcblog/2016/10/05/cmake-support-in-visual-studio/)
|
||||
from the Visual C++ Team.
|
||||
|
||||
You just need to open the root folder of the git repository in VS (which contains the file
|
||||
`CMakeSettings.json`):
|
||||
|
||||
You just need to open the root folder of the git repository in VS (which contains the file `CMakeSettings.json`):
|
||||
- In an Explorer window, right-click then `Open in Visual Studio`
|
||||
- In a VS2017 instance, `File > Open > Folder...`
|
||||
|
||||
You may need to edit the file `CMakeSettings.json` to set the environment variable for the dependencies.
|
||||
You may need to edit the file `CMakeSettings.json` to set the environment variable for the
|
||||
dependencies.
|
||||
|
||||
The developing experience is very similar than when using a solution file (Intellisense, build project with `F7`, debug, set breakpoints and watch variables, ...), except that the project configuration is done in 3 different files:
|
||||
The developing experience is very similar that when using a solution file (Intellisense, build
|
||||
project with `F7`, debug, set breakpoints and watch variables, ...), except that the project
|
||||
configuration is done in 3 different files:
|
||||
|
||||
- `CMakeList.txt`: this is the CMake source file from the original project.
|
||||
It is used to configure the build targets, add/remove files to compile and configure the compiler flags.
|
||||
- `CMakeList.txt`: this is the CMake source file from the original project.
|
||||
It is used to configure the build targets, add/remove files to compile and configure the
|
||||
compiler flags.
|
||||
|
||||
- `CMakeSettings.json`: this file is required to enable CMake integration in VS2017.
|
||||
Use this file to configure the environment variables and the parameters passed to CMake to generate the project.
|
||||
|
||||
- `.vs\launch.vs.json`: this is a user specific file and it is not commited in the Git repo
|
||||
Use this file to configure the debugging targets.
|
||||
- `CMakeSettings.json`: this file is required to enable CMake integration in VS2017.
|
||||
Use this file to configure the environment variables and the parameters passed to CMake to
|
||||
generate the project.
|
||||
|
||||
- `.vs\launch.vs.json`: this is a user specific file and it is not commited in the Git repo.
|
||||
Use this file to configure the debugging targets.
|
||||
For example:
|
||||
|
||||
{
|
||||
@ -116,8 +107,8 @@ The developing experience is very similar than when using a solution file (Intel
|
||||
"--keep-best",
|
||||
"--seed 1111",
|
||||
"--exponential-smoothing",
|
||||
"--normalize=1",
|
||||
"--beam-size=12",
|
||||
"--normalize 1",
|
||||
"--beam-size 12",
|
||||
"--quiet-translation"
|
||||
]
|
||||
}
|
||||
@ -125,12 +116,13 @@ The developing experience is very similar than when using a solution file (Intel
|
||||
}
|
||||
|
||||
|
||||
|
||||
### 2. Create solution and projects files for Visual Studio : `CreateVSProjects.bat`
|
||||
|
||||
If you have a previous version of Visual Studio, you will need to use CMake to generate the projects files.
|
||||
If you have a previous version of Visual Studio, you will need to use CMake to generate the projects
|
||||
files.
|
||||
|
||||
The provided script `CreateVSProjects.bat` runs the dependency checks then invokes CMake with the right parameters to create the solutions for Visual Studio.
|
||||
The provided script `CreateVSProjects.bat` runs the dependency checks then invokes CMake with the
|
||||
right parameters to create the solutions for Visual Studio.
|
||||
|
||||
|
||||
### 3. Use MSBuild : `BuildRelease.bat`
|
||||
@ -140,61 +132,31 @@ The last alternative is to use the script `BuildRelease.bat` that will:
|
||||
- Create the VS project files
|
||||
- Invoke MSBuild on these projects to build the targets in Release.
|
||||
|
||||
<!--
|
||||
This is interesting for developers, hiding away from users.
|
||||
|
||||
---
|
||||
## Changes from the master branch
|
||||
This part gives more information on all changes done in this PR. Refer to [this page](https://github.com/cedrou/marian-dev/commits/build_on_win) for commits.
|
||||
## Known issues
|
||||
|
||||
1. __Fix Cuda error : Unsupported Visual Studio Version Error__
|
||||
See above for justification and fixes
|
||||
1. __Patch for CUDA 9.2 error: Unsupported Visual Studio Version Error__
|
||||
|
||||
2. __Fix VS compiler flags / Build in Release, with improved debug info__
|
||||
Added VS specific compile and link flags
|
||||
When using CUDA 9.2, the latest versions of Visual Studio 2017 are not officially supported by
|
||||
CUDA. Two fixes are proposed:
|
||||
- Downgrade Visual Studio to a supported version
|
||||
- Edit the file `<CUDA install path>\include\crt\host_config.h` and change the line 131:
|
||||
|
||||
3. __Fix Warning: D9002: ignoring unknown option '-m64'__
|
||||
This one is related to a compiler flag added while finding the package MKL that does not exists for MS compiler.
|
||||
131 #if _MSC_VER < 1600 || _MSC_VER > 1914
|
||||
|
||||
4. __Fix marian::Backend, marian::cpu::Backend and marian::gpu::Backend conflicts__
|
||||
There were name conflicts between the 3 `Backend` classes that confused the compiler:
|
||||
|
||||
> template instantiation resulted in unexpected function type of "void(Ptr\<marian::gpu::Backend\> backend, [...])" (the meaning of a name may have changed since the template declaration -- the type of the template is "void(Ptr\<marian::Backend\> backend, [...]").
|
||||
into:
|
||||
|
||||
To solve this, I changed the declaration of 3 methods to specify the full name with namespace (`marian::Backend`, instead of `Backend`).
|
||||
131 #if _MSC_VER < 1600 || _MSC_VER > 1915
|
||||
|
||||
5. __Fix error : identifier "CUDA_FLT_MAX" is undefined in device code__
|
||||
`CUDA_FLT_MAX` is not seen by CUDA from the device code and I had to declare it as `__constant__`.
|
||||
For more information, read this [nVidia forum](https://devtalk.nvidia.com/default/topic/1022648/cuda-setup-and-installation/cuda-9-unsupported-visual-studio-version-error/4)
|
||||
|
||||
From [StackOverflow](https://stackoverflow.com/questions/20111409/how-to-pass-structures-into-cuda-device#comment29972423_20112013):
|
||||
> Undecorated constants get compiled into both host and device code with gcc based toolchains, but not with the Microsoft compiler.
|
||||
2. __It does not compile with Boost 1.73 or newer__
|
||||
|
||||
6. __Fix fatal error C1019: unexpected #else__
|
||||
There was preprocessor instructions (`#ifdef ... #else ... #endif`) in the middle of a call of a macro function (`CUDNN_CALL`), which is not allowed with MS compiler.
|
||||
At the moment (version 1.9.26) SimpleWebSocketServer, a 3rd party library that Marian uses for
|
||||
marian-server, does not support Boost newer than 1.72. Since vcpkg does not allow installing a
|
||||
specific library versions, you need to revert `ports/boost*` directories to install older Boost.
|
||||
See `CheckDeps.bat` for an example.
|
||||
|
||||
7. __Fix mismatched class/struct forward declarations__
|
||||
Microsoft's C++ name mangling makes a distinction between `class` and `struct` objects, so definitions and forward declaration must match.
|
||||
See [this pdf](https://www.agner.org/optimize/calling_conventions.pdf), page 27, for more information.
|
||||
Note that Boost is required only if you compile with marian-server, for compilation using CMake,
|
||||
it is if you set `COMPILE_SERVER` to `TRUE` in CMakeSettings.json.
|
||||
|
||||
_Note_: This fix was invalidated by commit # from @frankseide
|
||||
|
||||
8. __Fix unresolved external due to a removed #include directive__
|
||||
There was an include directive removed from MSVC compilation, but this prevented the build of the project.
|
||||
I'm not sure why this was removed; the comment is:
|
||||
|
||||
#ifndef _WIN32 // TODO: remove this once I updated the Linux-side makefile
|
||||
|
||||
9. __Fix CUDA+MSVC incompatibility with Boost.Preprocessor__
|
||||
The toolchain nvcc+msvc is not correctly handled in Boost.Preprocessor module. See [this issue](https://github.com/boostorg/preprocessor/issues/15). In the meantime, the recommended workaround is to disable Variadic Macro support in Boost.
|
||||
I created a [PR](https://github.com/boostorg/preprocessor/pull/18) in the Boost repo on GitHub to fix this.
|
||||
|
||||
_Note_: The library sources have been fixed, but this fix is still needed until the next release of Boost.Preprocessor
|
||||
|
||||
10. __Provide implementation for mkstemp / Fix temporary file creation__
|
||||
The code explicitely disabled the creation of temporary files because "mkstemp not available in Windows". In fact, `mktemp` and `unlink` are both implemented, but they don't work as expected. I used `tempnam` to replace `mkstemp`, and added the flag `_O_TEMPORARY` to the parameters of `open` to automatically delete the file when it is closed. If `unlinkEarly` is not set, I added a call to `remove` in the destructor to delete the file after its closure.
|
||||
I also handled the case of the default value for the `base` parameter: the path `\tmp` doesnot exist on Windows, so it is replaced by the value of the `%TMP%` environment variable in `NormalizeTempPrefix`.
|
||||
|
||||
11. __Revert commit #2f8b093 + Fix copy/paste error while fixing #301 + restrict fix to MSVC compiler.__
|
||||
cf [Issue #301](https://github.com/marian-nmt/marian-dev/issues/301) -->
|
||||
|
||||
|
||||
142
vs/pathspec-boost-1.72.txt
Executable file
142
vs/pathspec-boost-1.72.txt
Executable file
@ -0,0 +1,142 @@
|
||||
ports/boost/
|
||||
ports/boost-accumulators/
|
||||
ports/boost-algorithm/
|
||||
ports/boost-align/
|
||||
ports/boost-any/
|
||||
ports/boost-array/
|
||||
ports/boost-asio/
|
||||
ports/boost-assert/
|
||||
ports/boost-assign/
|
||||
ports/boost-atomic/
|
||||
ports/boost-beast/
|
||||
ports/boost-bimap/
|
||||
ports/boost-bind/
|
||||
ports/boost-build/
|
||||
ports/boost-callable-traits/
|
||||
ports/boost-chrono/
|
||||
ports/boost-circular-buffer/
|
||||
ports/boost-compatibility/
|
||||
ports/boost-compute/
|
||||
ports/boost-concept-check/
|
||||
ports/boost-config/
|
||||
ports/boost-container/
|
||||
ports/boost-container-hash/
|
||||
ports/boost-context/
|
||||
ports/boost-contract/
|
||||
ports/boost-conversion/
|
||||
ports/boost-convert/
|
||||
ports/boost-core/
|
||||
ports/boost-coroutine/
|
||||
ports/boost-coroutine2/
|
||||
ports/boost-crc/
|
||||
ports/boost-date-time/
|
||||
ports/boost-detail/
|
||||
ports/boost-di/
|
||||
ports/boost-dll/
|
||||
ports/boost-dynamic-bitset/
|
||||
ports/boost-endian/
|
||||
ports/boost-exception/
|
||||
ports/boost-fiber/
|
||||
ports/boost-filesystem/
|
||||
ports/boost-flyweight/
|
||||
ports/boost-foreach/
|
||||
ports/boost-format/
|
||||
ports/boost-function/
|
||||
ports/boost-functional/
|
||||
ports/boost-function-types/
|
||||
ports/boost-fusion/
|
||||
ports/boost-geometry/
|
||||
ports/boost-gil/
|
||||
ports/boost-graph/
|
||||
ports/boost-graph-parallel/
|
||||
ports/boost-hana/
|
||||
ports/boost-heap/
|
||||
ports/boost-histogram/
|
||||
ports/boost-hof/
|
||||
ports/boost-icl/
|
||||
ports/boost-integer/
|
||||
ports/boost-interprocess/
|
||||
ports/boost-interval/
|
||||
ports/boost-intrusive/
|
||||
ports/boost-io/
|
||||
ports/boost-iostreams/
|
||||
ports/boost-iterator/
|
||||
ports/boost-lambda/
|
||||
ports/boost-lexical-cast/
|
||||
ports/boost-locale/
|
||||
ports/boost-local-function/
|
||||
ports/boost-lockfree/
|
||||
ports/boost-log/
|
||||
ports/boost-logic/
|
||||
ports/boost-math/
|
||||
ports/boost-metaparse/
|
||||
ports/boost-modular-build-helper/
|
||||
ports/boost-move/
|
||||
ports/boost-mp11/
|
||||
ports/boost-mpi/
|
||||
ports/boost-mpl/
|
||||
ports/boost-msm/
|
||||
ports/boost-multi-array/
|
||||
ports/boost-multi-index/
|
||||
ports/boost-multiprecision/
|
||||
ports/boost-numeric-conversion/
|
||||
ports/boost-odeint/
|
||||
ports/boost-optional/
|
||||
ports/boost-outcome/
|
||||
ports/boost-parameter/
|
||||
ports/boost-parameter-python/
|
||||
ports/boost-phoenix/
|
||||
ports/boost-poly-collection/
|
||||
ports/boost-polygon/
|
||||
ports/boost-pool/
|
||||
ports/boost-predef/
|
||||
ports/boost-preprocessor/
|
||||
ports/boost-process/
|
||||
ports/boost-program-options/
|
||||
ports/boost-property-map/
|
||||
ports/boost-property-tree/
|
||||
ports/boost-proto/
|
||||
ports/boost-ptr-container/
|
||||
ports/boost-python/
|
||||
ports/boost-qvm/
|
||||
ports/boost-random/
|
||||
ports/boost-range/
|
||||
ports/boost-ratio/
|
||||
ports/boost-rational/
|
||||
ports/boost-regex/
|
||||
ports/boost-safe-numerics/
|
||||
ports/boost-scope-exit/
|
||||
ports/boost-serialization/
|
||||
ports/boost-signals/
|
||||
ports/boost-signals2/
|
||||
ports/boost-smart-ptr/
|
||||
ports/boost-sort/
|
||||
ports/boost-spirit/
|
||||
ports/boost-stacktrace/
|
||||
ports/boost-statechart/
|
||||
ports/boost-static-assert/
|
||||
ports/boost-system/
|
||||
ports/boost-test/
|
||||
ports/boost-thread/
|
||||
ports/boost-throw-exception/
|
||||
ports/boost-timer/
|
||||
ports/boost-tokenizer/
|
||||
ports/boost-tti/
|
||||
ports/boost-tuple/
|
||||
ports/boost-type-erasure/
|
||||
ports/boost-type-index/
|
||||
ports/boost-typeof/
|
||||
ports/boost-type-traits/
|
||||
ports/boost-ublas/
|
||||
ports/boost-units/
|
||||
ports/boost-unordered/
|
||||
ports/boost-utility/
|
||||
ports/boost-uuid/
|
||||
ports/boost-variant/
|
||||
ports/boost-variant2/
|
||||
ports/boost-vcpkg-helpers/
|
||||
ports/boost-vmd/
|
||||
ports/boost-wave/
|
||||
ports/boost-winapi/
|
||||
ports/boost-xpressive/
|
||||
ports/boost-yap/
|
||||
Loading…
Reference in New Issue
Block a user