mirror of
https://github.com/marian-nmt/marian.git
synced 2024-10-03 18:17:12 +03:00
Merged PR 18232: Update VS CMake builds and scripts
This PR updates Windows build using Visual Studio CMake compilation with Ninja. It does not affect standard VS compilation or Windows builds on Azure/GitHub CI. List of changes: - Fixed syntax in the script installing dependencies via vcpkg. - Removed installation of Protobuf (already included as a submodule) and Boost 1.72 (the previous solution no longer works with new vcpkg). - Disabled compilation of marian-server in the default setting due to Boost issues. - Disabled compilation of NCCL in the default setting due to an error (see comment in the code). - Updated vs/README.
This commit is contained in:
parent
272096c1d1
commit
326b9400e9
@ -17,10 +17,7 @@
|
||||
// Boost and OpenSSL are required if you compile with COMPILE_SERVER=TRUE
|
||||
"BOOST_INCLUDEDIR": "",
|
||||
"BOOST_LIBRARYDIR": "",
|
||||
"OPENSSL_ROOT_DIR": "",
|
||||
|
||||
// Protobuf is required if you compile with USE_SENTENCEPIECE=TRUE
|
||||
"Protobuf_SRC_ROOT_FOLDER": ""
|
||||
"OPENSSL_ROOT_DIR": ""
|
||||
}
|
||||
],
|
||||
"configurations": [
|
||||
@ -44,12 +41,13 @@
|
||||
{ "name": "COMPILE_CUDA:BOOL", "value": "TRUE" },
|
||||
{ "name": "COMPILE_CPU:BOOL", "value": "TRUE" },
|
||||
{ "name": "COMPILE_EXAMPLES:BOOL", "value": "FALSE" },
|
||||
{ "name": "COMPILE_SERVER:BOOL", "value": "TRUE" },
|
||||
{ "name": "COMPILE_SERVER:BOOL", "value": "FALSE" },
|
||||
{ "name": "COMPILE_TESTS:BOOL", "value": "FALSE" },
|
||||
|
||||
{ "name": "USE_CUDNN:BOOL", "value": "FALSE" },
|
||||
{ "name": "USE_FBGEMM:BOOL", "value": "TRUE" },
|
||||
{ "name": "USE_MPI:BOOL", "value": "FALSE" },
|
||||
{ "name": "USE_NCCL:BOOL", "value": "FALSE" },
|
||||
{ "name": "USE_SENTENCEPIECE:BOOL", "value": "TRUE" },
|
||||
{ "name": "USE_STATIC_LIBS:BOOL", "value": "TRUE" }
|
||||
]
|
||||
@ -74,12 +72,13 @@
|
||||
{ "name": "COMPILE_CUDA:BOOL", "value": "TRUE" },
|
||||
{ "name": "COMPILE_CPU:BOOL", "value": "TRUE" },
|
||||
{ "name": "COMPILE_EXAMPLES:BOOL", "value": "FALSE" },
|
||||
{ "name": "COMPILE_SERVER:BOOL", "value": "TRUE" },
|
||||
{ "name": "COMPILE_SERVER:BOOL", "value": "FALSE" },
|
||||
{ "name": "COMPILE_TESTS:BOOL", "value": "TRUE" },
|
||||
|
||||
{ "name": "USE_CUDNN:BOOL", "value": "FALSE" },
|
||||
{ "name": "USE_FBGEMM:BOOL", "value": "TRUE" },
|
||||
{ "name": "USE_MPI:BOOL", "value": "FALSE" },
|
||||
{ "name": "USE_NCCL:BOOL", "value": "FALSE" },
|
||||
{ "name": "USE_SENTENCEPIECE:BOOL", "value": "TRUE" },
|
||||
{ "name": "USE_STATIC_LIBS:BOOL", "value": "TRUE" }
|
||||
]
|
||||
|
4
src/3rd_party/CMakeLists.txt
vendored
4
src/3rd_party/CMakeLists.txt
vendored
@ -183,6 +183,10 @@ if(CUDA_FOUND)
|
||||
BINARY_DIR ${CMAKE_CURRENT_SOURCE_DIR}/nccl
|
||||
CONFIGURE_COMMAND ""
|
||||
BUILD_COMMAND
|
||||
# Note: $(MAKE) here causes CMake cache generation via Ninja failing (Windows build with
|
||||
# CMake only) because Ninja complains about unescaped $ signs. This happens only if
|
||||
# COMPILE_CUDA=on and because of that, USE_NCCL=off is used in CMakeSettings.json for now.
|
||||
# @TODO: find proper escaping that works for MSVC builds.
|
||||
"\$(MAKE)" -f ${CMAKE_CURRENT_SOURCE_DIR}/nccl/Makefile src.build
|
||||
BUILDDIR=${CMAKE_BINARY_DIR}/local CUDA_HOME=${CUDA_TOOLKIT_ROOT_DIR}
|
||||
CUDA8_GENCODE=${GENCODE} CXX=${CMAKE_CXX_COMPILER} CXX_FLAGS=${NCCL_FLAGS}
|
||||
|
@ -1,5 +1,5 @@
|
||||
::
|
||||
:: Usage: CheckDeps.bat
|
||||
:: Usage: CheckOrInstallDeps.bat
|
||||
::
|
||||
:: This script is used to verify that all the dependencies required to build Marian are available.
|
||||
:: The Cuda SDK and the Intel MKL must be installed beforehand by the user.
|
||||
@ -29,7 +29,6 @@ set ROOT=%~dp0
|
||||
::set BOOST_INCLUDEDIR=
|
||||
::set BOOST_LIBRARYDIR=
|
||||
::set OPENSSL_ROOT_DIR=
|
||||
::set Protobuf_SRC_ROOT_FOLDER=
|
||||
|
||||
|
||||
:: If all the variables are empty and vcpkg is found in a known path, the script will download and
|
||||
@ -45,7 +44,6 @@ set ROOT=%~dp0
|
||||
|
||||
if "%BOOST_INCLUDEDIR%" == "" goto :needVcPkg
|
||||
if "%OPENSSL_ROOT_DIR%" == "" goto :needVcPkg
|
||||
if "%Protobuf_SRC_ROOT_FOLDER%"=="" goto :needVcPkg
|
||||
|
||||
goto :checkDeps
|
||||
|
||||
@ -62,21 +60,15 @@ if "%VCPKG_ROOT%" == "" for /f "delims=" %%p in ('where vcpkg 2^>nul') do set VC
|
||||
if "%VCPKG_ROOT%" == "" set VCPKG_ROOT=%ROOT%deps\vcpkg
|
||||
|
||||
if not exist %VCPKG_ROOT% (
|
||||
|
||||
echo --- Cloning vcpkg...
|
||||
git clone https://github.com/Microsoft/vcpkg.git %VCPKG_ROOT%
|
||||
|
||||
set USE_BOOST_172=1
|
||||
set BOOTSTRAP_VCPKG=1
|
||||
|
||||
) else (
|
||||
|
||||
pushd %VCPKG_ROOT%
|
||||
|
||||
echo --- Updating vcpkg...
|
||||
for /f "delims=" %%p in ('git pull') do (
|
||||
if not "%%p" == "Already up to date." (
|
||||
set USE_BOOST_172=1
|
||||
set BOOTSTRAP_VCPKG=1
|
||||
)
|
||||
)
|
||||
@ -84,17 +76,6 @@ if not exist %VCPKG_ROOT% (
|
||||
popd
|
||||
)
|
||||
|
||||
:: Checkout to the last version of Boost that is supported by the 3rd party library
|
||||
:: SimpleWebSocketServer. Vcpkg does not allow installing a specific version of library yet, but
|
||||
:: the feature has a PR and should be available soon.
|
||||
:: For more details, see https://github.com/microsoft/vcpkg/issues/1681
|
||||
if "%USE_BOOST_172%"=="1" (
|
||||
pushd %VCPKG_ROOT%
|
||||
echo --- Checkout to Boost version 1.72...
|
||||
git checkout 597038559647776ee39d02dcf159da05d9342f1d --pathspec-from-file=../../pathspec-boost-1.72.txt
|
||||
popd
|
||||
)
|
||||
|
||||
if "%BOOTSTRAP_VCPKG%"=="1" (
|
||||
pushd %VCPKG_ROOT%
|
||||
call bootstrap-vcpkg.bat
|
||||
@ -124,8 +105,7 @@ echo.
|
||||
echo ... CUDA
|
||||
if "%CUDA_PATH%"=="" (
|
||||
echo The CUDA_PATH environment variable is not defined: this will compile only the CPU version.
|
||||
)
|
||||
else (
|
||||
) else (
|
||||
echo Found Cuda SDK in %CUDA_PATH%
|
||||
)
|
||||
|
||||
@ -211,17 +191,6 @@ if "%OPENSSL_ROOT_DIR%"=="" (
|
||||
set OPENSSL_ROOT_DIR=%VCPKG_INSTALL%
|
||||
)
|
||||
|
||||
:: -------------------------
|
||||
:: Protobuf_SRC_ROOT_FOLDER can be set to an existing Protobuf installation.
|
||||
:: If not, we use vcpkg to install the library
|
||||
::
|
||||
echo.
|
||||
echo ... Protobuf
|
||||
if "%Protobuf_SRC_ROOT_FOLDER%"=="" (
|
||||
%VCPKG% install protobuf
|
||||
set Protobuf_SRC_ROOT_FOLDER=%VCPKG_INSTALL%
|
||||
)
|
||||
|
||||
set CMAKE_PREFIX_PATH=%VCPKG_INSTALL%
|
||||
|
||||
echo.
|
||||
@ -233,7 +202,6 @@ echo CMAKE_PREFIX_PATH ^| %CMAKE_PREFIX_PATH%
|
||||
echo CUDA_PATH ^| %CUDA_PATH%
|
||||
echo MKLROOT ^| %MKLROOT%
|
||||
echo OPENSSL_ROOT_DIR ^| %OPENSSL_ROOT_DIR%
|
||||
echo Protobuf_SRC_ROOT_FOLDER ^| %Protobuf_SRC_ROOT_FOLDER%
|
||||
echo VCPKG_ROOT ^| %VCPKG_ROOT%
|
||||
echo --------------------------------------------------
|
||||
echo.
|
@ -25,7 +25,7 @@ if "%BUILD_ROOT%"=="" set BUILD_ROOT=%ROOT%build
|
||||
|
||||
set GENERATOR_TARGET="Visual Studio 15 2017 Win64"
|
||||
|
||||
call CheckDeps.bat
|
||||
call CheckOrInstallDeps.bat
|
||||
if errorlevel 1 exit /b 1
|
||||
|
||||
|
||||
@ -79,4 +79,4 @@ cmake %CMAKE_OPT% %MARIAN_ROOT%
|
||||
|
||||
popd
|
||||
|
||||
exit /b 0
|
||||
exit /b 0
|
||||
|
25
vs/README.md
25
vs/README.md
@ -8,23 +8,22 @@ installed. If only CUDA is installed but not MKL, a GPU-only version will be bui
|
||||
installed and not CUDA, only the CPU version will be built. So if you are interested in only one
|
||||
functionality, you can omit one of them. Install both for full functionality.
|
||||
|
||||
- [Cuda 10](https://developer.nvidia.com/cuda-downloads?target_os=Windows&target_arch=x86_64&target_version=10&target_type=exelocal),
|
||||
- [CUDA](https://developer.nvidia.com/cuda-downloads?target_os=Windows&target_arch=x86_64&target_version=10&target_type=exelocal),
|
||||
Base installer, CUDA 10.0+ is recommended, there might be issues with CUDA 9.2, see below
|
||||
- [Intel MKL](https://software.intel.com/en-us/mkl)
|
||||
|
||||
---
|
||||
## Check dependencies : `CheckDeps.bat`
|
||||
## Check dependencies : `CheckOrInstallDeps.bat`
|
||||
|
||||
In addition to the 2 previous prerequisites, Marian may need the following libraries that you may
|
||||
already have on your system:
|
||||
|
||||
- Boost (1.58-1.72), optional for marian-server (`COMPILE_SERVER=TRUE` in CMake)
|
||||
- OpenSSL, optional for marian-server
|
||||
- Protobuf, optional for compiling with SentencePiece (`USE_SENTENCEPIECE=TRUE` in CMake),
|
||||
recommended
|
||||
|
||||
The script `CheckDeps.bat` can be used to verify that all dependencies are found on your system. If
|
||||
not, it will use the `vcpkg` library manager to download and manage your dependencies for CMake.
|
||||
The script `CheckOrInstallDeps.bat` can be used to verify that all dependencies are found on your
|
||||
system. If not, it will use the `vcpkg` library manager to download and manage your dependencies for
|
||||
CMake.
|
||||
|
||||
If you already have a working `vcpkg` installation, this script can use it.
|
||||
If vcpkg is in your `PATH` environment variable, the script will find it and use it automatically.
|
||||
@ -67,9 +66,8 @@ configuration is done in 3 different files:
|
||||
Use this file to configure the environment variables and the parameters passed to CMake to
|
||||
generate the project.
|
||||
|
||||
- `.vs\launch.vs.json`: this is a user specific file and it is not commited in the Git repo.
|
||||
Use this file to configure the debugging targets.
|
||||
For example:
|
||||
- `.vs\launch.vs.json`: this is an optional user specific file and it is not commited in the Git
|
||||
repo. Use this file to configure the debugging targets. For example:
|
||||
|
||||
{
|
||||
"version": "0.2.1",
|
||||
@ -152,11 +150,10 @@ The last alternative is to use the script `BuildRelease.bat` that will:
|
||||
|
||||
2. __It does not compile with Boost 1.73 or newer__
|
||||
|
||||
At the moment (version 1.9.26) SimpleWebSocketServer, a 3rd party library that Marian uses for
|
||||
marian-server, does not support Boost newer than 1.72. Since vcpkg does not allow installing a
|
||||
specific library versions, you need to revert `ports/boost*` directories to install older Boost.
|
||||
See `CheckDeps.bat` for an example.
|
||||
It may happen that SimpleWebSocketServer, a 3rd party library that Marian uses for
|
||||
marian-server, does not support the version of Boost available in vcpkg. In such case install a
|
||||
supported version of Boost; if you use vcpkg, an option is to checkout to #5970385, which has
|
||||
Boost 1.72.
|
||||
|
||||
Note that Boost is required only if you compile with marian-server, for compilation using CMake,
|
||||
it is if you set `COMPILE_SERVER` to `TRUE` in CMakeSettings.json.
|
||||
|
||||
|
@ -1,142 +0,0 @@
|
||||
ports/boost/
|
||||
ports/boost-accumulators/
|
||||
ports/boost-algorithm/
|
||||
ports/boost-align/
|
||||
ports/boost-any/
|
||||
ports/boost-array/
|
||||
ports/boost-asio/
|
||||
ports/boost-assert/
|
||||
ports/boost-assign/
|
||||
ports/boost-atomic/
|
||||
ports/boost-beast/
|
||||
ports/boost-bimap/
|
||||
ports/boost-bind/
|
||||
ports/boost-build/
|
||||
ports/boost-callable-traits/
|
||||
ports/boost-chrono/
|
||||
ports/boost-circular-buffer/
|
||||
ports/boost-compatibility/
|
||||
ports/boost-compute/
|
||||
ports/boost-concept-check/
|
||||
ports/boost-config/
|
||||
ports/boost-container/
|
||||
ports/boost-container-hash/
|
||||
ports/boost-context/
|
||||
ports/boost-contract/
|
||||
ports/boost-conversion/
|
||||
ports/boost-convert/
|
||||
ports/boost-core/
|
||||
ports/boost-coroutine/
|
||||
ports/boost-coroutine2/
|
||||
ports/boost-crc/
|
||||
ports/boost-date-time/
|
||||
ports/boost-detail/
|
||||
ports/boost-di/
|
||||
ports/boost-dll/
|
||||
ports/boost-dynamic-bitset/
|
||||
ports/boost-endian/
|
||||
ports/boost-exception/
|
||||
ports/boost-fiber/
|
||||
ports/boost-filesystem/
|
||||
ports/boost-flyweight/
|
||||
ports/boost-foreach/
|
||||
ports/boost-format/
|
||||
ports/boost-function/
|
||||
ports/boost-functional/
|
||||
ports/boost-function-types/
|
||||
ports/boost-fusion/
|
||||
ports/boost-geometry/
|
||||
ports/boost-gil/
|
||||
ports/boost-graph/
|
||||
ports/boost-graph-parallel/
|
||||
ports/boost-hana/
|
||||
ports/boost-heap/
|
||||
ports/boost-histogram/
|
||||
ports/boost-hof/
|
||||
ports/boost-icl/
|
||||
ports/boost-integer/
|
||||
ports/boost-interprocess/
|
||||
ports/boost-interval/
|
||||
ports/boost-intrusive/
|
||||
ports/boost-io/
|
||||
ports/boost-iostreams/
|
||||
ports/boost-iterator/
|
||||
ports/boost-lambda/
|
||||
ports/boost-lexical-cast/
|
||||
ports/boost-locale/
|
||||
ports/boost-local-function/
|
||||
ports/boost-lockfree/
|
||||
ports/boost-log/
|
||||
ports/boost-logic/
|
||||
ports/boost-math/
|
||||
ports/boost-metaparse/
|
||||
ports/boost-modular-build-helper/
|
||||
ports/boost-move/
|
||||
ports/boost-mp11/
|
||||
ports/boost-mpi/
|
||||
ports/boost-mpl/
|
||||
ports/boost-msm/
|
||||
ports/boost-multi-array/
|
||||
ports/boost-multi-index/
|
||||
ports/boost-multiprecision/
|
||||
ports/boost-numeric-conversion/
|
||||
ports/boost-odeint/
|
||||
ports/boost-optional/
|
||||
ports/boost-outcome/
|
||||
ports/boost-parameter/
|
||||
ports/boost-parameter-python/
|
||||
ports/boost-phoenix/
|
||||
ports/boost-poly-collection/
|
||||
ports/boost-polygon/
|
||||
ports/boost-pool/
|
||||
ports/boost-predef/
|
||||
ports/boost-preprocessor/
|
||||
ports/boost-process/
|
||||
ports/boost-program-options/
|
||||
ports/boost-property-map/
|
||||
ports/boost-property-tree/
|
||||
ports/boost-proto/
|
||||
ports/boost-ptr-container/
|
||||
ports/boost-python/
|
||||
ports/boost-qvm/
|
||||
ports/boost-random/
|
||||
ports/boost-range/
|
||||
ports/boost-ratio/
|
||||
ports/boost-rational/
|
||||
ports/boost-regex/
|
||||
ports/boost-safe-numerics/
|
||||
ports/boost-scope-exit/
|
||||
ports/boost-serialization/
|
||||
ports/boost-signals/
|
||||
ports/boost-signals2/
|
||||
ports/boost-smart-ptr/
|
||||
ports/boost-sort/
|
||||
ports/boost-spirit/
|
||||
ports/boost-stacktrace/
|
||||
ports/boost-statechart/
|
||||
ports/boost-static-assert/
|
||||
ports/boost-system/
|
||||
ports/boost-test/
|
||||
ports/boost-thread/
|
||||
ports/boost-throw-exception/
|
||||
ports/boost-timer/
|
||||
ports/boost-tokenizer/
|
||||
ports/boost-tti/
|
||||
ports/boost-tuple/
|
||||
ports/boost-type-erasure/
|
||||
ports/boost-type-index/
|
||||
ports/boost-typeof/
|
||||
ports/boost-type-traits/
|
||||
ports/boost-ublas/
|
||||
ports/boost-units/
|
||||
ports/boost-unordered/
|
||||
ports/boost-utility/
|
||||
ports/boost-uuid/
|
||||
ports/boost-variant/
|
||||
ports/boost-variant2/
|
||||
ports/boost-vcpkg-helpers/
|
||||
ports/boost-vmd/
|
||||
ports/boost-wave/
|
||||
ports/boost-winapi/
|
||||
ports/boost-xpressive/
|
||||
ports/boost-yap/
|
Loading…
Reference in New Issue
Block a user