marian-nmt/marian
1
1
Fork 0
mirror of https://github.com/marian-nmt/marian.git synced 2024-09-17 09:47:34 +03:00
Code Issues Packages Projects Releases Wiki Activity

use ExternalProject for installation

Browse Source
This commit is contained in:
Marcin Junczys-Dowmunt 2019-02-04 23:07:00 -08:00
parent 10361f81d3
commit 4ab941b909
3 changed files with 42 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
CMakeLists.txt
View File

@ -32,7 +32,7 @@ message(STATUS "Project version: ${PROJECT_VERSION_STRING_FULL}")
execute_process(COMMAND git submodule update --init --recursive --no-fetch
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
# Set compilation flags
if(MSVC)
# These are used in src/CMakeLists.txt on a per-target basis
@ -42,7 +42,7 @@ if(MSVC)
# C4310: cast truncates constant value
# C4324: 'marian::cpu::int16::`anonymous-namespace'::ScatterPut': structure was padded due to alignment specifier
set(DISABLE_GLOBALLY "/wd\"4310\" /wd\"4324\"")
set(INTRINSICS "/arch:AVX")
# Or maybe use these?
@ -57,7 +57,7 @@ if(MSVC)
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /DEBUG /LTCG:incremental /INCREMENTAL:NO /NODEFAULTLIB:MSVCRT /ignore:4049")
set(CMAKE_STATIC_LINKER_FLAGS "${CMAKE_STATIC_LINKER_FLAGS} /LTCG:incremental")
find_library(SHLWAPI Shlwapi.lib)
find_library(SHLWAPI Shlwapi.lib)
set(EXT_LIBS ${EXT_LIBS} SHLWAPI)
else()
@ -172,37 +172,10 @@ endif(USE_STATIC_LIBS)
# Cmake. This is also fairly untested, let's hope it does not explode.
# @TODO: Make sure it does not use pre-installed NCCL headers
if(USE_NCCL)
# define and set the include dir for the generated nccl.h header
set(NCCL_HEADER_LOCATION "${CMAKE_CURRENT_BINARY_DIR}/nccl/include")
include_directories(${NCCL_HEADER_LOCATION})
# set the path for the generated static lib
set(NCCL_LIB_STATIC "${CMAKE_CURRENT_BINARY_DIR}/nccl/lib/libnccl_static.a")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DUSE_NCCL")
LIST(APPEND CUDA_NVCC_FLAGS -DUSE_NCCL; )
# disables compilation for sm_30 to avoid ptxas warning... that's general Kepler support. But K80s are supported for instance by sm_35
set(GENCODE "-gencode=arch=compute_35,code=sm_35 -gencode=arch=compute_50,code=sm_50 -gencode=arch=compute_60,code=sm_60 -gencode=arch=compute_61,code=sm_61")
# We build using NVidia's custom makefile, for that we pass a number of variables from CMake.
# Sets output to the chosen build folder, i.e. where the binaries and objects are generated.
# Also passes CUDA location from FindCUDA, sets c++ compiler to the same one CMake uses.
add_custom_command(OUTPUT ${NCCL_LIB_STATIC}
COMMAND ${CMAKE_MAKE_PROGRAM} src.build
BUILDDIR=${CMAKE_CURRENT_BINARY_DIR}/nccl
CUDA_HOME=${CUDA_TOOLKIT_ROOT_DIR}
CUDA8_GENCODE=${GENCODE}
CXX=${CMAKE_CXX_COMPILER}
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src/3rd_party/nccl)
add_custom_target(nccl_target DEPENDS ${NCCL_LIB_STATIC})
add_library(nccl STATIC IMPORTED)
set_target_properties(nccl PROPERTIES IMPORTED_LOCATION ${NCCL_LIB_STATIC})
add_dependencies(nccl nccl_target)
set(EXT_LIBS ${EXT_LIBS} nccl)
# adds the resulting files to be removed by `make clean`
set_directory_properties(PROPERTY ADDITIONAL_MAKE_CLEAN_FILES ${CMAKE_CURRENT_BINARY_DIR}/nccl)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DUSE_NCCL")
LIST(APPEND CUDA_NVCC_FLAGS -DUSE_NCCL; )
endif(USE_NCCL)
if(USE_STATIC_LIBS)

32
src/3rd_party/CMakeLists.txt vendored
View File

@ -38,3 +38,35 @@ include_directories(./pathie-cpp/include)
include_directories(./zlib)
include(ExternalProject)
set(INSTALLS "") # this will contain a list of 3rd part dependencies that we install locally
if(CUDA_FOUND)
if(USE_NCCL)
# disables compilation for sm_30 to avoid ptxas warning... that's general Kepler support. But K80s are supported for instance by sm_35
set(GENCODE "-gencode=arch=compute_35,code=sm_35 -gencode=arch=compute_50,code=sm_50 -gencode=arch=compute_60,code=sm_60 -gencode=arch=compute_61,code=sm_61")
# install nccl in ${CMAKE_BINARY_DIR}/local similar to /usr/local linux installation
ExternalProject_Add(nccl_install
SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/nccl
BINARY_DIR ${CMAKE_CURRENT_SOURCE_DIR}/nccl
CONFIGURE_COMMAND ""
BUILD_COMMAND
$(MAKE) -f ${CMAKE_CURRENT_SOURCE_DIR}/nccl/Makefile src.build
BUILDDIR=${CMAKE_BINARY_DIR}/local CUDA_HOME=${CUDA_TOOLKIT_ROOT_DIR}
CUDA8_GENCODE=${GENCODE} CXX=${CMAKE_CXX_COMPILER}
INSTALL_COMMAND "")
set_target_properties(nccl PROPERTIES IMPORTED_LOCATION ${CMAKE_BINARY_DIR}/local/lib/libnccl_static.a)
add_dependencies(nccl nccl_install)
set(INSTALLS ${INSTALLS} nccl_install)
endif(USE_NCCL)
endif(CUDA_FOUND)
# @TODO: do the same for SentencePiece, Protobuf etc.
# make clean will clean "${CMAKE_BINARY_DIR}/local"
set_directory_properties(PROPERTY ADDITIONAL_MAKE_CLEAN_FILES ${CMAKE_BINARY_DIR}/local)
add_custom_target(3rd_party_installs DEPENDS ${INSTALLS})

5
src/CMakeLists.txt
View File

@ -4,6 +4,7 @@ include_directories(.)
include_directories(3rd_party)
include_directories(3rd_party/SQLiteCpp/include)
include_directories(3rd_party/sentencepiece)
include_directories(${CMAKE_BINARY_DIR}/local/include)
add_library(marian STATIC
common/version.cpp
@ -98,6 +99,8 @@ add_custom_command(OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/common/git_revision.h
)
add_custom_target(marian_version DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/common/git_revision.h)
add_dependencies(marian marian_version) # marian must depend on it so that it gets created first
# make sure all local dependencies are installed first before this is built
add_dependencies(marian 3rd_party_installs)
if(CUDA_FOUND)
cuda_add_library(marian_cuda
@ -115,6 +118,8 @@ cuda_add_library(marian_cuda
STATIC)
target_compile_options(marian_cuda PUBLIC ${ALL_WARNINGS})
# make sure all local dependencies are installed first before this is built
add_dependencies(marian_cuda 3rd_party_installs)
endif(CUDA_FOUND)
set_target_properties(marian PROPERTIES LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}")