marian-nmt/marian
1
1
Fork 0
mirror of https://github.com/marian-nmt/marian.git synced 2024-11-03 20:13:47 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merged PR 14474: CMake build fixes for QuickSAND

Browse Source 
- Add installation targets (enabled by GENERATE_MARIAN_INSTALL_TARGETS; default: OFF to preserve CMake 3.5.1 compatibility)
- Add COMPILE_LIBRARY_ONLY option (default: OFF) to exclude in-source executables from the build
- Compiler warning flags are no longer exported as part of the public link interface, only when building privately
- Always set CPUINFO_BUILD_TOOLS=OFF when building fbgemm, not just for MSVC builds

Related work items: #108034
This commit is contained in:
Aaron Burke 2020-09-10 01:33:44 +00:00 committed by Martin Junczys-Dowmunt
parent e71e7e2469
commit 5c45a37fcc
5 changed files with 178 additions and 110 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
CMakeLists.txt
View File

@ -25,16 +25,36 @@ option(USE_MPI "Use MPI library" OFF)
option(USE_NCCL "Use NCCL library" ON)
option(USE_SENTENCEPIECE "Download and compile SentencePiece" OFF)
option(USE_STATIC_LIBS "Link statically against non-system libs" OFF)
option(GENERATE_MARIAN_INSTALL_TARGETS "Generate Marian install targets (requires CMake 3.12+)" OFF)
# fbgemm and sentencepiece are both defined with "non-local" installation targets (the source projects don't define them,
# so we define them in src\3rd_party\CMakeLists.txt), but that isn't supported until CMake 3.12. Prior to CMake 3.12,
# targets could only be install(...)ed in the same CMakeLists.txt they were defined. We currently target CMake 3.5.1
# as our minimum supported CMake version, so this option exists to provide compatibility by disabling install targets.
if(GENERATE_MARIAN_INSTALL_TARGETS AND ${CMAKE_VERSION} VERSION_LESS "3.12")
message(WARNING "Marian install targets cannot be generated on CMake <3.12.\
Please upgrade your CMake version or set GENERATE_MARIAN_INSTALL_TARGETS=OFF to remove this warning. Disabling installation targets.")
set(GENERATE_MARIAN_INSTALL_TARGETS OFF CACHE BOOL "Forcing disabled installation targets due to CMake <3.12." FORCE)
endif()
if(GENERATE_MARIAN_INSTALL_TARGETS)
include(GNUInstallDirs) # This defines default values for installation directories (all platforms even if named GNU)
include(InstallRequiredSystemLibraries) # Tell CMake that the `install` target needs to install required system libraries (eg: Windows SDK)
include(CMakePackageConfigHelpers) # Helper to create relocatable packages
install(EXPORT marian-targets # Installation target
DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake)
endif(GENERATE_MARIAN_INSTALL_TARGETS)
# use ccache (https://ccache.dev) for faster compilation if requested and available
if(USE_CCACHE)
find_program(CCACHE_PROGRAM ccache)
if(CCACHE_PROGRAM)
message(STATUS "Will be using ccache for faster repeat compilation (use cmake -DUSE_CCACHE=off to disable).")
set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE "${CCACHE_PROGRAM}")
else(CCACHE_PROGRAM)
message(WARNING "Compilation with ccache requested but no ccache found.")
endif(CCACHE_PROGRAM)
find_program(CCACHE_PROGRAM ccache)
if(CCACHE_PROGRAM)
message(STATUS "Will be using ccache for faster repeat compilation (use cmake -DUSE_CCACHE=off to disable).")
set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE "${CCACHE_PROGRAM}")
else(CCACHE_PROGRAM)
message(WARNING "Compilation with ccache requested but no ccache found.")
endif(CCACHE_PROGRAM)
endif(USE_CCACHE)
# Project versioning

24
src/3rd_party/CMakeLists.txt vendored
View File

@ -19,11 +19,11 @@ if(USE_FBGEMM)
# only locally disabled for the 3rd_party folder
# set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-value -Wno-unused-parameter -Wno-unused-variable -Wno-unused-function")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused")
else()
# Do not compile cpuinfo executables due to a linker error, and they are not needed
set(CPUINFO_BUILD_TOOLS OFF CACHE BOOL "Build command-line tools")
endif()
# Do not compile cpuinfo executables due to a linker error, and they are not needed
set(CPUINFO_BUILD_TOOLS OFF CACHE BOOL "Build command-line tools")
set(FBGEMM_BUILD_TESTS OFF CACHE BOOL "Disable fbgemm tests")
set(FBGEMM_BUILD_BENCHMARKS OFF CACHE BOOL "Disable fbgemm benchmark")
add_subdirectory(./fbgemm)
@ -36,6 +36,12 @@ if(USE_FBGEMM)
set_property(TARGET asmjit PROPERTY COMPILE_OPTIONS ${ASMJIT_COMPILE_OPTIONS})
message(" ASMJIT COMPILE FLAGS: ${ASMJIT_COMPILE_OPTIONS}")
if(GENERATE_MARIAN_INSTALL_TARGETS)
install(TARGETS fbgemm asmjit cpuinfo clog
EXPORT marian-targets
DESTINATION fbgemm)
endif(GENERATE_MARIAN_INSTALL_TARGETS)
endif(USE_FBGEMM)
if(USE_SENTENCEPIECE)
@ -83,6 +89,18 @@ if(USE_SENTENCEPIECE)
if(USE_STATIC_LIBS)
set(CMAKE_FIND_LIBRARY_SUFFIXES ${_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES})
endif()
if(GENERATE_MARIAN_INSTALL_TARGETS)
if(USE_STATIC_LIBS)
install(TARGETS sentencepiece-static sentencepiece_train-static
EXPORT marian-targets
DESTINATION sentencepiece)
else()
install(TARGETS sentencepiece sentencepiece_train
EXPORT marian-targets
DESTINATION sentencepiece)
endif()
endif(GENERATE_MARIAN_INSTALL_TARGETS)
endif(USE_SENTENCEPIECE)
include_directories(./SQLiteCpp/include)

2
src/3rd_party/fbgemm vendored

@ -1 +1 @@
Subproject commit da28b0abb0e54f4ed6e1444309e23879018689d7
Subproject commit 055d2a099c829563aff1fffdeb4594ad8cfe5d99

2
src/3rd_party/sentencepiece vendored

@ -1 +1 @@
Subproject commit 1d33bb67c3b6b2a51d3c9ffd55f37725801da39d
Subproject commit c0a84a4ff8bdc200480e179d57ece43d4929d242

226
src/CMakeLists.txt
View File

@ -7,7 +7,7 @@ include_directories(3rd_party/sentencepiece)
include_directories(3rd_party/fbgemm/include)
include_directories(${CMAKE_BINARY_DIR}/local/include)
add_library(marian STATIC
set(MARIAN_SOURCES
common/aliases.cpp
common/fastopt.cpp
common/version.cpp
@ -112,7 +112,9 @@ add_library(marian STATIC
$<TARGET_OBJECTS:faiss>
)
target_compile_options(marian PUBLIC ${ALL_WARNINGS})
add_library(marian STATIC ${MARIAN_SOURCES})
target_compile_options(marian PRIVATE ${ALL_WARNINGS})
# Generate git_revision.h to reflect current git revision information
# [https://stackoverflow.com/questions/1435953/how-can-i-pass-git-sha1-to-compiler-as-definition-using-cmake]
@ -146,106 +148,125 @@ add_dependencies(marian marian_version) # marian must depend on it so that it ge
# make sure all local dependencies are installed first before this is built
add_dependencies(marian 3rd_party_installs)
if(CUDA_FOUND)
cuda_add_library(marian_cuda
tensors/gpu/device.cu
tensors/gpu/algorithm.cu
tensors/gpu/prod.cpp
tensors/gpu/topk.cu
tensors/gpu/element.cu
tensors/gpu/add.cu
tensors/gpu/add_all.cu
tensors/gpu/tensor_operators.cu
tensors/gpu/cudnn_wrappers.cu
translator/nth_element.cu
translator/helpers.cu
STATIC)
target_compile_options(marian_cuda PUBLIC ${ALL_WARNINGS})
# make sure all local dependencies are installed first before this is built
add_dependencies(marian_cuda 3rd_party_installs)
endif(CUDA_FOUND)
set_target_properties(marian PROPERTIES LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}")
set_target_properties(marian PROPERTIES ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}")
add_executable(marian_train command/marian_main.cpp)
set_target_properties(marian_train PROPERTIES OUTPUT_NAME marian)
target_compile_options(marian_train PUBLIC ${ALL_WARNINGS})
add_executable(marian_decoder command/marian_decoder.cpp)
set_target_properties(marian_decoder PROPERTIES OUTPUT_NAME marian-decoder)
target_compile_options(marian_decoder PUBLIC ${ALL_WARNINGS})
add_executable(marian_scorer command/marian_scorer.cpp)
set_target_properties(marian_scorer PROPERTIES OUTPUT_NAME marian-scorer)
target_compile_options(marian_scorer PUBLIC ${ALL_WARNINGS})
add_executable(marian_vocab command/marian_vocab.cpp)
set_target_properties(marian_vocab PROPERTIES OUTPUT_NAME marian-vocab)
target_compile_options(marian_vocab PUBLIC ${ALL_WARNINGS})
add_executable(marian_conv command/marian_conv.cpp)
set_target_properties(marian_conv PROPERTIES OUTPUT_NAME marian-conv)
target_compile_options(marian_conv PUBLIC ${ALL_WARNINGS})
set(EXECUTABLES ${EXECUTABLES} marian_train marian_decoder marian_scorer marian_vocab marian_conv)
# marian.zip and marian.tgz
# This combines marian, marian_decoder in a single ZIP or TAR file for
# execution in MSFT internal tools FLO and Philly.
# For Philly submission, we need statically-linked versions to deal with
# library dependencies, so this target is only enabled for static builds.
if(USE_STATIC_LIBS)
add_custom_command(
OUTPUT "${CMAKE_BINARY_DIR}/marian.zip"
COMMAND zip -v -0 -j "${CMAKE_BINARY_DIR}/marian.zip"
"${CMAKE_BINARY_DIR}/marian"
"${CMAKE_BINARY_DIR}/marian-decoder"
"${CMAKE_BINARY_DIR}/marian-scorer"
"${CMAKE_BINARY_DIR}/marian-vocab"
"${CMAKE_BINARY_DIR}/marian-conv"
DEPENDS marian_train marian_decoder marian_scorer marian_vocab marian_conv)
add_custom_target(marian_zip DEPENDS "${CMAKE_BINARY_DIR}/marian.zip")
add_custom_command(
OUTPUT "${CMAKE_BINARY_DIR}/marian.tgz"
COMMAND tar -cvvzf "${CMAKE_BINARY_DIR}/marian.tgz" -C "${CMAKE_BINARY_DIR}"
"marian"
"marian-decoder"
"marian-scorer"
"marian-vocab"
"marian-conv"
DEPENDS marian_train marian_decoder marian_scorer marian_vocab marian_conv)
add_custom_target(marian_tgz DEPENDS "${CMAKE_BINARY_DIR}/marian.tgz")
add_custom_target(philly DEPENDS marian_tgz marian_zip)
endif(USE_STATIC_LIBS)
if(COMPILE_SERVER)
add_executable(marian_server command/marian_server.cpp)
set_target_properties(marian_server PROPERTIES OUTPUT_NAME marian-server)
if(MSVC)
# Disable warnings from the SimpleWebSocketServer library needed for compilation of marian-server
target_compile_options(marian_server PUBLIC ${ALL_WARNINGS} /wd4267 /wd4244 /wd4456 /wd4458)
else(MSVC)
# -Wno-suggest-override disables warnings from Boost 1.69+
target_compile_options(marian_server PUBLIC ${ALL_WARNINGS} -Wno-suggest-override)
endif(MSVC)
set(EXECUTABLES ${EXECUTABLES} marian_server)
endif(COMPILE_SERVER)
if(APPLE) # This is a dependency of pathie but I can't seem to link it into that CMakeLists because we're not compiling it as a library.
set(EXT_LIBS ${EXT_LIBS} iconv)
endif()
foreach(exec ${EXECUTABLES})
# @TODO: consider adding MKL and other libs to the library rather than the executables if at all possible
target_link_libraries(${exec} marian ${EXT_LIBS} ${EXT_LIBS} ${CMAKE_THREAD_LIBS_INIT})
if(CUDA_FOUND)
target_link_libraries(${exec} marian marian_cuda ${EXT_LIBS} ${CMAKE_THREAD_LIBS_INIT})
endif(CUDA_FOUND)
set_target_properties(${exec} PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}")
endforeach(exec)
# Add external libs to the public interface of the library
target_link_libraries(marian ${EXT_LIBS} ${EXT_LIBS} ${CMAKE_THREAD_LIBS_INIT})
set_target_properties(marian PROPERTIES LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}")
set_target_properties(marian PROPERTIES ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}")
if(CUDA_FOUND)
cuda_add_library(marian_cuda
tensors/gpu/device.cu
tensors/gpu/algorithm.cu
tensors/gpu/prod.cpp
tensors/gpu/topk.cu
tensors/gpu/element.cu
tensors/gpu/add.cu
tensors/gpu/add_all.cu
tensors/gpu/tensor_operators.cu
tensors/gpu/cudnn_wrappers.cu
translator/nth_element.cu
translator/helpers.cu
STATIC)
target_compile_options(marian_cuda PRIVATE ${ALL_WARNINGS})
# make sure all local dependencies are installed first before this is built
add_dependencies(marian_cuda 3rd_party_installs)
# Add external libs to the public interface of the library
target_link_libraries(marian_cuda ${EXT_LIBS} ${CMAKE_THREAD_LIBS_INIT})
# Add the CUDA Marian dependency to the core Marian target
target_link_libraries(marian marian_cuda)
if(GENERATE_MARIAN_INSTALL_TARGETS)
install(TARGETS marian_cuda
EXPORT marian-targets
ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
)
endif(GENERATE_MARIAN_INSTALL_TARGETS)
endif(CUDA_FOUND)
# If this option is ON, only build the Marian library. This is useful if this project is included
# as a sub-project of another build system that is only interested in the Marian output library.
option(COMPILE_LIBRARY_ONLY "Build only the Marian library and exclude all executables." OFF)
if (NOT COMPILE_LIBRARY_ONLY)
add_executable(marian_train command/marian_main.cpp)
set_target_properties(marian_train PROPERTIES OUTPUT_NAME marian)
target_compile_options(marian_train PRIVATE ${ALL_WARNINGS})
add_executable(marian_decoder command/marian_decoder.cpp)
set_target_properties(marian_decoder PROPERTIES OUTPUT_NAME marian-decoder)
target_compile_options(marian_decoder PRIVATE ${ALL_WARNINGS})
add_executable(marian_scorer command/marian_scorer.cpp)
set_target_properties(marian_scorer PROPERTIES OUTPUT_NAME marian-scorer)
target_compile_options(marian_scorer PRIVATE ${ALL_WARNINGS})
add_executable(marian_vocab command/marian_vocab.cpp)
set_target_properties(marian_vocab PROPERTIES OUTPUT_NAME marian-vocab)
target_compile_options(marian_vocab PRIVATE ${ALL_WARNINGS})
add_executable(marian_conv command/marian_conv.cpp)
set_target_properties(marian_conv PROPERTIES OUTPUT_NAME marian-conv)
target_compile_options(marian_conv PRIVATE ${ALL_WARNINGS})
set(EXECUTABLES ${EXECUTABLES} marian_train marian_decoder marian_scorer marian_vocab marian_conv)
# marian.zip and marian.tgz
# This combines marian, marian_decoder in a single ZIP or TAR file for
# execution in MSFT internal tools FLO and Philly.
# For Philly submission, we need statically-linked versions to deal with
# library dependencies, so this target is only enabled for static builds.
if(USE_STATIC_LIBS)
add_custom_command(
OUTPUT "${CMAKE_BINARY_DIR}/marian.zip"
COMMAND zip -v -0 -j "${CMAKE_BINARY_DIR}/marian.zip"
"${CMAKE_BINARY_DIR}/marian"
"${CMAKE_BINARY_DIR}/marian-decoder"
"${CMAKE_BINARY_DIR}/marian-scorer"
"${CMAKE_BINARY_DIR}/marian-vocab"
"${CMAKE_BINARY_DIR}/marian-conv"
DEPENDS marian_train marian_decoder marian_scorer marian_vocab marian_conv)
add_custom_target(marian_zip DEPENDS "${CMAKE_BINARY_DIR}/marian.zip")
add_custom_command(
OUTPUT "${CMAKE_BINARY_DIR}/marian.tgz"
COMMAND tar -cvvzf "${CMAKE_BINARY_DIR}/marian.tgz" -C "${CMAKE_BINARY_DIR}"
"marian"
"marian-decoder"
"marian-scorer"
"marian-vocab"
"marian-conv"
DEPENDS marian_train marian_decoder marian_scorer marian_vocab marian_conv)
add_custom_target(marian_tgz DEPENDS "${CMAKE_BINARY_DIR}/marian.tgz")
add_custom_target(philly DEPENDS marian_tgz marian_zip)
endif(USE_STATIC_LIBS)
if(COMPILE_SERVER)
add_executable(marian_server command/marian_server.cpp)
set_target_properties(marian_server PROPERTIES OUTPUT_NAME marian-server)
if(MSVC)
# Disable warnings from the SimpleWebSocketServer library needed for compilation of marian-server
target_compile_options(marian_server PRIVATE ${ALL_WARNINGS} /wd4267 /wd4244 /wd4456 /wd4458)
else(MSVC)
# -Wno-suggest-override disables warnings from Boost 1.69+
target_compile_options(marian_server PRIVATE ${ALL_WARNINGS} -Wno-suggest-override)
endif(MSVC)
set(EXECUTABLES ${EXECUTABLES} marian_server)
endif(COMPILE_SERVER)
foreach(exec ${EXECUTABLES})
target_link_libraries(${exec} marian)
if(CUDA_FOUND)
target_link_libraries(${exec} marian_cuda)
endif(CUDA_FOUND)
set_target_properties(${exec} PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}")
endforeach(exec)
endif(NOT COMPILE_LIBRARY_ONLY)
if(COMPILE_TESTS)
set(CATCH_INCLUDE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/3rd_party)
@ -258,3 +279,12 @@ endif(COMPILE_TESTS)
if(COMPILE_EXAMPLES)
add_subdirectory(examples)
endif(COMPILE_EXAMPLES)
if(GENERATE_MARIAN_INSTALL_TARGETS)
# Install the marian library if given a "make install" target
include(GNUInstallDirs) # This defines default values for installation directories (all platforms even if named GNU)
install(TARGETS marian
EXPORT marian-targets
ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
)
endif(GENERATE_MARIAN_INSTALL_TARGETS)