marian-nmt/marian
1
1
Fork 0
mirror of https://github.com/marian-nmt/marian.git synced 2024-09-19 02:37:14 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into rm-boost-2

Browse Source
This commit is contained in:
Marcin Junczys-Dowmunt 2018-09-26 13:42:07 -07:00 committed by GitHub
commit 5bc551c1a8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
47 changed files with 1036 additions and 246 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@ -57,3 +57,6 @@ examples/mnist/*ubyte
/vs/MarianDll.sln
/vs/MarianDll.VC.db
/vs/MarianDll.VC.VC.opendb
.vs
.vscode

45
.vscode/settings.json vendored
View File

@ -1,45 +0,0 @@
{
"files.associations": {
"*.cu": "cpp",
"*.ipp": "cpp",
"array": "cpp",
"slist": "cpp",
"initializer_list": "cpp",
"utility": "cpp",
"typeindex": "cpp",
"typeinfo": "cpp",
"cctype": "cpp",
"clocale": "cpp",
"cmath": "cpp",
"csignal": "cpp",
"cstdarg": "cpp",
"cstddef": "cpp",
"cstdio": "cpp",
"cstdlib": "cpp",
"cstring": "cpp",
"ctime": "cpp",
"cwchar": "cpp",
"cwctype": "cpp",
"atomic": "cpp",
"*.tcc": "cpp",
"chrono": "cpp",
"codecvt": "cpp",
"condition_variable": "cpp",
"cstdint": "cpp",
"exception": "cpp",
"functional": "cpp",
"future": "cpp",
"iosfwd": "cpp",
"limits": "cpp",
"memory": "cpp",
"mutex": "cpp",
"new": "cpp",
"ratio": "cpp",
"stdexcept": "cpp",
"system_error": "cpp",
"thread": "cpp",
"tuple": "cpp",
"type_traits": "cpp",
"algorithm": "cpp"
}
}

57
CMakeLists.txt
View File

@ -2,7 +2,6 @@ cmake_minimum_required(VERSION 3.5.1)
set(CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake)
set(CMAKE_LIBRARY_PATH ${CMAKE_LIBRARY_PATH} /usr/local/cuda-8.0/lib64)
project(marian CXX C)
# Custom CMake options
@ -24,14 +23,24 @@ message(STATUS "Project name: ${PROJECT_NAME}")
message(STATUS "Project version: ${PROJECT_VERSION_STRING_FULL}")
# Set compilation flags
set(CMAKE_CXX_FLAGS_RELEASE " -std=c++11 -O3 -Ofast -m64 -pthread -march=native -Wl,--no-as-needed -funroll-loops -ffinite-math-only -fPIC -Wno-unused-result -Wno-deprecated -Wno-deprecated-gpu-targets")
set(CMAKE_CXX_FLAGS_NONATIVE " -std=c++11 -O3 -Ofast -m64 -pthread -march=x86-64 -mavx -Wl,--no-as-needed -funroll-loops -ffinite-math-only -fPIC -Wno-unused-result -Wno-deprecated -Wno-deprecated-gpu-targets")
set(CMAKE_CXX_FLAGS_DEBUG " -std=c++11 -g -O0 -pthread -fPIC -Wno-unused-result -Wno-deprecated -Wno-deprecated-gpu-targets")
set(CMAKE_CXX_FLAGS_ST "${CMAKE_CXX_FLAGS_RELEASE} -DNDEBUG")
set(CMAKE_CXX_FLAGS_PROFILE "${CMAKE_CXX_FLAGS_RELEASE} -pg -g")
set(CMAKE_CXX_FLAGS_PROFGEN "${CMAKE_CXX_FLAGS_RELEASE} -fprofile-generate -fprofile-correction")
set(CMAKE_CXX_FLAGS_PROFUSE "${CMAKE_CXX_FLAGS_RELEASE} -fprofile-use -fprofile-correction")
set(CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS_RELEASE})
if (MSVC)
set(CMAKE_CXX_FLAGS "/EHsc /DWIN32 /D_WINDOWS /DUNICODE /D_UNICODE /D_CRT_NONSTDC_NO_WARNINGS /D_CRT_SECURE_NO_WARNINGS")
set(CMAKE_CXX_FLAGS_RELEASE "/MT /O2 /W4 /Zi /MP /GL /DNDEBUG")
set(CMAKE_CXX_FLAGS_DEBUG "/MTd /Od /Ob0 /RTC1 /Zi /D_DEBUG")
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /DEBUG /LTCG:incremental /INCREMENTAL:NO")
set(CMAKE_STATIC_LINKER_FLAGS "${CMAKE_STATIC_LINKER_FLAGS} /LTCG:incremental")
else()
set(CMAKE_CXX_FLAGS_RELEASE " -std=c++11 -O3 -Ofast -m64 -pthread -march=native -Wl,--no-as-needed -funroll-loops -ffinite-math-only -fPIC -Wno-unused-result -Wno-deprecated -Wno-deprecated-gpu-targets")
set(CMAKE_CXX_FLAGS_NONATIVE " -std=c++11 -O3 -Ofast -m64 -pthread -march=x86-64 -mavx -Wl,--no-as-needed -funroll-loops -ffinite-math-only -fPIC -Wno-unused-result -Wno-deprecated -Wno-deprecated-gpu-targets")
set(CMAKE_CXX_FLAGS_DEBUG " -std=c++11 -g -O0 -pthread -fPIC -Wno-unused-result -Wno-deprecated -Wno-deprecated-gpu-targets")
set(CMAKE_CXX_FLAGS_ST "${CMAKE_CXX_FLAGS_RELEASE} -DNDEBUG")
set(CMAKE_CXX_FLAGS_PROFILE "${CMAKE_CXX_FLAGS_RELEASE} -pg -g")
set(CMAKE_CXX_FLAGS_PROFGEN "${CMAKE_CXX_FLAGS_RELEASE} -fprofile-generate -fprofile-correction")
set(CMAKE_CXX_FLAGS_PROFUSE "${CMAKE_CXX_FLAGS_RELEASE} -fprofile-use -fprofile-correction")
set(CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS_RELEASE})
endif()
# Find packages
@ -64,6 +73,11 @@ if(CUDA_FOUND)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DCUDA_FOUND")
list(APPEND CUDA_NVCC_FLAGS -DCUDA_FOUND; )
if (MSVC)
list(APPEND CUDA_NVCC_FLAGS -DBOOST_PP_VARIADICS=0; )
endif()
else(CUDA_FOUND)
message(FATAL_ERROR "CUDA has not been found, set -DCOMPILE_CUDA=off to avoid this check and to compile the CPU version only")
endif(CUDA_FOUND)
@ -73,10 +87,15 @@ else(COMPILE_CUDA)
endif(COMPILE_CUDA)
if (CMAKE_BUILD_TYPE STREQUAL "Debug")
list(APPEND CUDA_NVCC_FLAGS -std=c++11; --default-stream per-thread; -O0; -g; -Xcompiler '-fPIC'; -arch=sm_30; -gencode=arch=compute_30,code=sm_30; -gencode=arch=compute_50,code=sm_50; -gencode=arch=compute_52,code=sm_52; -gencode=arch=compute_60,code=sm_60; -gencode=arch=compute_61,code=sm_61; -gencode=arch=compute_61,code=compute_61 ;)
list(APPEND CUDA_NVCC_FLAGS --default-stream per-thread; -O0; -g; -arch=sm_30; -gencode=arch=compute_30,code=sm_30; -gencode=arch=compute_50,code=sm_50; -gencode=arch=compute_52,code=sm_52; -gencode=arch=compute_60,code=sm_60; -gencode=arch=compute_61,code=sm_61; -gencode=arch=compute_61,code=compute_61 ;)
else(CMAKE_BUILD_TYPE STREQUAL "Debug")
list(APPEND CUDA_NVCC_FLAGS -std=c++11; --default-stream per-thread; -O3; -g; --use_fast_math; -Xcompiler '-fPIC'; -arch=sm_30; -gencode=arch=compute_30,code=sm_30; -gencode=arch=compute_50,code=sm_50; -gencode=arch=compute_52,code=sm_52; -gencode=arch=compute_60,code=sm_60; -gencode=arch=compute_61,code=sm_61; -gencode=arch=compute_61,code=compute_61 ;)
list(APPEND CUDA_NVCC_FLAGS --default-stream per-thread; -O3; -g; --use_fast_math; -arch=sm_30; -gencode=arch=compute_30,code=sm_30; -gencode=arch=compute_50,code=sm_50; -gencode=arch=compute_52,code=sm_52; -gencode=arch=compute_60,code=sm_60; -gencode=arch=compute_61,code=sm_61; -gencode=arch=compute_61,code=compute_61 ;)
endif(CMAKE_BUILD_TYPE STREQUAL "Debug")
if (NOT MSVC)
list(APPEND CUDA_NVCC_FLAGS -std=c++11; -Xcompiler\ -fPIC; )
else()
list(APPEND CUDA_NVCC_FLAGS -Xcompiler\ /FS; )
endif()
list(REMOVE_DUPLICATES CUDA_NVCC_FLAGS)
set(CUDA_PROPAGATE_HOST_FLAGS OFF)
@ -90,13 +109,15 @@ if(USE_STATIC_LIBS)
endif()
endif()
find_package(Tcmalloc)
if(Tcmalloc_FOUND)
include_directories(${Tcmalloc_INCLUDE_DIR})
set(EXT_LIBS ${EXT_LIBS} ${Tcmalloc_LIBRARIES})
else(Tcmalloc_FOUND)
message(WARNING "Cannot find TCMalloc library. Continuing.")
endif(Tcmalloc_FOUND)
if (NOT WIN32)
find_package(Tcmalloc)
if(Tcmalloc_FOUND)
include_directories(${Tcmalloc_INCLUDE_DIR})
set(EXT_LIBS ${EXT_LIBS} ${Tcmalloc_LIBRARIES})
else(Tcmalloc_FOUND)
message(WARNING "Cannot find TCMalloc library. Continuing.")
endif(Tcmalloc_FOUND)
endif()
if(USE_MPI)
find_package(MPI 2.0)

77
CMakeSettings.json Normal file
View File

@ -0,0 +1,77 @@
{
"environments": [
{
// Dependencies can be found automatically if you use vcpkg as your library manager.
// In this case, please set the VCPKG_ROOT variable to the directory that contains the vcpkg.exe
// If you prefer managing yourself the dependencies, please fill in the other variables.
"VCPKG_ROOT": "D:\\Perso\\Dev\\vcpkg",
"BOOST_INCLUDEDIR": "",
"BOOST_LIBRARYDIR": "",
"ZLIB_ROOT": "",
"OPENSSL_ROOT_DIR": "",
// The MKL library can be automatically found by CMake. However, if you installed it in a custom
// directory, please set the MKLROOT to this directory path.
// Default is c:\Program Files (x86)\IntelSWTools\compilers_and_libraries\windows\mkl on Windows, or
// /opt/intel/mkl on Linux
"MKLROOT": ""
}
],
"configurations": [
{
"name": "Release",
"generator": "Ninja",
"configurationType": "Release",
"inheritEnvironments": [ "msvc_x64_x64" ],
"buildRoot": "${projectDir}\\build",
"installRoot": "${projectDir}\\install",
"cmakeCommandArgs": "",
"buildCommandArgs": "-v",
"ctestCommandArgs": "",
"variables": [
{ "name": "CMAKE_TOOLCHAIN_FILE", "value": "${env.VCPKG_ROOT}\\scripts\\buildsystems\\vcpkg.cmake" },
{ "name": "VCPKG_TARGET_TRIPLET", "value": "x64-windows-static" },
{ "name": "OPENSSL_USE_STATIC_LIBS:BOOL", "value": "TRUE" },
{ "name": "OPENSSL_MSVC_STATIC_RT:BOOL", "value": "TRUE" },
{ "name": "COMPILE_SERVER:BOOL", "value": "TRUE" },
{ "name": "COMPILE_EXAMPLES:BOOL", "value": "FALSE" },
{ "name": "COMPILE_TESTS:BOOL", "value": "FALSE" },
{ "name": "COMPILE_CPU:BOOL", "value": "TRUE" },
{ "name": "COMPILE_CUDA:BOOL", "value": "TRUE" },
{ "name": "USE_CUDNN:BOOL", "value": "TRUE" },
{ "name": "USE_MPI:BOOL", "value": "FALSE" }
]
},
{
"name": "Debug",
"generator": "Ninja",
"configurationType": "Debug",
"inheritEnvironments": [ "msvc_x64_x64" ],
"buildRoot": "${projectDir}\\build\\debug",
"installRoot": "${projectDir}\\install\\debug",
"cmakeCommandArgs": "",
"buildCommandArgs": "-v",
"ctestCommandArgs": "",
"variables": [
{ "name": "CMAKE_TOOLCHAIN_FILE", "value": "${env.VCPKG_ROOT}\\scripts\\buildsystems\\vcpkg.cmake" },
{ "name": "VCPKG_TARGET_TRIPLET", "value": "x64-windows-static" },
{ "name": "OPENSSL_USE_STATIC_LIBS:BOOL", "value": "TRUE" },
{ "name": "OPENSSL_MSVC_STATIC_RT:BOOL", "value": "TRUE" },
{ "name": "COMPILE_SERVER:BOOL", "value": "TRUE" },
{ "name": "COMPILE_EXAMPLES:BOOL", "value": "FALSE" },
{ "name": "COMPILE_TESTS:BOOL", "value": "FALSE" },
{ "name": "COMPILE_CPU:BOOL", "value": "TRUE" },
{ "name": "COMPILE_CUDA:BOOL", "value": "TRUE" },
{ "name": "USE_CUDNN:BOOL", "value": "TRUE" },
{ "name": "USE_MPI:BOOL", "value": "FALSE" }
]
}
]
}

21
README.md
View File

@ -1,17 +1,17 @@
Marian
======
<!-- [![Join the chat at https://gitter.im/marian-nmt](https://badges.gitter.im/amunmt/marian.svg)](https://gitter.im/marian-nmt?
utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) -->
[![Build Status](http://vali.inf.ed.ac.uk/jenkins/buildStatus/icon?job=marian-dev)](http://vali.inf.ed.ac.uk/jenkins/job/marian-dev/)
[![CPU Build Status](http://vali.inf.ed.ac.uk/jenkins/buildStatus/icon?job=marian-dev)](http://vali.inf.ed.ac.uk/jenkins/job/marian-dev-cpu/)
[![Tests Status](http://vali.inf.ed.ac.uk/jenkins/buildStatus/icon?job=marian-regression-tests)](http://vali.inf.ed.ac.uk/jenkins/job/marian-regression-tests/)
[![Twitter](https://img.shields.io/twitter/follow/marian_nmt.svg?style=social&label=Follow)](https://twitter.com/intent/follow?screen_name=marian_nmt)
[![Build Status](https://img.shields.io/jenkins/s/http/vali.inf.ed.ac.uk/jenkins/view/marian/job/marian-dev.svg?label=CUDA)](http://vali.inf.ed.ac.uk/jenkins/job/marian-dev/)
[![CPU Build Status](https://img.shields.io/jenkins/s/http/vali.inf.ed.ac.uk/jenkins/view/marian/job/marian-dev-cpu.svg?label=CPU)](http://vali.inf.ed.ac.uk/jenkins/job/marian-dev-cpu/)
[![Tests Status](https://img.shields.io/jenkins/s/http/vali.inf.ed.ac.uk/jenkins/view/marian/job/marian-dev-cpu.svg?label=tests)](http://vali.inf.ed.ac.uk/jenkins/job/marian-regression-tests/)
[![Latest release](https://img.shields.io/github/release/marian-nmt/marian.svg?label=release)](https://github.com/marian-nmt/marian/releases)
[![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](./LICENSE.md)
[![Twitter](https://img.shields.io/twitter/follow/marian_nmt.svg?style=social)](https://twitter.com/intent/follow?screen_name=marian_nmt)
<p>
<b>Marian</b> is an efficient Neural Machine Translation framework written
in pure C++ with minimal dependencies. Named in honour of Marian Rejewski, a Polish mathematician and cryptologist.
Named in honour of Marian Rejewski, a Polish mathematician and cryptologist.
<!--It has mainly been developed at the
@ -38,7 +38,10 @@ Marian
If you use this, please cite:
Marcin Junczys-Dowmunt, Roman Grundkiewicz, Tomasz Dwojak, Hieu Hoang, Kenneth Heafield, Tom Neckermann, Frank Seide, Ulrich Germann, Alham Fikri Aji, Nikolay Bogoychev, André F. T. Martins, Alexandra Birch (2018). Marian: Fast Neural Machine Translation in C++ (http://www.aclweb.org/anthology/P18-4020)
Marcin Junczys-Dowmunt, Roman Grundkiewicz, Tomasz Dwojak, Hieu Hoang, Kenneth
Heafield, Tom Neckermann, Frank Seide, Ulrich Germann, Alham Fikri Aji, Nikolay
Bogoychev, André F. T. Martins, Alexandra Birch (2018). Marian: Fast Neural
Machine Translation in C++ (http://www.aclweb.org/anthology/P18-4020)
@InProceedings{mariannmt,
title = {Marian: Fast Neural Machine Translation in {C++}},
@ -69,7 +72,7 @@ make -j
-->
## Amun
The handwritten decoder for RNN models compatible with Marian and Nematus has been superseded by the Marian decoder. The code is available in a seperate repositoy: https://github.com/marian-nmt/amun
The handwritten decoder for RNN models compatible with Marian and Nematus has been superseded by the Marian decoder. The code is available in a separate repository: https://github.com/marian-nmt/amun
## Website

5
cmake/FindCUDNN.cmake
View File

@ -115,13 +115,10 @@ get_filename_component(__libpath_cudart "${CUDA_CUDART_LIBRARY}" PATH)
# More strict enforcement of minor/patch version is done if/when the header file is examined.
if(CUDNN_FIND_VERSION_EXACT)
SET(__cudnn_ver_suffix ".${CUDNN_FIND_VERSION_MAJOR}")
SET(__cudnn_lib_win_name cudnn64_${CUDNN_FIND_VERSION_MAJOR})
else()
SET(__cudnn_lib_win_name cudnn64)
endif()
find_library(CUDNN_LIBRARY
NAMES libcudnn.so${__cudnn_ver_suffix} libcudnn${__cudnn_ver_suffix}.dylib ${__cudnn_lib_win_name}
NAMES libcudnn.so${__cudnn_ver_suffix} libcudnn${__cudnn_ver_suffix}.dylib cudnn
PATHS $ENV{LD_LIBRARY_PATH} ${__libpath_cudart} ${CUDNN_ROOT_DIR} ${PC_CUDNN_LIBRARY_DIRS} ${CMAKE_INSTALL_PREFIX}
PATH_SUFFIXES lib lib64 bin
DOC "CUDNN library." )

11
cmake/FindMKL.cmake
View File

@ -53,7 +53,13 @@ else()
set(COR_LIB "mkl_core")
endif()
set(INTEL_ROOT "/opt/intel" CACHE PATH "Folder contains intel libs")
if(MSVC)
set(ProgramFilesx86 "ProgramFiles(x86)")
set(INTEL_ROOT_DEFAULT $ENV{${ProgramFilesx86}}/IntelSWTools/compilers_and_libraries/windows)
else()
set(INTEL_ROOT_DEFAULT "/opt/intel")
endif()
set(INTEL_ROOT ${INTEL_ROOT_DEFAULT} CACHE PATH "Folder contains intel libs")
find_path(MKL_ROOT include/mkl.h PATHS $ENV{MKLROOT} ${INTEL_ROOT}/mkl
DOC "Folder contains MKL")
@ -99,7 +105,8 @@ if (MKL_INCLUDE_DIR AND
if (NOT DEFINED ENV{CRAY_PRGENVPGI} AND
NOT DEFINED ENV{CRAY_PRGENVGNU} AND
NOT DEFINED ENV{CRAY_PRGENVCRAY} AND
NOT DEFINED ENV{CRAY_PRGENVINTEL})
NOT DEFINED ENV{CRAY_PRGENVINTEL} AND
NOT MSVC)
set(ABI "-m64")
endif()

8
src/3rd_party/CLI/ConfigFwd.hpp vendored
View File

@ -117,7 +117,7 @@ class ConfigINI : public Config {
std::vector<ConfigItem> output;
while(getline(input, line)) {
std::vector<std::string> items;
std::vector<std::string> sitems;
detail::trim(line);
size_t len = line.length();
@ -132,10 +132,10 @@ class ConfigINI : public Config {
if(pos != std::string::npos) {
out.name = detail::trim_copy(line.substr(0, pos));
std::string item = detail::trim_copy(line.substr(pos + 1));
items = detail::split_up(item);
sitems = detail::split_up(item);
} else {
out.name = detail::trim_copy(line);
items = {"ON"};
sitems = {"ON"};
}
if(detail::to_lower(section) != "default") {
@ -149,7 +149,7 @@ class ConfigINI : public Config {
out.parents.insert(out.parents.end(), plist.begin(), plist.end());
}
out.inputs.insert(std::end(out.inputs), std::begin(items), std::end(items));
out.inputs.insert(std::end(out.inputs), std::begin(sitems), std::end(sitems));
}
}
return output;

19
src/3rd_party/cnpy/cnpy.h vendored
View File

@ -260,8 +260,8 @@ namespace cnpy {
static inline
void npz_save(std::string zipname, const std::vector<NpzItem>& items)
{
auto tmpname = zipname + "$$"; // TODO: add thread id or something
unlink(tmpname.c_str()); // when saving to HDFS, we cannot overwrite an existing file
auto tmpname = zipname + "$$"; // TODO: add thread id or something
unlink(tmpname.c_str()); // when saving to HDFS, we cannot overwrite an existing file
FILE* fp = fopen(tmpname.c_str(),"wb");
if (!fp)
throw std::runtime_error("npz_save: error opening file for writing: " + tmpname);
@ -351,12 +351,15 @@ namespace cnpy {
fclose(fp);
// move to final location (atomically)
bad = bad || (rename(tmpname.c_str(), zipname.c_str()) == -1);
if (bad)
{
unlink(tmpname.c_str());
throw std::runtime_error("npz_save: error saving to file: " + zipname);
#ifdef _MSC_VER
unlink(zipname.c_str()); // needed for Windows
#endif
bad = bad || (rename(tmpname.c_str(), zipname.c_str()) == -1);
if (bad)
{
unlink(tmpname.c_str());
throw std::runtime_error("npz_save: error saving to file: " + zipname);
}
}

4
src/3rd_party/simple-websocket-server/crypto.hpp vendored
View File

@ -241,8 +241,8 @@ namespace SimpleWeb {
static std::string pbkdf2(const std::string &password, const std::string &salt, int iterations, int key_size) noexcept {
std::string key;
key.resize(static_cast<std::size_t>(key_size));
PKCS5_PBKDF2_HMAC_SHA1(password.c_str(), password.size(),
reinterpret_cast<const unsigned char *>(salt.c_str()), salt.size(), iterations,
PKCS5_PBKDF2_HMAC_SHA1(password.c_str(), (int)password.size(),
reinterpret_cast<const unsigned char *>(salt.c_str()), (int)salt.size(), iterations,
key_size, reinterpret_cast<unsigned char *>(&key[0]));
return key;
}

106
src/3rd_party/simple-websocket-server/server_ws.hpp vendored
View File

@ -334,8 +334,8 @@ namespace SimpleWeb {
auto send_stream = std::make_shared<SendStream>();
send_stream->put(status >> 8);
send_stream->put(status % 256);
send_stream->put((unsigned char)(status >> 8));
send_stream->put((unsigned char)(status % 256));
*send_stream << reason;
@ -419,17 +419,17 @@ namespace SimpleWeb {
if(io_service->stopped())
io_service->reset();
asio::ip::tcp::endpoint endpoint;
asio::ip::tcp::endpoint ep;
if(config.address.size() > 0)
endpoint = asio::ip::tcp::endpoint(asio::ip::address::from_string(config.address), config.port);
ep = asio::ip::tcp::endpoint(asio::ip::address::from_string(config.address), config.port);
else
endpoint = asio::ip::tcp::endpoint(asio::ip::tcp::v4(), config.port);
ep = asio::ip::tcp::endpoint(asio::ip::tcp::v4(), config.port);
if(!acceptor)
acceptor = std::unique_ptr<asio::ip::tcp::acceptor>(new asio::ip::tcp::acceptor(*io_service));
acceptor->open(endpoint.protocol());
acceptor->open(ep.protocol());
acceptor->set_option(asio::socket_base::reuse_address(config.reuse_address));
acceptor->bind(endpoint);
acceptor->bind(ep);
acceptor->listen();
accept();
@ -563,14 +563,14 @@ namespace SimpleWeb {
}
}
void read_message(const std::shared_ptr<Connection> &connection, Endpoint &endpoint) const {
asio::async_read(*connection->socket, connection->read_buffer, asio::transfer_exactly(2), [this, connection, &endpoint](const error_code &ec, std::size_t bytes_transferred) {
void read_message(const std::shared_ptr<Connection> &connection, Endpoint &ep) const {
asio::async_read(*connection->socket, connection->read_buffer, asio::transfer_exactly(2), [this, connection, &ep](const error_code &ec, std::size_t bytes_transferred) {
auto lock = connection->handler_runner->continue_lock();
if(!lock)
return;
if(!ec) {
if(bytes_transferred == 0) { // TODO: why does this happen sometimes?
read_message(connection, endpoint);
read_message(connection, ep);
return;
}
std::istream stream(&connection->read_buffer);
@ -584,7 +584,7 @@ namespace SimpleWeb {
if(first_bytes[1] < 128) {
const std::string reason("message from client not masked");
connection->send_close(1002, reason);
connection_close(connection, endpoint, 1002, reason);
connection_close(connection, ep, 1002, reason);
return;
}
@ -592,7 +592,7 @@ namespace SimpleWeb {
if(length == 126) {
// 2 next bytes is the size of content
asio::async_read(*connection->socket, connection->read_buffer, asio::transfer_exactly(2), [this, connection, &endpoint, fin_rsv_opcode](const error_code &ec, std::size_t /*bytes_transferred*/) {
asio::async_read(*connection->socket, connection->read_buffer, asio::transfer_exactly(2), [this, connection, &ep, fin_rsv_opcode](const error_code &ec, std::size_t /*bytes_transferred*/) {
auto lock = connection->handler_runner->continue_lock();
if(!lock)
return;
@ -607,15 +607,15 @@ namespace SimpleWeb {
for(std::size_t c = 0; c < num_bytes; c++)
length += static_cast<std::size_t>(length_bytes[c]) << (8 * (num_bytes - 1 - c));
read_message_content(connection, length, endpoint, fin_rsv_opcode);
read_message_content(connection, length, ep, fin_rsv_opcode);
}
else
connection_error(connection, endpoint, ec);
connection_error(connection, ep, ec);
});
}
else if(length == 127) {
// 8 next bytes is the size of content
asio::async_read(*connection->socket, connection->read_buffer, asio::transfer_exactly(8), [this, connection, &endpoint, fin_rsv_opcode](const error_code &ec, std::size_t /*bytes_transferred*/) {
asio::async_read(*connection->socket, connection->read_buffer, asio::transfer_exactly(8), [this, connection, &ep, fin_rsv_opcode](const error_code &ec, std::size_t /*bytes_transferred*/) {
auto lock = connection->handler_runner->continue_lock();
if(!lock)
return;
@ -630,30 +630,30 @@ namespace SimpleWeb {
for(std::size_t c = 0; c < num_bytes; c++)
length += static_cast<std::size_t>(length_bytes[c]) << (8 * (num_bytes - 1 - c));
read_message_content(connection, length, endpoint, fin_rsv_opcode);
read_message_content(connection, length, ep, fin_rsv_opcode);
}
else
connection_error(connection, endpoint, ec);
connection_error(connection, ep, ec);
});
}
else
read_message_content(connection, length, endpoint, fin_rsv_opcode);
read_message_content(connection, length, ep, fin_rsv_opcode);
}
else
connection_error(connection, endpoint, ec);
connection_error(connection, ep, ec);
});
}
void read_message_content(const std::shared_ptr<Connection> &connection, std::size_t length, Endpoint &endpoint, unsigned char fin_rsv_opcode) const {
void read_message_content(const std::shared_ptr<Connection> &connection, std::size_t length, Endpoint &ep, unsigned char fin_rsv_opcode) const {
if(length + (connection->fragmented_message ? connection->fragmented_message->length : 0) > config.max_message_size) {
connection_error(connection, endpoint, make_error_code::make_error_code(errc::message_size));
connection_error(connection, ep, make_error_code::make_error_code(errc::message_size));
const int status = 1009;
const std::string reason = "message too big";
connection->send_close(status, reason);
connection_close(connection, endpoint, status, reason);
connection_close(connection, ep, status, reason);
return;
}
asio::async_read(*connection->socket, connection->read_buffer, asio::transfer_exactly(4 + length), [this, connection, length, &endpoint, fin_rsv_opcode](const error_code &ec, std::size_t /*bytes_transferred*/) {
asio::async_read(*connection->socket, connection->read_buffer, asio::transfer_exactly(4 + length), [this, connection, length, &ep, fin_rsv_opcode](const error_code &ec, std::size_t /*bytes_transferred*/) {
auto lock = connection->handler_runner->continue_lock();
if(!lock)
return;
@ -680,7 +680,7 @@ namespace SimpleWeb {
message = std::shared_ptr<Message>(new Message(fin_rsv_opcode, length));
std::ostream ostream(&message->streambuf);
for(std::size_t c = 0; c < length; c++)
ostream.put(istream.get() ^ mask[c % 4]);
ostream.put((unsigned char)(istream.get() ^ mask[c % 4]));
// If connection close
if((fin_rsv_opcode & 0x0f) == 8) {
@ -689,14 +689,14 @@ namespace SimpleWeb {
int status = 0;
if(length >= 2) {
unsigned char byte1 = message->get();
unsigned char byte2 = message->get();
unsigned char byte1 = (unsigned char)(message->get());
unsigned char byte2 = (unsigned char)(message->get());
status = (static_cast<int>(byte1) << 8) + byte2;
}
auto reason = message->string();
connection->send_close(status, reason);
this->connection_close(connection, endpoint, status, reason);
this->connection_close(connection, ep, status, reason);
}
// If ping
else if((fin_rsv_opcode & 0x0f) == 9) {
@ -707,83 +707,83 @@ namespace SimpleWeb {
auto empty_send_stream = std::make_shared<SendStream>();
connection->send(empty_send_stream, nullptr, fin_rsv_opcode + 1);
if(endpoint.on_ping)
endpoint.on_ping(connection);
if(ep.on_ping)
ep.on_ping(connection);
// Next message
this->read_message(connection, endpoint);
this->read_message(connection, ep);
}
// If pong
else if((fin_rsv_opcode & 0x0f) == 10) {
connection->cancel_timeout();
connection->set_timeout();
if(endpoint.on_pong)
endpoint.on_pong(connection);
if(ep.on_pong)
ep.on_pong(connection);
// Next message
this->read_message(connection, endpoint);
this->read_message(connection, ep);
}
// If fragmented message and not final fragment
else if((fin_rsv_opcode & 0x80) == 0) {
// Next message
this->read_message(connection, endpoint);
this->read_message(connection, ep);
}
else {
connection->cancel_timeout();
connection->set_timeout();
if(endpoint.on_message)
endpoint.on_message(connection, message);
if(ep.on_message)
ep.on_message(connection, message);
// Next message
// Only reset fragmented_message for non-control frames (control frames can be in between a fragmented message)
connection->fragmented_message = nullptr;
this->read_message(connection, endpoint);
this->read_message(connection, ep);
}
}
else
this->connection_error(connection, endpoint, ec);
this->connection_error(connection, ep, ec);
});
}
void connection_open(const std::shared_ptr<Connection> &connection, Endpoint &endpoint) const {
void connection_open(const std::shared_ptr<Connection> &connection, Endpoint &ep) const {
connection->cancel_timeout();
connection->set_timeout();
{
std::unique_lock<std::mutex> lock(endpoint.connections_mutex);
endpoint.connections.insert(connection);
std::unique_lock<std::mutex> lock(ep.connections_mutex);
ep.connections.insert(connection);
}
if(endpoint.on_open)
endpoint.on_open(connection);
if(ep.on_open)
ep.on_open(connection);
}
void connection_close(const std::shared_ptr<Connection> &connection, Endpoint &endpoint, int status, const std::string &reason) const {
void connection_close(const std::shared_ptr<Connection> &connection, Endpoint &ep, int status, const std::string &reason) const {
connection->cancel_timeout();
connection->set_timeout();
{
std::unique_lock<std::mutex> lock(endpoint.connections_mutex);
endpoint.connections.erase(connection);
std::unique_lock<std::mutex> lock(ep.connections_mutex);
ep.connections.erase(connection);
}
if(endpoint.on_close)
endpoint.on_close(connection, status, reason);
if(ep.on_close)
ep.on_close(connection, status, reason);
}
void connection_error(const std::shared_ptr<Connection> &connection, Endpoint &endpoint, const error_code &ec) const {
void connection_error(const std::shared_ptr<Connection> &connection, Endpoint &ep, const error_code &ec) const {
connection->cancel_timeout();
connection->set_timeout();
{
std::unique_lock<std::mutex> lock(endpoint.connections_mutex);
endpoint.connections.erase(connection);
std::unique_lock<std::mutex> lock(ep.connections_mutex);
ep.connections.erase(connection);
}
if(endpoint.on_error)
endpoint.on_error(connection, ec);
if(ep.on_error)
ep.on_error(connection, ec);
}
};

1
src/3rd_party/spdlog/details/format.cc vendored
View File

@ -735,7 +735,6 @@ void fmt::internal::ArgMap<Char>::init(const ArgList &args)
/*nothing*/;
}
}
return;
}
for (unsigned i = 0; i != ArgList::MAX_PACKED_ARGS; ++i) {
internal::Arg::Type arg_type = args.type(i);

6
src/3rd_party/yaml-cpp/node/detail/impl.h vendored
View File

@ -140,9 +140,9 @@ inline bool node_data::remove(const Key& key, shared_memory_holder pMemory) {
it = jt;
}
for (node_map::iterator it = m_map.begin(); it != m_map.end(); ++it) {
if (it->first->equals(key, pMemory)) {
m_map.erase(it);
for (node_map::iterator it2 = m_map.begin(); it2 != m_map.end(); ++it2) {
if (it2->first->equals(key, pMemory)) {
m_map.erase(it2);
return true;
}
}

7
src/command/marian_conv.cpp
View File

@ -11,7 +11,12 @@ int main(int argc, char** argv) {
auto options = New<Options>();
{
auto cli = New<cli::CLIWrapper>(options, "Allowed options");
auto cli = New<cli::CLIWrapper>(
options,
"Convert a model in the .npz format to a mmap-able binary model",
"Allowed options",
"Examples:\n"
" ./marian-conv -f model.npz -t model.bin");
cli->add<std::string>("--from,-f", "Input model", "model.npz");
cli->add<std::string>("--to,-t", "Output model", "model.bin");
cli->parse(argc, argv);

2
src/command/marian_server.cpp
View File

@ -17,7 +17,7 @@ int main(int argc, char **argv) {
// create web service server
WsServer server;
server.config.port = options->get<size_t>("port");
server.config.port = options->get<unsigned short>("port");
auto &translate = server.endpoint["^/translate/?$"];
translate.on_message = [&task](Ptr<WsServer::Connection> connection,

10
src/command/marian_vocab.cpp
View File

@ -11,8 +11,14 @@ int main(int argc, char** argv) {
auto options = New<Options>();
{
auto cli = New<cli::CLIWrapper>(options, "Allowed options");
cli->add<size_t>("--max-size,-m", "Generate only arg most common vocabulary items", 0);
auto cli = New<cli::CLIWrapper>(
options,
"Create a vocabulary from text corpora given on STDIN",
"Allowed options",
"Examples:\n"
" ./marian-vocab < text.src > vocab.yml\n"
" cat text.src text.trg | ./marian-vocab > vocab.yml");
cli->add<size_t>("--max-size,-m", "Generate only UINT most common vocabulary items", 0);
cli->parse(argc, argv);
}

45
src/common/cli_wrapper.cpp
View File

@ -1,10 +1,12 @@
#include "common/cli_wrapper.h"
#include "common/options.h"
#include "common/version.h"
namespace marian {
namespace cli {
uint16_t guess_terminal_width(uint16_t max_width, uint16_t default_width) {
/*
static uint16_t guess_terminal_width(uint16_t max_width, uint16_t default_width) {
uint16_t cols = 0;
#ifdef TIOCGSIZE
struct ttysize ts;
@ -22,6 +24,7 @@ uint16_t guess_terminal_width(uint16_t max_width, uint16_t default_width) {
cols = default_width;
return max_width ? std::min(cols, max_width) : cols;
}
*/
CLIFormatter::CLIFormatter(size_t columnWidth, size_t screenWidth)
: CLI::Formatter(), screenWidth_(screenWidth) {
@ -58,28 +61,46 @@ std::string CLIFormatter::make_option_desc(const CLI::Option *opt) const {
}
CLIWrapper::CLIWrapper(YAML::Node &config,
const std::string &name,
const std::string &description,
const std::string &header,
const std::string &footer,
size_t columnWidth,
size_t screenWidth)
: app_(std::make_shared<CLI::App>()),
defaultGroup_(name),
currentGroup_(name),
: app_(std::make_shared<CLI::App>(description)),
defaultGroup_(header),
currentGroup_(header),
config_(config) {
// set group name for --help option
// set footer
if(!footer.empty())
app_->footer("\n" + footer);
// set group name for the automatically added --help option
app_->get_help_ptr()->group(defaultGroup_);
// set custom failure message
app_->failure_message(failureMessage);
// set custom formatter for help message
auto fmt = std::make_shared<CLIFormatter>(columnWidth, screenWidth);
app_->formatter(fmt);
// add --version option
optVersion_
= app_->add_flag("--version", "Print the version number and exit");
optVersion_->group(defaultGroup_);
}
CLIWrapper::CLIWrapper(Ptr<marian::Options> options,
const std::string &name,
const std::string &description,
const std::string &header,
const std::string &footer,
size_t columnWidth,
size_t screenWidth)
: CLIWrapper(options->getOptions(), name, columnWidth, screenWidth) {}
: CLIWrapper(options->getOptions(),
description,
header,
footer,
columnWidth,
screenWidth) {}
CLIWrapper::~CLIWrapper() {}
@ -96,6 +117,12 @@ void CLIWrapper::parse(int argc, char **argv) {
} catch(const CLI::ParseError &e) {
exit(app_->exit(e));
}
// handle --version flag
if(optVersion_->count()) {
std::cerr << PROJECT_VERSION_FULL << std::endl;
exit(0);
}
}
std::string CLIWrapper::failureMessage(const CLI::App *app,

39
src/common/cli_wrapper.h
View File

@ -71,15 +71,19 @@ private:
// Command-line argument parser
Ptr<CLI::App> app_;
// If this is a wrapper then this should just be a reference,
// then we do not have the added level of containment.
YAML::Node &config_;
// Name of the default option group
std::string defaultGroup_{""};
// Name of the current option group
std::string currentGroup_{""};
// If this is a wrapper then this should just be a reference,
// then we do not have the added level of containment.
YAML::Node &config_;
// Option for --version flag. This is a special flag and similarly to --help,
// the key "version" will be not added into the YAML config
CLI::Option* optVersion_;
static std::string failureMessage(const CLI::App *app, const CLI::Error &e);
// Extract an option name from comma-separated list of command-line arguments,
@ -98,14 +102,17 @@ public:
* Option --help, -h is automatically added.
*
* @param config A reference to the to-be-wrapped yaml tree
* @param name Header text for the main option group
* @param description Program description
* @param header Header text for the main option group
* @param footer Text displayed after the list of options
* @param columnWidth Width of the column with option names
* @param screenWidth Maximum allowed width for help messages, 0 means no
* limit
*/
CLIWrapper(YAML::Node &config,
const std::string &name = "General options",
const std::string &description = "",
const std::string &header = "General options",
const std::string &footer = "",
size_t columnWidth = 35,
size_t screenWidth = 0);
@ -115,15 +122,20 @@ public:
*
* Option --help, -h is automatically added.
*
* @param options A smart pointer to the Options object containing the to-be-wrapped yaml tree
* @param name Header text for the main option group
* @param options A smart pointer to the Options object containing the
* to-be-wrapped yaml tree
* @param description Program description
* @param header Header text for the main option group
* @param footer Text displayed after the list of options
* @param columnWidth Width of the column with option names
* @param screenWidth Maximum allowed width for help messages, 0 means no
* limit
*/
CLIWrapper(Ptr<Options> options,
const std::string &name = "General options",
size_t columnWidth = 35,
const std::string &description = "",
const std::string &header = "General options",
const std::string &footer = "",
size_t columnWidth = 30,
size_t screenWidth = 0);
virtual ~CLIWrapper();
@ -152,8 +164,9 @@ public:
* default value is T()
*
* The option will be defined in the config file even if not given as a
* command-line argument. The implicit default value for a numeric option is
* 0, for a string is an empty string, and for a vector is an empty vector.
* command-line argument. The implicit default value for a boolean or numeric
* option is 0, for a string is an empty string, and for a vector is an empty
* vector.
*
* @param args Comma-separated list of short and long option names
* @param help Help message

42
src/common/config.cpp
View File

@ -2,6 +2,7 @@
#include "common/file_stream.h"
#include "common/logging.h"
#include "common/utils.h"
#include "common/version.h"
#include <algorithm>
#include <set>
@ -27,36 +28,61 @@ void Config::initialize(int argc, char** argv, cli::mode mode, bool validate) {
createLoggers(this);
// set random seed
if(get<size_t>("seed") == 0)
seed = (size_t)time(0);
else
seed = get<size_t>("seed");
// load model parameters
if(mode != cli::mode::translation) {
if(filesystem::exists(get<std::string>("model"))
&& !get<bool>("no-reload")) {
auto model = get<std::string>("model");
if(filesystem::exists(model) && !get<bool>("no-reload")) {
try {
if(!get<bool>("ignore-model-config"))
loadModelParameters(get<std::string>("model"));
loadModelParameters(model);
} catch(std::runtime_error& e) {
LOG(info, "[config] No model configuration found in model file");
}
}
} else {
}
// if cli::mode::translation
else {
auto model = get<std::vector<std::string>>("models")[0];
try {
if(!get<bool>("ignore-model-config"))
loadModelParameters(model);
} catch(std::runtime_error& e) {
} catch(std::runtime_error& ) {
LOG(info, "[config] No model configuration found in model file");
}
}
log();
if(has("version"))
// Log version of Marian that has been used to create the model.
//
// Key "version" is present only if loaded from model parameters and is not
// related to --version flag
if(has("version")) {
auto version = get<std::string>("version");
if(mode == cli::mode::training && version != PROJECT_VERSION_FULL)
LOG(info,
"[config] Loaded model has been created with Marian {}, "
"will be overwritten with current version {} at saving",
version,
PROJECT_VERSION_FULL);
else
LOG(info,
"[config] Loaded model has been created with Marian {}",
version);
}
// If this is a newly started training
else if(mode == cli::mode::training) {
LOG(info,
"[config] Model created with Marian {}",
get("version").as<std::string>());
"[config] Model is being created with Marian {}",
PROJECT_VERSION_FULL);
}
}
bool Config::has(const std::string& key) const {

16
src/common/config_parser.cpp
View File

@ -6,7 +6,6 @@
#include "common/file_stream.h"
#include "common/logging.h"
#include "common/utils.h"
#include "common/version.h"
#include "3rd_party/exception.h"
#include <algorithm>
@ -42,9 +41,6 @@ void ConfigParser::addOptionsGeneral(cli::CLIWrapper& cli) {
cli.switchGroup("General options");
// clang-format off
cli.add<bool>("--version",
"Print version number and exit",
false);
cli.add<std::vector<std::string>>("--config,-c",
"Configuration file(s). If multiple, later overrides earlier");
cli.add<size_t>("--workspace,-w",
@ -603,7 +599,11 @@ void ConfigParser::expandAliases(cli::CLIWrapper& cli) {
}
void ConfigParser::parseOptions(int argc, char** argv, bool doValidate) {
cli::CLIWrapper cli(config_, "General options", 40);
cli::CLIWrapper cli(config_,
"Marian: Fast Neural Machine Translation in C++",
"General options",
"",
40);
addOptionsGeneral(cli);
addOptionsModel(cli);
@ -626,12 +626,6 @@ void ConfigParser::parseOptions(int argc, char** argv, bool doValidate) {
// parse command-line options and fill wrapped YAML config
cli.parse(argc, argv);
// handle version printing
if(get<bool>("version")) {
std::cerr << PROJECT_VERSION_FULL << std::endl;
exit(0);
}
// get paths to extra config files
auto configPaths = loadConfigPaths();

71
src/common/file_stream.h
View File

@ -18,59 +18,82 @@
#include <iostream>
#include "common/logging.h"
#ifdef _WIN32
#include <io.h>
#include <sys/stat.h>
#include <sys/types.h>
#endif
namespace io = boost::iostreams;
#ifdef _MSC_VER
#include <fcntl.h>
#include <io.h>
#include <stdlib.h>
#endif
class TemporaryFile {
private:
int fd_;
bool unlink_;
std::string name_;
#ifndef _MSC_VER
int mkstemp_and_unlink(char* tmpl) {
#ifdef _WIN32
ABORT_IF(true, "mkstemp not available in Windows");
int ret = -1;
#else
int ret = mkstemp(tmpl);
#endif
if(unlink_ && ret != -1) {
ABORT_IF(unlink(tmpl), "Error while deleting '{}'", tmpl);
}
return ret;
}
#endif
int MakeTemp(const std::string& base) {
#ifdef _MSC_VER
char* name = tempnam(base.c_str(), "marian.");
ABORT_IF(name == NULL,
"Error while making a temporary based on '{}'",
base);
int oflag = _O_RDWR | _O_CREAT | _O_EXCL;
if (unlink_) oflag |= _O_TEMPORARY;
int ret = open(name, oflag, _S_IREAD | _S_IWRITE);
ABORT_IF(ret == -1,
"Error while making a temporary based on '{}'",
base);
name_ = name;
free(name);
return ret;
#else
std::string name(base);
name += "marian.XXXXXX";
name.push_back(0);
int ret;
ABORT_IF(-1 == (ret = mkstemp_and_unlink(&name[0])),
"Error while making a temporary based on '{}'",
base);
"Error while making a temporary based on '{}'",
base);
name_ = name;
return ret;
#endif
}
void NormalizeTempPrefix(std::string& base) {
if(base.empty())
return;
#ifdef _MSC_VER
if(base.substr(0,4) == "/tmp")
base = getenv("TMP");
#else
if(base[base.size() - 1] == '/')
return;
struct stat sb;
// It's fine for it to not exist.
if(-1 == stat(base.c_str(), &sb))
if(stat(base.c_str(), &sb) == - 1)
return;
#ifdef _WIN32
#define S_ISDIR(m) (((m)&S_IFMT) == S_IFDIR) // TODO: unify this
#endif
if(S_ISDIR(sb.st_mode))
base += '/';
#endif
}
public:
@ -82,6 +105,19 @@ public:
}
~TemporaryFile() {
#ifdef _MSC_VER
if (fd_ == -1)
return;
if(close(fd_)) {
std::cerr << "Could not close file " << fd_ << std::endl;
std::abort();
}
if(!unlink_) {
ABORT_IF(remove(name_.c_str()), "Error while deleting '{}'", name_);
}
#else
if(fd_ != -1 && !unlink_) {
ABORT_IF(unlink(name_.c_str()), "Error while deleting '{}'", name_);
}
@ -89,6 +125,7 @@ public:
std::cerr << "Could not close file " << fd_ << std::endl;
std::abort();
}
#endif
}
int getFileDescriptor() { return fd_; }

3
src/data/corpus_base.cpp
View File

@ -42,6 +42,7 @@ CorpusBase::CorpusBase(std::vector<std::string> paths,
for(auto path : paths_) {
files_.emplace_back(new InputFileStream(path));
ABORT_IF(files_.back()->empty(), "File '{}' is empty", path);
}
}
@ -153,6 +154,7 @@ CorpusBase::CorpusBase(Ptr<Config> options, bool translate)
alignFileIdx_ = paths_.size();
paths_.emplace_back(path);
files_.emplace_back(new InputFileStream(path));
ABORT_IF(files_.back()->empty(), "File with alignments '{}' is empty", path);
}
if(training && options_->has("data-weighting")) {
@ -164,6 +166,7 @@ CorpusBase::CorpusBase(Ptr<Config> options, bool translate)
weightFileIdx_ = paths_.size();
paths_.emplace_back(path);
files_.emplace_back(new InputFileStream(path));
ABORT_IF(files_.back()->empty(), "File with weights '{}' is empty", path);
}
}

4
src/data/corpus_sqlite.cpp
View File

@ -82,7 +82,7 @@ void CorpusSQLite::fillSQLite() {
for(size_t i = 0; i < files_.size(); ++i) {
cont = cont && utils::getline((std::istream&)*files_[i], line);
if(cont)
ps.bind(i + 2, line);
ps.bind((int)(i + 2), line);
}
if(cont) {
@ -114,7 +114,7 @@ SentenceTuple CorpusSQLite::next() {
SentenceTuple tup(curId);
for(size_t i = 0; i < files_.size(); ++i) {
auto line = select_->getColumn(i + 1);
auto line = select_->getColumn((int)(i + 1));
if(i > 0 && i == alignFileIdx_) {
addAlignmentToSentenceTuple(line, tup);

2
src/data/corpus_sqlite.h
View File

@ -74,7 +74,7 @@ public:
maxDims.resize(ex.size(), 0);
for(size_t i = 0; i < ex.size(); ++i) {
if(ex[i].size() > (size_t)maxDims[i])
maxDims[i] = ex[i].size();
maxDims[i] = (int)ex[i].size();
}
sentenceIds.push_back(ex.getId());
}

2
src/data/shortlist.h
View File

@ -110,7 +110,7 @@ public:
return New<Shortlist>(idx, mapped, reverseMap);
}
virtual void dump(const std::string& prefix) {
virtual void dump(const std::string& /*prefix*/) {
ABORT("Not implemented");
}
};

5
src/data/vocab.cpp
View File

@ -70,6 +70,9 @@ int Vocab::loadOrCreate(const std::string& vocabPath,
if(filesystem::exists(trainPath + ".json")) {
return load(trainPath + ".json", max);
}
if(filesystem::exists(trainPath + ".yaml")) {
return load(trainPath + ".yaml", max);
}
if(filesystem::exists(trainPath + ".yml")) {
return load(trainPath + ".yml", max);
}
@ -92,7 +95,7 @@ Word Vocab::insertWord(Word id, const std::string& str) {
};
int Vocab::load(const std::string& vocabPath, int max) {
bool isJson = regex::regex_search(vocabPath, regex::regex("\\.(json|yml)$"));
bool isJson = regex::regex_search(vocabPath, regex::regex("\\.(json|yaml|yml)$"));
LOG(info,
"[data] Loading vocabulary from {} file {}",
isJson ? "JSON/Yaml" : "text",

2
src/layers/convolution.h
View File

@ -30,7 +30,7 @@ public:
std::vector<int> kernelNums,
int stride)
: name_(prefix),
size_(kernelNums.size()),
size_((int)kernelNums.size()),
kernelHeight_(kernelHeight),
kernelWidths_(kernelWidths),
kernelNums_(kernelNums),

6
src/models/char_s2s.h
View File

@ -56,7 +56,7 @@ protected:
int stride) {
auto subBatch = (*batch)[batchIndex_];
int dimBatch = subBatch->batchSize();
size_t dimBatch = subBatch->batchSize();
std::vector<float> strided;
for(size_t wordIdx = 0; wordIdx < subBatch->mask().size();
@ -65,9 +65,9 @@ protected:
strided.push_back(subBatch->mask()[j]);
}
}
int dimWords = strided.size() / dimBatch;
size_t dimWords = strided.size() / dimBatch;
auto stridedMask
= graph->constant({dimWords, dimBatch, 1}, inits::from_vector(strided));
= graph->constant({(int)dimWords, (int)dimBatch, 1}, inits::from_vector(strided));
return stridedMask;
}
};

2
src/models/costs.h
View File

@ -196,7 +196,7 @@ public:
virtual data::SoftAlignment getAlignment() override { return encdec_->getAlignment(); }
};
static Ptr<ModelBase> add_cost(Ptr<EncoderDecoder> encdec,
inline Ptr<ModelBase> add_cost(Ptr<EncoderDecoder> encdec,
Ptr<Options> options) {
switch(options->get<usage>("usage", usage::raw)) {
case usage::training:

2
src/models/transformer_factory.h
View File

@ -13,6 +13,4 @@ Ptr<EncoderBase> NewEncoderTransformer(Ptr<Options> options);
Ptr<DecoderBase> NewDecoderTransformer(Ptr<Options> options);
} // namespace marian
#ifndef _WIN32 // TODO: remove this once I updated the Linux-side makefile
#include "models/transformer.h"
#endif

6
src/rescorer/rescorer.h
View File

@ -79,7 +79,7 @@ public:
ThreadPool pool(graphs_.size(), graphs_.size());
for(size_t i = 0; i < graphs_.size(); ++i) {
pool.enqueue(
[=](int j) {
[=](size_t j) {
models_[j] = New<Model>(temp);
models_[j]->load(graphs_[j], modelFile);
},
@ -117,7 +117,7 @@ public:
while(*batchGenerator) {
auto batch = batchGenerator->next();
auto task = [=, &sumCost, &sumWords, &sumSamples, &smutex](int id) {
auto task = [=, &sumCost, &sumWords, &sumSamples, &smutex](size_t id) {
thread_local Ptr<ExpressionGraph> graph;
thread_local Ptr<Model> builder;
@ -146,7 +146,7 @@ public:
if(!summarize) {
for(size_t i = 0; i < batch->size(); ++i) {
output->Write(batch->getSentenceIds()[i], scores[i], aligns[i]);
output->Write((long)batch->getSentenceIds()[i], scores[i], aligns[i]);
}
}
};

1
src/rescorer/score_collector.cpp
View File

@ -82,7 +82,6 @@ ScoreCollectorNBest::ScoreCollectorNBest(const Ptr<Config>& options)
void ScoreCollectorNBest::Write(long id,
float score,
const data::SoftAlignment& align) {
std::string line;
{
std::lock_guard<std::mutex> lock(mutex_);
auto iter = buffer_.find(id);

1
src/tensors/backend.cpp
View File

@ -2,6 +2,7 @@
#ifdef CUDA_FOUND
#include "tensors/gpu/backend.h"
#pragma warning(disable:4505) // "unreferenced local function has been removed" in cuda\v9.2\include\cuda_fp16.hpp
#endif
#include "tensors/cpu/backend.h"

0
src/tensors/cpu/backend.h Executable file → Normal file
View File

6
src/tensors/gpu/algorithm.h
View File

@ -5,12 +5,12 @@
namespace marian {
namespace gpu {
template <typename T>
void copy(Ptr<Backend> backend, const T* begin, const T* end, T* dest);
void copy(Ptr<marian::Backend> backend, const T* begin, const T* end, T* dest);
template <typename T>
void fill(Ptr<Backend> backend, T* begin, T* end, T value);
void fill(Ptr<marian::Backend> backend, T* begin, T* end, T value);
void setSparse(Ptr<Backend> backend,
void setSparse(Ptr<marian::Backend> backend,
const std::vector<size_t>&,
const std::vector<float>&,
float*);

6
src/tensors/gpu/backend.h
View File

@ -25,7 +25,7 @@ public:
setHandles();
}
void setDevice() override { cudaSetDevice(deviceId_.no); }
void setDevice() override { cudaSetDevice((int)deviceId_.no); }
void synchronize() override { cudaStreamSynchronize(0); }
@ -43,7 +43,7 @@ private:
}
curandGenerator_t createCurandGenerator() {
cudaSetDevice(deviceId_.no);
cudaSetDevice((int)deviceId_.no);
curandGenerator_t generator;
CURAND_CALL(curandCreateGenerator(&generator, CURAND_RNG_PSEUDO_DEFAULT));
CURAND_CALL(curandSetPseudoRandomGeneratorSeed(generator, seed_));
@ -55,7 +55,7 @@ private:
}
cublasHandle_t create_handle() {
cudaSetDevice(deviceId_.no);
cudaSetDevice((int)deviceId_.no);
cublasHandle_t cublasHandle;
cublasCreate(&cublasHandle);
return cublasHandle;

3
src/tensors/gpu/cuda_helpers.h
View File

@ -4,7 +4,8 @@
#include "3rd_party/exception.h"
#include "common/logging.h"
const float CUDA_FLT_MAX = 1.70141e+38;
// fixes a missing constant in CUDA device code (specific to MSVC compiler)
__constant__ float CUDA_FLT_MAX = 1.70141e+38;
const int MAX_THREADS = 512;
const int MAX_BLOCKS = 65535;

9
src/tensors/gpu/cudnn_wrappers.cu
View File

@ -170,6 +170,7 @@ void ConvolutionWrapper::setConvDescriptor(int hPad,
int wStride) {
CUDNN_CALL(cudnnCreateConvolutionDescriptor(&convDesc_));
#if CUDNN_MAJOR > 5
CUDNN_CALL(cudnnSetConvolution2dDescriptor(convDesc_,
hPad,
wPad,
@ -177,10 +178,16 @@ void ConvolutionWrapper::setConvDescriptor(int hPad,
wStride,
1,
1, // upscales
#if CUDNN_MAJOR > 5
CUDNN_CROSS_CORRELATION,
CUDNN_DATA_FLOAT));
#else
CUDNN_CALL(cudnnSetConvolution2dDescriptor(convDesc_,
hPad,
wPad,
hStride,
wStride,
1,
1, // upscales
CUDNN_CROSS_CORRELATION));
#endif
}

6
src/training/graph_group_async.cpp
View File

@ -285,12 +285,12 @@ void AsyncGraphGroup::execute(Ptr<data::Batch> batch) {
for(auto g : graphs_)
fetchParams(g->params()->vals(), paramsAvg_, t_id);
if(scheduler_->saving())
this->save(graph);
if(scheduler_->validating())
scheduler_->validate(graphs_);
if(scheduler_->saving())
this->save(graph);
// Validation or saving is done, tell other threads to continue work.
pool_->notify_others();
}

6
src/training/graph_group_singleton.cpp
View File

@ -42,9 +42,6 @@ void SingletonGraph::execute(Ptr<data::Batch> batch) {
if(scheduler_) {
scheduler_->update(cost, batch);
if(scheduler_->saving())
this->save();
if(scheduler_->validating()) {
if(mvAvg_) {
graphAvg_->reuseWorkspace(graph_);
@ -53,6 +50,9 @@ void SingletonGraph::execute(Ptr<data::Batch> batch) {
scheduler_->validate({graph_});
}
}
if(scheduler_->saving())
this->save();
}
}
} // namespace marian

8
src/training/graph_group_sync.cpp
View File

@ -195,10 +195,6 @@ void SyncGraphGroup::execute(Ptr<data::Batch> batch) {
if(scheduler_) {
scheduler_->update(cost, batches);
if(scheduler_->saving()) {
this->save();
}
if(scheduler_->validating()) {
if(mvAvg_) {
comm_->swapParams(paramsAvg_);
@ -211,6 +207,10 @@ void SyncGraphGroup::execute(Ptr<data::Batch> batch) {
comm_->swapParams(paramsAvg_);
}
}
if(scheduler_->saving()) {
this->save();
}
}
}

2
src/translator/translator.h
View File

@ -207,7 +207,7 @@ public:
std::stringstream best1;
std::stringstream bestn;
printer->print(history, best1, bestn);
collector->add(history->GetLineNum(), best1.str(), bestn.str());
collector->add((long)history->GetLineNum(), best1.str(), bestn.str());
}
};

2
vs/.gitignore vendored Normal file
View File

@ -0,0 +1,2 @@
build-vs
deps

22
vs/BuildRelease.bat Normal file
View File

@ -0,0 +1,22 @@
::
:: Usage: BuildRelease.bat [<build-directory>=.\build]
::
:: This script runs the dependency checks, generate the projects/makefiles and then
:: build the project in Release configuration.
::
::
@echo off
setlocal
set ROOT=%~dp0
set MARIAN_ROOT=%ROOT%..
set BUILD_ROOT=%1
if "%BUILD_ROOT%"=="" set BUILD_ROOT=%ROOT%build
call CreateVSProjects.bat %BUILD_ROOT%
if errorlevel 1 exit /b 1
cmake --build %BUILD_ROOT% --config Release
exit /b 0

304
vs/CheckDeps.bat Normal file
View File

@ -0,0 +1,304 @@
::
:: Usage: CheckDeps.bat
::
:: This script is used to verify that all the dependencies required to build Marian are available.
:: The Cuda SDK, the CuDNN library and the Intel MKL must be installed beforehand by the user.
:: The Boost, zlib and OpenSSH libraries, if not found, will be installed by this script using vcpkg
::
::
@echo off
set ROOT=%~dp0
:: The vcpkg library manager can be used to manage your dependencies in CMake.
::
:: If you already have a working vcpkg installation, this script can use it.
:: If vcpkg is in your PATH variable, the script will use it automatically.
:: Otherwise, please set the VCPKG_ROOT variable to the directory that contains the vcpkg.exe
::
:: eg: set VCPKG_ROOT=D:\Perso\Dev\vcpkg
::----------------------------------------------------------------------------------------------
::set VCPKG_ROOT=
:: If you prefer to manage yourself the dependencies, please set the following variables
:: to the respective paths. These variable can also be already set in your environment.
::----------------------------------------------------------------------------------------------
::set BOOST_INCLUDEDIR=
::set BOOST_LIBRARYDIR=
::set ZLIB_ROOT=
::set OPENSSL_ROOT_DIR=
:: If all the variables are empty and vcpkg is found in a known path, the script will download and
:: install vcpkg and will use it to manage the dependencies.
:: The MKL library can be automatically found by CMake. However, if you installed it in a custom
:: directory, please set the MKLROOT to this directory path.
:: Default is c:\ProgramFiles (x86)\IntelSWTools\compilers_and_libraries\windows\mkl
::----------------------------------------------------------------------------------------------
::set MKLROOT=
if "%BOOST_INCLUDEDIR%" == "" goto :needVcPkg
if "%ZLIB_ROOT%" == "" goto :needVcPkg
if "%OPENSSL_ROOT_DIR%" == "" goto :needVcPkg
goto :checkDeps
:: -------------------------------------------------------
:: Download or update vcpkg
:needVcPkg
:: -------------------------------------------------------
:: First look if vcpkg is in a known path
if "%VCPKG_ROOT%" == "" for /f "delims=" %%p in ('where vcpkg 2^>nul') do set VCPKG_ROOT=%%~dpp
:: Otherwise install it in a subdirectory
if "%VCPKG_ROOT%" == "" set VCPKG_ROOT=%ROOT%deps\vcpkg
if not exist %VCPKG_ROOT% (
echo --- Cloning vcpkg...
git clone https://github.com/Microsoft/vcpkg.git %VCPKG_ROOT%
set BOOTSTRAP_VCPKG=1
) else (
pushd %VCPKG_ROOT%
echo --- Updating vcpkg...
for /f "delims=" %%p in ('git pull') do (
if not "%%p" == "Already up to date." (
set BOOTSTRAP_VCPKG=1
)
)
popd
)
if "%BOOTSTRAP_VCPKG%"=="1" (
pushd %VCPKG_ROOT%
call bootstrap-vcpkg.bat
popd
)
set VCPKG_DEFAULT_TRIPLET=x64-windows-static
set VCPKG_INSTALL=%VCPKG_ROOT%\installed\%VCPKG_DEFAULT_TRIPLET%
set VCPKG=%VCPKG_ROOT%\vcpkg
:: -------------------------------------------------------
:: Check dependencies and configure CMake
:checkDeps
:: -------------------------------------------------------
echo.
echo --- Checking dependencies...
set CMAKE_OPT=
:: -------------------------
:: The CUDA_PATH env variable should normally be set by the CUDA SDK installer
::
echo.
echo ... CUDA
if "%CUDA_PATH%"=="" (
echo The CUDA_PATH environment variable is not defined: please make sure CUDA 8.0+ is installed.
exit /b 1
)
if not exist "%CUDA_PATH%" (
echo CUDA_PATH is set to a non existing path:
echo %CUDA_PATH%
echo Please make sure CUDA 8.0+ is properly installed.
exit /b 1
)
if not exist "%CUDA_PATH%\include\cuda.h" (
echo CUDA header files were not found in this folder:
echo "%CUDA_PATH%"
echo Please make sure CUDA 8.0+ is properly installed.
exit /b 1
)
if not exist "%CUDA_PATH%\lib\x64\cuda.lib" (
echo CUDA library files were not found in this folder:
echo "%CUDA_PATH%"
echo Please make sure CUDA 8.0+ is properly installed.
exit /b 1
)
echo Found Cuda SDK in %CUDA_PATH%
:: -------------------------
:: CuDNN is installed manually into CUDA directories.
echo.
echo ... CUDNN
if not exist "%CUDA_PATH%\lib\x64\cudnn.lib" (
echo The CuDNN library was not found. Please make sure it is installed correctly in your CUDA setup.
exit /b 1
)
echo Found CuDNN library in %CUDA_PATH%\lib\x64
:: -------------------------
:: The MKL setup does not set any environment variable to the installation path.
:: The script look into the standard default installation dir
:: If you installed MKL in a custom directory, please set the variable MKLROOT at the top of this file.
::
echo.
echo ... Intel MKL
if "%MKLROOT%" == "" (
set "MKLROOT=C:\Program Files (x86)\IntelSWTools\compilers_and_libraries\windows\mkl"
)
if not exist "%MKLROOT%" (
echo MKLROOT is set to a non existing path:
echo "%MKLROOT%"
echo Please make sure the Intel MKL libraries are installed and set MKLROOT to the installation path.
exit /b 1
)
if not exist "%MKLROOT%\include\mkl_version.h" (
echo MKL header files were not found in this folder:
echo "%MKLROOT%"
echo Please make sure Intel MKL is properly installed.
exit /b 1
)
if not exist "%MKLROOT%\lib\intel64\mkl_core.lib" (
echo MKL library files were not found in this folder:
echo "%MKLROOT%"
echo Please make sure Intel MKL is properly installed.
exit /b 1
)
echo Found Intel MKL library in %MKLROOT%
:: -------------------------
:: BOOST_INCLUDEDIR and BOOST_LIBRARYDIR can be both set to an existing Boost installation.
:: If not, we use vcpkg to install the required Boost packages
::
echo.
echo ... Boost (1.58+)
if "%BOOST_INCLUDEDIR%" == "" (
"%VCPKG%" install boost-chrono boost-filesystem boost-iostreams boost-program-options boost-regex boost-system boost-thread boost-timer boost-asio
set BOOST_INCLUDEDIR=%VCPKG_INSTALL%\include
set BOOST_LIBRARYDIR=%VCPKG_INSTALL%\lib
)
if not exist "%BOOST_INCLUDEDIR%" (
echo BOOST_INCLUDEDIR is set to a non existing path:
echo "%BOOST_INCLUDEDIR%"
echo Please set BOOST_INCLUDEDIR and BOOST_LIBRARYDIR to the installation path of the Boost library.
exit /b 1
)
if not exist "%BOOST_INCLUDEDIR%\boost\version.hpp" (
echo Boost header files were not found in this folder:
echo "%BOOST_INCLUDEDIR%"
echo Please make sure Boost is correctly installed.
exit /b 1
)
if not exist "%BOOST_LIBRARYDIR%" (
echo BOOST_LIBRARYDIR is set to a non existing path:
echo "%BOOST_LIBRARYDIR%"
echo Please set BOOST_INCLUDEDIR and BOOST_LIBRARYDIR to the installation path of the Boost library.
exit /b 1
)
if not exist "%BOOST_LIBRARYDIR%\boost_*.lib" (
echo Boost library files were not found in this folder:
echo "%BOOST_LIBRARYDIR%"
echo Please make sure Boost is correctly installed.
exit /b 1
)
echo Found Boost headers in "%BOOST_INCLUDEDIR%" and libs in "%BOOST_LIBRARYDIR%"
:: -------------------------
:: ZLIB_ROOT can be set to an existing zlib installation.
:: If not, we use vcpkg to install the library
::
echo.
echo ... zlib
if "%ZLIB_ROOT%"=="" (
%VCPKG% install zlib
set ZLIB_ROOT=%VCPKG_INSTALL%
)
if not exist "%ZLIB_ROOT%" (
echo ZLIB_ROOT is set to a non existing path:
echo "%ZLIB_ROOT%"
echo Please set ZLIB_ROOT to the installation path of the zlib library.
exit /b 1
)
if not exist "%ZLIB_ROOT%\include\zlib.h" (
echo zlib header files were not found in this folder:
echo "%ZLIB_ROOT%"
echo Please make sure zlib is correctly installed.
exit /b 1
)
if not exist "%ZLIB_ROOT%\lib\zlib.lib" (
echo zlib library file were not found in this folder:
echo "%ZLIB_ROOT%"
echo Please make sure zlib is correctly installed.
exit /b 1
)
echo Found zlib library in "%ZLIB_ROOT%"
:: -------------------------
:: OPENSSL_ROOT_DIR can be set to an existing OpenSSL installation.
:: If not, we use vcpkg to install the library
::
echo.
echo ... OpenSSL
if "%OPENSSL_ROOT_DIR%"=="" (
%VCPKG% install openssl
set OPENSSL_ROOT_DIR=%VCPKG_INSTALL%
)
if not exist "%OPENSSL_ROOT_DIR%" (
echo OPENSSL_ROOT_DIR is set to a non existing path:
echo "%OPENSSL_ROOT_DIR%"
echo Please set OPENSSL_ROOT_DIR to the installation path of the OpenSLL library.
exit /b 1
)
if not exist "%OPENSSL_ROOT_DIR%\include\openssl\opensslv.h" (
echo OpenSSL header files were not found in this folder:
echo "%OPENSSL_ROOT_DIR%"
echo Please make sure OpenSSL is correctly installed.
exit /b 1
)
if not exist "%OPENSSL_ROOT_DIR%\lib\ssleay32.lib" (
echo OpenSSL library file were not found in this folder:
echo "%OPENSSL_ROOT_DIR%"
echo Please make sure OpenSSL is correctly installed.
exit /b 1
)
echo Found OpenSSL library in "%OPENSSL_ROOT_DIR%"
echo.
echo.
echo --------------------------------------------------
echo CUDA_PATH ^| %CUDA_PATH%
echo MKLROOT ^| %MKLROOT%
echo VCPKG_ROOT ^| %VCPKG_ROOT%
echo BOOST_INCLUDEDIR ^| %BOOST_INCLUDEDIR%
echo BOOST_LIBRARYDIR ^| %BOOST_LIBRARYDIR%
echo ZLIB_ROOT ^| %ZLIB_ROOT%
echo OPENSSL_ROOT_DIR ^| %OPENSSL_ROOT_DIR%
echo --------------------------------------------------
echo.
echo.
exit /b 0

77
vs/CreateVSProjects.bat Normal file
View File

@ -0,0 +1,77 @@
::
:: Usage: CreateVSProjects.bat [<build-directory>=.\build]
::
:: This script runs the dependency checks, then invokes CMake with the right parameters to create
:: the solutions for Visual Studio.
::
:: Run this script only if you have a previous version of Visual Studio (2015 and below), or if you
:: don't want to use the built-in CMake integration.
::
:: You may want to change the generator target to fit your installation. By default, the target is
:: "Visual Studio 15 2017 Win64". Run `cmake --help` for a list of supported targets.
::
:: Note: You don't need to run this script if you want to use Visual Studio with built-in support for CMake
:: (only available for VS 2017+)
::
::
@echo off
setlocal
set ROOT=%~dp0
set MARIAN_ROOT=%ROOT%..
set BUILD_ROOT=%1
if "%BUILD_ROOT%"=="" set BUILD_ROOT=%ROOT%build
set GENERATOR_TARGET="Visual Studio 15 2017 Win64"
call CheckDeps.bat
if errorlevel 1 exit /b 1
echo.
echo --- Configuring CMake...
set CMAKE_OPT=
:: ----- Configure OpenSLL dep
set CMAKE_OPT=%CMAKE_OPT% -D OPENSSL_USE_STATIC_LIBS:BOOL=TRUE
set CMAKE_OPT=%CMAKE_OPT% -D OPENSSL_MSVC_STATIC_RT:BOOL=TRUE
:: ----- Target Visual Studio 2017 64bits -----
set CMAKE_OPT=%CMAKE_OPT% -G %GENERATOR_TARGET%
:: Policy CMP0074: find_package uses <PackageName>_ROOT variables.
set CMAKE_OPT=%CMAKE_OPT% -D CMAKE_POLICY_DEFAULT_CMP0074=NEW
:: ----- Disable some tool build -----
set CMAKE_OPT=%CMAKE_OPT% -D COMPILE_SERVER:BOOL=TRUE
set CMAKE_OPT=%CMAKE_OPT% -D COMPILE_EXAMPLES:BOOL=FALSE
set CMAKE_OPT=%CMAKE_OPT% -D COMPILE_TESTS:BOOL=FALSE
set CMAKE_OPT=%CMAKE_OPT% -D COMPILE_CPU:BOOL=TRUE
set CMAKE_OPT=%CMAKE_OPT% -D COMPILE_CUDA:BOOL=TRUE
set CMAKE_OPT=%CMAKE_OPT% -D USE_CUDNN:BOOL=TRUE
set CMAKE_OPT=%CMAKE_OPT% -D USE_MPI:BOOL=FALSE
echo.
echo.
echo --------------------------------------------------
echo CMAKE configuration flags:
echo %CMAKE_OPT%
echo --------------------------------------------------
echo.
echo.
if not exist %BUILD_ROOT% mkdir %BUILD_ROOT%
pushd %BUILD_ROOT%
echo.
echo --- Creating Visual Studio projects...
cmake %CMAKE_OPT% %MARIAN_ROOT%
popd
exit /b 0

200
vs/README.md Normal file
View File

@ -0,0 +1,200 @@
# How to build Marian on Windows with GPU support
## Install prerequisites
The following SDK are required to build Marian with GPU support
- [Cuda 9.2+](https://developer.nvidia.com/cuda-downloads?target_os=Windows&target_arch=x86_64&target_version=10&target_type=exelocal)
- Base installer
- Patches
- [CuDNN 7.1+](https://developer.nvidia.com/rdp/cudnn-download)
- Requires nVidia Developper account
- [MKL](https://software.intel.com/en-us/mkl)
__Note: Patch for CUDA error: Unsupported Visual Studio Version Error__
The latest versions of Visual Studio 2017 are not officially supported by CUDA. Two fixes are proposed:
- Downgrade Visual Studio to a supported version
- Edit the file `<CUDA install path>\include\crt\host_config.h` and change the line 131:
131 #if _MSC_VER < 1600 || _MSC_VER > 1914
into:
131 #if _MSC_VER < 1600 || _MSC_VER > 1915
For more information, read this [nVidia forum](https://devtalk.nvidia.com/default/topic/1022648/cuda-setup-and-installation/cuda-9-unsupported-visual-studio-version-error/4)
---
## Check dependencies : `CheckDeps.bat`
In addition to the 3 previous prerequisites, Marian needs 3 libraries that you may already have on your system:
- Boost (1.58+)
- zlib
- OpenSSL
The script `CheckDeps.bat` can be used to verify that all dependencies are found on your system. If not, it will use the `vcpkg` library manager to download and manage your dependencies for CMake.
If you already have a working `vcpkg` installation, this script can use it:
- If vcpkg is in your `PATH` environment variable, the script will find it and use it automatically.
- Otherwise, you need to edit the script and set the `VCPKG_ROOT` variable to the directory that contains the vcpkg.exe
If you prefer to manage yourself the dependencies, you can edit the script file to set the following variables to the respective installation paths. These variable can also be already set in your environment.
- `BOOST_INCLUDE_PATH` and `BOOST_LIB_PATH`
- `ZLIB_ROOT`
- `OPENSSL_PATH`
---
## Build the project
There are 3 alternatives to build the project:
1. Use Visual Studio 2017 built-in support for CMake
2. Create a Solution file for Visual Studio
3. Use a script (MSBuild)
### 1. Use VS2017 with built-in support for CMake
VS2017 now allows to develop projects built with CMake without the need to generate VS projects and solutions. For more information, please read [this article](https://blogs.msdn.microsoft.com/vcblog/2016/10/05/cmake-support-in-visual-studio/) from the Visual C++ Team.
You just need to open the root folder of the git repository in VS (which contains the file `CMakeSettings.json`):
- In an Explorer window, right-click then `Open in Visual Studio`
- In a VS2017 instance, `File > Open > Folder...`
You may need to edit the file `CMakeSettings.json` to set the environment variable for the dependencies.
The developing experience is very similar than when using a solution file (Intellisense, build project with `F7`, debug, set breakpoints and watch variables, ...), except that the project configuration is done in 3 different files:
- `CMakeList.txt`: this is the CMake source file from the original project.
It is used to configure the build targets, add/remove files to compile and configure the compiler flags.
- `CMakeSettings.json`: this file is required to enable CMake integration in VS2017.
Use this file to configure the environment variables and the parameters passed to CMake to generate the project.
- `.vs\launch.vs.json`: this is a user specific file and it is not commited in the Git repo
Use this file to configure the debugging targets.
For example:
{
"version": "0.2.1",
"defaults": {},
"configurations": [
{
"type": "default",
"name": "Training Basics",
"project": "CMakeLists.txt",
"projectTarget": "marian.exe",
"currentDir": "D:\\Perso\\github\\marian\\marian-examples\\training-basics",
"args": [
"--devices 0",
"--type amun",
"--model model/model.npz",
"--train-sets data/corpus.bpe.ro data/corpus.bpe.en",
"--vocabs model/vocab.ro.yml model/vocab.en.yml",
"--dim-vocabs 66000 50000",
"--mini-batch-fit",
"-w 3000",
"--layer-normalization",
"--dropout-rnn 0.2",
"--dropout-src 0.1",
"--dropout-trg 0.1",
"--early-stopping 5",
"--valid-freq 100",
"--save-freq 10000",
"--disp-freq 100",
"--valid-metrics cross-entropy translation",
"--valid-sets data/newsdev2016.bpe.ro data/newsdev2016.bpe.en",
"--valid-script-path .\\scripts\\validate.bat",
"--log model/train.log",
"--valid-log model/valid.log",
"--overwrite",
"--keep-best",
"--seed 1111",
"--exponential-smoothing",
"--normalize=1",
"--beam-size=12",
"--quiet-translation"
]
}
]
}
### 2. Create solution and projects files for Visual Studio : `CreateVSProjects.bat`
If you have a previous version of Visual Studio, you will need to use CMake to generate the projects files.
The provided script `CreateVSProjects.bat` runs the dependency checks then invokes CMake with the right parameters to create the solutions for Visual Studio.
### 3. Use MSBuild : `Build.bat`
The last alternative is to use the script `Build.bat` that will:
- Check the dependencies
- Create the VS project files
- Invoke MSBuild on these projects to build the targets in Release.
---
## Changes from the master branch
This part gives more information on all changes done in this PR. Refer to [this page](https://github.com/cedrou/marian-dev/commits/build_on_win) for commits.
1. __Fix Cuda error : Unsupported Visual Studio Version Error__
See above for justification and fixes
2. __Fix VS compiler flags / Build in Release, with improved debug info__
Added VS specific compile and link flags
3. __Fix Warning: D9002: ignoring unknown option '-m64'__
This one is related to a compiler flag added while finding the package MKL that does not exists for MS compiler.
4. __Fix marian::Backend, marian::cpu::Backend and marian::gpu::Backend conflicts__
There were name conflicts between the 3 `Backend` classes that confused the compiler:
> template instantiation resulted in unexpected function type of "void(Ptr\<marian::gpu::Backend\> backend, [...])" (the meaning of a name may have changed since the template declaration -- the type of the template is "void(Ptr\<marian::Backend\> backend, [...]").
To solve this, I changed the declaration of 3 methods to specify the full name with namespace (`marian::Backend`, instead of `Backend`).
5. __Fix error : identifier "CUDA_FLT_MAX" is undefined in device code__
`CUDA_FLT_MAX` is not seen by CUDA from the device code and I had to declare it as `__constant__`.
From [StackOverflow](https://stackoverflow.com/questions/20111409/how-to-pass-structures-into-cuda-device#comment29972423_20112013):
> Undecorated constants get compiled into both host and device code with gcc based toolchains, but not with the Microsoft compiler.
6. __Fix fatal error C1019: unexpected #else__
There was preprocessor instructions (`#ifdef ... #else ... #endif`) in the middle of a call of a macro function (`CUDNN_CALL`), which is not allowed with MS compiler.
7. __Fix mismatched class/struct forward declarations__
Microsoft's C++ name mangling makes a distinction between `class` and `struct` objects, so definitions and forward declaration must match.
See [this pdf](https://www.agner.org/optimize/calling_conventions.pdf), page 27, for more information.
_Note_: This fix was invalidated by commit # from @frankseide
8. __Fix unresolved external due to a removed #include directive__
There was an include directive removed from MSVC compilation, but this prevented the build of the project.
I'm not sure why this was removed; the comment is:
#ifndef _WIN32 // TODO: remove this once I updated the Linux-side makefile
9. __Fix CUDA+MSVC incompatibility with Boost.Preprocessor__
The toolchain nvcc+msvc is not correctly handled in Boost.Preprocessor module. See [this issue](https://github.com/boostorg/preprocessor/issues/15). In the meantime, the recommended workaround is to disable Variadic Macro support in Boost.
I created a [PR](https://github.com/boostorg/preprocessor/pull/18) in the Boost repo on GitHub to fix this.
_Note_: The library sources have been fixed, but this fix is still needed until the next release of Boost.Preprocessor
10. __Provide implementation for mkstemp / Fix temporary file creation__
The code explicitely disabled the creation of temporary files because "mkstemp not available in Windows". In fact, `mktemp` and `unlink` are both implemented, but they don't work as expected. I used `tempnam` to replace `mkstemp`, and added the flag `_O_TEMPORARY` to the parameters of `open` to automatically delete the file when it is closed. If `unlinkEarly` is not set, I added a call to `remove` in the destructor to delete the file after its closure.
I also handled the case of the default value for the `base` parameter: the path `\tmp` doesnot exist on Windows, so it is replaced by the value of the `%TMP%` environment variable in `NormalizeTempPrefix`.
11. __Revert commit #2f8b093 + Fix copy/paste error while fixing #301 + restrict fix to MSVC compiler.__
cf [Issue #301](https://github.com/marian-nmt/marian-dev/issues/301)