marian-nmt/marian
1
1
Fork 0
mirror of https://github.com/marian-nmt/marian.git synced 2024-09-17 09:47:34 +03:00
Code Issues Packages Projects Releases Wiki Activity

Pile of truly horrible hacks to *almost* get things building on windows

Browse Source
This commit is contained in:
Jon Clark 2018-05-24 17:05:31 -07:00
parent fd5ea9f6b7
commit 3e68deff52
18 changed files with 76 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
CMakeLists.txt
View File

@ -23,9 +23,12 @@ message(STATUS "Project name: ${PROJECT_NAME}")
message(STATUS "Project version: ${PROJECT_VERSION_STRING_FULL}")
# Set compilation flags
set(CMAKE_CXX_FLAGS_RELEASE " -std=c++11 -g -O3 -Ofast -m64 -pthread -march=native -Wl,--no-as-needed -funroll-loops -ffinite-math-only -fPIC -Wno-unused-result -Wno-deprecated -Wno-deprecated-gpu-targets")
set(CMAKE_CXX_FLAGS_DEBUG " -std=c++11 -g -O0 -pthread -fPIC -Wno-unused-result -Wno-deprecated -Wno-deprecated-gpu-targets")
set(CMAKE_CXX_FLAGS_PROFILE "${CMAKE_CXX_FLAGS_RELEASE} -pg -g")
# -Wl,--no-as-needed -Wno-unused-result -Wno-deprecated -Wno-deprecated-gpu-targets -funroll-loops -ffinite-math-only
# -Wno-unused-result -Wno-deprecated -Wno-deprecated-gpu-targets
# -g
set(CMAKE_CXX_FLAGS_RELEASE " -std=c++ -O3 -Ofast -m64 -pthread -march=native -fPIC -DBOOST_NO_UNREACHABLE_RETURN_DETECTION")
set(CMAKE_CXX_FLAGS_DEBUG " -std=c++11 -O0 -pthread -fPIC")
set(CMAKE_CXX_FLAGS_PROFILE "${CMAKE_CXX_FLAGS_RELEASE} -pg")
#set(CMAKE_CXX_FLAGS_PROFGEN "${CMAKE_CXX_FLAGS_RELEASE} -fprofile-generate -fprofile-correction")
#set(CMAKE_CXX_FLAGS_PROFUSE "${CMAKE_CXX_FLAGS_RELEASE} -fprofile-use -fprofile-correction")
set(CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS_RELEASE})
@ -153,6 +156,9 @@ find_package(Boost COMPONENTS ${BOOST_COMPONENTS})
if(Boost_FOUND)
include_directories(${Boost_INCLUDE_DIRS})
set(EXT_LIBS ${EXT_LIBS} ${Boost_LIBRARIES})
#message("Found boost with toolset compiler ${_boost_COMPILER}")
#set(_boost_COMPILER "-vc140")
#message("Overwrote compiler version as ${_boost_COMPILER}")
else(Boost_FOUND)
message(SEND_ERROR "Cannot find Boost libraries. Terminating.")
endif(Boost_FOUND)

23
cmake/FindMKL.cmake
View File

@ -35,10 +35,17 @@ if (MKL_INCLUDE_DIRS AND MKL_LIBRARIES AND MKL_INTERFACE_LIBRARY AND
endif()
if(NOT BUILD_SHARED_LIBS)
set(INT_LIB "libmkl_intel_ilp64.a")
set(SEQ_LIB "libmkl_sequential.a")
set(THR_LIB "libmkl_intel_thread.a")
set(COR_LIB "libmkl_core.a")
if (WIN32)
set(INT_LIB "mkl_intel_ilp64.lib")
set(SEQ_LIB "mkl_sequential.lib")
set(THR_LIB "mkl_intel_thread.lib")
set(COR_LIB "mkl_core.lib")
else()
set(INT_LIB "libmkl_intel_ilp64.a")
set(SEQ_LIB "libmkl_sequential.a")
set(THR_LIB "libmkl_intel_thread.a")
set(COR_LIB "libmkl_core.a")
endif()
else()
set(INT_LIB "mkl_intel_ilp64")
set(SEQ_LIB "mkl_sequential")
@ -57,6 +64,7 @@ find_library(MKL_INTERFACE_LIBRARY
NAMES ${INT_LIB}
PATHS ${MKL_ROOT}/lib
${MKL_ROOT}/lib/intel64
${MKL_ROOT}/lib/intel64_win
${INTEL_ROOT}/mkl/lib/intel64
NO_DEFAULT_PATH)
@ -77,11 +85,17 @@ find_library(MKL_CORE_LIBRARY
set(MKL_INCLUDE_DIRS ${MKL_INCLUDE_DIR})
set(MKL_LIBRARIES ${MKL_INTERFACE_LIBRARY} ${MKL_SEQUENTIAL_LAYER_LIBRARY} ${MKL_CORE_LIBRARY})
message("1 ${MKL_INCLUDE_DIR}")
message("2 ${MKL_INTERFACE_LIBRARY}")
message("3 ${MKL_SEQUENTIAL_LAYER_LIBRARY}")
message("4 ${MKL_CORE_LIBRARY}")
if (MKL_INCLUDE_DIR AND
MKL_INTERFACE_LIBRARY AND
MKL_SEQUENTIAL_LAYER_LIBRARY AND
MKL_CORE_LIBRARY)
if (NOT DEFINED ENV{CRAY_PRGENVPGI} AND
NOT DEFINED ENV{CRAY_PRGENVGNU} AND
NOT DEFINED ENV{CRAY_PRGENVCRAY} AND
@ -93,7 +107,6 @@ if (MKL_INCLUDE_DIR AND
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DMKL_ILP64 ${ABI}")
else()
set(MKL_INCLUDE_DIRS "")
set(MKL_LIBRARIES "")
set(MKL_INTERFACE_LIBRARY "")

BIN
cmake_doze.txt Normal file
View File

Binary file not shown.

4
src/common/config.h
View File

@ -1,7 +1,7 @@
#pragma once
#include <sys/ioctl.h>
#include <unistd.h>
//#include <sys/ioctl.h>
//#include <unistd.h>
#include <boost/program_options.hpp>
#include "3rd_party/yaml-cpp/yaml.h"

4
src/common/config_parser.cpp
View File

@ -1134,11 +1134,11 @@ std::vector<DeviceId> ConfigParser::getDevices() {
// does this make sense?
devices.push_back({ds.size(), DeviceType::gpu});
for(auto d : ds)
devices.push_back({std::stoull(d), DeviceType::gpu});
devices.push_back({(size_t)std::stoull(d), DeviceType::gpu});
}
} else {
for(auto d : Split(devicesStr))
devices.push_back({std::stoull(d), DeviceType::gpu});
devices.push_back({(size_t)std::stoull(d), DeviceType::gpu});
}
if(config_["cpu-threads"].as<size_t>() > 0) {

4
src/common/config_parser.h
View File

@ -2,8 +2,8 @@
#include <boost/program_options.hpp>
#include <sys/ioctl.h>
#include <unistd.h>
//#include <sys/ioctl.h>
//#include <unistd.h>
#include "3rd_party/yaml-cpp/yaml.h"
#include "common/definitions.h"
#include "common/file_stream.h"

28
src/common/file_stream.h
View File

@ -1,9 +1,10 @@
#pragma once
#include <boost/filesystem.hpp>
#include <boost/filesystem/fstream.hpp>
#include <boost/iostreams/device/file_descriptor.hpp>
#include <boost/iostreams/filter/gzip.hpp>
//#include <boost/iostreams/filter/gzip.hpp>
#include <boost/iostreams/filtering_stream.hpp>
#include <iostream>
@ -21,11 +22,15 @@ private:
std::string name_;
int mkstemp_and_unlink(char* tmpl) {
ABORT_IF(true, "NYI");
return 0;
/*
int ret = mkstemp(tmpl);
if(unlink_ && ret != -1) {
ABORT_IF(unlink(tmpl), "Error while deleting '{}'", tmpl);
}
return ret;
*/
}
int MakeTemp(const std::string& base) {
@ -41,6 +46,9 @@ private:
}
void NormalizeTempPrefix(std::string& base) {
ABORT_IF(true, "NYI");
return;
/*
if(base.empty())
return;
if(base[base.size() - 1] == '/')
@ -51,11 +59,13 @@ private:
return;
if(S_ISDIR(sb.st_mode))
base += '/';
*/
}
public:
TemporaryFile(const std::string base = "/tmp/", bool earlyUnlink = true)
: unlink_(earlyUnlink) {
ABORT_IF(true, "NYI (see destructor)");
std::string baseTemp(base);
NormalizeTempPrefix(baseTemp);
fd_ = MakeTemp(baseTemp);
@ -65,10 +75,12 @@ public:
if(fd_ != -1 && !unlink_) {
ABORT_IF(unlink(name_.c_str()), "Error while deleting '{}'", name_);
}
/*
if(fd_ != -1 && close(fd_)) {
std::cerr << "Could not close file " << fd_ << std::endl;
std::abort();
}
*/
}
int getFileDescriptor() { return fd_; }
@ -82,14 +94,15 @@ public:
ABORT_IF(
!boost::filesystem::exists(file_), "File '{}' does not exist", file);
if(file_.extension() == ".gz")
istream_.push(io::gzip_decompressor());
//if(file_.extension() == ".gz")
// istream_.push(io::gzip_decompressor());
istream_.push(ifstream_);
}
InputFileStream(TemporaryFile& tempfile)
: fds_(tempfile.getFileDescriptor(), io::never_close_handle) {
lseek(tempfile.getFileDescriptor(), 0, SEEK_SET);
ABORT_IF(true, "NYI");
//lseek(tempfile.getFileDescriptor(), 0, SEEK_SET);
istream_.push(fds_, 1024);
}
@ -122,14 +135,15 @@ public:
ABORT_IF(
!boost::filesystem::exists(file_), "File '{}' does not exist", file);
if(file_.extension() == ".gz")
ostream_.push(io::gzip_compressor());
//if(file_.extension() == ".gz")
// ostream_.push(io::gzip_compressor());
ostream_.push(ofstream_);
}
OutputFileStream(TemporaryFile& tempfile)
: fds_(tempfile.getFileDescriptor(), io::never_close_handle) {
lseek(tempfile.getFileDescriptor(), 0, SEEK_SET);
ABORT_IF(true, "NYI");
//lseek(tempfile.getFileDescriptor(), 0, SEEK_SET);
ostream_.push(fds_, 1024);
}

2
src/common/logging.cpp
View File

@ -37,7 +37,7 @@ bool setLoggingLevel(spdlog::logger& logger, std::string const level) {
logger.set_level(spdlog::level::info);
else if(level == "warn")
logger.set_level(spdlog::level::warn);
else if(level == "err" or level == "error")
else if(level == "err" || level == "error")
logger.set_level(spdlog::level::err);
else if(level == "critical")
logger.set_level(spdlog::level::critical);

4
src/common/utils.cpp
View File

@ -64,6 +64,9 @@ std::string Join(const std::vector<std::string>& words,
}
std::string Exec(const std::string& cmd) {
ABORT_IF(true, "NYI");
return "";
/*
std::array<char, 128> buffer;
std::string result;
std::shared_ptr<std::FILE> pipe(popen(cmd.c_str(), "r"), pclose);
@ -75,4 +78,5 @@ std::string Exec(const std::string& cmd) {
result += buffer.data();
}
return result;
*/
}

1
src/data/corpus.cpp
View File

@ -1,4 +1,5 @@
#include <random>
#include <numeric>
#include "data/corpus.h"

2
src/functional/shape.h
View File

@ -36,7 +36,7 @@ struct ConstantShape {
bstride_(shape.bstride_),
elements_(shape.elements_) {}
ConstantShape(const Shape& shape) {
ConstantShape(const marian::Shape& shape) {
size_t filled = shape.size();
ABORT_IF(filled > N,

2
src/graph/node_operators_unary.h
View File

@ -293,7 +293,7 @@ struct PReLUNodeOp : public UnaryNodeOp {
}
private:
float alpha_{0.01};
float alpha_{0.01f};
};
/**

8
src/tensors/cpu/device.cpp
View File

@ -1,6 +1,9 @@
#include "tensors/device.h"
#include <iostream>
//#if DOZE
#include <malloc.h>
#include <stdlib.h>
namespace marian {
@ -12,6 +15,11 @@ Device::~Device() {
size_ = 0;
}
// #if DOZE
void* aligned_alloc(size_t alignment, size_t size) {
return _aligned_malloc(size, alignment);
}
void Device::reserve(size_t size) {
size = align(size);
ABORT_IF(size < size_ || size == 0,

2
src/training/graph_group_async.h
View File

@ -42,7 +42,7 @@ protected:
std::vector<Tensor> paramsAvg_;
std::vector<Ptr<TensorAllocator>> paramsAllocAvg_;
bool movingAvg_{false};
float mvDecay_{1e-4};
float mvDecay_{1e-4f};
std::unique_ptr<ThreadPool> pool_;

3
src/training/graph_group_async_drop.cpp
View File

@ -70,9 +70,10 @@ void AsyncGraphGroupDrop::fetchParams(Tensor oldParams,
pos += shardSize_;
}
/* OMG BAD
for(auto&& t : threads) {
t.join();
}
}*/
fetchStep_[device_id]++;
}

2
src/training/graph_group_singleton.h
View File

@ -21,7 +21,7 @@ private:
Ptr<ExpressionGraph> mvAvgGraph_;
bool mvAvg_{false};
float mvDecay_{1e-4};
float mvDecay_{1e-4f};
void updateMovingAverage(Tensor mvAvgParams, Tensor params, size_t batches);

2
src/training/graph_group_sync.h
View File

@ -29,7 +29,7 @@ private:
std::vector<Tensor> paramsAvg_;
std::vector<Ptr<TensorAllocator>> paramsAllocAvg_;
bool movingAvg_{false};
float mvDecay_{1e-4};
float mvDecay_{1e-4f};
size_t delay_{1};
void updateMovingAverage(Tensor paramsAvg, Tensor params, size_t batches);

2
src/translator/beam_search.h
View File

@ -27,7 +27,7 @@ public:
? options_->get<size_t>("beam-size")
: 3) {}
Beams toHyps(const std::vector<uint> keys,
Beams toHyps(const std::vector<uint32_t> keys,
const std::vector<float> costs,
size_t vocabSize,
const Beams& beams,