Merge branch 'master' into mjd/syncWithPublic

CMakeLists.txt

@@ -20,7 +20,7 @@ option(USE_NCCL "Use NCCL library" ON)
 option(USE_MPI "Use MPI library" OFF)
 option(COMPILE_EXAMPLES "Compile examples" OFF)
 option(COMPILE_TESTS "Compile tests" OFF)
-option(COMPILE_SERVER "Compile marian-server" ON)
+option(COMPILE_SERVER "Compile marian-server" OFF)
 option(USE_FBGEMM "Use FBGEMM" ON)
 option(USE_DOXYGEN "Build documentation with Doxygen" ON)
 
@@ -300,7 +300,7 @@ if(COMPILE_CPU)
   endif(MKL_FOUND)
 endif(COMPILE_CPU)
 
-set(BOOST_COMPONENTS timer iostreams filesystem system chrono)
+set(BOOST_COMPONENTS "")
 if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 4.9)
   add_definitions(-DUSE_BOOST_REGEX=1)
   set(BOOST_COMPONENTS ${BOOST_COMPONENTS} regex)
@@ -315,6 +315,7 @@ if(COMPILE_SERVER)
     message(STATUS "Found OpenSSL")
     include_directories(${OPENSSL_INCLUDE_DIR})
     set(EXT_LIBS ${EXT_LIBS} ${OPENSSL_CRYPTO_LIBRARY})
+    set(BOOST_COMPONENTS ${BOOST_COMPONENTS} system)
   else(OpenSSL_FOUND)
     message(WARNING "Cannot find OpenSSL library. Not compiling server.")
     set(COMPILE_SERVER "off")
@@ -329,14 +330,16 @@ if(USE_STATIC_LIBS)
   set(Boost_USE_STATIC_LIBS ON)
 endif()
 
-find_package(Boost COMPONENTS ${BOOST_COMPONENTS})
-if(Boost_FOUND)
-  include_directories(${Boost_INCLUDE_DIRS})
-  set(EXT_LIBS ${EXT_LIBS} ${Boost_LIBRARIES})
-  set(EXT_LIBS ${EXT_LIBS} ${ZLIB_LIBRARIES}) # hack for static compilation
-else(Boost_FOUND)
-  message(SEND_ERROR "Cannot find Boost libraries. Terminating.")
-endif(Boost_FOUND)
+if(BOOST_COMPONENTS)
+  find_package(Boost COMPONENTS ${BOOST_COMPONENTS})
+  if(Boost_FOUND)
+    include_directories(${Boost_INCLUDE_DIRS})
+    set(EXT_LIBS ${EXT_LIBS} ${Boost_LIBRARIES})
+    set(EXT_LIBS ${EXT_LIBS} ${ZLIB_LIBRARIES}) # hack for static compilation
+  else(Boost_FOUND)
+    message(SEND_ERROR "Cannot find Boost libraries. Terminating.")
+  endif(Boost_FOUND)
+endif(BOOST_COMPONENTS)
 
 if(COMPILE_TESTS)
   enable_testing()

src/CMakeLists.txt

@@ -217,13 +217,6 @@ foreach(exec ${EXECUTABLES})
   set_target_properties(${exec} PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}")
 endforeach(exec)
 
-#add_executable(
-#  align2steps
-#  tools/align2steps.cpp
-#)
-
-#set_target_properties(align2steps PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}")
-
 if(COMPILE_TESTS)
   set(CATCH_INCLUDE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/3rd_party)
   add_library(Catch INTERFACE)

src/common/timer.h

@@ -1,25 +1,8 @@
 #pragma once
 
-#ifdef __GNUC__
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wsuggest-override"
-#endif
-#ifndef NO_BOOST
-#include <boost/timer/timer.hpp>
-#endif
-#ifdef __GNUC__
-#pragma GCC diagnostic pop
-#endif
-
-#ifdef _MSC_VER
-// (needed on Windows only to resolve a link error, but causes a warning on Linux)
-#ifndef NO_BOOST
-#include <boost/chrono.hpp>
-#endif
-#endif
-
-#include <chrono>
+#include <iostream>
 #include <sstream>
+#include <chrono>
 
 namespace marian {
 namespace timer {
@@ -33,7 +16,6 @@ static inline std::string currentDate() {
 }
 
 // Timer measures elapsed time.
-// This is a wrapper around std::chrono providing wall time only
 class Timer {
 protected:
   using clock = std::chrono::steady_clock;
@@ -93,14 +75,9 @@ public:
   }
 };
 
-// @TODO: replace with a timer providing CPU/thread time on both Linux and Windows. This is required
-// for auto-tuner.
-// Check get_clocktime on Linux: https://linux.die.net/man/3/clock_gettime
-// Check GetThreadTimes on Windows:
-// https://docs.microsoft.com/en-gb/windows/desktop/api/processthreadsapi/nf-processthreadsapi-getthreadtimes
-#ifndef NO_BOOST
-using CPUTimer = boost::timer::cpu_timer;
-#endif
+// std::chrono::steady_clock seems to be the right choice here.
+using CPUTimer = Timer;
+
 
 }  // namespace timer
 }  // namespace marian

src/data/corpus.cpp

@@ -190,7 +190,6 @@ void Corpus::shuffleData(const std::vector<std::string>& paths) {
     LOG(info, "[data] Done shuffling {} sentences (cached in RAM)", numSentences);
   }
   else {
-#ifndef NO_BOOST
     // create temp files that contain the data in randomized order
     tempFiles_.resize(numStreams);
     for(size_t i = 0; i < numStreams; ++i) {
@@ -210,9 +209,6 @@ void Corpus::shuffleData(const std::vector<std::string>& paths) {
       files_[i] = std::move(inputStream);
     }
     LOG(info, "[data] Done shuffling {} sentences to temp files", numSentences);
-#else
-    ABORT("Shuffling to temp file is not available when compiling with NO_BOOST option. Use --shuffle-in-ram.");
-#endif
   }
   pos_ = 0;
 }

src/graph/auto_tuner.h

@@ -25,9 +25,7 @@ private:
   // collects the statistics.
   const size_t collectStatMax = 50;
 
-#ifndef NO_BOOST
   UPtr<timer::CPUTimer> timer_;
-#endif
 
   // This structure holds a hash key an algorithm function (e.g. int16, packed gemm, mkl gemm)
   // for a specific operation size
@@ -93,38 +91,28 @@ public:
   Return run(Args... args) { return algorithms_[choose()].algorithm(args...); }
 
   void start(size_t hash) override {
-#ifndef NO_BOOST
     if(!timer_ && done_.count(hash) == 0)
       timer_.reset(new timer::CPUTimer());
-#else
-    hash;
-    LOG_ONCE(warn, "Auto-tuner not available when compiling with NO_BOOST option. Will use the first option.");
-#endif
   }
 
   void stop(size_t hash, bool stop) override {
-#ifndef NO_BOOST
     if(stop && done_.count(hash) == 0) {
       timer_->stop();
 
-      typedef std::chrono::duration<double> sec;
-      sec seconds = std::chrono::nanoseconds(timer_->elapsed().user);
+      auto seconds = timer_->elapsed();
 
       auto it = stats_.find(hash);
       if(it != stats_.end()) {
         if(it->second.runs < collectStatMax) {
-          it->second.time += seconds.count();
+          it->second.time += seconds;
           it->second.runs += 1;
         }
       } else {
-        stats_.emplace(hash, Stat({seconds.count(), 1}));
+        stats_.emplace(hash, Stat({seconds, 1}));
       }
 
       timer_.reset(nullptr);
     }
-#else
-    hash; stop;
-#endif
   }
 };
 

src/tensors/cpu/expanded_gemm.h

@@ -4,7 +4,18 @@
 #include "tensors/cpu/sharp/packed_gemm.h"
 
 #if USE_FBGEMM
+
+#ifdef __GNUC__
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wunused-variable"
+#endif
+
 #include "3rd_party/fbgemm/include/fbgemm/FbgemmFP16.h"
+
+#ifdef __GNUC__
+#pragma GCC diagnostic pop
+#endif
+
 using namespace fbgemm;
 #endif  // USE_FBGEMM
 

src/tensors/cpu/sharp/packed_gemm.cpp

@@ -19,10 +19,21 @@
 //#pragma comment(linker, "/ignore:4049") // locally defined symbol ...asmjit... imported
 //#pragma comment(linker, "/ignore:4217") // locally defined symbol ...asmjit... imported
 #endif
+
+
+#ifdef __GNUC__
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wunused-variable"
+#endif
+
 #include "3rd_party/fbgemm/include/fbgemm/FbgemmFP16.h"
 #include "3rd_party/fbgemm/include/fbgemm/QuantUtils.h"
 #include "3rd_party/fbgemm/include/fbgemm/Fbgemm.h"
 
+#ifdef __GNUC__
+#pragma GCC diagnostic pop
+#endif
+
 #ifdef _OPENMP
 #include <omp.h>
 #endif