Add simple unit tests for binary files (#826)

* unit tests for binary file operations
* adjust changelog
* Set file_ in TemporaryFile for MSVC

Co-authored-by: Roman Grundkiewicz <rgrundkiewicz@gmail.com>

CHANGELOG.md

@@ -15,6 +15,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 - Correct training with fp16 via `--fp16`. 
 - Dynamic cost-scaling with `--cost-scaling`.
 - Dynamic gradient-scaling with `--dynamic-gradient-scaling`.
+- Add unit tests for binary files.
 - Fix compilation with OMP
 
 ### Fixed

src/common/file_stream.cpp

@@ -161,6 +161,7 @@ void TemporaryFile::MakeTemp(const std::string &base) {
   int fd = open(name, oflag, _S_IREAD | _S_IWRITE);
   ABORT_IF(fd == -1, "Error while making a temporary based on '{}'", base);
 
+  file_ = std::string(name);
 #else
   // create temp file
   std::string name(base);

src/tests/units/CMakeLists.txt

@@ -6,6 +6,7 @@ set(UNIT_TESTS
     attention_tests
     fastopt_tests
     utils_tests
+    binary_tests
     # cosmos_tests # optional, uncomment to test with specific files.
 )
 

src/tests/units/binary_tests.cpp

@@ -0,0 +1,61 @@
+#include "catch.hpp"
+#include "common/binary.h"
+#include "common/file_stream.h"
+
+#include "3rd_party/mio/mio.hpp"
+
+using namespace marian;
+
+TEST_CASE("a few operations on binary files", "[binary]") {
+
+  SECTION("Save two items to temporary binary file and then load and map") {
+
+    // Create a temporary file that we will only use for the file name
+    io::TemporaryFile temp("/tmp/", /*earlyUnlink=*/false);
+    io::Item item1, item2;
+
+    {
+      std::vector<float>    v1 = { 3.14, 2.71, 1.0, 0.0, 1.41 };
+      std::vector<uint16_t> v2 = { 5, 4, 3, 2, 1, 0 };
+
+      item1.name  = "item1";
+      item1.shape = { 5, 1 };
+      item1.type  = Type::float32;
+      item1.bytes.resize(v1.size() * sizeof(float));
+      std::copy((char*)v1.data(), (char*)v1.data() + v1.size() * sizeof(float), item1.bytes.data());
+
+      item2.name  = "item2";
+      item2.shape = { 2, 3 };
+      item2.type = Type::uint16;
+      item2.bytes.resize(v2.size() * sizeof(uint32_t));
+      std::copy((char*)v2.data(), (char*)v2.data() + v2.size() * sizeof(uint16_t), item2.bytes.data());
+      
+      std::vector<io::Item> items = {item1, item2};
+      io::binary::saveItems(temp.getFileName(), items);
+    }
+
+    { // test loading
+      std::vector<io::Item> items;
+      io::binary::loadItems(temp.getFileName(), items);
+
+      CHECK( item1.name == items[0].name );
+      CHECK( item2.name == items[1].name );
+    
+      CHECK( std::equal(item1.data(), item1.data() + item1.size(), items[0].data()) );
+      CHECK( std::equal(item2.data(), item2.data() + item2.size(), items[1].data()) );
+    }
+  
+    { // test mmapping
+      mio::mmap_source mmap(temp.getFileName());
+      
+      std::vector<io::Item> items;
+      io::binary::loadItems(mmap.data(), items, /*mapped=*/true);
+
+      CHECK( item1.name == items[0].name );
+      CHECK( item2.name == items[1].name );
+
+      CHECK( std::equal(item1.data(), item1.data() + item1.size(), items[0].data()) );
+      CHECK( std::equal(item2.data(), item2.data() + item2.size(), items[1].data()) );
+    }
+  }
+}