Rewrite FeatureData::setFeatureMap(); add the unit test.

mert/FeatureData.cpp

@@ -137,14 +137,14 @@ void FeatureData::setFeatureMap(const std::string& feat)
   number_of_features = 0;
   features = feat;
 
-  std::string substring, stringBuf;
-  stringBuf = features;
-  while (!stringBuf.empty()) {
-    getNextPound(stringBuf, substring);
-
-    featname2idx_[substring] = idx2featname_.size();
-    idx2featname_[idx2featname_.size()] = substring;
-    number_of_features++;
+  vector<string> buf;
+  Tokenize(feat.c_str(), ' ', &buf);
+  for (vector<string>::const_iterator it = buf.begin();
+       it != buf.end(); ++it) {
+    const size_t size = idx2featname_.size();
+    featname2idx_[*it] = size;
+    idx2featname_[size] = *it;
+    ++number_of_features;
   }
 }
 

mert/FeatureDataTest.cpp

@@ -0,0 +1,39 @@
+#include "FeatureData.h"
+
+#define BOOST_TEST_MODULE FeatureData
+#include <boost/test/unit_test.hpp>
+
+#include <cstdio>
+
+namespace {
+
+void CheckFeatureMap(const FeatureData* feature_data,
+                     const char* str, int num_feature, int* cnt) {
+  char tmp[32];
+  for (int i = 0; i < num_feature; ++i) {
+    std::snprintf(tmp, sizeof(tmp), "%s_%d", str, i);
+    BOOST_CHECK_EQUAL(feature_data->getFeatureIndex(tmp), *cnt);
+    BOOST_CHECK_EQUAL(feature_data->getFeatureName(*cnt).c_str(), tmp);
+    ++(*cnt);
+  }
+}
+
+} // namespace
+
+BOOST_AUTO_TEST_CASE(set_feature_map) {
+  std::string str("d_0 d_1 d_2 d_3 d_4 d_5 d_6 lm_0 lm_1 tm_0 tm_1 tm_2 tm_3 tm_4 w_0 ");
+  FeatureData feature_data;
+
+  feature_data.setFeatureMap(str);
+
+  BOOST_REQUIRE(feature_data.Features() == str);
+  BOOST_REQUIRE(feature_data.NumberOfFeatures() == 15);
+
+  int cnt = 0;
+  CheckFeatureMap(&feature_data, "d", 7, &cnt);
+  CheckFeatureMap(&feature_data, "lm", 2, &cnt);
+  CheckFeatureMap(&feature_data, "tm", 5, &cnt);
+
+  BOOST_CHECK_EQUAL(feature_data.getFeatureIndex("w_0"), cnt);
+  BOOST_CHECK_EQUAL(feature_data.getFeatureName(cnt).c_str(), "w_0");
+}

mert/Jamfile

@@ -44,6 +44,7 @@ exe pro : pro.cpp mert_lib ..//boost_program_options ;
 
 alias programs : mert extractor evaluator pro ;
 
+unit-test feature_data_test : FeatureDataTest.cpp mert_lib ..//boost_unit_test_framework ;
 unit-test data_test : DataTest.cpp mert_lib ..//boost_unit_test_framework ;
 unit-test timer_test : TimerTest.cpp mert_lib ..//boost_unit_test_framework ;
 unit-test util_test : UtilTest.cpp mert_lib ..//boost_unit_test_framework ;