mirror of
https://github.com/moses-smt/mosesdecoder.git
synced 2024-12-27 05:55:02 +03:00
Remove hard-coded "/dev/stdout".
This will improve the portability. We also change the interface of I/O functions for ease of the development unit tests.
This commit is contained in:
parent
3ce46da4cd
commit
5d600f2b50
@ -6,94 +6,104 @@
|
|||||||
*
|
*
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
#include <fstream>
|
||||||
#include "FeatureArray.h"
|
#include "FeatureArray.h"
|
||||||
#include "FileStream.h"
|
#include "FileStream.h"
|
||||||
#include "Util.h"
|
#include "Util.h"
|
||||||
|
|
||||||
|
|
||||||
FeatureArray::FeatureArray()
|
FeatureArray::FeatureArray()
|
||||||
: m_index(""), m_num_features(0), m_sparse_flag(false) {}
|
: m_index(""), m_num_features(0), m_sparse_flag(false) {}
|
||||||
|
|
||||||
FeatureArray::~FeatureArray() {}
|
FeatureArray::~FeatureArray() {}
|
||||||
|
|
||||||
void FeatureArray::savetxt(std::ofstream& outFile)
|
void FeatureArray::savetxt(ostream* os)
|
||||||
{
|
{
|
||||||
outFile << FEATURES_TXT_BEGIN << " " << m_index << " " << m_array.size()
|
*os << FEATURES_TXT_BEGIN << " " << m_index << " " << m_array.size()
|
||||||
<< " " << m_num_features << " " << m_features << std::endl;
|
<< " " << m_num_features << " " << m_features << endl;
|
||||||
for (featarray_t::iterator i = m_array.begin(); i != m_array.end(); ++i) {
|
for (featarray_t::iterator i = m_array.begin(); i != m_array.end(); ++i) {
|
||||||
i->savetxt(outFile);
|
i->savetxt(os);
|
||||||
outFile << std::endl;
|
*os << endl;
|
||||||
}
|
}
|
||||||
outFile << FEATURES_TXT_END << std::endl;
|
*os << FEATURES_TXT_END << endl;
|
||||||
}
|
}
|
||||||
|
|
||||||
void FeatureArray::savebin(std::ofstream& outFile)
|
void FeatureArray::savebin(ostream* os)
|
||||||
{
|
{
|
||||||
outFile << FEATURES_BIN_BEGIN << " " << m_index << " " << m_array.size()
|
*os << FEATURES_BIN_BEGIN << " " << m_index << " " << m_array.size()
|
||||||
<< " " << m_num_features << " " << m_features << std::endl;
|
<< " " << m_num_features << " " << m_features << endl;
|
||||||
for (featarray_t::iterator i = m_array.begin(); i != m_array.end(); ++i)
|
for (featarray_t::iterator i = m_array.begin(); i != m_array.end(); ++i)
|
||||||
i->savebin(outFile);
|
i->savebin(os);
|
||||||
|
|
||||||
outFile << FEATURES_BIN_END << std::endl;
|
*os << FEATURES_BIN_END << endl;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
void FeatureArray::save(std::ofstream& inFile, bool bin)
|
void FeatureArray::save(ostream* os, bool bin)
|
||||||
{
|
{
|
||||||
if (size()>0)
|
if (size() <= 0) return;
|
||||||
(bin)?savebin(inFile):savetxt(inFile);
|
if (bin) {
|
||||||
|
savebin(os);
|
||||||
|
} else {
|
||||||
|
savetxt(os);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void FeatureArray::save(const std::string &file, bool bin)
|
void FeatureArray::save(const string &file, bool bin)
|
||||||
{
|
{
|
||||||
|
ofstream ofs(file.c_str(), ios::out);
|
||||||
std::ofstream outFile(file.c_str(), std::ios::out); // matches a stream with a file. Opens the file
|
if (!ofs) {
|
||||||
|
cerr << "Failed to open " << file << endl;
|
||||||
save(outFile);
|
exit(1);
|
||||||
|
}
|
||||||
outFile.close();
|
ostream *os = &ofs;
|
||||||
|
save(os, bin);
|
||||||
|
ofs.close();
|
||||||
}
|
}
|
||||||
|
|
||||||
void FeatureArray::loadbin(ifstream& inFile, size_t n)
|
void FeatureArray::save(bool bin)
|
||||||
|
{
|
||||||
|
save(&cout, bin);
|
||||||
|
}
|
||||||
|
|
||||||
|
void FeatureArray::loadbin(istream* is, size_t n)
|
||||||
{
|
{
|
||||||
FeatureStats entry(m_num_features);
|
FeatureStats entry(m_num_features);
|
||||||
|
for (size_t i = 0 ; i < n; i++) {
|
||||||
for (size_t i=0 ; i < n; i++) {
|
entry.loadbin(is);
|
||||||
entry.loadbin(inFile);
|
|
||||||
add(entry);
|
add(entry);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void FeatureArray::loadtxt(ifstream& inFile, size_t n)
|
void FeatureArray::loadtxt(istream* is, size_t n)
|
||||||
{
|
{
|
||||||
FeatureStats entry(m_num_features);
|
FeatureStats entry(m_num_features);
|
||||||
|
|
||||||
for (size_t i=0 ; i < n; i++) {
|
for (size_t i = 0; i < n; i++) {
|
||||||
entry.loadtxt(inFile);
|
entry.loadtxt(is);
|
||||||
add(entry);
|
add(entry);
|
||||||
if (entry.getSparse().size()>0)
|
if (entry.getSparse().size()>0)
|
||||||
m_sparse_flag = true;
|
m_sparse_flag = true;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void FeatureArray::load(ifstream& inFile)
|
void FeatureArray::load(istream* is)
|
||||||
{
|
{
|
||||||
size_t number_of_entries=0;
|
size_t number_of_entries = 0;
|
||||||
bool binmode=false;
|
bool binmode = false;
|
||||||
|
|
||||||
std::string substring, stringBuf;
|
string substring, stringBuf;
|
||||||
std::string::size_type loc;
|
string::size_type loc;
|
||||||
|
|
||||||
std::getline(inFile, stringBuf);
|
getline(*is, stringBuf);
|
||||||
if (!inFile.good()) {
|
if (!is->good()) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!stringBuf.empty()) {
|
if (!stringBuf.empty()) {
|
||||||
if ((loc = stringBuf.find(FEATURES_TXT_BEGIN)) == 0) {
|
if ((loc = stringBuf.find(FEATURES_TXT_BEGIN)) == 0) {
|
||||||
binmode=false;
|
binmode = false;
|
||||||
} else if ((loc = stringBuf.find(FEATURES_BIN_BEGIN)) == 0) {
|
} else if ((loc = stringBuf.find(FEATURES_BIN_BEGIN)) == 0) {
|
||||||
binmode=true;
|
binmode = true;
|
||||||
} else {
|
} else {
|
||||||
TRACE_ERR("ERROR: FeatureArray::load(): Wrong header");
|
TRACE_ERR("ERROR: FeatureArray::load(): Wrong header");
|
||||||
return;
|
return;
|
||||||
@ -108,33 +118,35 @@ void FeatureArray::load(ifstream& inFile)
|
|||||||
m_features = stringBuf;
|
m_features = stringBuf;
|
||||||
}
|
}
|
||||||
|
|
||||||
(binmode)?loadbin(inFile, number_of_entries):loadtxt(inFile, number_of_entries);
|
if (binmode) {
|
||||||
|
loadbin(is, number_of_entries);
|
||||||
|
} else {
|
||||||
|
loadtxt(is, number_of_entries);
|
||||||
|
}
|
||||||
|
|
||||||
std::getline(inFile, stringBuf);
|
getline(*is, stringBuf);
|
||||||
if (!stringBuf.empty()) {
|
if (!stringBuf.empty()) {
|
||||||
if ((loc = stringBuf.find(FEATURES_TXT_END)) != 0 && (loc = stringBuf.find(FEATURES_BIN_END)) != 0) {
|
if ((loc = stringBuf.find(FEATURES_TXT_END)) != 0 &&
|
||||||
|
(loc = stringBuf.find(FEATURES_BIN_END)) != 0) {
|
||||||
TRACE_ERR("ERROR: FeatureArray::load(): Wrong footer");
|
TRACE_ERR("ERROR: FeatureArray::load(): Wrong footer");
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void FeatureArray::load(const std::string &file)
|
void FeatureArray::load(const string &file)
|
||||||
{
|
{
|
||||||
TRACE_ERR("loading data from " << file << std::endl);
|
TRACE_ERR("loading data from " << file << endl);
|
||||||
|
inputfilestream input_stream(file); // matches a stream with a file. Opens the file
|
||||||
inputfilestream inFile(file); // matches a stream with a file. Opens the file
|
istream* is = &input_stream;
|
||||||
|
load(is);
|
||||||
load((ifstream&) inFile);
|
input_stream.close();
|
||||||
|
|
||||||
inFile.close();
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
void FeatureArray::merge(FeatureArray& e)
|
void FeatureArray::merge(FeatureArray& e)
|
||||||
{
|
{
|
||||||
//dummy implementation
|
//dummy implementation
|
||||||
for (size_t i=0; i<e.size(); i++)
|
for (size_t i = 0; i < e.size(); i++)
|
||||||
add(e.get(i));
|
add(e.get(i));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -11,7 +11,6 @@
|
|||||||
|
|
||||||
#include <vector>
|
#include <vector>
|
||||||
#include <iostream>
|
#include <iostream>
|
||||||
#include <fstream>
|
|
||||||
#include "FeatureStats.h"
|
#include "FeatureStats.h"
|
||||||
|
|
||||||
using namespace std;
|
using namespace std;
|
||||||
@ -89,17 +88,16 @@ public:
|
|||||||
m_features = f;
|
m_features = f;
|
||||||
}
|
}
|
||||||
|
|
||||||
void savetxt(ofstream& outFile);
|
void savetxt(std::ostream* os);
|
||||||
void savebin(ofstream& outFile);
|
void savebin(std::ostream* os);
|
||||||
void save(ofstream& outFile, bool bin=false);
|
void save(std::ostream* os, bool bin=false);
|
||||||
void save(const std::string &file, bool bin=false);
|
|
||||||
inline void save(bool bin=false) {
|
|
||||||
save("/dev/stdout",bin);
|
|
||||||
}
|
|
||||||
|
|
||||||
void loadtxt(ifstream& inFile, size_t n);
|
void save(const std::string &file, bool bin=false);
|
||||||
void loadbin(ifstream& inFile, size_t n);
|
void save(bool bin=false);
|
||||||
void load(ifstream& inFile);
|
|
||||||
|
void loadtxt(std::istream* is, size_t n);
|
||||||
|
void loadbin(std::istream* is, size_t n);
|
||||||
|
void load(std::istream* is);
|
||||||
void load(const std::string &file);
|
void load(const std::string &file);
|
||||||
|
|
||||||
bool check_consistency() const;
|
bool check_consistency() const;
|
||||||
|
@ -13,44 +13,41 @@
|
|||||||
#include "Util.h"
|
#include "Util.h"
|
||||||
#include <cstdio>
|
#include <cstdio>
|
||||||
|
|
||||||
static const float MIN_FLOAT=-1.0*numeric_limits<float>::max();
|
static const float MIN_FLOAT = -1.0 * numeric_limits<float>::max();
|
||||||
static const float MAX_FLOAT=numeric_limits<float>::max();
|
static const float MAX_FLOAT = numeric_limits<float>::max();
|
||||||
|
|
||||||
FeatureData::FeatureData()
|
FeatureData::FeatureData()
|
||||||
: m_num_features(0),
|
: m_num_features(0),
|
||||||
m_sparse_flag(false) {}
|
m_sparse_flag(false) {}
|
||||||
|
|
||||||
void FeatureData::save(std::ofstream& outFile, bool bin)
|
void FeatureData::save(ostream* os, bool bin)
|
||||||
{
|
{
|
||||||
for (featdata_t::iterator i = m_array.begin(); i != m_array.end(); i++)
|
for (featdata_t::iterator i = m_array.begin(); i != m_array.end(); i++)
|
||||||
i->save(outFile, bin);
|
i->save(os, bin);
|
||||||
}
|
}
|
||||||
|
|
||||||
void FeatureData::save(const std::string &file, bool bin)
|
void FeatureData::save(const string &file, bool bin)
|
||||||
{
|
{
|
||||||
if (file.empty()) return;
|
if (file.empty()) return;
|
||||||
|
TRACE_ERR("saving the array into " << file << endl);
|
||||||
TRACE_ERR("saving the array into " << file << std::endl);
|
ofstream ofs(file.c_str(), ios::out); // matches a stream with a file. Opens the file
|
||||||
|
ostream* os = &ofs;
|
||||||
std::ofstream outFile(file.c_str(), std::ios::out); // matches a stream with a file. Opens the file
|
save(os, bin);
|
||||||
|
ofs.close();
|
||||||
save(outFile, bin);
|
|
||||||
|
|
||||||
outFile.close();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
void FeatureData::load(ifstream& inFile)
|
void FeatureData::load(istream* is)
|
||||||
{
|
{
|
||||||
FeatureArray entry;
|
FeatureArray entry;
|
||||||
|
|
||||||
while (!inFile.eof()) {
|
while (!is->eof()) {
|
||||||
|
|
||||||
if (!inFile.good()) {
|
if (!is->good()) {
|
||||||
std::cerr << "ERROR FeatureData::load inFile.good()" << std::endl;
|
cerr << "ERROR FeatureData::load inFile.good()" << endl;
|
||||||
}
|
}
|
||||||
|
|
||||||
entry.clear();
|
entry.clear();
|
||||||
entry.load(inFile);
|
entry.load(is);
|
||||||
|
|
||||||
if (entry.size() == 0)
|
if (entry.size() == 0)
|
||||||
break;
|
break;
|
||||||
@ -66,19 +63,16 @@ void FeatureData::load(ifstream& inFile)
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
void FeatureData::load(const std::string &file)
|
void FeatureData::load(const string &file)
|
||||||
{
|
{
|
||||||
TRACE_ERR("loading feature data from " << file << std::endl);
|
TRACE_ERR("loading feature data from " << file << endl);
|
||||||
|
inputfilestream input_stream(file); // matches a stream with a file. Opens the file
|
||||||
inputfilestream inFile(file); // matches a stream with a file. Opens the file
|
if (!input_stream) {
|
||||||
|
|
||||||
if (!inFile) {
|
|
||||||
throw runtime_error("Unable to open feature file: " + file);
|
throw runtime_error("Unable to open feature file: " + file);
|
||||||
}
|
}
|
||||||
|
istream* is = &input_stream;
|
||||||
load((ifstream&) inFile);
|
load(is);
|
||||||
|
input_stream.close();
|
||||||
inFile.close();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
void FeatureData::add(FeatureArray& e)
|
void FeatureData::add(FeatureArray& e)
|
||||||
@ -93,7 +87,7 @@ void FeatureData::add(FeatureArray& e)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void FeatureData::add(FeatureStats& e, const std::string& sent_idx)
|
void FeatureData::add(FeatureStats& e, const string& sent_idx)
|
||||||
{
|
{
|
||||||
if (exists(sent_idx)) { // array at position e.getIndex() already exists
|
if (exists(sent_idx)) { // array at position e.getIndex() already exists
|
||||||
//enlarge array at position e.getIndex()
|
//enlarge array at position e.getIndex()
|
||||||
@ -132,7 +126,7 @@ void FeatureData::setIndex()
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void FeatureData::setFeatureMap(const std::string& feat)
|
void FeatureData::setFeatureMap(const string& feat)
|
||||||
{
|
{
|
||||||
m_num_features = 0;
|
m_num_features = 0;
|
||||||
m_features = feat;
|
m_features = feat;
|
||||||
|
@ -92,12 +92,13 @@ public:
|
|||||||
}
|
}
|
||||||
|
|
||||||
void save(const std::string &file, bool bin=false);
|
void save(const std::string &file, bool bin=false);
|
||||||
void save(ofstream& outFile, bool bin=false);
|
void save(std::ostream* os, bool bin=false);
|
||||||
|
|
||||||
inline void save(bool bin=false) {
|
inline void save(bool bin=false) {
|
||||||
save("/dev/stdout", bin);
|
save(&cout, bin);
|
||||||
}
|
}
|
||||||
|
|
||||||
void load(ifstream& inFile);
|
void load(std::istream* is);
|
||||||
void load(const std::string &file);
|
void load(const std::string &file);
|
||||||
|
|
||||||
bool check_consistency() const;
|
bool check_consistency() const;
|
||||||
|
@ -8,6 +8,7 @@
|
|||||||
|
|
||||||
#include "FeatureStats.h"
|
#include "FeatureStats.h"
|
||||||
|
|
||||||
|
#include <fstream>
|
||||||
#include <cmath>
|
#include <cmath>
|
||||||
#include "Util.h"
|
#include "Util.h"
|
||||||
|
|
||||||
@ -89,7 +90,7 @@ FeatureStats::FeatureStats(const size_t size)
|
|||||||
memset(m_array, 0, GetArraySizeWithBytes());
|
memset(m_array, 0, GetArraySizeWithBytes());
|
||||||
}
|
}
|
||||||
|
|
||||||
FeatureStats::FeatureStats(std::string &theString)
|
FeatureStats::FeatureStats(string &theString)
|
||||||
: m_available_size(0), m_entries(0), m_array(NULL)
|
: m_available_size(0), m_entries(0), m_array(NULL)
|
||||||
{
|
{
|
||||||
set(theString);
|
set(theString);
|
||||||
@ -144,9 +145,9 @@ void FeatureStats::addSparse(const string& name, FeatureStatsType v)
|
|||||||
m_map.set(name,v);
|
m_map.set(name,v);
|
||||||
}
|
}
|
||||||
|
|
||||||
void FeatureStats::set(std::string &theString)
|
void FeatureStats::set(string &theString)
|
||||||
{
|
{
|
||||||
std::string substring, stringBuf;
|
string substring, stringBuf;
|
||||||
reset();
|
reset();
|
||||||
|
|
||||||
while (!theString.empty()) {
|
while (!theString.empty()) {
|
||||||
@ -163,48 +164,50 @@ void FeatureStats::set(std::string &theString)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void FeatureStats::loadbin(istream* is)
|
||||||
void FeatureStats::loadbin(std::ifstream& inFile)
|
|
||||||
{
|
{
|
||||||
inFile.read((char*) m_array, GetArraySizeWithBytes());
|
is->read(reinterpret_cast<char*>(m_array),
|
||||||
|
static_cast<streamsize>(GetArraySizeWithBytes()));
|
||||||
}
|
}
|
||||||
|
|
||||||
void FeatureStats::loadtxt(std::ifstream& inFile)
|
void FeatureStats::loadtxt(istream* is)
|
||||||
{
|
{
|
||||||
std::string theString;
|
string line;
|
||||||
std::getline(inFile, theString);
|
getline(*is, line);
|
||||||
set(theString);
|
set(line);
|
||||||
}
|
}
|
||||||
|
|
||||||
void FeatureStats::loadtxt(const std::string &file)
|
void FeatureStats::loadtxt(const string &file)
|
||||||
{
|
{
|
||||||
// TRACE_ERR("loading the stats from " << file << std::endl);
|
ifstream ifs(file.c_str(), ios::in);
|
||||||
|
if (!ifs) {
|
||||||
std::ifstream inFile(file.c_str(), std::ios::in); // matches a stream with a file. Opens the file
|
cerr << "Failed to open " << file << endl;
|
||||||
|
exit(1);
|
||||||
loadtxt(inFile);
|
}
|
||||||
|
istream* is = &ifs;
|
||||||
|
loadtxt(is);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void FeatureStats::savetxt(const string &file)
|
||||||
void FeatureStats::savetxt(const std::string &file)
|
|
||||||
{
|
{
|
||||||
// TRACE_ERR("saving the stats into " << file << std::endl);
|
ofstream ofs(file.c_str(), ios::out);
|
||||||
|
ostream* os = &ofs;
|
||||||
std::ofstream outFile(file.c_str(), std::ios::out); // matches a stream with a file. Opens the file
|
savetxt(os);
|
||||||
|
|
||||||
savetxt(outFile);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void FeatureStats::savetxt(ostream* os)
|
||||||
void FeatureStats::savetxt(std::ofstream& outFile)
|
|
||||||
{
|
{
|
||||||
// TRACE_ERR("saving the stats" << std::endl);
|
*os << *this;
|
||||||
outFile << *this;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
void FeatureStats::savebin(std::ofstream& outFile)
|
void FeatureStats::savetxt() {
|
||||||
|
savetxt(&cout);
|
||||||
|
}
|
||||||
|
|
||||||
|
void FeatureStats::savebin(ostream* os)
|
||||||
{
|
{
|
||||||
outFile.write((char*) m_array, GetArraySizeWithBytes());
|
os->write(reinterpret_cast<char*>(m_array),
|
||||||
|
static_cast<streamsize>(GetArraySizeWithBytes()));
|
||||||
}
|
}
|
||||||
|
|
||||||
ostream& operator<<(ostream& o, const FeatureStats& e)
|
ostream& operator<<(ostream& o, const FeatureStats& e)
|
||||||
|
@ -10,7 +10,6 @@
|
|||||||
#define MERT_FEATURE_STATS_H_
|
#define MERT_FEATURE_STATS_H_
|
||||||
|
|
||||||
#include <cstring>
|
#include <cstring>
|
||||||
#include <fstream>
|
|
||||||
#include <iostream>
|
#include <iostream>
|
||||||
#include <map>
|
#include <map>
|
||||||
#include <string>
|
#include <string>
|
||||||
@ -118,15 +117,13 @@ public:
|
|||||||
}
|
}
|
||||||
|
|
||||||
void savetxt(const std::string &file);
|
void savetxt(const std::string &file);
|
||||||
void savetxt(ofstream& outFile);
|
void savetxt(std::ostream* os);
|
||||||
void savebin(ofstream& outFile);
|
void savebin(std::ostream* os);
|
||||||
inline void savetxt() {
|
void savetxt();
|
||||||
savetxt("/dev/stdout");
|
|
||||||
}
|
|
||||||
|
|
||||||
void loadtxt(const std::string &file);
|
void loadtxt(const std::string &file);
|
||||||
void loadtxt(ifstream& inFile);
|
void loadtxt(std::istream* is);
|
||||||
void loadbin(ifstream& inFile);
|
void loadbin(std::istream* is);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Write the whole object to a stream.
|
* Write the whole object to a stream.
|
||||||
|
@ -2,6 +2,7 @@
|
|||||||
#define MERT_FILE_STREAM_H_
|
#define MERT_FILE_STREAM_H_
|
||||||
|
|
||||||
#include <fstream>
|
#include <fstream>
|
||||||
|
#include <iostream>
|
||||||
#include <streambuf>
|
#include <streambuf>
|
||||||
#include <string>
|
#include <string>
|
||||||
|
|
||||||
|
@ -13,74 +13,82 @@
|
|||||||
ScoreArray::ScoreArray()
|
ScoreArray::ScoreArray()
|
||||||
: m_num_scores(0), m_index("") {}
|
: m_num_scores(0), m_index("") {}
|
||||||
|
|
||||||
void ScoreArray::savetxt(std::ofstream& outFile, const std::string& sctype)
|
void ScoreArray::savetxt(ostream* os, const string& sctype)
|
||||||
{
|
{
|
||||||
outFile << SCORES_TXT_BEGIN << " " << m_index << " " << m_array.size()
|
*os << SCORES_TXT_BEGIN << " " << m_index << " " << m_array.size()
|
||||||
<< " " << m_num_scores << " " << sctype << std::endl;
|
<< " " << m_num_scores << " " << sctype << endl;
|
||||||
for (scorearray_t::iterator i = m_array.begin(); i !=m_array.end(); i++) {
|
for (scorearray_t::iterator i = m_array.begin(); i !=m_array.end(); i++) {
|
||||||
i->savetxt(outFile);
|
i->savetxt(os);
|
||||||
outFile << std::endl;
|
*os << endl;
|
||||||
}
|
}
|
||||||
outFile << SCORES_TXT_END << std::endl;
|
*os << SCORES_TXT_END << endl;
|
||||||
}
|
}
|
||||||
|
|
||||||
void ScoreArray::savebin(std::ofstream& outFile, const std::string& sctype)
|
void ScoreArray::savebin(ostream* os, const string& score_type)
|
||||||
{
|
{
|
||||||
outFile << SCORES_BIN_BEGIN << " " << m_index << " " << m_array.size()
|
*os << SCORES_BIN_BEGIN << " " << m_index << " " << m_array.size()
|
||||||
<< " " << m_num_scores << " " << sctype << std::endl;
|
<< " " << m_num_scores << " " << score_type << endl;
|
||||||
for (scorearray_t::iterator i = m_array.begin(); i !=m_array.end(); i++)
|
for (scorearray_t::iterator i = m_array.begin();
|
||||||
i->savebin(outFile);
|
i != m_array.end(); i++) {
|
||||||
|
i->savebin(os);
|
||||||
outFile << SCORES_BIN_END << std::endl;
|
}
|
||||||
|
*os << SCORES_BIN_END << endl;
|
||||||
}
|
}
|
||||||
|
|
||||||
void ScoreArray::save(std::ofstream& inFile, const std::string& sctype, bool bin)
|
void ScoreArray::save(ostream* os, const string& score_type, bool bin)
|
||||||
{
|
{
|
||||||
if (size() <= 0) return;
|
if (size() <= 0) return;
|
||||||
if (bin) {
|
if (bin) {
|
||||||
savebin(inFile, sctype);
|
savebin(os, score_type);
|
||||||
} else {
|
} else {
|
||||||
savetxt(inFile, sctype);
|
savetxt(os, score_type);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void ScoreArray::save(const std::string &file, const std::string& sctype, bool bin)
|
void ScoreArray::save(const string &file, const string& score_type, bool bin)
|
||||||
{
|
{
|
||||||
std::ofstream outFile(file.c_str(), std::ios::out); // matches a stream with a file. Opens the file
|
ofstream ofs(file.c_str(), ios::out);
|
||||||
save(outFile, sctype, bin);
|
if (!ofs) {
|
||||||
outFile.close();
|
cerr << "Failed to open " << file << endl;
|
||||||
}
|
exit(1);
|
||||||
|
|
||||||
void ScoreArray::loadbin(ifstream& inFile, size_t n)
|
|
||||||
{
|
|
||||||
ScoreStats entry(m_num_scores);
|
|
||||||
|
|
||||||
for (size_t i=0 ; i < n; i++) {
|
|
||||||
entry.loadbin(inFile);
|
|
||||||
add(entry);
|
|
||||||
}
|
}
|
||||||
|
ostream* os = &ofs;
|
||||||
|
save(os, score_type, bin);
|
||||||
|
ofs.close();
|
||||||
}
|
}
|
||||||
|
|
||||||
void ScoreArray::loadtxt(ifstream& inFile, size_t n)
|
void ScoreArray::save(const string& score_type, bool bin) {
|
||||||
|
save(&cout, score_type, bin);
|
||||||
|
}
|
||||||
|
|
||||||
|
void ScoreArray::loadbin(istream* is, size_t n)
|
||||||
{
|
{
|
||||||
ScoreStats entry(m_num_scores);
|
ScoreStats entry(m_num_scores);
|
||||||
|
|
||||||
for (size_t i = 0; i < n; i++) {
|
for (size_t i = 0; i < n; i++) {
|
||||||
entry.loadtxt(inFile);
|
entry.loadbin(is);
|
||||||
add(entry);
|
add(entry);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void ScoreArray::load(ifstream& inFile)
|
void ScoreArray::loadtxt(istream* is, size_t n)
|
||||||
|
{
|
||||||
|
ScoreStats entry(m_num_scores);
|
||||||
|
for (size_t i = 0; i < n; i++) {
|
||||||
|
entry.loadtxt(is);
|
||||||
|
add(entry);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void ScoreArray::load(istream* is)
|
||||||
{
|
{
|
||||||
size_t number_of_entries = 0;
|
size_t number_of_entries = 0;
|
||||||
bool binmode = false;
|
bool binmode = false;
|
||||||
|
|
||||||
std::string substring, stringBuf;
|
string substring, stringBuf;
|
||||||
std::string::size_type loc;
|
string::size_type loc;
|
||||||
|
|
||||||
std::getline(inFile, stringBuf);
|
getline(*is, stringBuf);
|
||||||
if (!inFile.good()) {
|
if (!is->good()) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -105,12 +113,12 @@ void ScoreArray::load(ifstream& inFile)
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (binmode) {
|
if (binmode) {
|
||||||
loadbin(inFile, number_of_entries);
|
loadbin(is, number_of_entries);
|
||||||
} else {
|
} else {
|
||||||
loadtxt(inFile, number_of_entries);
|
loadtxt(is, number_of_entries);
|
||||||
}
|
}
|
||||||
|
|
||||||
std::getline(inFile, stringBuf);
|
getline(*is, stringBuf);
|
||||||
if (!stringBuf.empty()) {
|
if (!stringBuf.empty()) {
|
||||||
if ((loc = stringBuf.find(SCORES_TXT_END)) != 0 &&
|
if ((loc = stringBuf.find(SCORES_TXT_END)) != 0 &&
|
||||||
(loc = stringBuf.find(SCORES_BIN_END)) != 0) {
|
(loc = stringBuf.find(SCORES_BIN_END)) != 0) {
|
||||||
@ -120,15 +128,13 @@ void ScoreArray::load(ifstream& inFile)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void ScoreArray::load(const std::string &file)
|
void ScoreArray::load(const string &file)
|
||||||
{
|
{
|
||||||
TRACE_ERR("loading data from " << file << std::endl);
|
TRACE_ERR("loading data from " << file << endl);
|
||||||
|
inputfilestream input_stream(file); // matches a stream with a file. Opens the file
|
||||||
inputfilestream inFile(file); // matches a stream with a file. Opens the file
|
istream* is = &input_stream;
|
||||||
|
load(is);
|
||||||
load((ifstream&) inFile);
|
input_stream.close();
|
||||||
|
|
||||||
inFile.close();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@ -95,17 +95,15 @@ public:
|
|||||||
m_num_scores = v;
|
m_num_scores = v;
|
||||||
}
|
}
|
||||||
|
|
||||||
void savetxt(ofstream& outFile, const std::string& sctype);
|
void savetxt(std::ostream* os, const std::string& score_type);
|
||||||
void savebin(ofstream& outFile, const std::string& sctype);
|
void savebin(std::ostream* os, const std::string& score_type);
|
||||||
void save(ofstream& outFile, const std::string& sctype, bool bin=false);
|
void save(std::ostream* os, const std::string& score_type, bool bin=false);
|
||||||
void save(const std::string &file, const std::string& sctype, bool bin=false);
|
void save(const std::string &file, const std::string& score_type, bool bin=false);
|
||||||
inline void save(const std::string& sctype, bool bin=false) {
|
void save(const std::string& score_type, bool bin=false);
|
||||||
save("/dev/stdout", sctype, bin);
|
|
||||||
}
|
|
||||||
|
|
||||||
void loadtxt(ifstream& inFile, size_t n);
|
void loadtxt(std::istream* is, size_t n);
|
||||||
void loadbin(ifstream& inFile, size_t n);
|
void loadbin(std::istream* is, size_t n);
|
||||||
void load(ifstream& inFile);
|
void load(std::istream* is);
|
||||||
void load(const std::string &file);
|
void load(const std::string &file);
|
||||||
|
|
||||||
bool check_consistency() const;
|
bool check_consistency() const;
|
||||||
|
@ -7,6 +7,8 @@
|
|||||||
*/
|
*/
|
||||||
|
|
||||||
#include "ScoreData.h"
|
#include "ScoreData.h"
|
||||||
|
|
||||||
|
#include <fstream>
|
||||||
#include "Scorer.h"
|
#include "Scorer.h"
|
||||||
#include "Util.h"
|
#include "Util.h"
|
||||||
#include "FileStream.h"
|
#include "FileStream.h"
|
||||||
@ -21,34 +23,40 @@ ScoreData::ScoreData(Scorer* scorer) :
|
|||||||
// TRACE_ERR("ScoreData: m_num_scores: " << m_num_scores << std::endl);
|
// TRACE_ERR("ScoreData: m_num_scores: " << m_num_scores << std::endl);
|
||||||
}
|
}
|
||||||
|
|
||||||
void ScoreData::save(std::ofstream& outFile, bool bin)
|
void ScoreData::save(ostream* os, bool bin)
|
||||||
{
|
{
|
||||||
for (scoredata_t::iterator i = m_array.begin(); i != m_array.end(); ++i) {
|
for (scoredata_t::iterator i = m_array.begin();
|
||||||
i->save(outFile, m_score_type, bin);
|
i != m_array.end(); ++i) {
|
||||||
|
i->save(os, m_score_type, bin);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void ScoreData::save(const std::string &file, bool bin)
|
void ScoreData::save(const string &file, bool bin)
|
||||||
{
|
{
|
||||||
if (file.empty()) return;
|
if (file.empty()) return;
|
||||||
TRACE_ERR("saving the array into " << file << std::endl);
|
TRACE_ERR("saving the array into " << file << endl);
|
||||||
|
|
||||||
// matches a stream with a file. Opens the file.
|
// matches a stream with a file. Opens the file.
|
||||||
std::ofstream outFile(file.c_str(), std::ios::out);
|
ofstream ofs(file.c_str(), ios::out);
|
||||||
save(outFile, bin);
|
ostream* os = &ofs;
|
||||||
outFile.close();
|
save(os, bin);
|
||||||
|
ofs.close();
|
||||||
}
|
}
|
||||||
|
|
||||||
void ScoreData::load(ifstream& inFile)
|
void ScoreData::save(bool bin) {
|
||||||
|
save(&cout, bin);
|
||||||
|
}
|
||||||
|
|
||||||
|
void ScoreData::load(istream* is)
|
||||||
{
|
{
|
||||||
ScoreArray entry;
|
ScoreArray entry;
|
||||||
|
|
||||||
while (!inFile.eof()) {
|
while (!is->eof()) {
|
||||||
if (!inFile.good()) {
|
if (!is->good()) {
|
||||||
std::cerr << "ERROR ScoreData::load inFile.good()" << std::endl;
|
cerr << "ERROR ScoreData::load inFile.good()" << endl;
|
||||||
}
|
}
|
||||||
entry.clear();
|
entry.clear();
|
||||||
entry.load(inFile);
|
entry.load(is);
|
||||||
if (entry.size() == 0) {
|
if (entry.size() == 0) {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
@ -56,15 +64,16 @@ void ScoreData::load(ifstream& inFile)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void ScoreData::load(const std::string &file)
|
void ScoreData::load(const string &file)
|
||||||
{
|
{
|
||||||
TRACE_ERR("loading score data from " << file << std::endl);
|
TRACE_ERR("loading score data from " << file << endl);
|
||||||
inputfilestream inFile(file); // matches a stream with a file. Opens the file
|
inputfilestream input_stream(file); // matches a stream with a file. Opens the file
|
||||||
if (!inFile) {
|
if (!input_stream) {
|
||||||
throw runtime_error("Unable to open score file: " + file);
|
throw runtime_error("Unable to open score file: " + file);
|
||||||
}
|
}
|
||||||
load((ifstream&) inFile);
|
istream* is = &input_stream;
|
||||||
inFile.close();
|
load(is);
|
||||||
|
input_stream.close();
|
||||||
}
|
}
|
||||||
|
|
||||||
void ScoreData::add(ScoreArray& e)
|
void ScoreData::add(ScoreArray& e)
|
||||||
@ -79,7 +88,7 @@ void ScoreData::add(ScoreArray& e)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void ScoreData::add(const ScoreStats& e, const std::string& sent_idx)
|
void ScoreData::add(const ScoreStats& e, const string& sent_idx)
|
||||||
{
|
{
|
||||||
if (exists(sent_idx)) { // array at position e.getIndex() already exists
|
if (exists(sent_idx)) { // array at position e.getIndex() already exists
|
||||||
// Enlarge array at position e.getIndex()
|
// Enlarge array at position e.getIndex()
|
||||||
|
@ -9,7 +9,7 @@
|
|||||||
#ifndef MERT_SCORE_DATA_H_
|
#ifndef MERT_SCORE_DATA_H_
|
||||||
#define MERT_SCORE_DATA_H_
|
#define MERT_SCORE_DATA_H_
|
||||||
|
|
||||||
#include <fstream>
|
#include <iostream>
|
||||||
#include <vector>
|
#include <vector>
|
||||||
#include <stdexcept>
|
#include <stdexcept>
|
||||||
#include <string>
|
#include <string>
|
||||||
@ -86,12 +86,10 @@ public:
|
|||||||
}
|
}
|
||||||
|
|
||||||
void save(const std::string &file, bool bin=false);
|
void save(const std::string &file, bool bin=false);
|
||||||
void save(ofstream& outFile, bool bin=false);
|
void save(std::ostream* os, bool bin=false);
|
||||||
inline void save(bool bin=false) {
|
void save(bool bin=false);
|
||||||
save("/dev/stdout", bin);
|
|
||||||
}
|
|
||||||
|
|
||||||
void load(ifstream& inFile);
|
void load(std::istream* is);
|
||||||
void load(const std::string &file);
|
void load(const std::string &file);
|
||||||
|
|
||||||
bool check_consistency() const;
|
bool check_consistency() const;
|
||||||
|
@ -67,7 +67,7 @@ void ScoreStats::add(ScoreStatsType v)
|
|||||||
m_array[m_entries++]=v;
|
m_array[m_entries++]=v;
|
||||||
}
|
}
|
||||||
|
|
||||||
void ScoreStats::set(const std::string& str)
|
void ScoreStats::set(const string& str)
|
||||||
{
|
{
|
||||||
reset();
|
reset();
|
||||||
vector<string> out;
|
vector<string> out;
|
||||||
@ -78,46 +78,51 @@ void ScoreStats::set(const std::string& str)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void ScoreStats::loadbin(std::ifstream& inFile)
|
void ScoreStats::loadbin(istream* is)
|
||||||
{
|
{
|
||||||
inFile.read((char*)m_array, GetArraySizeWithBytes());
|
is->read(reinterpret_cast<char*>(m_array),
|
||||||
|
static_cast<streamsize>(GetArraySizeWithBytes()));
|
||||||
}
|
}
|
||||||
|
|
||||||
void ScoreStats::loadtxt(std::ifstream& inFile)
|
void ScoreStats::loadtxt(istream* is)
|
||||||
{
|
{
|
||||||
std::string theString;
|
string line;
|
||||||
std::getline(inFile, theString);
|
getline(*is, line);
|
||||||
set(theString);
|
set(line);
|
||||||
}
|
}
|
||||||
|
|
||||||
void ScoreStats::loadtxt(const std::string &file)
|
void ScoreStats::loadtxt(const string &file)
|
||||||
{
|
{
|
||||||
// TRACE_ERR("loading the stats from " << file << std::endl);
|
ifstream ifs(file.c_str(), ios::in); // matches a stream with a file. Opens the file
|
||||||
|
if (!ifs) {
|
||||||
std::ifstream inFile(file.c_str(), std::ios::in); // matches a stream with a file. Opens the file
|
cerr << "Failed to open " << file << endl;
|
||||||
|
exit(1);
|
||||||
loadtxt(inFile);
|
}
|
||||||
|
istream* is = &ifs;
|
||||||
|
loadtxt(is);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
void ScoreStats::savetxt(const std::string &file)
|
void ScoreStats::savetxt(const string &file)
|
||||||
{
|
{
|
||||||
// TRACE_ERR("saving the stats into " << file << std::endl);
|
ofstream ofs(file.c_str(), ios::out); // matches a stream with a file. Opens the file
|
||||||
|
ostream* os = &ofs;
|
||||||
std::ofstream outFile(file.c_str(), std::ios::out); // matches a stream with a file. Opens the file
|
savetxt(os);
|
||||||
|
|
||||||
savetxt(outFile);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void ScoreStats::savetxt(ostream* os)
|
||||||
void ScoreStats::savetxt(std::ofstream& outFile)
|
|
||||||
{
|
{
|
||||||
outFile << *this;
|
*os << *this;
|
||||||
}
|
}
|
||||||
|
|
||||||
void ScoreStats::savebin(std::ofstream& outFile)
|
void ScoreStats::savetxt() {
|
||||||
|
savetxt(&cout);
|
||||||
|
}
|
||||||
|
|
||||||
|
void ScoreStats::savebin(ostream* os)
|
||||||
{
|
{
|
||||||
outFile.write((char*)m_array, GetArraySizeWithBytes());
|
os->write(reinterpret_cast<char*>(m_array),
|
||||||
|
static_cast<streamsize>(GetArraySizeWithBytes()));
|
||||||
}
|
}
|
||||||
|
|
||||||
ostream& operator<<(ostream& o, const ScoreStats& e)
|
ostream& operator<<(ostream& o, const ScoreStats& e)
|
||||||
|
@ -81,15 +81,13 @@ public:
|
|||||||
inline size_t available() const { return m_available_size; }
|
inline size_t available() const { return m_available_size; }
|
||||||
|
|
||||||
void savetxt(const std::string &file);
|
void savetxt(const std::string &file);
|
||||||
void savetxt(ofstream& outFile);
|
void savetxt(ostream* os);
|
||||||
void savebin(ofstream& outFile);
|
void savebin(ostream* os);
|
||||||
inline void savetxt() {
|
void savetxt();
|
||||||
savetxt("/dev/stdout");
|
|
||||||
}
|
|
||||||
|
|
||||||
void loadtxt(const std::string &file);
|
void loadtxt(const std::string &file);
|
||||||
void loadtxt(ifstream& inFile);
|
void loadtxt(istream* is);
|
||||||
void loadbin(ifstream& inFile);
|
void loadbin(istream* is);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Write the whole object to a stream.
|
* Write the whole object to a stream.
|
||||||
|
Loading…
Reference in New Issue
Block a user