mirror of
https://github.com/moses-smt/mosesdecoder.git
synced 2024-09-11 11:25:40 +03:00
fix memory leaks in binarisation
This commit is contained in:
parent
236611193e
commit
576bc09fa5
@ -77,7 +77,7 @@ int main (int argc, char * const argv[])
|
||||
std::vector<float> misc(1);
|
||||
SourcePhrase sourcePhrase;
|
||||
TargetPhrase *targetPhrase = new TargetPhrase(numScores);
|
||||
OnDiskPt::Phrase *spShort = Tokenize(sourcePhrase, *targetPhrase, line, onDiskWrapper, numScores, misc);
|
||||
OnDiskPt::PhrasePtr spShort = Tokenize(sourcePhrase, *targetPhrase, line, onDiskWrapper, numScores, misc);
|
||||
assert(misc.size() == onDiskWrapper.GetNumCounts());
|
||||
|
||||
rootNode.AddTargetPhrase(sourcePhrase, targetPhrase, onDiskWrapper, tableLimit, misc, spShort);
|
||||
@ -105,7 +105,7 @@ bool Flush(const OnDiskPt::SourcePhrase *prevSourcePhrase, const OnDiskPt::Sourc
|
||||
return ret;
|
||||
}
|
||||
|
||||
OnDiskPt::Phrase *Tokenize(SourcePhrase &sourcePhrase, TargetPhrase &targetPhrase, char *line, OnDiskWrapper &onDiskWrapper, int numScores, vector<float> &misc)
|
||||
OnDiskPt::PhrasePtr Tokenize(SourcePhrase &sourcePhrase, TargetPhrase &targetPhrase, char *line, OnDiskWrapper &onDiskWrapper, int numScores, vector<float> &misc)
|
||||
{
|
||||
size_t scoreInd = 0;
|
||||
|
||||
@ -118,14 +118,14 @@ OnDiskPt::Phrase *Tokenize(SourcePhrase &sourcePhrase, TargetPhrase &targetPhras
|
||||
4 = count
|
||||
*/
|
||||
char *tok = strtok (line," ");
|
||||
OnDiskPt::Phrase *out = new Phrase();
|
||||
OnDiskPt::PhrasePtr out(new Phrase());
|
||||
while (tok != NULL) {
|
||||
if (0 == strcmp(tok, "|||")) {
|
||||
++stage;
|
||||
} else {
|
||||
switch (stage) {
|
||||
case 0: {
|
||||
Word *w = Tokenize(sourcePhrase, tok, true, true, onDiskWrapper);
|
||||
WordPtr w = Tokenize(sourcePhrase, tok, true, true, onDiskWrapper);
|
||||
if (w != NULL)
|
||||
out->AddWord(w);
|
||||
|
||||
@ -184,7 +184,7 @@ OnDiskPt::Phrase *Tokenize(SourcePhrase &sourcePhrase, TargetPhrase &targetPhras
|
||||
return out;
|
||||
} // Tokenize()
|
||||
|
||||
OnDiskPt::Word *Tokenize(OnDiskPt::Phrase &phrase
|
||||
OnDiskPt::WordPtr Tokenize(OnDiskPt::Phrase &phrase
|
||||
, const std::string &token, bool addSourceNonTerm, bool addTargetNonTerm
|
||||
, OnDiskPt::OnDiskWrapper &onDiskWrapper)
|
||||
{
|
||||
@ -198,7 +198,7 @@ OnDiskPt::Word *Tokenize(OnDiskPt::Phrase &phrase
|
||||
nonTerm = comStr == 0;
|
||||
}
|
||||
|
||||
OnDiskPt::Word *out = NULL;
|
||||
OnDiskPt::WordPtr out;
|
||||
if (nonTerm) {
|
||||
// non-term
|
||||
size_t splitPos = token.find_first_of("[", 2);
|
||||
@ -206,20 +206,20 @@ OnDiskPt::Word *Tokenize(OnDiskPt::Phrase &phrase
|
||||
|
||||
if (splitPos == string::npos) {
|
||||
// lhs - only 1 word
|
||||
Word *word = new Word();
|
||||
WordPtr word(new Word());
|
||||
word->CreateFromString(wordStr, onDiskWrapper.GetVocab());
|
||||
phrase.AddWord(word);
|
||||
} else {
|
||||
// source & target non-terms
|
||||
if (addSourceNonTerm) {
|
||||
Word *word = new Word();
|
||||
WordPtr word(new Word());
|
||||
word->CreateFromString(wordStr, onDiskWrapper.GetVocab());
|
||||
phrase.AddWord(word);
|
||||
}
|
||||
|
||||
wordStr = token.substr(splitPos, tokSize - splitPos);
|
||||
if (addTargetNonTerm) {
|
||||
Word *word = new Word();
|
||||
WordPtr word(new Word());
|
||||
word->CreateFromString(wordStr, onDiskWrapper.GetVocab());
|
||||
phrase.AddWord(word);
|
||||
out = word;
|
||||
@ -228,7 +228,7 @@ OnDiskPt::Word *Tokenize(OnDiskPt::Phrase &phrase
|
||||
}
|
||||
} else {
|
||||
// term
|
||||
Word *word = new Word();
|
||||
WordPtr word(new Word());
|
||||
word->CreateFromString(token, onDiskWrapper.GetVocab());
|
||||
phrase.AddWord(word);
|
||||
out = word;
|
||||
|
@ -25,10 +25,10 @@
|
||||
typedef std::pair<size_t, size_t> AlignPair;
|
||||
typedef std::vector<AlignPair> AlignType;
|
||||
|
||||
OnDiskPt::Word *Tokenize(OnDiskPt::Phrase &phrase
|
||||
OnDiskPt::WordPtr Tokenize(OnDiskPt::Phrase &phrase
|
||||
, const std::string &token, bool addSourceNonTerm, bool addTargetNonTerm
|
||||
, OnDiskPt::OnDiskWrapper &onDiskWrapper);
|
||||
OnDiskPt::Phrase *Tokenize(OnDiskPt::SourcePhrase &sourcePhrase, OnDiskPt::TargetPhrase &targetPhrase
|
||||
OnDiskPt::PhrasePtr Tokenize(OnDiskPt::SourcePhrase &sourcePhrase, OnDiskPt::TargetPhrase &targetPhrase
|
||||
, char *line, OnDiskPt::OnDiskWrapper &onDiskWrapper
|
||||
, int numScores
|
||||
, std::vector<float> &misc);
|
||||
|
@ -27,27 +27,13 @@ using namespace std;
|
||||
namespace OnDiskPt
|
||||
{
|
||||
|
||||
Phrase::Phrase(const Phrase ©)
|
||||
:m_words(copy.GetSize())
|
||||
{
|
||||
for (size_t pos = 0; pos < copy.GetSize(); ++pos) {
|
||||
const Word &oldWord = copy.GetWord(pos);
|
||||
Word *newWord = new Word(oldWord);
|
||||
m_words[pos] = newWord;
|
||||
}
|
||||
}
|
||||
|
||||
Phrase::~Phrase()
|
||||
{
|
||||
Moses::RemoveAllInColl(m_words);
|
||||
}
|
||||
|
||||
void Phrase::AddWord(Word *word)
|
||||
void Phrase::AddWord(WordPtr word)
|
||||
{
|
||||
m_words.push_back(word);
|
||||
}
|
||||
|
||||
void Phrase::AddWord(Word *word, size_t pos)
|
||||
void Phrase::AddWord(WordPtr word, size_t pos)
|
||||
{
|
||||
CHECK(pos < m_words.size());
|
||||
m_words.insert(m_words.begin() + pos + 1, word);
|
||||
|
@ -20,12 +20,14 @@
|
||||
***********************************************************************/
|
||||
#include <vector>
|
||||
#include <iostream>
|
||||
#include <boost/shared_ptr.hpp>
|
||||
#include "Word.h"
|
||||
|
||||
namespace OnDiskPt
|
||||
{
|
||||
class Vocab;
|
||||
|
||||
|
||||
/** A contiguous phrase. SourcePhrase & TargetPhrase inherit from this and add the on-disk functionality
|
||||
*/
|
||||
class Phrase
|
||||
@ -33,16 +35,14 @@ class Phrase
|
||||
friend std::ostream& operator<<(std::ostream&, const Phrase&);
|
||||
|
||||
protected:
|
||||
std::vector<Word*> m_words;
|
||||
std::vector<WordPtr> m_words;
|
||||
|
||||
public:
|
||||
Phrase()
|
||||
{}
|
||||
Phrase(const Phrase ©);
|
||||
virtual ~Phrase();
|
||||
|
||||
void AddWord(Word *word);
|
||||
void AddWord(Word *word, size_t pos);
|
||||
void AddWord(WordPtr word);
|
||||
void AddWord(WordPtr word, size_t pos);
|
||||
|
||||
const Word &GetWord(size_t pos) const {
|
||||
return *m_words[pos];
|
||||
@ -59,4 +59,6 @@ public:
|
||||
bool operator==(const Phrase &compare) const;
|
||||
};
|
||||
|
||||
typedef boost::shared_ptr<Phrase> PhrasePtr;
|
||||
|
||||
}
|
||||
|
@ -160,14 +160,14 @@ void PhraseNode::Save(OnDiskWrapper &onDiskWrapper, size_t pos, size_t tableLimi
|
||||
|
||||
void PhraseNode::AddTargetPhrase(const SourcePhrase &sourcePhrase, TargetPhrase *targetPhrase
|
||||
, OnDiskWrapper &onDiskWrapper, size_t tableLimit
|
||||
, const std::vector<float> &counts, OnDiskPt::Phrase *spShort)
|
||||
, const std::vector<float> &counts, OnDiskPt::PhrasePtr spShort)
|
||||
{
|
||||
AddTargetPhrase(0, sourcePhrase, targetPhrase, onDiskWrapper, tableLimit, counts, spShort);
|
||||
}
|
||||
|
||||
void PhraseNode::AddTargetPhrase(size_t pos, const SourcePhrase &sourcePhrase
|
||||
, TargetPhrase *targetPhrase, OnDiskWrapper &onDiskWrapper
|
||||
, size_t tableLimit, const std::vector<float> &counts, OnDiskPt::Phrase *spShort)
|
||||
, size_t tableLimit, const std::vector<float> &counts, OnDiskPt::PhrasePtr spShort)
|
||||
{
|
||||
size_t phraseSize = sourcePhrase.GetSize();
|
||||
if (pos < phraseSize) {
|
||||
|
@ -52,7 +52,7 @@ protected:
|
||||
|
||||
void AddTargetPhrase(size_t pos, const SourcePhrase &sourcePhrase
|
||||
, TargetPhrase *targetPhrase, OnDiskWrapper &onDiskWrapper
|
||||
, size_t tableLimit, const std::vector<float> &counts, OnDiskPt::Phrase *spShort);
|
||||
, size_t tableLimit, const std::vector<float> &counts, OnDiskPt::PhrasePtr spShort);
|
||||
size_t ReadChild(Word &wordFound, UINT64 &childFilePos, const char *mem) const;
|
||||
void GetChild(Word &wordFound, UINT64 &childFilePos, size_t ind, OnDiskWrapper &onDiskWrapper) const;
|
||||
|
||||
@ -68,7 +68,7 @@ public:
|
||||
|
||||
void AddTargetPhrase(const SourcePhrase &sourcePhrase, TargetPhrase *targetPhrase
|
||||
, OnDiskWrapper &onDiskWrapper, size_t tableLimit
|
||||
, const std::vector<float> &counts, OnDiskPt::Phrase *spShort);
|
||||
, const std::vector<float> &counts, OnDiskPt::PhrasePtr spShort);
|
||||
|
||||
UINT64 GetFilePos() const {
|
||||
return m_filePos;
|
||||
|
@ -34,4 +34,5 @@ protected:
|
||||
public:
|
||||
};
|
||||
|
||||
|
||||
}
|
||||
|
@ -50,7 +50,7 @@ TargetPhrase::~TargetPhrase()
|
||||
{
|
||||
}
|
||||
|
||||
void TargetPhrase::SetLHS(Word *lhs)
|
||||
void TargetPhrase::SetLHS(WordPtr lhs)
|
||||
{
|
||||
AddWord(lhs);
|
||||
}
|
||||
@ -99,7 +99,7 @@ char *TargetPhrase::WriteToMemory(OnDiskWrapper &onDiskWrapper, size_t &memUsed)
|
||||
size_t phraseSize = GetSize();
|
||||
size_t targetWordSize = onDiskWrapper.GetTargetWordSize();
|
||||
|
||||
const Phrase* sp = GetSourcePhrase();
|
||||
const PhrasePtr sp = GetSourcePhrase();
|
||||
size_t spSize = sp->GetSize();
|
||||
size_t sourceWordSize = onDiskWrapper.GetSourceWordSize();
|
||||
|
||||
@ -252,7 +252,7 @@ Moses::TargetPhrase *TargetPhrase::ConvertToMoses(const std::vector<Moses::Facto
|
||||
int indicator[m_align.size()];
|
||||
int index = 0;
|
||||
std::set<std::pair<size_t, size_t> > alignmentInfo;
|
||||
const Phrase* sp = GetSourcePhrase();
|
||||
const PhrasePtr sp = GetSourcePhrase();
|
||||
for (size_t ind = 0; ind < m_align.size(); ++ind) {
|
||||
const std::pair<size_t, size_t> &entry = m_align[ind];
|
||||
alignmentInfo.insert(entry);
|
||||
@ -306,7 +306,7 @@ UINT64 TargetPhrase::ReadFromFile(std::fstream &fileTP)
|
||||
bytesRead += sizeof(UINT64);
|
||||
|
||||
for (size_t ind = 0; ind < numWords; ++ind) {
|
||||
Word *word = new Word();
|
||||
WordPtr word(new Word());
|
||||
bytesRead += word->ReadFromFile(fileTP);
|
||||
AddWord(word);
|
||||
}
|
||||
@ -316,9 +316,9 @@ UINT64 TargetPhrase::ReadFromFile(std::fstream &fileTP)
|
||||
fileTP.read((char*) &numSourceWords, sizeof(UINT64));
|
||||
bytesRead += sizeof(UINT64);
|
||||
|
||||
SourcePhrase *sp = new SourcePhrase();
|
||||
PhrasePtr sp(new SourcePhrase());
|
||||
for (size_t ind = 0; ind < numSourceWords; ++ind) {
|
||||
Word *word = new Word();
|
||||
WordPtr word( new Word());
|
||||
bytesRead += word->ReadFromFile(fileTP);
|
||||
sp->AddWord(word);
|
||||
}
|
||||
|
@ -51,7 +51,7 @@ class TargetPhrase: public Phrase
|
||||
friend std::ostream& operator<<(std::ostream&, const TargetPhrase&);
|
||||
protected:
|
||||
AlignType m_align;
|
||||
Phrase* m_sourcePhrase;
|
||||
PhrasePtr m_sourcePhrase;
|
||||
|
||||
std::vector<float> m_scores;
|
||||
UINT64 m_filePos;
|
||||
@ -67,15 +67,14 @@ public:
|
||||
TargetPhrase(const TargetPhrase ©);
|
||||
virtual ~TargetPhrase();
|
||||
|
||||
void SetSourcePhrase(Phrase *p) {
|
||||
Phrase *copy = new Phrase(*p);
|
||||
m_sourcePhrase = copy;
|
||||
void SetSourcePhrase(PhrasePtr p) {
|
||||
m_sourcePhrase = p;
|
||||
}
|
||||
const Phrase* GetSourcePhrase() const {
|
||||
return m_sourcePhrase;
|
||||
const PhrasePtr GetSourcePhrase() const {
|
||||
return m_sourcePhrase;
|
||||
}
|
||||
|
||||
void SetLHS(Word *lhs);
|
||||
void SetLHS(WordPtr lhs);
|
||||
|
||||
void Create1AlignFromString(const std::string &align1Str);
|
||||
void CreateAlignFromString(const std::string &align1Str);
|
||||
|
@ -22,6 +22,7 @@
|
||||
#include <vector>
|
||||
#include <iostream>
|
||||
#include <fstream>
|
||||
#include <boost/shared_ptr.hpp>
|
||||
#include "Vocab.h"
|
||||
|
||||
namespace Moses
|
||||
@ -81,5 +82,7 @@ public:
|
||||
bool operator==(const Word &compare) const;
|
||||
|
||||
};
|
||||
|
||||
typedef boost::shared_ptr<Word> WordPtr;
|
||||
}
|
||||
|
||||
|
@ -38,20 +38,20 @@ void Tokenize(OnDiskPt::Phrase &phrase
|
||||
|
||||
if (splitPos == string::npos) {
|
||||
// lhs - only 1 word
|
||||
Word *word = new Word();
|
||||
WordPtr word (new Word());
|
||||
word->CreateFromString(wordStr, onDiskWrapper.GetVocab());
|
||||
phrase.AddWord(word);
|
||||
} else {
|
||||
// source & target non-terms
|
||||
if (addSourceNonTerm) {
|
||||
Word *word = new Word();
|
||||
WordPtr word( new Word());
|
||||
word->CreateFromString(wordStr, onDiskWrapper.GetVocab());
|
||||
phrase.AddWord(word);
|
||||
}
|
||||
|
||||
wordStr = token.substr(splitPos, tokSize - splitPos);
|
||||
if (addTargetNonTerm) {
|
||||
Word *word = new Word();
|
||||
WordPtr word(new Word());
|
||||
word->CreateFromString(wordStr, onDiskWrapper.GetVocab());
|
||||
phrase.AddWord(word);
|
||||
}
|
||||
@ -59,7 +59,7 @@ void Tokenize(OnDiskPt::Phrase &phrase
|
||||
}
|
||||
} else {
|
||||
// term
|
||||
Word *word = new Word();
|
||||
WordPtr word(new Word());
|
||||
word->CreateFromString(token, onDiskWrapper.GetVocab());
|
||||
phrase.AddWord(word);
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user