moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2024-09-11 11:25:40 +03:00
Code Issues Packages Projects Releases Wiki Activity

fix memory leaks in binarisation

Browse Source
This commit is contained in:
Barry Haddow 2012-09-19 21:48:10 +01:00
parent 236611193e
commit 576bc09fa5
11 changed files with 45 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
OnDiskPt/Main.cpp
View File

@ -77,7 +77,7 @@ int main (int argc, char * const argv[])
std::vector<float> misc(1);
SourcePhrase sourcePhrase;
TargetPhrase *targetPhrase = new TargetPhrase(numScores);
OnDiskPt::Phrase *spShort = Tokenize(sourcePhrase, *targetPhrase, line, onDiskWrapper, numScores, misc);
OnDiskPt::PhrasePtr spShort = Tokenize(sourcePhrase, *targetPhrase, line, onDiskWrapper, numScores, misc);
assert(misc.size() == onDiskWrapper.GetNumCounts());
rootNode.AddTargetPhrase(sourcePhrase, targetPhrase, onDiskWrapper, tableLimit, misc, spShort);
@ -105,7 +105,7 @@ bool Flush(const OnDiskPt::SourcePhrase *prevSourcePhrase, const OnDiskPt::Sourc
return ret;
}
OnDiskPt::Phrase *Tokenize(SourcePhrase &sourcePhrase, TargetPhrase &targetPhrase, char *line, OnDiskWrapper &onDiskWrapper, int numScores, vector<float> &misc)
OnDiskPt::PhrasePtr Tokenize(SourcePhrase &sourcePhrase, TargetPhrase &targetPhrase, char *line, OnDiskWrapper &onDiskWrapper, int numScores, vector<float> &misc)
{
size_t scoreInd = 0;
@ -118,14 +118,14 @@ OnDiskPt::Phrase *Tokenize(SourcePhrase &sourcePhrase, TargetPhrase &targetPhras
4 = count
*/
char *tok = strtok (line," ");
OnDiskPt::Phrase *out = new Phrase();
OnDiskPt::PhrasePtr out(new Phrase());
while (tok != NULL) {
if (0 == strcmp(tok, "|||")) {
++stage;
} else {
switch (stage) {
case 0: {
Word *w = Tokenize(sourcePhrase, tok, true, true, onDiskWrapper);
WordPtr w = Tokenize(sourcePhrase, tok, true, true, onDiskWrapper);
if (w != NULL)
out->AddWord(w);
@ -184,7 +184,7 @@ OnDiskPt::Phrase *Tokenize(SourcePhrase &sourcePhrase, TargetPhrase &targetPhras
return out;
} // Tokenize()
OnDiskPt::Word *Tokenize(OnDiskPt::Phrase &phrase
OnDiskPt::WordPtr Tokenize(OnDiskPt::Phrase &phrase
, const std::string &token, bool addSourceNonTerm, bool addTargetNonTerm
, OnDiskPt::OnDiskWrapper &onDiskWrapper)
{
@ -198,7 +198,7 @@ OnDiskPt::Word *Tokenize(OnDiskPt::Phrase &phrase
nonTerm = comStr == 0;
}
OnDiskPt::Word *out = NULL;
OnDiskPt::WordPtr out;
if (nonTerm) {
// non-term
size_t splitPos = token.find_first_of("[", 2);
@ -206,20 +206,20 @@ OnDiskPt::Word *Tokenize(OnDiskPt::Phrase &phrase
if (splitPos == string::npos) {
// lhs - only 1 word
Word *word = new Word();
WordPtr word(new Word());
word->CreateFromString(wordStr, onDiskWrapper.GetVocab());
phrase.AddWord(word);
} else {
// source & target non-terms
if (addSourceNonTerm) {
Word *word = new Word();
WordPtr word(new Word());
word->CreateFromString(wordStr, onDiskWrapper.GetVocab());
phrase.AddWord(word);
}
wordStr = token.substr(splitPos, tokSize - splitPos);
if (addTargetNonTerm) {
Word *word = new Word();
WordPtr word(new Word());
word->CreateFromString(wordStr, onDiskWrapper.GetVocab());
phrase.AddWord(word);
out = word;
@ -228,7 +228,7 @@ OnDiskPt::Word *Tokenize(OnDiskPt::Phrase &phrase
}
} else {
// term
Word *word = new Word();
WordPtr word(new Word());
word->CreateFromString(token, onDiskWrapper.GetVocab());
phrase.AddWord(word);
out = word;

4
OnDiskPt/Main.h
View File

@ -25,10 +25,10 @@
typedef std::pair<size_t, size_t> AlignPair;
typedef std::vector<AlignPair> AlignType;
OnDiskPt::Word *Tokenize(OnDiskPt::Phrase &phrase
OnDiskPt::WordPtr Tokenize(OnDiskPt::Phrase &phrase
, const std::string &token, bool addSourceNonTerm, bool addTargetNonTerm
, OnDiskPt::OnDiskWrapper &onDiskWrapper);
OnDiskPt::Phrase *Tokenize(OnDiskPt::SourcePhrase &sourcePhrase, OnDiskPt::TargetPhrase &targetPhrase
OnDiskPt::PhrasePtr Tokenize(OnDiskPt::SourcePhrase &sourcePhrase, OnDiskPt::TargetPhrase &targetPhrase
, char *line, OnDiskPt::OnDiskWrapper &onDiskWrapper
, int numScores
, std::vector<float> &misc);

18
OnDiskPt/Phrase.cpp
View File

@ -27,27 +27,13 @@ using namespace std;
namespace OnDiskPt
{
Phrase::Phrase(const Phrase &copy)
:m_words(copy.GetSize())
{
for (size_t pos = 0; pos < copy.GetSize(); ++pos) {
const Word &oldWord = copy.GetWord(pos);
Word *newWord = new Word(oldWord);
m_words[pos] = newWord;
}
}
Phrase::~Phrase()
{
Moses::RemoveAllInColl(m_words);
}
void Phrase::AddWord(Word *word)
void Phrase::AddWord(WordPtr word)
{
m_words.push_back(word);
}
void Phrase::AddWord(Word *word, size_t pos)
void Phrase::AddWord(WordPtr word, size_t pos)
{
CHECK(pos < m_words.size());
m_words.insert(m_words.begin() + pos + 1, word);

12
OnDiskPt/Phrase.h
View File

@ -20,12 +20,14 @@
***********************************************************************/
#include <vector>
#include <iostream>
#include <boost/shared_ptr.hpp>
#include "Word.h"
namespace OnDiskPt
{
class Vocab;
/** A contiguous phrase. SourcePhrase & TargetPhrase inherit from this and add the on-disk functionality
*/
class Phrase
@ -33,16 +35,14 @@ class Phrase
friend std::ostream& operator<<(std::ostream&, const Phrase&);
protected:
std::vector<Word*> m_words;
std::vector<WordPtr> m_words;
public:
Phrase()
{}
Phrase(const Phrase &copy);
virtual ~Phrase();
void AddWord(Word *word);
void AddWord(Word *word, size_t pos);
void AddWord(WordPtr word);
void AddWord(WordPtr word, size_t pos);
const Word &GetWord(size_t pos) const {
return *m_words[pos];
@ -59,4 +59,6 @@ public:
bool operator==(const Phrase &compare) const;
};
typedef boost::shared_ptr<Phrase> PhrasePtr;
}

4
OnDiskPt/PhraseNode.cpp
View File

@ -160,14 +160,14 @@ void PhraseNode::Save(OnDiskWrapper &onDiskWrapper, size_t pos, size_t tableLimi
void PhraseNode::AddTargetPhrase(const SourcePhrase &sourcePhrase, TargetPhrase *targetPhrase
, OnDiskWrapper &onDiskWrapper, size_t tableLimit
, const std::vector<float> &counts, OnDiskPt::Phrase *spShort)
, const std::vector<float> &counts, OnDiskPt::PhrasePtr spShort)
{
AddTargetPhrase(0, sourcePhrase, targetPhrase, onDiskWrapper, tableLimit, counts, spShort);
}
void PhraseNode::AddTargetPhrase(size_t pos, const SourcePhrase &sourcePhrase
, TargetPhrase *targetPhrase, OnDiskWrapper &onDiskWrapper
, size_t tableLimit, const std::vector<float> &counts, OnDiskPt::Phrase *spShort)
, size_t tableLimit, const std::vector<float> &counts, OnDiskPt::PhrasePtr spShort)
{
size_t phraseSize = sourcePhrase.GetSize();
if (pos < phraseSize) {

4
OnDiskPt/PhraseNode.h
View File

@ -52,7 +52,7 @@ protected:
void AddTargetPhrase(size_t pos, const SourcePhrase &sourcePhrase
, TargetPhrase *targetPhrase, OnDiskWrapper &onDiskWrapper
, size_t tableLimit, const std::vector<float> &counts, OnDiskPt::Phrase *spShort);
, size_t tableLimit, const std::vector<float> &counts, OnDiskPt::PhrasePtr spShort);
size_t ReadChild(Word &wordFound, UINT64 &childFilePos, const char *mem) const;
void GetChild(Word &wordFound, UINT64 &childFilePos, size_t ind, OnDiskWrapper &onDiskWrapper) const;
@ -68,7 +68,7 @@ public:
void AddTargetPhrase(const SourcePhrase &sourcePhrase, TargetPhrase *targetPhrase
, OnDiskWrapper &onDiskWrapper, size_t tableLimit
, const std::vector<float> &counts, OnDiskPt::Phrase *spShort);
, const std::vector<float> &counts, OnDiskPt::PhrasePtr spShort);
UINT64 GetFilePos() const {
return m_filePos;

1
OnDiskPt/SourcePhrase.h
View File

@ -34,4 +34,5 @@ protected:
public:
};
}

12
OnDiskPt/TargetPhrase.cpp
View File

@ -50,7 +50,7 @@ TargetPhrase::~TargetPhrase()
{
}
void TargetPhrase::SetLHS(Word *lhs)
void TargetPhrase::SetLHS(WordPtr lhs)
{
AddWord(lhs);
}
@ -99,7 +99,7 @@ char *TargetPhrase::WriteToMemory(OnDiskWrapper &onDiskWrapper, size_t &memUsed)
size_t phraseSize = GetSize();
size_t targetWordSize = onDiskWrapper.GetTargetWordSize();
const Phrase* sp = GetSourcePhrase();
const PhrasePtr sp = GetSourcePhrase();
size_t spSize = sp->GetSize();
size_t sourceWordSize = onDiskWrapper.GetSourceWordSize();
@ -252,7 +252,7 @@ Moses::TargetPhrase *TargetPhrase::ConvertToMoses(const std::vector<Moses::Facto
int indicator[m_align.size()];
int index = 0;
std::set<std::pair<size_t, size_t> > alignmentInfo;
const Phrase* sp = GetSourcePhrase();
const PhrasePtr sp = GetSourcePhrase();
for (size_t ind = 0; ind < m_align.size(); ++ind) {
const std::pair<size_t, size_t> &entry = m_align[ind];
alignmentInfo.insert(entry);
@ -306,7 +306,7 @@ UINT64 TargetPhrase::ReadFromFile(std::fstream &fileTP)
bytesRead += sizeof(UINT64);
for (size_t ind = 0; ind < numWords; ++ind) {
Word *word = new Word();
WordPtr word(new Word());
bytesRead += word->ReadFromFile(fileTP);
AddWord(word);
}
@ -316,9 +316,9 @@ UINT64 TargetPhrase::ReadFromFile(std::fstream &fileTP)
fileTP.read((char*) &numSourceWords, sizeof(UINT64));
bytesRead += sizeof(UINT64);
SourcePhrase *sp = new SourcePhrase();
PhrasePtr sp(new SourcePhrase());
for (size_t ind = 0; ind < numSourceWords; ++ind) {
Word *word = new Word();
WordPtr word( new Word());
bytesRead += word->ReadFromFile(fileTP);
sp->AddWord(word);
}

13
OnDiskPt/TargetPhrase.h
View File

@ -51,7 +51,7 @@ class TargetPhrase: public Phrase
friend std::ostream& operator<<(std::ostream&, const TargetPhrase&);
protected:
AlignType m_align;
Phrase* m_sourcePhrase;
PhrasePtr m_sourcePhrase;
std::vector<float> m_scores;
UINT64 m_filePos;
@ -67,15 +67,14 @@ public:
TargetPhrase(const TargetPhrase &copy);
virtual ~TargetPhrase();
void SetSourcePhrase(Phrase *p) {
Phrase *copy = new Phrase(*p);
m_sourcePhrase = copy;
void SetSourcePhrase(PhrasePtr p) {
m_sourcePhrase = p;
}
const Phrase* GetSourcePhrase() const {
return m_sourcePhrase;
const PhrasePtr GetSourcePhrase() const {
return m_sourcePhrase;
}
void SetLHS(Word *lhs);
void SetLHS(WordPtr lhs);
void Create1AlignFromString(const std::string &align1Str);
void CreateAlignFromString(const std::string &align1Str);

3
OnDiskPt/Word.h
View File

@ -22,6 +22,7 @@
#include <vector>
#include <iostream>
#include <fstream>
#include <boost/shared_ptr.hpp>
#include "Vocab.h"
namespace Moses
@ -81,5 +82,7 @@ public:
bool operator==(const Word &compare) const;
};
typedef boost::shared_ptr<Word> WordPtr;
}

8
OnDiskPt/queryOnDiskPt.cpp
View File

@ -38,20 +38,20 @@ void Tokenize(OnDiskPt::Phrase &phrase
if (splitPos == string::npos) {
// lhs - only 1 word
Word *word = new Word();
WordPtr word (new Word());
word->CreateFromString(wordStr, onDiskWrapper.GetVocab());
phrase.AddWord(word);
} else {
// source & target non-terms
if (addSourceNonTerm) {
Word *word = new Word();
WordPtr word( new Word());
word->CreateFromString(wordStr, onDiskWrapper.GetVocab());
phrase.AddWord(word);
}
wordStr = token.substr(splitPos, tokSize - splitPos);
if (addTargetNonTerm) {
Word *word = new Word();
WordPtr word(new Word());
word->CreateFromString(wordStr, onDiskWrapper.GetVocab());
phrase.AddWord(word);
}
@ -59,7 +59,7 @@ void Tokenize(OnDiskPt::Phrase &phrase
}
} else {
// term
Word *word = new Word();
WordPtr word(new Word());
word->CreateFromString(token, onDiskWrapper.GetVocab());
phrase.AddWord(word);
}