mirror of
https://github.com/moses-smt/mosesdecoder.git
synced 2024-09-11 19:27:11 +03:00
roll back last change. seg fault in release build
git-svn-id: https://mosesdecoder.svn.sourceforge.net/svnroot/mosesdecoder/trunk@1549 1f5c12ca-751b-0410-a591-d2e778427230
This commit is contained in:
parent
060d26cbb3
commit
fd60fe93b9
@ -41,7 +41,7 @@
|
||||
Name="VCCLCompilerTool"
|
||||
Optimization="0"
|
||||
AdditionalIncludeDirectories=""$(SolutionDir)\src";"$(SolutionDir)..\mysqlpp\lib";"$(SolutionDir)..\irstlm\src";"$(SolutionDir)..\srilm\src""
|
||||
PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE;LM_INTERNAL;TRACE_ENABLE;_CRT_SECURE_NO_DEPRECATE;_SCL_SECURE_NO_DEPRECATE"
|
||||
PreprocessorDefinitions="WIN32;_DEBUG;_CONSOLE;LM_INTERNAL;TRACE_ENABLE;_CRT_SECURE_NO_DEPRECATE;_SCL_SECURE_NO_DEPRECATE;EXIT_RETURN"
|
||||
MinimalRebuild="true"
|
||||
BasicRuntimeChecks="3"
|
||||
RuntimeLibrary="3"
|
||||
|
@ -195,12 +195,12 @@ int main(int argc, char* argv[])
|
||||
IFVERBOSE(1)
|
||||
PrintUserTime("End.");
|
||||
|
||||
//#ifdef HACK_EXIT
|
||||
#ifndef EXIT_RETURN
|
||||
//This avoids that detructors are called (it can take a long time)
|
||||
exit(EXIT_SUCCESS);
|
||||
//#else
|
||||
// return EXIT_SUCCESS;
|
||||
//#endif
|
||||
#else
|
||||
return EXIT_SUCCESS;
|
||||
#endif
|
||||
}
|
||||
|
||||
IOStream *GetIODevice(const StaticData &staticData)
|
||||
|
@ -185,8 +185,10 @@ int Hypothesis::NGramCompare(const Hypothesis &compare) const
|
||||
// 0 = this ==compare
|
||||
if (m_languageModelStates < compare.m_languageModelStates) return -1;
|
||||
if (m_languageModelStates > compare.m_languageModelStates) return 1;
|
||||
if (m_sourceCompleted.GetCompressedRepresentation() < compare.m_sourceCompleted.GetCompressedRepresentation()) return -1;
|
||||
if (m_sourceCompleted.GetCompressedRepresentation() > compare.m_sourceCompleted.GetCompressedRepresentation()) return 1;
|
||||
|
||||
int compareBitmap = m_sourceCompleted.Compare(compare.m_sourceCompleted);
|
||||
if (compareBitmap != 0)
|
||||
return compareBitmap;
|
||||
if (m_currSourceWordsRange.GetEndPos() < compare.m_currSourceWordsRange.GetEndPos()) return -1;
|
||||
if (m_currSourceWordsRange.GetEndPos() > compare.m_currSourceWordsRange.GetEndPos()) return 1;
|
||||
if (! StaticData::Instance().GetSourceStartPosMattersForRecombination()) return 0;
|
||||
|
@ -147,10 +147,10 @@ void Manager::ProcessOneHypothesis(const Hypothesis &hypothesis)
|
||||
|
||||
for (size_t startPos = hypoFirstGapPos ; startPos < sourceSize ; ++startPos)
|
||||
{
|
||||
size_t maxSize = sourceSize - startPos;
|
||||
size_t maxSizePhrase = StaticData::Instance().GetMaxPhraseLength();
|
||||
maxSize = (maxSize < maxSizePhrase) ? maxSize : maxSizePhrase;
|
||||
|
||||
size_t maxSize = sourceSize - startPos;
|
||||
size_t maxSizePhrase = StaticData::Instance().GetMaxPhraseLength();
|
||||
maxSize = (maxSize < maxSizePhrase) ? maxSize : maxSizePhrase;
|
||||
|
||||
for (size_t endPos = startPos ; endPos < startPos + maxSize ; ++endPos)
|
||||
{
|
||||
if (!hypoBitmap.Overlap(WordsRange(startPos, endPos)))
|
||||
@ -296,20 +296,6 @@ void Manager::ExpandHypothesis(const Hypothesis &hypothesis, const TranslationOp
|
||||
if (debug2) { std::cerr << "::EXT: " << transOpt << "\n"; }
|
||||
#endif
|
||||
Hypothesis *newHypo = hypothesis.CreateNext(transOpt);
|
||||
// expand hypothesis further if transOpt was linked
|
||||
for (std::vector<TranslationOption*>::const_iterator iterLinked = transOpt.GetLinkedTransOpts().begin();
|
||||
iterLinked != transOpt.GetLinkedTransOpts().end(); iterLinked++) {
|
||||
const WordsBitmap hypoBitmap = newHypo->GetWordsBitmap();
|
||||
if (hypoBitmap.Overlap((**iterLinked).GetSourceWordsRange())) {
|
||||
// don't want to add a hypothesis that has some but not all of a linked TO set, so return
|
||||
return;
|
||||
}
|
||||
else
|
||||
{
|
||||
newHypo->CalcScore(m_transOptColl->GetFutureScore());
|
||||
newHypo = newHypo->CreateNext(**iterLinked);
|
||||
}
|
||||
}
|
||||
newHypo->CalcScore(m_transOptColl->GetFutureScore());
|
||||
|
||||
// logging for the curious
|
||||
|
@ -42,7 +42,7 @@ int Sentence::Read(std::istream& in,const std::vector<FactorType>& factorOrder)
|
||||
//parse XML markup in translation line
|
||||
const StaticData &staticData = StaticData::Instance();
|
||||
if (staticData.GetXmlInputType() != XmlPassThrough)
|
||||
m_xmlOptionsList = ProcessAndStripXMLTags(line, *this);
|
||||
m_xmlOptionsList = ProcessAndStripXMLTags(line);
|
||||
Phrase::CreateFromString(factorOrder, line, factorDelimiter);
|
||||
|
||||
//only fill the vector if we are parsing XML
|
||||
@ -50,10 +50,9 @@ int Sentence::Read(std::istream& in,const std::vector<FactorType>& factorOrder)
|
||||
for (size_t i=0; i<GetSize();i++) {
|
||||
m_xmlCoverageMap.push_back(false);
|
||||
}
|
||||
for (std::vector<TranslationOption*>::const_iterator iterXMLOpts = m_xmlOptionsList.begin();
|
||||
iterXMLOpts != m_xmlOptionsList.end(); iterXMLOpts++) {
|
||||
for (size_t i=0; i< m_xmlOptionsList.size();i++) {
|
||||
//m_xmlOptionsList will be empty for XmlIgnore
|
||||
for(size_t j=(**iterXMLOpts).GetSourceWordsRange().GetStartPos();j<=(**iterXMLOpts).GetSourceWordsRange().GetEndPos();j++) {
|
||||
for(size_t j=m_xmlOptionsList[i].startPos;j<=m_xmlOptionsList[i].endPos;j++) {
|
||||
m_xmlCoverageMap[j]=true;
|
||||
|
||||
}
|
||||
@ -89,12 +88,24 @@ bool Sentence::XmlOverlap(size_t startPos, size_t endPos) const {
|
||||
|
||||
void Sentence::GetXmlTranslationOptions(std::vector <TranslationOption*> &list, size_t startPos, size_t endPos) const {
|
||||
//iterate over XmlOptions list, find exact source/target matches
|
||||
//we don't worry about creating the objects ahead of time because this should only be called once for each unique start/end when a given sentence is processed
|
||||
const std::vector<FactorType> &outputFactorOrder = StaticData::Instance().GetOutputFactorOrder();
|
||||
|
||||
for (std::vector<TranslationOption*>::const_iterator iterXMLOpts = m_xmlOptionsList.begin();
|
||||
iterXMLOpts != m_xmlOptionsList.end(); iterXMLOpts++) {
|
||||
if (startPos == (**iterXMLOpts).GetSourceWordsRange().GetStartPos() && endPos == (**iterXMLOpts).GetSourceWordsRange().GetEndPos()) {
|
||||
list.push_back(*iterXMLOpts);
|
||||
for(size_t i=0;i<m_xmlOptionsList.size();i++) {
|
||||
if (startPos == m_xmlOptionsList[i].startPos && endPos == m_xmlOptionsList[i].endPos) {
|
||||
//create TranslationOptions
|
||||
|
||||
for (size_t j=0;j<m_xmlOptionsList[i].targetPhrases.size();j++) {
|
||||
TargetPhrase targetPhrase(Output);
|
||||
targetPhrase.CreateFromString(outputFactorOrder,m_xmlOptionsList[i].targetPhrases[j],StaticData::Instance().GetFactorDelimiter());
|
||||
targetPhrase.SetScore(m_xmlOptionsList[i].targetScores[j]);
|
||||
WordsRange range(m_xmlOptionsList[i].startPos,m_xmlOptionsList[i].endPos);
|
||||
|
||||
TranslationOption *option = new TranslationOption(range,targetPhrase,*this);
|
||||
assert(option);
|
||||
list.push_back(option);
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -48,7 +48,7 @@ class Sentence : public Phrase, public InputType
|
||||
* and returns the value of that tag if present, empty string otherwise
|
||||
*/
|
||||
static std::string ParseXmlTagAttribute(const std::string& tag,const std::string& attributeName);
|
||||
std::vector <TranslationOption*> m_xmlOptionsList;
|
||||
std::vector <XmlOption> m_xmlOptionsList;
|
||||
std::vector <bool> m_xmlCoverageMap;
|
||||
|
||||
public:
|
||||
@ -90,4 +90,3 @@ class Sentence : public Phrase, public InputType
|
||||
TranslationOptionCollection* CreateTranslationOptionCollection() const;
|
||||
};
|
||||
|
||||
|
||||
|
@ -60,7 +60,6 @@ protected:
|
||||
Phrase *m_sourcePhrase; /*< input phrase translated by this */
|
||||
const WordsRange m_sourceWordsRange; /*< word position in the input that are covered by this translation option */
|
||||
float m_futureScore; /*< estimate of total cost when using this translation option, includes language model probabilities */
|
||||
std::vector<TranslationOption*> m_linkedTransOpts; /* list of linked TOs which must be included with this in any hypothesis */
|
||||
|
||||
//! in TranslationOption, m_scoreBreakdown is not complete. It cannot,
|
||||
//! for example, know the full n-gram score since the length of the
|
||||
@ -113,18 +112,6 @@ public:
|
||||
{
|
||||
return m_sourcePhrase;
|
||||
}
|
||||
|
||||
/** returns linked TOs */
|
||||
inline const std::vector<TranslationOption*> &GetLinkedTransOpts() const
|
||||
{
|
||||
return m_linkedTransOpts;
|
||||
}
|
||||
|
||||
/** add link to another TO */
|
||||
inline void AddLinkedTransOpt(TranslationOption* to)
|
||||
{
|
||||
m_linkedTransOpts.push_back(to);
|
||||
}
|
||||
|
||||
/** whether source span overlaps with those of a hypothesis */
|
||||
bool Overlap(const Hypothesis &hypothesis) const;
|
||||
@ -180,4 +167,3 @@ public:
|
||||
};
|
||||
|
||||
|
||||
|
||||
|
@ -56,23 +56,3 @@ int WordsBitmap::GetFutureCosts(int lastPos) const
|
||||
return sum;
|
||||
}
|
||||
|
||||
|
||||
std::vector<size_t> WordsBitmap::GetCompressedRepresentation() const
|
||||
{
|
||||
int vectorSize = 1 + m_size / (sizeof(int) * CHAR_BIT);
|
||||
std::vector<size_t> res(vectorSize, 0);
|
||||
size_t x=0; size_t ci=0;
|
||||
for(size_t i=0;i<m_size;++i) {
|
||||
x <<= 1;
|
||||
x |= (size_t)m_bitmap[i];
|
||||
if ((i + 1) % (sizeof(int) * CHAR_BIT) == 0) {
|
||||
res[ci++] = x; x = 0;
|
||||
}
|
||||
}
|
||||
x <<= vectorSize * sizeof(int) * CHAR_BIT - m_size;
|
||||
res[ci] = x;
|
||||
return res;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
@ -149,11 +149,7 @@ public:
|
||||
{
|
||||
return m_size;
|
||||
}
|
||||
/** represent this bitmap as 1 or more vector of integers.
|
||||
* Used for exact matching of source words translated in hypothesis recombination
|
||||
*/
|
||||
std::vector<size_t> GetCompressedRepresentation() const;
|
||||
|
||||
|
||||
//! transitive comparison of WordsBitmap
|
||||
inline int Compare (const WordsBitmap &compare) const
|
||||
{
|
||||
|
@ -26,7 +26,6 @@
|
||||
#include <iostream>
|
||||
#include "Util.h"
|
||||
#include "StaticData.h"
|
||||
#include "TranslationOption.h"
|
||||
|
||||
namespace {
|
||||
|
||||
@ -92,11 +91,10 @@ inline std::vector<std::string> TokenizeXml(const std::string& str)
|
||||
|
||||
}
|
||||
|
||||
std::vector<TranslationOption*> ProcessAndStripXMLTags(std::string& line, const InputType &source) {
|
||||
std::vector<XmlOption> ProcessAndStripXMLTags(std::string& line) {
|
||||
//parse XML markup in translation line
|
||||
std::vector<TranslationOption*> res;
|
||||
std::vector<XmlOption> res;
|
||||
std::string rstr;
|
||||
std::string linkedStr;
|
||||
if (line.find_first_of('<') == std::string::npos) { return res; }
|
||||
std::vector<std::string> xmlTokens = TokenizeXml(line);
|
||||
std::string tagName = "";
|
||||
@ -108,7 +106,6 @@ std::vector<TranslationOption*> ProcessAndStripXMLTags(std::string& line, const
|
||||
size_t curWord=0;
|
||||
int numUnary = 0;
|
||||
bool doClose = false;
|
||||
bool isLinked = false;
|
||||
for (size_t xmlTokenPos = 0 ; xmlTokenPos < xmlTokens.size() ; xmlTokenPos++)
|
||||
{
|
||||
if(!isXmlTag(xmlTokens[xmlTokenPos]))
|
||||
@ -130,30 +127,7 @@ std::vector<TranslationOption*> ProcessAndStripXMLTags(std::string& line, const
|
||||
nextTagName = tag.substr(0,endOfName);
|
||||
tagContents = tag.substr(endOfName+1);
|
||||
}
|
||||
if (nextTagName == "linked") {
|
||||
isLinked = true;
|
||||
linkedStr = "";
|
||||
}
|
||||
else if (nextTagName == "/linked") {
|
||||
isLinked = false;
|
||||
// recurse to process linked tags
|
||||
std::vector<TranslationOption*> tOptions = ProcessAndStripXMLTags(linkedStr, source);
|
||||
// link them together
|
||||
std::vector<TranslationOption*>::const_iterator iterTransOpts1;
|
||||
std::vector<TranslationOption*>::const_iterator iterTransOpts2;
|
||||
for (iterTransOpts1 = tOptions.begin(); iterTransOpts1 != tOptions.end(); iterTransOpts1++) {
|
||||
for (iterTransOpts2 = tOptions.begin(); iterTransOpts2 != tOptions.end(); iterTransOpts2++) {
|
||||
if (iterTransOpts1 != iterTransOpts2) {
|
||||
(**iterTransOpts1).AddLinkedTransOpt(*iterTransOpts2);
|
||||
}
|
||||
}
|
||||
res.push_back(*iterTransOpts1);
|
||||
}
|
||||
}
|
||||
else if (isLinked) {
|
||||
linkedStr += xmlTokens[xmlTokenPos];
|
||||
}
|
||||
else if (isOpen)
|
||||
if (isOpen)
|
||||
{
|
||||
//this is an open tag
|
||||
tagName = nextTagName;
|
||||
@ -208,7 +182,6 @@ std::vector<TranslationOption*> ProcessAndStripXMLTags(std::string& line, const
|
||||
//TODO: deal with multiple XML options here
|
||||
|
||||
if (StaticData::Instance().GetXmlInputType() != XmlIgnore) {
|
||||
const std::vector<FactorType> &outputFactorOrder = StaticData::Instance().GetOutputFactorOrder();
|
||||
for (size_t i=0; i<altTexts.size(); ++i) {
|
||||
//only store options if we aren't ignoring them
|
||||
//set default probability
|
||||
@ -216,15 +189,7 @@ std::vector<TranslationOption*> ProcessAndStripXMLTags(std::string& line, const
|
||||
if (altProbs[i] != "") probValue = Scan<float>(altProbs[i]);
|
||||
//Convert from prob to log-prob
|
||||
float scoreValue = FloorScore(TransformScore(probValue));
|
||||
|
||||
TargetPhrase targetPhrase(Output);
|
||||
targetPhrase.CreateFromString(outputFactorOrder,altTexts[i],StaticData::Instance().GetFactorDelimiter());
|
||||
targetPhrase.SetScore(scoreValue);
|
||||
WordsRange range(tagStart,tagEnd);
|
||||
|
||||
TranslationOption *option = new TranslationOption(range,targetPhrase,source);
|
||||
assert(option);
|
||||
|
||||
XmlOption option(tagStart,tagEnd,altTexts[i],scoreValue);
|
||||
res.push_back(option);
|
||||
}
|
||||
}
|
||||
@ -240,4 +205,3 @@ std::vector<TranslationOption*> ProcessAndStripXMLTags(std::string& line, const
|
||||
return res;
|
||||
}
|
||||
|
||||
|
||||
|
@ -2,7 +2,6 @@
|
||||
|
||||
#include <vector>
|
||||
#include <string>
|
||||
#include "InputType.h"
|
||||
|
||||
/** This struct is used for storing XML force translation data for a given range in the sentence
|
||||
*/
|
||||
@ -19,6 +18,5 @@ struct XmlOption {
|
||||
|
||||
};
|
||||
|
||||
std::vector<TranslationOption*> ProcessAndStripXMLTags(std::string& line, const InputType &source);
|
||||
|
||||
std::vector<XmlOption> ProcessAndStripXMLTags(std::string& line);
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user