2008-06-11 14:52:57 +04:00
|
|
|
// $Id$
|
|
|
|
// vim:tabstop=2
|
|
|
|
|
|
|
|
/***********************************************************************
|
|
|
|
Moses - factored phrase-based language decoder
|
|
|
|
Copyright (C) 2006 University of Edinburgh
|
|
|
|
|
|
|
|
This library is free software; you can redistribute it and/or
|
|
|
|
modify it under the terms of the GNU Lesser General Public
|
|
|
|
License as published by the Free Software Foundation; either
|
|
|
|
version 2.1 of the License, or (at your option) any later version.
|
|
|
|
|
|
|
|
This library is distributed in the hope that it will be useful,
|
|
|
|
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
|
|
Lesser General Public License for more details.
|
|
|
|
|
|
|
|
You should have received a copy of the GNU Lesser General Public
|
|
|
|
License along with this library; if not, write to the Free Software
|
|
|
|
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
|
|
|
***********************************************************************/
|
|
|
|
|
2009-10-02 20:58:02 +04:00
|
|
|
#include <stdexcept>
|
|
|
|
|
2008-06-11 14:52:57 +04:00
|
|
|
#include "Sentence.h"
|
|
|
|
#include "PhraseDictionaryMemory.h"
|
|
|
|
#include "TranslationOptionCollectionText.h"
|
|
|
|
#include "StaticData.h"
|
|
|
|
#include "Util.h"
|
|
|
|
|
2010-04-08 21:16:10 +04:00
|
|
|
using namespace std;
|
|
|
|
|
2008-10-09 03:51:26 +04:00
|
|
|
namespace Moses
|
|
|
|
{
|
2010-01-28 15:12:57 +03:00
|
|
|
|
2010-04-08 21:16:10 +04:00
|
|
|
Sentence::Sentence(FactorDirection direction)
|
|
|
|
: Phrase(direction)
|
|
|
|
, InputType()
|
|
|
|
{
|
|
|
|
assert(direction == Input);
|
2011-01-13 03:25:10 +03:00
|
|
|
const StaticData& staticData = StaticData::Instance();
|
|
|
|
if (staticData.GetSearchAlgorithm() == ChartDecoding)
|
|
|
|
{
|
|
|
|
m_defaultLabelSet.insert(StaticData::Instance().GetInputDefaultNonTerminal());
|
|
|
|
}
|
2010-04-08 21:16:10 +04:00
|
|
|
}
|
|
|
|
|
2008-06-11 14:52:57 +04:00
|
|
|
int Sentence::Read(std::istream& in,const std::vector<FactorType>& factorOrder)
|
|
|
|
{
|
|
|
|
const std::string& factorDelimiter = StaticData::Instance().GetFactorDelimiter();
|
|
|
|
std::string line;
|
|
|
|
std::map<std::string, std::string> meta;
|
|
|
|
|
|
|
|
if (getline(in, line, '\n').eof())
|
|
|
|
return 0;
|
2010-05-04 01:39:23 +04:00
|
|
|
|
2010-05-04 01:39:43 +04:00
|
|
|
//get covered words - if continual-partial-translation is switched on, parse input
|
2010-05-04 01:39:23 +04:00
|
|
|
const StaticData &staticData = StaticData::Instance();
|
2010-05-04 01:41:28 +04:00
|
|
|
m_frontSpanCoveredLength = 0;
|
2010-05-04 01:40:17 +04:00
|
|
|
m_sourceCompleted.resize(0);
|
2010-05-04 01:39:23 +04:00
|
|
|
if (staticData.ContinuePartialTranslation()){
|
2010-05-04 01:40:17 +04:00
|
|
|
string initialTargetPhrase;
|
|
|
|
string sourceCompletedStr;
|
2010-05-04 01:39:23 +04:00
|
|
|
int loc1 = line.find( "|||", 0 );
|
|
|
|
int loc2 = line.find( "|||", loc1 + 3 );
|
|
|
|
if (loc1 > -1 && loc2 > -1){
|
2010-05-04 01:40:17 +04:00
|
|
|
initialTargetPhrase = line.substr(0, loc1);
|
|
|
|
sourceCompletedStr = line.substr(loc1 + 3, loc2 - loc1 - 3);
|
2010-05-04 01:39:23 +04:00
|
|
|
line = line.substr(loc2 + 3);
|
2010-05-04 01:40:17 +04:00
|
|
|
sourceCompletedStr = Trim(sourceCompletedStr);
|
|
|
|
initialTargetPhrase = Trim(initialTargetPhrase);
|
|
|
|
m_initialTargetPhrase = initialTargetPhrase;
|
|
|
|
int len = sourceCompletedStr.size();
|
|
|
|
m_sourceCompleted.resize(len);
|
2010-05-04 01:41:28 +04:00
|
|
|
int contiguous = 1;
|
2010-05-04 01:39:23 +04:00
|
|
|
for (int i = 0; i < len; ++i){
|
2010-05-04 01:41:28 +04:00
|
|
|
if (sourceCompletedStr.at(i) == '1')
|
|
|
|
{
|
|
|
|
m_sourceCompleted[i] = true;
|
|
|
|
if (contiguous)
|
|
|
|
m_frontSpanCoveredLength ++;
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
m_sourceCompleted[i] = false;
|
|
|
|
contiguous = 0;
|
|
|
|
}
|
2010-05-04 01:39:23 +04:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2008-12-15 15:52:38 +03:00
|
|
|
// remove extra spaces
|
2008-06-11 14:52:57 +04:00
|
|
|
line = Trim(line);
|
|
|
|
|
2008-12-15 15:52:38 +03:00
|
|
|
// if sentences is specified as "<seg id=1> ... </seg>", extract id
|
|
|
|
meta = ProcessAndStripSGML(line);
|
2008-06-11 14:52:57 +04:00
|
|
|
if (meta.find("id") != meta.end()) { this->SetTranslationId(atol(meta["id"].c_str())); }
|
|
|
|
|
2008-12-15 15:52:38 +03:00
|
|
|
// parse XML markup in translation line
|
2010-05-04 01:39:23 +04:00
|
|
|
//const StaticData &staticData = StaticData::Instance();
|
2011-02-02 14:06:19 +03:00
|
|
|
std::vector<XmlOption*> xmlOptionsList(0);
|
2008-12-15 15:52:38 +03:00
|
|
|
std::vector< size_t > xmlWalls;
|
2008-12-13 16:12:54 +03:00
|
|
|
if (staticData.GetXmlInputType() != XmlPassThrough) {
|
2008-12-15 15:52:38 +03:00
|
|
|
if (!ProcessAndStripXMLTags(line, xmlOptionsList, m_reorderingConstraint, xmlWalls )) {
|
2009-10-02 20:58:02 +04:00
|
|
|
const string msg("Unable to parse XML in line: " + line);
|
|
|
|
TRACE_ERR(msg << endl);
|
|
|
|
throw runtime_error(msg);
|
2008-12-13 16:12:54 +03:00
|
|
|
}
|
|
|
|
}
|
2008-06-11 14:52:57 +04:00
|
|
|
Phrase::CreateFromString(factorOrder, line, factorDelimiter);
|
|
|
|
|
2010-04-08 21:16:10 +04:00
|
|
|
if (staticData.GetSearchAlgorithm() == ChartDecoding)
|
|
|
|
{
|
|
|
|
InitStartEndWord();
|
|
|
|
}
|
|
|
|
|
2008-12-13 16:12:54 +03:00
|
|
|
//now that we have final word positions in phrase (from CreateFromString),
|
|
|
|
//we can make input phrase objects to go with our XmlOptions and create TranslationOptions
|
|
|
|
|
2008-06-11 14:52:57 +04:00
|
|
|
//only fill the vector if we are parsing XML
|
|
|
|
if (staticData.GetXmlInputType() != XmlPassThrough ) {
|
|
|
|
for (size_t i=0; i<GetSize();i++) {
|
|
|
|
m_xmlCoverageMap.push_back(false);
|
|
|
|
}
|
2008-12-13 16:12:54 +03:00
|
|
|
|
|
|
|
//iterXMLOpts will be empty for XmlIgnore
|
|
|
|
//look at each column
|
2011-02-02 14:06:19 +03:00
|
|
|
for(std::vector<XmlOption*>::const_iterator iterXmlOpts = xmlOptionsList.begin();
|
2008-12-13 16:12:54 +03:00
|
|
|
iterXmlOpts != xmlOptionsList.end(); iterXmlOpts++) {
|
|
|
|
|
2011-02-02 14:06:19 +03:00
|
|
|
const XmlOption *xmlOption = *iterXmlOpts;
|
2008-12-13 16:12:54 +03:00
|
|
|
|
2011-02-02 14:06:19 +03:00
|
|
|
TranslationOption *transOpt = new TranslationOption(xmlOption->range, xmlOption->targetPhrase, *this);
|
|
|
|
m_xmlOptionsList.push_back(transOpt);
|
|
|
|
|
|
|
|
for(size_t j=transOpt->GetSourceWordsRange().GetStartPos();j<=transOpt->GetSourceWordsRange().GetEndPos();j++) {
|
|
|
|
m_xmlCoverageMap[j]=true;
|
2008-06-11 14:52:57 +04:00
|
|
|
}
|
2008-12-13 16:12:54 +03:00
|
|
|
|
2011-02-02 14:06:19 +03:00
|
|
|
delete xmlOption;
|
2008-06-11 14:52:57 +04:00
|
|
|
}
|
|
|
|
|
2008-12-13 16:12:54 +03:00
|
|
|
}
|
2008-12-15 15:52:38 +03:00
|
|
|
|
|
|
|
m_reorderingConstraint.InitializeWalls( GetSize() );
|
|
|
|
|
|
|
|
// set reordering walls, if "-monotone-at-punction" is set
|
|
|
|
if (staticData.UseReorderingConstraint())
|
|
|
|
{
|
|
|
|
m_reorderingConstraint.SetMonotoneAtPunctuation( GetSubString( WordsRange(0,GetSize()-1 ) ) );
|
|
|
|
}
|
|
|
|
|
|
|
|
// set walls obtained from xml
|
|
|
|
for(size_t i=0; i<xmlWalls.size(); i++)
|
|
|
|
if( xmlWalls[i] < GetSize() ) // no buggy walls, please
|
|
|
|
m_reorderingConstraint.SetWall( xmlWalls[i], true );
|
|
|
|
m_reorderingConstraint.FinalizeWalls();
|
|
|
|
|
2008-06-11 14:52:57 +04:00
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
|
2010-04-08 21:16:10 +04:00
|
|
|
void Sentence::InitStartEndWord()
|
|
|
|
{
|
|
|
|
FactorCollection &factorCollection = FactorCollection::Instance();
|
|
|
|
|
|
|
|
Word startWord(Input);
|
|
|
|
const Factor *factor = factorCollection.AddFactor(Input, 0, BOS_); // TODO - non-factored
|
|
|
|
startWord.SetFactor(0, factor);
|
|
|
|
PrependWord(startWord);
|
|
|
|
|
|
|
|
Word endWord(Input);
|
|
|
|
factor = factorCollection.AddFactor(Input, 0, EOS_); // TODO - non-factored
|
|
|
|
endWord.SetFactor(0, factor);
|
|
|
|
AddWord(endWord);
|
|
|
|
}
|
|
|
|
|
2008-06-11 14:52:57 +04:00
|
|
|
TranslationOptionCollection*
|
2010-08-10 17:12:00 +04:00
|
|
|
Sentence::CreateTranslationOptionCollection(const TranslationSystem* system) const
|
2008-06-11 14:52:57 +04:00
|
|
|
{
|
|
|
|
size_t maxNoTransOptPerCoverage = StaticData::Instance().GetMaxNoTransOptPerCoverage();
|
2008-12-13 15:08:55 +03:00
|
|
|
float transOptThreshold = StaticData::Instance().GetTranslationOptionThreshold();
|
2010-08-10 17:12:00 +04:00
|
|
|
TranslationOptionCollection *rv= new TranslationOptionCollectionText(system, *this, maxNoTransOptPerCoverage, transOptThreshold);
|
2008-06-11 14:52:57 +04:00
|
|
|
assert(rv);
|
|
|
|
return rv;
|
|
|
|
}
|
|
|
|
void Sentence::Print(std::ostream& out) const
|
|
|
|
{
|
|
|
|
out<<*static_cast<Phrase const*>(this)<<"\n";
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
bool Sentence::XmlOverlap(size_t startPos, size_t endPos) const {
|
|
|
|
for (size_t pos = startPos; pos <= endPos ; pos++)
|
|
|
|
{
|
|
|
|
if (pos < m_xmlCoverageMap.size() && m_xmlCoverageMap[pos]) {
|
|
|
|
return true;
|
2009-05-26 23:30:35 +04:00
|
|
|
}
|
2008-06-11 14:52:57 +04:00
|
|
|
}
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
void Sentence::GetXmlTranslationOptions(std::vector <TranslationOption*> &list, size_t startPos, size_t endPos) const {
|
|
|
|
//iterate over XmlOptions list, find exact source/target matches
|
|
|
|
|
|
|
|
for (std::vector<TranslationOption*>::const_iterator iterXMLOpts = m_xmlOptionsList.begin();
|
|
|
|
iterXMLOpts != m_xmlOptionsList.end(); iterXMLOpts++) {
|
|
|
|
if (startPos == (**iterXMLOpts).GetSourceWordsRange().GetStartPos() && endPos == (**iterXMLOpts).GetSourceWordsRange().GetEndPos()) {
|
|
|
|
list.push_back(*iterXMLOpts);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2009-06-01 00:32:18 +04:00
|
|
|
void Sentence::CreateFromString(const std::vector<FactorType> &factorOrder
|
|
|
|
, const std::string &phraseString
|
|
|
|
, const std::string &factorDelimiter)
|
|
|
|
{
|
|
|
|
Phrase::CreateFromString(factorOrder, phraseString, factorDelimiter);
|
|
|
|
}
|
|
|
|
|
|
|
|
|
2008-10-09 03:51:26 +04:00
|
|
|
|
|
|
|
}
|
|
|
|
|