mirror of
https://github.com/moses-smt/mosesdecoder.git
synced 2024-09-11 19:27:11 +03:00
In extract-rules, if the source or target syntax contains an unsupported
escape sequence (anything other than "<", ">", "&", "&apos", and """) then write a warning message and skip the sentence pair (instead of asserting). git-svn-id: https://mosesdecoder.svn.sourceforge.net/svnroot/mosesdecoder/trunk@3350 1f5c12ca-751b-0410-a591-d2e778427230
This commit is contained in:
parent
1f6e9b488b
commit
fab2e96d2f
@ -25,22 +25,30 @@
|
||||
|
||||
#include "tables-core.h"
|
||||
|
||||
void SentenceAlignment::processTargetSentence(const char * targetString)
|
||||
bool SentenceAlignment::processTargetSentence(const char * targetString, int)
|
||||
{
|
||||
target = tokenize(targetString);
|
||||
return true;
|
||||
}
|
||||
|
||||
void SentenceAlignment::processSourceSentence(const char * sourceString)
|
||||
bool SentenceAlignment::processSourceSentence(const char * sourceString, int)
|
||||
{
|
||||
source = tokenize(sourceString);
|
||||
return true;
|
||||
}
|
||||
|
||||
bool SentenceAlignment::create( char targetString[], char sourceString[], char alignmentString[], int sentenceID) {
|
||||
using namespace std;
|
||||
|
||||
// process sentence strings and store in target and source members.
|
||||
processTargetSentence(targetString);
|
||||
processSourceSentence(sourceString);
|
||||
if (!processTargetSentence(targetString, sentenceID))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
if (!processSourceSentence(sourceString, sentenceID))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
// check if sentences are empty
|
||||
if (target.size() == 0 || source.size() == 0) {
|
||||
|
@ -32,9 +32,9 @@ class SentenceAlignment
|
||||
std::vector<int> alignedCountS;
|
||||
std::vector<std::vector<int> > alignedToT;
|
||||
|
||||
virtual void processTargetSentence(const char *);
|
||||
virtual bool processTargetSentence(const char *, int);
|
||||
|
||||
virtual void processSourceSentence(const char *);
|
||||
virtual bool processSourceSentence(const char *, int);
|
||||
|
||||
bool create(char targetString[], char sourceString[],
|
||||
char alignmentString[], int sentenceID);
|
||||
|
@ -24,32 +24,49 @@
|
||||
#include <string>
|
||||
|
||||
#include "tables-core.h"
|
||||
#include "XmlException.h"
|
||||
#include "XmlTree.h"
|
||||
|
||||
void SentenceAlignmentWithSyntax::processTargetSentence(const char * targetString)
|
||||
bool SentenceAlignmentWithSyntax::processTargetSentence(const char * targetString, int sentenceID)
|
||||
{
|
||||
// tokenizing target (and potentially extract syntax spans)
|
||||
if (m_options.targetSyntax) {
|
||||
string targetStringCPP = string(targetString);
|
||||
ProcessAndStripXMLTags(targetStringCPP, targetTree, m_targetLabelCollection ,
|
||||
if (!m_options.targetSyntax) {
|
||||
return SentenceAlignment::processTargetSentence(targetString, sentenceID);
|
||||
}
|
||||
|
||||
string targetStringCPP(targetString);
|
||||
try {
|
||||
ProcessAndStripXMLTags(targetStringCPP, targetTree,
|
||||
m_targetLabelCollection,
|
||||
m_targetTopLabelCollection);
|
||||
target = tokenize(targetStringCPP.c_str());
|
||||
}
|
||||
else {
|
||||
target = tokenize(targetString);
|
||||
catch (const XmlException & e)
|
||||
{
|
||||
std::cerr << "WARNING: failed to process target sentence at line "
|
||||
<< sentenceID << ": " << e.getMsg() << std::endl;
|
||||
return false;
|
||||
}
|
||||
target = tokenize(targetStringCPP.c_str());
|
||||
return true;
|
||||
}
|
||||
|
||||
void SentenceAlignmentWithSyntax::processSourceSentence(const char * sourceString)
|
||||
bool SentenceAlignmentWithSyntax::processSourceSentence(const char * sourceString, int sentenceID)
|
||||
{
|
||||
// tokenizing source (and potentially extract syntax spans)
|
||||
if (m_options.sourceSyntax) {
|
||||
string sourceStringCPP = string(sourceString);
|
||||
ProcessAndStripXMLTags(sourceStringCPP, sourceTree, m_sourceLabelCollection ,
|
||||
if (!m_options.sourceSyntax) {
|
||||
return SentenceAlignment::processSourceSentence(sourceString, sentenceID);
|
||||
}
|
||||
|
||||
string sourceStringCPP(sourceString);
|
||||
try {
|
||||
ProcessAndStripXMLTags(sourceStringCPP, sourceTree,
|
||||
m_sourceLabelCollection ,
|
||||
m_sourceTopLabelCollection);
|
||||
source = tokenize(sourceStringCPP.c_str());
|
||||
}
|
||||
else {
|
||||
source = tokenize(sourceString);
|
||||
catch (const XmlException & e)
|
||||
{
|
||||
std::cerr << "WARNING: failed to process source sentence at line "
|
||||
<< sentenceID << ": " << e.getMsg() << std::endl;
|
||||
return false;
|
||||
}
|
||||
source = tokenize(sourceStringCPP.c_str());
|
||||
return true;
|
||||
}
|
||||
|
@ -53,11 +53,11 @@ class SentenceAlignmentWithSyntax : public SentenceAlignment
|
||||
, m_options(options)
|
||||
{}
|
||||
|
||||
void
|
||||
processTargetSentence(const char *);
|
||||
bool
|
||||
processTargetSentence(const char *, int);
|
||||
|
||||
void
|
||||
processSourceSentence(const char *);
|
||||
bool
|
||||
processSourceSentence(const char *, int);
|
||||
};
|
||||
|
||||
#endif
|
||||
|
39
scripts/training/phrase-extract/XmlException.h
Normal file
39
scripts/training/phrase-extract/XmlException.h
Normal file
@ -0,0 +1,39 @@
|
||||
/***********************************************************************
|
||||
Moses - factored phrase-based language decoder
|
||||
Copyright (C) 2010 University of Edinburgh
|
||||
|
||||
This library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
This library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with this library; if not, write to the Free Software
|
||||
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
***********************************************************************/
|
||||
#pragma once
|
||||
#ifndef XMLEXCEPTION_H_INCLUDED_
|
||||
#define XMLEXCEPTION_H_INCLUDED_
|
||||
|
||||
#include <string>
|
||||
|
||||
class XmlException
|
||||
{
|
||||
public:
|
||||
XmlException(const std::string & msg)
|
||||
: m_msg(msg)
|
||||
{}
|
||||
|
||||
const std::string &
|
||||
getMsg() const { return m_msg; }
|
||||
|
||||
private:
|
||||
std::string m_msg;
|
||||
};
|
||||
|
||||
#endif
|
@ -26,7 +26,9 @@
|
||||
#include <set>
|
||||
#include <iostream>
|
||||
#include <stdlib.h>
|
||||
#include <sstream>
|
||||
#include "SyntaxTree.h"
|
||||
#include "XmlException.h"
|
||||
|
||||
using namespace std;
|
||||
|
||||
@ -134,13 +136,26 @@ string unescape(const string& str)
|
||||
{
|
||||
s += string("&");
|
||||
}
|
||||
else if (name == "apos")
|
||||
{
|
||||
s += string("'");
|
||||
}
|
||||
else if (name == "quot")
|
||||
{
|
||||
s += string("\"");
|
||||
}
|
||||
else
|
||||
{
|
||||
// TODO Currently only handles '<', '>', and '&' because those are
|
||||
// the only chracters that the parsing wrapper scripts will
|
||||
// escape. Should handle all possible &...; forms in case the
|
||||
// input is ever generated by another program.
|
||||
assert(!"Unsupported XML escape sequence");
|
||||
// Currently only handles the following five XML escape sequences:
|
||||
// < <
|
||||
// > >
|
||||
// & &
|
||||
// ' '
|
||||
// " "
|
||||
// Numeric character references (like ö) are not supported.
|
||||
std::ostringstream msg;
|
||||
msg << "unsupported XML escape sequence: &" << name << ";";
|
||||
throw XmlException(msg.str());
|
||||
}
|
||||
if (end == str.size()-1)
|
||||
{
|
||||
|
Loading…
Reference in New Issue
Block a user