In extract-rules, if the source or target syntax contains an unsupported

escape sequence (anything other than "<", ">", "&", "&apos",
and "&quot") then write a warning message and skip the sentence pair
(instead of asserting).


git-svn-id: https://mosesdecoder.svn.sourceforge.net/svnroot/mosesdecoder/trunk@3350 1f5c12ca-751b-0410-a591-d2e778427230

scripts/training/phrase-extract/SentenceAlignment.cpp

@@ -25,22 +25,30 @@
 
 #include "tables-core.h"
 
-void SentenceAlignment::processTargetSentence(const char * targetString)
+bool SentenceAlignment::processTargetSentence(const char * targetString, int)
 {
     target = tokenize(targetString);
+    return true;
 }
 
-void SentenceAlignment::processSourceSentence(const char * sourceString)
+bool SentenceAlignment::processSourceSentence(const char * sourceString, int)
 {
     source = tokenize(sourceString);
+    return true;
 }
 
 bool SentenceAlignment::create( char targetString[], char sourceString[], char alignmentString[], int sentenceID) {
     using namespace std;
 
     // process sentence strings and store in target and source members.
-    processTargetSentence(targetString);
-    processSourceSentence(sourceString);
+    if (!processTargetSentence(targetString, sentenceID))
+    {
+        return false;
+    }
+    if (!processSourceSentence(sourceString, sentenceID))
+    {
+        return false;
+    }
 
   // check if sentences are empty
   if (target.size() == 0 || source.size() == 0) {

scripts/training/phrase-extract/SentenceAlignment.h

@@ -32,9 +32,9 @@ class SentenceAlignment
         std::vector<int> alignedCountS;
         std::vector<std::vector<int> > alignedToT;
 
-        virtual void processTargetSentence(const char *);
+        virtual bool processTargetSentence(const char *, int);
 
-        virtual void processSourceSentence(const char *);
+        virtual bool processSourceSentence(const char *, int);
 
         bool create(char targetString[], char sourceString[],
                     char alignmentString[], int sentenceID);

scripts/training/phrase-extract/SentenceAlignmentWithSyntax.cpp

@@ -24,32 +24,49 @@
 #include <string>
 
 #include "tables-core.h"
+#include "XmlException.h"
 #include "XmlTree.h"
 
-void SentenceAlignmentWithSyntax::processTargetSentence(const char * targetString)
+bool SentenceAlignmentWithSyntax::processTargetSentence(const char * targetString, int sentenceID)
 {
-    // tokenizing target (and potentially extract syntax spans)
-    if (m_options.targetSyntax) {
-        string targetStringCPP = string(targetString);
-        ProcessAndStripXMLTags(targetStringCPP, targetTree, m_targetLabelCollection ,
+    if (!m_options.targetSyntax) {
+        return SentenceAlignment::processTargetSentence(targetString, sentenceID);
+    }
+
+    string targetStringCPP(targetString);
+    try {
+        ProcessAndStripXMLTags(targetStringCPP, targetTree,
+                               m_targetLabelCollection,
                                m_targetTopLabelCollection);
-        target = tokenize(targetStringCPP.c_str());
     }
-    else {
-        target = tokenize(targetString);
+    catch (const XmlException & e)
+    {
+        std::cerr << "WARNING: failed to process target sentence at line "
+                  << sentenceID << ": " << e.getMsg() << std::endl;
+        return false;
     }
+    target = tokenize(targetStringCPP.c_str());
+    return true;
 }
 
-void SentenceAlignmentWithSyntax::processSourceSentence(const char * sourceString)
+bool SentenceAlignmentWithSyntax::processSourceSentence(const char * sourceString, int sentenceID)
 {
-    // tokenizing source (and potentially extract syntax spans)
-    if (m_options.sourceSyntax) {
-        string sourceStringCPP = string(sourceString);
-        ProcessAndStripXMLTags(sourceStringCPP, sourceTree, m_sourceLabelCollection ,
+    if (!m_options.sourceSyntax) {
+        return SentenceAlignment::processSourceSentence(sourceString, sentenceID);
+    }
+
+    string sourceStringCPP(sourceString);
+    try {
+        ProcessAndStripXMLTags(sourceStringCPP, sourceTree,
+                               m_sourceLabelCollection ,
                                m_sourceTopLabelCollection);
-        source = tokenize(sourceStringCPP.c_str());
     }
-    else {
-        source = tokenize(sourceString);
+    catch (const XmlException & e)
+    {
+        std::cerr << "WARNING: failed to process source sentence at line "
+                  << sentenceID << ": " << e.getMsg() << std::endl;
+        return false;
     }
+    source = tokenize(sourceStringCPP.c_str());
+    return true;
 }

scripts/training/phrase-extract/SentenceAlignmentWithSyntax.h

@@ -53,11 +53,11 @@ class SentenceAlignmentWithSyntax : public SentenceAlignment
           , m_options(options)
         {}
 
-        void
-        processTargetSentence(const char *);
+        bool
+        processTargetSentence(const char *, int);
 
-        void
-        processSourceSentence(const char *);
+        bool
+        processSourceSentence(const char *, int);
 };
 
 #endif

scripts/training/phrase-extract/XmlException.h

@@ -0,0 +1,39 @@
+/***********************************************************************
+  Moses - factored phrase-based language decoder
+  Copyright (C) 2010 University of Edinburgh
+
+  This library is free software; you can redistribute it and/or
+  modify it under the terms of the GNU Lesser General Public
+  License as published by the Free Software Foundation; either
+  version 2.1 of the License, or (at your option) any later version.
+
+  This library is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+  Lesser General Public License for more details.
+
+  You should have received a copy of the GNU Lesser General Public
+  License along with this library; if not, write to the Free Software
+  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ ***********************************************************************/
+#pragma once
+#ifndef XMLEXCEPTION_H_INCLUDED_
+#define XMLEXCEPTION_H_INCLUDED_
+
+#include <string>
+
+class XmlException
+{
+    public:
+        XmlException(const std::string & msg)
+          : m_msg(msg)
+        {}
+
+        const std::string &
+        getMsg() const { return m_msg; }
+
+    private:
+        std::string m_msg;
+};
+
+#endif

scripts/training/phrase-extract/XmlTree.cpp

@@ -26,7 +26,9 @@
 #include <set>
 #include <iostream>
 #include <stdlib.h>
+#include <sstream>
 #include "SyntaxTree.h"
+#include "XmlException.h"
 
 using namespace std;
 
@@ -134,13 +136,26 @@ string unescape(const string& str)
         {
             s += string("&");
         }
+        else if (name == "apos")
+        {
+            s += string("'");
+        }
+        else if (name == "quot")
+        {
+            s += string("\"");
+        }
         else
         {
-            // TODO Currently only handles '<', '>', and '&' because those are
-            //      the only chracters that the parsing wrapper scripts will
-            //      escape.  Should handle all possible &...; forms in case the
-            //      input is ever generated by another program.
-            assert(!"Unsupported XML escape sequence");
+            // Currently only handles the following five XML escape sequences:
+            //      &lt;        <
+            //      &gt;        >
+            //      &amp;       &
+            //      &apos;      '
+            //      &quot;      "
+            // Numeric character references (like &#xf6;) are not supported.
+            std::ostringstream msg;
+            msg << "unsupported XML escape sequence: &" << name << ";";
+            throw XmlException(msg.str());
         }
         if (end == str.size()-1)
         {