Merge branch 'moses-server-placeholder-patch' of https://github.com/ebay-hlt/mosesdecoder into ebay-hlt-moses-server-placeholder-patch

Conflicts:
	regression-testing/Jamfile
	run-regtests.sh

moses/server/TranslationRequest.cpp

@@ -3,6 +3,8 @@
 #include "moses/ContextScope.h"
 #include <boost/foreach.hpp>
 #include "moses/Util.h"
+#include "moses/Hypothesis.h"
+
 namespace MosesServer
 {
 using namespace std;
@@ -262,8 +264,11 @@ bool
 check(std::map<std::string, xmlrpc_c::value> const& param, 
       std::string const key)
 {
-  std::map<std::string, xmlrpc_c::value>::const_iterator m;
-  return (param.find(key) != param.end());
+  std::map<std::string, xmlrpc_c::value>::const_iterator m = param.find(key);
+  if(m == param.end()) return false;
+  std::string val = string(xmlrpc_c::value_string(m->second));
+  if(val == "true" || val == "True" || val == "TRUE" || val == "1") return true;
+  return false;
 }
 
 void
@@ -367,21 +372,17 @@ run_chart_decoder()
 
 void
 TranslationRequest::
-pack_hypothesis(Moses::Manager const& manager, 
-		vector<Hypothesis const* > const& edges, 
-		string const& key, map<string, xmlrpc_c::value> & dest) const
+pack_hypothesis(const Moses::Manager& manager, vector<Hypothesis const* > const& edges, string const& key,
+                map<string, xmlrpc_c::value> & dest) const
 {
   // target string
   ostringstream target;
-  BOOST_REVERSE_FOREACH(Hypothesis const* e, edges)
-    {
-      // output_phrase(target, e->GetCurrTargetPhrase());
-      manager.OutputSurface(target,*e, m_options.output.factor_order, 
-			    m_options.output.ReportSegmentation, 
-			    m_options.output.ReportAllFactors);
-    }
-  XVERBOSE(1,"SERVER TRANSLATION: " << target.str() << std::endl);
-  
+  BOOST_REVERSE_FOREACH(Hypothesis const* e, edges) {
+    manager.OutputSurface(target, *e, m_options.output.factor_order,
+                          m_options.output.ReportSegmentation, m_options.output.ReportAllFactors);
+  }
+  XVERBOSE(1, "BEST TRANSLATION: " << *(manager.GetBestHypothesis()) << std::endl);
+//  XVERBOSE(1,"SERVER TRANSLATION: " << target.str() << std::endl);
   dest[key] = xmlrpc_c::value_string(target.str());
 
   if (m_withAlignInfo) {
@@ -389,7 +390,7 @@ pack_hypothesis(Moses::Manager const& manager,
 
     vector<xmlrpc_c::value> p_aln;
     BOOST_REVERSE_FOREACH(Hypothesis const* e, edges)
-    add_phrase_aln_info(*e, p_aln);
+      add_phrase_aln_info(*e, p_aln);
     dest["align"] = xmlrpc_c::value_array(p_aln);
   }
 
@@ -397,15 +398,14 @@ pack_hypothesis(Moses::Manager const& manager,
     // word alignment, if requested
     vector<xmlrpc_c::value> w_aln;
     BOOST_FOREACH(Hypothesis const* e, edges)
-    e->OutputLocalWordAlignment(w_aln);
+      e->OutputLocalWordAlignment(w_aln);
     dest["word-align"] = xmlrpc_c::value_array(w_aln);
   }
 }
 
 void
 TranslationRequest::
-pack_hypothesis(Moses::Manager const& manager, 
-		Hypothesis const* h, string const& key,
+pack_hypothesis(const Moses::Manager& manager, Hypothesis const* h, string const& key,
                 map<string, xmlrpc_c::value>& dest) const
 {
   using namespace std;

moses/server/TranslationRequest.h

@@ -58,17 +58,14 @@ TranslationRequest : public virtual Moses::TranslationTask
   run_phrase_decoder();
 
   void
-  pack_hypothesis(Moses::Manager const& manager, 
-		  std::vector<Moses::Hypothesis const* > const& edges,
+  pack_hypothesis(const Moses::Manager& manager, std::vector<Moses::Hypothesis const* > const& edges,
                   std::string const& key,
                   std::map<std::string, xmlrpc_c::value> & dest) const;
 
   void
-  pack_hypothesis(Moses::Manager const& manager,
-		  Moses::Hypothesis const* h, std::string const& key,
+  pack_hypothesis(const Moses::Manager& manager, Moses::Hypothesis const* h, std::string const& key,
                   std::map<std::string, xmlrpc_c::value> & dest) const;
 
-
   // void
   // output_phrase(std::ostream& out, Moses::Phrase const& phrase) const;
 

regression-testing/Jamfile

@@ -1,6 +1,7 @@
 import option path ;
 
 with-regtest = [ option.get "with-regtest" ] ;
+with-xmlrpc = [ option.get "with-xmlrpc-c" ] ;
 
 if $(with-regtest) {
   with-regtest = [ path.root $(with-regtest) [ path.pwd ] ] ;
@@ -24,6 +25,12 @@ if $(with-regtest) {
   actions reg_test_decode {
     $(TOP)/regression-testing/run-single-test.perl --decoder=$(>) --test=$(<:B) --data-dir=$(with-regtest) --test-dir=$(test-dir) && touch $(<)
   }
+
+  if $(with-xmlrpc)  {
+    actions reg_test_decode_server {
+      $(TOP)/regression-testing/run-single-test.perl --server --decoder=$(>) --test=$(<:B) --data-dir=$(with-regtest) --test-dir=$(test-dir) && touch $(<)
+    }
+    reg_test phrase-server : [ glob $(test-dir)/phrase-server.* ] : ../moses-cmd//moses : @reg_test_decode_server ;
   reg_test phrase : [ glob $(test-dir)/phrase.* : $(test-dir)/*withDALM ] : ../moses-cmd//moses : @reg_test_decode ;
   reg_test chart : [ glob $(test-dir)/chart.* : $(test-dir)/*withDALM ] : ../moses-cmd//moses : @reg_test_decode ;
   if [ option.get "with-dalm" : : "yes" ] {

regression-testing/run-single-test.perl

@@ -2,6 +2,8 @@
 
 # $Id$
 
+use Encode;
+use utf8;
 use warnings;
 use strict;
 my $script_dir; BEGIN { use Cwd qw/ abs_path /; use File::Basename; $script_dir = dirname(abs_path($0)); push @INC, $script_dir; }
@@ -9,6 +11,7 @@ use MosesRegressionTesting;
 use Getopt::Long;
 use File::Temp qw ( tempfile );
 use POSIX qw ( strftime );
+use POSIX ":sys_wait_h";
 my @SIGS = qw ( SIGHUP SIGINT SIGQUIT SIGILL SIGTRAP SIGABRT SIGIOT SIGBUS SIGFPE SIGKILL SIGUSR1 SIGSEGV SIGUSR2 SIGPIPE SIGALRM SIGTERM SIGSTKFLT SIGCHLD SIGCONT SIGSTOP SIGTSTP SIGTTIN SIGTTOU SIGURG SIGXCPU SIGXFSZ SIGVTALRM SIGPROF SIGWINCH SIGIO SIGPWR SIGSYS SIGUNUSED SIGRTMIN );
 my ($decoder, $test_name);
 
@@ -17,14 +20,31 @@ my $data_dir;
 my $BIN_TEST = $script_dir;
 my $results_dir;
 my $NBEST = 0;
-
+my $run_server_test = 0;
+my $serverport = int(rand(9999)) + 10001;
+my $url = "http://localhost:$serverport/RPC2";
+my $startupTest = 0;
 GetOptions("decoder=s" => \$decoder,
            "test=s"    => \$test_name,
            "data-dir=s"=> \$data_dir,
            "test-dir=s"=> \$test_dir,
            "results-dir=s"=> \$results_dir,
+           "server"=> \$run_server_test,
+           "startuptest"=> \$startupTest
           ) or exit 1;
 
+if($run_server_test)
+{
+  eval {
+    require XMLRPC::Lite;
+    import XMLRPC::Lite;
+  };
+  if ($@) {
+    die "Error: XMLRPC::Lite not installed, moses server regression tests will not be run. $@";
+  }
+  exit(0) if($startupTest);
+}
+
 die "Please specify a decoder with --decoder\n" unless $decoder;
 die "Please specify a test to run with --test\n" unless $test_name;
 
@@ -72,8 +92,13 @@ if (!-d $truth) {
 }
 
 print "RESULTS AVAILABLE IN: $results\n\n";
-
-my ($o, $elapsed, $ec, $sig) = exec_moses($decoder, $local_moses_ini, $input, $results);
+my ($o, $elapsed, $ec, $sig);
+if($run_server_test) {
+  ($o, $elapsed, $ec, $sig) = exec_moses_server($decoder, $local_moses_ini, $input, $results);
+}
+else {
+  ($o, $elapsed, $ec, $sig) = exec_moses($decoder, $local_moses_ini, $input, $results);
+}
 my $error = ($sig || $ec > 0);
 if ($error) {
   open OUT, ">$results/Summary";
@@ -139,6 +164,54 @@ sub exec_moses {
   return ($o, $elapsed, $ec, $sig);
 }
 
+sub exec_moses_server {
+  my ($decoder, $conf, $input, $results) = @_;
+  my $start_time = time;
+  my ($o, $ec, $sig);
+  $ec = 0; $sig = 0; $o = 0;
+  my $pid = fork();
+  if (not defined $pid) {
+      warn "resources not avilable to fork Moses server\n";
+      $ec = 1; # to generate error
+  } elsif ($pid == 0) {
+      setpgrp(0, 0);
+      warn "Starting Moses server on port $serverport ...\n";
+      ($o, $ec, $sig) = run_command("$decoder --server --server-port $serverport -f $conf -verbose 2 --server-log $results/run.stderr.server 2> $results/run.stderr ");
+      exit;
+      # this should not be reached unless the server fails to start
+  }
+  while( 1==1 ) # wait until the server is listening for requests
+  {
+    sleep 5;
+    my $str = `grep "Listening on port $serverport" $results/run.stderr`;
+    last if($str =~ /Listening/);
+  }
+  my $proxy = XMLRPC::Lite->proxy($url);
+  warn "Opening file $input to write to $results\n";
+  open(TEXTIN, "$input") or die "Can not open the input file to translate with Moses server\n";
+  binmode TEXTIN, ':utf8';
+  open(TEXTOUT, ">$results/run.stdout");
+  binmode TEXTOUT, ':utf8';
+  while(<TEXTIN>)
+  {
+    chop;
+    my $encoded = SOAP::Data->type(string => $_); # NOTE: assuming properly encoded UTF-8 input: check tests before adding them!
+    my %param = ("text" => $encoded);
+    my $result = $proxy->call("translate",\%param)->result;
+    print TEXTOUT $result->{'text'} . "\n";
+  }
+  close(TEXTIN);
+  close(TEXTOUT);
+  my $elapsed = time - $start_time;
+  print STDERR "Finished translating file $input\n";
+  if(waitpid($pid, WNOHANG) <= 0)
+  {
+    warn "Killing process group $pid of the $decoder --server ... \n";
+    kill 9, -$pid;
+  }
+  return ($o, $elapsed, $ec, $sig);
+}
+
 sub run_command {
   my ($cmd) = @_;
   my $o = `$cmd`;

regtest

@@ -1 +1 @@
-Subproject commit e07a00c9733e0fecb8433f1c9d5805d3f0b35c6f
+Subproject commit f434f7e9b04057d82b4e5c55bd49962c8a2852be

run-regtests.sh

@@ -5,11 +5,18 @@
 set -e -o pipefail -x
 
 opt=$(pwd)/opt
-# git submodule init
-# git submodule update regtest
+git submodule init
+git submodule update regtest
+if [ "$RECOMPILE" == "NO" ] ; then
+  RECOMPILE=
+else
+  RECOMPILE="-a"
+fi
 
 # test compilation without xmlrpc-c
-./bjam -j$(nproc) --with-irstlm=$opt --with-boost=$opt --with-cmph=$opt --no-xmlrpc-c --with-regtest=./regtest -a -q $@ || exit $?
+./bjam -j$(nproc) --with-irstlm=$opt --with-boost=$opt --with-cmph=$opt --no-xmlrpc-c --with-regtest=$(pwd)/regtest -a -q $@ || exit $?
 
 # test compilation with xmlrpc-c
-./bjam -j$(nproc) --with-irstlm=$opt --with-boost=$opt --with-cmph=$opt --with-xmlrpc-c=$opt --with-regtest=./regtest -a -q $@
+if ./regression-testing/run-single-test.perl --server --startuptest  ; then
+  ./bjam -j$(nproc) --with-irstlm=$opt --with-boost=$opt --with-cmph=$opt --with-xmlrpc-c=$opt --with-regtest=$(pwd)/regtest $RECOMPILE -q $@
+fi