Merge branch 'master' of github.com:moses-smt/mosesdecoder

Jamroot

@@ -89,7 +89,7 @@ if [ path.exists $(home)/moses-environment.jam ]
 include $(TOP)/jam-files/check-environment.jam ; # get resource locations 
                                                  # from environment variables
 include $(TOP)/jam-files/xmlrpc-c.jam ; # xmlrpc-c stuff for the server
-include $(TOP)/jam-files/curlpp.jam ;   # curlpp stuff for bias lookup (MMT only)
+# include $(TOP)/jam-files/curlpp.jam ;   # curlpp stuff for bias lookup (MMT only)
 
 # exit "done" : 0 ;
 

moses/TranslationModel/UG/mm/test-http-client.cc

@@ -0,0 +1,27 @@
+// -*- c++ -*-
+#include "ug_http_client.h"
+
+int main(int argc, char* argv[])
+{
+  try
+    {
+      if (argc != 2)
+	{
+	  std::cout << "Usage: async_client <url>\n";
+	  std::cout << "Example:\n";
+	  std::cout << "  async_client www.boost.org/LICENSE_1_0.txt\n";
+	  return 1;
+	}
+
+      boost::asio::io_service io_service;
+      Moses::http_client c(io_service, argv[1]);
+      io_service.run();
+      std::cout << c.content() << std::endl;
+    }
+  catch (std::exception& e)
+    {
+      std::cout << "Exception: " << e.what() << "\n";
+    }
+
+  return 0;
+}

moses/TranslationModel/UG/mm/test-xml-escaping.cc

@@ -0,0 +1,13 @@
+#include <iostream>
+#include <string>
+#include <iomanip>
+#include "ug_http_client.h"
+
+using namespace std;
+int main()
+{
+  string line;
+  while (getline(cin,line))
+    cout << Moses::uri_encode(line) << endl;
+}
+

moses/TranslationModel/UG/mm/ug_http_client.cc

@@ -0,0 +1,200 @@
+#include "ug_http_client.h"
+namespace Moses
+{
+using boost::asio::ip::tcp;
+
+std::string http_client::content() const { return m_content.str(); }
+
+http_client::
+http_client(boost::asio::io_service& io_service,
+	    const std::string& server, const std::string& path)
+  : resolver_(io_service), socket_(io_service)
+{
+  init(server,path);
+}
+
+http_client::
+http_client(boost::asio::io_service& io_service, std::string url)
+  : resolver_(io_service), socket_(io_service)
+{
+  size_t p = url.find("://");
+  if (p < url.size()) url.erase(0,p+3);
+  p = url.find("/");
+  if (p < url.size()) 
+    init(url.substr(0,p),url.substr(p));
+  else 
+    init(url,"/");
+}
+
+void 
+http_client::
+init(std::string const& server, std::string const& path)
+{
+  // Form the request. We specify the "Connection: close" header so
+  // that the server will close the socket after transmitting the
+  // response. This will allow us to treat all data up until the EOF
+  // as the content.
+
+  std::ostream request_stream(&request_);
+  request_stream << "GET " << path << " HTTP/1.0\r\n";
+  request_stream << "Host: " << server << "\r\n";
+  request_stream << "Accept: */*\r\n";
+  request_stream << "Connection: close\r\n\r\n";
+  
+  // Start an asynchronous resolve to translate the server and service names
+  // into a list of endpoints.
+  tcp::resolver::query query(server, "http");
+  resolver_.async_resolve(query,
+			  boost::bind(&http_client::handle_resolve, this,
+				      boost::asio::placeholders::error,
+				      boost::asio::placeholders::iterator));
+  
+}
+
+void 
+http_client::
+handle_resolve(const boost::system::error_code& err,
+	       tcp::resolver::iterator endpoint_iterator)
+{
+  if (!err)
+    {
+      // Attempt a connection to the first endpoint in the list. Each endpoint
+      // will be tried until we successfully establish a connection.
+      tcp::endpoint endpoint = *endpoint_iterator;
+      socket_.async_connect(endpoint,
+			    boost::bind(&http_client::handle_connect, this,
+					boost::asio::placeholders::error, ++endpoint_iterator));
+    }
+  else
+    {
+      m_error << "Error: " << err.message() << "\n";
+    }
+}
+
+void 
+http_client::
+handle_connect(const boost::system::error_code& err,
+		      tcp::resolver::iterator endpoint_iterator)
+{
+  if (!err)
+    {
+      // The connection was successful. Send the request.
+      boost::asio::async_write(socket_, request_,
+			       boost::bind(&http_client::handle_write_request, this,
+					   boost::asio::placeholders::error));
+    }
+  else if (endpoint_iterator != tcp::resolver::iterator())
+    {
+      // The connection failed. Try the next endpoint in the list.
+      socket_.close();
+      tcp::endpoint endpoint = *endpoint_iterator;
+      socket_.async_connect(endpoint,
+			    boost::bind(&http_client::handle_connect, this,
+					boost::asio::placeholders::error, ++endpoint_iterator));
+    }
+  else m_error << "Error: " << err.message() << "\n";
+}
+
+void 
+http_client::
+handle_write_request(const boost::system::error_code& err)
+{
+  using namespace boost::asio;
+  if (err) { m_error << "Error: " << err.message() << "\n"; return; }
+  
+  // Read the response status line. The response_ streambuf will
+  // automatically grow to accommodate the entire line. The growth may be
+  // limited by passing a maximum size to the streambuf constructor.
+  async_read_until(socket_, response_, "\r\n",
+		   boost::bind(&http_client::handle_read_status_line,
+			       this, placeholders::error));
+}
+
+void 
+http_client::
+handle_read_status_line(const boost::system::error_code& err)
+{
+  if (err) { m_error << "Error: " << err << "\n"; return; }
+
+  using namespace boost::asio;
+  // Check that response is OK.
+  std::istream response_stream(&response_);
+  response_stream >> m_http_version >> m_status_code;
+  std::getline(response_stream, m_status_message);
+  if (!response_stream || m_http_version.substr(0, 5) != "HTTP/")
+    m_error << "Invalid response\n"; 
+  else if (m_status_code != 200)
+    m_error << "Response returned with status code " << m_status_code << "\n";
+  else // Read the response headers, which are terminated by a blank line.
+    async_read_until(socket_, response_, "\r\n\r\n",
+		     boost::bind(&http_client::handle_read_headers, this,
+				 placeholders::error));
+}
+
+
+void 
+http_client::
+handle_read_headers(const boost::system::error_code& err)
+{
+  if (err) { m_error << "Error: " << err << "\n"; return; }
+  
+  // Process the response headers.
+  std::istream response_stream(&response_);
+  std::string line;
+  while (std::getline(response_stream, line) && line != "\r")
+    m_header.push_back(line);
+  
+  // Write whatever content we already have to output.
+  if (response_.size() > 0)
+    m_content << &response_;
+  
+  using namespace boost::asio;
+  // Start reading remaining data until EOF.
+  async_read(socket_, response_, transfer_at_least(1),
+	     boost::bind(&http_client::handle_read_content, this,
+			 placeholders::error));
+}
+
+void 
+http_client::
+handle_read_content(const boost::system::error_code& err)
+{
+  using namespace boost::asio;
+  if(!err)
+    {
+      // Write all of the data that has been read so far.
+      // Then continue reading remaining data until EOF.
+      m_content << &response_; 
+      async_read(socket_, response_, transfer_at_least(1),
+		 boost::bind(&http_client::handle_read_content, this,
+			     placeholders::error));
+    }
+  else if (err != boost::asio::error::eof) 
+    { 
+      m_error << "Error: " << err << "\n"; 
+    }
+}
+
+std::string
+uri_encode(std::string const& in)
+{
+  char buf[3 * in.size() + 1];
+  size_t i = 0;
+  for (unsigned char const* c = (unsigned char const*)in.c_str(); *c; ++c)
+    {
+      // cout << *c << " " << int(*c) << endl;
+      if (*c == ' ') buf[i++] = '+';
+      else if (*c == '.' || *c == '~' || *c == '_' || *c == '-') buf[i++] = *c;
+      else if (*c <  '0') i += sprintf(buf+i, "%%%x", int(*c));
+      else if (*c <= '9') buf[i++] = *c;
+      else if (*c <  'A') i += sprintf(buf+i, "%%%x", int(*c));
+      else if (*c <= 'Z') buf[i++] = *c;
+      else if (*c <  'a') i += sprintf(buf+i, "%%%x", int(*c));
+      else if (*c <= 'z') buf[i++] = *c;
+      else i += sprintf(buf+i, "%%%x", int(*c));
+    }
+  buf[i] = 0;
+  return std::string(buf);
+}
+
+}

moses/TranslationModel/UG/mm/ug_http_client.h

@@ -0,0 +1,57 @@
+// -*- c++ -*-
+// Adapted by Ulrich Germann from: 
+// async_client.cpp
+// ~~~~~~~~~~~~~~~~
+//
+// Copyright (c) 2003-2011 Christopher M. Kohlhoff (chris at kohlhoff dot com)
+//
+// Distributed under the Boost Software License, Version 1.0. (See accompanying
+// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+//
+#pragma once
+#include <iostream>
+#include <istream>
+#include <ostream>
+#include <string>
+#include <sstream>
+#include <boost/asio.hpp>
+#include <boost/bind.hpp>
+
+namespace Moses
+{
+using boost::asio::ip::tcp;
+
+std::string uri_encode(std::string const& in);
+
+class http_client
+{
+  std::ostringstream m_content;
+  std::vector<std::string> m_header;
+  std::string m_http_version;
+  unsigned int m_status_code;
+  std::string m_status_message;
+  std::ostringstream m_error;
+
+public:
+  http_client(boost::asio::io_service& io_service, std::string url);
+  http_client(boost::asio::io_service& io_service,
+	      const std::string& server, const std::string& path);
+private:
+  void init(std::string const& server, std::string const& path);
+  void handle_resolve(const boost::system::error_code& err,
+		      tcp::resolver::iterator endpoint_iterator);
+  void handle_connect(const boost::system::error_code& err,
+		      tcp::resolver::iterator endpoint_iterator);
+  void handle_write_request(const boost::system::error_code& err);
+  void handle_read_status_line(const boost::system::error_code& err);
+  void handle_read_headers(const boost::system::error_code& err);
+  void handle_read_content(const boost::system::error_code& err);
+  tcp::resolver resolver_;
+  tcp::socket socket_;
+  boost::asio::streambuf request_;
+  boost::asio::streambuf response_;
+public:
+  std::string content() const;
+};
+
+}

moses/TranslationModel/UG/mm/ug_sampling_bias.cc

@@ -3,11 +3,15 @@
 #include <boost/foreach.hpp>
 #include "moses/Timer.h"
 
-#ifdef HAVE_CURLPP
-#include <curlpp/Options.hpp>
-#include <curlpp/cURLpp.hpp>
-#include <curlpp/Easy.hpp>
-#endif
+// #ifdef HAVE_CURLPP
+// #include <curlpp/Options.hpp>
+// #include <curlpp/cURLpp.hpp>
+// #include <curlpp/Easy.hpp>
+// #endif
+
+// #ifdef WITH_MMT_BIAS_CLIENT
+#include "ug_http_client.h"
+// #endif
 
 namespace Moses
 {
@@ -15,21 +19,17 @@ namespace Moses
   {
     using ugdiss::id_type;
 
-#ifdef HAVE_CURLPP
+    // #ifdef WITH_MMT_BIAS_CLIENT
     std::string
     query_bias_server(std::string const& url, std::string const& text)
     {
-      // communicate with the bias server; resuts will be in ...
-      std::ostringstream os;
-      curlpp::Easy myRequest;
-      std::string query = url+curlpp::escape(text);
-      myRequest.setOpt(new curlpp::options::Url(query));
-      curlpp::options::WriteStream ws(&os);
-      myRequest.setOpt(ws); // Give it to your request
-      myRequest.perform();  // This will output to os
-      return os.str();
+      std::string query = url+uri_encode(text);
+      boost::asio::io_service io_service;
+      Moses::http_client c(io_service, query);
+      io_service.run();
+      return c.content();
     }
-#endif
+    // #endif
 
     DocumentBias
     ::DocumentBias
@@ -40,13 +40,13 @@ namespace Moses
       : m_sid2docid(sid2doc)
       , m_bias(docname2docid.size(), 0)
     {
-#ifdef HAVE_CURLPP
+      // #ifdef HAVE_CURLPP
       Timer timer;
       if (log) timer.start(NULL);
       std::string json = query_bias_server(server_url, text);
       init_from_json(json, docname2docid, log);
       if (log) *log << "Bias query took " << timer << " seconds." << std::endl;
-#endif
+      // #endif
     }
 
     void

moses/TranslationModel/UG/mmsapt.cpp

@@ -1,8 +1,8 @@
-#ifdef HAVE_CURLPP
-#include <curlpp/Options.hpp>
-#include <curlpp/cURLpp.hpp>
-#include <curlpp/Easy.hpp>
-#endif
+// #ifdef HAVE_CURLPP
+// #include <curlpp/Options.hpp>
+// #include <curlpp/cURLpp.hpp>
+// #include <curlpp/Easy.hpp>
+// #endif
 
 #include "mmsapt.h"
 #include <boost/foreach.hpp>