mirror of
https://github.com/moses-smt/mosesdecoder.git
synced 2024-09-11 11:25:40 +03:00
Removed dependency on libcurlpp, as it was difficult to link that staticly.
This commit is contained in:
parent
3a0cd0518d
commit
da052b7f2b
2
Jamroot
2
Jamroot
@ -89,7 +89,7 @@ if [ path.exists $(home)/moses-environment.jam ]
|
||||
include $(TOP)/jam-files/check-environment.jam ; # get resource locations
|
||||
# from environment variables
|
||||
include $(TOP)/jam-files/xmlrpc-c.jam ; # xmlrpc-c stuff for the server
|
||||
include $(TOP)/jam-files/curlpp.jam ; # curlpp stuff for bias lookup (MMT only)
|
||||
# include $(TOP)/jam-files/curlpp.jam ; # curlpp stuff for bias lookup (MMT only)
|
||||
|
||||
# exit "done" : 0 ;
|
||||
|
||||
|
27
moses/TranslationModel/UG/mm/test-http-client.cc
Normal file
27
moses/TranslationModel/UG/mm/test-http-client.cc
Normal file
@ -0,0 +1,27 @@
|
||||
// -*- c++ -*-
|
||||
#include "ug_http_client.h"
|
||||
|
||||
int main(int argc, char* argv[])
|
||||
{
|
||||
try
|
||||
{
|
||||
if (argc != 2)
|
||||
{
|
||||
std::cout << "Usage: async_client <url>\n";
|
||||
std::cout << "Example:\n";
|
||||
std::cout << " async_client www.boost.org/LICENSE_1_0.txt\n";
|
||||
return 1;
|
||||
}
|
||||
|
||||
boost::asio::io_service io_service;
|
||||
Moses::http_client c(io_service, argv[1]);
|
||||
io_service.run();
|
||||
std::cout << c.content() << std::endl;
|
||||
}
|
||||
catch (std::exception& e)
|
||||
{
|
||||
std::cout << "Exception: " << e.what() << "\n";
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
13
moses/TranslationModel/UG/mm/test-xml-escaping.cc
Normal file
13
moses/TranslationModel/UG/mm/test-xml-escaping.cc
Normal file
@ -0,0 +1,13 @@
|
||||
#include <iostream>
|
||||
#include <string>
|
||||
#include <iomanip>
|
||||
#include "ug_http_client.h"
|
||||
|
||||
using namespace std;
|
||||
int main()
|
||||
{
|
||||
string line;
|
||||
while (getline(cin,line))
|
||||
cout << Moses::uri_encode(line) << endl;
|
||||
}
|
||||
|
200
moses/TranslationModel/UG/mm/ug_http_client.cc
Normal file
200
moses/TranslationModel/UG/mm/ug_http_client.cc
Normal file
@ -0,0 +1,200 @@
|
||||
#include "ug_http_client.h"
|
||||
namespace Moses
|
||||
{
|
||||
using boost::asio::ip::tcp;
|
||||
|
||||
std::string http_client::content() const { return m_content.str(); }
|
||||
|
||||
http_client::
|
||||
http_client(boost::asio::io_service& io_service,
|
||||
const std::string& server, const std::string& path)
|
||||
: resolver_(io_service), socket_(io_service)
|
||||
{
|
||||
init(server,path);
|
||||
}
|
||||
|
||||
http_client::
|
||||
http_client(boost::asio::io_service& io_service, std::string url)
|
||||
: resolver_(io_service), socket_(io_service)
|
||||
{
|
||||
size_t p = url.find("://");
|
||||
if (p < url.size()) url.erase(0,p+3);
|
||||
p = url.find("/");
|
||||
if (p < url.size())
|
||||
init(url.substr(0,p),url.substr(p));
|
||||
else
|
||||
init(url,"/");
|
||||
}
|
||||
|
||||
void
|
||||
http_client::
|
||||
init(std::string const& server, std::string const& path)
|
||||
{
|
||||
// Form the request. We specify the "Connection: close" header so
|
||||
// that the server will close the socket after transmitting the
|
||||
// response. This will allow us to treat all data up until the EOF
|
||||
// as the content.
|
||||
|
||||
std::ostream request_stream(&request_);
|
||||
request_stream << "GET " << path << " HTTP/1.0\r\n";
|
||||
request_stream << "Host: " << server << "\r\n";
|
||||
request_stream << "Accept: */*\r\n";
|
||||
request_stream << "Connection: close\r\n\r\n";
|
||||
|
||||
// Start an asynchronous resolve to translate the server and service names
|
||||
// into a list of endpoints.
|
||||
tcp::resolver::query query(server, "http");
|
||||
resolver_.async_resolve(query,
|
||||
boost::bind(&http_client::handle_resolve, this,
|
||||
boost::asio::placeholders::error,
|
||||
boost::asio::placeholders::iterator));
|
||||
|
||||
}
|
||||
|
||||
void
|
||||
http_client::
|
||||
handle_resolve(const boost::system::error_code& err,
|
||||
tcp::resolver::iterator endpoint_iterator)
|
||||
{
|
||||
if (!err)
|
||||
{
|
||||
// Attempt a connection to the first endpoint in the list. Each endpoint
|
||||
// will be tried until we successfully establish a connection.
|
||||
tcp::endpoint endpoint = *endpoint_iterator;
|
||||
socket_.async_connect(endpoint,
|
||||
boost::bind(&http_client::handle_connect, this,
|
||||
boost::asio::placeholders::error, ++endpoint_iterator));
|
||||
}
|
||||
else
|
||||
{
|
||||
m_error << "Error: " << err.message() << "\n";
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
http_client::
|
||||
handle_connect(const boost::system::error_code& err,
|
||||
tcp::resolver::iterator endpoint_iterator)
|
||||
{
|
||||
if (!err)
|
||||
{
|
||||
// The connection was successful. Send the request.
|
||||
boost::asio::async_write(socket_, request_,
|
||||
boost::bind(&http_client::handle_write_request, this,
|
||||
boost::asio::placeholders::error));
|
||||
}
|
||||
else if (endpoint_iterator != tcp::resolver::iterator())
|
||||
{
|
||||
// The connection failed. Try the next endpoint in the list.
|
||||
socket_.close();
|
||||
tcp::endpoint endpoint = *endpoint_iterator;
|
||||
socket_.async_connect(endpoint,
|
||||
boost::bind(&http_client::handle_connect, this,
|
||||
boost::asio::placeholders::error, ++endpoint_iterator));
|
||||
}
|
||||
else m_error << "Error: " << err.message() << "\n";
|
||||
}
|
||||
|
||||
void
|
||||
http_client::
|
||||
handle_write_request(const boost::system::error_code& err)
|
||||
{
|
||||
using namespace boost::asio;
|
||||
if (err) { m_error << "Error: " << err.message() << "\n"; return; }
|
||||
|
||||
// Read the response status line. The response_ streambuf will
|
||||
// automatically grow to accommodate the entire line. The growth may be
|
||||
// limited by passing a maximum size to the streambuf constructor.
|
||||
async_read_until(socket_, response_, "\r\n",
|
||||
boost::bind(&http_client::handle_read_status_line,
|
||||
this, placeholders::error));
|
||||
}
|
||||
|
||||
void
|
||||
http_client::
|
||||
handle_read_status_line(const boost::system::error_code& err)
|
||||
{
|
||||
if (err) { m_error << "Error: " << err << "\n"; return; }
|
||||
|
||||
using namespace boost::asio;
|
||||
// Check that response is OK.
|
||||
std::istream response_stream(&response_);
|
||||
response_stream >> m_http_version >> m_status_code;
|
||||
std::getline(response_stream, m_status_message);
|
||||
if (!response_stream || m_http_version.substr(0, 5) != "HTTP/")
|
||||
m_error << "Invalid response\n";
|
||||
else if (m_status_code != 200)
|
||||
m_error << "Response returned with status code " << m_status_code << "\n";
|
||||
else // Read the response headers, which are terminated by a blank line.
|
||||
async_read_until(socket_, response_, "\r\n\r\n",
|
||||
boost::bind(&http_client::handle_read_headers, this,
|
||||
placeholders::error));
|
||||
}
|
||||
|
||||
|
||||
void
|
||||
http_client::
|
||||
handle_read_headers(const boost::system::error_code& err)
|
||||
{
|
||||
if (err) { m_error << "Error: " << err << "\n"; return; }
|
||||
|
||||
// Process the response headers.
|
||||
std::istream response_stream(&response_);
|
||||
std::string line;
|
||||
while (std::getline(response_stream, line) && line != "\r")
|
||||
m_header.push_back(line);
|
||||
|
||||
// Write whatever content we already have to output.
|
||||
if (response_.size() > 0)
|
||||
m_content << &response_;
|
||||
|
||||
using namespace boost::asio;
|
||||
// Start reading remaining data until EOF.
|
||||
async_read(socket_, response_, transfer_at_least(1),
|
||||
boost::bind(&http_client::handle_read_content, this,
|
||||
placeholders::error));
|
||||
}
|
||||
|
||||
void
|
||||
http_client::
|
||||
handle_read_content(const boost::system::error_code& err)
|
||||
{
|
||||
using namespace boost::asio;
|
||||
if(!err)
|
||||
{
|
||||
// Write all of the data that has been read so far.
|
||||
// Then continue reading remaining data until EOF.
|
||||
m_content << &response_;
|
||||
async_read(socket_, response_, transfer_at_least(1),
|
||||
boost::bind(&http_client::handle_read_content, this,
|
||||
placeholders::error));
|
||||
}
|
||||
else if (err != boost::asio::error::eof)
|
||||
{
|
||||
m_error << "Error: " << err << "\n";
|
||||
}
|
||||
}
|
||||
|
||||
std::string
|
||||
uri_encode(std::string const& in)
|
||||
{
|
||||
char buf[3 * in.size() + 1];
|
||||
size_t i = 0;
|
||||
for (unsigned char const* c = (unsigned char const*)in.c_str(); *c; ++c)
|
||||
{
|
||||
// cout << *c << " " << int(*c) << endl;
|
||||
if (*c == ' ') buf[i++] = '+';
|
||||
else if (*c == '.' || *c == '~' || *c == '_' || *c == '-') buf[i++] = *c;
|
||||
else if (*c < '0') i += sprintf(buf+i, "%%%x", int(*c));
|
||||
else if (*c <= '9') buf[i++] = *c;
|
||||
else if (*c < 'A') i += sprintf(buf+i, "%%%x", int(*c));
|
||||
else if (*c <= 'Z') buf[i++] = *c;
|
||||
else if (*c < 'a') i += sprintf(buf+i, "%%%x", int(*c));
|
||||
else if (*c <= 'z') buf[i++] = *c;
|
||||
else i += sprintf(buf+i, "%%%x", int(*c));
|
||||
}
|
||||
buf[i] = 0;
|
||||
return std::string(buf);
|
||||
}
|
||||
|
||||
}
|
57
moses/TranslationModel/UG/mm/ug_http_client.h
Normal file
57
moses/TranslationModel/UG/mm/ug_http_client.h
Normal file
@ -0,0 +1,57 @@
|
||||
// -*- c++ -*-
|
||||
// Adapted by Ulrich Germann from:
|
||||
// async_client.cpp
|
||||
// ~~~~~~~~~~~~~~~~
|
||||
//
|
||||
// Copyright (c) 2003-2011 Christopher M. Kohlhoff (chris at kohlhoff dot com)
|
||||
//
|
||||
// Distributed under the Boost Software License, Version 1.0. (See accompanying
|
||||
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
|
||||
//
|
||||
#pragma once
|
||||
#include <iostream>
|
||||
#include <istream>
|
||||
#include <ostream>
|
||||
#include <string>
|
||||
#include <sstream>
|
||||
#include <boost/asio.hpp>
|
||||
#include <boost/bind.hpp>
|
||||
|
||||
namespace Moses
|
||||
{
|
||||
using boost::asio::ip::tcp;
|
||||
|
||||
std::string uri_encode(std::string const& in);
|
||||
|
||||
class http_client
|
||||
{
|
||||
std::ostringstream m_content;
|
||||
std::vector<std::string> m_header;
|
||||
std::string m_http_version;
|
||||
unsigned int m_status_code;
|
||||
std::string m_status_message;
|
||||
std::ostringstream m_error;
|
||||
|
||||
public:
|
||||
http_client(boost::asio::io_service& io_service, std::string url);
|
||||
http_client(boost::asio::io_service& io_service,
|
||||
const std::string& server, const std::string& path);
|
||||
private:
|
||||
void init(std::string const& server, std::string const& path);
|
||||
void handle_resolve(const boost::system::error_code& err,
|
||||
tcp::resolver::iterator endpoint_iterator);
|
||||
void handle_connect(const boost::system::error_code& err,
|
||||
tcp::resolver::iterator endpoint_iterator);
|
||||
void handle_write_request(const boost::system::error_code& err);
|
||||
void handle_read_status_line(const boost::system::error_code& err);
|
||||
void handle_read_headers(const boost::system::error_code& err);
|
||||
void handle_read_content(const boost::system::error_code& err);
|
||||
tcp::resolver resolver_;
|
||||
tcp::socket socket_;
|
||||
boost::asio::streambuf request_;
|
||||
boost::asio::streambuf response_;
|
||||
public:
|
||||
std::string content() const;
|
||||
};
|
||||
|
||||
}
|
@ -3,11 +3,15 @@
|
||||
#include <boost/foreach.hpp>
|
||||
#include "moses/Timer.h"
|
||||
|
||||
#ifdef HAVE_CURLPP
|
||||
#include <curlpp/Options.hpp>
|
||||
#include <curlpp/cURLpp.hpp>
|
||||
#include <curlpp/Easy.hpp>
|
||||
#endif
|
||||
// #ifdef HAVE_CURLPP
|
||||
// #include <curlpp/Options.hpp>
|
||||
// #include <curlpp/cURLpp.hpp>
|
||||
// #include <curlpp/Easy.hpp>
|
||||
// #endif
|
||||
|
||||
// #ifdef WITH_MMT_BIAS_CLIENT
|
||||
#include "ug_http_client.h"
|
||||
// #endif
|
||||
|
||||
namespace Moses
|
||||
{
|
||||
@ -15,21 +19,17 @@ namespace Moses
|
||||
{
|
||||
using ugdiss::id_type;
|
||||
|
||||
#ifdef HAVE_CURLPP
|
||||
// #ifdef WITH_MMT_BIAS_CLIENT
|
||||
std::string
|
||||
query_bias_server(std::string const& url, std::string const& text)
|
||||
{
|
||||
// communicate with the bias server; resuts will be in ...
|
||||
std::ostringstream os;
|
||||
curlpp::Easy myRequest;
|
||||
std::string query = url+curlpp::escape(text);
|
||||
myRequest.setOpt(new curlpp::options::Url(query));
|
||||
curlpp::options::WriteStream ws(&os);
|
||||
myRequest.setOpt(ws); // Give it to your request
|
||||
myRequest.perform(); // This will output to os
|
||||
return os.str();
|
||||
std::string query = url+uri_encode(text);
|
||||
boost::asio::io_service io_service;
|
||||
Moses::client c(io_service, query);
|
||||
io_service.run();
|
||||
return c.content();
|
||||
}
|
||||
#endif
|
||||
// #endif
|
||||
|
||||
DocumentBias
|
||||
::DocumentBias
|
||||
@ -40,13 +40,13 @@ namespace Moses
|
||||
: m_sid2docid(sid2doc)
|
||||
, m_bias(docname2docid.size(), 0)
|
||||
{
|
||||
#ifdef HAVE_CURLPP
|
||||
// #ifdef HAVE_CURLPP
|
||||
Timer timer;
|
||||
if (log) timer.start(NULL);
|
||||
std::string json = query_bias_server(server_url, text);
|
||||
init_from_json(json, docname2docid, log);
|
||||
if (log) *log << "Bias query took " << timer << " seconds." << std::endl;
|
||||
#endif
|
||||
// #endif
|
||||
}
|
||||
|
||||
void
|
||||
|
@ -1,8 +1,8 @@
|
||||
#ifdef HAVE_CURLPP
|
||||
#include <curlpp/Options.hpp>
|
||||
#include <curlpp/cURLpp.hpp>
|
||||
#include <curlpp/Easy.hpp>
|
||||
#endif
|
||||
// #ifdef HAVE_CURLPP
|
||||
// #include <curlpp/Options.hpp>
|
||||
// #include <curlpp/cURLpp.hpp>
|
||||
// #include <curlpp/Easy.hpp>
|
||||
// #endif
|
||||
|
||||
#include "mmsapt.h"
|
||||
#include <boost/foreach.hpp>
|
||||
|
Loading…
Reference in New Issue
Block a user