mirror of
https://github.com/moses-smt/mosesdecoder.git
synced 2024-09-11 19:27:11 +03:00
Merge 3791-3842 from trunk
git-svn-id: http://svn.statmt.org/repository/mira@3873 cc96ff50-19ce-11e0-b349-13d7f0bd23df
This commit is contained in:
parent
ec6272b3fc
commit
e664b4a4b3
@ -2,8 +2,8 @@ bin_PROGRAMS = CreateOnDiskPt
|
||||
CreateOnDiskPt_SOURCES = Main.cpp
|
||||
AM_CPPFLAGS = -W -Wall -ffor-scope -D_FILE_OFFSET_BITS=64 -D_LARGE_FILES -DUSE_HYPO_POOL -I$(top_srcdir)/moses/src $(BOOST_CPPFLAGS)
|
||||
|
||||
CreateOnDiskPt_LDADD = -L$(top_srcdir)/OnDiskPt/src -L$(top_srcdir)/moses/src -lOnDiskPt -lmoses
|
||||
CreateOnDiskPt_DEPENDENCIES = $(top_srcdir)/OnDiskPt/src/libOnDiskPt.a $(top_srcdir)/moses/src/libmoses.la
|
||||
CreateOnDiskPt_LDADD = -L$(top_srcdir)/OnDiskPt/src -L$(top_srcdir)/moses/src -lOnDiskPt -lmoses $(BOOST_THREAD_LDFLAGS) $(BOOST_THREAD_LIBS) @KENLM_LDFLAGS@
|
||||
CreateOnDiskPt_DEPENDENCIES = $(top_srcdir)/OnDiskPt/src/libOnDiskPt.a $(top_srcdir)/moses/src/libmoses.la @KENLM_DEPS@
|
||||
|
||||
|
||||
|
||||
|
@ -14,4 +14,4 @@ endif
|
||||
if WITH_UNITTEST
|
||||
UNITTEST = unittest
|
||||
endif
|
||||
SUBDIRS = moses/src moses-chart/src OnDiskPt/src kenlm moses-cmd/src misc moses-chart-cmd/src CreateOnDisk/src mira $(MERT) $(SERVER) $(UNITTEST)
|
||||
SUBDIRS = kenlm moses/src moses-chart/src OnDiskPt/src moses-cmd/src misc moses-chart-cmd/src CreateOnDisk/src mira $(MERT) $(SERVER) $(UNITTEST)
|
||||
|
21
config.h.in
21
config.h.in
@ -1,19 +1,19 @@
|
||||
/* config.h.in. Generated from configure.in by autoheader. */
|
||||
|
||||
/* define if the Boost library is available */
|
||||
/* Defined if the requested minimum BOOST version is satisfied */
|
||||
#undef HAVE_BOOST
|
||||
|
||||
/* define if the Boost::MPI library is available */
|
||||
#undef HAVE_BOOST_MPI
|
||||
/* Define to 1 if you have <boost/archive/text_oarchive.hpp> */
|
||||
#undef HAVE_BOOST_ARCHIVE_TEXT_OARCHIVE_HPP
|
||||
|
||||
/* define if the Boost::PROGRAM_OPTIONS library is available */
|
||||
#undef HAVE_BOOST_PROGRAM_OPTIONS
|
||||
/* Define to 1 if you have <boost/mpi/communicator.hpp> */
|
||||
#undef HAVE_BOOST_MPI_COMMUNICATOR_HPP
|
||||
|
||||
/* define if the Boost::Serialization library is available */
|
||||
#undef HAVE_BOOST_SERIALIZATION
|
||||
/* Define to 1 if you have <boost/program_options.hpp> */
|
||||
#undef HAVE_BOOST_PROGRAM_OPTIONS_HPP
|
||||
|
||||
/* define if the Boost::Thread library is available */
|
||||
#undef HAVE_BOOST_THREAD
|
||||
/* Define to 1 if you have <boost/thread.hpp> */
|
||||
#undef HAVE_BOOST_THREAD_HPP
|
||||
|
||||
/* Define to 1 if you have the <dlfcn.h> header file. */
|
||||
#undef HAVE_DLFCN_H
|
||||
@ -88,6 +88,9 @@
|
||||
/* Define to the one symbol short name of this package. */
|
||||
#undef PACKAGE_TARNAME
|
||||
|
||||
/* Define to the home page for this package. */
|
||||
#undef PACKAGE_URL
|
||||
|
||||
/* Define to the version of this package. */
|
||||
#undef PACKAGE_VERSION
|
||||
|
||||
|
28
configure.in
28
configure.in
@ -16,8 +16,8 @@ AC_PROG_LIBTOOL
|
||||
|
||||
AX_XMLRPC_C
|
||||
|
||||
AX_BOOST_BASE([1.37.0])
|
||||
AX_BOOST_PROGRAM_OPTIONS
|
||||
BOOST_REQUIRE([1.37.0])
|
||||
BOOST_PROGRAM_OPTIONS
|
||||
|
||||
ac_have_mpi=no
|
||||
AC_ARG_WITH(mpi,
|
||||
@ -31,8 +31,8 @@ if test $ac_have_mpi = yes ; then
|
||||
if test $CXX = none ; then
|
||||
AC_MSG_ERROR([Cannot locate MPI compiler drivers])
|
||||
fi
|
||||
AX_BOOST_SERIALIZATION
|
||||
AX_BOOST_MPI
|
||||
BOOST_SERIALIZATION
|
||||
BOOST_MPI
|
||||
AC_DEFINE(MPI_ENABLED,1,[Define if compiling with MPI.])
|
||||
CPPFLAGS="$CPPFLAGS -DMPI_ENABLE=1"
|
||||
fi
|
||||
@ -133,13 +133,13 @@ fi
|
||||
if test "x$with_threads" = 'xyes' || test "x$enable_boost" = 'xyes'
|
||||
then
|
||||
AC_MSG_NOTICE([Using Boost library])
|
||||
AX_BOOST_BASE([1.36.0])
|
||||
BOOST_REQUIRE([1.36.0])
|
||||
fi
|
||||
|
||||
if test "x$with_threads" = 'xyes'
|
||||
then
|
||||
AC_MSG_NOTICE([Building threaded moses])
|
||||
AX_BOOST_THREAD
|
||||
BOOST_THREADS
|
||||
CPPFLAGS="$CPPFLAGS -DWITH_THREADS"
|
||||
AM_CONDITIONAL([WITH_THREADS],true)
|
||||
else
|
||||
@ -149,10 +149,6 @@ fi
|
||||
AM_CONDITIONAL([WITH_UNITTEST], false)
|
||||
if test "x$enable_unittest" = 'xyes'
|
||||
then
|
||||
if test "x$BOOST_CPPFLAGS" = 'x'
|
||||
then
|
||||
AC_MSG_ERROR([Require boost to build unit tests])
|
||||
fi
|
||||
AC_MSG_NOTICE([Building unit tests])
|
||||
AM_CONDITIONAL([WITH_UNITTEST], true)
|
||||
fi
|
||||
@ -236,12 +232,16 @@ then
|
||||
[AC_DEFINE([HAVE_KENLM], [], [flag for KENLM])],
|
||||
[AC_MSG_ERROR([Cannot find KEN-LM in ${PWD}/kenlm])])
|
||||
|
||||
LIB_KENLM="-lkenlm"
|
||||
LDFLAGS="$LDFLAGS -L${PWD}/kenlm"
|
||||
LIBS="$LIBS $LIB_KENLM"
|
||||
FMTLIBS="$FMTLIBS libkenlm.a"
|
||||
KENLM_LDFLAGS="-L\$(top_srcdir)/kenlm -lkenlm -lz"
|
||||
KENLM_DEPS="\$(top_srcdir)/kenlm/libkenlm.la"
|
||||
FMTLIBS="$FMTLIBS libkenlm.la"
|
||||
AM_CONDITIONAL([KEN_LM], true)
|
||||
else
|
||||
KENLM_LDFLAGS=""
|
||||
KENLM_DEPS=""
|
||||
fi
|
||||
AC_SUBST(KENLM_LDFLAGS)
|
||||
AC_SUBST(KENLM_DEPS)
|
||||
|
||||
if test "x$with_randlm" != 'xno'
|
||||
then
|
||||
|
@ -1,8 +1,8 @@
|
||||
lib_LIBRARIES = libkenlm.a
|
||||
lib_LTLIBRARIES = libkenlm.la
|
||||
bin_PROGRAMS = query build_binary
|
||||
|
||||
AM_CPPFLAGS = -W -Wall -ffor-scope -D_FILE_OFFSET_BITS=64 -D_LARGE_FILES $(BOOST_CPPFLAGS)
|
||||
libkenlm_a_SOURCES = \
|
||||
libkenlm_la_SOURCES = \
|
||||
lm/lm_exception.cc \
|
||||
lm/config.cc \
|
||||
lm/model.cc \
|
||||
@ -24,10 +24,8 @@ libkenlm_a_SOURCES = \
|
||||
util/bit_packing.cc
|
||||
|
||||
query_SOURCES = lm/ngram_query.cc
|
||||
query_DEPENDENCIES = libkenlm.a
|
||||
query_LDADD = -L$(top_srcdir)/kenlm -lkenlm -lz
|
||||
query_LDADD = libkenlm.la
|
||||
|
||||
build_binary_SOURCES = lm/build_binary.cc
|
||||
build_binary_DEPENDENCIES = libkenlm.a
|
||||
build_binary_LDADD = -L$(top_srcdir)/kenlm -lkenlm -lz
|
||||
build_binary_LDADD = libkenlm.la
|
||||
|
||||
|
@ -35,17 +35,11 @@
|
||||
#endif
|
||||
|
||||
#include <algorithm>
|
||||
#include <iostream>
|
||||
|
||||
#ifdef HAVE_ICU
|
||||
U_NAMESPACE_BEGIN
|
||||
#endif
|
||||
|
||||
std::ostream& operator<<(std::ostream& o, const StringPiece& piece) {
|
||||
o.write(piece.data(), static_cast<std::streamsize>(piece.size()));
|
||||
return o;
|
||||
}
|
||||
|
||||
#ifdef HAVE_BOOST
|
||||
size_t hash_value(const StringPiece &str) {
|
||||
return boost::hash_range(str.data(), str.data() + str.length());
|
||||
|
@ -56,13 +56,14 @@
|
||||
#include <cstring>
|
||||
#include <iosfwd>
|
||||
|
||||
#include <ostream>
|
||||
|
||||
#ifdef HAVE_ICU
|
||||
#include <unicode/stringpiece.h>
|
||||
U_NAMESPACE_BEGIN
|
||||
#else
|
||||
|
||||
#include <algorithm>
|
||||
#include <ostream>
|
||||
#include <string>
|
||||
#include <string.h>
|
||||
|
||||
@ -228,7 +229,9 @@ inline bool operator>=(const StringPiece& x, const StringPiece& y) {
|
||||
}
|
||||
|
||||
// allow StringPiece to be logged (needed for unit testing).
|
||||
extern std::ostream& operator<<(std::ostream& o, const StringPiece& piece);
|
||||
inline std::ostream& operator<<(std::ostream& o, const StringPiece& piece) {
|
||||
return o.write(piece.data(), static_cast<std::streamsize>(piece.size()));
|
||||
}
|
||||
|
||||
#ifdef HAVE_BOOST
|
||||
size_t hash_value(const StringPiece &str);
|
||||
|
@ -1,223 +0,0 @@
|
||||
# ===========================================================================
|
||||
# http://autoconf-archive.cryp.to/ax_boost_base.html
|
||||
# ===========================================================================
|
||||
#
|
||||
# SYNOPSIS
|
||||
#
|
||||
# AX_BOOST_BASE([MINIMUM-VERSION])
|
||||
#
|
||||
# DESCRIPTION
|
||||
#
|
||||
# Test for the Boost C++ libraries of a particular version (or newer)
|
||||
#
|
||||
# If no path to the installed boost library is given the macro searchs
|
||||
# under /usr, /usr/local, /opt and /opt/local and evaluates the
|
||||
# $BOOST_ROOT environment variable. Further documentation is available at
|
||||
# <http://randspringer.de/boost/index.html>.
|
||||
#
|
||||
# This macro calls:
|
||||
#
|
||||
# AC_SUBST(BOOST_CPPFLAGS) / AC_SUBST(BOOST_LDFLAGS)
|
||||
#
|
||||
# And sets:
|
||||
#
|
||||
# HAVE_BOOST
|
||||
#
|
||||
# LAST MODIFICATION
|
||||
#
|
||||
# 2008-04-12
|
||||
#
|
||||
# COPYLEFT
|
||||
#
|
||||
# Copyright (c) 2008 Thomas Porschberg <thomas@randspringer.de>
|
||||
#
|
||||
# Copying and distribution of this file, with or without modification, are
|
||||
# permitted in any medium without royalty provided the copyright notice
|
||||
# and this notice are preserved.
|
||||
|
||||
AC_DEFUN([AX_BOOST_BASE],
|
||||
[
|
||||
AC_ARG_WITH([boost],
|
||||
AS_HELP_STRING([--with-boost@<:@=DIR@:>@], [use boost (default is yes) - it is possible to specify the root directory for boost (optional)]),
|
||||
[
|
||||
if test "$withval" = "no"; then
|
||||
want_boost="no"
|
||||
elif test "$withval" = "yes"; then
|
||||
want_boost="yes"
|
||||
ac_boost_path=""
|
||||
else
|
||||
want_boost="yes"
|
||||
ac_boost_path="$withval"
|
||||
fi
|
||||
],
|
||||
[want_boost="yes"])
|
||||
|
||||
|
||||
AC_ARG_WITH([boost-libdir],
|
||||
AS_HELP_STRING([--with-boost-libdir=LIB_DIR],
|
||||
[Force given directory for boost libraries. Note that this will overwrite library path detection, so use this parameter only if default library detection fails and you know exactly where your boost libraries are located.]),
|
||||
[
|
||||
if test -d $withval
|
||||
then
|
||||
ac_boost_lib_path="$withval"
|
||||
else
|
||||
AC_MSG_ERROR(--with-boost-libdir expected directory name)
|
||||
fi
|
||||
],
|
||||
[ac_boost_lib_path=""]
|
||||
)
|
||||
|
||||
if test "x$want_boost" = "xyes"; then
|
||||
boost_lib_version_req=ifelse([$1], ,1.20.0,$1)
|
||||
boost_lib_version_req_shorten=`expr $boost_lib_version_req : '\([[0-9]]*\.[[0-9]]*\)'`
|
||||
boost_lib_version_req_major=`expr $boost_lib_version_req : '\([[0-9]]*\)'`
|
||||
boost_lib_version_req_minor=`expr $boost_lib_version_req : '[[0-9]]*\.\([[0-9]]*\)'`
|
||||
boost_lib_version_req_sub_minor=`expr $boost_lib_version_req : '[[0-9]]*\.[[0-9]]*\.\([[0-9]]*\)'`
|
||||
if test "x$boost_lib_version_req_sub_minor" = "x" ; then
|
||||
boost_lib_version_req_sub_minor="0"
|
||||
fi
|
||||
WANT_BOOST_VERSION=`expr $boost_lib_version_req_major \* 100000 \+ $boost_lib_version_req_minor \* 100 \+ $boost_lib_version_req_sub_minor`
|
||||
AC_MSG_CHECKING(for boostlib >= $boost_lib_version_req)
|
||||
succeeded=no
|
||||
|
||||
dnl first we check the system location for boost libraries
|
||||
dnl this location ist chosen if boost libraries are installed with the --layout=system option
|
||||
dnl or if you install boost with RPM
|
||||
if test "$ac_boost_path" != ""; then
|
||||
BOOST_LDFLAGS="-L$ac_boost_path/lib"
|
||||
BOOST_CPPFLAGS="-I$ac_boost_path/include"
|
||||
else
|
||||
for ac_boost_path_tmp in /usr /usr/local /opt /opt/local ; do
|
||||
if test -d "$ac_boost_path_tmp/include/boost" && test -r "$ac_boost_path_tmp/include/boost"; then
|
||||
BOOST_LDFLAGS="-L$ac_boost_path_tmp/lib"
|
||||
BOOST_CPPFLAGS="-I$ac_boost_path_tmp/include"
|
||||
break;
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
dnl overwrite ld flags if we have required special directory with
|
||||
dnl --with-boost-libdir parameter
|
||||
if test "$ac_boost_lib_path" != ""; then
|
||||
BOOST_LDFLAGS="-L$ac_boost_lib_path"
|
||||
fi
|
||||
|
||||
CPPFLAGS_SAVED="$CPPFLAGS"
|
||||
CPPFLAGS="$CPPFLAGS $BOOST_CPPFLAGS"
|
||||
export CPPFLAGS
|
||||
|
||||
LDFLAGS_SAVED="$LDFLAGS"
|
||||
LDFLAGS="$LDFLAGS $BOOST_LDFLAGS"
|
||||
export LDFLAGS
|
||||
|
||||
AC_LANG_PUSH(C++)
|
||||
AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
|
||||
@%:@include <boost/version.hpp>
|
||||
]], [[
|
||||
#if BOOST_VERSION >= $WANT_BOOST_VERSION
|
||||
// Everything is okay
|
||||
#else
|
||||
# error Boost version is too old
|
||||
#endif
|
||||
]])],[
|
||||
AC_MSG_RESULT(yes)
|
||||
succeeded=yes
|
||||
found_system=yes
|
||||
],[
|
||||
])
|
||||
AC_LANG_POP([C++])
|
||||
|
||||
|
||||
|
||||
dnl if we found no boost with system layout we search for boost libraries
|
||||
dnl built and installed without the --layout=system option or for a staged(not installed) version
|
||||
if test "x$succeeded" != "xyes"; then
|
||||
_version=0
|
||||
if test "$ac_boost_path" != ""; then
|
||||
if test -d "$ac_boost_path" && test -r "$ac_boost_path"; then
|
||||
for i in `ls -d $ac_boost_path/include/boost-* 2>/dev/null`; do
|
||||
_version_tmp=`echo $i | sed "s#$ac_boost_path##" | sed 's/\/include\/boost-//' | sed 's/_/./'`
|
||||
V_CHECK=`expr $_version_tmp \> $_version`
|
||||
if test "$V_CHECK" = "1" ; then
|
||||
_version=$_version_tmp
|
||||
fi
|
||||
VERSION_UNDERSCORE=`echo $_version | sed 's/\./_/'`
|
||||
BOOST_CPPFLAGS="-I$ac_boost_path/include/boost-$VERSION_UNDERSCORE"
|
||||
done
|
||||
fi
|
||||
else
|
||||
for ac_boost_path in /usr /usr/local /opt /opt/local ; do
|
||||
if test -d "$ac_boost_path" && test -r "$ac_boost_path"; then
|
||||
for i in `ls -d $ac_boost_path/include/boost-* 2>/dev/null`; do
|
||||
_version_tmp=`echo $i | sed "s#$ac_boost_path##" | sed 's/\/include\/boost-//' | sed 's/_/./'`
|
||||
V_CHECK=`expr $_version_tmp \> $_version`
|
||||
if test "$V_CHECK" = "1" ; then
|
||||
_version=$_version_tmp
|
||||
best_path=$ac_boost_path
|
||||
fi
|
||||
done
|
||||
fi
|
||||
done
|
||||
|
||||
VERSION_UNDERSCORE=`echo $_version | sed 's/\./_/'`
|
||||
BOOST_CPPFLAGS="-I$best_path/include/boost-$VERSION_UNDERSCORE"
|
||||
if test "$ac_boost_lib_path" = ""
|
||||
then
|
||||
BOOST_LDFLAGS="-L$best_path/lib"
|
||||
fi
|
||||
|
||||
if test "x$BOOST_ROOT" != "x"; then
|
||||
if test -d "$BOOST_ROOT" && test -r "$BOOST_ROOT" && test -d "$BOOST_ROOT/stage/lib" && test -r "$BOOST_ROOT/stage/lib"; then
|
||||
version_dir=`expr //$BOOST_ROOT : '.*/\(.*\)'`
|
||||
stage_version=`echo $version_dir | sed 's/boost_//' | sed 's/_/./g'`
|
||||
stage_version_shorten=`expr $stage_version : '\([[0-9]]*\.[[0-9]]*\)'`
|
||||
V_CHECK=`expr $stage_version_shorten \>\= $_version`
|
||||
if test "$V_CHECK" = "1" -a "$ac_boost_lib_path" = "" ; then
|
||||
AC_MSG_NOTICE(We will use a staged boost library from $BOOST_ROOT)
|
||||
BOOST_CPPFLAGS="-I$BOOST_ROOT"
|
||||
BOOST_LDFLAGS="-L$BOOST_ROOT/stage/lib"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
CPPFLAGS="$CPPFLAGS $BOOST_CPPFLAGS"
|
||||
export CPPFLAGS
|
||||
LDFLAGS="$LDFLAGS $BOOST_LDFLAGS"
|
||||
export LDFLAGS
|
||||
|
||||
AC_LANG_PUSH(C++)
|
||||
AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
|
||||
@%:@include <boost/version.hpp>
|
||||
]], [[
|
||||
#if BOOST_VERSION >= $WANT_BOOST_VERSION
|
||||
// Everything is okay
|
||||
#else
|
||||
# error Boost version is too old
|
||||
#endif
|
||||
]])],[
|
||||
AC_MSG_RESULT(yes)
|
||||
succeeded=yes
|
||||
found_system=yes
|
||||
],[
|
||||
])
|
||||
AC_LANG_POP([C++])
|
||||
fi
|
||||
|
||||
if test "$succeeded" != "yes" ; then
|
||||
if test "$_version" = "0" ; then
|
||||
AC_MSG_ERROR([[We could not detect the boost libraries (version $boost_lib_version_req_shorten or higher). If you have a staged boost library (still not installed) please specify \$BOOST_ROOT in your environment and do not give a PATH to --with-boost option. If you are sure you have boost installed, then check your version number looking in <boost/version.hpp>. See http://randspringer.de/boost for more documentation.]])
|
||||
else
|
||||
AC_MSG_NOTICE([Your boost libraries seems to old (version $_version).])
|
||||
fi
|
||||
else
|
||||
AC_SUBST(BOOST_CPPFLAGS)
|
||||
AC_SUBST(BOOST_LDFLAGS)
|
||||
AC_DEFINE(HAVE_BOOST,,[define if the Boost library is available])
|
||||
fi
|
||||
|
||||
CPPFLAGS="$CPPFLAGS_SAVED"
|
||||
LDFLAGS="$LDFLAGS_SAVED"
|
||||
fi
|
||||
|
||||
])
|
@ -1,122 +0,0 @@
|
||||
# ===========================================================================
|
||||
#
|
||||
# SYNOPSIS
|
||||
#
|
||||
# AX_BOOST_MPI
|
||||
#
|
||||
# DESCRIPTION
|
||||
#
|
||||
# Test for MPI library from the Boost C++ libraries. The macro
|
||||
# requires a preceding call to AX_BOOST_BASE, AX_BOOST_SERIALIZATION
|
||||
# and AX_MPI. You also need to set CXX="$MPICXX" before calling the
|
||||
# macro.
|
||||
#
|
||||
# This macro calls:
|
||||
#
|
||||
# AC_SUBST(BOOST_MPI_LIB)
|
||||
#
|
||||
# And sets:
|
||||
#
|
||||
# HAVE_BOOST_MPI
|
||||
#
|
||||
# LICENSE
|
||||
#
|
||||
# Based on Boost Serialize by:
|
||||
# Copyright (c) 2008 Thomas Porschberg <thomas@randspringer.de>
|
||||
#
|
||||
# Copyright (c) 2010 Mirko Maischberger <mirko.maischberger@gmail.com>
|
||||
#
|
||||
# Copying and distribution of this file, with or without modification, are
|
||||
# permitted in any medium without royalty provided the copyright notice
|
||||
# and this notice are preserved. This file is offered as-is, without any
|
||||
# warranty.
|
||||
|
||||
#serial 1
|
||||
|
||||
AC_DEFUN([AX_BOOST_MPI],
|
||||
[
|
||||
AC_ARG_WITH([boost-mpi],
|
||||
AS_HELP_STRING([--with-boost-mpi@<:@=special-lib@:>@],
|
||||
[use the MPI library from boost - it is possible to
|
||||
specify a certain library for the linker
|
||||
e.g. --with-boost-mpi=boost_mpi-gcc-mt-d-1_33_1 ]),
|
||||
[
|
||||
if test "$withval" = "no"; then
|
||||
want_boost="no"
|
||||
elif test "$withval" = "yes"; then
|
||||
want_boost="yes"
|
||||
ax_boost_user_mpi_lib=""
|
||||
else
|
||||
want_boost="yes"
|
||||
ax_boost_user_mpi_lib="$withval"
|
||||
fi
|
||||
],
|
||||
[want_boost="yes"]
|
||||
)
|
||||
|
||||
if test "x$want_boost" = "xyes"; then
|
||||
AC_REQUIRE([AC_PROG_CC])
|
||||
CPPFLAGS_SAVED="$CPPFLAGS"
|
||||
CPPFLAGS="$CPPFLAGS $BOOST_CPPFLAGS"
|
||||
AC_MSG_WARN(BOOST_CPPFLAGS $BOOST_CPPFLAGS)
|
||||
export CPPFLAGS
|
||||
|
||||
LDFLAGS_SAVED="$LDFLAGS"
|
||||
LDFLAGS="$LDFLAGS $BOOST_LDFLAGS"
|
||||
export LDFLAGS
|
||||
|
||||
LIBS_SAVED="$LIBS"
|
||||
LIBS="$LIBS $BOOST_SERIALIZATION_LIB"
|
||||
export LIBS
|
||||
|
||||
AC_CACHE_CHECK(whether the Boost::MPI library is available,
|
||||
ax_cv_boost_mpi,
|
||||
[AC_LANG_PUSH([C++])
|
||||
AC_COMPILE_IFELSE(AC_LANG_PROGRAM([[@%:@include <boost/mpi.hpp>
|
||||
]],
|
||||
[[int argc = 0;
|
||||
char **argv = 0;
|
||||
boost::mpi::environment env(argc,argv);
|
||||
return 0;
|
||||
]]),
|
||||
ax_cv_boost_mpi=yes, ax_cv_boost_mpi=no)
|
||||
AC_LANG_POP([C++])
|
||||
])
|
||||
if test "x$ax_cv_boost_mpi" = "xyes"; then
|
||||
AC_DEFINE(HAVE_BOOST_MPI,,[define if the Boost::MPI library is available])
|
||||
BOOSTLIBDIR=`echo $BOOST_LDFLAGS | sed -e 's/@<:@^\/@:>@*//'`
|
||||
if test "x$ax_boost_user_mpi_lib" = "x"; then
|
||||
for libextension in `ls $BOOSTLIBDIR/libboost_mpi*.{so,a}* 2>/dev/null | grep -v python | sed 's,.*/,,' | sed -e 's;^lib\(boost_mpi.*\)\.so.*$;\1;' -e 's;^lib\(boost_mpi.*\)\.a*$;\1;'` ; do
|
||||
ax_lib=${libextension}
|
||||
AC_CHECK_LIB($ax_lib, exit,
|
||||
[BOOST_MPI_LIB="-l$ax_lib"; AC_SUBST(BOOST_MPI_LIB) link_mpi="yes"; break],
|
||||
[link_mpi="no"])
|
||||
done
|
||||
if test "x$link_mpi" != "xyes"; then
|
||||
for libextension in `ls $BOOSTLIBDIR/boost_mpi*.{dll,a}* 2>/dev/null | grep -v python | sed 's,.*/,,' | sed -e 's;^\(boost_mpi.*\)\.dll.*$;\1;' -e 's;^\(boost_mpi.*\)\.a*$;\1;'` ; do
|
||||
ax_lib=${libextension}
|
||||
AC_CHECK_LIB($ax_lib, exit,
|
||||
[BOOST_MPI_LIB="-l$ax_lib"; AC_SUBST(BOOST_MPI_LIB) link_mpi="yes"; break],
|
||||
[link_mpi="no"])
|
||||
done
|
||||
fi
|
||||
|
||||
else
|
||||
for ax_lib in $ax_boost_user_mpi_lib boost_mpi-$ax_boost_user_mpi_lib; do
|
||||
AC_CHECK_LIB($ax_lib, exit,
|
||||
[BOOST_MPI_LIB="-l$ax_lib"; AC_SUBST(BOOST_MPI_LIB) link_mpi="yes"; break],
|
||||
[link_mpi="no"])
|
||||
done
|
||||
|
||||
fi
|
||||
if test "x$link_mpi" != "xyes"; then
|
||||
AC_MSG_ERROR(Could not link against $ax_lib !)
|
||||
fi
|
||||
fi
|
||||
LIBS="$LIBS_SAVED"
|
||||
CPPFLAGS="$CPPFLAGS_SAVED"
|
||||
LDFLAGS="$LDFLAGS_SAVED"
|
||||
fi
|
||||
])
|
||||
|
||||
|
@ -1,106 +0,0 @@
|
||||
# ===========================================================================
|
||||
# http://autoconf-archive.cryp.to/ax_boost_program_options.html
|
||||
# ===========================================================================
|
||||
#
|
||||
# SYNOPSIS
|
||||
#
|
||||
# AX_BOOST_PROGRAM_OPTIONS
|
||||
#
|
||||
# DESCRIPTION
|
||||
#
|
||||
# Test for program options library from the Boost C++ libraries. The macro
|
||||
# requires a preceding call to AX_BOOST_BASE. Further documentation is
|
||||
# available at <http://randspringer.de/boost/index.html>.
|
||||
#
|
||||
# This macro calls:
|
||||
#
|
||||
# AC_SUBST(BOOST_PROGRAM_OPTIONS_LIB)
|
||||
#
|
||||
# And sets:
|
||||
#
|
||||
# HAVE_BOOST_PROGRAM_OPTIONS
|
||||
#
|
||||
# LAST MODIFICATION
|
||||
#
|
||||
# 2008-04-12
|
||||
#
|
||||
# COPYLEFT
|
||||
#
|
||||
# Copyright (c) 2008 Thomas Porschberg <thomas@randspringer.de>
|
||||
#
|
||||
# Copying and distribution of this file, with or without modification, are
|
||||
# permitted in any medium without royalty provided the copyright notice
|
||||
# and this notice are preserved.
|
||||
|
||||
AC_DEFUN([AX_BOOST_PROGRAM_OPTIONS],
|
||||
[
|
||||
AC_ARG_WITH([boost-program-options],
|
||||
AS_HELP_STRING([--with-boost-program-options@<:@=special-lib@:>@],
|
||||
[use the program options library from boost - it is possible to specify a certain library for the linker
|
||||
e.g. --with-boost-program-options=boost_program_options-gcc-mt-1_33_1 ]),
|
||||
[
|
||||
if test "$withval" = "no"; then
|
||||
want_boost="no"
|
||||
elif test "$withval" = "yes"; then
|
||||
want_boost="yes"
|
||||
ax_boost_user_program_options_lib=""
|
||||
else
|
||||
want_boost="yes"
|
||||
ax_boost_user_program_options_lib="$withval"
|
||||
fi
|
||||
],
|
||||
[want_boost="yes"]
|
||||
)
|
||||
|
||||
if test "x$want_boost" = "xyes"; then
|
||||
AC_REQUIRE([AC_PROG_CC])
|
||||
export want_boost
|
||||
CPPFLAGS_SAVED="$CPPFLAGS"
|
||||
CPPFLAGS="$CPPFLAGS $BOOST_CPPFLAGS"
|
||||
export CPPFLAGS
|
||||
LDFLAGS_SAVED="$LDFLAGS"
|
||||
LDFLAGS="$LDFLAGS $BOOST_LDFLAGS"
|
||||
export LDFLAGS
|
||||
AC_CACHE_CHECK([whether the Boost::Program_Options library is available],
|
||||
ax_cv_boost_program_options,
|
||||
[AC_LANG_PUSH(C++)
|
||||
AC_COMPILE_IFELSE(AC_LANG_PROGRAM([[@%:@include <boost/program_options.hpp>
|
||||
]],
|
||||
[[boost::program_options::options_description generic("Generic options");
|
||||
return 0;]]),
|
||||
ax_cv_boost_program_options=yes, ax_cv_boost_program_options=no)
|
||||
AC_LANG_POP([C++])
|
||||
])
|
||||
if test "$ax_cv_boost_program_options" = yes; then
|
||||
AC_DEFINE(HAVE_BOOST_PROGRAM_OPTIONS,,[define if the Boost::PROGRAM_OPTIONS library is available])
|
||||
BOOSTLIBDIR=`echo $BOOST_LDFLAGS | sed -e 's/@<:@^\/@:>@*//'`
|
||||
if test "x$ax_boost_user_program_options_lib" = "x"; then
|
||||
for libextension in `ls $BOOSTLIBDIR/libboost_program_options*.{so,a}* 2>/dev/null | sed 's,.*/,,' | sed -e 's;^lib\(boost_program_options.*\)\.so.*$;\1;' -e 's;^lib\(boost_program_options.*\)\.a*$;\1;'` ; do
|
||||
ax_lib=${libextension}
|
||||
AC_CHECK_LIB($ax_lib, exit,
|
||||
[BOOST_PROGRAM_OPTIONS_LIB="-l$ax_lib"; AC_SUBST(BOOST_PROGRAM_OPTIONS_LIB) link_program_options="yes"; break],
|
||||
[link_program_options="no"])
|
||||
done
|
||||
if test "x$link_program_options" != "xyes"; then
|
||||
for libextension in `ls $BOOSTLIBDIR/boost_program_options*.{dll,a}* 2>/dev/null | sed 's,.*/,,' | sed -e 's;^\(boost_program_options.*\)\.dll.*$;\1;' -e 's;^\(boost_program_options.*\)\.a*$;\1;'` ; do
|
||||
ax_lib=${libextension}
|
||||
AC_CHECK_LIB($ax_lib, exit,
|
||||
[BOOST_PROGRAM_OPTIONS_LIB="-l$ax_lib"; AC_SUBST(BOOST_PROGRAM_OPTIONS_LIB) link_program_options="yes"; break],
|
||||
[link_program_options="no"])
|
||||
done
|
||||
fi
|
||||
else
|
||||
for ax_lib in $ax_boost_user_program_options_lib boost_program_options-$ax_boost_user_program_options_lib; do
|
||||
AC_CHECK_LIB($ax_lib, main,
|
||||
[BOOST_PROGRAM_OPTIONS_LIB="-l$ax_lib"; AC_SUBST(BOOST_PROGRAM_OPTIONS_LIB) link_program_options="yes"; break],
|
||||
[link_program_options="no"])
|
||||
done
|
||||
fi
|
||||
if test "x$link_program_options" != "xyes"; then
|
||||
AC_MSG_ERROR([Could not link against [$ax_lib] !])
|
||||
fi
|
||||
fi
|
||||
CPPFLAGS="$CPPFLAGS_SAVED"
|
||||
LDFLAGS="$LDFLAGS_SAVED"
|
||||
fi
|
||||
])
|
@ -1,115 +0,0 @@
|
||||
# ===========================================================================
|
||||
# http://autoconf-archive.cryp.to/ax_boost_serialization.html
|
||||
# ===========================================================================
|
||||
#
|
||||
# SYNOPSIS
|
||||
#
|
||||
# AX_BOOST_SERIALIZATION
|
||||
#
|
||||
# DESCRIPTION
|
||||
#
|
||||
# Test for Serialization library from the Boost C++ libraries. The macro
|
||||
# requires a preceding call to AX_BOOST_BASE. Further documentation is
|
||||
# available at <http://randspringer.de/boost/index.html>.
|
||||
#
|
||||
# This macro calls:
|
||||
#
|
||||
# AC_SUBST(BOOST_SERIALIZATION_LIB)
|
||||
#
|
||||
# And sets:
|
||||
#
|
||||
# HAVE_BOOST_SERIALIZATION
|
||||
#
|
||||
# LAST MODIFICATION
|
||||
#
|
||||
# 2008-04-12
|
||||
#
|
||||
# COPYLEFT
|
||||
#
|
||||
# Copyright (c) 2008 Thomas Porschberg <thomas@randspringer.de>
|
||||
#
|
||||
# Copying and distribution of this file, with or without modification, are
|
||||
# permitted in any medium without royalty provided the copyright notice
|
||||
# and this notice are preserved.
|
||||
|
||||
AC_DEFUN([AX_BOOST_SERIALIZATION],
|
||||
[
|
||||
AC_ARG_WITH([boost-serialization],
|
||||
AS_HELP_STRING([--with-boost-serialization@<:@=special-lib@:>@],
|
||||
[use the Serialization library from boost - it is possible to specify a certain library for the linker
|
||||
e.g. --with-boost-serialization=boost_serialization-gcc-mt-d-1_33_1 ]),
|
||||
[
|
||||
if test "$withval" = "no"; then
|
||||
want_boost="no"
|
||||
elif test "$withval" = "yes"; then
|
||||
want_boost="yes"
|
||||
ax_boost_user_serialization_lib=""
|
||||
else
|
||||
want_boost="yes"
|
||||
ax_boost_user_serialization_lib="$withval"
|
||||
fi
|
||||
],
|
||||
[want_boost="yes"]
|
||||
)
|
||||
|
||||
if test "x$want_boost" = "xyes"; then
|
||||
AC_REQUIRE([AC_PROG_CC])
|
||||
CPPFLAGS_SAVED="$CPPFLAGS"
|
||||
CPPFLAGS="$CPPFLAGS $BOOST_CPPFLAGS"
|
||||
AC_MSG_WARN(BOOST_CPPFLAGS $BOOST_CPPFLAGS)
|
||||
export CPPFLAGS
|
||||
|
||||
LDFLAGS_SAVED="$LDFLAGS"
|
||||
LDFLAGS="$LDFLAGS $BOOST_LDFLAGS"
|
||||
export LDFLAGS
|
||||
|
||||
AC_CACHE_CHECK(whether the Boost::Serialization library is available,
|
||||
ax_cv_boost_serialization,
|
||||
[AC_LANG_PUSH([C++])
|
||||
AC_COMPILE_IFELSE(AC_LANG_PROGRAM([[@%:@include <fstream>
|
||||
@%:@include <boost/archive/text_oarchive.hpp>
|
||||
@%:@include <boost/archive/text_iarchive.hpp>
|
||||
]],
|
||||
[[std::ofstream ofs("filename");
|
||||
boost::archive::text_oarchive oa(ofs);
|
||||
return 0;
|
||||
]]),
|
||||
ax_cv_boost_serialization=yes, ax_cv_boost_serialization=no)
|
||||
AC_LANG_POP([C++])
|
||||
])
|
||||
if test "x$ax_cv_boost_serialization" = "xyes"; then
|
||||
AC_DEFINE(HAVE_BOOST_SERIALIZATION,,[define if the Boost::Serialization library is available])
|
||||
BOOSTLIBDIR=`echo $BOOST_LDFLAGS | sed -e 's/@<:@^\/@:>@*//'`
|
||||
if test "x$ax_boost_user_serialization_lib" = "x"; then
|
||||
for libextension in `ls $BOOSTLIBDIR/libboost_serialization*.{so,a}* 2>/dev/null | sed 's,.*/,,' | sed -e 's;^lib\(boost_serialization.*\)\.so.*$;\1;' -e 's;^lib\(boost_serialization.*\)\.a*$;\1;'` ; do
|
||||
ax_lib=${libextension}
|
||||
AC_CHECK_LIB($ax_lib, exit,
|
||||
[BOOST_SERIALIZATION_LIB="-l$ax_lib"; AC_SUBST(BOOST_SERIALIZATION_LIB) link_serialization="yes"; break],
|
||||
[link_serialization="no"])
|
||||
done
|
||||
if test "x$link_serialization" != "xyes"; then
|
||||
for libextension in `ls $BOOSTLIBDIR/boost_serialization*.{dll,a}* 2>/dev/null | sed 's,.*/,,' | sed -e 's;^\(boost_serialization.*\)\.dll.*$;\1;' -e 's;^\(boost_serialization.*\)\.a*$;\1;'` ; do
|
||||
ax_lib=${libextension}
|
||||
AC_CHECK_LIB($ax_lib, exit,
|
||||
[BOOST_SERIALIZATION_LIB="-l$ax_lib"; AC_SUBST(BOOST_SERIALIZATION_LIB) link_serialization="yes"; break],
|
||||
[link_serialization="no"])
|
||||
done
|
||||
fi
|
||||
|
||||
else
|
||||
for ax_lib in $ax_boost_user_serialization_lib boost_serialization-$ax_boost_user_serialization_lib; do
|
||||
AC_CHECK_LIB($ax_lib, main,
|
||||
[BOOST_SERIALIZATION_LIB="-l$ax_lib"; AC_SUBST(BOOST_SERIALIZATION_LIB) link_serialization="yes"; break],
|
||||
[link_serialization="no"])
|
||||
done
|
||||
|
||||
fi
|
||||
if test "x$link_serialization" != "xyes"; then
|
||||
AC_MSG_ERROR(Could not link against $ax_lib !)
|
||||
fi
|
||||
fi
|
||||
|
||||
CPPFLAGS="$CPPFLAGS_SAVED"
|
||||
LDFLAGS="$LDFLAGS_SAVED"
|
||||
fi
|
||||
])
|
@ -1,143 +0,0 @@
|
||||
# ===========================================================================
|
||||
# http://www.nongnu.org/autoconf-archive/ax_boost_thread.html
|
||||
# ===========================================================================
|
||||
#
|
||||
# SYNOPSIS
|
||||
#
|
||||
# AX_BOOST_THREAD
|
||||
#
|
||||
# DESCRIPTION
|
||||
#
|
||||
# Test for Thread library from the Boost C++ libraries. The macro requires
|
||||
# a preceding call to AX_BOOST_BASE. Further documentation is available at
|
||||
# <http://randspringer.de/boost/index.html>.
|
||||
#
|
||||
# This macro calls:
|
||||
#
|
||||
# AC_SUBST(BOOST_THREAD_LIB)
|
||||
#
|
||||
# And sets:
|
||||
#
|
||||
# HAVE_BOOST_THREAD
|
||||
#
|
||||
# LICENSE
|
||||
#
|
||||
# Copyright (c) 2009 Thomas Porschberg <thomas@randspringer.de>
|
||||
# Copyright (c) 2009 Michael Tindal
|
||||
#
|
||||
# Copying and distribution of this file, with or without modification, are
|
||||
# permitted in any medium without royalty provided the copyright notice
|
||||
# and this notice are preserved.
|
||||
|
||||
AC_DEFUN([AX_BOOST_THREAD],
|
||||
[
|
||||
AC_ARG_WITH([boost-thread],
|
||||
AS_HELP_STRING([--with-boost-thread@<:@=special-lib@:>@],
|
||||
[use the Thread library from boost - it is possible to specify a certain library for the linker
|
||||
e.g. --with-boost-thread=boost_thread-gcc-mt ]),
|
||||
[
|
||||
if test "$withval" = "no"; then
|
||||
want_boost="no"
|
||||
elif test "$withval" = "yes"; then
|
||||
want_boost="yes"
|
||||
ax_boost_user_thread_lib=""
|
||||
else
|
||||
want_boost="yes"
|
||||
ax_boost_user_thread_lib="$withval"
|
||||
fi
|
||||
],
|
||||
[want_boost="yes"]
|
||||
)
|
||||
|
||||
if test "x$want_boost" = "xyes"; then
|
||||
AC_REQUIRE([AC_PROG_CC])
|
||||
AC_REQUIRE([AC_CANONICAL_BUILD])
|
||||
CPPFLAGS_SAVED="$CPPFLAGS"
|
||||
CPPFLAGS="$CPPFLAGS $BOOST_CPPFLAGS"
|
||||
export CPPFLAGS
|
||||
|
||||
LDFLAGS_SAVED="$LDFLAGS"
|
||||
LDFLAGS="$LDFLAGS $BOOST_LDFLAGS"
|
||||
export LDFLAGS
|
||||
|
||||
AC_CACHE_CHECK(whether the Boost::Thread library is available,
|
||||
ax_cv_boost_thread,
|
||||
[AC_LANG_PUSH([C++])
|
||||
CXXFLAGS_SAVE=$CXXFLAGS
|
||||
|
||||
if test "x$build_os" = "xsolaris" ; then
|
||||
CXXFLAGS="-pthreads $CXXFLAGS"
|
||||
elif test "x$build_os" = "xming32" ; then
|
||||
CXXFLAGS="-mthreads $CXXFLAGS"
|
||||
else
|
||||
CXXFLAGS="-pthread $CXXFLAGS"
|
||||
fi
|
||||
AC_COMPILE_IFELSE(AC_LANG_PROGRAM([[@%:@include <boost/thread/thread.hpp>]],
|
||||
[[boost::thread_group thrds;
|
||||
return 0;]]),
|
||||
ax_cv_boost_thread=yes, ax_cv_boost_thread=no)
|
||||
CXXFLAGS=$CXXFLAGS_SAVE
|
||||
AC_LANG_POP([C++])
|
||||
])
|
||||
if test "x$ax_cv_boost_thread" = "xyes"; then
|
||||
if test "x$build_os" = "xsolaris" ; then
|
||||
BOOST_CPPFLAGS="-pthreads $BOOST_CPPFLAGS"
|
||||
elif test "x$build_os" = "xming32" ; then
|
||||
BOOST_CPPFLAGS="-mthreads $BOOST_CPPFLAGS"
|
||||
else
|
||||
BOOST_CPPFLAGS="-pthread $BOOST_CPPFLAGS"
|
||||
fi
|
||||
|
||||
AC_SUBST(BOOST_CPPFLAGS)
|
||||
|
||||
AC_DEFINE(HAVE_BOOST_THREAD,,[define if the Boost::Thread library is available])
|
||||
BOOSTLIBDIR=`echo $BOOST_LDFLAGS | sed -e 's/@<:@^\/@:>@*//'`
|
||||
|
||||
LDFLAGS_SAVE=$LDFLAGS
|
||||
case "x$build_os" in
|
||||
*bsd* )
|
||||
LDFLAGS="-pthread $LDFLAGS"
|
||||
break;
|
||||
;;
|
||||
esac
|
||||
if test "x$ax_boost_user_thread_lib" = "x"; then
|
||||
for libextension in `ls $BOOSTLIBDIR/libboost_thread*.so* 2>/dev/null | sed 's,.*/,,' | sed -e 's;^lib\(boost_thread.*\)\.so.*$;\1;'` `ls $BOOSTLIBDIR/libboost_thread*.a* 2>/dev/null | sed 's,.*/,,' | sed -e 's;^lib\(boost_thread.*\)\.a*$;\1;'`; do
|
||||
ax_lib=${libextension}
|
||||
AC_CHECK_LIB($ax_lib, exit,
|
||||
[BOOST_THREAD_LIB="-l$ax_lib"; AC_SUBST(BOOST_THREAD_LIB) link_thread="yes"; break],
|
||||
[link_thread="no"])
|
||||
done
|
||||
if test "x$link_thread" != "xyes"; then
|
||||
for libextension in `ls $BOOSTLIBDIR/boost_thread*.dll* 2>/dev/null | sed 's,.*/,,' | sed -e 's;^\(boost_thread.*\)\.dll.*$;\1;'` `ls $BOOSTLIBDIR/boost_thread*.a* 2>/dev/null | sed 's,.*/,,' | sed -e 's;^\(boost_thread.*\)\.a*$;\1;'` ; do
|
||||
ax_lib=${libextension}
|
||||
AC_CHECK_LIB($ax_lib, exit,
|
||||
[BOOST_THREAD_LIB="-l$ax_lib"; AC_SUBST(BOOST_THREAD_LIB) link_thread="yes"; break],
|
||||
[link_thread="no"])
|
||||
done
|
||||
fi
|
||||
|
||||
else
|
||||
for ax_lib in $ax_boost_user_thread_lib boost_thread-$ax_boost_user_thread_lib; do
|
||||
AC_CHECK_LIB($ax_lib, exit,
|
||||
[BOOST_THREAD_LIB="-l$ax_lib"; AC_SUBST(BOOST_THREAD_LIB) link_thread="yes"; break],
|
||||
[link_thread="no"])
|
||||
done
|
||||
|
||||
fi
|
||||
if test "x$link_thread" = "xno"; then
|
||||
AC_MSG_ERROR(Could not link against $ax_lib !)
|
||||
else
|
||||
case "x$build_os" in
|
||||
*bsd* )
|
||||
BOOST_LDFLAGS="-pthread $BOOST_LDFLAGS"
|
||||
break;
|
||||
;;
|
||||
esac
|
||||
|
||||
fi
|
||||
fi
|
||||
|
||||
CPPFLAGS="$CPPFLAGS_SAVED"
|
||||
LDFLAGS="$LDFLAGS_SAVED"
|
||||
fi
|
||||
])
|
1045
m4/boost.m4
Normal file
1045
m4/boost.m4
Normal file
File diff suppressed because it is too large
Load Diff
@ -10,4 +10,5 @@ mira_SOURCES = \
|
||||
AM_CPPFLAGS = -W -Wall -Wno-unused -ffor-scope -D_FILE_OFFSET_BITS=64 -D_LARGE_FILES -I$(top_srcdir)/moses/src -I$(top_srcdir)/moses-chart/src $(BOOST_CPPFLAGS)
|
||||
|
||||
|
||||
mira_LDADD = $(top_builddir)/moses/src/libmoses.la -L$(top_srcdir)/moses/src -L$(top_srcdir)/moses-chart/src -lmoses -lmoses-chart $(BOOST_LDFLAGS) $(BOOST_SERIALIZATION_LIB) $(BOOST_PROGRAM_OPTIONS_LIB) $(BOOST_MPI_LIB) -L$(top_srcdir)/OnDiskPt/src -lOnDiskPt
|
||||
mira_LDADD = $(top_builddir)/moses/src/libmoses.la -L$(top_srcdir)/moses/src -L$(top_srcdir)/moses-chart/src -lmoses -lmoses-chart -L$(top_srcdir)/OnDiskPt/src -lOnDiskPt @KENLM_LDFLAGS@ $(BOOST_LDFLAGS) $(BOOST_SERIALIZATION_LIBS) $(BOOST_PROGRAM_OPTIONS_LDFLAGS) $(BOOST_PROGRAM_OPTIONS_LIBS) $(BOOST_MPI_LIBS)
|
||||
mira_DEPENDENCIES = $(top_srcdir)/moses/src/libmoses.la
|
||||
|
@ -7,10 +7,10 @@ queryPhraseTable_SOURCES = queryPhraseTable.cpp
|
||||
|
||||
AM_CPPFLAGS = -W -Wall -ffor-scope -D_FILE_OFFSET_BITS=64 -D_LARGE_FILES -I$(top_srcdir)/moses/src $(BOOST_CPPFLAGS)
|
||||
|
||||
processPhraseTable_LDADD = $(top_builddir)/moses/src/libmoses.la $(BOOST_LDFLAGS) $(BOOST_THREAD_LIB) -L$(top_srcdir)/moses/src -L$(top_srcdir)/OnDiskPt/src -lmoses -lOnDiskPt
|
||||
processPhraseTable_LDADD = $(top_builddir)/moses/src/libmoses.la -L$(top_srcdir)/moses/src -L$(top_srcdir)/OnDiskPt/src -lmoses -lOnDiskPt @KENLM_LDFLAGS@ $(BOOST_THREAD_LDFLAGS) $(BOOST_THREAD_LIBS)
|
||||
|
||||
processLexicalTable_LDADD = $(top_builddir)/moses/src/libmoses.la $(BOOST_LDFLAGS) $(BOOST_THREAD_LIB) -L$(top_srcdir)/moses/src -L$(top_srcdir)/OnDiskPt/src -lmoses -lOnDiskPt
|
||||
processLexicalTable_LDADD = $(top_builddir)/moses/src/libmoses.la -L$(top_srcdir)/moses/src -L$(top_srcdir)/OnDiskPt/src -lmoses -lOnDiskPt @KENLM_LDFLAGS@ $(BOOST_THREAD_LDFLAGS) $(BOOST_THREAD_LIBS)
|
||||
|
||||
queryLexicalTable_LDADD = $(top_builddir)/moses/src/libmoses.la $(BOOST_LDFLAGS) $(BOOST_THREAD_LIB) -L$(top_srcdir)/moses/src -L$(top_srcdir)/OnDiskPt/src -lmoses -lOnDiskPt
|
||||
queryLexicalTable_LDADD = $(top_builddir)/moses/src/libmoses.la -L$(top_srcdir)/moses/src -L$(top_srcdir)/OnDiskPt/src -lmoses -lOnDiskPt @KENLM_LDFLAGS@ $(BOOST_THREAD_LDFLAGS) $(BOOST_THREAD_LIBS)
|
||||
|
||||
queryPhraseTable_LDADD = $(top_builddir)/moses/src/libmoses.la $(BOOST_LDFLAGS) $(BOOST_THREAD_LIB) -L$(top_srcdir)/moses/src -L$(top_srcdir)/OnDiskPt/src -lmoses -lOnDiskPt
|
||||
queryPhraseTable_LDADD = $(top_builddir)/moses/src/libmoses.la -L$(top_srcdir)/moses/src -L$(top_srcdir)/OnDiskPt/src -lmoses -lOnDiskPt @KENLM_LDFLAGS@ $(BOOST_THREAD_LDFLAGS) $(BOOST_THREAD_LIBS)
|
||||
|
@ -2,8 +2,8 @@ bin_PROGRAMS = moses_chart
|
||||
moses_chart_SOURCES = Main.cpp mbr.cpp IOWrapper.cpp TranslationAnalysis.cpp
|
||||
AM_CPPFLAGS = -W -Wall -ffor-scope -D_FILE_OFFSET_BITS=64 -D_LARGE_FILES -DUSE_HYPO_POOL -I$(top_srcdir)/moses/src $(BOOST_CPPFLAGS)
|
||||
|
||||
moses_chart_LDADD = -L$(top_srcdir)/moses/src -L$(top_srcdir)/moses-chart/src -L$(top_srcdir)/OnDiskPt/src -lmoses -lmoses-chart -lOnDiskPt $(BOOST_LDFLAGS) $(BOOST_THREAD_LIB)
|
||||
moses_chart_DEPENDENCIES = $(top_srcdir)/moses/src/libmoses.la $(top_srcdir)/moses-chart/src/libmoses-chart.a $(top_srcdir)/OnDiskPt/src/libOnDiskPt.a
|
||||
moses_chart_LDADD = -L$(top_srcdir)/moses/src -L$(top_srcdir)/moses-chart/src -L$(top_srcdir)/OnDiskPt/src -lmoses -lmoses-chart -lOnDiskPt @KENLM_LDFLAGS@ $(BOOST_THREAD_LDFLAGS) $(BOOST_THREAD_LIBS)
|
||||
moses_chart_DEPENDENCIES = $(top_srcdir)/moses/src/libmoses.la $(top_srcdir)/moses-chart/src/libmoses-chart.a $(top_srcdir)/OnDiskPt/src/libOnDiskPt.a @KENLM_DEPS@
|
||||
|
||||
|
||||
|
||||
|
@ -114,11 +114,10 @@ void ChartCell::SortHypotheses()
|
||||
{
|
||||
// sort each mini cells & fill up target lhs list
|
||||
assert(m_headWords.empty());
|
||||
m_headWords.reserve(m_hypoColl.size());
|
||||
std::map<Moses::Word, HypothesisCollection>::iterator iter;
|
||||
for (iter = m_hypoColl.begin(); iter != m_hypoColl.end(); ++iter)
|
||||
{
|
||||
m_headWords.push_back(iter->first);
|
||||
m_headWords.insert(iter->first);
|
||||
|
||||
HypothesisCollection &coll = iter->second;
|
||||
coll.SortHypotheses();
|
||||
|
@ -27,6 +27,7 @@
|
||||
#include <vector>
|
||||
#include "../../moses/src/Word.h"
|
||||
#include "../../moses/src/WordsRange.h"
|
||||
#include "../../moses/src/NonTerminal.h"
|
||||
#include "ChartHypothesis.h"
|
||||
#include "ChartHypothesisCollection.h"
|
||||
#include "QueueEntry.h"
|
||||
@ -52,7 +53,7 @@ public:
|
||||
|
||||
protected:
|
||||
std::map<Moses::Word, HypothesisCollection> m_hypoColl;
|
||||
std::vector<Moses::Word> m_headWords;
|
||||
Moses::NonTerminalSet m_headWords;
|
||||
|
||||
Moses::WordsRange m_coverage;
|
||||
|
||||
@ -74,7 +75,7 @@ public:
|
||||
const Hypothesis *GetBestHypothesis() const;
|
||||
|
||||
bool HeadwordExists(const Moses::Word &headWord) const;
|
||||
const std::vector<Moses::Word> &GetHeadwords() const
|
||||
const Moses::NonTerminalSet &GetHeadwords() const
|
||||
{ return m_headWords; }
|
||||
|
||||
void CleanupArcList();
|
||||
|
@ -55,7 +55,7 @@ public:
|
||||
return *m_hypoStackColl[coverage.GetStartPos()][coverage.GetEndPos() - coverage.GetStartPos()];
|
||||
}
|
||||
|
||||
const std::vector<Moses::Word> &GetHeadwords(const Moses::WordsRange &coverage) const
|
||||
const Moses::NonTerminalSet &GetHeadwords(const Moses::WordsRange &coverage) const
|
||||
{
|
||||
const ChartCell &cell = Get(coverage);
|
||||
return cell.GetHeadwords();
|
||||
|
@ -43,17 +43,28 @@ namespace MosesChart
|
||||
Manager::Manager(InputType const& source, const TranslationSystem* system)
|
||||
:m_source(source)
|
||||
,m_hypoStackColl(source, *this)
|
||||
,m_transOptColl(source, system, m_hypoStackColl)
|
||||
,m_transOptColl(source, system, m_hypoStackColl, m_ruleLookupManagers)
|
||||
,m_system(system)
|
||||
,m_start(clock())
|
||||
|
||||
{
|
||||
m_system->InitializeBeforeSentenceProcessing(source);
|
||||
const std::vector<PhraseDictionaryFeature*> &dictionaries = m_system->GetPhraseDictionaries();
|
||||
m_ruleLookupManagers.reserve(dictionaries.size());
|
||||
for (std::vector<PhraseDictionaryFeature*>::const_iterator p = dictionaries.begin();
|
||||
p != dictionaries.end(); ++p)
|
||||
{
|
||||
PhraseDictionaryFeature *pdf = *p;
|
||||
const PhraseDictionary *dict = pdf->GetDictionary();
|
||||
PhraseDictionary *nonConstDict = const_cast<PhraseDictionary*>(dict);
|
||||
m_ruleLookupManagers.push_back(nonConstDict->CreateRuleLookupManager(source, m_hypoStackColl));
|
||||
}
|
||||
}
|
||||
|
||||
Manager::~Manager()
|
||||
{
|
||||
m_system->CleanUpAfterSentenceProcessing();
|
||||
|
||||
RemoveAllInColl(m_ruleLookupManagers);
|
||||
|
||||
clock_t end = clock();
|
||||
float et = (end - m_start);
|
||||
|
@ -30,6 +30,7 @@
|
||||
#include "../../moses/src/TrellisPathList.h"
|
||||
#include "../../moses/src/SentenceStats.h"
|
||||
#include "../../moses/src/TranslationSystem.h"
|
||||
#include "../../moses/src/ChartRuleLookupManager.h"
|
||||
|
||||
namespace MosesChart
|
||||
{
|
||||
@ -46,6 +47,7 @@ protected:
|
||||
std::auto_ptr<Moses::SentenceStats> m_sentenceStats;
|
||||
const Moses::TranslationSystem* m_system;
|
||||
clock_t m_start; /**< starting time, used for logging */
|
||||
std::vector<Moses::ChartRuleLookupManager*> m_ruleLookupManagers;
|
||||
|
||||
public:
|
||||
Manager(Moses::InputType const& source, const Moses::TranslationSystem* system);
|
||||
|
@ -37,12 +37,14 @@ namespace MosesChart
|
||||
|
||||
TranslationOptionCollection::TranslationOptionCollection(InputType const& source
|
||||
, const Moses::TranslationSystem* system
|
||||
, const ChartCellCollection &hypoStackColl)
|
||||
, const ChartCellCollection &hypoStackColl
|
||||
, const std::vector<ChartRuleLookupManager*> &ruleLookupManagers)
|
||||
:m_source(source)
|
||||
,m_system(system)
|
||||
,m_decodeGraphList(system->GetDecodeGraphs())
|
||||
,m_hypoStackColl(hypoStackColl)
|
||||
,m_collection(source.GetSize())
|
||||
,m_ruleLookupManagers(ruleLookupManagers)
|
||||
{
|
||||
// create 2-d vector
|
||||
size_t size = source.GetSize();
|
||||
@ -76,14 +78,21 @@ void TranslationOptionCollection::CreateTranslationOptionsForRange(
|
||||
size_t startPos
|
||||
, size_t endPos)
|
||||
{
|
||||
ChartTranslationOptionList &chartRuleColl = GetTranslationOptionList(startPos, endPos);
|
||||
const WordsRange &wordsRange = chartRuleColl.GetSourceRange();
|
||||
|
||||
assert(m_decodeGraphList.size() == m_ruleLookupManagers.size());
|
||||
std::vector <DecodeGraph*>::const_iterator iterDecodeGraph;
|
||||
for (iterDecodeGraph = m_decodeGraphList.begin(); iterDecodeGraph != m_decodeGraphList.end(); ++iterDecodeGraph)
|
||||
std::vector <ChartRuleLookupManager*>::const_iterator iterRuleLookupManagers = m_ruleLookupManagers.begin();
|
||||
for (iterDecodeGraph = m_decodeGraphList.begin(); iterDecodeGraph != m_decodeGraphList.end(); ++iterDecodeGraph, ++iterRuleLookupManagers)
|
||||
{
|
||||
const DecodeGraph &decodeGraph = **iterDecodeGraph;
|
||||
assert(decodeGraph.GetSize() == 1);
|
||||
ChartRuleLookupManager &ruleLookupManager = **iterRuleLookupManagers;
|
||||
size_t maxSpan = decodeGraph.GetMaxChartSpan();
|
||||
if (maxSpan == 0 || (endPos-startPos+1) <= maxSpan)
|
||||
{
|
||||
CreateTranslationOptionsForRange(decodeGraph, startPos, endPos, true);
|
||||
ruleLookupManager.GetChartRuleCollection(wordsRange, true, chartRuleColl);
|
||||
}
|
||||
}
|
||||
|
||||
@ -104,18 +113,22 @@ void TranslationOptionCollection::ProcessUnknownWord(size_t startPos, size_t end
|
||||
}
|
||||
|
||||
ChartTranslationOptionList &fullList = GetTranslationOptionList(startPos, startPos);
|
||||
const WordsRange &wordsRange = fullList.GetSourceRange();
|
||||
|
||||
// try to translation for coverage with no trans by expanding table limit
|
||||
std::vector <DecodeGraph*>::const_iterator iterDecodeGraph;
|
||||
for (iterDecodeGraph = m_decodeGraphList.begin(); iterDecodeGraph != m_decodeGraphList.end(); ++iterDecodeGraph)
|
||||
std::vector <ChartRuleLookupManager*>::const_iterator iterRuleLookupManagers = m_ruleLookupManagers.begin();
|
||||
for (iterDecodeGraph = m_decodeGraphList.begin(); iterDecodeGraph != m_decodeGraphList.end(); ++iterDecodeGraph, ++iterRuleLookupManagers)
|
||||
{
|
||||
const DecodeGraph &decodeGraph = **iterDecodeGraph;
|
||||
ChartRuleLookupManager &ruleLookupManager = **iterRuleLookupManagers;
|
||||
size_t numTransOpt = fullList.GetSize();
|
||||
if (numTransOpt == 0)
|
||||
{
|
||||
CreateTranslationOptionsForRange(decodeGraph, startPos, startPos, false);
|
||||
ruleLookupManager.GetChartRuleCollection(wordsRange, false, fullList);
|
||||
}
|
||||
}
|
||||
assert(iterRuleLookupManagers == m_ruleLookupManagers.end());
|
||||
|
||||
bool alwaysCreateDirectTranslationOption = StaticData::Instance().IsAlwaysCreateDirectTranslationOption();
|
||||
// create unknown words for 1 word coverage where we don't have any trans options
|
||||
@ -124,33 +137,6 @@ void TranslationOptionCollection::ProcessUnknownWord(size_t startPos, size_t end
|
||||
}
|
||||
|
||||
|
||||
void TranslationOptionCollection::CreateTranslationOptionsForRange(
|
||||
const DecodeGraph &decodeGraph
|
||||
, size_t startPos
|
||||
, size_t endPos
|
||||
, bool adhereTableLimit)
|
||||
{
|
||||
assert(decodeGraph.GetSize() == 1);
|
||||
const DecodeStep &decodeStep = **decodeGraph.begin();
|
||||
|
||||
// get wordsrange that doesn't go away until after sentence processing
|
||||
const WordsRange &wordsRange = GetTranslationOptionList(startPos, endPos).GetSourceRange();
|
||||
|
||||
ChartTranslationOptionList &translationOptionList = GetTranslationOptionList(startPos, endPos);
|
||||
const PhraseDictionary* phraseDictionary =
|
||||
decodeStep.GetPhraseDictionaryFeature()->GetDictionary();
|
||||
//cerr << phraseDictionary.GetScoreProducerDescription() << endl;
|
||||
|
||||
ChartTranslationOptionList &chartRuleCollection = GetTranslationOptionList(startPos, endPos);
|
||||
|
||||
phraseDictionary->GetChartRuleCollection(chartRuleCollection
|
||||
, m_source
|
||||
, wordsRange
|
||||
, adhereTableLimit
|
||||
, m_hypoStackColl);
|
||||
//cerr << "chartRuleCollection size=" << chartRuleCollection->GetSize();
|
||||
}
|
||||
|
||||
ChartTranslationOptionList &TranslationOptionCollection::GetTranslationOptionList(size_t startPos, size_t endPos)
|
||||
{
|
||||
size_t sizeVec = m_collection[startPos].size();
|
||||
|
@ -25,6 +25,7 @@
|
||||
#include "../../moses/src/InputType.h"
|
||||
#include "../../moses/src/DecodeGraph.h"
|
||||
#include "../../moses/src/ChartTranslationOptionList.h"
|
||||
#include "../../moses/src/ChartRuleLookupManager.h"
|
||||
|
||||
namespace Moses
|
||||
{
|
||||
@ -47,17 +48,13 @@ protected:
|
||||
const Moses::TranslationSystem* m_system;
|
||||
std::vector <Moses::DecodeGraph*> m_decodeGraphList;
|
||||
const ChartCellCollection &m_hypoStackColl;
|
||||
const std::vector<Moses::ChartRuleLookupManager*> &m_ruleLookupManagers;
|
||||
|
||||
std::vector< std::vector< Moses::ChartTranslationOptionList > > m_collection; /*< contains translation options */
|
||||
std::vector<Moses::Phrase*> m_unksrcs;
|
||||
std::list<Moses::TargetPhrase*> m_cacheTargetPhrase;
|
||||
std::list<std::vector<Moses::WordConsumed*>* > m_cachedWordsConsumed;
|
||||
|
||||
virtual void CreateTranslationOptionsForRange(const Moses::DecodeGraph& decodeGraph,
|
||||
size_t startPosition
|
||||
, size_t endPosition
|
||||
, bool adhereTableLimit);
|
||||
|
||||
// for adding 1 trans opt in unknown word proc
|
||||
void Add(Moses::ChartTranslationOption *transOpt, size_t pos);
|
||||
|
||||
@ -82,9 +79,9 @@ protected:
|
||||
public:
|
||||
TranslationOptionCollection(Moses::InputType const& source
|
||||
, const Moses::TranslationSystem* system
|
||||
, const ChartCellCollection &hypoStackColl);
|
||||
, const ChartCellCollection &hypoStackColl
|
||||
, const std::vector<Moses::ChartRuleLookupManager*> &ruleLookupManagers);
|
||||
virtual ~TranslationOptionCollection();
|
||||
//virtual void CreateTranslationOptions(const std::vector <Moses::DecodeGraph*> &decodeGraphList);
|
||||
void CreateTranslationOptionsForRange(size_t startPos
|
||||
, size_t endPos);
|
||||
|
||||
|
@ -3,11 +3,11 @@ bin_PROGRAMS = moses lmbrgrid checkplf
|
||||
AM_CPPFLAGS = -W -Wall -ffor-scope -D_FILE_OFFSET_BITS=64 -D_LARGE_FILES -DUSE_HYPO_POOL -I$(top_srcdir)/moses/src $(BOOST_CPPFLAGS)
|
||||
|
||||
checkplf_SOURCES = checkplf.cpp
|
||||
checkplf_LDADD = $(top_builddir)/moses/src/libmoses.la
|
||||
checkplf_LDADD = $(top_builddir)/moses/src/libmoses.la -L$(top_srcdir)/OnDiskPt/src -lOnDiskPt @KENLM_LDFLAGS@ $(BOOST_THREAD_LDFLAGS) $(BOOST_THREAD_LIBS)
|
||||
|
||||
moses_SOURCES = Main.cpp mbr.cpp IOWrapper.cpp TranslationAnalysis.cpp LatticeMBR.cpp ThreadPool.cpp
|
||||
moses_LDADD = $(top_builddir)/moses/src/libmoses.la $(BOOST_LDFLAGS) $(BOOST_THREAD_LIB) -L$(top_srcdir)/OnDiskPt/src -lOnDiskPt
|
||||
moses_LDADD = $(top_builddir)/moses/src/libmoses.la -L$(top_srcdir)/OnDiskPt/src -lOnDiskPt @KENLM_LDFLAGS@ $(BOOST_THREAD_LDFLAGS) $(BOOST_THREAD_LIBS)
|
||||
|
||||
|
||||
lmbrgrid_SOURCES = LatticeMBRGrid.cpp LatticeMBR.cpp IOWrapper.cpp
|
||||
lmbrgrid_LDADD = $(top_builddir)/moses/src/libmoses.la $(BOOST_LDFLAGS) $(BOOST_THREAD_LIB) -L$(top_srcdir)/OnDiskPt/src -lOnDiskPt
|
||||
lmbrgrid_LDADD = $(top_builddir)/moses/src/libmoses.la -L$(top_srcdir)/OnDiskPt/src -lOnDiskPt @KENLM_LDFLAGS@ $(BOOST_THREAD_LDFLAGS) $(BOOST_THREAD_LIBS)
|
||||
|
@ -407,6 +407,10 @@
|
||||
RelativePath=".\src\LanguageModelFactory.cpp"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath=".\src\LanguageModelImplementation.cpp"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath=".\src\LanguageModelInternal.cpp"
|
||||
>
|
||||
@ -885,6 +889,10 @@
|
||||
RelativePath=".\src\LanguageModelFactory.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath=".\src\LanguageModelImplementation.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath=".\src\LanguageModelInternal.h"
|
||||
>
|
||||
|
@ -21,6 +21,10 @@
|
||||
|
||||
#include <vector>
|
||||
|
||||
#include "NonTerminal.h"
|
||||
#include "Word.h"
|
||||
#include "WordsRange.h"
|
||||
|
||||
namespace Moses
|
||||
{
|
||||
class Word;
|
||||
@ -30,8 +34,7 @@ class CellCollection
|
||||
public:
|
||||
virtual ~CellCollection()
|
||||
{}
|
||||
virtual const std::vector<Word> &GetHeadwords(const Moses::WordsRange &coverage) const = 0;
|
||||
|
||||
virtual const NonTerminalSet &GetHeadwords(const Moses::WordsRange &coverage) const = 0;
|
||||
};
|
||||
|
||||
}
|
||||
|
67
moses/src/ChartRuleLookupManager.h
Normal file
67
moses/src/ChartRuleLookupManager.h
Normal file
@ -0,0 +1,67 @@
|
||||
/***********************************************************************
|
||||
Moses - factored phrase-based language decoder
|
||||
Copyright (C) 2011 University of Edinburgh
|
||||
|
||||
This library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
This library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with this library; if not, write to the Free Software
|
||||
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
***********************************************************************/
|
||||
|
||||
#pragma once
|
||||
#ifndef moses_ChartRuleLookupManager_h
|
||||
#define moses_ChartRuleLookupManager_h
|
||||
|
||||
#include "CellCollection.h"
|
||||
#include "InputType.h"
|
||||
|
||||
namespace Moses
|
||||
{
|
||||
|
||||
class ChartTranslationOptionList;
|
||||
class WordsRange;
|
||||
|
||||
// Defines an interface for looking up rules in a rule table. Concrete
|
||||
// implementation classes should correspond to specific PhraseDictionary
|
||||
// subclasses (memory or on-disk). Since a ChartRuleLookupManager object
|
||||
// maintains sentence-specific state, exactly one should be created for
|
||||
// each sentence that is to be decoded.
|
||||
class ChartRuleLookupManager
|
||||
{
|
||||
public:
|
||||
ChartRuleLookupManager(const InputType &sentence,
|
||||
const CellCollection &cellColl)
|
||||
: m_sentence(sentence)
|
||||
, m_cellCollection(cellColl) {}
|
||||
|
||||
virtual ~ChartRuleLookupManager() {}
|
||||
|
||||
const InputType &GetSentence() const { return m_sentence; }
|
||||
const CellCollection &GetCellCollection() const { return m_cellCollection; }
|
||||
|
||||
virtual void GetChartRuleCollection(
|
||||
const WordsRange &range,
|
||||
bool adhereTableLimit,
|
||||
ChartTranslationOptionList &outColl) = 0;
|
||||
|
||||
private:
|
||||
// Non-copyable: copy constructor and assignment operator not implemented.
|
||||
ChartRuleLookupManager(const ChartRuleLookupManager &);
|
||||
ChartRuleLookupManager &operator=(const ChartRuleLookupManager &);
|
||||
|
||||
const InputType &m_sentence;
|
||||
const CellCollection &m_cellCollection;
|
||||
};
|
||||
|
||||
} // namespace Moses
|
||||
|
||||
#endif
|
233
moses/src/ChartRuleLookupManagerMemory.cpp
Normal file
233
moses/src/ChartRuleLookupManagerMemory.cpp
Normal file
@ -0,0 +1,233 @@
|
||||
/***********************************************************************
|
||||
Moses - factored phrase-based language decoder
|
||||
Copyright (C) 2011 University of Edinburgh
|
||||
|
||||
This library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
This library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with this library; if not, write to the Free Software
|
||||
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
***********************************************************************/
|
||||
|
||||
#include "ChartRuleLookupManagerMemory.h"
|
||||
|
||||
#include "PhraseDictionarySCFG.h"
|
||||
#include "InputType.h"
|
||||
#include "ChartTranslationOptionList.h"
|
||||
#include "CellCollection.h"
|
||||
#include "DotChart.h"
|
||||
#include "StaticData.h"
|
||||
#include "NonTerminal.h"
|
||||
|
||||
namespace Moses
|
||||
{
|
||||
|
||||
ChartRuleLookupManagerMemory::ChartRuleLookupManagerMemory(
|
||||
const InputType &src,
|
||||
const CellCollection &cellColl,
|
||||
const PhraseDictionarySCFG &ruleTable)
|
||||
: ChartRuleLookupManager(src, cellColl)
|
||||
, m_ruleTable(ruleTable)
|
||||
{
|
||||
assert(m_processedRuleColls.size() == 0);
|
||||
size_t sourceSize = src.GetSize();
|
||||
m_processedRuleColls.resize(sourceSize);
|
||||
|
||||
const PhraseDictionaryNodeSCFG &rootNode = m_ruleTable.GetRootNode();
|
||||
|
||||
for (size_t ind = 0; ind < m_processedRuleColls.size(); ++ind)
|
||||
{
|
||||
ProcessedRule *initProcessedRule = new ProcessedRule(rootNode);
|
||||
|
||||
ProcessedRuleColl *processedRuleColl = new ProcessedRuleColl(sourceSize - ind + 1);
|
||||
processedRuleColl->Add(0, initProcessedRule); // init rule. stores the top node in tree
|
||||
|
||||
m_processedRuleColls[ind] = processedRuleColl;
|
||||
}
|
||||
}
|
||||
|
||||
ChartRuleLookupManagerMemory::~ChartRuleLookupManagerMemory()
|
||||
{
|
||||
RemoveAllInColl(m_processedRuleColls);
|
||||
}
|
||||
|
||||
void ChartRuleLookupManagerMemory::GetChartRuleCollection(
|
||||
const WordsRange &range,
|
||||
bool adhereTableLimit,
|
||||
ChartTranslationOptionList &outColl)
|
||||
{
|
||||
size_t relEndPos = range.GetEndPos() - range.GetStartPos();
|
||||
size_t absEndPos = range.GetEndPos();
|
||||
|
||||
// MAIN LOOP. create list of nodes of target phrases
|
||||
|
||||
ProcessedRuleColl &processedRuleCol = *m_processedRuleColls[range.GetStartPos()];
|
||||
const ProcessedRuleList &runningNodes = processedRuleCol.GetRunningNodes();
|
||||
// Note that runningNodes can be expanded as the loop runs (through calls to
|
||||
// ExtendPartialRuleApplication()).
|
||||
for (size_t ind = 0; ind < runningNodes.size(); ++ind)
|
||||
{
|
||||
const ProcessedRule &prevProcessedRule = *runningNodes[ind];
|
||||
const PhraseDictionaryNodeSCFG &prevNode = prevProcessedRule.GetLastNode();
|
||||
const WordConsumed *prevWordConsumed = prevProcessedRule.GetLastWordConsumed();
|
||||
size_t startPos = (prevWordConsumed == NULL) ? range.GetStartPos() : prevWordConsumed->GetWordsRange().GetEndPos() + 1;
|
||||
|
||||
// search for terminal symbol
|
||||
if (startPos == absEndPos)
|
||||
{
|
||||
const Word &sourceWord = GetSentence().GetWord(absEndPos);
|
||||
const PhraseDictionaryNodeSCFG *node = prevNode.GetChild(sourceWord);
|
||||
if (node != NULL)
|
||||
{
|
||||
WordConsumed *newWordConsumed = new WordConsumed(absEndPos, absEndPos
|
||||
, sourceWord
|
||||
, prevWordConsumed);
|
||||
ProcessedRule *processedRule = new ProcessedRule(*node, newWordConsumed);
|
||||
processedRuleCol.Add(relEndPos+1, processedRule);
|
||||
}
|
||||
}
|
||||
|
||||
// search for non-terminals
|
||||
size_t endPos, stackInd;
|
||||
if (startPos > absEndPos)
|
||||
continue;
|
||||
else if (startPos == range.GetStartPos() && range.GetEndPos() > range.GetStartPos())
|
||||
{ // start.
|
||||
endPos = absEndPos - 1;
|
||||
stackInd = relEndPos;
|
||||
}
|
||||
else
|
||||
{
|
||||
endPos = absEndPos;
|
||||
stackInd = relEndPos + 1;
|
||||
}
|
||||
|
||||
const NonTerminalSet &sourceNonTerms =
|
||||
GetSentence().GetLabelSet(startPos, endPos);
|
||||
|
||||
const NonTerminalSet &targetNonTerms =
|
||||
GetCellCollection().GetHeadwords(WordsRange(startPos, endPos));
|
||||
|
||||
ExtendPartialRuleApplication(prevNode, prevWordConsumed, startPos,
|
||||
endPos, stackInd, sourceNonTerms,
|
||||
targetNonTerms, processedRuleCol);
|
||||
}
|
||||
|
||||
// return list of target phrases
|
||||
ProcessedRuleList &nodes = processedRuleCol.Get(relEndPos + 1);
|
||||
|
||||
size_t rulesLimit = StaticData::Instance().GetRuleLimit();
|
||||
ProcessedRuleList::const_iterator iterNode;
|
||||
for (iterNode = nodes.begin(); iterNode != nodes.end(); ++iterNode)
|
||||
{
|
||||
const ProcessedRule &processedRule = **iterNode;
|
||||
const PhraseDictionaryNodeSCFG &node = processedRule.GetLastNode();
|
||||
const WordConsumed *wordConsumed = processedRule.GetLastWordConsumed();
|
||||
assert(wordConsumed);
|
||||
|
||||
const TargetPhraseCollection *targetPhraseCollection = node.GetTargetPhraseCollection();
|
||||
|
||||
if (targetPhraseCollection != NULL)
|
||||
{
|
||||
outColl.Add(*targetPhraseCollection, *wordConsumed, adhereTableLimit, rulesLimit);
|
||||
}
|
||||
}
|
||||
outColl.CreateChartRules(rulesLimit);
|
||||
}
|
||||
|
||||
// Given a partial rule application ending at startPos-1 and given the sets of
|
||||
// source and target non-terminals covering the span [startPos, endPos],
|
||||
// determines the full or partial rule applications that can be produced through
|
||||
// extending the current rule application by a single non-terminal.
|
||||
void ChartRuleLookupManagerMemory::ExtendPartialRuleApplication(
|
||||
const PhraseDictionaryNodeSCFG & node,
|
||||
const WordConsumed *prevWordConsumed,
|
||||
size_t startPos,
|
||||
size_t endPos,
|
||||
size_t stackInd,
|
||||
const NonTerminalSet & sourceNonTerms,
|
||||
const NonTerminalSet & targetNonTerms,
|
||||
ProcessedRuleColl & processedRuleColl)
|
||||
{
|
||||
const PhraseDictionaryNodeSCFG::NonTerminalMap & nonTermMap =
|
||||
node.GetNonTerminalMap();
|
||||
|
||||
const size_t numChildren = nonTermMap.size();
|
||||
if (numChildren == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
const size_t numSourceNonTerms = sourceNonTerms.size();
|
||||
const size_t numTargetNonTerms = targetNonTerms.size();
|
||||
const size_t numCombinations = numSourceNonTerms * numTargetNonTerms;
|
||||
|
||||
// We can search by either:
|
||||
// 1. Enumerating all possible source-target NT pairs that are valid for
|
||||
// the span and then searching for matching children in the node,
|
||||
// or
|
||||
// 2. Iterating over all the NT children in the node, searching
|
||||
// for each source and target NT in the span's sets.
|
||||
// We'll do whichever minimises the number of lookups:
|
||||
if (numCombinations <= numChildren*2)
|
||||
{
|
||||
NonTerminalSet::const_iterator p = sourceNonTerms.begin();
|
||||
NonTerminalSet::const_iterator sEnd = sourceNonTerms.end();
|
||||
for (; p != sEnd; ++p)
|
||||
{
|
||||
const Word & sourceNonTerm = *p;
|
||||
NonTerminalSet::const_iterator q = targetNonTerms.begin();
|
||||
NonTerminalSet::const_iterator tEnd = targetNonTerms.end();
|
||||
for (; q != tEnd; ++q)
|
||||
{
|
||||
const Word & targetNonTerm = *q;
|
||||
const PhraseDictionaryNodeSCFG * child =
|
||||
node.GetChild(sourceNonTerm, targetNonTerm);
|
||||
if (child == NULL)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
WordConsumed * wc = new WordConsumed(startPos, endPos,
|
||||
targetNonTerm,
|
||||
prevWordConsumed);
|
||||
ProcessedRule * rule = new ProcessedRule(*child, wc);
|
||||
processedRuleColl.Add(stackInd, rule);
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
PhraseDictionaryNodeSCFG::NonTerminalMap::const_iterator p;
|
||||
PhraseDictionaryNodeSCFG::NonTerminalMap::const_iterator end =
|
||||
nonTermMap.end();
|
||||
for (p = nonTermMap.begin(); p != end; ++p)
|
||||
{
|
||||
const PhraseDictionaryNodeSCFG::NonTerminalMapKey & key = p->first;
|
||||
const Word & sourceNonTerm = key.first;
|
||||
if (sourceNonTerms.find(sourceNonTerm) == sourceNonTerms.end())
|
||||
{
|
||||
continue;
|
||||
}
|
||||
const Word & targetNonTerm = key.second;
|
||||
if (targetNonTerms.find(targetNonTerm) == targetNonTerms.end())
|
||||
{
|
||||
continue;
|
||||
}
|
||||
const PhraseDictionaryNodeSCFG & child = p->second;
|
||||
WordConsumed * wc = new WordConsumed(startPos, endPos,
|
||||
targetNonTerm,
|
||||
prevWordConsumed);
|
||||
ProcessedRule * rule = new ProcessedRule(child, wc);
|
||||
processedRuleColl.Add(stackInd, rule);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace Moses
|
71
moses/src/ChartRuleLookupManagerMemory.h
Normal file
71
moses/src/ChartRuleLookupManagerMemory.h
Normal file
@ -0,0 +1,71 @@
|
||||
/***********************************************************************
|
||||
Moses - factored phrase-based language decoder
|
||||
Copyright (C) 2011 University of Edinburgh
|
||||
|
||||
This library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
This library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with this library; if not, write to the Free Software
|
||||
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
***********************************************************************/
|
||||
|
||||
#pragma once
|
||||
#ifndef moses_ChartRuleLookupManagerMemory_h
|
||||
#define moses_ChartRuleLookupManagerMemory_h
|
||||
|
||||
#include <vector>
|
||||
|
||||
#include "ChartRuleLookupManager.h"
|
||||
#include "NonTerminal.h"
|
||||
#include "PhraseDictionaryNodeSCFG.h"
|
||||
#include "PhraseDictionarySCFG.h"
|
||||
|
||||
namespace Moses
|
||||
{
|
||||
|
||||
class ChartTranslationOptionList;
|
||||
class ProcessedRuleColl;
|
||||
class WordsRange;
|
||||
class WordConsumed;
|
||||
|
||||
// Implementation of ChartRuleLookupManager for in-memory rule tables.
|
||||
class ChartRuleLookupManagerMemory : public ChartRuleLookupManager
|
||||
{
|
||||
public:
|
||||
ChartRuleLookupManagerMemory(const InputType &sentence,
|
||||
const CellCollection &cellColl,
|
||||
const PhraseDictionarySCFG &ruleTable);
|
||||
|
||||
~ChartRuleLookupManagerMemory();
|
||||
|
||||
virtual void GetChartRuleCollection(
|
||||
const WordsRange &range,
|
||||
bool adhereTableLimit,
|
||||
ChartTranslationOptionList &outColl);
|
||||
|
||||
private:
|
||||
void ExtendPartialRuleApplication(
|
||||
const PhraseDictionaryNodeSCFG &node,
|
||||
const WordConsumed *prevWordConsumed,
|
||||
size_t startPos,
|
||||
size_t endPos,
|
||||
size_t stackInd,
|
||||
const NonTerminalSet &sourceNonTerms,
|
||||
const NonTerminalSet &targetNonTerms,
|
||||
ProcessedRuleColl &processedRuleColl);
|
||||
|
||||
std::vector<ProcessedRuleColl*> m_processedRuleColls;
|
||||
const PhraseDictionarySCFG &m_ruleTable;
|
||||
};
|
||||
|
||||
} // namespace Moses
|
||||
|
||||
#endif
|
311
moses/src/ChartRuleLookupManagerOnDisk.cpp
Normal file
311
moses/src/ChartRuleLookupManagerOnDisk.cpp
Normal file
@ -0,0 +1,311 @@
|
||||
/***********************************************************************
|
||||
Moses - factored phrase-based language decoder
|
||||
Copyright (C) 2011 University of Edinburgh
|
||||
|
||||
This library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
This library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with this library; if not, write to the Free Software
|
||||
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
***********************************************************************/
|
||||
|
||||
#include "ChartRuleLookupManagerOnDisk.h"
|
||||
|
||||
#include <algorithm>
|
||||
|
||||
#include "PhraseDictionaryOnDisk.h"
|
||||
#include "StaticData.h"
|
||||
#include "DotChartOnDisk.h"
|
||||
#include "CellCollection.h"
|
||||
#include "ChartTranslationOptionList.h"
|
||||
#include "../../OnDiskPt/src/TargetPhraseCollection.h"
|
||||
|
||||
using namespace std;
|
||||
|
||||
namespace Moses
|
||||
{
|
||||
|
||||
ChartRuleLookupManagerOnDisk::ChartRuleLookupManagerOnDisk(
|
||||
const InputType &sentence,
|
||||
const CellCollection &cellColl,
|
||||
const PhraseDictionaryOnDisk &dictionary,
|
||||
OnDiskPt::OnDiskWrapper &dbWrapper,
|
||||
const LMList *languageModels,
|
||||
const WordPenaltyProducer *wpProducer,
|
||||
const std::vector<FactorType> &inputFactorsVec,
|
||||
const std::vector<FactorType> &outputFactorsVec,
|
||||
const std::vector<float> &weight,
|
||||
const std::string &filePath)
|
||||
: ChartRuleLookupManager(sentence, cellColl)
|
||||
, m_dictionary(dictionary)
|
||||
, m_dbWrapper(dbWrapper)
|
||||
, m_languageModels(languageModels)
|
||||
, m_wpProducer(wpProducer)
|
||||
, m_inputFactorsVec(inputFactorsVec)
|
||||
, m_outputFactorsVec(outputFactorsVec)
|
||||
, m_weight(weight)
|
||||
, m_filePath(filePath)
|
||||
{
|
||||
assert(m_runningNodesVec.size() == 0);
|
||||
size_t sourceSize = sentence.GetSize();
|
||||
m_runningNodesVec.resize(sourceSize);
|
||||
|
||||
for (size_t ind = 0; ind < m_runningNodesVec.size(); ++ind)
|
||||
{
|
||||
ProcessedRuleOnDisk *initProcessedRule = new ProcessedRuleOnDisk(m_dbWrapper.GetRootSourceNode());
|
||||
|
||||
ProcessedRuleStackOnDisk *processedStack = new ProcessedRuleStackOnDisk(sourceSize - ind + 1);
|
||||
processedStack->Add(0, initProcessedRule); // init rule. stores the top node in tree
|
||||
|
||||
m_runningNodesVec[ind] = processedStack;
|
||||
}
|
||||
}
|
||||
|
||||
ChartRuleLookupManagerOnDisk::~ChartRuleLookupManagerOnDisk()
|
||||
{
|
||||
std::map<UINT64, const TargetPhraseCollection*>::const_iterator iterCache;
|
||||
for (iterCache = m_cache.begin(); iterCache != m_cache.end(); ++iterCache)
|
||||
{
|
||||
delete iterCache->second;
|
||||
}
|
||||
m_cache.clear();
|
||||
|
||||
RemoveAllInColl(m_runningNodesVec);
|
||||
RemoveAllInColl(m_sourcePhraseNode);
|
||||
}
|
||||
|
||||
void ChartRuleLookupManagerOnDisk::GetChartRuleCollection(
|
||||
const WordsRange &range,
|
||||
bool adhereTableLimit,
|
||||
ChartTranslationOptionList &outColl)
|
||||
{
|
||||
const StaticData &staticData = StaticData::Instance();
|
||||
size_t rulesLimit = StaticData::Instance().GetRuleLimit();
|
||||
|
||||
size_t relEndPos = range.GetEndPos() - range.GetStartPos();
|
||||
size_t absEndPos = range.GetEndPos();
|
||||
|
||||
// MAIN LOOP. create list of nodes of target phrases
|
||||
ProcessedRuleStackOnDisk &runningNodes = *m_runningNodesVec[range.GetStartPos()];
|
||||
|
||||
// sort save nodes so only do nodes with most counts
|
||||
runningNodes.SortSavedNodes();
|
||||
size_t numDerivations = 0
|
||||
,maxDerivations = 999999; // staticData.GetMaxDerivations();
|
||||
bool overThreshold = true;
|
||||
|
||||
const ProcessedRuleStackOnDisk::SavedNodeColl &savedNodeColl = runningNodes.GetSavedNodeColl();
|
||||
//cerr << "savedNodeColl=" << savedNodeColl.size() << " ";
|
||||
|
||||
for (size_t ind = 0; ind < (savedNodeColl.size()) && ((numDerivations < maxDerivations) || overThreshold) ; ++ind)
|
||||
{
|
||||
const SavedNodeOnDisk &savedNode = *savedNodeColl[ind];
|
||||
|
||||
const ProcessedRuleOnDisk &prevProcessedRule = savedNode.GetProcessedRule();
|
||||
const OnDiskPt::PhraseNode &prevNode = prevProcessedRule.GetLastNode();
|
||||
const WordConsumed *prevWordConsumed = prevProcessedRule.GetLastWordConsumed();
|
||||
size_t startPos = (prevWordConsumed == NULL) ? range.GetStartPos() : prevWordConsumed->GetWordsRange().GetEndPos() + 1;
|
||||
|
||||
// search for terminal symbol
|
||||
if (startPos == absEndPos)
|
||||
{
|
||||
const Word &sourceWord = GetSentence().GetWord(absEndPos);
|
||||
OnDiskPt::Word *sourceWordBerkeleyDb = m_dbWrapper.ConvertFromMoses(Input, m_inputFactorsVec, sourceWord);
|
||||
|
||||
if (sourceWordBerkeleyDb != NULL)
|
||||
{
|
||||
const OnDiskPt::PhraseNode *node = prevNode.GetChild(*sourceWordBerkeleyDb, m_dbWrapper);
|
||||
if (node != NULL)
|
||||
{
|
||||
// TODO figure out why source word is needed from node, not from sentence
|
||||
// prob to do with factors or non-term
|
||||
//const Word &sourceWord = node->GetSourceWord();
|
||||
WordConsumed *newWordConsumed = new WordConsumed(absEndPos, absEndPos
|
||||
, sourceWord
|
||||
, prevWordConsumed);
|
||||
ProcessedRuleOnDisk *processedRule = new ProcessedRuleOnDisk(*node, newWordConsumed);
|
||||
runningNodes.Add(relEndPos+1, processedRule);
|
||||
|
||||
// cache for cleanup
|
||||
m_sourcePhraseNode.push_back(node);
|
||||
}
|
||||
|
||||
delete sourceWordBerkeleyDb;
|
||||
}
|
||||
}
|
||||
|
||||
// search for non-terminals
|
||||
size_t endPos, stackInd;
|
||||
if (startPos > absEndPos)
|
||||
continue;
|
||||
else if (startPos == range.GetStartPos() && range.GetEndPos() > range.GetStartPos())
|
||||
{ // start.
|
||||
endPos = absEndPos - 1;
|
||||
stackInd = relEndPos;
|
||||
}
|
||||
else
|
||||
{
|
||||
endPos = absEndPos;
|
||||
stackInd = relEndPos + 1;
|
||||
}
|
||||
|
||||
size_t nonTermNumWordsCovered = endPos - startPos + 1;
|
||||
|
||||
// get target headwords in this span from chart
|
||||
const NonTerminalSet &headWords = GetCellCollection().GetHeadwords(WordsRange(startPos, endPos));
|
||||
|
||||
const Word &defaultSourceNonTerm = staticData.GetInputDefaultNonTerminal()
|
||||
,&defaultTargetNonTerm = staticData.GetOutputDefaultNonTerminal();
|
||||
|
||||
// go through each SOURCE lhs
|
||||
const NonTerminalSet &sourceLHSSet = GetSentence().GetLabelSet(startPos, endPos);
|
||||
|
||||
NonTerminalSet::const_iterator iterSourceLHS;
|
||||
for (iterSourceLHS = sourceLHSSet.begin(); iterSourceLHS != sourceLHSSet.end(); ++iterSourceLHS)
|
||||
{
|
||||
const Word &sourceLHS = *iterSourceLHS;
|
||||
|
||||
OnDiskPt::Word *sourceLHSBerkeleyDb = m_dbWrapper.ConvertFromMoses(Input, m_inputFactorsVec, sourceLHS);
|
||||
|
||||
if (sourceLHSBerkeleyDb == NULL)
|
||||
{
|
||||
delete sourceLHSBerkeleyDb;
|
||||
continue; // vocab not in pt. node definately won't be in there
|
||||
}
|
||||
|
||||
const OnDiskPt::PhraseNode *sourceNode = prevNode.GetChild(*sourceLHSBerkeleyDb, m_dbWrapper);
|
||||
delete sourceLHSBerkeleyDb;
|
||||
|
||||
if (sourceNode == NULL)
|
||||
continue; // didn't find source node
|
||||
|
||||
// go through each TARGET lhs
|
||||
NonTerminalSet::const_iterator iterTargetLHS;
|
||||
for (iterTargetLHS = headWords.begin(); iterTargetLHS != headWords.end(); ++iterTargetLHS)
|
||||
{
|
||||
const Word &targetLHS = *iterTargetLHS;
|
||||
|
||||
//cerr << sourceLHS << " " << defaultSourceNonTerm << " " << targetLHS << " " << defaultTargetNonTerm << endl;
|
||||
|
||||
//bool isSyntaxNonTerm = (sourceLHS != defaultSourceNonTerm) || (targetLHS != defaultTargetNonTerm);
|
||||
bool doSearch = true; //isSyntaxNonTerm ? nonTermNumWordsCovered <= maxSyntaxSpan :
|
||||
// nonTermNumWordsCovered <= maxDefaultSpan;
|
||||
|
||||
if (doSearch)
|
||||
{
|
||||
|
||||
OnDiskPt::Word *targetLHSBerkeleyDb = m_dbWrapper.ConvertFromMoses(Output, m_outputFactorsVec, targetLHS);
|
||||
|
||||
if (targetLHSBerkeleyDb == NULL)
|
||||
continue;
|
||||
|
||||
const OnDiskPt::PhraseNode *node = sourceNode->GetChild(*targetLHSBerkeleyDb, m_dbWrapper);
|
||||
delete targetLHSBerkeleyDb;
|
||||
|
||||
if (node == NULL)
|
||||
continue;
|
||||
|
||||
// found matching entry
|
||||
//const Word &sourceWord = node->GetSourceWord();
|
||||
WordConsumed *newWordConsumed = new WordConsumed(startPos, endPos
|
||||
, targetLHS
|
||||
, prevWordConsumed);
|
||||
|
||||
ProcessedRuleOnDisk *processedRule = new ProcessedRuleOnDisk(*node, newWordConsumed);
|
||||
runningNodes.Add(stackInd, processedRule);
|
||||
|
||||
m_sourcePhraseNode.push_back(node);
|
||||
}
|
||||
} // for (iterHeadWords
|
||||
|
||||
delete sourceNode;
|
||||
|
||||
} // for (iterLabelListf
|
||||
|
||||
// return list of target phrases
|
||||
ProcessedRuleCollOnDisk &nodes = runningNodes.Get(relEndPos + 1);
|
||||
|
||||
// source LHS
|
||||
ProcessedRuleCollOnDisk::const_iterator iterProcessedRuleColl;
|
||||
for (iterProcessedRuleColl = nodes.begin(); iterProcessedRuleColl != nodes.end(); ++iterProcessedRuleColl)
|
||||
{
|
||||
// node of last source word
|
||||
const ProcessedRuleOnDisk &prevProcessedRule = **iterProcessedRuleColl;
|
||||
if (prevProcessedRule.Done())
|
||||
continue;
|
||||
prevProcessedRule.Done(true);
|
||||
|
||||
const WordConsumed *wordConsumed = prevProcessedRule.GetLastWordConsumed();
|
||||
assert(wordConsumed);
|
||||
|
||||
const OnDiskPt::PhraseNode &prevNode = prevProcessedRule.GetLastNode();
|
||||
|
||||
//get node for each source LHS
|
||||
const NonTerminalSet &lhsSet = GetSentence().GetLabelSet(range.GetStartPos(), range.GetEndPos());
|
||||
NonTerminalSet::const_iterator iterLabelSet;
|
||||
for (iterLabelSet = lhsSet.begin(); iterLabelSet != lhsSet.end(); ++iterLabelSet)
|
||||
{
|
||||
const Word &sourceLHS = *iterLabelSet;
|
||||
|
||||
OnDiskPt::Word *sourceLHSBerkeleyDb = m_dbWrapper.ConvertFromMoses(Input, m_inputFactorsVec, sourceLHS);
|
||||
if (sourceLHSBerkeleyDb == NULL)
|
||||
continue;
|
||||
|
||||
const TargetPhraseCollection *targetPhraseCollection = NULL;
|
||||
const OnDiskPt::PhraseNode *node = prevNode.GetChild(*sourceLHSBerkeleyDb, m_dbWrapper);
|
||||
if (node)
|
||||
{
|
||||
UINT64 tpCollFilePos = node->GetValue();
|
||||
std::map<UINT64, const TargetPhraseCollection*>::const_iterator iterCache = m_cache.find(tpCollFilePos);
|
||||
if (iterCache == m_cache.end())
|
||||
{ // not in case
|
||||
overThreshold = node->GetCount(0) > staticData.GetRuleCountThreshold();
|
||||
//cerr << node->GetCount(0) << " ";
|
||||
|
||||
const OnDiskPt::TargetPhraseCollection *tpcollBerkeleyDb = node->GetTargetPhraseCollection(m_dictionary.GetTableLimit(), m_dbWrapper);
|
||||
|
||||
targetPhraseCollection
|
||||
= tpcollBerkeleyDb->ConvertToMoses(m_inputFactorsVec
|
||||
,m_outputFactorsVec
|
||||
,m_dictionary
|
||||
,m_weight
|
||||
,m_wpProducer
|
||||
,*m_languageModels
|
||||
,m_filePath
|
||||
, m_dbWrapper.GetVocab());
|
||||
|
||||
delete tpcollBerkeleyDb;
|
||||
m_cache[tpCollFilePos] = targetPhraseCollection;
|
||||
}
|
||||
else
|
||||
{ // jsut get out of cache
|
||||
targetPhraseCollection = iterCache->second;
|
||||
}
|
||||
|
||||
assert(targetPhraseCollection);
|
||||
outColl.Add(*targetPhraseCollection, *wordConsumed, adhereTableLimit, rulesLimit);
|
||||
|
||||
numDerivations++;
|
||||
|
||||
} // if (node)
|
||||
|
||||
delete node;
|
||||
delete sourceLHSBerkeleyDb;
|
||||
}
|
||||
}
|
||||
} // for (size_t ind = 0; ind < savedNodeColl.size(); ++ind)
|
||||
|
||||
outColl.CreateChartRules(rulesLimit);
|
||||
|
||||
//cerr << numDerivations << " ";
|
||||
}
|
||||
|
||||
} // namespace Moses
|
73
moses/src/ChartRuleLookupManagerOnDisk.h
Normal file
73
moses/src/ChartRuleLookupManagerOnDisk.h
Normal file
@ -0,0 +1,73 @@
|
||||
/***********************************************************************
|
||||
Moses - factored phrase-based language decoder
|
||||
Copyright (C) 2011 University of Edinburgh
|
||||
|
||||
This library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
This library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with this library; if not, write to the Free Software
|
||||
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
***********************************************************************/
|
||||
|
||||
#pragma once
|
||||
#ifndef moses_ChartRuleLookupManagerOnDisk_h
|
||||
#define moses_ChartRuleLookupManagerOnDisk_h
|
||||
|
||||
#include "../../OnDiskPt/src/OnDiskWrapper.h"
|
||||
|
||||
#include "CellCollection.h"
|
||||
#include "ChartRuleLookupManager.h"
|
||||
#include "ChartTranslationOptionList.h"
|
||||
#include "DotChartOnDisk.h"
|
||||
#include "InputType.h"
|
||||
#include "PhraseDictionaryOnDisk.h"
|
||||
|
||||
namespace Moses
|
||||
{
|
||||
|
||||
// Implementation of ChartRuleLookupManager for on-disk rule tables.
|
||||
class ChartRuleLookupManagerOnDisk : public ChartRuleLookupManager
|
||||
{
|
||||
public:
|
||||
ChartRuleLookupManagerOnDisk(const InputType &sentence,
|
||||
const CellCollection &cellColl,
|
||||
const PhraseDictionaryOnDisk &dictionary,
|
||||
OnDiskPt::OnDiskWrapper &dbWrapper,
|
||||
const LMList *languageModels,
|
||||
const WordPenaltyProducer *wpProducer,
|
||||
const std::vector<FactorType> &inputFactorsVec,
|
||||
const std::vector<FactorType> &outputFactorsVec,
|
||||
const std::vector<float> &weight,
|
||||
const std::string &filePath);
|
||||
|
||||
~ChartRuleLookupManagerOnDisk();
|
||||
|
||||
virtual void GetChartRuleCollection(const WordsRange &range,
|
||||
bool adhereTableLimit,
|
||||
ChartTranslationOptionList &outColl);
|
||||
|
||||
private:
|
||||
const PhraseDictionaryOnDisk &m_dictionary;
|
||||
OnDiskPt::OnDiskWrapper &m_dbWrapper;
|
||||
const LMList *m_languageModels;
|
||||
const WordPenaltyProducer *m_wpProducer;
|
||||
const std::vector<FactorType> &m_inputFactorsVec;
|
||||
const std::vector<FactorType> &m_outputFactorsVec;
|
||||
const std::vector<float> &m_weight;
|
||||
const std::string &m_filePath;
|
||||
std::vector<ProcessedRuleStackOnDisk*> m_runningNodesVec;
|
||||
std::map<UINT64, const TargetPhraseCollection*> m_cache;
|
||||
std::list<const OnDiskPt::PhraseNode*> m_sourcePhraseNode;
|
||||
};
|
||||
|
||||
} // namespace Moses
|
||||
|
||||
#endif
|
@ -55,10 +55,10 @@ class ConfusionNet : public InputType {
|
||||
|
||||
TranslationOptionCollection* CreateTranslationOptionCollection(const TranslationSystem* system) const;
|
||||
|
||||
const LabelList &GetLabelList(size_t /*startPos*/, size_t /*endPos*/) const
|
||||
const NonTerminalSet &GetLabelSet(size_t /*startPos*/, size_t /*endPos*/) const
|
||||
{
|
||||
assert(false);
|
||||
return *(new LabelList());
|
||||
return *(new NonTerminalSet());
|
||||
}
|
||||
|
||||
};
|
||||
|
@ -28,6 +28,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
#include "Phrase.h"
|
||||
#include "TargetPhraseCollection.h"
|
||||
#include "ReorderingConstraint.h"
|
||||
#include "NonTerminal.h"
|
||||
|
||||
namespace Moses
|
||||
{
|
||||
@ -38,8 +39,6 @@ class PhraseDictionary;
|
||||
class TranslationOptionCollection;
|
||||
class TranslationSystem;
|
||||
|
||||
typedef std::vector<Word> LabelList;
|
||||
|
||||
//! base class for sentences and confusion networks
|
||||
class InputType
|
||||
{
|
||||
@ -130,7 +129,7 @@ public:
|
||||
return m_reorderingConstraint;
|
||||
};
|
||||
|
||||
virtual const LabelList &GetLabelList(size_t startPos, size_t endPos) const = 0;
|
||||
virtual const NonTerminalSet &GetLabelSet(size_t startPos, size_t endPos) const = 0;
|
||||
|
||||
TO_STRING();
|
||||
|
||||
|
@ -9,6 +9,9 @@ libmoses_la_HEADERS = \
|
||||
BitmapContainer.h \
|
||||
BleuScoreFeature.h \
|
||||
CellCollection.h \
|
||||
ChartRuleLookupManager.h \
|
||||
ChartRuleLookupManagerMemory.h \
|
||||
ChartRuleLookupManagerOnDisk.h \
|
||||
ChartTranslationOption.h \
|
||||
ChartTranslationOptionList.h \
|
||||
ConfusionNet.h \
|
||||
@ -58,6 +61,7 @@ libmoses_la_HEADERS = \
|
||||
LexicalReorderingState.h \
|
||||
LexicalReorderingTable.h \
|
||||
Manager.h \
|
||||
NonTerminal.h \
|
||||
ObjectPool.h \
|
||||
OnlineCommand.h \
|
||||
PCNTools.h \
|
||||
@ -140,7 +144,9 @@ libmoses_la_SOURCES = \
|
||||
AlignmentInfo.cpp \
|
||||
BilingualDynSuffixArray.cpp \
|
||||
BitmapContainer.cpp \
|
||||
BleuScoreFeature.cpp \
|
||||
BleuScoreFeature.cpp \
|
||||
ChartRuleLookupManagerMemory.cpp \
|
||||
ChartRuleLookupManagerOnDisk.cpp \
|
||||
ChartTranslationOption.cpp \
|
||||
ChartTranslationOptionList.cpp \
|
||||
ConfusionNet.cpp \
|
||||
@ -198,11 +204,9 @@ libmoses_la_SOURCES = \
|
||||
PhraseDictionaryDynSuffixArray.cpp \
|
||||
PhraseDictionaryMemory.cpp \
|
||||
PhraseDictionarySCFG.cpp \
|
||||
PhraseDictionarySCFGChart.cpp \
|
||||
PhraseDictionaryNode.cpp \
|
||||
PhraseDictionaryNodeSCFG.cpp \
|
||||
PhraseDictionaryOnDisk.cpp \
|
||||
PhraseDictionaryOnDiskChart.cpp \
|
||||
PhraseDictionaryTree.cpp \
|
||||
PhraseDictionaryTreeAdaptor.cpp \
|
||||
PhrasePairFeature.cpp \
|
||||
@ -288,4 +292,4 @@ if KEN_LM
|
||||
libmoses_la_SOURCES += LanguageModelKen.cpp
|
||||
endif
|
||||
|
||||
libmoses_la_LIBADD = $(BOOST_LDFLAGS) $(BOOST_THREAD_LIB)
|
||||
libmoses_la_LIBADD = $(BOOST_THREAD_LDFLAGS) $(BOOST_THREAD_LIBS)
|
||||
|
68
moses/src/NonTerminal.h
Normal file
68
moses/src/NonTerminal.h
Normal file
@ -0,0 +1,68 @@
|
||||
/***********************************************************************
|
||||
Moses - factored phrase-based language decoder
|
||||
Copyright (C) 2011 University of Edinburgh
|
||||
|
||||
This library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
This library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with this library; if not, write to the Free Software
|
||||
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
***********************************************************************/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "Factor.h"
|
||||
#include "Word.h"
|
||||
|
||||
#if HAVE_CONFIG_H
|
||||
#include "config.h"
|
||||
#ifdef HAVE_BOOST
|
||||
#include <boost/functional/hash.hpp>
|
||||
#include <boost/unordered_set.hpp>
|
||||
#endif
|
||||
#endif
|
||||
|
||||
#include <set>
|
||||
|
||||
namespace Moses
|
||||
{
|
||||
#ifdef HAVE_BOOST
|
||||
class NonTerminalHasher
|
||||
{
|
||||
public:
|
||||
size_t operator()(const Word & k) const
|
||||
{
|
||||
// Assumes that only the first factor is relevant.
|
||||
const Factor * f = k[0];
|
||||
return hash_value(*f);
|
||||
}
|
||||
};
|
||||
|
||||
class NonTerminalEqualityPred
|
||||
{
|
||||
public:
|
||||
bool operator()(const Word & k1, const Word & k2) const
|
||||
{
|
||||
// Assumes that only the first factor is relevant.
|
||||
const Factor * f1 = k1[0];
|
||||
const Factor * f2 = k2[0];
|
||||
return !(f1->Compare(*f2));
|
||||
}
|
||||
};
|
||||
|
||||
typedef boost::unordered_set<Word,
|
||||
NonTerminalHasher,
|
||||
NonTerminalEqualityPred> NonTerminalSet;
|
||||
#else
|
||||
typedef std::set<Word> NonTerminalSet;
|
||||
#endif
|
||||
|
||||
} // namespace Moses
|
@ -48,8 +48,10 @@ class WordsRange;
|
||||
class ChartTranslationOptionList;
|
||||
class CellCollection;
|
||||
class TranslationSystem;
|
||||
class ChartRuleLookupManager;
|
||||
|
||||
class PhraseDictionaryFeature;
|
||||
|
||||
/**
|
||||
* Abstract base class for phrase dictionaries (tables).
|
||||
**/
|
||||
@ -71,11 +73,10 @@ class PhraseDictionary: public Dictionary {
|
||||
virtual void AddEquivPhrase(const Phrase &source, const TargetPhrase &targetPhrase)=0;
|
||||
virtual void InitializeForInput(InputType const& source) = 0;
|
||||
|
||||
virtual void GetChartRuleCollection(ChartTranslationOptionList &outColl
|
||||
, InputType const& src
|
||||
, WordsRange const& range
|
||||
, bool adhereTableLimit
|
||||
, const CellCollection &cellColl) const=0;
|
||||
//! Create a sentence-specific manager for SCFG rule lookup.
|
||||
virtual ChartRuleLookupManager *CreateRuleLookupManager(
|
||||
const InputType &,
|
||||
const CellCollection &) = 0;
|
||||
|
||||
protected:
|
||||
size_t m_tableLimit;
|
||||
|
@ -66,14 +66,6 @@ const TargetPhraseCollection *PhraseDictionaryDynSuffixArray::GetTargetPhraseCol
|
||||
return ret;
|
||||
}
|
||||
|
||||
void PhraseDictionaryDynSuffixArray::GetChartRuleCollection(ChartTranslationOptionList &outColl
|
||||
,InputType const& src
|
||||
,WordsRange const& range
|
||||
,bool adhereTableLimit
|
||||
,const CellCollection &cellColl) const
|
||||
{
|
||||
assert(false);
|
||||
}
|
||||
void PhraseDictionaryDynSuffixArray::insertSnt(string& source, string& target, string& alignment) {
|
||||
m_biSA->addSntPair(source, target, alignment);
|
||||
}
|
||||
@ -81,4 +73,9 @@ void PhraseDictionaryDynSuffixArray::deleteSnt(unsigned idx, unsigned num2Del) {
|
||||
// need to implement --
|
||||
}
|
||||
|
||||
ChartRuleLookupManager *PhraseDictionaryDynSuffixArray::CreateRuleLookupManager(const InputType&, const CellCollection&) {
|
||||
assert(false);
|
||||
return 0;
|
||||
}
|
||||
|
||||
}// end namepsace
|
||||
|
@ -28,6 +28,7 @@ public:
|
||||
void CleanUp();
|
||||
void insertSnt(string&, string&, string&);
|
||||
void deleteSnt(unsigned, unsigned);
|
||||
ChartRuleLookupManager *CreateRuleLookupManager(const InputType&, const CellCollection&);
|
||||
private:
|
||||
BilingualDynSuffixArray *m_biSA;
|
||||
std::vector<float> m_weight;
|
||||
@ -35,12 +36,6 @@ private:
|
||||
const LMList *m_languageModels;
|
||||
float m_weightWP;
|
||||
|
||||
virtual void GetChartRuleCollection(ChartTranslationOptionList &outColl
|
||||
,InputType const& src
|
||||
,WordsRange const& range
|
||||
,bool adhereTableLimit
|
||||
,const CellCollection &cellColl) const;
|
||||
|
||||
|
||||
};
|
||||
|
||||
|
@ -62,15 +62,14 @@ public:
|
||||
// for mert
|
||||
virtual void InitializeForInput(InputType const&)
|
||||
{/* Don't do anything source specific here as this object is shared between threads.*/}
|
||||
|
||||
void GetChartRuleCollection(ChartTranslationOptionList &/*outColl*/
|
||||
,InputType const& /*src*/
|
||||
, WordsRange const& /*range*/
|
||||
, bool /*adhereTableLimit*/
|
||||
,const CellCollection &/*cellColl*/) const
|
||||
{
|
||||
assert(false);
|
||||
}
|
||||
|
||||
virtual ChartRuleLookupManager *CreateRuleLookupManager(
|
||||
const InputType &,
|
||||
const CellCollection &)
|
||||
{
|
||||
assert(false);
|
||||
return 0;
|
||||
}
|
||||
|
||||
TO_STRING();
|
||||
|
||||
|
@ -139,6 +139,7 @@ public:
|
||||
*/
|
||||
class PhraseDictionaryNodeSCFG
|
||||
{
|
||||
public:
|
||||
typedef std::pair<Word, Word> NonTerminalMapKey;
|
||||
|
||||
#if defined(BOOST_VERSION) && (BOOST_VERSION >= 104200)
|
||||
@ -156,6 +157,7 @@ class PhraseDictionaryNodeSCFG
|
||||
typedef std::map<NonTerminalMapKey, PhraseDictionaryNodeSCFG> NonTerminalMap;
|
||||
#endif
|
||||
|
||||
private:
|
||||
friend std::ostream& operator<<(std::ostream&, const PhraseDictionarySCFG&);
|
||||
|
||||
// only these classes are allowed to instantiate this class
|
||||
@ -191,6 +193,9 @@ public:
|
||||
return *m_targetPhraseCollection;
|
||||
}
|
||||
|
||||
const NonTerminalMap & GetNonTerminalMap() const
|
||||
{ return m_nonTermMap; }
|
||||
|
||||
TO_STRING();
|
||||
};
|
||||
|
||||
|
@ -24,6 +24,7 @@
|
||||
#include "StaticData.h"
|
||||
#include "TargetPhraseCollection.h"
|
||||
#include "DotChartOnDisk.h"
|
||||
#include "ChartRuleLookupManagerOnDisk.h"
|
||||
|
||||
using namespace std;
|
||||
|
||||
@ -87,33 +88,12 @@ void PhraseDictionaryOnDisk::AddEquivPhrase(const Phrase &source, TargetPhrase *
|
||||
|
||||
void PhraseDictionaryOnDisk::InitializeForInput(const InputType& input)
|
||||
{
|
||||
assert(m_runningNodesVec.size() == 0);
|
||||
size_t sourceSize = input.GetSize();
|
||||
m_runningNodesVec.resize(sourceSize);
|
||||
|
||||
for (size_t ind = 0; ind < m_runningNodesVec.size(); ++ind)
|
||||
{
|
||||
ProcessedRuleOnDisk *initProcessedRule = new ProcessedRuleOnDisk(m_dbWrapper.GetRootSourceNode());
|
||||
|
||||
ProcessedRuleStackOnDisk *processedStack = new ProcessedRuleStackOnDisk(sourceSize - ind + 1);
|
||||
processedStack->Add(0, initProcessedRule); // init rule. stores the top node in tree
|
||||
|
||||
m_runningNodesVec[ind] = processedStack;
|
||||
}
|
||||
|
||||
// Nothing to do: sentence-specific state is stored in ChartRuleLookupManager
|
||||
}
|
||||
|
||||
void PhraseDictionaryOnDisk::CleanUp()
|
||||
{
|
||||
std::map<UINT64, const TargetPhraseCollection*>::const_iterator iterCache;
|
||||
for (iterCache = m_cache.begin(); iterCache != m_cache.end(); ++iterCache)
|
||||
{
|
||||
delete iterCache->second;
|
||||
}
|
||||
m_cache.clear();
|
||||
|
||||
RemoveAllInColl(m_runningNodesVec);
|
||||
RemoveAllInColl(m_sourcePhraseNode);
|
||||
// Nothing to do: sentence-specific state is stored in ChartRuleLookupManager
|
||||
}
|
||||
|
||||
void PhraseDictionaryOnDisk::LoadTargetLookup()
|
||||
@ -121,6 +101,16 @@ void PhraseDictionaryOnDisk::LoadTargetLookup()
|
||||
// TODO
|
||||
}
|
||||
|
||||
|
||||
ChartRuleLookupManager *PhraseDictionaryOnDisk::CreateRuleLookupManager(
|
||||
const InputType &sentence,
|
||||
const CellCollection &cellCollection)
|
||||
{
|
||||
return new ChartRuleLookupManagerOnDisk(sentence, cellCollection, *this,
|
||||
m_dbWrapper, m_languageModels,
|
||||
m_wpProducer, m_inputFactorsVec,
|
||||
m_outputFactorsVec, m_weight,
|
||||
m_filePath);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
@ -42,19 +42,13 @@ class PhraseDictionaryOnDisk : public PhraseDictionary
|
||||
friend std::ostream& operator<<(std::ostream&, const PhraseDictionaryOnDisk&);
|
||||
|
||||
protected:
|
||||
OnDiskPt::OnDiskWrapper m_dbWrapper;
|
||||
const LMList* m_languageModels;
|
||||
const WordPenaltyProducer* m_wpProducer;;
|
||||
const WordPenaltyProducer* m_wpProducer;
|
||||
std::vector<FactorType> m_inputFactorsVec, m_outputFactorsVec;
|
||||
std::vector<float> m_weight;
|
||||
std::string m_filePath;
|
||||
|
||||
mutable OnDiskPt::OnDiskWrapper m_dbWrapper;
|
||||
|
||||
mutable std::map<UINT64, const TargetPhraseCollection*> m_cache;
|
||||
mutable std::list<const OnDiskPt::PhraseNode*> m_sourcePhraseNode;
|
||||
|
||||
mutable std::vector<ProcessedRuleStackOnDisk*> m_runningNodesVec;
|
||||
|
||||
void LoadTargetLookup();
|
||||
|
||||
public:
|
||||
@ -86,18 +80,12 @@ public:
|
||||
//! Create entry for translation of source to targetPhrase
|
||||
virtual void AddEquivPhrase(const Phrase &source, TargetPhrase *targetPhrase);
|
||||
|
||||
virtual void GetChartRuleCollection(ChartTranslationOptionList &outColl
|
||||
,InputType const& src
|
||||
,WordsRange const& range
|
||||
,bool adhereTableLimit
|
||||
,const CellCollection &cellColl) const;
|
||||
|
||||
void InitializeForInput(const InputType& input);
|
||||
void CleanUp();
|
||||
|
||||
virtual ChartRuleLookupManager *CreateRuleLookupManager(
|
||||
const InputType &,
|
||||
const CellCollection &);
|
||||
};
|
||||
|
||||
|
||||
};
|
||||
|
||||
|
||||
} // namespace Moses
|
||||
|
@ -1,262 +0,0 @@
|
||||
// $Id$
|
||||
// vim:tabstop=2
|
||||
/***********************************************************************
|
||||
Moses - factored phrase-based language decoder
|
||||
Copyright (C) 2010 Hieu Hoang
|
||||
|
||||
This library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
This library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with this library; if not, write to the Free Software
|
||||
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
***********************************************************************/
|
||||
|
||||
#include <algorithm>
|
||||
#include "PhraseDictionaryOnDisk.h"
|
||||
#include "StaticData.h"
|
||||
#include "DotChartOnDisk.h"
|
||||
#include "CellCollection.h"
|
||||
#include "ChartTranslationOptionList.h"
|
||||
#include "../../OnDiskPt/src/TargetPhraseCollection.h"
|
||||
|
||||
using namespace std;
|
||||
|
||||
namespace Moses
|
||||
{
|
||||
void PhraseDictionaryOnDisk::GetChartRuleCollection(ChartTranslationOptionList &outColl
|
||||
,InputType const& src
|
||||
, WordsRange const& range
|
||||
,bool adhereTableLimit
|
||||
,const CellCollection &cellColl) const
|
||||
{
|
||||
const StaticData &staticData = StaticData::Instance();
|
||||
size_t rulesLimit = StaticData::Instance().GetRuleLimit();
|
||||
|
||||
size_t relEndPos = range.GetEndPos() - range.GetStartPos();
|
||||
size_t absEndPos = range.GetEndPos();
|
||||
|
||||
// MAIN LOOP. create list of nodes of target phrases
|
||||
ProcessedRuleStackOnDisk &runningNodes = *m_runningNodesVec[range.GetStartPos()];
|
||||
|
||||
// sort save nodes so only do nodes with most counts
|
||||
runningNodes.SortSavedNodes();
|
||||
size_t numDerivations = 0
|
||||
,maxDerivations = 999999; // staticData.GetMaxDerivations();
|
||||
bool overThreshold = true;
|
||||
|
||||
const ProcessedRuleStackOnDisk::SavedNodeColl &savedNodeColl = runningNodes.GetSavedNodeColl();
|
||||
//cerr << "savedNodeColl=" << savedNodeColl.size() << " ";
|
||||
|
||||
for (size_t ind = 0; ind < (savedNodeColl.size()) && ((numDerivations < maxDerivations) || overThreshold) ; ++ind)
|
||||
{
|
||||
const SavedNodeOnDisk &savedNode = *savedNodeColl[ind];
|
||||
|
||||
const ProcessedRuleOnDisk &prevProcessedRule = savedNode.GetProcessedRule();
|
||||
const OnDiskPt::PhraseNode &prevNode = prevProcessedRule.GetLastNode();
|
||||
const WordConsumed *prevWordConsumed = prevProcessedRule.GetLastWordConsumed();
|
||||
size_t startPos = (prevWordConsumed == NULL) ? range.GetStartPos() : prevWordConsumed->GetWordsRange().GetEndPos() + 1;
|
||||
|
||||
// search for terminal symbol
|
||||
if (startPos == absEndPos)
|
||||
{
|
||||
const Word &sourceWord = src.GetWord(absEndPos);
|
||||
OnDiskPt::Word *sourceWordBerkeleyDb = m_dbWrapper.ConvertFromMoses(Input, m_inputFactorsVec, sourceWord);
|
||||
|
||||
if (sourceWordBerkeleyDb != NULL)
|
||||
{
|
||||
const OnDiskPt::PhraseNode *node = prevNode.GetChild(*sourceWordBerkeleyDb, m_dbWrapper);
|
||||
if (node != NULL)
|
||||
{
|
||||
// TODO figure out why source word is needed from node, not from sentence
|
||||
// prob to do with factors or non-term
|
||||
//const Word &sourceWord = node->GetSourceWord();
|
||||
WordConsumed *newWordConsumed = new WordConsumed(absEndPos, absEndPos
|
||||
, sourceWord
|
||||
, prevWordConsumed);
|
||||
ProcessedRuleOnDisk *processedRule = new ProcessedRuleOnDisk(*node, newWordConsumed);
|
||||
runningNodes.Add(relEndPos+1, processedRule);
|
||||
|
||||
// cache for cleanup
|
||||
m_sourcePhraseNode.push_back(node);
|
||||
}
|
||||
|
||||
delete sourceWordBerkeleyDb;
|
||||
}
|
||||
}
|
||||
|
||||
// search for non-terminals
|
||||
size_t endPos, stackInd;
|
||||
if (startPos > absEndPos)
|
||||
continue;
|
||||
else if (startPos == range.GetStartPos() && range.GetEndPos() > range.GetStartPos())
|
||||
{ // start.
|
||||
endPos = absEndPos - 1;
|
||||
stackInd = relEndPos;
|
||||
}
|
||||
else
|
||||
{
|
||||
endPos = absEndPos;
|
||||
stackInd = relEndPos + 1;
|
||||
}
|
||||
|
||||
size_t nonTermNumWordsCovered = endPos - startPos + 1;
|
||||
|
||||
// get target headwords in this span from chart
|
||||
const vector<Word> &headWords = cellColl.GetHeadwords(WordsRange(startPos, endPos));
|
||||
|
||||
const Word &defaultSourceNonTerm = staticData.GetInputDefaultNonTerminal()
|
||||
,&defaultTargetNonTerm = staticData.GetOutputDefaultNonTerminal();
|
||||
|
||||
// go through each SOURCE lhs
|
||||
const LabelList &sourceLHSList = src.GetLabelList(startPos, endPos);
|
||||
|
||||
LabelList::const_iterator iterSourceLHS;
|
||||
for (iterSourceLHS = sourceLHSList.begin(); iterSourceLHS != sourceLHSList.end(); ++iterSourceLHS)
|
||||
{
|
||||
const Word &sourceLHS = *iterSourceLHS;
|
||||
|
||||
OnDiskPt::Word *sourceLHSBerkeleyDb = m_dbWrapper.ConvertFromMoses(Input, m_inputFactorsVec, sourceLHS);
|
||||
|
||||
if (sourceLHSBerkeleyDb == NULL)
|
||||
{
|
||||
delete sourceLHSBerkeleyDb;
|
||||
continue; // vocab not in pt. node definately won't be in there
|
||||
}
|
||||
|
||||
const OnDiskPt::PhraseNode *sourceNode = prevNode.GetChild(*sourceLHSBerkeleyDb, m_dbWrapper);
|
||||
delete sourceLHSBerkeleyDb;
|
||||
|
||||
if (sourceNode == NULL)
|
||||
continue; // didn't find source node
|
||||
|
||||
// go through each TARGET lhs
|
||||
vector<Word>::const_iterator iterTargetLHS;
|
||||
for (iterTargetLHS = headWords.begin(); iterTargetLHS != headWords.end(); ++iterTargetLHS)
|
||||
{
|
||||
const Word &targetLHS = *iterTargetLHS;
|
||||
|
||||
//cerr << sourceLHS << " " << defaultSourceNonTerm << " " << targetLHS << " " << defaultTargetNonTerm << endl;
|
||||
|
||||
//bool isSyntaxNonTerm = (sourceLHS != defaultSourceNonTerm) || (targetLHS != defaultTargetNonTerm);
|
||||
bool doSearch = true; //isSyntaxNonTerm ? nonTermNumWordsCovered <= maxSyntaxSpan :
|
||||
// nonTermNumWordsCovered <= maxDefaultSpan;
|
||||
|
||||
if (doSearch)
|
||||
{
|
||||
|
||||
OnDiskPt::Word *targetLHSBerkeleyDb = m_dbWrapper.ConvertFromMoses(Output, m_outputFactorsVec, targetLHS);
|
||||
|
||||
if (targetLHSBerkeleyDb == NULL)
|
||||
continue;
|
||||
|
||||
const OnDiskPt::PhraseNode *node = sourceNode->GetChild(*targetLHSBerkeleyDb, m_dbWrapper);
|
||||
delete targetLHSBerkeleyDb;
|
||||
|
||||
if (node == NULL)
|
||||
continue;
|
||||
|
||||
// found matching entry
|
||||
//const Word &sourceWord = node->GetSourceWord();
|
||||
WordConsumed *newWordConsumed = new WordConsumed(startPos, endPos
|
||||
, targetLHS
|
||||
, prevWordConsumed);
|
||||
|
||||
ProcessedRuleOnDisk *processedRule = new ProcessedRuleOnDisk(*node, newWordConsumed);
|
||||
runningNodes.Add(stackInd, processedRule);
|
||||
|
||||
m_sourcePhraseNode.push_back(node);
|
||||
}
|
||||
} // for (iterHeadWords
|
||||
|
||||
delete sourceNode;
|
||||
|
||||
} // for (iterLabelListf
|
||||
|
||||
// return list of target phrases
|
||||
ProcessedRuleCollOnDisk &nodes = runningNodes.Get(relEndPos + 1);
|
||||
|
||||
// source LHS
|
||||
ProcessedRuleCollOnDisk::const_iterator iterProcessedRuleColl;
|
||||
for (iterProcessedRuleColl = nodes.begin(); iterProcessedRuleColl != nodes.end(); ++iterProcessedRuleColl)
|
||||
{
|
||||
// node of last source word
|
||||
const ProcessedRuleOnDisk &prevProcessedRule = **iterProcessedRuleColl;
|
||||
if (prevProcessedRule.Done())
|
||||
continue;
|
||||
prevProcessedRule.Done(true);
|
||||
|
||||
const WordConsumed *wordConsumed = prevProcessedRule.GetLastWordConsumed();
|
||||
assert(wordConsumed);
|
||||
|
||||
const OnDiskPt::PhraseNode &prevNode = prevProcessedRule.GetLastNode();
|
||||
|
||||
//get node for each source LHS
|
||||
const LabelList &lhsList = src.GetLabelList(range.GetStartPos(), range.GetEndPos());
|
||||
LabelList::const_iterator iterLabelList;
|
||||
for (iterLabelList = lhsList.begin(); iterLabelList != lhsList.end(); ++iterLabelList)
|
||||
{
|
||||
const Word &sourceLHS = *iterLabelList;
|
||||
|
||||
OnDiskPt::Word *sourceLHSBerkeleyDb = m_dbWrapper.ConvertFromMoses(Input, m_inputFactorsVec, sourceLHS);
|
||||
if (sourceLHSBerkeleyDb == NULL)
|
||||
continue;
|
||||
|
||||
const TargetPhraseCollection *targetPhraseCollection = NULL;
|
||||
const OnDiskPt::PhraseNode *node = prevNode.GetChild(*sourceLHSBerkeleyDb, m_dbWrapper);
|
||||
if (node)
|
||||
{
|
||||
UINT64 tpCollFilePos = node->GetValue();
|
||||
std::map<UINT64, const TargetPhraseCollection*>::const_iterator iterCache = m_cache.find(tpCollFilePos);
|
||||
if (iterCache == m_cache.end())
|
||||
{ // not in case
|
||||
overThreshold = node->GetCount(0) > staticData.GetRuleCountThreshold();
|
||||
//cerr << node->GetCount(0) << " ";
|
||||
|
||||
const OnDiskPt::TargetPhraseCollection *tpcollBerkeleyDb = node->GetTargetPhraseCollection(GetTableLimit(), m_dbWrapper);
|
||||
|
||||
targetPhraseCollection
|
||||
= tpcollBerkeleyDb->ConvertToMoses(m_inputFactorsVec
|
||||
,m_outputFactorsVec
|
||||
,*this
|
||||
,m_weight
|
||||
,m_wpProducer
|
||||
,*m_languageModels
|
||||
,m_filePath
|
||||
, m_dbWrapper.GetVocab());
|
||||
|
||||
delete tpcollBerkeleyDb;
|
||||
m_cache[tpCollFilePos] = targetPhraseCollection;
|
||||
}
|
||||
else
|
||||
{ // jsut get out of cache
|
||||
targetPhraseCollection = iterCache->second;
|
||||
}
|
||||
|
||||
assert(targetPhraseCollection);
|
||||
outColl.Add(*targetPhraseCollection, *wordConsumed, adhereTableLimit, rulesLimit);
|
||||
|
||||
numDerivations++;
|
||||
|
||||
} // if (node)
|
||||
|
||||
delete node;
|
||||
delete sourceLHSBerkeleyDb;
|
||||
}
|
||||
}
|
||||
} // for (size_t ind = 0; ind < savedNodeColl.size(); ++ind)
|
||||
|
||||
outColl.CreateChartRules(rulesLimit);
|
||||
|
||||
//cerr << numDerivations << " ";
|
||||
}
|
||||
|
||||
}; // namespace
|
||||
|
@ -36,6 +36,7 @@
|
||||
#include "ChartTranslationOptionList.h"
|
||||
#include "DotChart.h"
|
||||
#include "FactorCollection.h"
|
||||
#include "ChartRuleLookupManagerMemory.h"
|
||||
|
||||
using namespace std;
|
||||
|
||||
@ -264,19 +265,7 @@ const TargetPhraseCollection *PhraseDictionarySCFG::GetTargetPhraseCollection(co
|
||||
|
||||
void PhraseDictionarySCFG::InitializeForInput(const InputType& input)
|
||||
{
|
||||
assert(m_processedRuleColls.size() == 0);
|
||||
size_t sourceSize = input.GetSize();
|
||||
m_processedRuleColls.resize(sourceSize);
|
||||
|
||||
for (size_t ind = 0; ind < m_processedRuleColls.size(); ++ind)
|
||||
{
|
||||
ProcessedRule *initProcessedRule = new ProcessedRule(m_collection);
|
||||
|
||||
ProcessedRuleColl *processedRuleColl = new ProcessedRuleColl(sourceSize - ind + 1);
|
||||
processedRuleColl->Add(0, initProcessedRule); // init rule. stores the top node in tree
|
||||
|
||||
m_processedRuleColls[ind] = processedRuleColl;
|
||||
}
|
||||
// Nothing to do: sentence-specific state is stored in ChartRuleLookupManager
|
||||
}
|
||||
|
||||
PhraseDictionarySCFG::~PhraseDictionarySCFG()
|
||||
@ -286,7 +275,14 @@ PhraseDictionarySCFG::~PhraseDictionarySCFG()
|
||||
|
||||
void PhraseDictionarySCFG::CleanUp()
|
||||
{
|
||||
RemoveAllInColl(m_processedRuleColls);
|
||||
// Nothing to do: sentence-specific state is stored in ChartRuleLookupManager
|
||||
}
|
||||
|
||||
ChartRuleLookupManager *PhraseDictionarySCFG::CreateRuleLookupManager(
|
||||
const InputType &sentence,
|
||||
const CellCollection &cellCollection)
|
||||
{
|
||||
return new ChartRuleLookupManagerMemory(sentence, cellCollection, *this);
|
||||
}
|
||||
|
||||
TO_STRING_BODY(PhraseDictionarySCFG);
|
||||
|
@ -25,6 +25,9 @@
|
||||
#include "PhraseDictionary.h"
|
||||
#include "PhraseDictionaryNodeSCFG.h"
|
||||
#include "CellCollection.h"
|
||||
#include "InputType.h"
|
||||
#include "WordConsumed.h"
|
||||
#include "NonTerminal.h"
|
||||
|
||||
namespace Moses
|
||||
{
|
||||
@ -42,8 +45,6 @@ namespace Moses
|
||||
|
||||
protected:
|
||||
PhraseDictionaryNodeSCFG m_collection;
|
||||
mutable std::vector<ProcessedRuleColl*> m_processedRuleColls;
|
||||
|
||||
std::string m_filePath;
|
||||
|
||||
TargetPhraseCollection &GetOrCreateTargetPhraseCollection(const Phrase &source, const TargetPhrase &target);
|
||||
@ -81,6 +82,10 @@ namespace Moses
|
||||
, size_t tableLimit
|
||||
, const LMList &languageModels
|
||||
, const WordPenaltyProducer* wpProducer);
|
||||
|
||||
const PhraseDictionaryNodeSCFG &GetRootNode() const
|
||||
{ return m_collection; }
|
||||
|
||||
|
||||
const TargetPhraseCollection *GetTargetPhraseCollection(const Phrase &source) const;
|
||||
|
||||
@ -95,13 +100,11 @@ namespace Moses
|
||||
|
||||
void InitializeForInput(const InputType& i);
|
||||
|
||||
virtual void GetChartRuleCollection(ChartTranslationOptionList &outColl
|
||||
,InputType const& src
|
||||
,WordsRange const& range
|
||||
,bool adhereTableLimit
|
||||
,const CellCollection &cellColl) const;
|
||||
|
||||
void CleanUp();
|
||||
|
||||
ChartRuleLookupManager *CreateRuleLookupManager(
|
||||
const InputType &,
|
||||
const CellCollection &);
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
} // namespace Moses
|
||||
|
@ -1,136 +0,0 @@
|
||||
// $Id$
|
||||
// vim:tabstop=2
|
||||
/***********************************************************************
|
||||
Moses - factored phrase-based language decoder
|
||||
Copyright (C) 2010 Hieu Hoang
|
||||
|
||||
This library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
This library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with this library; if not, write to the Free Software
|
||||
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
***********************************************************************/
|
||||
|
||||
#include "PhraseDictionarySCFG.h"
|
||||
#include "FactorCollection.h"
|
||||
#include "InputType.h"
|
||||
#include "ChartTranslationOptionList.h"
|
||||
#include "CellCollection.h"
|
||||
#include "DotChart.h"
|
||||
#include "StaticData.h"
|
||||
#include "TreeInput.h"
|
||||
|
||||
using namespace std;
|
||||
using namespace Moses;
|
||||
|
||||
void PhraseDictionarySCFG::GetChartRuleCollection(ChartTranslationOptionList &outColl
|
||||
,InputType const& src
|
||||
,WordsRange const& range
|
||||
,bool adhereTableLimit
|
||||
,const CellCollection &cellColl) const
|
||||
{
|
||||
size_t relEndPos = range.GetEndPos() - range.GetStartPos();
|
||||
size_t absEndPos = range.GetEndPos();
|
||||
|
||||
// MAIN LOOP. create list of nodes of target phrases
|
||||
|
||||
ProcessedRuleColl &processedRuleCol = *m_processedRuleColls[range.GetStartPos()];
|
||||
const ProcessedRuleList &runningNodes = processedRuleCol.GetRunningNodes();
|
||||
// Note that runningNodes can be expanded as the loop runs (through calls to processedRuleCol::Add())
|
||||
for (size_t ind = 0; ind < runningNodes.size(); ++ind)
|
||||
{
|
||||
const ProcessedRule &prevProcessedRule = *runningNodes[ind];
|
||||
const PhraseDictionaryNodeSCFG &prevNode = prevProcessedRule.GetLastNode();
|
||||
const WordConsumed *prevWordConsumed = prevProcessedRule.GetLastWordConsumed();
|
||||
size_t startPos = (prevWordConsumed == NULL) ? range.GetStartPos() : prevWordConsumed->GetWordsRange().GetEndPos() + 1;
|
||||
|
||||
// search for terminal symbol
|
||||
if (startPos == absEndPos)
|
||||
{
|
||||
const Word &sourceWord = src.GetWord(absEndPos);
|
||||
const PhraseDictionaryNodeSCFG *node = prevNode.GetChild(sourceWord);
|
||||
if (node != NULL)
|
||||
{
|
||||
WordConsumed *newWordConsumed = new WordConsumed(absEndPos, absEndPos
|
||||
, sourceWord
|
||||
, prevWordConsumed);
|
||||
ProcessedRule *processedRule = new ProcessedRule(*node, newWordConsumed);
|
||||
processedRuleCol.Add(relEndPos+1, processedRule);
|
||||
}
|
||||
}
|
||||
|
||||
// search for non-terminals
|
||||
size_t endPos, stackInd;
|
||||
if (startPos > absEndPos)
|
||||
continue;
|
||||
else if (startPos == range.GetStartPos() && range.GetEndPos() > range.GetStartPos())
|
||||
{ // start.
|
||||
endPos = absEndPos - 1;
|
||||
stackInd = relEndPos;
|
||||
}
|
||||
else
|
||||
{
|
||||
endPos = absEndPos;
|
||||
stackInd = relEndPos + 1;
|
||||
}
|
||||
|
||||
// get headwords in this span from chart
|
||||
const vector<Word> &headWords = cellColl.GetHeadwords(WordsRange(startPos, endPos));
|
||||
|
||||
// go thru each source span
|
||||
const LabelList &labelList = src.GetLabelList(startPos, endPos);
|
||||
|
||||
LabelList::const_iterator iterLabelList;
|
||||
for (iterLabelList = labelList.begin(); iterLabelList != labelList.end(); ++iterLabelList)
|
||||
{
|
||||
const Word &sourceLabel = *iterLabelList;
|
||||
|
||||
// go thru each headword & see if in phrase table
|
||||
vector<Word>::const_iterator iterHeadWords;
|
||||
for (iterHeadWords = headWords.begin(); iterHeadWords != headWords.end(); ++iterHeadWords)
|
||||
{
|
||||
const Word &headWord = *iterHeadWords;
|
||||
|
||||
const PhraseDictionaryNodeSCFG *node = prevNode.GetChild(sourceLabel, headWord);
|
||||
if (node != NULL)
|
||||
{
|
||||
WordConsumed *newWordConsumed = new WordConsumed(startPos, endPos
|
||||
, headWord
|
||||
, prevWordConsumed);
|
||||
|
||||
ProcessedRule *processedRule = new ProcessedRule(*node, newWordConsumed);
|
||||
processedRuleCol.Add(stackInd, processedRule);
|
||||
}
|
||||
} // for (iterHeadWords
|
||||
} // for (iterLabelList
|
||||
}
|
||||
|
||||
// return list of target phrases
|
||||
ProcessedRuleList &nodes = processedRuleCol.Get(relEndPos + 1);
|
||||
|
||||
size_t rulesLimit = StaticData::Instance().GetRuleLimit();
|
||||
ProcessedRuleList::const_iterator iterNode;
|
||||
for (iterNode = nodes.begin(); iterNode != nodes.end(); ++iterNode)
|
||||
{
|
||||
const ProcessedRule &processedRule = **iterNode;
|
||||
const PhraseDictionaryNodeSCFG &node = processedRule.GetLastNode();
|
||||
const WordConsumed *wordConsumed = processedRule.GetLastWordConsumed();
|
||||
assert(wordConsumed);
|
||||
|
||||
const TargetPhraseCollection *targetPhraseCollection = node.GetTargetPhraseCollection();
|
||||
|
||||
if (targetPhraseCollection != NULL)
|
||||
{
|
||||
outColl.Add(*targetPhraseCollection, *wordConsumed, adhereTableLimit, rulesLimit);
|
||||
}
|
||||
}
|
||||
outColl.CreateChartRules(rulesLimit);
|
||||
}
|
@ -67,15 +67,14 @@ class PhraseDictionaryTreeAdaptor : public PhraseDictionary {
|
||||
|
||||
size_t GetNumInputScores() const;
|
||||
virtual void InitializeForInput(InputType const& source);
|
||||
|
||||
void GetChartRuleCollection(ChartTranslationOptionList &outColl
|
||||
,InputType const& /*src*/
|
||||
,WordsRange const& /*range*/
|
||||
,bool /*adhereTableLimit*/
|
||||
,const CellCollection &/*cellColl*/) const
|
||||
{
|
||||
assert(false);
|
||||
}
|
||||
|
||||
virtual ChartRuleLookupManager *CreateRuleLookupManager(
|
||||
const InputType &,
|
||||
const CellCollection &)
|
||||
{
|
||||
assert(false);
|
||||
return 0;
|
||||
}
|
||||
};
|
||||
|
||||
}
|
||||
|
@ -38,7 +38,11 @@ Sentence::Sentence(FactorDirection direction)
|
||||
, InputType()
|
||||
{
|
||||
assert(direction == Input);
|
||||
m_defaultLabelList.push_back(StaticData::Instance().GetInputDefaultNonTerminal());
|
||||
const StaticData& staticData = StaticData::Instance();
|
||||
if (staticData.GetSearchAlgorithm() == ChartDecoding)
|
||||
{
|
||||
m_defaultLabelSet.insert(StaticData::Instance().GetInputDefaultNonTerminal());
|
||||
}
|
||||
}
|
||||
|
||||
int Sentence::Read(std::istream& in,const std::vector<FactorType>& factorOrder)
|
||||
|
@ -54,7 +54,7 @@ class Sentence : public Phrase, public InputType
|
||||
std::vector <TranslationOption*> m_xmlOptionsList;
|
||||
std::vector <bool> m_xmlCoverageMap;
|
||||
|
||||
LabelList m_defaultLabelList;
|
||||
NonTerminalSet m_defaultLabelSet;
|
||||
|
||||
void InitStartEndWord();
|
||||
|
||||
@ -98,8 +98,8 @@ class Sentence : public Phrase, public InputType
|
||||
, const std::string &phraseString
|
||||
, const std::string &factorDelimiter);
|
||||
|
||||
const LabelList &GetLabelList(size_t /*startPos*/, size_t /*endPos*/) const
|
||||
{ return m_defaultLabelList; }
|
||||
const NonTerminalSet &GetLabelSet(size_t /*startPos*/, size_t /*endPos*/) const
|
||||
{ return m_defaultLabelSet; }
|
||||
|
||||
};
|
||||
|
||||
|
@ -1375,6 +1375,7 @@ void StaticData::ReduceTransOptCache() const
|
||||
|
||||
void StaticData::AddTransOptListToCache(const DecodeGraph &decodeGraph, const Phrase &sourcePhrase, const TranslationOptionList &transOptList) const
|
||||
{
|
||||
if (m_transOptCacheMaxSize == 0) return;
|
||||
std::pair<size_t, Phrase> key(decodeGraph.GetPosition(), sourcePhrase);
|
||||
TranslationOptionList* storedTransOptList = new TranslationOptionList(transOptList);
|
||||
#ifdef WITH_THREADS
|
||||
|
@ -269,21 +269,20 @@ void TreeInput::AddChartLabel(size_t startPos, size_t endPos, const Word &label
|
||||
assert(label.IsNonTerminal());
|
||||
|
||||
SourceLabelOverlap overlapType = StaticData::Instance().GetSourceLabelOverlap();
|
||||
LabelList &list = GetLabelList(startPos, endPos);
|
||||
NonTerminalSet &list = GetLabelSet(startPos, endPos);
|
||||
switch (overlapType)
|
||||
{
|
||||
case SourceLabelOverlapAdd:
|
||||
list.push_back(label);
|
||||
list.insert(label);
|
||||
break;
|
||||
case SourceLabelOverlapReplace:
|
||||
if (list.size() > 0) // replace existing label
|
||||
list[0] = label;
|
||||
else
|
||||
list.push_back(label);
|
||||
list.clear();
|
||||
list.insert(label);
|
||||
break;
|
||||
case SourceLabelOverlapDiscard:
|
||||
if (list.size() == 0)
|
||||
list.push_back(label);
|
||||
list.insert(label);
|
||||
break;
|
||||
}
|
||||
}
|
||||
@ -307,9 +306,9 @@ std::ostream& operator<<(std::ostream &out, const TreeInput &input)
|
||||
{
|
||||
for (size_t endPos = startPos; endPos < size; ++endPos)
|
||||
{
|
||||
const LabelList &labelList = input.GetLabelList(startPos, endPos);
|
||||
LabelList::const_iterator iter;
|
||||
for (iter = labelList.begin(); iter != labelList.end(); ++iter)
|
||||
const NonTerminalSet &labelSet = input.GetLabelSet(startPos, endPos);
|
||||
NonTerminalSet::const_iterator iter;
|
||||
for (iter = labelSet.begin(); iter != labelSet.end(); ++iter)
|
||||
{
|
||||
const Word &word = *iter;
|
||||
out << "[" << startPos <<"," << endPos << "]="
|
||||
|
@ -24,13 +24,13 @@ class TreeInput : public Sentence
|
||||
friend std::ostream& operator<<(std::ostream&, const TreeInput&);
|
||||
|
||||
protected:
|
||||
std::vector<std::vector<LabelList> > m_sourceChart;
|
||||
std::vector<std::vector<NonTerminalSet> > m_sourceChart;
|
||||
|
||||
void AddChartLabel(size_t startPos, size_t endPos, const std::string &label
|
||||
,const std::vector<FactorType>& factorOrder);
|
||||
void AddChartLabel(size_t startPos, size_t endPos, const Word &label
|
||||
,const std::vector<FactorType>& factorOrder);
|
||||
LabelList &GetLabelList(size_t startPos, size_t endPos)
|
||||
NonTerminalSet &GetLabelSet(size_t startPos, size_t endPos)
|
||||
{
|
||||
return m_sourceChart[startPos][endPos - startPos];
|
||||
}
|
||||
@ -54,7 +54,7 @@ public:
|
||||
//! create trans options specific to this InputType
|
||||
virtual TranslationOptionCollection* CreateTranslationOptionCollection() const;
|
||||
|
||||
virtual const LabelList &GetLabelList(size_t startPos, size_t endPos) const
|
||||
virtual const NonTerminalSet &GetLabelSet(size_t startPos, size_t endPos) const
|
||||
{ return m_sourceChart[startPos][endPos - startPos]; }
|
||||
|
||||
};
|
||||
|
@ -30,10 +30,10 @@ public:
|
||||
*/
|
||||
void GetAsEdgeMatrix(std::vector<std::vector<bool> >& edges) const;
|
||||
|
||||
const LabelList &GetLabelList(size_t /*startPos*/, size_t /*endPos*/) const
|
||||
const NonTerminalSet &GetLabelSet(size_t /*startPos*/, size_t /*endPos*/) const
|
||||
{
|
||||
assert(false);
|
||||
return *(new LabelList());
|
||||
return *(new NonTerminalSet());
|
||||
}
|
||||
|
||||
};
|
||||
|
@ -8,8 +8,8 @@ DS?=$(shell date '+%Y%m%d')
|
||||
# Set TARGETDIR to directory where you want the compiled scripts to be copied
|
||||
# to.
|
||||
# Set BINDIR to the directory where GIZA++ and other tools are installed.
|
||||
TARGETDIR=/Users/hieuhoang/temp
|
||||
BINDIR=/Users/hieuhoang/temp
|
||||
TARGETDIR=/mnt/odin3/bhaddow/moses
|
||||
BINDIR=/mnt/odin3/bhaddow/moses/bin
|
||||
|
||||
MAIN_SCRIPTS_TARGET_DIR=$(TARGETDIR)
|
||||
# MAIN_SCRIPTS_TARGET_DIR=$(shell echo `pwd`/temp)
|
||||
|
@ -702,12 +702,19 @@ ibm-bleu-c
|
||||
rerun-on-change: ibm-bleu-c
|
||||
template: $ibm-bleu-c -s $input-sgm -r IN1 -t IN > OUT
|
||||
multi-bleu
|
||||
in: recased-output reference
|
||||
in: cleaned-output reference
|
||||
out: multi-bleu-score
|
||||
default-name: evaluation/multi-bleu
|
||||
ignore-unless: multi-bleu
|
||||
rerun-on-change: multi-bleu
|
||||
template: $multi-bleu IN1 < IN > OUT
|
||||
multi-bleu-c
|
||||
in: recased-output tokenized-reference
|
||||
out: multi-bleu-c-score
|
||||
default-name: evaluation/multi-bleu-c
|
||||
ignore-unless: multi-bleu-c
|
||||
rerun-on-change: multi-bleu-c
|
||||
template: $multi-bleu-c IN1 < IN > OUT
|
||||
ter
|
||||
in: wrapped-output reference-sgm
|
||||
out: ter-score
|
||||
@ -740,6 +747,6 @@ analysis-coverage
|
||||
|
||||
[REPORTING] single
|
||||
report
|
||||
in: EVALUATION:nist-bleu-score EVALUATION:nist-bleu-c-score EVALUATION:multi-bleu-score EVALUATION:meteor-score EVALUATION:ter-score EVALUATION:wer-score EVALUATION:ibm-bleu-score EVALUATION:ibm-bleu-c-score EVALUATION:analysis EVALUATION:analysis-coverage
|
||||
in: EVALUATION:nist-bleu-score EVALUATION:nist-bleu-c-score EVALUATION:multi-bleu-score EVALUATION:multi-bleu-c-score EVALUATION:meteor-score EVALUATION:ter-score EVALUATION:wer-score EVALUATION:ibm-bleu-score EVALUATION:ibm-bleu-c-score EVALUATION:analysis EVALUATION:analysis-coverage
|
||||
out: report
|
||||
default-name: evaluation/report
|
||||
|
@ -10,6 +10,7 @@ my %TYPE;
|
||||
$TYPE{"nist-bleu"} = "BLEU";
|
||||
$TYPE{"multi-bleu"} = "BLEU";
|
||||
$TYPE{"nist-bleu-c"} = "BLEU-c";
|
||||
$TYPE{"multi-bleu-c"}= "BLEU-c";
|
||||
$TYPE{"ibm-bleu"} = "IBM";
|
||||
$TYPE{"ibm-bleu-c"} = "IBM-c";
|
||||
|
||||
@ -52,8 +53,8 @@ sub process {
|
||||
elsif ($type eq 'ibm-bleu' || $type eq 'ibm-bleu-c') {
|
||||
$SCORE{$set} .= &extract_ibm_bleu($file,$type)." ";
|
||||
}
|
||||
elsif ($type eq 'multi-bleu') {
|
||||
$SCORE{$set} .= &extract_multi_bleu($file)." ";
|
||||
elsif ($type eq 'multi-bleu' || $type eq 'multi-bleu-c') {
|
||||
$SCORE{$set} .= &extract_multi_bleu($file,$type)." ";
|
||||
}
|
||||
}
|
||||
|
||||
@ -96,7 +97,7 @@ sub extract_ibm_bleu {
|
||||
}
|
||||
|
||||
sub extract_multi_bleu {
|
||||
my ($file) = @_;
|
||||
my ($file,$type) = @_;
|
||||
my ($bleu,$ratio);
|
||||
foreach (`cat $file`) {
|
||||
$bleu = $1 if /BLEU = (\S+), /;
|
||||
@ -107,5 +108,5 @@ sub extract_multi_bleu {
|
||||
|
||||
$AVERAGE{"multi-bleu"} += $bleu;
|
||||
|
||||
return $output."BLEU";
|
||||
return $output.$TYPE{$type};
|
||||
}
|
||||
|
@ -60,6 +60,7 @@
|
||||
/>
|
||||
<Tool
|
||||
Name="VCLinkerTool"
|
||||
AdditionalDependencies="zlib.lib"
|
||||
LinkIncremental="2"
|
||||
GenerateDebugInformation="true"
|
||||
SubSystem="1"
|
||||
@ -132,6 +133,7 @@
|
||||
/>
|
||||
<Tool
|
||||
Name="VCLinkerTool"
|
||||
AdditionalDependencies="zlib.lib"
|
||||
LinkIncremental="1"
|
||||
GenerateDebugInformation="true"
|
||||
SubSystem="1"
|
||||
@ -174,12 +176,20 @@
|
||||
RelativePath=".\consolidate-direct.cpp"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath=".\InputFileStream.cpp"
|
||||
>
|
||||
</File>
|
||||
</Filter>
|
||||
<Filter
|
||||
Name="Header Files"
|
||||
Filter="h;hpp;hxx;hm;inl;inc;xsd"
|
||||
UniqueIdentifier="{93995380-89BD-4b04-88EB-625FBE52EBFB}"
|
||||
>
|
||||
<File
|
||||
RelativePath=".\InputFileStream.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath=".\SafeGetline.h"
|
||||
>
|
||||
|
@ -61,6 +61,7 @@
|
||||
/>
|
||||
<Tool
|
||||
Name="VCLinkerTool"
|
||||
AdditionalDependencies="zlib.lib"
|
||||
LinkIncremental="2"
|
||||
GenerateDebugInformation="true"
|
||||
SubSystem="1"
|
||||
@ -133,6 +134,7 @@
|
||||
/>
|
||||
<Tool
|
||||
Name="VCLinkerTool"
|
||||
AdditionalDependencies="zlib.lib"
|
||||
LinkIncremental="1"
|
||||
GenerateDebugInformation="true"
|
||||
SubSystem="1"
|
||||
@ -177,6 +179,10 @@
|
||||
RelativePath=".\AlignmentPhrase.cpp"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath=".\InputFileStream.cpp"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath=".\PhraseAlignment.cpp"
|
||||
>
|
||||
@ -199,6 +205,10 @@
|
||||
RelativePath=".\AlignmentPhrase.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath=".\InputFileStream.h"
|
||||
>
|
||||
</File>
|
||||
<File
|
||||
RelativePath=".\PhraseAlignment.h"
|
||||
>
|
||||
|
@ -1601,6 +1601,7 @@ sub create_ini {
|
||||
print INI "$phrase_table_impl $ff 5 $file\n";
|
||||
}
|
||||
if ($_GLUE_GRAMMAR) {
|
||||
&full_path(\$___GLUE_GRAMMAR_FILE);
|
||||
print INI "6 0 0 1 $___GLUE_GRAMMAR_FILE\n";
|
||||
}
|
||||
if ($num_of_ttables != $stepsused{"T"}) {
|
||||
|
@ -1,5 +1,5 @@
|
||||
bin_PROGRAMS = mosesserver
|
||||
mosesserver_SOURCES = mosesserver.cpp
|
||||
mosesserver_CPPFLAGS = -W -Wall -I$(top_srcdir)/moses/src $(XMLRPC_C_CPPFLAGS) $(BOOST_CPPFLAGS)
|
||||
mosesserver_LDADD = -L$(top_srcdir)/moses/src -lmoses $(BOOST_LDFLAGS) $(XMLRPC_C_LIBS) $(BOOST_THREAD_LIB) -L$(top_srcdir)/OnDiskPt/src -lOnDiskPt
|
||||
mosesserver_DEPENDENCIES = $(top_srcdir)/moses/src/libmoses.la $(top_srcdir)/OnDiskPt/src/libOnDiskPt.a
|
||||
mosesserver_LDADD = -L$(top_srcdir)/moses/src -lmoses -L$(top_srcdir)/OnDiskPt/src -lOnDiskPt @KENLM_LDFLAGS@ $(BOOST_THREAD_LDFLAGS) $(XMLRPC_C_LIBS) $(BOOST_THREAD_LIBS)
|
||||
mosesserver_DEPENDENCIES = $(top_srcdir)/moses/src/libmoses.la $(top_srcdir)/OnDiskPt/src/libOnDiskPt.a @KENLM_DEPS@
|
||||
|
@ -7,5 +7,5 @@ moses_test_SOURCES = MockHypothesis.cpp \
|
||||
../mira/Hildreth.cpp
|
||||
|
||||
moses_test_CPPFLAGS = -W -Wall -I$(top_srcdir)/moses/src -I$(top_srcdir)/mira $(BOOST_CPPFLAGS) -DBOOST_TEST_DYN_LINK
|
||||
moses_test_LDADD = -L$(top_srcdir)/moses/src -lmoses $(BOOST_LDFLAGS) $(BOOST_THREAD_LIB) -L$(top_srcdir)/OnDiskPt/src -lOnDiskPt -lboost_unit_test_framework
|
||||
moses_test_LDADD = -L$(top_srcdir)/moses/src -lmoses $(BOOST_LDFLAGS) $(BOOST_THREAD_LIBS) -L$(top_srcdir)/OnDiskPt/src -lOnDiskPt @KENLM_LDFLAGS@ -lboost_unit_test_framework
|
||||
moses_test_DEPENDENCIES = $(top_srcdir)/moses/src/libmoses.la $(top_srcdir)/OnDiskPt/src/libOnDiskPt.a
|
||||
|
Loading…
Reference in New Issue
Block a user