moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2024-09-11 19:27:11 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge 3791-3842 from trunk

Browse Source 
git-svn-id: http://svn.statmt.org/repository/mira@3873 cc96ff50-19ce-11e0-b349-13d7f0bd23df
This commit is contained in:
bhaddow 2011-04-13 10:38:27 +00:00 committed by Ondrej Bojar
parent ec6272b3fc
commit e664b4a4b3
61 changed files with 2122 additions and 1346 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
CreateOnDisk/src/Makefile.am
View File

@ -2,8 +2,8 @@ bin_PROGRAMS = CreateOnDiskPt
CreateOnDiskPt_SOURCES = Main.cpp
AM_CPPFLAGS = -W -Wall -ffor-scope -D_FILE_OFFSET_BITS=64 -D_LARGE_FILES -DUSE_HYPO_POOL -I$(top_srcdir)/moses/src $(BOOST_CPPFLAGS)
CreateOnDiskPt_LDADD = -L$(top_srcdir)/OnDiskPt/src -L$(top_srcdir)/moses/src -lOnDiskPt -lmoses
CreateOnDiskPt_DEPENDENCIES = $(top_srcdir)/OnDiskPt/src/libOnDiskPt.a $(top_srcdir)/moses/src/libmoses.la
CreateOnDiskPt_LDADD = -L$(top_srcdir)/OnDiskPt/src -L$(top_srcdir)/moses/src -lOnDiskPt -lmoses $(BOOST_THREAD_LDFLAGS) $(BOOST_THREAD_LIBS) @KENLM_LDFLAGS@
CreateOnDiskPt_DEPENDENCIES = $(top_srcdir)/OnDiskPt/src/libOnDiskPt.a $(top_srcdir)/moses/src/libmoses.la @KENLM_DEPS@

2
Makefile.am
View File

@ -14,4 +14,4 @@ endif
if WITH_UNITTEST
UNITTEST = unittest
endif
SUBDIRS = moses/src moses-chart/src OnDiskPt/src kenlm moses-cmd/src misc moses-chart-cmd/src CreateOnDisk/src mira $(MERT) $(SERVER) $(UNITTEST)
SUBDIRS = kenlm moses/src moses-chart/src OnDiskPt/src moses-cmd/src misc moses-chart-cmd/src CreateOnDisk/src mira $(MERT) $(SERVER) $(UNITTEST)

21
config.h.in
View File

@ -1,19 +1,19 @@
/* config.h.in. Generated from configure.in by autoheader. */
/* define if the Boost library is available */
/* Defined if the requested minimum BOOST version is satisfied */
#undef HAVE_BOOST
/* define if the Boost::MPI library is available */
#undef HAVE_BOOST_MPI
/* Define to 1 if you have <boost/archive/text_oarchive.hpp> */
#undef HAVE_BOOST_ARCHIVE_TEXT_OARCHIVE_HPP
/* define if the Boost::PROGRAM_OPTIONS library is available */
#undef HAVE_BOOST_PROGRAM_OPTIONS
/* Define to 1 if you have <boost/mpi/communicator.hpp> */
#undef HAVE_BOOST_MPI_COMMUNICATOR_HPP
/* define if the Boost::Serialization library is available */
#undef HAVE_BOOST_SERIALIZATION
/* Define to 1 if you have <boost/program_options.hpp> */
#undef HAVE_BOOST_PROGRAM_OPTIONS_HPP
/* define if the Boost::Thread library is available */
#undef HAVE_BOOST_THREAD
/* Define to 1 if you have <boost/thread.hpp> */
#undef HAVE_BOOST_THREAD_HPP
/* Define to 1 if you have the <dlfcn.h> header file. */
#undef HAVE_DLFCN_H
@ -88,6 +88,9 @@
/* Define to the one symbol short name of this package. */
#undef PACKAGE_TARNAME
/* Define to the home page for this package. */
#undef PACKAGE_URL
/* Define to the version of this package. */
#undef PACKAGE_VERSION

28
configure.in
View File

@ -16,8 +16,8 @@ AC_PROG_LIBTOOL
AX_XMLRPC_C
AX_BOOST_BASE([1.37.0])
AX_BOOST_PROGRAM_OPTIONS
BOOST_REQUIRE([1.37.0])
BOOST_PROGRAM_OPTIONS
ac_have_mpi=no
AC_ARG_WITH(mpi,
@ -31,8 +31,8 @@ if test $ac_have_mpi = yes ; then
if test $CXX = none ; then
AC_MSG_ERROR([Cannot locate MPI compiler drivers])
fi
AX_BOOST_SERIALIZATION
AX_BOOST_MPI
BOOST_SERIALIZATION
BOOST_MPI
AC_DEFINE(MPI_ENABLED,1,[Define if compiling with MPI.])
CPPFLAGS="$CPPFLAGS -DMPI_ENABLE=1"
fi
@ -133,13 +133,13 @@ fi
if test "x$with_threads" = 'xyes' || test "x$enable_boost" = 'xyes'
then
AC_MSG_NOTICE([Using Boost library])
AX_BOOST_BASE([1.36.0])
BOOST_REQUIRE([1.36.0])
fi
if test "x$with_threads" = 'xyes'
then
AC_MSG_NOTICE([Building threaded moses])
AX_BOOST_THREAD
BOOST_THREADS
CPPFLAGS="$CPPFLAGS -DWITH_THREADS"
AM_CONDITIONAL([WITH_THREADS],true)
else
@ -149,10 +149,6 @@ fi
AM_CONDITIONAL([WITH_UNITTEST], false)
if test "x$enable_unittest" = 'xyes'
then
if test "x$BOOST_CPPFLAGS" = 'x'
then
AC_MSG_ERROR([Require boost to build unit tests])
fi
AC_MSG_NOTICE([Building unit tests])
AM_CONDITIONAL([WITH_UNITTEST], true)
fi
@ -236,12 +232,16 @@ then
[AC_DEFINE([HAVE_KENLM], [], [flag for KENLM])],
[AC_MSG_ERROR([Cannot find KEN-LM in ${PWD}/kenlm])])
LIB_KENLM="-lkenlm"
LDFLAGS="$LDFLAGS -L${PWD}/kenlm"
LIBS="$LIBS $LIB_KENLM"
FMTLIBS="$FMTLIBS libkenlm.a"
KENLM_LDFLAGS="-L\$(top_srcdir)/kenlm -lkenlm -lz"
KENLM_DEPS="\$(top_srcdir)/kenlm/libkenlm.la"
FMTLIBS="$FMTLIBS libkenlm.la"
AM_CONDITIONAL([KEN_LM], true)
else
KENLM_LDFLAGS=""
KENLM_DEPS=""
fi
AC_SUBST(KENLM_LDFLAGS)
AC_SUBST(KENLM_DEPS)
if test "x$with_randlm" != 'xno'
then

10
kenlm/Makefile.am
View File

@ -1,8 +1,8 @@
lib_LIBRARIES = libkenlm.a
lib_LTLIBRARIES = libkenlm.la
bin_PROGRAMS = query build_binary
AM_CPPFLAGS = -W -Wall -ffor-scope -D_FILE_OFFSET_BITS=64 -D_LARGE_FILES $(BOOST_CPPFLAGS)
libkenlm_a_SOURCES = \
libkenlm_la_SOURCES = \
lm/lm_exception.cc \
lm/config.cc \
lm/model.cc \
@ -24,10 +24,8 @@ libkenlm_a_SOURCES = \
util/bit_packing.cc
query_SOURCES = lm/ngram_query.cc
query_DEPENDENCIES = libkenlm.a
query_LDADD = -L$(top_srcdir)/kenlm -lkenlm -lz
query_LDADD = libkenlm.la
build_binary_SOURCES = lm/build_binary.cc
build_binary_DEPENDENCIES = libkenlm.a
build_binary_LDADD = -L$(top_srcdir)/kenlm -lkenlm -lz
build_binary_LDADD = libkenlm.la

6
kenlm/util/string_piece.cc
View File

@ -35,17 +35,11 @@
#endif
#include <algorithm>
#include <iostream>
#ifdef HAVE_ICU
U_NAMESPACE_BEGIN
#endif
std::ostream& operator<<(std::ostream& o, const StringPiece& piece) {
o.write(piece.data(), static_cast<std::streamsize>(piece.size()));
return o;
}
#ifdef HAVE_BOOST
size_t hash_value(const StringPiece &str) {
return boost::hash_range(str.data(), str.data() + str.length());

7
kenlm/util/string_piece.hh
View File

@ -56,13 +56,14 @@
#include <cstring>
#include <iosfwd>
#include <ostream>
#ifdef HAVE_ICU
#include <unicode/stringpiece.h>
U_NAMESPACE_BEGIN
#else
#include <algorithm>
#include <ostream>
#include <string>
#include <string.h>
@ -228,7 +229,9 @@ inline bool operator>=(const StringPiece& x, const StringPiece& y) {
}
// allow StringPiece to be logged (needed for unit testing).
extern std::ostream& operator<<(std::ostream& o, const StringPiece& piece);
inline std::ostream& operator<<(std::ostream& o, const StringPiece& piece) {
return o.write(piece.data(), static_cast<std::streamsize>(piece.size()));
}
#ifdef HAVE_BOOST
size_t hash_value(const StringPiece &str);

223
m4/ax_boost_base.m4
View File

@ -1,223 +0,0 @@
# ===========================================================================
# http://autoconf-archive.cryp.to/ax_boost_base.html
# ===========================================================================
#
# SYNOPSIS
#
# AX_BOOST_BASE([MINIMUM-VERSION])
#
# DESCRIPTION
#
# Test for the Boost C++ libraries of a particular version (or newer)
#
# If no path to the installed boost library is given the macro searchs
# under /usr, /usr/local, /opt and /opt/local and evaluates the
# $BOOST_ROOT environment variable. Further documentation is available at
# <http://randspringer.de/boost/index.html>.
#
# This macro calls:
#
# AC_SUBST(BOOST_CPPFLAGS) / AC_SUBST(BOOST_LDFLAGS)
#
# And sets:
#
# HAVE_BOOST
#
# LAST MODIFICATION
#
# 2008-04-12
#
# COPYLEFT
#
# Copyright (c) 2008 Thomas Porschberg <thomas@randspringer.de>
#
# Copying and distribution of this file, with or without modification, are
# permitted in any medium without royalty provided the copyright notice
# and this notice are preserved.
AC_DEFUN([AX_BOOST_BASE],
[
AC_ARG_WITH([boost],
AS_HELP_STRING([--with-boost@<:@=DIR@:>@], [use boost (default is yes) - it is possible to specify the root directory for boost (optional)]),
[
if test "$withval" = "no"; then
want_boost="no"
elif test "$withval" = "yes"; then
want_boost="yes"
ac_boost_path=""
else
want_boost="yes"
ac_boost_path="$withval"
fi
],
[want_boost="yes"])
AC_ARG_WITH([boost-libdir],
AS_HELP_STRING([--with-boost-libdir=LIB_DIR],
[Force given directory for boost libraries. Note that this will overwrite library path detection, so use this parameter only if default library detection fails and you know exactly where your boost libraries are located.]),
[
if test -d $withval
then
ac_boost_lib_path="$withval"
else
AC_MSG_ERROR(--with-boost-libdir expected directory name)
fi
],
[ac_boost_lib_path=""]
)
if test "x$want_boost" = "xyes"; then
boost_lib_version_req=ifelse([$1], ,1.20.0,$1)
boost_lib_version_req_shorten=`expr $boost_lib_version_req : '\([[0-9]]*\.[[0-9]]*\)'`
boost_lib_version_req_major=`expr $boost_lib_version_req : '\([[0-9]]*\)'`
boost_lib_version_req_minor=`expr $boost_lib_version_req : '[[0-9]]*\.\([[0-9]]*\)'`
boost_lib_version_req_sub_minor=`expr $boost_lib_version_req : '[[0-9]]*\.[[0-9]]*\.\([[0-9]]*\)'`
if test "x$boost_lib_version_req_sub_minor" = "x" ; then
boost_lib_version_req_sub_minor="0"
fi
WANT_BOOST_VERSION=`expr $boost_lib_version_req_major \* 100000 \+ $boost_lib_version_req_minor \* 100 \+ $boost_lib_version_req_sub_minor`
AC_MSG_CHECKING(for boostlib >= $boost_lib_version_req)
succeeded=no
dnl first we check the system location for boost libraries
dnl this location ist chosen if boost libraries are installed with the --layout=system option
dnl or if you install boost with RPM
if test "$ac_boost_path" != ""; then
BOOST_LDFLAGS="-L$ac_boost_path/lib"
BOOST_CPPFLAGS="-I$ac_boost_path/include"
else
for ac_boost_path_tmp in /usr /usr/local /opt /opt/local ; do
if test -d "$ac_boost_path_tmp/include/boost" && test -r "$ac_boost_path_tmp/include/boost"; then
BOOST_LDFLAGS="-L$ac_boost_path_tmp/lib"
BOOST_CPPFLAGS="-I$ac_boost_path_tmp/include"
break;
fi
done
fi
dnl overwrite ld flags if we have required special directory with
dnl --with-boost-libdir parameter
if test "$ac_boost_lib_path" != ""; then
BOOST_LDFLAGS="-L$ac_boost_lib_path"
fi
CPPFLAGS_SAVED="$CPPFLAGS"
CPPFLAGS="$CPPFLAGS $BOOST_CPPFLAGS"
export CPPFLAGS
LDFLAGS_SAVED="$LDFLAGS"
LDFLAGS="$LDFLAGS $BOOST_LDFLAGS"
export LDFLAGS
AC_LANG_PUSH(C++)
AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
@%:@include <boost/version.hpp>
]], [[
#if BOOST_VERSION >= $WANT_BOOST_VERSION
// Everything is okay
#else
# error Boost version is too old
#endif
]])],[
AC_MSG_RESULT(yes)
succeeded=yes
found_system=yes
],[
])
AC_LANG_POP([C++])
dnl if we found no boost with system layout we search for boost libraries
dnl built and installed without the --layout=system option or for a staged(not installed) version
if test "x$succeeded" != "xyes"; then
_version=0
if test "$ac_boost_path" != ""; then
if test -d "$ac_boost_path" && test -r "$ac_boost_path"; then
for i in `ls -d $ac_boost_path/include/boost-* 2>/dev/null`; do
_version_tmp=`echo $i | sed "s#$ac_boost_path##" | sed 's/\/include\/boost-//' | sed 's/_/./'`
V_CHECK=`expr $_version_tmp \> $_version`
if test "$V_CHECK" = "1" ; then
_version=$_version_tmp
fi
VERSION_UNDERSCORE=`echo $_version | sed 's/\./_/'`
BOOST_CPPFLAGS="-I$ac_boost_path/include/boost-$VERSION_UNDERSCORE"
done
fi
else
for ac_boost_path in /usr /usr/local /opt /opt/local ; do
if test -d "$ac_boost_path" && test -r "$ac_boost_path"; then
for i in `ls -d $ac_boost_path/include/boost-* 2>/dev/null`; do
_version_tmp=`echo $i | sed "s#$ac_boost_path##" | sed 's/\/include\/boost-//' | sed 's/_/./'`
V_CHECK=`expr $_version_tmp \> $_version`
if test "$V_CHECK" = "1" ; then
_version=$_version_tmp
best_path=$ac_boost_path
fi
done
fi
done
VERSION_UNDERSCORE=`echo $_version | sed 's/\./_/'`
BOOST_CPPFLAGS="-I$best_path/include/boost-$VERSION_UNDERSCORE"
if test "$ac_boost_lib_path" = ""
then
BOOST_LDFLAGS="-L$best_path/lib"
fi
if test "x$BOOST_ROOT" != "x"; then
if test -d "$BOOST_ROOT" && test -r "$BOOST_ROOT" && test -d "$BOOST_ROOT/stage/lib" && test -r "$BOOST_ROOT/stage/lib"; then
version_dir=`expr //$BOOST_ROOT : '.*/\(.*\)'`
stage_version=`echo $version_dir | sed 's/boost_//' | sed 's/_/./g'`
stage_version_shorten=`expr $stage_version : '\([[0-9]]*\.[[0-9]]*\)'`
V_CHECK=`expr $stage_version_shorten \>\= $_version`
if test "$V_CHECK" = "1" -a "$ac_boost_lib_path" = "" ; then
AC_MSG_NOTICE(We will use a staged boost library from $BOOST_ROOT)
BOOST_CPPFLAGS="-I$BOOST_ROOT"
BOOST_LDFLAGS="-L$BOOST_ROOT/stage/lib"
fi
fi
fi
fi
CPPFLAGS="$CPPFLAGS $BOOST_CPPFLAGS"
export CPPFLAGS
LDFLAGS="$LDFLAGS $BOOST_LDFLAGS"
export LDFLAGS
AC_LANG_PUSH(C++)
AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
@%:@include <boost/version.hpp>
]], [[
#if BOOST_VERSION >= $WANT_BOOST_VERSION
// Everything is okay
#else
# error Boost version is too old
#endif
]])],[
AC_MSG_RESULT(yes)
succeeded=yes
found_system=yes
],[
])
AC_LANG_POP([C++])
fi
if test "$succeeded" != "yes" ; then
if test "$_version" = "0" ; then
AC_MSG_ERROR([[We could not detect the boost libraries (version $boost_lib_version_req_shorten or higher). If you have a staged boost library (still not installed) please specify \$BOOST_ROOT in your environment and do not give a PATH to --with-boost option. If you are sure you have boost installed, then check your version number looking in <boost/version.hpp>. See http://randspringer.de/boost for more documentation.]])
else
AC_MSG_NOTICE([Your boost libraries seems to old (version $_version).])
fi
else
AC_SUBST(BOOST_CPPFLAGS)
AC_SUBST(BOOST_LDFLAGS)
AC_DEFINE(HAVE_BOOST,,[define if the Boost library is available])
fi
CPPFLAGS="$CPPFLAGS_SAVED"
LDFLAGS="$LDFLAGS_SAVED"
fi
])

122
m4/ax_boost_mpi.m4
View File

@ -1,122 +0,0 @@
# ===========================================================================
#
# SYNOPSIS
#
# AX_BOOST_MPI
#
# DESCRIPTION
#
# Test for MPI library from the Boost C++ libraries. The macro
# requires a preceding call to AX_BOOST_BASE, AX_BOOST_SERIALIZATION
# and AX_MPI. You also need to set CXX="$MPICXX" before calling the
# macro.
#
# This macro calls:
#
# AC_SUBST(BOOST_MPI_LIB)
#
# And sets:
#
# HAVE_BOOST_MPI
#
# LICENSE
#
# Based on Boost Serialize by:
# Copyright (c) 2008 Thomas Porschberg <thomas@randspringer.de>
#
# Copyright (c) 2010 Mirko Maischberger <mirko.maischberger@gmail.com>
#
# Copying and distribution of this file, with or without modification, are
# permitted in any medium without royalty provided the copyright notice
# and this notice are preserved. This file is offered as-is, without any
# warranty.
#serial 1
AC_DEFUN([AX_BOOST_MPI],
[
AC_ARG_WITH([boost-mpi],
AS_HELP_STRING([--with-boost-mpi@<:@=special-lib@:>@],
[use the MPI library from boost - it is possible to
specify a certain library for the linker
e.g. --with-boost-mpi=boost_mpi-gcc-mt-d-1_33_1 ]),
[
if test "$withval" = "no"; then
want_boost="no"
elif test "$withval" = "yes"; then
want_boost="yes"
ax_boost_user_mpi_lib=""
else
want_boost="yes"
ax_boost_user_mpi_lib="$withval"
fi
],
[want_boost="yes"]
)
if test "x$want_boost" = "xyes"; then
AC_REQUIRE([AC_PROG_CC])
CPPFLAGS_SAVED="$CPPFLAGS"
CPPFLAGS="$CPPFLAGS $BOOST_CPPFLAGS"
AC_MSG_WARN(BOOST_CPPFLAGS $BOOST_CPPFLAGS)
export CPPFLAGS
LDFLAGS_SAVED="$LDFLAGS"
LDFLAGS="$LDFLAGS $BOOST_LDFLAGS"
export LDFLAGS
LIBS_SAVED="$LIBS"
LIBS="$LIBS $BOOST_SERIALIZATION_LIB"
export LIBS
AC_CACHE_CHECK(whether the Boost::MPI library is available,
ax_cv_boost_mpi,
[AC_LANG_PUSH([C++])
AC_COMPILE_IFELSE(AC_LANG_PROGRAM([[@%:@include <boost/mpi.hpp>
]],
[[int argc = 0;
char **argv = 0;
boost::mpi::environment env(argc,argv);
return 0;
]]),
ax_cv_boost_mpi=yes, ax_cv_boost_mpi=no)
AC_LANG_POP([C++])
])
if test "x$ax_cv_boost_mpi" = "xyes"; then
AC_DEFINE(HAVE_BOOST_MPI,,[define if the Boost::MPI library is available])
BOOSTLIBDIR=`echo $BOOST_LDFLAGS | sed -e 's/@<:@^\/@:>@*//'`
if test "x$ax_boost_user_mpi_lib" = "x"; then
for libextension in `ls $BOOSTLIBDIR/libboost_mpi*.{so,a}* 2>/dev/null | grep -v python | sed 's,.*/,,' | sed -e 's;^lib\(boost_mpi.*\)\.so.*$;\1;' -e 's;^lib\(boost_mpi.*\)\.a*$;\1;'` ; do
ax_lib=${libextension}
AC_CHECK_LIB($ax_lib, exit,
[BOOST_MPI_LIB="-l$ax_lib"; AC_SUBST(BOOST_MPI_LIB) link_mpi="yes"; break],
[link_mpi="no"])
done
if test "x$link_mpi" != "xyes"; then
for libextension in `ls $BOOSTLIBDIR/boost_mpi*.{dll,a}* 2>/dev/null | grep -v python | sed 's,.*/,,' | sed -e 's;^\(boost_mpi.*\)\.dll.*$;\1;' -e 's;^\(boost_mpi.*\)\.a*$;\1;'` ; do
ax_lib=${libextension}
AC_CHECK_LIB($ax_lib, exit,
[BOOST_MPI_LIB="-l$ax_lib"; AC_SUBST(BOOST_MPI_LIB) link_mpi="yes"; break],
[link_mpi="no"])
done
fi
else
for ax_lib in $ax_boost_user_mpi_lib boost_mpi-$ax_boost_user_mpi_lib; do
AC_CHECK_LIB($ax_lib, exit,
[BOOST_MPI_LIB="-l$ax_lib"; AC_SUBST(BOOST_MPI_LIB) link_mpi="yes"; break],
[link_mpi="no"])
done
fi
if test "x$link_mpi" != "xyes"; then
AC_MSG_ERROR(Could not link against $ax_lib !)
fi
fi
LIBS="$LIBS_SAVED"
CPPFLAGS="$CPPFLAGS_SAVED"
LDFLAGS="$LDFLAGS_SAVED"
fi
])

106
m4/ax_boost_program_options.m4
View File

@ -1,106 +0,0 @@
# ===========================================================================
# http://autoconf-archive.cryp.to/ax_boost_program_options.html
# ===========================================================================
#
# SYNOPSIS
#
# AX_BOOST_PROGRAM_OPTIONS
#
# DESCRIPTION
#
# Test for program options library from the Boost C++ libraries. The macro
# requires a preceding call to AX_BOOST_BASE. Further documentation is
# available at <http://randspringer.de/boost/index.html>.
#
# This macro calls:
#
# AC_SUBST(BOOST_PROGRAM_OPTIONS_LIB)
#
# And sets:
#
# HAVE_BOOST_PROGRAM_OPTIONS
#
# LAST MODIFICATION
#
# 2008-04-12
#
# COPYLEFT
#
# Copyright (c) 2008 Thomas Porschberg <thomas@randspringer.de>
#
# Copying and distribution of this file, with or without modification, are
# permitted in any medium without royalty provided the copyright notice
# and this notice are preserved.
AC_DEFUN([AX_BOOST_PROGRAM_OPTIONS],
[
AC_ARG_WITH([boost-program-options],
AS_HELP_STRING([--with-boost-program-options@<:@=special-lib@:>@],
[use the program options library from boost - it is possible to specify a certain library for the linker
e.g. --with-boost-program-options=boost_program_options-gcc-mt-1_33_1 ]),
[
if test "$withval" = "no"; then
want_boost="no"
elif test "$withval" = "yes"; then
want_boost="yes"
ax_boost_user_program_options_lib=""
else
want_boost="yes"
ax_boost_user_program_options_lib="$withval"
fi
],
[want_boost="yes"]
)
if test "x$want_boost" = "xyes"; then
AC_REQUIRE([AC_PROG_CC])
export want_boost
CPPFLAGS_SAVED="$CPPFLAGS"
CPPFLAGS="$CPPFLAGS $BOOST_CPPFLAGS"
export CPPFLAGS
LDFLAGS_SAVED="$LDFLAGS"
LDFLAGS="$LDFLAGS $BOOST_LDFLAGS"
export LDFLAGS
AC_CACHE_CHECK([whether the Boost::Program_Options library is available],
ax_cv_boost_program_options,
[AC_LANG_PUSH(C++)
AC_COMPILE_IFELSE(AC_LANG_PROGRAM([[@%:@include <boost/program_options.hpp>
]],
[[boost::program_options::options_description generic("Generic options");
return 0;]]),
ax_cv_boost_program_options=yes, ax_cv_boost_program_options=no)
AC_LANG_POP([C++])
])
if test "$ax_cv_boost_program_options" = yes; then
AC_DEFINE(HAVE_BOOST_PROGRAM_OPTIONS,,[define if the Boost::PROGRAM_OPTIONS library is available])
BOOSTLIBDIR=`echo $BOOST_LDFLAGS | sed -e 's/@<:@^\/@:>@*//'`
if test "x$ax_boost_user_program_options_lib" = "x"; then
for libextension in `ls $BOOSTLIBDIR/libboost_program_options*.{so,a}* 2>/dev/null | sed 's,.*/,,' | sed -e 's;^lib\(boost_program_options.*\)\.so.*$;\1;' -e 's;^lib\(boost_program_options.*\)\.a*$;\1;'` ; do
ax_lib=${libextension}
AC_CHECK_LIB($ax_lib, exit,
[BOOST_PROGRAM_OPTIONS_LIB="-l$ax_lib"; AC_SUBST(BOOST_PROGRAM_OPTIONS_LIB) link_program_options="yes"; break],
[link_program_options="no"])
done
if test "x$link_program_options" != "xyes"; then
for libextension in `ls $BOOSTLIBDIR/boost_program_options*.{dll,a}* 2>/dev/null | sed 's,.*/,,' | sed -e 's;^\(boost_program_options.*\)\.dll.*$;\1;' -e 's;^\(boost_program_options.*\)\.a*$;\1;'` ; do
ax_lib=${libextension}
AC_CHECK_LIB($ax_lib, exit,
[BOOST_PROGRAM_OPTIONS_LIB="-l$ax_lib"; AC_SUBST(BOOST_PROGRAM_OPTIONS_LIB) link_program_options="yes"; break],
[link_program_options="no"])
done
fi
else
for ax_lib in $ax_boost_user_program_options_lib boost_program_options-$ax_boost_user_program_options_lib; do
AC_CHECK_LIB($ax_lib, main,
[BOOST_PROGRAM_OPTIONS_LIB="-l$ax_lib"; AC_SUBST(BOOST_PROGRAM_OPTIONS_LIB) link_program_options="yes"; break],
[link_program_options="no"])
done
fi
if test "x$link_program_options" != "xyes"; then
AC_MSG_ERROR([Could not link against [$ax_lib] !])
fi
fi
CPPFLAGS="$CPPFLAGS_SAVED"
LDFLAGS="$LDFLAGS_SAVED"
fi
])

115
m4/ax_boost_serialization.m4
View File

@ -1,115 +0,0 @@
# ===========================================================================
# http://autoconf-archive.cryp.to/ax_boost_serialization.html
# ===========================================================================
#
# SYNOPSIS
#
# AX_BOOST_SERIALIZATION
#
# DESCRIPTION
#
# Test for Serialization library from the Boost C++ libraries. The macro
# requires a preceding call to AX_BOOST_BASE. Further documentation is
# available at <http://randspringer.de/boost/index.html>.
#
# This macro calls:
#
# AC_SUBST(BOOST_SERIALIZATION_LIB)
#
# And sets:
#
# HAVE_BOOST_SERIALIZATION
#
# LAST MODIFICATION
#
# 2008-04-12
#
# COPYLEFT
#
# Copyright (c) 2008 Thomas Porschberg <thomas@randspringer.de>
#
# Copying and distribution of this file, with or without modification, are
# permitted in any medium without royalty provided the copyright notice
# and this notice are preserved.
AC_DEFUN([AX_BOOST_SERIALIZATION],
[
AC_ARG_WITH([boost-serialization],
AS_HELP_STRING([--with-boost-serialization@<:@=special-lib@:>@],
[use the Serialization library from boost - it is possible to specify a certain library for the linker
e.g. --with-boost-serialization=boost_serialization-gcc-mt-d-1_33_1 ]),
[
if test "$withval" = "no"; then
want_boost="no"
elif test "$withval" = "yes"; then
want_boost="yes"
ax_boost_user_serialization_lib=""
else
want_boost="yes"
ax_boost_user_serialization_lib="$withval"
fi
],
[want_boost="yes"]
)
if test "x$want_boost" = "xyes"; then
AC_REQUIRE([AC_PROG_CC])
CPPFLAGS_SAVED="$CPPFLAGS"
CPPFLAGS="$CPPFLAGS $BOOST_CPPFLAGS"
AC_MSG_WARN(BOOST_CPPFLAGS $BOOST_CPPFLAGS)
export CPPFLAGS
LDFLAGS_SAVED="$LDFLAGS"
LDFLAGS="$LDFLAGS $BOOST_LDFLAGS"
export LDFLAGS
AC_CACHE_CHECK(whether the Boost::Serialization library is available,
ax_cv_boost_serialization,
[AC_LANG_PUSH([C++])
AC_COMPILE_IFELSE(AC_LANG_PROGRAM([[@%:@include <fstream>
@%:@include <boost/archive/text_oarchive.hpp>
@%:@include <boost/archive/text_iarchive.hpp>
]],
[[std::ofstream ofs("filename");
boost::archive::text_oarchive oa(ofs);
return 0;
]]),
ax_cv_boost_serialization=yes, ax_cv_boost_serialization=no)
AC_LANG_POP([C++])
])
if test "x$ax_cv_boost_serialization" = "xyes"; then
AC_DEFINE(HAVE_BOOST_SERIALIZATION,,[define if the Boost::Serialization library is available])
BOOSTLIBDIR=`echo $BOOST_LDFLAGS | sed -e 's/@<:@^\/@:>@*//'`
if test "x$ax_boost_user_serialization_lib" = "x"; then
for libextension in `ls $BOOSTLIBDIR/libboost_serialization*.{so,a}* 2>/dev/null | sed 's,.*/,,' | sed -e 's;^lib\(boost_serialization.*\)\.so.*$;\1;' -e 's;^lib\(boost_serialization.*\)\.a*$;\1;'` ; do
ax_lib=${libextension}
AC_CHECK_LIB($ax_lib, exit,
[BOOST_SERIALIZATION_LIB="-l$ax_lib"; AC_SUBST(BOOST_SERIALIZATION_LIB) link_serialization="yes"; break],
[link_serialization="no"])
done
if test "x$link_serialization" != "xyes"; then
for libextension in `ls $BOOSTLIBDIR/boost_serialization*.{dll,a}* 2>/dev/null | sed 's,.*/,,' | sed -e 's;^\(boost_serialization.*\)\.dll.*$;\1;' -e 's;^\(boost_serialization.*\)\.a*$;\1;'` ; do
ax_lib=${libextension}
AC_CHECK_LIB($ax_lib, exit,
[BOOST_SERIALIZATION_LIB="-l$ax_lib"; AC_SUBST(BOOST_SERIALIZATION_LIB) link_serialization="yes"; break],
[link_serialization="no"])
done
fi
else
for ax_lib in $ax_boost_user_serialization_lib boost_serialization-$ax_boost_user_serialization_lib; do
AC_CHECK_LIB($ax_lib, main,
[BOOST_SERIALIZATION_LIB="-l$ax_lib"; AC_SUBST(BOOST_SERIALIZATION_LIB) link_serialization="yes"; break],
[link_serialization="no"])
done
fi
if test "x$link_serialization" != "xyes"; then
AC_MSG_ERROR(Could not link against $ax_lib !)
fi
fi
CPPFLAGS="$CPPFLAGS_SAVED"
LDFLAGS="$LDFLAGS_SAVED"
fi
])

143
m4/ax_boost_thread.m4
View File

@ -1,143 +0,0 @@
# ===========================================================================
# http://www.nongnu.org/autoconf-archive/ax_boost_thread.html
# ===========================================================================
#
# SYNOPSIS
#
# AX_BOOST_THREAD
#
# DESCRIPTION
#
# Test for Thread library from the Boost C++ libraries. The macro requires
# a preceding call to AX_BOOST_BASE. Further documentation is available at
# <http://randspringer.de/boost/index.html>.
#
# This macro calls:
#
# AC_SUBST(BOOST_THREAD_LIB)
#
# And sets:
#
# HAVE_BOOST_THREAD
#
# LICENSE
#
# Copyright (c) 2009 Thomas Porschberg <thomas@randspringer.de>
# Copyright (c) 2009 Michael Tindal
#
# Copying and distribution of this file, with or without modification, are
# permitted in any medium without royalty provided the copyright notice
# and this notice are preserved.
AC_DEFUN([AX_BOOST_THREAD],
[
AC_ARG_WITH([boost-thread],
AS_HELP_STRING([--with-boost-thread@<:@=special-lib@:>@],
[use the Thread library from boost - it is possible to specify a certain library for the linker
e.g. --with-boost-thread=boost_thread-gcc-mt ]),
[
if test "$withval" = "no"; then
want_boost="no"
elif test "$withval" = "yes"; then
want_boost="yes"
ax_boost_user_thread_lib=""
else
want_boost="yes"
ax_boost_user_thread_lib="$withval"
fi
],
[want_boost="yes"]
)
if test "x$want_boost" = "xyes"; then
AC_REQUIRE([AC_PROG_CC])
AC_REQUIRE([AC_CANONICAL_BUILD])
CPPFLAGS_SAVED="$CPPFLAGS"
CPPFLAGS="$CPPFLAGS $BOOST_CPPFLAGS"
export CPPFLAGS
LDFLAGS_SAVED="$LDFLAGS"
LDFLAGS="$LDFLAGS $BOOST_LDFLAGS"
export LDFLAGS
AC_CACHE_CHECK(whether the Boost::Thread library is available,
ax_cv_boost_thread,
[AC_LANG_PUSH([C++])
CXXFLAGS_SAVE=$CXXFLAGS
if test "x$build_os" = "xsolaris" ; then
CXXFLAGS="-pthreads $CXXFLAGS"
elif test "x$build_os" = "xming32" ; then
CXXFLAGS="-mthreads $CXXFLAGS"
else
CXXFLAGS="-pthread $CXXFLAGS"
fi
AC_COMPILE_IFELSE(AC_LANG_PROGRAM([[@%:@include <boost/thread/thread.hpp>]],
[[boost::thread_group thrds;
return 0;]]),
ax_cv_boost_thread=yes, ax_cv_boost_thread=no)
CXXFLAGS=$CXXFLAGS_SAVE
AC_LANG_POP([C++])
])
if test "x$ax_cv_boost_thread" = "xyes"; then
if test "x$build_os" = "xsolaris" ; then
BOOST_CPPFLAGS="-pthreads $BOOST_CPPFLAGS"
elif test "x$build_os" = "xming32" ; then
BOOST_CPPFLAGS="-mthreads $BOOST_CPPFLAGS"
else
BOOST_CPPFLAGS="-pthread $BOOST_CPPFLAGS"
fi
AC_SUBST(BOOST_CPPFLAGS)
AC_DEFINE(HAVE_BOOST_THREAD,,[define if the Boost::Thread library is available])
BOOSTLIBDIR=`echo $BOOST_LDFLAGS | sed -e 's/@<:@^\/@:>@*//'`
LDFLAGS_SAVE=$LDFLAGS
case "x$build_os" in
*bsd* )
LDFLAGS="-pthread $LDFLAGS"
break;
;;
esac
if test "x$ax_boost_user_thread_lib" = "x"; then
for libextension in `ls $BOOSTLIBDIR/libboost_thread*.so* 2>/dev/null | sed 's,.*/,,' | sed -e 's;^lib\(boost_thread.*\)\.so.*$;\1;'` `ls $BOOSTLIBDIR/libboost_thread*.a* 2>/dev/null | sed 's,.*/,,' | sed -e 's;^lib\(boost_thread.*\)\.a*$;\1;'`; do
ax_lib=${libextension}
AC_CHECK_LIB($ax_lib, exit,
[BOOST_THREAD_LIB="-l$ax_lib"; AC_SUBST(BOOST_THREAD_LIB) link_thread="yes"; break],
[link_thread="no"])
done
if test "x$link_thread" != "xyes"; then
for libextension in `ls $BOOSTLIBDIR/boost_thread*.dll* 2>/dev/null | sed 's,.*/,,' | sed -e 's;^\(boost_thread.*\)\.dll.*$;\1;'` `ls $BOOSTLIBDIR/boost_thread*.a* 2>/dev/null | sed 's,.*/,,' | sed -e 's;^\(boost_thread.*\)\.a*$;\1;'` ; do
ax_lib=${libextension}
AC_CHECK_LIB($ax_lib, exit,
[BOOST_THREAD_LIB="-l$ax_lib"; AC_SUBST(BOOST_THREAD_LIB) link_thread="yes"; break],
[link_thread="no"])
done
fi
else
for ax_lib in $ax_boost_user_thread_lib boost_thread-$ax_boost_user_thread_lib; do
AC_CHECK_LIB($ax_lib, exit,
[BOOST_THREAD_LIB="-l$ax_lib"; AC_SUBST(BOOST_THREAD_LIB) link_thread="yes"; break],
[link_thread="no"])
done
fi
if test "x$link_thread" = "xno"; then
AC_MSG_ERROR(Could not link against $ax_lib !)
else
case "x$build_os" in
*bsd* )
BOOST_LDFLAGS="-pthread $BOOST_LDFLAGS"
break;
;;
esac
fi
fi
CPPFLAGS="$CPPFLAGS_SAVED"
LDFLAGS="$LDFLAGS_SAVED"
fi
])

1045
m4/boost.m4 Normal file
View File

File diff suppressed because it is too large Load Diff

3
mira/Makefile.am
View File

@ -10,4 +10,5 @@ mira_SOURCES = \
AM_CPPFLAGS = -W -Wall -Wno-unused -ffor-scope -D_FILE_OFFSET_BITS=64 -D_LARGE_FILES -I$(top_srcdir)/moses/src -I$(top_srcdir)/moses-chart/src $(BOOST_CPPFLAGS)
mira_LDADD = $(top_builddir)/moses/src/libmoses.la -L$(top_srcdir)/moses/src -L$(top_srcdir)/moses-chart/src -lmoses -lmoses-chart $(BOOST_LDFLAGS) $(BOOST_SERIALIZATION_LIB) $(BOOST_PROGRAM_OPTIONS_LIB) $(BOOST_MPI_LIB) -L$(top_srcdir)/OnDiskPt/src -lOnDiskPt
mira_LDADD = $(top_builddir)/moses/src/libmoses.la -L$(top_srcdir)/moses/src -L$(top_srcdir)/moses-chart/src -lmoses -lmoses-chart -L$(top_srcdir)/OnDiskPt/src -lOnDiskPt @KENLM_LDFLAGS@ $(BOOST_LDFLAGS) $(BOOST_SERIALIZATION_LIBS) $(BOOST_PROGRAM_OPTIONS_LDFLAGS) $(BOOST_PROGRAM_OPTIONS_LIBS) $(BOOST_MPI_LIBS)
mira_DEPENDENCIES = $(top_srcdir)/moses/src/libmoses.la

8
misc/Makefile.am
View File

@ -7,10 +7,10 @@ queryPhraseTable_SOURCES = queryPhraseTable.cpp
AM_CPPFLAGS = -W -Wall -ffor-scope -D_FILE_OFFSET_BITS=64 -D_LARGE_FILES -I$(top_srcdir)/moses/src $(BOOST_CPPFLAGS)
processPhraseTable_LDADD = $(top_builddir)/moses/src/libmoses.la $(BOOST_LDFLAGS) $(BOOST_THREAD_LIB) -L$(top_srcdir)/moses/src -L$(top_srcdir)/OnDiskPt/src -lmoses -lOnDiskPt
processPhraseTable_LDADD = $(top_builddir)/moses/src/libmoses.la -L$(top_srcdir)/moses/src -L$(top_srcdir)/OnDiskPt/src -lmoses -lOnDiskPt @KENLM_LDFLAGS@ $(BOOST_THREAD_LDFLAGS) $(BOOST_THREAD_LIBS)
processLexicalTable_LDADD = $(top_builddir)/moses/src/libmoses.la $(BOOST_LDFLAGS) $(BOOST_THREAD_LIB) -L$(top_srcdir)/moses/src -L$(top_srcdir)/OnDiskPt/src -lmoses -lOnDiskPt
processLexicalTable_LDADD = $(top_builddir)/moses/src/libmoses.la -L$(top_srcdir)/moses/src -L$(top_srcdir)/OnDiskPt/src -lmoses -lOnDiskPt @KENLM_LDFLAGS@ $(BOOST_THREAD_LDFLAGS) $(BOOST_THREAD_LIBS)
queryLexicalTable_LDADD = $(top_builddir)/moses/src/libmoses.la $(BOOST_LDFLAGS) $(BOOST_THREAD_LIB) -L$(top_srcdir)/moses/src -L$(top_srcdir)/OnDiskPt/src -lmoses -lOnDiskPt
queryLexicalTable_LDADD = $(top_builddir)/moses/src/libmoses.la -L$(top_srcdir)/moses/src -L$(top_srcdir)/OnDiskPt/src -lmoses -lOnDiskPt @KENLM_LDFLAGS@ $(BOOST_THREAD_LDFLAGS) $(BOOST_THREAD_LIBS)
queryPhraseTable_LDADD = $(top_builddir)/moses/src/libmoses.la $(BOOST_LDFLAGS) $(BOOST_THREAD_LIB) -L$(top_srcdir)/moses/src -L$(top_srcdir)/OnDiskPt/src -lmoses -lOnDiskPt
queryPhraseTable_LDADD = $(top_builddir)/moses/src/libmoses.la -L$(top_srcdir)/moses/src -L$(top_srcdir)/OnDiskPt/src -lmoses -lOnDiskPt @KENLM_LDFLAGS@ $(BOOST_THREAD_LDFLAGS) $(BOOST_THREAD_LIBS)

4
moses-chart-cmd/src/Makefile.am
View File

@ -2,8 +2,8 @@ bin_PROGRAMS = moses_chart
moses_chart_SOURCES = Main.cpp mbr.cpp IOWrapper.cpp TranslationAnalysis.cpp
AM_CPPFLAGS = -W -Wall -ffor-scope -D_FILE_OFFSET_BITS=64 -D_LARGE_FILES -DUSE_HYPO_POOL -I$(top_srcdir)/moses/src $(BOOST_CPPFLAGS)
moses_chart_LDADD = -L$(top_srcdir)/moses/src -L$(top_srcdir)/moses-chart/src -L$(top_srcdir)/OnDiskPt/src -lmoses -lmoses-chart -lOnDiskPt $(BOOST_LDFLAGS) $(BOOST_THREAD_LIB)
moses_chart_DEPENDENCIES = $(top_srcdir)/moses/src/libmoses.la $(top_srcdir)/moses-chart/src/libmoses-chart.a $(top_srcdir)/OnDiskPt/src/libOnDiskPt.a
moses_chart_LDADD = -L$(top_srcdir)/moses/src -L$(top_srcdir)/moses-chart/src -L$(top_srcdir)/OnDiskPt/src -lmoses -lmoses-chart -lOnDiskPt @KENLM_LDFLAGS@ $(BOOST_THREAD_LDFLAGS) $(BOOST_THREAD_LIBS)
moses_chart_DEPENDENCIES = $(top_srcdir)/moses/src/libmoses.la $(top_srcdir)/moses-chart/src/libmoses-chart.a $(top_srcdir)/OnDiskPt/src/libOnDiskPt.a @KENLM_DEPS@

3
moses-chart/src/ChartCell.cpp
View File

@ -114,11 +114,10 @@ void ChartCell::SortHypotheses()
{
// sort each mini cells & fill up target lhs list
assert(m_headWords.empty());
m_headWords.reserve(m_hypoColl.size());
std::map<Moses::Word, HypothesisCollection>::iterator iter;
for (iter = m_hypoColl.begin(); iter != m_hypoColl.end(); ++iter)
{
m_headWords.push_back(iter->first);
m_headWords.insert(iter->first);
HypothesisCollection &coll = iter->second;
coll.SortHypotheses();

5
moses-chart/src/ChartCell.h
View File

@ -27,6 +27,7 @@
#include <vector>
#include "../../moses/src/Word.h"
#include "../../moses/src/WordsRange.h"
#include "../../moses/src/NonTerminal.h"
#include "ChartHypothesis.h"
#include "ChartHypothesisCollection.h"
#include "QueueEntry.h"
@ -52,7 +53,7 @@ public:
protected:
std::map<Moses::Word, HypothesisCollection> m_hypoColl;
std::vector<Moses::Word> m_headWords;
Moses::NonTerminalSet m_headWords;
Moses::WordsRange m_coverage;
@ -74,7 +75,7 @@ public:
const Hypothesis *GetBestHypothesis() const;
bool HeadwordExists(const Moses::Word &headWord) const;
const std::vector<Moses::Word> &GetHeadwords() const
const Moses::NonTerminalSet &GetHeadwords() const
{ return m_headWords; }
void CleanupArcList();

2
moses-chart/src/ChartCellCollection.h
View File

@ -55,7 +55,7 @@ public:
return *m_hypoStackColl[coverage.GetStartPos()][coverage.GetEndPos() - coverage.GetStartPos()];
}
const std::vector<Moses::Word> &GetHeadwords(const Moses::WordsRange &coverage) const
const Moses::NonTerminalSet &GetHeadwords(const Moses::WordsRange &coverage) const
{
const ChartCell &cell = Get(coverage);
return cell.GetHeadwords();

15
moses-chart/src/ChartManager.cpp
View File

@ -43,17 +43,28 @@ namespace MosesChart
Manager::Manager(InputType const& source, const TranslationSystem* system)
:m_source(source)
,m_hypoStackColl(source, *this)
,m_transOptColl(source, system, m_hypoStackColl)
,m_transOptColl(source, system, m_hypoStackColl, m_ruleLookupManagers)
,m_system(system)
,m_start(clock())
{
m_system->InitializeBeforeSentenceProcessing(source);
const std::vector<PhraseDictionaryFeature*> &dictionaries = m_system->GetPhraseDictionaries();
m_ruleLookupManagers.reserve(dictionaries.size());
for (std::vector<PhraseDictionaryFeature*>::const_iterator p = dictionaries.begin();
p != dictionaries.end(); ++p)
{
PhraseDictionaryFeature *pdf = *p;
const PhraseDictionary *dict = pdf->GetDictionary();
PhraseDictionary *nonConstDict = const_cast<PhraseDictionary*>(dict);
m_ruleLookupManagers.push_back(nonConstDict->CreateRuleLookupManager(source, m_hypoStackColl));
}
}
Manager::~Manager()
{
m_system->CleanUpAfterSentenceProcessing();
RemoveAllInColl(m_ruleLookupManagers);
clock_t end = clock();
float et = (end - m_start);

2
moses-chart/src/ChartManager.h
View File

@ -30,6 +30,7 @@
#include "../../moses/src/TrellisPathList.h"
#include "../../moses/src/SentenceStats.h"
#include "../../moses/src/TranslationSystem.h"
#include "../../moses/src/ChartRuleLookupManager.h"
namespace MosesChart
{
@ -46,6 +47,7 @@ protected:
std::auto_ptr<Moses::SentenceStats> m_sentenceStats;
const Moses::TranslationSystem* m_system;
clock_t m_start; /**< starting time, used for logging */
std::vector<Moses::ChartRuleLookupManager*> m_ruleLookupManagers;
public:
Manager(Moses::InputType const& source, const Moses::TranslationSystem* system);

50
moses-chart/src/ChartTranslationOptionCollection.cpp
View File

@ -37,12 +37,14 @@ namespace MosesChart
TranslationOptionCollection::TranslationOptionCollection(InputType const& source
, const Moses::TranslationSystem* system
, const ChartCellCollection &hypoStackColl)
, const ChartCellCollection &hypoStackColl
, const std::vector<ChartRuleLookupManager*> &ruleLookupManagers)
:m_source(source)
,m_system(system)
,m_decodeGraphList(system->GetDecodeGraphs())
,m_hypoStackColl(hypoStackColl)
,m_collection(source.GetSize())
,m_ruleLookupManagers(ruleLookupManagers)
{
// create 2-d vector
size_t size = source.GetSize();
@ -76,14 +78,21 @@ void TranslationOptionCollection::CreateTranslationOptionsForRange(
size_t startPos
, size_t endPos)
{
ChartTranslationOptionList &chartRuleColl = GetTranslationOptionList(startPos, endPos);
const WordsRange &wordsRange = chartRuleColl.GetSourceRange();
assert(m_decodeGraphList.size() == m_ruleLookupManagers.size());
std::vector <DecodeGraph*>::const_iterator iterDecodeGraph;
for (iterDecodeGraph = m_decodeGraphList.begin(); iterDecodeGraph != m_decodeGraphList.end(); ++iterDecodeGraph)
std::vector <ChartRuleLookupManager*>::const_iterator iterRuleLookupManagers = m_ruleLookupManagers.begin();
for (iterDecodeGraph = m_decodeGraphList.begin(); iterDecodeGraph != m_decodeGraphList.end(); ++iterDecodeGraph, ++iterRuleLookupManagers)
{
const DecodeGraph &decodeGraph = **iterDecodeGraph;
assert(decodeGraph.GetSize() == 1);
ChartRuleLookupManager &ruleLookupManager = **iterRuleLookupManagers;
size_t maxSpan = decodeGraph.GetMaxChartSpan();
if (maxSpan == 0 || (endPos-startPos+1) <= maxSpan)
{
CreateTranslationOptionsForRange(decodeGraph, startPos, endPos, true);
ruleLookupManager.GetChartRuleCollection(wordsRange, true, chartRuleColl);
}
}
@ -104,18 +113,22 @@ void TranslationOptionCollection::ProcessUnknownWord(size_t startPos, size_t end
}
ChartTranslationOptionList &fullList = GetTranslationOptionList(startPos, startPos);
const WordsRange &wordsRange = fullList.GetSourceRange();
// try to translation for coverage with no trans by expanding table limit
std::vector <DecodeGraph*>::const_iterator iterDecodeGraph;
for (iterDecodeGraph = m_decodeGraphList.begin(); iterDecodeGraph != m_decodeGraphList.end(); ++iterDecodeGraph)
std::vector <ChartRuleLookupManager*>::const_iterator iterRuleLookupManagers = m_ruleLookupManagers.begin();
for (iterDecodeGraph = m_decodeGraphList.begin(); iterDecodeGraph != m_decodeGraphList.end(); ++iterDecodeGraph, ++iterRuleLookupManagers)
{
const DecodeGraph &decodeGraph = **iterDecodeGraph;
ChartRuleLookupManager &ruleLookupManager = **iterRuleLookupManagers;
size_t numTransOpt = fullList.GetSize();
if (numTransOpt == 0)
{
CreateTranslationOptionsForRange(decodeGraph, startPos, startPos, false);
ruleLookupManager.GetChartRuleCollection(wordsRange, false, fullList);
}
}
assert(iterRuleLookupManagers == m_ruleLookupManagers.end());
bool alwaysCreateDirectTranslationOption = StaticData::Instance().IsAlwaysCreateDirectTranslationOption();
// create unknown words for 1 word coverage where we don't have any trans options
@ -124,33 +137,6 @@ void TranslationOptionCollection::ProcessUnknownWord(size_t startPos, size_t end
}
void TranslationOptionCollection::CreateTranslationOptionsForRange(
const DecodeGraph &decodeGraph
, size_t startPos
, size_t endPos
, bool adhereTableLimit)
{
assert(decodeGraph.GetSize() == 1);
const DecodeStep &decodeStep = **decodeGraph.begin();
// get wordsrange that doesn't go away until after sentence processing
const WordsRange &wordsRange = GetTranslationOptionList(startPos, endPos).GetSourceRange();
ChartTranslationOptionList &translationOptionList = GetTranslationOptionList(startPos, endPos);
const PhraseDictionary* phraseDictionary =
decodeStep.GetPhraseDictionaryFeature()->GetDictionary();
//cerr << phraseDictionary.GetScoreProducerDescription() << endl;
ChartTranslationOptionList &chartRuleCollection = GetTranslationOptionList(startPos, endPos);
phraseDictionary->GetChartRuleCollection(chartRuleCollection
, m_source
, wordsRange
, adhereTableLimit
, m_hypoStackColl);
//cerr << "chartRuleCollection size=" << chartRuleCollection->GetSize();
}
ChartTranslationOptionList &TranslationOptionCollection::GetTranslationOptionList(size_t startPos, size_t endPos)
{
size_t sizeVec = m_collection[startPos].size();

11
moses-chart/src/ChartTranslationOptionCollection.h
View File

@ -25,6 +25,7 @@
#include "../../moses/src/InputType.h"
#include "../../moses/src/DecodeGraph.h"
#include "../../moses/src/ChartTranslationOptionList.h"
#include "../../moses/src/ChartRuleLookupManager.h"
namespace Moses
{
@ -47,17 +48,13 @@ protected:
const Moses::TranslationSystem* m_system;
std::vector <Moses::DecodeGraph*> m_decodeGraphList;
const ChartCellCollection &m_hypoStackColl;
const std::vector<Moses::ChartRuleLookupManager*> &m_ruleLookupManagers;
std::vector< std::vector< Moses::ChartTranslationOptionList > > m_collection; /*< contains translation options */
std::vector<Moses::Phrase*> m_unksrcs;
std::list<Moses::TargetPhrase*> m_cacheTargetPhrase;
std::list<std::vector<Moses::WordConsumed*>* > m_cachedWordsConsumed;
virtual void CreateTranslationOptionsForRange(const Moses::DecodeGraph& decodeGraph,
size_t startPosition
, size_t endPosition
, bool adhereTableLimit);
// for adding 1 trans opt in unknown word proc
void Add(Moses::ChartTranslationOption *transOpt, size_t pos);
@ -82,9 +79,9 @@ protected:
public:
TranslationOptionCollection(Moses::InputType const& source
, const Moses::TranslationSystem* system
, const ChartCellCollection &hypoStackColl);
, const ChartCellCollection &hypoStackColl
, const std::vector<Moses::ChartRuleLookupManager*> &ruleLookupManagers);
virtual ~TranslationOptionCollection();
//virtual void CreateTranslationOptions(const std::vector <Moses::DecodeGraph*> &decodeGraphList);
void CreateTranslationOptionsForRange(size_t startPos
, size_t endPos);

6
moses-cmd/src/Makefile.am
View File

@ -3,11 +3,11 @@ bin_PROGRAMS = moses lmbrgrid checkplf
AM_CPPFLAGS = -W -Wall -ffor-scope -D_FILE_OFFSET_BITS=64 -D_LARGE_FILES -DUSE_HYPO_POOL -I$(top_srcdir)/moses/src $(BOOST_CPPFLAGS)
checkplf_SOURCES = checkplf.cpp
checkplf_LDADD = $(top_builddir)/moses/src/libmoses.la
checkplf_LDADD = $(top_builddir)/moses/src/libmoses.la -L$(top_srcdir)/OnDiskPt/src -lOnDiskPt @KENLM_LDFLAGS@ $(BOOST_THREAD_LDFLAGS) $(BOOST_THREAD_LIBS)
moses_SOURCES = Main.cpp mbr.cpp IOWrapper.cpp TranslationAnalysis.cpp LatticeMBR.cpp ThreadPool.cpp
moses_LDADD = $(top_builddir)/moses/src/libmoses.la $(BOOST_LDFLAGS) $(BOOST_THREAD_LIB) -L$(top_srcdir)/OnDiskPt/src -lOnDiskPt
moses_LDADD = $(top_builddir)/moses/src/libmoses.la -L$(top_srcdir)/OnDiskPt/src -lOnDiskPt @KENLM_LDFLAGS@ $(BOOST_THREAD_LDFLAGS) $(BOOST_THREAD_LIBS)
lmbrgrid_SOURCES = LatticeMBRGrid.cpp LatticeMBR.cpp IOWrapper.cpp
lmbrgrid_LDADD = $(top_builddir)/moses/src/libmoses.la $(BOOST_LDFLAGS) $(BOOST_THREAD_LIB) -L$(top_srcdir)/OnDiskPt/src -lOnDiskPt
lmbrgrid_LDADD = $(top_builddir)/moses/src/libmoses.la -L$(top_srcdir)/OnDiskPt/src -lOnDiskPt @KENLM_LDFLAGS@ $(BOOST_THREAD_LDFLAGS) $(BOOST_THREAD_LIBS)

8
moses/moses.vcproj
View File

@ -407,6 +407,10 @@
RelativePath=".\src\LanguageModelFactory.cpp"
>
</File>
<File
RelativePath=".\src\LanguageModelImplementation.cpp"
>
</File>
<File
RelativePath=".\src\LanguageModelInternal.cpp"
>
@ -885,6 +889,10 @@
RelativePath=".\src\LanguageModelFactory.h"
>
</File>
<File
RelativePath=".\src\LanguageModelImplementation.h"
>
</File>
<File
RelativePath=".\src\LanguageModelInternal.h"
>

7
moses/src/CellCollection.h
View File

@ -21,6 +21,10 @@
#include <vector>
#include "NonTerminal.h"
#include "Word.h"
#include "WordsRange.h"
namespace Moses
{
class Word;
@ -30,8 +34,7 @@ class CellCollection
public:
virtual ~CellCollection()
{}
virtual const std::vector<Word> &GetHeadwords(const Moses::WordsRange &coverage) const = 0;
virtual const NonTerminalSet &GetHeadwords(const Moses::WordsRange &coverage) const = 0;
};
}

67
moses/src/ChartRuleLookupManager.h Normal file
View File

@ -0,0 +1,67 @@
/***********************************************************************
Moses - factored phrase-based language decoder
Copyright (C) 2011 University of Edinburgh
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
***********************************************************************/
#pragma once
#ifndef moses_ChartRuleLookupManager_h
#define moses_ChartRuleLookupManager_h
#include "CellCollection.h"
#include "InputType.h"
namespace Moses
{
class ChartTranslationOptionList;
class WordsRange;
// Defines an interface for looking up rules in a rule table. Concrete
// implementation classes should correspond to specific PhraseDictionary
// subclasses (memory or on-disk). Since a ChartRuleLookupManager object
// maintains sentence-specific state, exactly one should be created for
// each sentence that is to be decoded.
class ChartRuleLookupManager
{
public:
ChartRuleLookupManager(const InputType &sentence,
const CellCollection &cellColl)
: m_sentence(sentence)
, m_cellCollection(cellColl) {}
virtual ~ChartRuleLookupManager() {}
const InputType &GetSentence() const { return m_sentence; }
const CellCollection &GetCellCollection() const { return m_cellCollection; }
virtual void GetChartRuleCollection(
const WordsRange &range,
bool adhereTableLimit,
ChartTranslationOptionList &outColl) = 0;
private:
// Non-copyable: copy constructor and assignment operator not implemented.
ChartRuleLookupManager(const ChartRuleLookupManager &);
ChartRuleLookupManager &operator=(const ChartRuleLookupManager &);
const InputType &m_sentence;
const CellCollection &m_cellCollection;
};
} // namespace Moses
#endif

233
moses/src/ChartRuleLookupManagerMemory.cpp Normal file
View File

@ -0,0 +1,233 @@
/***********************************************************************
Moses - factored phrase-based language decoder
Copyright (C) 2011 University of Edinburgh
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
***********************************************************************/
#include "ChartRuleLookupManagerMemory.h"
#include "PhraseDictionarySCFG.h"
#include "InputType.h"
#include "ChartTranslationOptionList.h"
#include "CellCollection.h"
#include "DotChart.h"
#include "StaticData.h"
#include "NonTerminal.h"
namespace Moses
{
ChartRuleLookupManagerMemory::ChartRuleLookupManagerMemory(
const InputType &src,
const CellCollection &cellColl,
const PhraseDictionarySCFG &ruleTable)
: ChartRuleLookupManager(src, cellColl)
, m_ruleTable(ruleTable)
{
assert(m_processedRuleColls.size() == 0);
size_t sourceSize = src.GetSize();
m_processedRuleColls.resize(sourceSize);
const PhraseDictionaryNodeSCFG &rootNode = m_ruleTable.GetRootNode();
for (size_t ind = 0; ind < m_processedRuleColls.size(); ++ind)
{
ProcessedRule *initProcessedRule = new ProcessedRule(rootNode);
ProcessedRuleColl *processedRuleColl = new ProcessedRuleColl(sourceSize - ind + 1);
processedRuleColl->Add(0, initProcessedRule); // init rule. stores the top node in tree
m_processedRuleColls[ind] = processedRuleColl;
}
}
ChartRuleLookupManagerMemory::~ChartRuleLookupManagerMemory()
{
RemoveAllInColl(m_processedRuleColls);
}
void ChartRuleLookupManagerMemory::GetChartRuleCollection(
const WordsRange &range,
bool adhereTableLimit,
ChartTranslationOptionList &outColl)
{
size_t relEndPos = range.GetEndPos() - range.GetStartPos();
size_t absEndPos = range.GetEndPos();
// MAIN LOOP. create list of nodes of target phrases
ProcessedRuleColl &processedRuleCol = *m_processedRuleColls[range.GetStartPos()];
const ProcessedRuleList &runningNodes = processedRuleCol.GetRunningNodes();
// Note that runningNodes can be expanded as the loop runs (through calls to
// ExtendPartialRuleApplication()).
for (size_t ind = 0; ind < runningNodes.size(); ++ind)
{
const ProcessedRule &prevProcessedRule = *runningNodes[ind];
const PhraseDictionaryNodeSCFG &prevNode = prevProcessedRule.GetLastNode();
const WordConsumed *prevWordConsumed = prevProcessedRule.GetLastWordConsumed();
size_t startPos = (prevWordConsumed == NULL) ? range.GetStartPos() : prevWordConsumed->GetWordsRange().GetEndPos() + 1;
// search for terminal symbol
if (startPos == absEndPos)
{
const Word &sourceWord = GetSentence().GetWord(absEndPos);
const PhraseDictionaryNodeSCFG *node = prevNode.GetChild(sourceWord);
if (node != NULL)
{
WordConsumed *newWordConsumed = new WordConsumed(absEndPos, absEndPos
, sourceWord
, prevWordConsumed);
ProcessedRule *processedRule = new ProcessedRule(*node, newWordConsumed);
processedRuleCol.Add(relEndPos+1, processedRule);
}
}
// search for non-terminals
size_t endPos, stackInd;
if (startPos > absEndPos)
continue;
else if (startPos == range.GetStartPos() && range.GetEndPos() > range.GetStartPos())
{ // start.
endPos = absEndPos - 1;
stackInd = relEndPos;
}
else
{
endPos = absEndPos;
stackInd = relEndPos + 1;
}
const NonTerminalSet &sourceNonTerms =
GetSentence().GetLabelSet(startPos, endPos);
const NonTerminalSet &targetNonTerms =
GetCellCollection().GetHeadwords(WordsRange(startPos, endPos));
ExtendPartialRuleApplication(prevNode, prevWordConsumed, startPos,
endPos, stackInd, sourceNonTerms,
targetNonTerms, processedRuleCol);
}
// return list of target phrases
ProcessedRuleList &nodes = processedRuleCol.Get(relEndPos + 1);
size_t rulesLimit = StaticData::Instance().GetRuleLimit();
ProcessedRuleList::const_iterator iterNode;
for (iterNode = nodes.begin(); iterNode != nodes.end(); ++iterNode)
{
const ProcessedRule &processedRule = **iterNode;
const PhraseDictionaryNodeSCFG &node = processedRule.GetLastNode();
const WordConsumed *wordConsumed = processedRule.GetLastWordConsumed();
assert(wordConsumed);
const TargetPhraseCollection *targetPhraseCollection = node.GetTargetPhraseCollection();
if (targetPhraseCollection != NULL)
{
outColl.Add(*targetPhraseCollection, *wordConsumed, adhereTableLimit, rulesLimit);
}
}
outColl.CreateChartRules(rulesLimit);
}
// Given a partial rule application ending at startPos-1 and given the sets of
// source and target non-terminals covering the span [startPos, endPos],
// determines the full or partial rule applications that can be produced through
// extending the current rule application by a single non-terminal.
void ChartRuleLookupManagerMemory::ExtendPartialRuleApplication(
const PhraseDictionaryNodeSCFG & node,
const WordConsumed *prevWordConsumed,
size_t startPos,
size_t endPos,
size_t stackInd,
const NonTerminalSet & sourceNonTerms,
const NonTerminalSet & targetNonTerms,
ProcessedRuleColl & processedRuleColl)
{
const PhraseDictionaryNodeSCFG::NonTerminalMap & nonTermMap =
node.GetNonTerminalMap();
const size_t numChildren = nonTermMap.size();
if (numChildren == 0)
{
return;
}
const size_t numSourceNonTerms = sourceNonTerms.size();
const size_t numTargetNonTerms = targetNonTerms.size();
const size_t numCombinations = numSourceNonTerms * numTargetNonTerms;
// We can search by either:
// 1. Enumerating all possible source-target NT pairs that are valid for
// the span and then searching for matching children in the node,
// or
// 2. Iterating over all the NT children in the node, searching
// for each source and target NT in the span's sets.
// We'll do whichever minimises the number of lookups:
if (numCombinations <= numChildren*2)
{
NonTerminalSet::const_iterator p = sourceNonTerms.begin();
NonTerminalSet::const_iterator sEnd = sourceNonTerms.end();
for (; p != sEnd; ++p)
{
const Word & sourceNonTerm = *p;
NonTerminalSet::const_iterator q = targetNonTerms.begin();
NonTerminalSet::const_iterator tEnd = targetNonTerms.end();
for (; q != tEnd; ++q)
{
const Word & targetNonTerm = *q;
const PhraseDictionaryNodeSCFG * child =
node.GetChild(sourceNonTerm, targetNonTerm);
if (child == NULL)
{
continue;
}
WordConsumed * wc = new WordConsumed(startPos, endPos,
targetNonTerm,
prevWordConsumed);
ProcessedRule * rule = new ProcessedRule(*child, wc);
processedRuleColl.Add(stackInd, rule);
}
}
}
else
{
PhraseDictionaryNodeSCFG::NonTerminalMap::const_iterator p;
PhraseDictionaryNodeSCFG::NonTerminalMap::const_iterator end =
nonTermMap.end();
for (p = nonTermMap.begin(); p != end; ++p)
{
const PhraseDictionaryNodeSCFG::NonTerminalMapKey & key = p->first;
const Word & sourceNonTerm = key.first;
if (sourceNonTerms.find(sourceNonTerm) == sourceNonTerms.end())
{
continue;
}
const Word & targetNonTerm = key.second;
if (targetNonTerms.find(targetNonTerm) == targetNonTerms.end())
{
continue;
}
const PhraseDictionaryNodeSCFG & child = p->second;
WordConsumed * wc = new WordConsumed(startPos, endPos,
targetNonTerm,
prevWordConsumed);
ProcessedRule * rule = new ProcessedRule(child, wc);
processedRuleColl.Add(stackInd, rule);
}
}
}
} // namespace Moses

71
moses/src/ChartRuleLookupManagerMemory.h Normal file
View File

@ -0,0 +1,71 @@
/***********************************************************************
Moses - factored phrase-based language decoder
Copyright (C) 2011 University of Edinburgh
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
***********************************************************************/
#pragma once
#ifndef moses_ChartRuleLookupManagerMemory_h
#define moses_ChartRuleLookupManagerMemory_h
#include <vector>
#include "ChartRuleLookupManager.h"
#include "NonTerminal.h"
#include "PhraseDictionaryNodeSCFG.h"
#include "PhraseDictionarySCFG.h"
namespace Moses
{
class ChartTranslationOptionList;
class ProcessedRuleColl;
class WordsRange;
class WordConsumed;
// Implementation of ChartRuleLookupManager for in-memory rule tables.
class ChartRuleLookupManagerMemory : public ChartRuleLookupManager
{
public:
ChartRuleLookupManagerMemory(const InputType &sentence,
const CellCollection &cellColl,
const PhraseDictionarySCFG &ruleTable);
~ChartRuleLookupManagerMemory();
virtual void GetChartRuleCollection(
const WordsRange &range,
bool adhereTableLimit,
ChartTranslationOptionList &outColl);
private:
void ExtendPartialRuleApplication(
const PhraseDictionaryNodeSCFG &node,
const WordConsumed *prevWordConsumed,
size_t startPos,
size_t endPos,
size_t stackInd,
const NonTerminalSet &sourceNonTerms,
const NonTerminalSet &targetNonTerms,
ProcessedRuleColl &processedRuleColl);
std::vector<ProcessedRuleColl*> m_processedRuleColls;
const PhraseDictionarySCFG &m_ruleTable;
};
} // namespace Moses
#endif

311
moses/src/ChartRuleLookupManagerOnDisk.cpp Normal file
View File

@ -0,0 +1,311 @@
/***********************************************************************
Moses - factored phrase-based language decoder
Copyright (C) 2011 University of Edinburgh
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
***********************************************************************/
#include "ChartRuleLookupManagerOnDisk.h"
#include <algorithm>
#include "PhraseDictionaryOnDisk.h"
#include "StaticData.h"
#include "DotChartOnDisk.h"
#include "CellCollection.h"
#include "ChartTranslationOptionList.h"
#include "../../OnDiskPt/src/TargetPhraseCollection.h"
using namespace std;
namespace Moses
{
ChartRuleLookupManagerOnDisk::ChartRuleLookupManagerOnDisk(
const InputType &sentence,
const CellCollection &cellColl,
const PhraseDictionaryOnDisk &dictionary,
OnDiskPt::OnDiskWrapper &dbWrapper,
const LMList *languageModels,
const WordPenaltyProducer *wpProducer,
const std::vector<FactorType> &inputFactorsVec,
const std::vector<FactorType> &outputFactorsVec,
const std::vector<float> &weight,
const std::string &filePath)
: ChartRuleLookupManager(sentence, cellColl)
, m_dictionary(dictionary)
, m_dbWrapper(dbWrapper)
, m_languageModels(languageModels)
, m_wpProducer(wpProducer)
, m_inputFactorsVec(inputFactorsVec)
, m_outputFactorsVec(outputFactorsVec)
, m_weight(weight)
, m_filePath(filePath)
{
assert(m_runningNodesVec.size() == 0);
size_t sourceSize = sentence.GetSize();
m_runningNodesVec.resize(sourceSize);
for (size_t ind = 0; ind < m_runningNodesVec.size(); ++ind)
{
ProcessedRuleOnDisk *initProcessedRule = new ProcessedRuleOnDisk(m_dbWrapper.GetRootSourceNode());
ProcessedRuleStackOnDisk *processedStack = new ProcessedRuleStackOnDisk(sourceSize - ind + 1);
processedStack->Add(0, initProcessedRule); // init rule. stores the top node in tree
m_runningNodesVec[ind] = processedStack;
}
}
ChartRuleLookupManagerOnDisk::~ChartRuleLookupManagerOnDisk()
{
std::map<UINT64, const TargetPhraseCollection*>::const_iterator iterCache;
for (iterCache = m_cache.begin(); iterCache != m_cache.end(); ++iterCache)
{
delete iterCache->second;
}
m_cache.clear();
RemoveAllInColl(m_runningNodesVec);
RemoveAllInColl(m_sourcePhraseNode);
}
void ChartRuleLookupManagerOnDisk::GetChartRuleCollection(
const WordsRange &range,
bool adhereTableLimit,
ChartTranslationOptionList &outColl)
{
const StaticData &staticData = StaticData::Instance();
size_t rulesLimit = StaticData::Instance().GetRuleLimit();
size_t relEndPos = range.GetEndPos() - range.GetStartPos();
size_t absEndPos = range.GetEndPos();
// MAIN LOOP. create list of nodes of target phrases
ProcessedRuleStackOnDisk &runningNodes = *m_runningNodesVec[range.GetStartPos()];
// sort save nodes so only do nodes with most counts
runningNodes.SortSavedNodes();
size_t numDerivations = 0
,maxDerivations = 999999; // staticData.GetMaxDerivations();
bool overThreshold = true;
const ProcessedRuleStackOnDisk::SavedNodeColl &savedNodeColl = runningNodes.GetSavedNodeColl();
//cerr << "savedNodeColl=" << savedNodeColl.size() << " ";
for (size_t ind = 0; ind < (savedNodeColl.size()) && ((numDerivations < maxDerivations) || overThreshold) ; ++ind)
{
const SavedNodeOnDisk &savedNode = *savedNodeColl[ind];
const ProcessedRuleOnDisk &prevProcessedRule = savedNode.GetProcessedRule();
const OnDiskPt::PhraseNode &prevNode = prevProcessedRule.GetLastNode();
const WordConsumed *prevWordConsumed = prevProcessedRule.GetLastWordConsumed();
size_t startPos = (prevWordConsumed == NULL) ? range.GetStartPos() : prevWordConsumed->GetWordsRange().GetEndPos() + 1;
// search for terminal symbol
if (startPos == absEndPos)
{
const Word &sourceWord = GetSentence().GetWord(absEndPos);
OnDiskPt::Word *sourceWordBerkeleyDb = m_dbWrapper.ConvertFromMoses(Input, m_inputFactorsVec, sourceWord);
if (sourceWordBerkeleyDb != NULL)
{
const OnDiskPt::PhraseNode *node = prevNode.GetChild(*sourceWordBerkeleyDb, m_dbWrapper);
if (node != NULL)
{
// TODO figure out why source word is needed from node, not from sentence
// prob to do with factors or non-term
//const Word &sourceWord = node->GetSourceWord();
WordConsumed *newWordConsumed = new WordConsumed(absEndPos, absEndPos
, sourceWord
, prevWordConsumed);
ProcessedRuleOnDisk *processedRule = new ProcessedRuleOnDisk(*node, newWordConsumed);
runningNodes.Add(relEndPos+1, processedRule);
// cache for cleanup
m_sourcePhraseNode.push_back(node);
}
delete sourceWordBerkeleyDb;
}
}
// search for non-terminals
size_t endPos, stackInd;
if (startPos > absEndPos)
continue;
else if (startPos == range.GetStartPos() && range.GetEndPos() > range.GetStartPos())
{ // start.
endPos = absEndPos - 1;
stackInd = relEndPos;
}
else
{
endPos = absEndPos;
stackInd = relEndPos + 1;
}
size_t nonTermNumWordsCovered = endPos - startPos + 1;
// get target headwords in this span from chart
const NonTerminalSet &headWords = GetCellCollection().GetHeadwords(WordsRange(startPos, endPos));
const Word &defaultSourceNonTerm = staticData.GetInputDefaultNonTerminal()
,&defaultTargetNonTerm = staticData.GetOutputDefaultNonTerminal();
// go through each SOURCE lhs
const NonTerminalSet &sourceLHSSet = GetSentence().GetLabelSet(startPos, endPos);
NonTerminalSet::const_iterator iterSourceLHS;
for (iterSourceLHS = sourceLHSSet.begin(); iterSourceLHS != sourceLHSSet.end(); ++iterSourceLHS)
{
const Word &sourceLHS = *iterSourceLHS;
OnDiskPt::Word *sourceLHSBerkeleyDb = m_dbWrapper.ConvertFromMoses(Input, m_inputFactorsVec, sourceLHS);
if (sourceLHSBerkeleyDb == NULL)
{
delete sourceLHSBerkeleyDb;
continue; // vocab not in pt. node definately won't be in there
}
const OnDiskPt::PhraseNode *sourceNode = prevNode.GetChild(*sourceLHSBerkeleyDb, m_dbWrapper);
delete sourceLHSBerkeleyDb;
if (sourceNode == NULL)
continue; // didn't find source node
// go through each TARGET lhs
NonTerminalSet::const_iterator iterTargetLHS;
for (iterTargetLHS = headWords.begin(); iterTargetLHS != headWords.end(); ++iterTargetLHS)
{
const Word &targetLHS = *iterTargetLHS;
//cerr << sourceLHS << " " << defaultSourceNonTerm << " " << targetLHS << " " << defaultTargetNonTerm << endl;
//bool isSyntaxNonTerm = (sourceLHS != defaultSourceNonTerm) || (targetLHS != defaultTargetNonTerm);
bool doSearch = true; //isSyntaxNonTerm ? nonTermNumWordsCovered <= maxSyntaxSpan :
// nonTermNumWordsCovered <= maxDefaultSpan;
if (doSearch)
{
OnDiskPt::Word *targetLHSBerkeleyDb = m_dbWrapper.ConvertFromMoses(Output, m_outputFactorsVec, targetLHS);
if (targetLHSBerkeleyDb == NULL)
continue;
const OnDiskPt::PhraseNode *node = sourceNode->GetChild(*targetLHSBerkeleyDb, m_dbWrapper);
delete targetLHSBerkeleyDb;
if (node == NULL)
continue;
// found matching entry
//const Word &sourceWord = node->GetSourceWord();
WordConsumed *newWordConsumed = new WordConsumed(startPos, endPos
, targetLHS
, prevWordConsumed);
ProcessedRuleOnDisk *processedRule = new ProcessedRuleOnDisk(*node, newWordConsumed);
runningNodes.Add(stackInd, processedRule);
m_sourcePhraseNode.push_back(node);
}
} // for (iterHeadWords
delete sourceNode;
} // for (iterLabelListf
// return list of target phrases
ProcessedRuleCollOnDisk &nodes = runningNodes.Get(relEndPos + 1);
// source LHS
ProcessedRuleCollOnDisk::const_iterator iterProcessedRuleColl;
for (iterProcessedRuleColl = nodes.begin(); iterProcessedRuleColl != nodes.end(); ++iterProcessedRuleColl)
{
// node of last source word
const ProcessedRuleOnDisk &prevProcessedRule = **iterProcessedRuleColl;
if (prevProcessedRule.Done())
continue;
prevProcessedRule.Done(true);
const WordConsumed *wordConsumed = prevProcessedRule.GetLastWordConsumed();
assert(wordConsumed);
const OnDiskPt::PhraseNode &prevNode = prevProcessedRule.GetLastNode();
//get node for each source LHS
const NonTerminalSet &lhsSet = GetSentence().GetLabelSet(range.GetStartPos(), range.GetEndPos());
NonTerminalSet::const_iterator iterLabelSet;
for (iterLabelSet = lhsSet.begin(); iterLabelSet != lhsSet.end(); ++iterLabelSet)
{
const Word &sourceLHS = *iterLabelSet;
OnDiskPt::Word *sourceLHSBerkeleyDb = m_dbWrapper.ConvertFromMoses(Input, m_inputFactorsVec, sourceLHS);
if (sourceLHSBerkeleyDb == NULL)
continue;
const TargetPhraseCollection *targetPhraseCollection = NULL;
const OnDiskPt::PhraseNode *node = prevNode.GetChild(*sourceLHSBerkeleyDb, m_dbWrapper);
if (node)
{
UINT64 tpCollFilePos = node->GetValue();
std::map<UINT64, const TargetPhraseCollection*>::const_iterator iterCache = m_cache.find(tpCollFilePos);
if (iterCache == m_cache.end())
{ // not in case
overThreshold = node->GetCount(0) > staticData.GetRuleCountThreshold();
//cerr << node->GetCount(0) << " ";
const OnDiskPt::TargetPhraseCollection *tpcollBerkeleyDb = node->GetTargetPhraseCollection(m_dictionary.GetTableLimit(), m_dbWrapper);
targetPhraseCollection
= tpcollBerkeleyDb->ConvertToMoses(m_inputFactorsVec
,m_outputFactorsVec
,m_dictionary
,m_weight
,m_wpProducer
,*m_languageModels
,m_filePath
, m_dbWrapper.GetVocab());
delete tpcollBerkeleyDb;
m_cache[tpCollFilePos] = targetPhraseCollection;
}
else
{ // jsut get out of cache
targetPhraseCollection = iterCache->second;
}
assert(targetPhraseCollection);
outColl.Add(*targetPhraseCollection, *wordConsumed, adhereTableLimit, rulesLimit);
numDerivations++;
} // if (node)
delete node;
delete sourceLHSBerkeleyDb;
}
}
} // for (size_t ind = 0; ind < savedNodeColl.size(); ++ind)
outColl.CreateChartRules(rulesLimit);
//cerr << numDerivations << " ";
}
} // namespace Moses

73
moses/src/ChartRuleLookupManagerOnDisk.h Normal file
View File

@ -0,0 +1,73 @@
/***********************************************************************
Moses - factored phrase-based language decoder
Copyright (C) 2011 University of Edinburgh
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
***********************************************************************/
#pragma once
#ifndef moses_ChartRuleLookupManagerOnDisk_h
#define moses_ChartRuleLookupManagerOnDisk_h
#include "../../OnDiskPt/src/OnDiskWrapper.h"
#include "CellCollection.h"
#include "ChartRuleLookupManager.h"
#include "ChartTranslationOptionList.h"
#include "DotChartOnDisk.h"
#include "InputType.h"
#include "PhraseDictionaryOnDisk.h"
namespace Moses
{
// Implementation of ChartRuleLookupManager for on-disk rule tables.
class ChartRuleLookupManagerOnDisk : public ChartRuleLookupManager
{
public:
ChartRuleLookupManagerOnDisk(const InputType &sentence,
const CellCollection &cellColl,
const PhraseDictionaryOnDisk &dictionary,
OnDiskPt::OnDiskWrapper &dbWrapper,
const LMList *languageModels,
const WordPenaltyProducer *wpProducer,
const std::vector<FactorType> &inputFactorsVec,
const std::vector<FactorType> &outputFactorsVec,
const std::vector<float> &weight,
const std::string &filePath);
~ChartRuleLookupManagerOnDisk();
virtual void GetChartRuleCollection(const WordsRange &range,
bool adhereTableLimit,
ChartTranslationOptionList &outColl);
private:
const PhraseDictionaryOnDisk &m_dictionary;
OnDiskPt::OnDiskWrapper &m_dbWrapper;
const LMList *m_languageModels;
const WordPenaltyProducer *m_wpProducer;
const std::vector<FactorType> &m_inputFactorsVec;
const std::vector<FactorType> &m_outputFactorsVec;
const std::vector<float> &m_weight;
const std::string &m_filePath;
std::vector<ProcessedRuleStackOnDisk*> m_runningNodesVec;
std::map<UINT64, const TargetPhraseCollection*> m_cache;
std::list<const OnDiskPt::PhraseNode*> m_sourcePhraseNode;
};
} // namespace Moses
#endif

4
moses/src/ConfusionNet.h
View File

@ -55,10 +55,10 @@ class ConfusionNet : public InputType {
TranslationOptionCollection* CreateTranslationOptionCollection(const TranslationSystem* system) const;
const LabelList &GetLabelList(size_t /*startPos*/, size_t /*endPos*/) const
const NonTerminalSet &GetLabelSet(size_t /*startPos*/, size_t /*endPos*/) const
{
assert(false);
return *(new LabelList());
return *(new NonTerminalSet());
}
};

5
moses/src/InputType.h
View File

@ -28,6 +28,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#include "Phrase.h"
#include "TargetPhraseCollection.h"
#include "ReorderingConstraint.h"
#include "NonTerminal.h"
namespace Moses
{
@ -38,8 +39,6 @@ class PhraseDictionary;
class TranslationOptionCollection;
class TranslationSystem;
typedef std::vector<Word> LabelList;
//! base class for sentences and confusion networks
class InputType
{
@ -130,7 +129,7 @@ public:
return m_reorderingConstraint;
};
virtual const LabelList &GetLabelList(size_t startPos, size_t endPos) const = 0;
virtual const NonTerminalSet &GetLabelSet(size_t startPos, size_t endPos) const = 0;
TO_STRING();

12
moses/src/Makefile.am
View File

@ -9,6 +9,9 @@ libmoses_la_HEADERS = \
BitmapContainer.h \
BleuScoreFeature.h \
CellCollection.h \
ChartRuleLookupManager.h \
ChartRuleLookupManagerMemory.h \
ChartRuleLookupManagerOnDisk.h \
ChartTranslationOption.h \
ChartTranslationOptionList.h \
ConfusionNet.h \
@ -58,6 +61,7 @@ libmoses_la_HEADERS = \
LexicalReorderingState.h \
LexicalReorderingTable.h \
Manager.h \
NonTerminal.h \
ObjectPool.h \
OnlineCommand.h \
PCNTools.h \
@ -140,7 +144,9 @@ libmoses_la_SOURCES = \
AlignmentInfo.cpp \
BilingualDynSuffixArray.cpp \
BitmapContainer.cpp \
BleuScoreFeature.cpp \
BleuScoreFeature.cpp \
ChartRuleLookupManagerMemory.cpp \
ChartRuleLookupManagerOnDisk.cpp \
ChartTranslationOption.cpp \
ChartTranslationOptionList.cpp \
ConfusionNet.cpp \
@ -198,11 +204,9 @@ libmoses_la_SOURCES = \
PhraseDictionaryDynSuffixArray.cpp \
PhraseDictionaryMemory.cpp \
PhraseDictionarySCFG.cpp \
PhraseDictionarySCFGChart.cpp \
PhraseDictionaryNode.cpp \
PhraseDictionaryNodeSCFG.cpp \
PhraseDictionaryOnDisk.cpp \
PhraseDictionaryOnDiskChart.cpp \
PhraseDictionaryTree.cpp \
PhraseDictionaryTreeAdaptor.cpp \
PhrasePairFeature.cpp \
@ -288,4 +292,4 @@ if KEN_LM
libmoses_la_SOURCES += LanguageModelKen.cpp
endif
libmoses_la_LIBADD = $(BOOST_LDFLAGS) $(BOOST_THREAD_LIB)
libmoses_la_LIBADD = $(BOOST_THREAD_LDFLAGS) $(BOOST_THREAD_LIBS)

68
moses/src/NonTerminal.h Normal file
View File

@ -0,0 +1,68 @@
/***********************************************************************
Moses - factored phrase-based language decoder
Copyright (C) 2011 University of Edinburgh
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
***********************************************************************/
#pragma once
#include "Factor.h"
#include "Word.h"
#if HAVE_CONFIG_H
#include "config.h"
#ifdef HAVE_BOOST
#include <boost/functional/hash.hpp>
#include <boost/unordered_set.hpp>
#endif
#endif
#include <set>
namespace Moses
{
#ifdef HAVE_BOOST
class NonTerminalHasher
{
public:
size_t operator()(const Word & k) const
{
// Assumes that only the first factor is relevant.
const Factor * f = k[0];
return hash_value(*f);
}
};
class NonTerminalEqualityPred
{
public:
bool operator()(const Word & k1, const Word & k2) const
{
// Assumes that only the first factor is relevant.
const Factor * f1 = k1[0];
const Factor * f2 = k2[0];
return !(f1->Compare(*f2));
}
};
typedef boost::unordered_set<Word,
NonTerminalHasher,
NonTerminalEqualityPred> NonTerminalSet;
#else
typedef std::set<Word> NonTerminalSet;
#endif
} // namespace Moses

11
moses/src/PhraseDictionary.h
View File

@ -48,8 +48,10 @@ class WordsRange;
class ChartTranslationOptionList;
class CellCollection;
class TranslationSystem;
class ChartRuleLookupManager;
class PhraseDictionaryFeature;
/**
* Abstract base class for phrase dictionaries (tables).
**/
@ -71,11 +73,10 @@ class PhraseDictionary: public Dictionary {
virtual void AddEquivPhrase(const Phrase &source, const TargetPhrase &targetPhrase)=0;
virtual void InitializeForInput(InputType const& source) = 0;
virtual void GetChartRuleCollection(ChartTranslationOptionList &outColl
, InputType const& src
, WordsRange const& range
, bool adhereTableLimit
, const CellCollection &cellColl) const=0;
//! Create a sentence-specific manager for SCFG rule lookup.
virtual ChartRuleLookupManager *CreateRuleLookupManager(
const InputType &,
const CellCollection &) = 0;
protected:
size_t m_tableLimit;

13
moses/src/PhraseDictionaryDynSuffixArray.cpp
View File

@ -66,14 +66,6 @@ const TargetPhraseCollection *PhraseDictionaryDynSuffixArray::GetTargetPhraseCol
return ret;
}
void PhraseDictionaryDynSuffixArray::GetChartRuleCollection(ChartTranslationOptionList &outColl
,InputType const& src
,WordsRange const& range
,bool adhereTableLimit
,const CellCollection &cellColl) const
{
assert(false);
}
void PhraseDictionaryDynSuffixArray::insertSnt(string& source, string& target, string& alignment) {
m_biSA->addSntPair(source, target, alignment);
}
@ -81,4 +73,9 @@ void PhraseDictionaryDynSuffixArray::deleteSnt(unsigned idx, unsigned num2Del) {
// need to implement --
}
ChartRuleLookupManager *PhraseDictionaryDynSuffixArray::CreateRuleLookupManager(const InputType&, const CellCollection&) {
assert(false);
return 0;
}
}// end namepsace

7
moses/src/PhraseDictionaryDynSuffixArray.h
View File

@ -28,6 +28,7 @@ public:
void CleanUp();
void insertSnt(string&, string&, string&);
void deleteSnt(unsigned, unsigned);
ChartRuleLookupManager *CreateRuleLookupManager(const InputType&, const CellCollection&);
private:
BilingualDynSuffixArray *m_biSA;
std::vector<float> m_weight;
@ -35,12 +36,6 @@ private:
const LMList *m_languageModels;
float m_weightWP;
virtual void GetChartRuleCollection(ChartTranslationOptionList &outColl
,InputType const& src
,WordsRange const& range
,bool adhereTableLimit
,const CellCollection &cellColl) const;
};

17
moses/src/PhraseDictionaryMemory.h
View File

@ -62,15 +62,14 @@ public:
// for mert
virtual void InitializeForInput(InputType const&)
{/* Don't do anything source specific here as this object is shared between threads.*/}
void GetChartRuleCollection(ChartTranslationOptionList &/*outColl*/
,InputType const& /*src*/
, WordsRange const& /*range*/
, bool /*adhereTableLimit*/
,const CellCollection &/*cellColl*/) const
{
assert(false);
}
virtual ChartRuleLookupManager *CreateRuleLookupManager(
const InputType &,
const CellCollection &)
{
assert(false);
return 0;
}
TO_STRING();

5
moses/src/PhraseDictionaryNodeSCFG.h
View File

@ -139,6 +139,7 @@ public:
*/
class PhraseDictionaryNodeSCFG
{
public:
typedef std::pair<Word, Word> NonTerminalMapKey;
#if defined(BOOST_VERSION) && (BOOST_VERSION >= 104200)
@ -156,6 +157,7 @@ class PhraseDictionaryNodeSCFG
typedef std::map<NonTerminalMapKey, PhraseDictionaryNodeSCFG> NonTerminalMap;
#endif
private:
friend std::ostream& operator<<(std::ostream&, const PhraseDictionarySCFG&);
// only these classes are allowed to instantiate this class
@ -191,6 +193,9 @@ public:
return *m_targetPhraseCollection;
}
const NonTerminalMap & GetNonTerminalMap() const
{ return m_nonTermMap; }
TO_STRING();
};

38
moses/src/PhraseDictionaryOnDisk.cpp
View File

@ -24,6 +24,7 @@
#include "StaticData.h"
#include "TargetPhraseCollection.h"
#include "DotChartOnDisk.h"
#include "ChartRuleLookupManagerOnDisk.h"
using namespace std;
@ -87,33 +88,12 @@ void PhraseDictionaryOnDisk::AddEquivPhrase(const Phrase &source, TargetPhrase *
void PhraseDictionaryOnDisk::InitializeForInput(const InputType& input)
{
assert(m_runningNodesVec.size() == 0);
size_t sourceSize = input.GetSize();
m_runningNodesVec.resize(sourceSize);
for (size_t ind = 0; ind < m_runningNodesVec.size(); ++ind)
{
ProcessedRuleOnDisk *initProcessedRule = new ProcessedRuleOnDisk(m_dbWrapper.GetRootSourceNode());
ProcessedRuleStackOnDisk *processedStack = new ProcessedRuleStackOnDisk(sourceSize - ind + 1);
processedStack->Add(0, initProcessedRule); // init rule. stores the top node in tree
m_runningNodesVec[ind] = processedStack;
}
// Nothing to do: sentence-specific state is stored in ChartRuleLookupManager
}
void PhraseDictionaryOnDisk::CleanUp()
{
std::map<UINT64, const TargetPhraseCollection*>::const_iterator iterCache;
for (iterCache = m_cache.begin(); iterCache != m_cache.end(); ++iterCache)
{
delete iterCache->second;
}
m_cache.clear();
RemoveAllInColl(m_runningNodesVec);
RemoveAllInColl(m_sourcePhraseNode);
// Nothing to do: sentence-specific state is stored in ChartRuleLookupManager
}
void PhraseDictionaryOnDisk::LoadTargetLookup()
@ -121,6 +101,16 @@ void PhraseDictionaryOnDisk::LoadTargetLookup()
// TODO
}
ChartRuleLookupManager *PhraseDictionaryOnDisk::CreateRuleLookupManager(
const InputType &sentence,
const CellCollection &cellCollection)
{
return new ChartRuleLookupManagerOnDisk(sentence, cellCollection, *this,
m_dbWrapper, m_languageModels,
m_wpProducer, m_inputFactorsVec,
m_outputFactorsVec, m_weight,
m_filePath);
}
}

24
moses/src/PhraseDictionaryOnDisk.h
View File

@ -42,19 +42,13 @@ class PhraseDictionaryOnDisk : public PhraseDictionary
friend std::ostream& operator<<(std::ostream&, const PhraseDictionaryOnDisk&);
protected:
OnDiskPt::OnDiskWrapper m_dbWrapper;
const LMList* m_languageModels;
const WordPenaltyProducer* m_wpProducer;;
const WordPenaltyProducer* m_wpProducer;
std::vector<FactorType> m_inputFactorsVec, m_outputFactorsVec;
std::vector<float> m_weight;
std::string m_filePath;
mutable OnDiskPt::OnDiskWrapper m_dbWrapper;
mutable std::map<UINT64, const TargetPhraseCollection*> m_cache;
mutable std::list<const OnDiskPt::PhraseNode*> m_sourcePhraseNode;
mutable std::vector<ProcessedRuleStackOnDisk*> m_runningNodesVec;
void LoadTargetLookup();
public:
@ -86,18 +80,12 @@ public:
//! Create entry for translation of source to targetPhrase
virtual void AddEquivPhrase(const Phrase &source, TargetPhrase *targetPhrase);
virtual void GetChartRuleCollection(ChartTranslationOptionList &outColl
,InputType const& src
,WordsRange const& range
,bool adhereTableLimit
,const CellCollection &cellColl) const;
void InitializeForInput(const InputType& input);
void CleanUp();
virtual ChartRuleLookupManager *CreateRuleLookupManager(
const InputType &,
const CellCollection &);
};
};
} // namespace Moses

262
moses/src/PhraseDictionaryOnDiskChart.cpp
View File

@ -1,262 +0,0 @@
// $Id$
// vim:tabstop=2
/***********************************************************************
Moses - factored phrase-based language decoder
Copyright (C) 2010 Hieu Hoang
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
***********************************************************************/
#include <algorithm>
#include "PhraseDictionaryOnDisk.h"
#include "StaticData.h"
#include "DotChartOnDisk.h"
#include "CellCollection.h"
#include "ChartTranslationOptionList.h"
#include "../../OnDiskPt/src/TargetPhraseCollection.h"
using namespace std;
namespace Moses
{
void PhraseDictionaryOnDisk::GetChartRuleCollection(ChartTranslationOptionList &outColl
,InputType const& src
, WordsRange const& range
,bool adhereTableLimit
,const CellCollection &cellColl) const
{
const StaticData &staticData = StaticData::Instance();
size_t rulesLimit = StaticData::Instance().GetRuleLimit();
size_t relEndPos = range.GetEndPos() - range.GetStartPos();
size_t absEndPos = range.GetEndPos();
// MAIN LOOP. create list of nodes of target phrases
ProcessedRuleStackOnDisk &runningNodes = *m_runningNodesVec[range.GetStartPos()];
// sort save nodes so only do nodes with most counts
runningNodes.SortSavedNodes();
size_t numDerivations = 0
,maxDerivations = 999999; // staticData.GetMaxDerivations();
bool overThreshold = true;
const ProcessedRuleStackOnDisk::SavedNodeColl &savedNodeColl = runningNodes.GetSavedNodeColl();
//cerr << "savedNodeColl=" << savedNodeColl.size() << " ";
for (size_t ind = 0; ind < (savedNodeColl.size()) && ((numDerivations < maxDerivations) || overThreshold) ; ++ind)
{
const SavedNodeOnDisk &savedNode = *savedNodeColl[ind];
const ProcessedRuleOnDisk &prevProcessedRule = savedNode.GetProcessedRule();
const OnDiskPt::PhraseNode &prevNode = prevProcessedRule.GetLastNode();
const WordConsumed *prevWordConsumed = prevProcessedRule.GetLastWordConsumed();
size_t startPos = (prevWordConsumed == NULL) ? range.GetStartPos() : prevWordConsumed->GetWordsRange().GetEndPos() + 1;
// search for terminal symbol
if (startPos == absEndPos)
{
const Word &sourceWord = src.GetWord(absEndPos);
OnDiskPt::Word *sourceWordBerkeleyDb = m_dbWrapper.ConvertFromMoses(Input, m_inputFactorsVec, sourceWord);
if (sourceWordBerkeleyDb != NULL)
{
const OnDiskPt::PhraseNode *node = prevNode.GetChild(*sourceWordBerkeleyDb, m_dbWrapper);
if (node != NULL)
{
// TODO figure out why source word is needed from node, not from sentence
// prob to do with factors or non-term
//const Word &sourceWord = node->GetSourceWord();
WordConsumed *newWordConsumed = new WordConsumed(absEndPos, absEndPos
, sourceWord
, prevWordConsumed);
ProcessedRuleOnDisk *processedRule = new ProcessedRuleOnDisk(*node, newWordConsumed);
runningNodes.Add(relEndPos+1, processedRule);
// cache for cleanup
m_sourcePhraseNode.push_back(node);
}
delete sourceWordBerkeleyDb;
}
}
// search for non-terminals
size_t endPos, stackInd;
if (startPos > absEndPos)
continue;
else if (startPos == range.GetStartPos() && range.GetEndPos() > range.GetStartPos())
{ // start.
endPos = absEndPos - 1;
stackInd = relEndPos;
}
else
{
endPos = absEndPos;
stackInd = relEndPos + 1;
}
size_t nonTermNumWordsCovered = endPos - startPos + 1;
// get target headwords in this span from chart
const vector<Word> &headWords = cellColl.GetHeadwords(WordsRange(startPos, endPos));
const Word &defaultSourceNonTerm = staticData.GetInputDefaultNonTerminal()
,&defaultTargetNonTerm = staticData.GetOutputDefaultNonTerminal();
// go through each SOURCE lhs
const LabelList &sourceLHSList = src.GetLabelList(startPos, endPos);
LabelList::const_iterator iterSourceLHS;
for (iterSourceLHS = sourceLHSList.begin(); iterSourceLHS != sourceLHSList.end(); ++iterSourceLHS)
{
const Word &sourceLHS = *iterSourceLHS;
OnDiskPt::Word *sourceLHSBerkeleyDb = m_dbWrapper.ConvertFromMoses(Input, m_inputFactorsVec, sourceLHS);
if (sourceLHSBerkeleyDb == NULL)
{
delete sourceLHSBerkeleyDb;
continue; // vocab not in pt. node definately won't be in there
}
const OnDiskPt::PhraseNode *sourceNode = prevNode.GetChild(*sourceLHSBerkeleyDb, m_dbWrapper);
delete sourceLHSBerkeleyDb;
if (sourceNode == NULL)
continue; // didn't find source node
// go through each TARGET lhs
vector<Word>::const_iterator iterTargetLHS;
for (iterTargetLHS = headWords.begin(); iterTargetLHS != headWords.end(); ++iterTargetLHS)
{
const Word &targetLHS = *iterTargetLHS;
//cerr << sourceLHS << " " << defaultSourceNonTerm << " " << targetLHS << " " << defaultTargetNonTerm << endl;
//bool isSyntaxNonTerm = (sourceLHS != defaultSourceNonTerm) || (targetLHS != defaultTargetNonTerm);
bool doSearch = true; //isSyntaxNonTerm ? nonTermNumWordsCovered <= maxSyntaxSpan :
// nonTermNumWordsCovered <= maxDefaultSpan;
if (doSearch)
{
OnDiskPt::Word *targetLHSBerkeleyDb = m_dbWrapper.ConvertFromMoses(Output, m_outputFactorsVec, targetLHS);
if (targetLHSBerkeleyDb == NULL)
continue;
const OnDiskPt::PhraseNode *node = sourceNode->GetChild(*targetLHSBerkeleyDb, m_dbWrapper);
delete targetLHSBerkeleyDb;
if (node == NULL)
continue;
// found matching entry
//const Word &sourceWord = node->GetSourceWord();
WordConsumed *newWordConsumed = new WordConsumed(startPos, endPos
, targetLHS
, prevWordConsumed);
ProcessedRuleOnDisk *processedRule = new ProcessedRuleOnDisk(*node, newWordConsumed);
runningNodes.Add(stackInd, processedRule);
m_sourcePhraseNode.push_back(node);
}
} // for (iterHeadWords
delete sourceNode;
} // for (iterLabelListf
// return list of target phrases
ProcessedRuleCollOnDisk &nodes = runningNodes.Get(relEndPos + 1);
// source LHS
ProcessedRuleCollOnDisk::const_iterator iterProcessedRuleColl;
for (iterProcessedRuleColl = nodes.begin(); iterProcessedRuleColl != nodes.end(); ++iterProcessedRuleColl)
{
// node of last source word
const ProcessedRuleOnDisk &prevProcessedRule = **iterProcessedRuleColl;
if (prevProcessedRule.Done())
continue;
prevProcessedRule.Done(true);
const WordConsumed *wordConsumed = prevProcessedRule.GetLastWordConsumed();
assert(wordConsumed);
const OnDiskPt::PhraseNode &prevNode = prevProcessedRule.GetLastNode();
//get node for each source LHS
const LabelList &lhsList = src.GetLabelList(range.GetStartPos(), range.GetEndPos());
LabelList::const_iterator iterLabelList;
for (iterLabelList = lhsList.begin(); iterLabelList != lhsList.end(); ++iterLabelList)
{
const Word &sourceLHS = *iterLabelList;
OnDiskPt::Word *sourceLHSBerkeleyDb = m_dbWrapper.ConvertFromMoses(Input, m_inputFactorsVec, sourceLHS);
if (sourceLHSBerkeleyDb == NULL)
continue;
const TargetPhraseCollection *targetPhraseCollection = NULL;
const OnDiskPt::PhraseNode *node = prevNode.GetChild(*sourceLHSBerkeleyDb, m_dbWrapper);
if (node)
{
UINT64 tpCollFilePos = node->GetValue();
std::map<UINT64, const TargetPhraseCollection*>::const_iterator iterCache = m_cache.find(tpCollFilePos);
if (iterCache == m_cache.end())
{ // not in case
overThreshold = node->GetCount(0) > staticData.GetRuleCountThreshold();
//cerr << node->GetCount(0) << " ";
const OnDiskPt::TargetPhraseCollection *tpcollBerkeleyDb = node->GetTargetPhraseCollection(GetTableLimit(), m_dbWrapper);
targetPhraseCollection
= tpcollBerkeleyDb->ConvertToMoses(m_inputFactorsVec
,m_outputFactorsVec
,*this
,m_weight
,m_wpProducer
,*m_languageModels
,m_filePath
, m_dbWrapper.GetVocab());
delete tpcollBerkeleyDb;
m_cache[tpCollFilePos] = targetPhraseCollection;
}
else
{ // jsut get out of cache
targetPhraseCollection = iterCache->second;
}
assert(targetPhraseCollection);
outColl.Add(*targetPhraseCollection, *wordConsumed, adhereTableLimit, rulesLimit);
numDerivations++;
} // if (node)
delete node;
delete sourceLHSBerkeleyDb;
}
}
} // for (size_t ind = 0; ind < savedNodeColl.size(); ++ind)
outColl.CreateChartRules(rulesLimit);
//cerr << numDerivations << " ";
}
}; // namespace

24
moses/src/PhraseDictionarySCFG.cpp
View File

@ -36,6 +36,7 @@
#include "ChartTranslationOptionList.h"
#include "DotChart.h"
#include "FactorCollection.h"
#include "ChartRuleLookupManagerMemory.h"
using namespace std;
@ -264,19 +265,7 @@ const TargetPhraseCollection *PhraseDictionarySCFG::GetTargetPhraseCollection(co
void PhraseDictionarySCFG::InitializeForInput(const InputType& input)
{
assert(m_processedRuleColls.size() == 0);
size_t sourceSize = input.GetSize();
m_processedRuleColls.resize(sourceSize);
for (size_t ind = 0; ind < m_processedRuleColls.size(); ++ind)
{
ProcessedRule *initProcessedRule = new ProcessedRule(m_collection);
ProcessedRuleColl *processedRuleColl = new ProcessedRuleColl(sourceSize - ind + 1);
processedRuleColl->Add(0, initProcessedRule); // init rule. stores the top node in tree
m_processedRuleColls[ind] = processedRuleColl;
}
// Nothing to do: sentence-specific state is stored in ChartRuleLookupManager
}
PhraseDictionarySCFG::~PhraseDictionarySCFG()
@ -286,7 +275,14 @@ PhraseDictionarySCFG::~PhraseDictionarySCFG()
void PhraseDictionarySCFG::CleanUp()
{
RemoveAllInColl(m_processedRuleColls);
// Nothing to do: sentence-specific state is stored in ChartRuleLookupManager
}
ChartRuleLookupManager *PhraseDictionarySCFG::CreateRuleLookupManager(
const InputType &sentence,
const CellCollection &cellCollection)
{
return new ChartRuleLookupManagerMemory(sentence, cellCollection, *this);
}
TO_STRING_BODY(PhraseDictionarySCFG);

23
moses/src/PhraseDictionarySCFG.h
View File

@ -25,6 +25,9 @@
#include "PhraseDictionary.h"
#include "PhraseDictionaryNodeSCFG.h"
#include "CellCollection.h"
#include "InputType.h"
#include "WordConsumed.h"
#include "NonTerminal.h"
namespace Moses
{
@ -42,8 +45,6 @@ namespace Moses
protected:
PhraseDictionaryNodeSCFG m_collection;
mutable std::vector<ProcessedRuleColl*> m_processedRuleColls;
std::string m_filePath;
TargetPhraseCollection &GetOrCreateTargetPhraseCollection(const Phrase &source, const TargetPhrase &target);
@ -81,6 +82,10 @@ namespace Moses
, size_t tableLimit
, const LMList &languageModels
, const WordPenaltyProducer* wpProducer);
const PhraseDictionaryNodeSCFG &GetRootNode() const
{ return m_collection; }
const TargetPhraseCollection *GetTargetPhraseCollection(const Phrase &source) const;
@ -95,13 +100,11 @@ namespace Moses
void InitializeForInput(const InputType& i);
virtual void GetChartRuleCollection(ChartTranslationOptionList &outColl
,InputType const& src
,WordsRange const& range
,bool adhereTableLimit
,const CellCollection &cellColl) const;
void CleanUp();
ChartRuleLookupManager *CreateRuleLookupManager(
const InputType &,
const CellCollection &);
};
}
} // namespace Moses

136
moses/src/PhraseDictionarySCFGChart.cpp
View File

@ -1,136 +0,0 @@
// $Id$
// vim:tabstop=2
/***********************************************************************
Moses - factored phrase-based language decoder
Copyright (C) 2010 Hieu Hoang
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
***********************************************************************/
#include "PhraseDictionarySCFG.h"
#include "FactorCollection.h"
#include "InputType.h"
#include "ChartTranslationOptionList.h"
#include "CellCollection.h"
#include "DotChart.h"
#include "StaticData.h"
#include "TreeInput.h"
using namespace std;
using namespace Moses;
void PhraseDictionarySCFG::GetChartRuleCollection(ChartTranslationOptionList &outColl
,InputType const& src
,WordsRange const& range
,bool adhereTableLimit
,const CellCollection &cellColl) const
{
size_t relEndPos = range.GetEndPos() - range.GetStartPos();
size_t absEndPos = range.GetEndPos();
// MAIN LOOP. create list of nodes of target phrases
ProcessedRuleColl &processedRuleCol = *m_processedRuleColls[range.GetStartPos()];
const ProcessedRuleList &runningNodes = processedRuleCol.GetRunningNodes();
// Note that runningNodes can be expanded as the loop runs (through calls to processedRuleCol::Add())
for (size_t ind = 0; ind < runningNodes.size(); ++ind)
{
const ProcessedRule &prevProcessedRule = *runningNodes[ind];
const PhraseDictionaryNodeSCFG &prevNode = prevProcessedRule.GetLastNode();
const WordConsumed *prevWordConsumed = prevProcessedRule.GetLastWordConsumed();
size_t startPos = (prevWordConsumed == NULL) ? range.GetStartPos() : prevWordConsumed->GetWordsRange().GetEndPos() + 1;
// search for terminal symbol
if (startPos == absEndPos)
{
const Word &sourceWord = src.GetWord(absEndPos);
const PhraseDictionaryNodeSCFG *node = prevNode.GetChild(sourceWord);
if (node != NULL)
{
WordConsumed *newWordConsumed = new WordConsumed(absEndPos, absEndPos
, sourceWord
, prevWordConsumed);
ProcessedRule *processedRule = new ProcessedRule(*node, newWordConsumed);
processedRuleCol.Add(relEndPos+1, processedRule);
}
}
// search for non-terminals
size_t endPos, stackInd;
if (startPos > absEndPos)
continue;
else if (startPos == range.GetStartPos() && range.GetEndPos() > range.GetStartPos())
{ // start.
endPos = absEndPos - 1;
stackInd = relEndPos;
}
else
{
endPos = absEndPos;
stackInd = relEndPos + 1;
}
// get headwords in this span from chart
const vector<Word> &headWords = cellColl.GetHeadwords(WordsRange(startPos, endPos));
// go thru each source span
const LabelList &labelList = src.GetLabelList(startPos, endPos);
LabelList::const_iterator iterLabelList;
for (iterLabelList = labelList.begin(); iterLabelList != labelList.end(); ++iterLabelList)
{
const Word &sourceLabel = *iterLabelList;
// go thru each headword & see if in phrase table
vector<Word>::const_iterator iterHeadWords;
for (iterHeadWords = headWords.begin(); iterHeadWords != headWords.end(); ++iterHeadWords)
{
const Word &headWord = *iterHeadWords;
const PhraseDictionaryNodeSCFG *node = prevNode.GetChild(sourceLabel, headWord);
if (node != NULL)
{
WordConsumed *newWordConsumed = new WordConsumed(startPos, endPos
, headWord
, prevWordConsumed);
ProcessedRule *processedRule = new ProcessedRule(*node, newWordConsumed);
processedRuleCol.Add(stackInd, processedRule);
}
} // for (iterHeadWords
} // for (iterLabelList
}
// return list of target phrases
ProcessedRuleList &nodes = processedRuleCol.Get(relEndPos + 1);
size_t rulesLimit = StaticData::Instance().GetRuleLimit();
ProcessedRuleList::const_iterator iterNode;
for (iterNode = nodes.begin(); iterNode != nodes.end(); ++iterNode)
{
const ProcessedRule &processedRule = **iterNode;
const PhraseDictionaryNodeSCFG &node = processedRule.GetLastNode();
const WordConsumed *wordConsumed = processedRule.GetLastWordConsumed();
assert(wordConsumed);
const TargetPhraseCollection *targetPhraseCollection = node.GetTargetPhraseCollection();
if (targetPhraseCollection != NULL)
{
outColl.Add(*targetPhraseCollection, *wordConsumed, adhereTableLimit, rulesLimit);
}
}
outColl.CreateChartRules(rulesLimit);
}

17
moses/src/PhraseDictionaryTreeAdaptor.h
View File

@ -67,15 +67,14 @@ class PhraseDictionaryTreeAdaptor : public PhraseDictionary {
size_t GetNumInputScores() const;
virtual void InitializeForInput(InputType const& source);
void GetChartRuleCollection(ChartTranslationOptionList &outColl
,InputType const& /*src*/
,WordsRange const& /*range*/
,bool /*adhereTableLimit*/
,const CellCollection &/*cellColl*/) const
{
assert(false);
}
virtual ChartRuleLookupManager *CreateRuleLookupManager(
const InputType &,
const CellCollection &)
{
assert(false);
return 0;
}
};
}

6
moses/src/Sentence.cpp
View File

@ -38,7 +38,11 @@ Sentence::Sentence(FactorDirection direction)
, InputType()
{
assert(direction == Input);
m_defaultLabelList.push_back(StaticData::Instance().GetInputDefaultNonTerminal());
const StaticData& staticData = StaticData::Instance();
if (staticData.GetSearchAlgorithm() == ChartDecoding)
{
m_defaultLabelSet.insert(StaticData::Instance().GetInputDefaultNonTerminal());
}
}
int Sentence::Read(std::istream& in,const std::vector<FactorType>& factorOrder)

6
moses/src/Sentence.h
View File

@ -54,7 +54,7 @@ class Sentence : public Phrase, public InputType
std::vector <TranslationOption*> m_xmlOptionsList;
std::vector <bool> m_xmlCoverageMap;
LabelList m_defaultLabelList;
NonTerminalSet m_defaultLabelSet;
void InitStartEndWord();
@ -98,8 +98,8 @@ class Sentence : public Phrase, public InputType
, const std::string &phraseString
, const std::string &factorDelimiter);
const LabelList &GetLabelList(size_t /*startPos*/, size_t /*endPos*/) const
{ return m_defaultLabelList; }
const NonTerminalSet &GetLabelSet(size_t /*startPos*/, size_t /*endPos*/) const
{ return m_defaultLabelSet; }
};

1
moses/src/StaticData.cpp
View File

@ -1375,6 +1375,7 @@ void StaticData::ReduceTransOptCache() const
void StaticData::AddTransOptListToCache(const DecodeGraph &decodeGraph, const Phrase &sourcePhrase, const TranslationOptionList &transOptList) const
{
if (m_transOptCacheMaxSize == 0) return;
std::pair<size_t, Phrase> key(decodeGraph.GetPosition(), sourcePhrase);
TranslationOptionList* storedTransOptList = new TranslationOptionList(transOptList);
#ifdef WITH_THREADS

17
moses/src/TreeInput.cpp
View File

@ -269,21 +269,20 @@ void TreeInput::AddChartLabel(size_t startPos, size_t endPos, const Word &label
assert(label.IsNonTerminal());
SourceLabelOverlap overlapType = StaticData::Instance().GetSourceLabelOverlap();
LabelList &list = GetLabelList(startPos, endPos);
NonTerminalSet &list = GetLabelSet(startPos, endPos);
switch (overlapType)
{
case SourceLabelOverlapAdd:
list.push_back(label);
list.insert(label);
break;
case SourceLabelOverlapReplace:
if (list.size() > 0) // replace existing label
list[0] = label;
else
list.push_back(label);
list.clear();
list.insert(label);
break;
case SourceLabelOverlapDiscard:
if (list.size() == 0)
list.push_back(label);
list.insert(label);
break;
}
}
@ -307,9 +306,9 @@ std::ostream& operator<<(std::ostream &out, const TreeInput &input)
{
for (size_t endPos = startPos; endPos < size; ++endPos)
{
const LabelList &labelList = input.GetLabelList(startPos, endPos);
LabelList::const_iterator iter;
for (iter = labelList.begin(); iter != labelList.end(); ++iter)
const NonTerminalSet &labelSet = input.GetLabelSet(startPos, endPos);
NonTerminalSet::const_iterator iter;
for (iter = labelSet.begin(); iter != labelSet.end(); ++iter)
{
const Word &word = *iter;
out << "[" << startPos <<"," << endPos << "]="

6
moses/src/TreeInput.h
View File

@ -24,13 +24,13 @@ class TreeInput : public Sentence
friend std::ostream& operator<<(std::ostream&, const TreeInput&);
protected:
std::vector<std::vector<LabelList> > m_sourceChart;
std::vector<std::vector<NonTerminalSet> > m_sourceChart;
void AddChartLabel(size_t startPos, size_t endPos, const std::string &label
,const std::vector<FactorType>& factorOrder);
void AddChartLabel(size_t startPos, size_t endPos, const Word &label
,const std::vector<FactorType>& factorOrder);
LabelList &GetLabelList(size_t startPos, size_t endPos)
NonTerminalSet &GetLabelSet(size_t startPos, size_t endPos)
{
return m_sourceChart[startPos][endPos - startPos];
}
@ -54,7 +54,7 @@ public:
//! create trans options specific to this InputType
virtual TranslationOptionCollection* CreateTranslationOptionCollection() const;
virtual const LabelList &GetLabelList(size_t startPos, size_t endPos) const
virtual const NonTerminalSet &GetLabelSet(size_t startPos, size_t endPos) const
{ return m_sourceChart[startPos][endPos - startPos]; }
};

4
moses/src/WordLattice.h
View File

@ -30,10 +30,10 @@ public:
*/
void GetAsEdgeMatrix(std::vector<std::vector<bool> >& edges) const;
const LabelList &GetLabelList(size_t /*startPos*/, size_t /*endPos*/) const
const NonTerminalSet &GetLabelSet(size_t /*startPos*/, size_t /*endPos*/) const
{
assert(false);
return *(new LabelList());
return *(new NonTerminalSet());
}
};

4
scripts/Makefile
View File

@ -8,8 +8,8 @@ DS?=$(shell date '+%Y%m%d')
# Set TARGETDIR to directory where you want the compiled scripts to be copied
# to.
# Set BINDIR to the directory where GIZA++ and other tools are installed.
TARGETDIR=/Users/hieuhoang/temp
BINDIR=/Users/hieuhoang/temp
TARGETDIR=/mnt/odin3/bhaddow/moses
BINDIR=/mnt/odin3/bhaddow/moses/bin
MAIN_SCRIPTS_TARGET_DIR=$(TARGETDIR)
# MAIN_SCRIPTS_TARGET_DIR=$(shell echo `pwd`/temp)

11
scripts/ems/experiment.meta
View File

@ -702,12 +702,19 @@ ibm-bleu-c
rerun-on-change: ibm-bleu-c
template: $ibm-bleu-c -s $input-sgm -r IN1 -t IN > OUT
multi-bleu
in: recased-output reference
in: cleaned-output reference
out: multi-bleu-score
default-name: evaluation/multi-bleu
ignore-unless: multi-bleu
rerun-on-change: multi-bleu
template: $multi-bleu IN1 < IN > OUT
multi-bleu-c
in: recased-output tokenized-reference
out: multi-bleu-c-score
default-name: evaluation/multi-bleu-c
ignore-unless: multi-bleu-c
rerun-on-change: multi-bleu-c
template: $multi-bleu-c IN1 < IN > OUT
ter
in: wrapped-output reference-sgm
out: ter-score
@ -740,6 +747,6 @@ analysis-coverage
[REPORTING] single
report
in: EVALUATION:nist-bleu-score EVALUATION:nist-bleu-c-score EVALUATION:multi-bleu-score EVALUATION:meteor-score EVALUATION:ter-score EVALUATION:wer-score EVALUATION:ibm-bleu-score EVALUATION:ibm-bleu-c-score EVALUATION:analysis EVALUATION:analysis-coverage
in: EVALUATION:nist-bleu-score EVALUATION:nist-bleu-c-score EVALUATION:multi-bleu-score EVALUATION:multi-bleu-c-score EVALUATION:meteor-score EVALUATION:ter-score EVALUATION:wer-score EVALUATION:ibm-bleu-score EVALUATION:ibm-bleu-c-score EVALUATION:analysis EVALUATION:analysis-coverage
out: report
default-name: evaluation/report

9
scripts/ems/support/report-experiment-scores.perl
View File

@ -10,6 +10,7 @@ my %TYPE;
$TYPE{"nist-bleu"} = "BLEU";
$TYPE{"multi-bleu"} = "BLEU";
$TYPE{"nist-bleu-c"} = "BLEU-c";
$TYPE{"multi-bleu-c"}= "BLEU-c";
$TYPE{"ibm-bleu"} = "IBM";
$TYPE{"ibm-bleu-c"} = "IBM-c";
@ -52,8 +53,8 @@ sub process {
elsif ($type eq 'ibm-bleu' || $type eq 'ibm-bleu-c') {
$SCORE{$set} .= &extract_ibm_bleu($file,$type)." ";
}
elsif ($type eq 'multi-bleu') {
$SCORE{$set} .= &extract_multi_bleu($file)." ";
elsif ($type eq 'multi-bleu' || $type eq 'multi-bleu-c') {
$SCORE{$set} .= &extract_multi_bleu($file,$type)." ";
}
}
@ -96,7 +97,7 @@ sub extract_ibm_bleu {
}
sub extract_multi_bleu {
my ($file) = @_;
my ($file,$type) = @_;
my ($bleu,$ratio);
foreach (`cat $file`) {
$bleu = $1 if /BLEU = (\S+), /;
@ -107,5 +108,5 @@ sub extract_multi_bleu {
$AVERAGE{"multi-bleu"} += $bleu;
return $output."BLEU";
return $output.$TYPE{$type};
}

10
scripts/training/phrase-extract/consolidate-direct.vcproj
View File

@ -60,6 +60,7 @@
/>
<Tool
Name="VCLinkerTool"
AdditionalDependencies="zlib.lib"
LinkIncremental="2"
GenerateDebugInformation="true"
SubSystem="1"
@ -132,6 +133,7 @@
/>
<Tool
Name="VCLinkerTool"
AdditionalDependencies="zlib.lib"
LinkIncremental="1"
GenerateDebugInformation="true"
SubSystem="1"
@ -174,12 +176,20 @@
RelativePath=".\consolidate-direct.cpp"
>
</File>
<File
RelativePath=".\InputFileStream.cpp"
>
</File>
</Filter>
<Filter
Name="Header Files"
Filter="h;hpp;hxx;hm;inl;inc;xsd"
UniqueIdentifier="{93995380-89BD-4b04-88EB-625FBE52EBFB}"
>
<File
RelativePath=".\InputFileStream.h"
>
</File>
<File
RelativePath=".\SafeGetline.h"
>

10
scripts/training/phrase-extract/score.vcproj
View File

@ -61,6 +61,7 @@
/>
<Tool
Name="VCLinkerTool"
AdditionalDependencies="zlib.lib"
LinkIncremental="2"
GenerateDebugInformation="true"
SubSystem="1"
@ -133,6 +134,7 @@
/>
<Tool
Name="VCLinkerTool"
AdditionalDependencies="zlib.lib"
LinkIncremental="1"
GenerateDebugInformation="true"
SubSystem="1"
@ -177,6 +179,10 @@
RelativePath=".\AlignmentPhrase.cpp"
>
</File>
<File
RelativePath=".\InputFileStream.cpp"
>
</File>
<File
RelativePath=".\PhraseAlignment.cpp"
>
@ -199,6 +205,10 @@
RelativePath=".\AlignmentPhrase.h"
>
</File>
<File
RelativePath=".\InputFileStream.h"
>
</File>
<File
RelativePath=".\PhraseAlignment.h"
>

1
scripts/training/train-model.perl
View File

@ -1601,6 +1601,7 @@ sub create_ini {
print INI "$phrase_table_impl $ff 5 $file\n";
}
if ($_GLUE_GRAMMAR) {
&full_path(\$___GLUE_GRAMMAR_FILE);
print INI "6 0 0 1 $___GLUE_GRAMMAR_FILE\n";
}
if ($num_of_ttables != $stepsused{"T"}) {

4
server/Makefile.am
View File

@ -1,5 +1,5 @@
bin_PROGRAMS = mosesserver
mosesserver_SOURCES = mosesserver.cpp
mosesserver_CPPFLAGS = -W -Wall -I$(top_srcdir)/moses/src $(XMLRPC_C_CPPFLAGS) $(BOOST_CPPFLAGS)
mosesserver_LDADD = -L$(top_srcdir)/moses/src -lmoses $(BOOST_LDFLAGS) $(XMLRPC_C_LIBS) $(BOOST_THREAD_LIB) -L$(top_srcdir)/OnDiskPt/src -lOnDiskPt
mosesserver_DEPENDENCIES = $(top_srcdir)/moses/src/libmoses.la $(top_srcdir)/OnDiskPt/src/libOnDiskPt.a
mosesserver_LDADD = -L$(top_srcdir)/moses/src -lmoses -L$(top_srcdir)/OnDiskPt/src -lOnDiskPt @KENLM_LDFLAGS@ $(BOOST_THREAD_LDFLAGS) $(XMLRPC_C_LIBS) $(BOOST_THREAD_LIBS)
mosesserver_DEPENDENCIES = $(top_srcdir)/moses/src/libmoses.la $(top_srcdir)/OnDiskPt/src/libOnDiskPt.a @KENLM_DEPS@

2
unittest/Makefile.am
View File

@ -7,5 +7,5 @@ moses_test_SOURCES = MockHypothesis.cpp \
../mira/Hildreth.cpp
moses_test_CPPFLAGS = -W -Wall -I$(top_srcdir)/moses/src -I$(top_srcdir)/mira $(BOOST_CPPFLAGS) -DBOOST_TEST_DYN_LINK
moses_test_LDADD = -L$(top_srcdir)/moses/src -lmoses $(BOOST_LDFLAGS) $(BOOST_THREAD_LIB) -L$(top_srcdir)/OnDiskPt/src -lOnDiskPt -lboost_unit_test_framework
moses_test_LDADD = -L$(top_srcdir)/moses/src -lmoses $(BOOST_LDFLAGS) $(BOOST_THREAD_LIBS) -L$(top_srcdir)/OnDiskPt/src -lOnDiskPt @KENLM_LDFLAGS@ -lboost_unit_test_framework
moses_test_DEPENDENCIES = $(top_srcdir)/moses/src/libmoses.la $(top_srcdir)/OnDiskPt/src/libOnDiskPt.a