Jam: Clean up boost library handling, assimilate mert

Jamroot

@@ -5,8 +5,14 @@ project : default-build
   <link>static
   ;
 
-lib boost_thread : : <link>shared ;
+#The linked Boost libraries we use so far.  Link shared because most people don't have the static version.  
+import boost ;
+boost.use-project ;
+alias boost_thread : /boost//thread : <link>shared ;
+alias boost_unit_test_framework : /boost//unit_test_framework : <link>shared ;
+alias boost_program_options : /boost//program_options : <link>shared ;
 
+import option ;
 if [ option.get "notrace" : no : yes ] = yes
 {
   trace = ;
@@ -17,9 +23,21 @@ if [ option.get "notrace" : no : yes ] = yes
 project : requirements 
   <threading>multi:<define>WITH_THREADS
   <threading>multi:<library>boost_thread
-  <define>_FILE_OFFSET_BITS=64 <define>_LARGE_FILES $(trace)
+  <define>_FILE_OFFSET_BITS=64 <define>_LARGE_FILES <define>USE_HYPO_POOL
+  $(trace)
   ;
 
+# Shell with trailing line removed http://lists.boost.org/boost-build/2007/08/17051.php
+rule trim-nl ( str )
+{
+  return [ MATCH "([^
+]*)" : $(str) ] ;
+}
+rule _shell ( cmd )
+{
+  return [ trim-nl [ SHELL $(cmd) ] ] ;
+}
+
 path-constant TOP : . ;
 
 build-project lm ; 
@@ -29,6 +47,7 @@ build-project OnDiskPt/src ;
 build-project CreateOnDisk/src ;
 build-project moses-chart-cmd/src ;
 build-project moses-cmd/src ;
+build-project mert ;
 
 install dist : 
   lm//query
@@ -36,4 +55,5 @@ install dist :
   moses-chart-cmd/src//moses_chart 
   moses-cmd/src//programs
   CreateOnDisk/src//CreateOnDisk
+  mert//programs
   : <location>dist <install-type>EXE <install-dependencies>on <dll-path>$(TOP)/dist <link>shared:<install-type>LIB ;

boost.jam

@@ -0,0 +1,321 @@
+# $Id: boost.jam 63913 2010-07-12 07:37:43Z vladimir_prus $
+# Copyright 2008 Roland Schwarz
+# Distributed under the Boost Software License, Version 1.0.
+# (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt)
+
+# Boost library support module.
+# 
+# This module allows to use the boost library from boost-build projects.
+# The location of a boost source tree or the path to a pre-built
+# version of the library can be configured from either site-config.jam
+# or user-config.jam. If no location  is configured the module looks for
+# a BOOST_ROOT environment variable, which should point to a boost source
+# tree. As a last resort it tries to use pre-built libraries from the standard
+# search path of the compiler.
+#
+# If the location to a source tree is known, the module can be configured 
+# from the *-config.jam files:
+#
+# using boost : 1.35 : <root>/path-to-boost-root ;
+#
+# If the location to a pre-built version is known:
+#
+# using boost : 1.34 
+#   : <include>/usr/local/include/boost_1_34 
+#     <library>/usr/local/lib
+#   ;
+#
+# It is legal to configure more than one boost library version in the config
+# files. The version identifier is used to disambiguate between them.
+# The first configured version becomes the default.
+#
+# To use a boost library you need to put a 'use' statement into your
+# Jamfile:
+#
+# import boost ;
+#
+# boost.use-project 1.35 ;
+#
+# If you don't care about a specific version you just can omit the version
+# part, in which case the default is picked up:
+#
+# boost.use-project ;
+#
+# The library can be referenced with the project identifier '/boost'. To 
+# reference the program_options you would specify:
+#
+# exe myexe : mysrc.cpp : <library>/boost//program_options ;
+#
+# Note that the requirements are automatically transformed into suitable
+# tags to find the correct pre-built library.
+#
+
+import modules ;
+import errors ;
+import project ;
+import string ;
+import toolset ;
+import property-set ;
+import regex ;
+import common ;
+import option ;
+import numbers ;
+
+.boost.auto_config = [ property-set.create <layout>system ] ;
+
+if [ MATCH (--debug-configuration) : [ modules.peek : ARGV ] ]
+{
+    .debug-configuration = true ;
+}
+
+# Configuration of the boost library to use.
+#
+# This can either be a boost source tree or
+# pre-built libraries. The 'version' parameter must be a valid boost
+# version number, e.g. 1.35, if specifying a pre-built version with
+# versioned layout. It may be a symbolic name, e.g. 'trunk' if specifying
+# a source tree. The options are specified as named parameters (like
+# properties). The following paramters are available:
+#
+# <root>/path-to-boost-root: Specify a source tree.
+#
+# <include>/path-to-include: The include directory to search.
+#
+# <library>/path-to-library: The library directory to search.
+# 
+# <layout>system or <layout>versioned.
+# 
+# <build-id>my_build_id: The custom build id to use.
+#
+rule init 
+( 
+    version     # Version identifier.
+    : options * # Set the option properties.
+)
+{
+    if $(.boost.$(version)) {
+        errors.user-error 
+            "Boost " $(version) "already configured." ;
+    }
+    else {
+        if $(.debug-configuration) {
+            if ! $(.boost_default) {
+                echo notice: configuring default boost library $(version) ;
+            }
+            echo notice: configuring boost library $(version) ;
+        }
+        .boost_default ?= $(version) ; # the first configured is default
+        .boost.$(version) = [ property-set.create $(options) ] ;
+    }
+}
+
+# Use a certain version of the library.
+#
+# The use-project rule causes the module to define a boost project of
+# searchable pre-built boost libraries, or references a source tree
+# of the boost library. If the 'version' parameter is omitted either
+# the configured default (first in config files) is used or an auto
+# configuration will be attempted.
+# 
+rule use-project 
+( 
+    version ? # The version of the library to use.
+)
+{
+    project.push-current [ project.current ] ;
+    version ?= $(.boost_default) ;
+    version ?= auto_config ;
+    
+    if $(.initialized) {
+        if $(.initialized) != $(version) {
+            errors.user-error
+                "Attempt to use" $(__name__) "with different parameters" ;
+        }
+    }
+    else {
+        if $(.boost.$(version)) {
+            local opt  = $(.boost.$(version)) ;
+            local root = [ $(opt).get <root> ] ;
+            local inc  = [ $(opt).get <include> ] ;
+            local lib  = [ $(opt).get <library> ] ;
+            
+            if $(.debug-configuration) {
+                echo notice: using boost library $(version) [ $(opt).raw ] ;
+            }
+            
+            .layout = [ $(opt).get <layout> ] ;
+            .layout ?= versioned ;
+            .build_id = [ $(opt).get <build-id> ] ;
+            .version_tag = [ regex.replace $(version) "[*\\/:.\"\' ]" "_" ] ;
+            .initialized = $(version) ;
+
+            if  ( $(root) && $(inc) ) 
+                || ( $(root) && $(lib) )
+                || ( $(lib) && ! $(inc) )
+                || ( ! $(lib) && $(inc) ) {
+                errors.user-error
+                    "Ambiguous parameters,"
+                    "use either <root> or <inlude> with <library>." ;
+            }
+            else if ! $(root) && ! $(inc) {
+                root = [ modules.peek : BOOST_ROOT ] ;
+            }
+
+            local prj = [ project.current ] ;
+            local mod = [ $(prj).project-module ] ;
+
+            if $(root) {
+                modules.call-in $(mod) : use-project boost : $(root) ;
+            }
+            else {
+                project.initialize $(__name__) ;
+                # It is possible to overide the setup of the searched
+                # libraries per version. The (unlikely) tag 0.0.1 is
+                # meant as an example template only.
+                switch $(version) {
+                    case 0.0.1 : boost_0_0_1 $(inc) $(lib) ;
+                    case *    : boost_std $(inc) $(lib) ;
+                }
+            }
+        }
+        else {
+            errors.user-error 
+                "Reference to unconfigured boost version." ;
+        }
+    }
+    project.pop-current ;
+}
+
+rule boost_std ( inc ? lib ? )
+{
+#   The default definitions for pre-built libraries.
+
+    project boost
+        : usage-requirements <include>$(inc) <define>BOOST_ALL_NO_LIB
+        : requirements <tag>@tag_std <search>$(lib)
+        ;
+
+    alias headers ;
+    lib date_time        : : : : 
+        <link>shared:<define>BOOST_DATE_TIME_DYN_LINK ;
+    lib filesystem       : : : : 
+        <link>shared:<define>BOOST_FILE_SYSTEM_DYN_LINK ;
+    lib graph            : : : : 
+        <link>shared:<define>BOOST_GRAPH_DYN_LINK ;
+    lib graph_parallel   : : : : 
+        <link>shared:<define>BOOST_GRAPH_DYN_LINK ;
+    lib iostreams        : : : :
+        <link>shared:<define>BOOST_IOSTREAMS_DYN_LINK ;
+    lib math_tr1         : : : :
+        <link>shared:<define>BOOST_MATH_TR1_DYN_LINK ;
+    lib math_tr1f         : : : :
+        <link>shared:<define>BOOST_MATH_TR1_DYN_LINK ;
+    lib math_tr1l        : : : :
+        <link>shared:<define>BOOST_MATH_TR1_DYN_LINK ;
+    lib math_c99         : : : :
+        <link>shared:<define>BOOST_MATH_TR1_DYN_LINK ;
+    lib math_c99f         : : : :
+        <link>shared:<define>BOOST_MATH_TR1_DYN_LINK ;
+    lib math_c99l        : : : :
+        <link>shared:<define>BOOST_MATH_TR1_DYN_LINK ;
+    lib mpi              : : : : 
+        <link>shared:<define>BOOST_MPI_DYN_LINK  ;    
+    lib program_options  : : : : 
+        <link>shared:<define>BOOST_PROGRAM_OPTIONS_DYN_LINK ;
+    lib python           : : : : 
+        <link>shared:<define>BOOST_PYTHON_DYN_LINK ;
+    lib random           : : : :
+        <link>shared:<define>BOOST_RANDOM_DYN_LINK ;
+    lib regex            : : : : 
+        <link>shared:<define>BOOST_REGEX_DYN_LINK  ;
+    lib serialization    : : : :
+        <link>shared:<define>BOOST_SERIALIZATION_DYN_LINK ;
+    lib wserialization   : : : :
+        <link>shared:<define>BOOST_SERIALIZATION_DYN_LINK ;
+    lib signals          : : : : 
+        <link>shared:<define>BOOST_SIGNALS_DYN_LINK  ;
+    lib system           : : : : 
+        <link>shared:<define>BOOST_SYSTEM_DYN_LINK  ;
+    lib unit_test_framework : : : : 
+        <link>shared:<define>BOOST_TEST_DYN_LINK  ;
+    lib prg_exec_monitor : : : :
+        <link>shared:<define>BOOST_TEST_DYN_LINK ;
+    lib test_exec_monitor : : : :
+        <link>shared:<define>BOOST_TEST_DYN_LINK ;
+    lib thread           : : : : 
+        <link>shared:<define>BOOST_THREAD_DYN_DLL  ;
+    lib wave             : : : : 
+        <link>shared:<define>BOOST_WAVE_DYN_LINK  ;
+}
+
+rule boost_0_0_1 ( inc ? lib ? )
+{
+    echo "You are trying to use an example placeholder for boost libs." ;
+    # Copy this template to another place (in the file boost.jam)
+    # and define a project and libraries modelled after the
+    # boost_std rule. Please note that it is also possible to have
+    # a per version taging rule in case they are different between
+    # versions.
+}
+
+rule tag_std ( name : type ? : property-set )
+{
+    name = boost_$(name) ;
+    if  ( [ $(property-set).get <link> ] in static ) &&
+        ( [ $(property-set).get <target-os> ] in windows )
+    {
+        name = lib$(name) ;
+    }
+
+    local result ;
+    if $(.layout) = system
+    {
+        local version = [ MATCH ^([0-9]+)_([0-9]+) : $(.version_tag) ] ;
+        if $(version[1]) = "1" && [ numbers.less $(version[2]) 39 ]
+        {
+            result = [ tag_tagged $(name) : $(type) : $(property-set) ] ;
+        }
+        else
+        {
+            result = [ tag_system $(name) : $(type) : $(property-set) ] ;
+        }
+    }
+    else if $(.layout) = tagged
+    {
+        result = [ tag_tagged $(name) : $(type) : $(property-set) ] ;
+    }
+    else if $(.layout) = versioned
+    {
+        result = [ tag_versioned $(name) : $(type) : $(property-set) ] ;
+    }
+    else
+    {
+        errors.error "Missing layout" ;
+    }
+
+    return $(result) ;
+}
+
+rule tag_system ( name : type ? : property-set )
+{
+    return [ common.format-name
+             <base> 
+             -$(.build_id)
+             : $(name) : $(type) : $(property-set) ] ;
+}
+
+rule tag_tagged ( name : type ? : property-set )
+{
+    return [ common.format-name
+             <base> <threading> <runtime>
+             -$(.build_id)
+             : $(name) : $(type) : $(property-set) ] ; 
+}
+
+rule tag_versioned ( name : type ? : property-set )
+{
+    return [ common.format-name
+             <base> <toolset> <threading> <runtime> -$(.version_tag)
+             -$(.build_id)
+             : $(name) : $(type) : $(property-set) ] ;
+}

lm/Jamfile

@@ -1,10 +1,9 @@
 lib lm : bhiksha.cc binary_format.cc config.cc lm_exception.cc model.cc quantize.cc read_arpa.cc search_hashed.cc search_trie.cc trie.cc trie_sort.cc virtual_interface.cc vocab.cc ../util//util : <include>.. : : <include>.. ;
 
 import testing ;
-lib boost_unit_test_framework : : <link>shared : : <link>shared:<define>BOOST_TEST_DYN_LINK ;
 
-run left_test.cc lm boost_unit_test_framework : : test.arpa ;
-run model_test.cc lm boost_unit_test_framework : : test.arpa test_nounk.arpa ;
+run left_test.cc lm ..//boost_unit_test_framework : : test.arpa ;
+run model_test.cc lm ..//boost_unit_test_framework : : test.arpa test_nounk.arpa ;
 
 exe query : ngram_query.cc lm ;
 exe build_binary : build_binary.cc lm ;

mert/Jamfile

@@ -0,0 +1,43 @@
+lib m ;
+lib z ;
+
+lib mert_lib :
+Util.cpp
+FileStream.cpp
+Timer.cpp
+ScoreStats.cpp ScoreArray.cpp ScoreData.cpp
+ScoreDataIterator.cpp
+FeatureStats.cpp FeatureArray.cpp FeatureData.cpp
+FeatureDataIterator.cpp
+Data.cpp
+BleuScorer.cpp
+Point.cpp
+PerScorer.cpp
+Scorer.cpp
+ScorerFactory.cpp
+Optimizer.cpp
+TERsrc/alignmentStruct.cpp
+TERsrc/hashMap.cpp
+TERsrc/hashMapStringInfos.cpp
+TERsrc/stringHasher.cpp
+TERsrc/terAlignment.cpp
+TERsrc/terShift.cpp
+TERsrc/hashMapInfos.cpp
+TERsrc/infosHasher.cpp
+TERsrc/stringInfosHasher.cpp
+TERsrc/tercalc.cpp
+TERsrc/tools.cpp
+TerScorer.cpp
+CderScorer.cpp
+MergeScorer.cpp
+../util//util m z ;
+
+exe mert : mert.cpp mert_lib ../moses/src//ThreadPool ;
+
+exe extractor : extractor.cpp mert_lib ;
+
+exe evaluator : evaluator.cpp mert_lib ;
+
+exe pro : pro.cpp mert_lib ..//boost_program_options ;
+
+alias programs : mert extractor evaluator pro ;

moses/src/Jamfile

@@ -3,6 +3,8 @@ alias headers : ../../util//util : : : <include>. ;
 lib z ;
 alias InputFileStream : InputFileStream.cpp z headers ;
 
+alias ThreadPool : ThreadPool.cpp ;
+
 if [ option.get "with-synlm" : no : yes ] = yes
 {
   lib m ;
@@ -103,7 +105,6 @@ SquareMatrix.cpp
 StaticData.cpp
 TargetPhrase.cpp
 TargetPhraseCollection.cpp
-ThreadPool.cpp
 Timer.cpp
 TranslationOption.cpp
 TranslationOptionCollection.cpp
@@ -122,5 +123,4 @@ WordsBitmap.cpp
 WordsRange.cpp
 XmlOption.cpp
 
-synlm
-InputFileStream LM//LM headers ;
+synlm ThreadPool InputFileStream LM//LM headers ;

moses/src/LM/Jamfile

@@ -1,16 +1,5 @@
 import option ;
 
-# Shell with trailing line removed http://lists.boost.org/boost-build/2007/08/17051.php
-rule trim-nl ( str )
-{
-  return [ MATCH "([^
-]*)" : $(str) ] ;
-}
-rule _shell ( cmd )
-{
-  return [ trim-nl [ SHELL $(cmd) ] ] ;
-}
-
 with-irstlm = [ option.get "with-irstlm" ] ;
 if $(with-irstlm) != ""
 {

util/Jamfile

@@ -3,11 +3,10 @@ lib z ;
 lib util : bit_packing.cc ersatz_progress.cc exception.cc file.cc file_piece.cc mmap.cc murmur_hash.cc z : <include>.. : : <include>.. ;
 
 import testing ;
-lib boost_unit_test_framework : : <link>shared : : <link>shared:<define>BOOST_TEST_DYN_LINK ;
 
-unit-test bit_packing_test : bit_packing_test.cc util boost_unit_test_framework ;
-run file_piece_test.cc util boost_unit_test_framework : : file_piece.cc ;
-unit-test joint_sort_test : joint_sort_test.cc util boost_unit_test_framework ;
-unit-test probing_hash_table_test : probing_hash_table_test.cc util boost_unit_test_framework ;
-unit-test sorted_uniform_test : sorted_uniform_test.cc util boost_unit_test_framework ;
-unit-test tokenize_piece_test : tokenize_piece_test.cc util boost_unit_test_framework ;
+unit-test bit_packing_test : bit_packing_test.cc util ..//boost_unit_test_framework ;
+run file_piece_test.cc util ..//boost_unit_test_framework : : file_piece.cc ;
+unit-test joint_sort_test : joint_sort_test.cc util ..//boost_unit_test_framework ;
+unit-test probing_hash_table_test : probing_hash_table_test.cc util ..//boost_unit_test_framework ;
+unit-test sorted_uniform_test : sorted_uniform_test.cc util ..//boost_unit_test_framework ;
+unit-test tokenize_piece_test : tokenize_piece_test.cc util ..//boost_unit_test_framework ;