Merge branch 'master' of https://github.com/moses-smt/mosesdecoder

2024-08-17 07:20:48 +03:00 · 2017-03-04 13:15:29 +00:00 · 2017-03-04 13:15:29 +00:00 · c6509118fe
commit c6509118fe
parent 5282ad667c b95f091d18
513 changed files with 55099 additions and 4111 deletions
--- a/BUILD-INSTRUCTIONS.txt
+++ b/BUILD-INSTRUCTIONS.txt
@ -7,5 +7,3 @@ into the source tree from elsewhere:

 * "bjam-files" is taken from Boost.
 * "util" and "lm" are taken from KenLM: https://github.com/kpu/kenlm
-
-
--- a/6
+++ b/6
@ -183,7 +183,6 @@ requirements += [ option.get "with-mm" : : <define>MAX_NUM_FACTORS=4 ] ;
 requirements += [ option.get "unlabelled-source" : : <define>UNLABELLED_SOURCE ] ;

 if [ option.get "with-oxlm" ] {
-  external-lib boost_serialization ;
  external-lib gomp ;
  requirements += <library>boost_serialization ;
  requirements += <library>gomp ;
@ -316,6 +315,9 @@ rephraser
 contrib/c++tokenizer//tokenizer
 contrib/expected-bleu-training//train-expected-bleu
 contrib/expected-bleu-training//prepare-expected-bleu-training
+
+probingpt//programs
+moses2//programs
 ;


@ -339,3 +341,5 @@ if [ path.exists $(TOP)/dist ] && $(prefix) != dist {
 local temp = [ _shell "mkdir -p $(TOP)/bin" ] ;
 local temp = [ _shell "rm -f $(TOP)/bin/moses_chart" ] ;
 local temp = [ _shell "cd $(TOP)/bin && ln -s moses moses_chart" ] ;
+local temp = [ _shell "cd $(TOP)/bin && ln -s CreateProbingPT CreateProbingPT2" ] ;
+
--- a/OnDiskPt/OnDiskWrapper.cpp
+++ b/OnDiskPt/OnDiskWrapper.cpp
@ -23,7 +23,7 @@
 #include <sys/stat.h>
 #include <string>
 #include "OnDiskWrapper.h"
-#include "moses/Factor.h"
+#include "moses/Util.h"
 #include "util/exception.hh"
 #include "util/string_stream.hh"

@ -219,42 +219,5 @@ uint64_t OnDiskWrapper::GetMisc(const std::string &key) const
  return iter->second;
 }

-Word *OnDiskWrapper::ConvertFromMoses(const std::vector<Moses::FactorType> &factorsVec
-                                      , const Moses::Word &origWord) const
-{
-  bool isNonTerminal = origWord.IsNonTerminal();
-  Word *newWord = new Word(isNonTerminal);
-
-  util::StringStream strme;
-
-  size_t factorType = factorsVec[0];
-  const Moses::Factor *factor = origWord.GetFactor(factorType);
-  UTIL_THROW_IF2(factor == NULL, "Expecting factor " << factorType);
-  strme << factor->GetString();
-
-  for (size_t ind = 1 ; ind < factorsVec.size() ; ++ind) {
-    size_t factorType = factorsVec[ind];
-    const Moses::Factor *factor = origWord.GetFactor(factorType);
-    if (factor == NULL) {
-      // can have less factors than factorType.size()
-      break;
-    }
-    UTIL_THROW_IF2(factor == NULL,
-                   "Expecting factor " << factorType << " at position " << ind);
-    strme << "|" << factor->GetString();
-  } // for (size_t factorType
-
-  bool found;
-  uint64_t vocabId = m_vocab.GetVocabId(strme.str(), found);
-  if (!found) {
-    // factor not in phrase table -> phrse definately not in. exit
-    delete newWord;
-    return NULL;
-  } else {
-    newWord->SetVocabId(vocabId);
-    return newWord;
-  }
-}
-

 }
--- a/OnDiskPt/OnDiskWrapper.h
+++ b/OnDiskPt/OnDiskWrapper.h
@ -22,7 +22,6 @@
 #include <fstream>
 #include "Vocab.h"
 #include "PhraseNode.h"
-#include "moses/Word.h"

 namespace OnDiskPt
 {
@ -107,9 +106,6 @@ public:

  uint64_t GetMisc(const std::string &key) const;

-  Word *ConvertFromMoses(const std::vector<Moses::FactorType> &factorsVec
-                         , const Moses::Word &origWord) const;
-
 };

 }
--- a/OnDiskPt/TargetPhrase.cpp
+++ b/OnDiskPt/TargetPhrase.cpp
@ -21,8 +21,6 @@
 #include <algorithm>
 #include <iostream>
 #include "moses/Util.h"
-#include "moses/TargetPhrase.h"
-#include "moses/TranslationModel/PhraseDictionary.h"
 #include "TargetPhrase.h"
 #include "OnDiskWrapper.h"
 #include "util/exception.hh"
@ -251,74 +249,6 @@ size_t TargetPhrase::WriteScoresToMemory(char *mem) const
  return memUsed;
 }

-
-Moses::TargetPhrase *TargetPhrase::ConvertToMoses(const std::vector<Moses::FactorType> & inputFactors
-    , const std::vector<Moses::FactorType> &outputFactors
-    , const Vocab &vocab
-    , const Moses::PhraseDictionary &phraseDict
-    , const std::vector<float> &weightT
-    , bool isSyntax) const
-{
-  Moses::TargetPhrase *ret = new Moses::TargetPhrase(&phraseDict);
-
-  // words
-  size_t phraseSize = GetSize();
-  UTIL_THROW_IF2(phraseSize == 0, "Target phrase cannot be empty"); // last word is lhs
-  if (isSyntax) {
-    --phraseSize;
-  }
-
-  for (size_t pos = 0; pos < phraseSize; ++pos) {
-    GetWord(pos).ConvertToMoses(outputFactors, vocab, ret->AddWord());
-  }
-
-  // alignments
-  // int index = 0;
-  Moses::AlignmentInfo::CollType alignTerm, alignNonTerm;
-  std::set<std::pair<size_t, size_t> > alignmentInfo;
-  const PhrasePtr sp = GetSourcePhrase();
-  for (size_t ind = 0; ind < m_align.size(); ++ind) {
-    const std::pair<size_t, size_t> &entry = m_align[ind];
-    alignmentInfo.insert(entry);
-    size_t sourcePos = entry.first;
-    size_t targetPos = entry.second;
-
-    if (GetWord(targetPos).IsNonTerminal()) {
-      alignNonTerm.insert(std::pair<size_t,size_t>(sourcePos, targetPos));
-    } else {
-      alignTerm.insert(std::pair<size_t,size_t>(sourcePos, targetPos));
-    }
-
-  }
-  ret->SetAlignTerm(alignTerm);
-  ret->SetAlignNonTerm(alignNonTerm);
-
-  if (isSyntax) {
-    Moses::Word *lhsTarget = new Moses::Word(true);
-    GetWord(GetSize() - 1).ConvertToMoses(outputFactors, vocab, *lhsTarget);
-    ret->SetTargetLHS(lhsTarget);
-  }
-
-  // set source phrase
-  Moses::Phrase mosesSP(Moses::Input);
-  for (size_t pos = 0; pos < sp->GetSize(); ++pos) {
-    sp->GetWord(pos).ConvertToMoses(inputFactors, vocab, mosesSP.AddWord());
-  }
-
-  // scores
-  ret->GetScoreBreakdown().Assign(&phraseDict, m_scores);
-
-  // sparse features
-  ret->GetScoreBreakdown().Assign(&phraseDict, m_sparseFeatures);
-
-  // property
-  ret->SetProperties(m_property);
-
-  ret->EvaluateInIsolation(mosesSP, phraseDict.GetFeaturesToApply());
-
-  return ret;
-}
-
 uint64_t TargetPhrase::ReadOtherInfoFromFile(uint64_t filePos, std::fstream &fileTPColl)
 {
  assert(filePos == (uint64_t)fileTPColl.tellg());
--- a/OnDiskPt/TargetPhrase.h
+++ b/OnDiskPt/TargetPhrase.h
@ -102,21 +102,23 @@ public:
    return m_scores[ind];
  }

-  Moses::TargetPhrase *ConvertToMoses(const std::vector<Moses::FactorType> &inputFactors
-                                      , const std::vector<Moses::FactorType> &outputFactors
-                                      , const Vocab &vocab
-                                      , const Moses::PhraseDictionary &phraseDict
-                                      , const std::vector<float> &weightT
-                                      , bool isSyntax) const;
  uint64_t ReadOtherInfoFromFile(uint64_t filePos, std::fstream &fileTPColl);
  uint64_t ReadFromFile(std::fstream &fileTP);

  virtual void DebugPrint(std::ostream &out, const Vocab &vocab) const;

+  const std::string &GetProperty() const {
+    return m_property;
+  }
+
  void SetProperty(const std::string &value) {
    m_property = value;
  }

+  const std::string &GetSparseFeatures() const {
+    return m_sparseFeatures;
+  }
+
  void SetSparseFeatures(const std::string &value) {
    m_sparseFeatures = value;
  }
--- a/OnDiskPt/TargetPhraseCollection.cpp
+++ b/OnDiskPt/TargetPhraseCollection.cpp
@ -21,8 +21,6 @@
 #include <algorithm>
 #include <iostream>
 #include "moses/Util.h"
-#include "moses/TargetPhraseCollection.h"
-#include "moses/TranslationModel/PhraseDictionary.h"
 #include "TargetPhraseCollection.h"
 #include "Vocab.h"
 #include "OnDiskWrapper.h"
@ -114,39 +112,6 @@ void TargetPhraseCollection::Save(OnDiskWrapper &onDiskWrapper)

 }

-Moses::TargetPhraseCollection::shared_ptr TargetPhraseCollection::ConvertToMoses(const std::vector<Moses::FactorType> &inputFactors
-    , const std::vector<Moses::FactorType> &outputFactors
-    , const Moses::PhraseDictionary &phraseDict
-    , const std::vector<float> &weightT
-    , Vocab &vocab
-    , bool isSyntax) const
-{
-  Moses::TargetPhraseCollection::shared_ptr ret;
-  ret.reset(new Moses::TargetPhraseCollection);
-
-  CollType::const_iterator iter;
-  for (iter = m_coll.begin(); iter != m_coll.end(); ++iter) {
-    const TargetPhrase &tp = **iter;
-    Moses::TargetPhrase *mosesPhrase
-    = tp.ConvertToMoses(inputFactors, outputFactors, vocab,
-                        phraseDict, weightT, isSyntax);
-
-    /*
-    // debugging output
-    stringstream strme;
-    strme << filePath << " " << *mosesPhrase;
-    mosesPhrase->SetDebugOutput(strme.str());
-    */
-
-    ret->Add(mosesPhrase);
-  }
-
-  ret->Sort(true, phraseDict.GetTableLimit());
-
-  return ret;
-
-}
-
 void TargetPhraseCollection::ReadFromFile(size_t tableLimit, uint64_t filePos, OnDiskWrapper &onDiskWrapper)
 {
  fstream &fileTPColl = onDiskWrapper.GetFileTargetColl();
--- a/OnDiskPt/TargetPhraseCollection.h
+++ b/OnDiskPt/TargetPhraseCollection.h
@ -21,7 +21,6 @@

 #include "TargetPhrase.h"
 #include "Vocab.h"
-#include "moses/TargetPhraseCollection.h"
 #include <boost/shared_ptr.hpp>

 namespace Moses
@ -74,12 +73,6 @@ public:

  uint64_t GetFilePos() const;

-  Moses::TargetPhraseCollection::shared_ptr ConvertToMoses(const std::vector<Moses::FactorType> &inputFactors
-      , const std::vector<Moses::FactorType> &outputFactors
-      , const Moses::PhraseDictionary &phraseDict
-      , const std::vector<float> &weightT
-      , Vocab &vocab
-      , bool isSyntax) const;
  void ReadFromFile(size_t tableLimit, uint64_t filePos, OnDiskWrapper &onDiskWrapper);

  const std::string GetDebugStr() const;
--- a/OnDiskPt/Vocab.cpp
+++ b/OnDiskPt/Vocab.cpp
@ -21,6 +21,7 @@
 #include <fstream>
 #include "OnDiskWrapper.h"
 #include "Vocab.h"
+#include "moses/Util.h"
 #include "util/exception.hh"

 using namespace std;
--- a/OnDiskPt/Word.cpp
+++ b/OnDiskPt/Word.cpp
@ -19,9 +19,7 @@
 ***********************************************************************/

 #include <boost/algorithm/string/predicate.hpp>
-#include "moses/FactorCollection.h"
 #include "moses/Util.h"
-#include "moses/Word.h"
 #include "Word.h"

 #include "util/tokenize_piece.hh"
@ -98,29 +96,6 @@ size_t Word::ReadFromFile(std::fstream &file)
  return memAlloc;
 }

-void Word::ConvertToMoses(
-  const std::vector<Moses::FactorType> &outputFactorsVec,
-  const Vocab &vocab,
-  Moses::Word &overwrite) const
-{
-  Moses::FactorCollection &factorColl = Moses::FactorCollection::Instance();
-  overwrite = Moses::Word(m_isNonTerminal);
-
-  if (m_isNonTerminal) {
-    const std::string &tok = vocab.GetString(m_vocabId);
-    overwrite.SetFactor(0, factorColl.AddFactor(tok, m_isNonTerminal));
-  } else {
-    // TODO: this conversion should have been done at load time.
-    util::TokenIter<util::SingleCharacter> tok(vocab.GetString(m_vocabId), '|');
-
-    for (std::vector<Moses::FactorType>::const_iterator t = outputFactorsVec.begin(); t != outputFactorsVec.end(); ++t, ++tok) {
-      UTIL_THROW_IF2(!tok, "Too few factors in \"" << vocab.GetString(m_vocabId) << "\"; was expecting " << outputFactorsVec.size());
-      overwrite.SetFactor(*t, factorColl.AddFactor(*tok, m_isNonTerminal));
-    }
-    UTIL_THROW_IF2(tok, "Too many factors in \"" << vocab.GetString(m_vocabId) << "\"; was expecting " << outputFactorsVec.size());
-  }
-}
-
 int Word::Compare(const Word &compare) const
 {
  int ret;
--- a/OnDiskPt/Word.h
+++ b/OnDiskPt/Word.h
@ -67,14 +67,13 @@ public:
  size_t ReadFromMemory(const char *mem);
  size_t ReadFromFile(std::fstream &file);

-  void SetVocabId(uint32_t vocabId) {
-    m_vocabId = vocabId;
+  uint64_t GetVocabId() const {
+    return m_vocabId;
  }

-  void ConvertToMoses(
-    const std::vector<Moses::FactorType> &outputFactorsVec,
-    const Vocab &vocab,
-    Moses::Word &overwrite) const;
+  void SetVocabId(uint64_t vocabId) {
+    m_vocabId = vocabId;
+  }

  void DebugPrint(std::ostream &out, const Vocab &vocab) const;
  inline const std::string &GetString(const Vocab &vocab) const {
--- a/contrib/Makefiles/install-dependencies.gmake
+++ b/contrib/Makefiles/install-dependencies.gmake
@ -91,11 +91,11 @@ $(call safepath,$(IRSTLM_PREFIX)/bin/build-lm.sh):
 	rm -rf ${TMP}

 # boost 
-boost: URL=http://sourceforge.net/projects/boost/files/boost/1.59.0/boost_1_59_0.tar.gz/download
+boost: URL=http://sourceforge.net/projects/boost/files/boost/1.63.0/boost_1_63_0.tar.gz/download
 boost: TMP=$(CWD)/build/boost
 boost: override PREFIX=${BOOST_PREFIX}
 boost: | $(call safepath,${BOOST_PREFIX}/include/boost)
 $(call safepath,${BOOST_PREFIX}/include/boost):
 	$(sfget)
-	cd '${TMP}/boost_1_59_0' && ./bootstrap.sh && ./b2 --prefix=${PREFIX} -j${nproc} install
+	cd '${TMP}/boost_1_63_0' && ./bootstrap.sh && ./b2 --prefix=${PREFIX} -j${nproc} install
 	rm -rf ${TMP}
--- a/contrib/goshen-chrome/README.md
+++ b/contrib/goshen-chrome/README.md
@ -0,0 +1,93 @@
+# goshen
+
+Goshen is a Chrome extension that duplicates the utility of the Google Translate chrome extension for on-page website translation, using the Goshen JavaScript library with Moses as a backend translator. (It also has the ability to swap in an arbitrary translation engine, if the appropriate adapters are written.)
+
+
+## 1. The Goshen.js Library
+As Google Translate is the current go-to machine-translation system for developers, I intend to make Moses a viable alternative for even the non-savvy developer. This is in large part simplified by having an easily deployed (perhaps Dockerized) Moses server, as mentioned in the section above. However, it is also greatly simplified by exposing a comprehensive and well-formed JavaScript API that allows the same level of flexibility as the existing Google API.
+
+Instead of trying to duplicate the Google Translate API, I instead chose to write a wrapper for *any* translation engine. An engine with an exposed HTTP endpoint can be added to the Goshen translation library by implementing `GoshenAdapter`, for which I have provided a complete `moses-mt-server` implementation (`MosesGoshenAdapter`) and a partially complete proof of concept for Google Translate (`GoogleTranslateGoshenAdapter`). This is to illustrate that the engines can be used interchangeably for simple translation tasks, but the entirety of Moses functionality can be accessed whereas Google Translate's public API fails to accommodate some more technical tasks.
+
+The library is both commented and minified, available in the `goshenlib/` directory, [here](https://github.com/j6k4m8/goshen-moses). It is also possible to import the unminified, importable version from `goshenlib/dist`. The complete documentation, as well as usage examples and implementation explanations and justifications, are available in `goshenlib/docs` at the above repository.
+
+## 2. Chrome Extension
+This directory contains a Chrome extension that utilizes the CASMACAT moses-mt-server/Moses backend to provide a frontend website translation service. The extension automatically detects the relevant content of most articles or body-text on the page, and at the user's request, translates it to the requested language. Usage is explained below, as well as inside the extension popup after installation, for quick reference.
+
+### Usage
+1. **Install the unpacked extension.** Go to `chrome://extensions` and click <kbd>Load Unpacked Extension</kbd>. Navigate to this `goshen-chrome/` directory, and load.
+2. This adds a Goshen icon to your Chrome toolbar. Clicking it brings up a simple modal that allows the switching of languages.
+3. Use the <kbd>Alt</kbd>+<kbd>T</kbd> key-chord ("T" for "Translate") to begin text-selection. The Goshen-translate extension will highlight elements of text in cyan as you mouse over them: To translate what is currently highlighted, click.
+
+## Goshen.js Documentatio
+
+### Overview
+The Goshen library provides a web-developer-facing library for handling machine translation. It allows interaction with arbitrary machine translation services, agnostic of the technology or algorithm stack.
+
+### Usage
+A very brief tutorial is provided here:
+
+- Create a new Goshen object. Use the MosesGoshenAdapter, so that translations are handled by a Moses MT server.
+    ```JavaScript
+    g = new Goshen('localhost:3000', 'http', MosesGoshenAdapter);
+    ```
+- Use the Goshen object to pass a translation job to the Moses adapter. The adapter will pass back a completed translation once the job completes.
+    ```JavaScript
+    g.translate('This is a simple sentence.', Languages.ENGLISH, Languages.SPANISH);
+    ```
+- You can also optionally pass a callback function to the .translate method:
+    ```JavaScript
+    g.translate('This is a simple sentence.',
+                Languages.ENGLISH,
+                Languages.SPANISH,
+                function(err, val) {
+        if (!!err) {
+            console.warn("Encountered an error: " + err);
+        } else {
+            console.info("Translated to: " + val);
+        }
+    });
+    ```
+    If a callback is supplied, the function is run on a new thread, and is non-blocking. If one is not supplied, then the return value of the function contains the translated text. `undefined` is returned if the translation fails.
+
+
+### `Goshen`
+The generic class for a Goshen.js object, the object that handles translation with an arbitrary translation backend. In order to specify a backend, pass a `type` parameter to the constructor. (Default is Moses, of course!)
+
+- `Goshen`
+    - Arguments:
+        - `hostname`: A string hostname, such as `locahost:8000`. This is the base URL for formulating the RESTful API endpoint.
+        - `protocol`: The HTTP protocol. Either `http` or `https`.
+        - `type`: What type of GoshenAdapter to use. Options are currently `GoogleTranslateGoshenAdapter` or `MosesGoshenAdapter`.
+        - `opts`: A dictonary of options to pass to the adapter constructor. Currently, none are required for existing adapters.
+
+- function `url`
+
+    Generate a complete URI. If `hostname` is `localhost:8000` and `protocol` is `https`, then `this.url('foo')` returns `https://localhost:8000/foo`
+    - Arguments:
+        - `suffix`: A suffix to concatenate onto the end of a well-formed URI.
+    - Returns:
+        - String: The complete web-accessible URL.
+
+- function `translate`
+
+    Translate a text from a source language to a target language.
+    - Arguments:
+        - `text`: The text to translate. If this is too long, a series of truncated versions are translated, splitting on sentence-delimiters if possible.
+        - `source`: An item from the `LANGUAGES` set (e.g. `'en-us'`)
+        - `target`: An item from the `LANGUAGES` set (e.g. `'en-us'`)
+        - `callback`: Optional. If supplied, must be a function (or be of a callable type) that will be run with `errors` and `value` as its two arguments.
+    - Returns:
+        - String: The translated text. All supplementary data, such as alignments or language detections, are ignored by this function.
+
+
+### `GoshenAdapter`
+The `Goshen` class secretly outsources all of its computation to a GoshenAdapter class attribute, which is responsible for performing the machine translation. `GoshenAdapter`s should expose `url` and `translate` functions unambiguously, with the same signatures as those in the `Goshen` class. Other functions may be optionally exposed.
+
+#### `MosesGoshenAdapter`
+This is one particular implementation of the `GoshenAdapter` type, that uses the `moses-mt-server` backend as its translation engine API endpoint. It splits text into manageable chunks when translating, to avoid crashing the underlying Moses server (RAM allocation fail).
+
+#### `GoogleTranslateGoshenAdapter`
+This is another implementation of the `GoshenAdapter` type, that uses the Google Translate API as its translation engine endpoint. Because Google handles arbitrarily long text, this adapter does not split text, as `MosesGoshenAdapter`s do.
+
+
+For more information, see [this full report](https://github.com/j6k4m8/goshen-moses/blob/master/report/report.md), or contact Jordan Matelsky (@j6k4m8).
--- a/contrib/goshen-chrome/icon.png
+++ b/contrib/goshen-chrome/icon.png
--- a/contrib/goshen-chrome/manifest.json
+++ b/contrib/goshen-chrome/manifest.json
@ -0,0 +1,29 @@
+{
+    "manifest_version": 2,
+
+    "name": "Goshen Web Translator",
+    "description": "Translate entire webpages with a casmacat-moses backend",
+    "version": "1.0",
+
+    "browser_action": {
+        "default_icon": "icon.png",
+        "default_popup": "popup/popup.html"
+    },
+    "permissions": [
+        "activeTab",
+        "storage",
+        "https://ajax.googleapis.com/"
+    ],
+    "options_page" : "options/index.html",
+
+    "content_scripts": [{
+        "matches": ["http://*/*", "https://*/*", "file:///*"],
+        "css": ["onpage/onpage.css"],
+        "js": [
+            "onpage/onpage.js",
+            "onpage/goshen.js",
+            "onpage/chromegoshen.js"
+        ],
+        "all_frames": true
+    }]
+}
--- a/contrib/goshen-chrome/onpage/chromegoshen.js
+++ b/contrib/goshen-chrome/onpage/chromegoshen.js
@ -0,0 +1,166 @@
+(function(window) {
+
+    var demo_url = "ec2-52-23-242-15.compute-1.amazonaws.com:8081";
+
+    var _goshen = window._goshen;
+
+    on = function(event, cb) {
+        window.addEventListener(event, cb);
+    }
+
+    off = function(event, cb) {
+        window.removeEventListener(event, cb);
+    }
+
+    class ChromeGoshen {
+        constructor() {
+            this.G = new _goshen.Goshen(demo_url);
+            console.info("Goshenjs engine loaded successfully.")
+        }
+
+        /**
+        * Begin interactive dom node selection.
+        */
+        selectMode() {
+            var self = this;
+            var selection = [];
+            var previousElement = null;
+
+            var showSelection = function() {
+                var olds = document.querySelectorAll('._goshen-selected');
+                for (var i = 0; i < olds.length; i++) {
+                    olds[i].classList.remove('_goshen-selected');
+                }
+                for (var i = 0; i < selection.length; i++) {
+                    selection[i].classList.add('_goshen-selected');
+                }
+            };
+
+            var setSelection = function(sel) {
+                selection = sel;
+                showSelection();
+            };
+
+            var validParents = [
+                "DIV", "ARTICLE", "BLOCKQUOTE", "MAIN",
+                "SECTION", "UL", "OL", "DL"
+            ];
+
+            var validChildren = [
+                "P", "H1", "H2", "H3", "H4", "H5", "H6", "SPAN", "DL",
+                "OL", "UL", "BLOCKQUOTE", "SECTION"
+            ];
+
+            var selectSiblings = function(el) {
+                var firstChild = el;
+                var parent = el.parentNode;
+                while (parent && !~validParents.indexOf(parent.tagName)) {
+                    firstChild = parent;
+                    parent = firstChild.parentNode;
+                }
+
+                if (parent) {
+                    var kids = parent.childNodes,
+                        len = kids.length,
+                        result = [],
+                        i = 0;
+
+                    while (kids[i] !== firstChild) { i++; }
+
+                    for (; i < len; i++) {
+                        var kid = kids[i];
+                        if (!!~validChildren.indexOf(kid.tagName)) {
+                            result.push(kid);
+                        }
+                    }
+                    return result;
+
+                } else { return [el]; }
+            };
+
+            var stop = function() {
+                off("mouseover", mouseoverHandler);
+                off("mousemove", moveHandler);
+                off("keydown", keydownHandler);
+                off("keyup", keyupHandler);
+                off("click", clickHandler);
+                self.performSelectTranslation(selection);
+            };
+
+            var mouseoverHandler = function(ev) {
+                previousElement = ev.target;
+
+                if (ev.altKey) {
+                    setSelection([ev.target]);
+                } else {
+                    setSelection(selectSiblings(ev.target));
+                }
+            };
+
+            var clickHandler = function(ev) {
+                stop();
+            };
+
+            var moveHandler = function(ev) {
+                mouseoverHandler(ev);
+                off("mousemove", moveHandler);
+            };
+
+            var keydownHandler = function(ev) {
+                if (ev.keyCode === 27) {
+                    stop();
+                } else if (ev.altKey && selection.length > 1) {
+                    setSelection([selection[0]]);
+                }
+            };
+
+            var keyupHandler = function(ev) {
+                if (!ev.altKey && selection.length === 1) {
+                    setSelection(selectSiblings(selection[0]));
+                }
+            };
+
+            on("mouseover", mouseoverHandler);
+            on("click", clickHandler);
+            on("mousemove", moveHandler);
+            on("keydown", keydownHandler);
+            on("keyup", keyupHandler);
+        }
+
+        select(contextData) {
+            var text;
+            if (contextData === undefined) {
+                text = window.getSelection().toString();
+            } else {
+                text = contextData.selectionText;
+            }
+            if (text.trim().length > 0) {
+                this.init(this.parse.string(text));
+                window.getSelection().removeAllRanges();
+            } else {
+                selectMode();
+            }
+        };
+
+        _chunkedTranslation(text) {
+            // We need to find a way to split on sentences, or long things.
+            var texts = text.split('.');
+            for (var i = 0; i < texts.length; i++) {
+                texts[i] = this.G.translate(texts[i]);
+            }
+            return texts.join('.');
+        }
+
+        performSelectTranslation(selection) {
+            for (var i = 0; i < selection.length; i++) {
+                selection[i].classList.add('_goshen-active');
+                selection[i].innerText = this._chunkedTranslation(selection[i].innerText);
+                selection[i].classList.remove('_goshen-active');
+                selection[i].classList.remove('_goshen-selected');
+            }
+        }
+    };
+
+    _goshen._cg = new ChromeGoshen();
+
+})(this);
--- a/contrib/goshen-chrome/onpage/goshen.js
+++ b/contrib/goshen-chrome/onpage/goshen.js
@ -0,0 +1,107 @@
+(function (root) {
+
+    var _goshen = root._goshen;
+
+    LANGUAGES = {
+        English: 'en',
+        en: 'en',
+        German: 'de',
+        de: 'de'
+    }
+
+    LOCALES = {
+        English: 'en-US',
+        en: 'en-US',
+        German: 'de',
+        de: 'de'
+    }
+
+
+    serialize = function(obj) {
+        var str = [];
+        for (var p in obj) {
+            str.push(encodeURIComponent(p) + "=" + encodeURIComponent(obj[p]));
+        }
+        return str.join("&");
+    };
+
+
+    class MosesGoshenAdapter {
+        constructor(hostname, protocol, opts) {
+            this.hostname = hostname;
+            this.protocol = protocol || 'http';
+        }
+
+        url(suffix) {
+            suffix = suffix || '';
+            return `${this.protocol}://${this.hostname}/translate?${suffix}`;
+        }
+
+        translate(text, target, source, callback) {
+            /* Translate a string `text`, using `opts` as corequisite options.
+
+            Arguments:
+            text (str): The text to translate.
+            target (str): The language to translate to
+            source (str): The language to translate from
+            callback (function): The function to call on the translated text
+
+            Returns:
+            str: The translated text
+            */
+
+            var requestURL = this.url(serialize({
+                q: text,
+                key: 'x',
+                target: target || LANGUAGES.en,
+                source: source || LANGUAGES.de
+            }));
+
+            if (!!root.Meteor && !!root.HTTP) {
+                var response = HTTP.call('GET', requestURL, {});
+                var translated = response.data;
+                if (callback) callback(text, translated);
+
+            } else if (!!root.XMLHttpRequest) {
+                var request = new XMLHttpRequest();
+                request.open('GET', requestURL, false);
+                request.send(null);
+
+                if (request.status === 200) {
+                    var translated = root.JSON.parse(request.responseText);
+                    if (callback) callback(text, translated);
+                }
+            }
+            return translated.data.translations[0].translatedText
+        }
+    }
+
+
+    _goshen.Goshen = class Goshen {
+        constructor(hostname, protocol, type, opts) {
+            /* Create a new Goshen object.
+
+            Arguments:
+            hostname (str): A protocol-less URI such as `255.255.0.0:3000`
+            protocol (str: 'http'): An http protocol (either 'http' or 'https')
+            type (class): The type of adapter to use by default.
+            opts (dict): Options for configuration.
+
+            The options configuration dictionary can contain
+            */
+            type = type || MosesGoshenAdapter;
+            this.ga = new type(hostname, protocol, opts);
+        }
+
+        url(suffix) {
+            return this.ga.url(suffix);
+        }
+
+        translate(text, target, source, callback) {
+            /* Calls the local GoshenAdapter#translate. */
+            return this.ga.translate(text, target, source, callback);
+        }
+
+
+    };
+})(this);
--- a/contrib/goshen-chrome/onpage/onpage.css
+++ b/contrib/goshen-chrome/onpage/onpage.css
@ -0,0 +1,7 @@
+._goshen-selected {
+    background-color: rgba(100, 250, 250, 0.2);
+}
+
+._goshen-selected._goshen-active {
+    background-color: rgba(250, 100, 250, 0.2);
+}
--- a/contrib/goshen-chrome/onpage/onpage.js
+++ b/contrib/goshen-chrome/onpage/onpage.js
@ -0,0 +1,25 @@
+// This is run inside the scope of a page, and so we have direct access to the
+// page's HTML from here.
+
+(function (window) {
+
+    if (typeof window._goshen !== 'undefined') {
+        console.warn("_goshen unable to initialize!");
+        return;
+    } else {
+        window._goshen = {};
+    }
+
+    // We can now request the contents of window.
+
+    window.addEventListener('keyup', function(ev) {
+        // This is a bit heavy-handed, and we almost assuredly don't need to be
+        // capturing every keyup event. But it's lightweight, and serves as a
+        // decent proof of concept.
+        if (ev.altKey && ev.keyCode == 84) {
+            // They pressed Alt+T. Call _goshen's get-text function!
+            window._goshen._cg.selectMode();
+        }
+    });
+
+})(this);
--- a/contrib/goshen-chrome/options/index.html
+++ b/contrib/goshen-chrome/options/index.html
@ -0,0 +1,10 @@
+<!DOCTYPE html>
+<html>
+    <head>
+        <meta charset="utf-8">
+        <title>Goshen Options</title>
+    </head>
+    <body>
+        
+    </body>
+</html>
--- a/contrib/goshen-chrome/popup/popup.html
+++ b/contrib/goshen-chrome/popup/popup.html
@ -0,0 +1,44 @@
+<html>
+<head>
+    <script src="../vendor/ustr.min.js"></script>
+    <!-- <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script> -->
+    <script src="popup.js"></script>
+    <link rel="stylesheet/less" type="text/css" href="style.less" />
+    <script src="../vendor/less.js"></script>
+</head>
+<body style="width: 400px">
+    <div class="container">
+        <h1>Goshen Translator</h1>
+        <p>
+            Goshen uses <code>python-mt-server</code> and <code>moses</code> to
+            translate webpage text. For more information, see the repository
+            on <a href="https://github.com/j6k4m8/en600.468-final/">GitHub</a>.
+        </p>
+        <hr>
+        <div class="dropdown-container from-container">
+            <h2>Translate from:</h2>
+            <select class="from-select">
+                <!-- <option value="English">English</option> -->
+                <!-- <option value="French">French</option> -->
+                <option value="German">German</option>
+            </select>
+        </div>
+        <div class="dropdown-container to-container">
+            <h2>Translate to:</h2>
+            <select class="to-select">
+                <option value="English">English</option>
+                <!-- <option value="French">French</option> -->
+                <!-- <option value="German">German</option> -->
+            </select>
+        </div>
+        <hr>
+        <p>
+            To translate the webpage, press the <kbd>Alt</kbd>+<kbd>T</kbd>
+            keychord and mouse over the element(s) that you want to queue for
+            translation. Click to begin the translation &mdash; the selected
+            elements will turn blue to indicate that they're queued.
+        </p>
+        <!-- <button type="button" style="float: right;" class="js-translate">Translate</button> -->
+    </div>
+</body>
+</html>
--- a/contrib/goshen-chrome/popup/popup.js
+++ b/contrib/goshen-chrome/popup/popup.js
--- a/contrib/goshen-chrome/style.less
+++ b/contrib/goshen-chrome/style.less
@ -0,0 +1,22 @@
+* {
+    box-sizing: border-box;
+}
+.container {
+    width: 100%;
+    .dropdown-container, .arrow-container {
+        display: inline-block;
+        width: 40%;
+    }
+
+    button {
+        border-radius: 0;
+        background: #09f;
+        color: white;
+        text-transform: uppercase;
+        padding: 1em;
+        border: none;
+        cursor: pointer;
+        letter-spacing: 0.1em;
+        font-size: 1.1em;
+    }
+}
--- a/contrib/goshen-chrome/vendor/less.js
+++ b/contrib/goshen-chrome/vendor/less.js
--- a/contrib/other-builds/CreateOnDiskPt/.cproject
+++ b/contrib/other-builds/CreateOnDiskPt/.cproject
@ -40,6 +40,7 @@
 							<tool id="cdt.managedbuild.tool.gnu.c.linker.exe.debug.606542044" name="GCC C Linker" superClass="cdt.managedbuild.tool.gnu.c.linker.exe.debug"/>
 							<tool id="cdt.managedbuild.tool.gnu.cpp.linker.exe.debug.813817495" name="GCC C++ Linker" superClass="cdt.managedbuild.tool.gnu.cpp.linker.exe.debug">
 								<option id="gnu.cpp.link.option.libs.1325292383" name="Libraries (-l)" superClass="gnu.cpp.link.option.libs" valueType="libs">
+									<listOptionValue builtIn="false" value="moses"/>
 									<listOptionValue builtIn="false" value="xmlrpc_xmltok"/>
 									<listOptionValue builtIn="false" value="xmlrpc_xmlparse"/>
 									<listOptionValue builtIn="false" value="xmlrpc_util++"/>
@ -52,7 +53,6 @@
 									<listOptionValue builtIn="false" value="xmlrpc++"/>
 									<listOptionValue builtIn="false" value="xmlrpc"/>
 									<listOptionValue builtIn="false" value="OnDiskPt"/>
-									<listOptionValue builtIn="false" value="moses"/>
 									<listOptionValue builtIn="false" value="cmph"/>
 									<listOptionValue builtIn="false" value="search"/>
 									<listOptionValue builtIn="false" value="lm"/>
@ -65,12 +65,14 @@
 									<listOptionValue builtIn="false" value="boost_program_options"/>
 									<listOptionValue builtIn="false" value="pthread"/>
 									<listOptionValue builtIn="false" value="z"/>
+									<listOptionValue builtIn="false" value="probingpt"/>
 									<listOptionValue builtIn="false" value="bz2"/>
 									<listOptionValue builtIn="false" value="dl"/>
 									<listOptionValue builtIn="false" value="rt"/>
 								</option>
 								<option id="gnu.cpp.link.option.paths.815001500" name="Library search path (-L)" superClass="gnu.cpp.link.option.paths" valueType="libPaths">
 									<listOptionValue builtIn="false" value="&quot;${workspace_loc:}/../../boost/lib64&quot;"/>
+									<listOptionValue builtIn="false" value="&quot;${workspace_loc:}/probingpt/Debug&quot;"/>
 									<listOptionValue builtIn="false" value="&quot;${workspace_loc}/../../cmph/lib&quot;"/>
 									<listOptionValue builtIn="false" value="&quot;${workspace_loc}/../../xmlrpc-c/lib&quot;"/>
 									<listOptionValue builtIn="false" value="&quot;${workspace_loc:}/search/Debug&quot;"/>
@ -78,6 +80,7 @@
 									<listOptionValue builtIn="false" value="&quot;${workspace_loc:}/util/Debug&quot;"/>
 									<listOptionValue builtIn="false" value="&quot;${workspace_loc:}/moses/Debug&quot;"/>
 									<listOptionValue builtIn="false" value="&quot;${workspace_loc:}/lm/Debug&quot;"/>
+									<listOptionValue builtIn="false" value="/opt/local/lib"/>
 								</option>
 								<inputType id="cdt.managedbuild.tool.gnu.cpp.linker.input.2077999464" superClass="cdt.managedbuild.tool.gnu.cpp.linker.input">
 									<additionalInput kind="additionalinputdependency" paths="$(USER_OBJS)"/>
--- a/contrib/other-builds/CreateProbingPT/.cproject
+++ b/contrib/other-builds/CreateProbingPT/.cproject
@ -0,0 +1,150 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<?fileVersion 4.0.0?><cproject storage_type_id="org.eclipse.cdt.core.XmlProjectDescriptionStorage">
+	<storageModule moduleId="org.eclipse.cdt.core.settings">
+		<cconfiguration id="cdt.managedbuild.config.gnu.cross.exe.debug.1570028742">
+			<storageModule buildSystemId="org.eclipse.cdt.managedbuilder.core.configurationDataProvider" id="cdt.managedbuild.config.gnu.cross.exe.debug.1570028742" moduleId="org.eclipse.cdt.core.settings" name="Debug">
+				<externalSettings/>
+				<extensions>
+					<extension id="org.eclipse.cdt.core.ELF" point="org.eclipse.cdt.core.BinaryParser"/>
+					<extension id="org.eclipse.cdt.core.GmakeErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
+					<extension id="org.eclipse.cdt.core.CWDLocator" point="org.eclipse.cdt.core.ErrorParser"/>
+					<extension id="org.eclipse.cdt.core.GCCErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
+					<extension id="org.eclipse.cdt.core.GASErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
+					<extension id="org.eclipse.cdt.core.GLDErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
+				</extensions>
+			</storageModule>
+			<storageModule moduleId="cdtBuildSystem" version="4.0.0">
+				<configuration artifactName="${ProjName}" buildArtefactType="org.eclipse.cdt.build.core.buildArtefactType.exe" buildProperties="org.eclipse.cdt.build.core.buildArtefactType=org.eclipse.cdt.build.core.buildArtefactType.exe,org.eclipse.cdt.build.core.buildType=org.eclipse.cdt.build.core.buildType.debug" cleanCommand="rm -rf" description="" id="cdt.managedbuild.config.gnu.cross.exe.debug.1570028742" name="Debug" parent="cdt.managedbuild.config.gnu.cross.exe.debug">
+					<folderInfo id="cdt.managedbuild.config.gnu.cross.exe.debug.1570028742." name="/" resourcePath="">
+						<toolChain id="cdt.managedbuild.toolchain.gnu.cross.exe.debug.2140911490" name="Cross GCC" superClass="cdt.managedbuild.toolchain.gnu.cross.exe.debug">
+							<targetPlatform archList="all" binaryParser="org.eclipse.cdt.core.ELF" id="cdt.managedbuild.targetPlatform.gnu.cross.1880937408" isAbstract="false" osList="all" superClass="cdt.managedbuild.targetPlatform.gnu.cross"/>
+							<builder buildPath="${workspace_loc:/CreateProbingPT}/Debug" id="cdt.managedbuild.builder.gnu.cross.959886208" keepEnvironmentInBuildfile="false" managedBuildOn="true" name="Gnu Make Builder" superClass="cdt.managedbuild.builder.gnu.cross"/>
+							<tool id="cdt.managedbuild.tool.gnu.cross.c.compiler.1929025978" name="Cross GCC Compiler" superClass="cdt.managedbuild.tool.gnu.cross.c.compiler">
+								<option defaultValue="gnu.c.optimization.level.none" id="gnu.c.compiler.option.optimization.level.1823080879" name="Optimization Level" superClass="gnu.c.compiler.option.optimization.level" useByScannerDiscovery="false" valueType="enumerated"/>
+								<option id="gnu.c.compiler.option.debugging.level.512837716" name="Debug Level" superClass="gnu.c.compiler.option.debugging.level" useByScannerDiscovery="false" value="gnu.c.debugging.level.max" valueType="enumerated"/>
+								<inputType id="cdt.managedbuild.tool.gnu.c.compiler.input.1056616988" superClass="cdt.managedbuild.tool.gnu.c.compiler.input"/>
+							</tool>
+							<tool id="cdt.managedbuild.tool.gnu.cross.cpp.compiler.345399823" name="Cross G++ Compiler" superClass="cdt.managedbuild.tool.gnu.cross.cpp.compiler">
+								<option id="gnu.cpp.compiler.option.optimization.level.582934544" name="Optimization Level" superClass="gnu.cpp.compiler.option.optimization.level" useByScannerDiscovery="false" value="gnu.cpp.compiler.optimization.level.none" valueType="enumerated"/>
+								<option id="gnu.cpp.compiler.option.debugging.level.1251534431" name="Debug Level" superClass="gnu.cpp.compiler.option.debugging.level" useByScannerDiscovery="false" value="gnu.cpp.compiler.debugging.level.max" valueType="enumerated"/>
+								<option id="gnu.cpp.compiler.option.include.paths.1543859005" name="Include paths (-I)" superClass="gnu.cpp.compiler.option.include.paths" useByScannerDiscovery="false" valueType="includePath">
+									<listOptionValue builtIn="false" value="&quot;${workspace_loc}/../..&quot;"/>
+									<listOptionValue builtIn="false" value="&quot;${workspace_loc}/../../boost/include&quot;"/>
+								</option>
+								<option id="gnu.cpp.compiler.option.dialect.std.582049299" name="Language standard" superClass="gnu.cpp.compiler.option.dialect.std" useByScannerDiscovery="true" value="gnu.cpp.compiler.dialect.default" valueType="enumerated"/>
+								<inputType id="cdt.managedbuild.tool.gnu.cpp.compiler.input.1025890847" superClass="cdt.managedbuild.tool.gnu.cpp.compiler.input"/>
+							</tool>
+							<tool id="cdt.managedbuild.tool.gnu.cross.c.linker.901839598" name="Cross GCC Linker" superClass="cdt.managedbuild.tool.gnu.cross.c.linker"/>
+							<tool id="cdt.managedbuild.tool.gnu.cross.cpp.linker.1207177372" name="Cross G++ Linker" superClass="cdt.managedbuild.tool.gnu.cross.cpp.linker">
+								<option id="gnu.cpp.link.option.paths.1842543896" name="Library search path (-L)" superClass="gnu.cpp.link.option.paths" valueType="libPaths">
+									<listOptionValue builtIn="false" value="&quot;${workspace_loc:}/moses/Debug&quot;"/>
+									<listOptionValue builtIn="false" value="&quot;${workspace_loc:}/probingpt/Debug&quot;"/>
+									<listOptionValue builtIn="false" value="&quot;${workspace_loc:}/util/Debug&quot;"/>
+									<listOptionValue builtIn="false" value="&quot;${workspace_loc:}/../../boost/lib64&quot;"/>
+									<listOptionValue builtIn="false" value="/opt/local/lib"/>
+								</option>
+								<option id="gnu.cpp.link.option.libs.1975921106" name="Libraries (-l)" superClass="gnu.cpp.link.option.libs" valueType="libs">
+									<listOptionValue builtIn="false" value="moses"/>
+									<listOptionValue builtIn="false" value="util"/>
+									<listOptionValue builtIn="false" value="z"/>
+									<listOptionValue builtIn="false" value="boost_filesystem"/>
+									<listOptionValue builtIn="false" value="boost_thread"/>
+									<listOptionValue builtIn="false" value="boost_iostreams"/>
+									<listOptionValue builtIn="false" value="boost_serialization"/>
+									<listOptionValue builtIn="false" value="boost_system"/>
+									<listOptionValue builtIn="false" value="boost_program_options"/>
+									<listOptionValue builtIn="false" value="dl"/>
+									<listOptionValue builtIn="false" value="pthread"/>
+									<listOptionValue builtIn="false" value="probingpt"/>
+									<listOptionValue builtIn="false" value="rt"/>
+								</option>
+								<inputType id="cdt.managedbuild.tool.gnu.cpp.linker.input.1820802929" superClass="cdt.managedbuild.tool.gnu.cpp.linker.input">
+									<additionalInput kind="additionalinputdependency" paths="$(USER_OBJS)"/>
+									<additionalInput kind="additionalinput" paths="$(LIBS)"/>
+								</inputType>
+							</tool>
+							<tool id="cdt.managedbuild.tool.gnu.cross.archiver.1265750554" name="Cross GCC Archiver" superClass="cdt.managedbuild.tool.gnu.cross.archiver"/>
+							<tool id="cdt.managedbuild.tool.gnu.cross.assembler.58722959" name="Cross GCC Assembler" superClass="cdt.managedbuild.tool.gnu.cross.assembler">
+								<inputType id="cdt.managedbuild.tool.gnu.assembler.input.242758144" superClass="cdt.managedbuild.tool.gnu.assembler.input"/>
+							</tool>
+						</toolChain>
+					</folderInfo>
+				</configuration>
+			</storageModule>
+			<storageModule moduleId="org.eclipse.cdt.core.externalSettings"/>
+		</cconfiguration>
+		<cconfiguration id="cdt.managedbuild.config.gnu.cross.exe.release.1619501710">
+			<storageModule buildSystemId="org.eclipse.cdt.managedbuilder.core.configurationDataProvider" id="cdt.managedbuild.config.gnu.cross.exe.release.1619501710" moduleId="org.eclipse.cdt.core.settings" name="Release">
+				<externalSettings/>
+				<extensions>
+					<extension id="org.eclipse.cdt.core.ELF" point="org.eclipse.cdt.core.BinaryParser"/>
+					<extension id="org.eclipse.cdt.core.GmakeErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
+					<extension id="org.eclipse.cdt.core.CWDLocator" point="org.eclipse.cdt.core.ErrorParser"/>
+					<extension id="org.eclipse.cdt.core.GCCErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
+					<extension id="org.eclipse.cdt.core.GASErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
+					<extension id="org.eclipse.cdt.core.GLDErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
+				</extensions>
+			</storageModule>
+			<storageModule moduleId="cdtBuildSystem" version="4.0.0">
+				<configuration artifactName="${ProjName}" buildArtefactType="org.eclipse.cdt.build.core.buildArtefactType.exe" buildProperties="org.eclipse.cdt.build.core.buildArtefactType=org.eclipse.cdt.build.core.buildArtefactType.exe,org.eclipse.cdt.build.core.buildType=org.eclipse.cdt.build.core.buildType.release" cleanCommand="rm -rf" description="" id="cdt.managedbuild.config.gnu.cross.exe.release.1619501710" name="Release" parent="cdt.managedbuild.config.gnu.cross.exe.release">
+					<folderInfo id="cdt.managedbuild.config.gnu.cross.exe.release.1619501710." name="/" resourcePath="">
+						<toolChain id="cdt.managedbuild.toolchain.gnu.cross.exe.release.1969822952" name="Cross GCC" superClass="cdt.managedbuild.toolchain.gnu.cross.exe.release">
+							<targetPlatform archList="all" binaryParser="org.eclipse.cdt.core.ELF" id="cdt.managedbuild.targetPlatform.gnu.cross.1371996557" isAbstract="false" osList="all" superClass="cdt.managedbuild.targetPlatform.gnu.cross"/>
+							<builder buildPath="${workspace_loc:/CreateProbingPT}/Release" id="cdt.managedbuild.builder.gnu.cross.1372031461" keepEnvironmentInBuildfile="false" managedBuildOn="true" name="Gnu Make Builder" superClass="cdt.managedbuild.builder.gnu.cross"/>
+							<tool id="cdt.managedbuild.tool.gnu.cross.c.compiler.492064703" name="Cross GCC Compiler" superClass="cdt.managedbuild.tool.gnu.cross.c.compiler">
+								<option defaultValue="gnu.c.optimization.level.most" id="gnu.c.compiler.option.optimization.level.1156954368" name="Optimization Level" superClass="gnu.c.compiler.option.optimization.level" useByScannerDiscovery="false" valueType="enumerated"/>
+								<option id="gnu.c.compiler.option.debugging.level.15136302" name="Debug Level" superClass="gnu.c.compiler.option.debugging.level" useByScannerDiscovery="false" value="gnu.c.debugging.level.none" valueType="enumerated"/>
+								<inputType id="cdt.managedbuild.tool.gnu.c.compiler.input.199674880" superClass="cdt.managedbuild.tool.gnu.c.compiler.input"/>
+							</tool>
+							<tool id="cdt.managedbuild.tool.gnu.cross.cpp.compiler.291504447" name="Cross G++ Compiler" superClass="cdt.managedbuild.tool.gnu.cross.cpp.compiler">
+								<option id="gnu.cpp.compiler.option.optimization.level.1996596419" name="Optimization Level" superClass="gnu.cpp.compiler.option.optimization.level" useByScannerDiscovery="false" value="gnu.cpp.compiler.optimization.level.most" valueType="enumerated"/>
+								<option id="gnu.cpp.compiler.option.debugging.level.126290725" name="Debug Level" superClass="gnu.cpp.compiler.option.debugging.level" useByScannerDiscovery="false" value="gnu.cpp.compiler.debugging.level.none" valueType="enumerated"/>
+								<inputType id="cdt.managedbuild.tool.gnu.cpp.compiler.input.2087951440" superClass="cdt.managedbuild.tool.gnu.cpp.compiler.input"/>
+							</tool>
+							<tool id="cdt.managedbuild.tool.gnu.cross.c.linker.1394946632" name="Cross GCC Linker" superClass="cdt.managedbuild.tool.gnu.cross.c.linker"/>
+							<tool id="cdt.managedbuild.tool.gnu.cross.cpp.linker.253350906" name="Cross G++ Linker" superClass="cdt.managedbuild.tool.gnu.cross.cpp.linker">
+								<inputType id="cdt.managedbuild.tool.gnu.cpp.linker.input.720543399" superClass="cdt.managedbuild.tool.gnu.cpp.linker.input">
+									<additionalInput kind="additionalinputdependency" paths="$(USER_OBJS)"/>
+									<additionalInput kind="additionalinput" paths="$(LIBS)"/>
+								</inputType>
+							</tool>
+							<tool id="cdt.managedbuild.tool.gnu.cross.archiver.1354920365" name="Cross GCC Archiver" superClass="cdt.managedbuild.tool.gnu.cross.archiver"/>
+							<tool id="cdt.managedbuild.tool.gnu.cross.assembler.1318834738" name="Cross GCC Assembler" superClass="cdt.managedbuild.tool.gnu.cross.assembler">
+								<inputType id="cdt.managedbuild.tool.gnu.assembler.input.725479607" superClass="cdt.managedbuild.tool.gnu.assembler.input"/>
+							</tool>
+						</toolChain>
+					</folderInfo>
+				</configuration>
+			</storageModule>
+			<storageModule moduleId="org.eclipse.cdt.core.externalSettings"/>
+		</cconfiguration>
+	</storageModule>
+	<storageModule moduleId="cdtBuildSystem" version="4.0.0">
+		<project id="CreateProbingPT.cdt.managedbuild.target.gnu.cross.exe.1649652220" name="Executable" projectType="cdt.managedbuild.target.gnu.cross.exe"/>
+	</storageModule>
+	<storageModule moduleId="scannerConfiguration">
+		<autodiscovery enabled="true" problemReportingEnabled="true" selectedProfileId=""/>
+		<scannerConfigBuildInfo instanceId="cdt.managedbuild.config.gnu.cross.exe.debug.1570028742;cdt.managedbuild.config.gnu.cross.exe.debug.1570028742.;cdt.managedbuild.tool.gnu.cross.cpp.compiler.345399823;cdt.managedbuild.tool.gnu.cpp.compiler.input.1025890847">
+			<autodiscovery enabled="true" problemReportingEnabled="true" selectedProfileId=""/>
+		</scannerConfigBuildInfo>
+		<scannerConfigBuildInfo instanceId="cdt.managedbuild.config.gnu.cross.exe.release.1619501710;cdt.managedbuild.config.gnu.cross.exe.release.1619501710.;cdt.managedbuild.tool.gnu.cross.cpp.compiler.291504447;cdt.managedbuild.tool.gnu.cpp.compiler.input.2087951440">
+			<autodiscovery enabled="true" problemReportingEnabled="true" selectedProfileId=""/>
+		</scannerConfigBuildInfo>
+		<scannerConfigBuildInfo instanceId="cdt.managedbuild.config.gnu.cross.exe.debug.1570028742;cdt.managedbuild.config.gnu.cross.exe.debug.1570028742.;cdt.managedbuild.tool.gnu.cross.c.compiler.1929025978;cdt.managedbuild.tool.gnu.c.compiler.input.1056616988">
+			<autodiscovery enabled="true" problemReportingEnabled="true" selectedProfileId=""/>
+		</scannerConfigBuildInfo>
+		<scannerConfigBuildInfo instanceId="cdt.managedbuild.config.gnu.cross.exe.release.1619501710;cdt.managedbuild.config.gnu.cross.exe.release.1619501710.;cdt.managedbuild.tool.gnu.cross.c.compiler.492064703;cdt.managedbuild.tool.gnu.c.compiler.input.199674880">
+			<autodiscovery enabled="true" problemReportingEnabled="true" selectedProfileId=""/>
+		</scannerConfigBuildInfo>
+	</storageModule>
+	<storageModule moduleId="org.eclipse.cdt.core.LanguageSettingsProviders"/>
+	<storageModule moduleId="refreshScope" versionNumber="2">
+		<configuration configurationName="Release">
+			<resource resourceType="PROJECT" workspacePath="/CreateProbingPT"/>
+		</configuration>
+		<configuration configurationName="Debug">
+			<resource resourceType="PROJECT" workspacePath="/CreateProbingPT"/>
+		</configuration>
+	</storageModule>
+	<storageModule moduleId="org.eclipse.cdt.internal.ui.text.commentOwnerProjectMappings"/>
+</cproject>
--- a/contrib/other-builds/CreateProbingPT/.project
+++ b/contrib/other-builds/CreateProbingPT/.project
@ -0,0 +1,36 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<projectDescription>
+	<name>CreateProbingPT</name>
+	<comment></comment>
+	<projects>
+		<project>moses</project>
+		<project>util</project>
+	</projects>
+	<buildSpec>
+		<buildCommand>
+			<name>org.eclipse.cdt.managedbuilder.core.genmakebuilder</name>
+			<triggers>clean,full,incremental,</triggers>
+			<arguments>
+			</arguments>
+		</buildCommand>
+		<buildCommand>
+			<name>org.eclipse.cdt.managedbuilder.core.ScannerConfigBuilder</name>
+			<triggers>full,incremental,</triggers>
+			<arguments>
+			</arguments>
+		</buildCommand>
+	</buildSpec>
+	<natures>
+		<nature>org.eclipse.cdt.core.cnature</nature>
+		<nature>org.eclipse.cdt.core.ccnature</nature>
+		<nature>org.eclipse.cdt.managedbuilder.core.managedBuildNature</nature>
+		<nature>org.eclipse.cdt.managedbuilder.core.ScannerConfigNature</nature>
+	</natures>
+	<linkedResources>
+		<link>
+			<name>CreateProbingPT.cpp</name>
+			<type>1</type>
+			<locationURI>PARENT-3-PROJECT_LOC/probingpt/CreateProbingPT.cpp</locationURI>
+		</link>
+	</linkedResources>
+</projectDescription>
--- a/contrib/other-builds/consolidate/.cproject
+++ b/contrib/other-builds/consolidate/.cproject
@ -61,6 +61,7 @@
 									<listOptionValue builtIn="false" value="&quot;${workspace_loc:}/search/Debug&quot;"/>
 									<listOptionValue builtIn="false" value="&quot;${workspace_loc:}/lm/Debug&quot;"/>
 									<listOptionValue builtIn="false" value="&quot;${workspace_loc:}/OnDiskPt/Debug&quot;"/>
+									<listOptionValue builtIn="false" value="/opt/local/lib"/>
 								</option>
 								<inputType id="cdt.managedbuild.tool.gnu.cpp.linker.input.1093223502" superClass="cdt.managedbuild.tool.gnu.cpp.linker.input">
 									<additionalInput kind="additionalinputdependency" paths="$(USER_OBJS)"/>
--- a/contrib/other-builds/extractor/.cproject
+++ b/contrib/other-builds/extractor/.cproject
@ -14,7 +14,7 @@
 				</extensions>
 			</storageModule>
 			<storageModule moduleId="cdtBuildSystem" version="4.0.0">
-				<configuration artifactName="${ProjName}" buildArtefactType="org.eclipse.cdt.build.core.buildArtefactType.exe" buildProperties="org.eclipse.cdt.build.core.buildType=org.eclipse.cdt.build.core.buildType.debug,org.eclipse.cdt.build.core.buildArtefactType=org.eclipse.cdt.build.core.buildArtefactType.exe" cleanCommand="rm -rf" description="" id="cdt.managedbuild.config.gnu.exe.debug.1133345948" name="Debug" parent="cdt.managedbuild.config.gnu.exe.debug">
+				<configuration artifactName="${ProjName}" buildArtefactType="org.eclipse.cdt.build.core.buildArtefactType.exe" buildProperties="org.eclipse.cdt.build.core.buildArtefactType=org.eclipse.cdt.build.core.buildArtefactType.exe,org.eclipse.cdt.build.core.buildType=org.eclipse.cdt.build.core.buildType.debug" cleanCommand="rm -rf" description="" id="cdt.managedbuild.config.gnu.exe.debug.1133345948" name="Debug" parent="cdt.managedbuild.config.gnu.exe.debug">
 					<folderInfo id="cdt.managedbuild.config.gnu.exe.debug.1133345948." name="/" resourcePath="">
 						<toolChain id="cdt.managedbuild.toolchain.gnu.exe.debug.1405862229" name="Linux GCC" superClass="cdt.managedbuild.toolchain.gnu.exe.debug">
 							<targetPlatform id="cdt.managedbuild.target.gnu.platform.exe.debug.605722566" name="Debug Platform" superClass="cdt.managedbuild.target.gnu.platform.exe.debug"/>
@ -40,6 +40,7 @@
 									<listOptionValue builtIn="false" value="&quot;${workspace_loc:}/../../boost/lib64&quot;"/>
 									<listOptionValue builtIn="false" value="&quot;${workspace_loc:}/mert_lib/Debug&quot;"/>
 									<listOptionValue builtIn="false" value="&quot;${workspace_loc:}/util/Debug&quot;"/>
+									<listOptionValue builtIn="false" value="/opt/local/lib"/>
 								</option>
 								<option id="gnu.cpp.link.option.libs.585257079" name="Libraries (-l)" superClass="gnu.cpp.link.option.libs" valueType="libs">
 									<listOptionValue builtIn="false" value="mert_lib"/>
@ -77,7 +78,7 @@
 				</extensions>
 			</storageModule>
 			<storageModule moduleId="cdtBuildSystem" version="4.0.0">
-				<configuration artifactName="${ProjName}" buildArtefactType="org.eclipse.cdt.build.core.buildArtefactType.exe" buildProperties="org.eclipse.cdt.build.core.buildType=org.eclipse.cdt.build.core.buildType.release,org.eclipse.cdt.build.core.buildArtefactType=org.eclipse.cdt.build.core.buildArtefactType.exe" cleanCommand="rm -rf" description="" id="cdt.managedbuild.config.gnu.exe.release.1385955159" name="Release" parent="cdt.managedbuild.config.gnu.exe.release">
+				<configuration artifactName="${ProjName}" buildArtefactType="org.eclipse.cdt.build.core.buildArtefactType.exe" buildProperties="org.eclipse.cdt.build.core.buildArtefactType=org.eclipse.cdt.build.core.buildArtefactType.exe,org.eclipse.cdt.build.core.buildType=org.eclipse.cdt.build.core.buildType.release" cleanCommand="rm -rf" description="" id="cdt.managedbuild.config.gnu.exe.release.1385955159" name="Release" parent="cdt.managedbuild.config.gnu.exe.release">
 					<folderInfo id="cdt.managedbuild.config.gnu.exe.release.1385955159." name="/" resourcePath="">
 						<toolChain id="cdt.managedbuild.toolchain.gnu.exe.release.887500021" name="Linux GCC" superClass="cdt.managedbuild.toolchain.gnu.exe.release">
 							<targetPlatform id="cdt.managedbuild.target.gnu.platform.exe.release.1965146498" name="Debug Platform" superClass="cdt.managedbuild.target.gnu.platform.exe.release"/>
--- a/contrib/other-builds/mert_lib/.project
+++ b/contrib/other-builds/mert_lib/.project
@ -246,6 +246,26 @@
 			<type>1</type>
 			<locationURI>PARENT-3-PROJECT_LOC/mert/Jamfile</locationURI>
 		</link>
+		<link>
+			<name>M2.cpp</name>
+			<type>1</type>
+			<locationURI>PARENT-3-PROJECT_LOC/mert/M2.cpp</locationURI>
+		</link>
+		<link>
+			<name>M2.h</name>
+			<type>1</type>
+			<locationURI>PARENT-3-PROJECT_LOC/mert/M2.h</locationURI>
+		</link>
+		<link>
+			<name>M2Scorer.cpp</name>
+			<type>1</type>
+			<locationURI>PARENT-3-PROJECT_LOC/mert/M2Scorer.cpp</locationURI>
+		</link>
+		<link>
+			<name>M2Scorer.h</name>
+			<type>1</type>
+			<locationURI>PARENT-3-PROJECT_LOC/mert/M2Scorer.h</locationURI>
+		</link>
 		<link>
 			<name>MeteorScorer.cpp</name>
 			<type>1</type>
--- a/contrib/other-builds/moses-cmd/.cproject
+++ b/contrib/other-builds/moses-cmd/.cproject
@ -19,7 +19,7 @@
 					<folderInfo id="cdt.managedbuild.config.gnu.exe.debug.461114338." name="/" resourcePath="">
 						<toolChain id="cdt.managedbuild.toolchain.gnu.exe.debug.1896491482" name="Linux GCC" superClass="cdt.managedbuild.toolchain.gnu.exe.debug">
 							<targetPlatform binaryParser="org.eclipse.cdt.core.ELF;org.eclipse.cdt.core.MachO64" id="cdt.managedbuild.target.gnu.platform.exe.debug.2144309834" name="Debug Platform" superClass="cdt.managedbuild.target.gnu.platform.exe.debug"/>
-							<builder buildPath="${workspace_loc:/moses-cmd/Debug}" id="cdt.managedbuild.target.gnu.builder.exe.debug.56664170" keepEnvironmentInBuildfile="false" managedBuildOn="true" name="Gnu Make Builder" parallelBuildOn="true" parallelizationNumber="3" superClass="cdt.managedbuild.target.gnu.builder.exe.debug"/>
+							<builder buildPath="${workspace_loc:/moses-cmd/Debug}" id="cdt.managedbuild.target.gnu.builder.exe.debug.56664170" keepEnvironmentInBuildfile="false" managedBuildOn="true" name="Gnu Make Builder" parallelBuildOn="true" parallelizationNumber="optimal" superClass="cdt.managedbuild.target.gnu.builder.exe.debug"/>
 							<tool id="cdt.managedbuild.tool.gnu.archiver.base.1278274354" name="GCC Archiver" superClass="cdt.managedbuild.tool.gnu.archiver.base"/>
 							<tool id="cdt.managedbuild.tool.gnu.cpp.compiler.exe.debug.626095182" name="GCC C++ Compiler" superClass="cdt.managedbuild.tool.gnu.cpp.compiler.exe.debug">
 								<option id="gnu.cpp.compiler.exe.debug.option.optimization.level.2084031389" name="Optimization Level" superClass="gnu.cpp.compiler.exe.debug.option.optimization.level" value="gnu.cpp.compiler.optimization.level.none" valueType="enumerated"/>
@ -47,6 +47,8 @@
 							<tool id="cdt.managedbuild.tool.gnu.cpp.linker.exe.debug.1546774818" name="GCC C++ Linker" superClass="cdt.managedbuild.tool.gnu.cpp.linker.exe.debug">
 								<option id="gnu.cpp.link.option.paths.523170942" name="Library search path (-L)" superClass="gnu.cpp.link.option.paths" valueType="libPaths">
 									<listOptionValue builtIn="false" value="&quot;${workspace_loc:}/../../boost/lib64&quot;"/>
+									<listOptionValue builtIn="false" value="&quot;${workspace_loc:}/probingpt/Debug&quot;"/>
+									<listOptionValue builtIn="false" value="/home/hieu/workspace/xmlrpc-c/xmlrpc-c-1.39.07/lib"/>
 									<listOptionValue builtIn="false" value="&quot;${workspace_loc}/../../cmph/lib&quot;"/>
 									<listOptionValue builtIn="false" value="&quot;${workspace_loc}/../../xmlrpc-c/lib&quot;"/>
 									<listOptionValue builtIn="false" value="&quot;${workspace_loc:}/moses/Debug&quot;"/>
@ -57,6 +59,7 @@
 									<listOptionValue builtIn="false" value="/opt/local/lib"/>
 								</option>
 								<option id="gnu.cpp.link.option.libs.998577284" name="Libraries (-l)" superClass="gnu.cpp.link.option.libs" valueType="libs">
+									<listOptionValue builtIn="false" value="moses"/>
 									<listOptionValue builtIn="false" value="xmlrpc_xmltok"/>
 									<listOptionValue builtIn="false" value="xmlrpc_xmlparse"/>
 									<listOptionValue builtIn="false" value="xmlrpc_util++"/>
@ -68,7 +71,6 @@
 									<listOptionValue builtIn="false" value="xmlrpc_abyss"/>
 									<listOptionValue builtIn="false" value="xmlrpc++"/>
 									<listOptionValue builtIn="false" value="xmlrpc"/>
-									<listOptionValue builtIn="false" value="moses"/>
 									<listOptionValue builtIn="false" value="cmph"/>
 									<listOptionValue builtIn="false" value="search"/>
 									<listOptionValue builtIn="false" value="OnDiskPt"/>
@ -84,6 +86,7 @@
 									<listOptionValue builtIn="false" value="z"/>
 									<listOptionValue builtIn="false" value="bz2"/>
 									<listOptionValue builtIn="false" value="dl"/>
+									<listOptionValue builtIn="false" value="probingpt"/>
 									<listOptionValue builtIn="false" value="rt"/>
 								</option>
 								<option id="gnu.cpp.link.option.userobjs.1542590830" name="Other objects" superClass="gnu.cpp.link.option.userobjs"/>
--- a/contrib/other-builds/moses-cmd/.project
+++ b/contrib/other-builds/moses-cmd/.project
@ -6,6 +6,7 @@
 		<project>lm</project>
 		<project>moses</project>
 		<project>OnDiskPt</project>
+		<project>probingpt</project>
 		<project>search</project>
 		<project>util</project>
 	</projects>
--- a/contrib/other-builds/moses/.cproject
+++ b/contrib/other-builds/moses/.cproject
@ -26,17 +26,16 @@
 							<builder buildPath="${workspace_loc:/moses}/Debug" id="cdt.managedbuild.target.gnu.builder.exe.debug.1778877633" keepEnvironmentInBuildfile="false" managedBuildOn="true" name="Gnu Make Builder" parallelBuildOn="true" parallelizationNumber="optimal" superClass="cdt.managedbuild.target.gnu.builder.exe.debug"/>
 							<tool id="cdt.managedbuild.tool.gnu.archiver.base.1097285966" name="GCC Archiver" superClass="cdt.managedbuild.tool.gnu.archiver.base"/>
 							<tool id="cdt.managedbuild.tool.gnu.cpp.compiler.exe.debug.1729217620" name="GCC C++ Compiler" superClass="cdt.managedbuild.tool.gnu.cpp.compiler.exe.debug">
-								<option id="gnu.cpp.compiler.exe.debug.option.optimization.level.1455257477" name="Optimization Level" superClass="gnu.cpp.compiler.exe.debug.option.optimization.level" value="gnu.cpp.compiler.optimization.level.none" valueType="enumerated"/>
-								<option id="gnu.cpp.compiler.exe.debug.option.debugging.level.227767392" name="Debug Level" superClass="gnu.cpp.compiler.exe.debug.option.debugging.level" value="gnu.cpp.compiler.debugging.level.max" valueType="enumerated"/>
-								<option id="gnu.cpp.compiler.option.include.paths.876218169" name="Include paths (-I)" superClass="gnu.cpp.compiler.option.include.paths" valueType="includePath">
+								<option id="gnu.cpp.compiler.exe.debug.option.optimization.level.1455257477" name="Optimization Level" superClass="gnu.cpp.compiler.exe.debug.option.optimization.level" useByScannerDiscovery="false" value="gnu.cpp.compiler.optimization.level.none" valueType="enumerated"/>
+								<option id="gnu.cpp.compiler.exe.debug.option.debugging.level.227767392" name="Debug Level" superClass="gnu.cpp.compiler.exe.debug.option.debugging.level" useByScannerDiscovery="false" value="gnu.cpp.compiler.debugging.level.max" valueType="enumerated"/>
+								<option id="gnu.cpp.compiler.option.include.paths.876218169" name="Include paths (-I)" superClass="gnu.cpp.compiler.option.include.paths" useByScannerDiscovery="false" valueType="includePath">
 									<listOptionValue builtIn="false" value="&quot;${workspace_loc}/../..&quot;"/>
 									<listOptionValue builtIn="false" value="&quot;${workspace_loc}/../../boost/include&quot;"/>
 									<listOptionValue builtIn="false" value="&quot;${workspace_loc}/../../cmph/include&quot;"/>
 									<listOptionValue builtIn="false" value="&quot;${workspace_loc}/../../xmlrpc-c/include&quot;"/>
 								</option>
-								<option id="gnu.cpp.compiler.option.preprocessor.def.53427549" name="Defined symbols (-D)" superClass="gnu.cpp.compiler.option.preprocessor.def" valueType="definedSymbols">
+								<option id="gnu.cpp.compiler.option.preprocessor.def.53427549" name="Defined symbols (-D)" superClass="gnu.cpp.compiler.option.preprocessor.def" useByScannerDiscovery="false" valueType="definedSymbols">
 									<listOptionValue builtIn="false" value="HAVE_XMLRPC_C"/>
-									<listOptionValue builtIn="false" value="PT_UG"/>
 									<listOptionValue builtIn="false" value="MOSES_VERSION_ID=0"/>
 									<listOptionValue builtIn="false" value="HAVE_CMPH"/>
 									<listOptionValue builtIn="false" value="MAX_NUM_FACTORS=4"/>
@ -45,12 +44,12 @@
 									<listOptionValue builtIn="false" value="_FILE_OFFSET_BITS=64"/>
 									<listOptionValue builtIn="false" value="_LARGE_FILES"/>
 								</option>
-								<option id="gnu.cpp.compiler.option.dialect.std.1869920346" name="Language standard" superClass="gnu.cpp.compiler.option.dialect.std" value="gnu.cpp.compiler.dialect.default" valueType="enumerated"/>
+								<option id="gnu.cpp.compiler.option.dialect.std.1869920346" name="Language standard" superClass="gnu.cpp.compiler.option.dialect.std" useByScannerDiscovery="true" value="gnu.cpp.compiler.dialect.default" valueType="enumerated"/>
 								<inputType id="cdt.managedbuild.tool.gnu.cpp.compiler.input.1023855536" superClass="cdt.managedbuild.tool.gnu.cpp.compiler.input"/>
 							</tool>
 							<tool id="cdt.managedbuild.tool.gnu.c.compiler.exe.debug.1313249282" name="GCC C Compiler" superClass="cdt.managedbuild.tool.gnu.c.compiler.exe.debug">
-								<option defaultValue="gnu.c.optimization.level.none" id="gnu.c.compiler.exe.debug.option.optimization.level.146557271" name="Optimization Level" superClass="gnu.c.compiler.exe.debug.option.optimization.level" valueType="enumerated"/>
-								<option id="gnu.c.compiler.exe.debug.option.debugging.level.1656486500" name="Debug Level" superClass="gnu.c.compiler.exe.debug.option.debugging.level" value="gnu.c.debugging.level.max" valueType="enumerated"/>
+								<option defaultValue="gnu.c.optimization.level.none" id="gnu.c.compiler.exe.debug.option.optimization.level.146557271" name="Optimization Level" superClass="gnu.c.compiler.exe.debug.option.optimization.level" useByScannerDiscovery="false" valueType="enumerated"/>
+								<option id="gnu.c.compiler.exe.debug.option.debugging.level.1656486500" name="Debug Level" superClass="gnu.c.compiler.exe.debug.option.debugging.level" useByScannerDiscovery="false" value="gnu.c.debugging.level.max" valueType="enumerated"/>
 								<inputType id="cdt.managedbuild.tool.gnu.c.compiler.input.570559630" superClass="cdt.managedbuild.tool.gnu.c.compiler.input"/>
 							</tool>
 							<tool id="cdt.managedbuild.tool.gnu.c.linker.exe.debug.1471271407" name="GCC C Linker" superClass="cdt.managedbuild.tool.gnu.c.linker.exe.debug"/>
@ -76,7 +75,7 @@
 					<fileInfo id="cdt.managedbuild.config.gnu.exe.debug.1846963597.1459438132" name="DALMWrapper.h" rcbsApplicability="disable" resourcePath="LM/DALMWrapper.h" toolsToInvoke=""/>
 					<fileInfo id="cdt.managedbuild.config.gnu.exe.debug.1846963597.871386239" name="LDHT.h" rcbsApplicability="disable" resourcePath="LM/LDHT.h" toolsToInvoke=""/>
 					<sourceEntries>
-						<entry excluding="TranslationModel/UG/ptable-lookup.cc|TranslationModel/UG/ptable-lookup-corpus.cc|TranslationModel/UG/mm/test-http-client.cc|TranslationModel/UG/ptable-describe-features.cc|TranslationModel/UG/count-ptable-features.cc|TranslationModel/UG/try-align2.cc|TranslationModel/UG/try-align.cc|TranslationModel/UG/spe-check-coverage3.cc|TranslationModel/UG/spe-check-coverage2.cc|TranslationModel/UG/spe-check-coverage.cc|TranslationModel/UG/sim-pe.cc|TranslationModel/UG/generic/stringdist|TranslationModel/UG/mm/test-dynamic-im-tsa.cc|TranslationModel/UG/mm/mtt.count.cc|LM/ParallelBackoff.h|LM/ParallelBackoff.cpp|LM/bilingual-lm|LM/MaxEntSRI.h|LM/MaxEntSRI.cpp|LM/BilingualLM.h|LM/BilingualLM.cpp|LM/Rand.h|LM/Rand.cpp|LM/LDHT.h|LM/LDHT.cpp|LM/ORLM.h|LM/ORLM.cpp|LM/NeuralLMWrapper.h|LM/NeuralLMWrapper.cpp|LM/SRI.h|LM/SRI.cpp|LM/IRST.h|LM/IRST.cpp|LM/DALMWrapper.h|LM/DALMWrapper.cpp|LM/oxlm|TranslationModel/UG/util" flags="VALUE_WORKSPACE_PATH|RESOLVED" kind="sourcePath" name=""/>
+						<entry excluding="TranslationModel/UG|TranslationModel/UG/ptable-lookup.cc|TranslationModel/UG/ptable-lookup-corpus.cc|TranslationModel/UG/mm/test-http-client.cc|TranslationModel/UG/ptable-describe-features.cc|TranslationModel/UG/count-ptable-features.cc|TranslationModel/UG/try-align2.cc|TranslationModel/UG/try-align.cc|TranslationModel/UG/spe-check-coverage3.cc|TranslationModel/UG/spe-check-coverage2.cc|TranslationModel/UG/spe-check-coverage.cc|TranslationModel/UG/sim-pe.cc|TranslationModel/UG/generic/stringdist|TranslationModel/UG/mm/test-dynamic-im-tsa.cc|TranslationModel/UG/mm/mtt.count.cc|LM/ParallelBackoff.h|LM/ParallelBackoff.cpp|LM/bilingual-lm|LM/MaxEntSRI.h|LM/MaxEntSRI.cpp|LM/BilingualLM.h|LM/BilingualLM.cpp|LM/Rand.h|LM/Rand.cpp|LM/LDHT.h|LM/LDHT.cpp|LM/ORLM.h|LM/ORLM.cpp|LM/NeuralLMWrapper.h|LM/NeuralLMWrapper.cpp|LM/SRI.h|LM/SRI.cpp|LM/IRST.h|LM/IRST.cpp|LM/DALMWrapper.h|LM/DALMWrapper.cpp|LM/oxlm|TranslationModel/UG/util" flags="VALUE_WORKSPACE_PATH|RESOLVED" kind="sourcePath" name=""/>
 					</sourceEntries>
 				</configuration>
 			</storageModule>
--- a/contrib/other-builds/moses/.project
+++ b/contrib/other-builds/moses/.project
--- a/contrib/other-builds/moses2-cmd/.cproject
+++ b/contrib/other-builds/moses2-cmd/.cproject
@ -0,0 +1,176 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<?fileVersion 4.0.0?><cproject storage_type_id="org.eclipse.cdt.core.XmlProjectDescriptionStorage">
+	<storageModule moduleId="org.eclipse.cdt.core.settings">
+		<cconfiguration id="cdt.managedbuild.config.gnu.exe.debug.597260676">
+			<storageModule buildSystemId="org.eclipse.cdt.managedbuilder.core.configurationDataProvider" id="cdt.managedbuild.config.gnu.exe.debug.597260676" moduleId="org.eclipse.cdt.core.settings" name="Debug">
+				<externalSettings/>
+				<extensions>
+					<extension id="org.eclipse.cdt.core.ELF" point="org.eclipse.cdt.core.BinaryParser"/>
+					<extension id="org.eclipse.cdt.core.GASErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
+					<extension id="org.eclipse.cdt.core.GmakeErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
+					<extension id="org.eclipse.cdt.core.GLDErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
+					<extension id="org.eclipse.cdt.core.CWDLocator" point="org.eclipse.cdt.core.ErrorParser"/>
+					<extension id="org.eclipse.cdt.core.GCCErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
+				</extensions>
+			</storageModule>
+			<storageModule moduleId="cdtBuildSystem" version="4.0.0">
+				<configuration artifactName="${ProjName}" buildArtefactType="org.eclipse.cdt.build.core.buildArtefactType.exe" buildProperties="org.eclipse.cdt.build.core.buildArtefactType=org.eclipse.cdt.build.core.buildArtefactType.exe,org.eclipse.cdt.build.core.buildType=org.eclipse.cdt.build.core.buildType.debug" cleanCommand="rm -rf" description="" id="cdt.managedbuild.config.gnu.exe.debug.597260676" name="Debug" parent="cdt.managedbuild.config.gnu.exe.debug">
+					<folderInfo id="cdt.managedbuild.config.gnu.exe.debug.597260676." name="/" resourcePath="">
+						<toolChain id="cdt.managedbuild.toolchain.gnu.exe.debug.1894543739" name="Linux GCC" superClass="cdt.managedbuild.toolchain.gnu.exe.debug">
+							<targetPlatform id="cdt.managedbuild.target.gnu.platform.exe.debug.607512381" name="Debug Platform" superClass="cdt.managedbuild.target.gnu.platform.exe.debug"/>
+							<builder buildPath="${workspace_loc:/moses2-cmd}/Debug" id="cdt.managedbuild.target.gnu.builder.exe.debug.219597164" keepEnvironmentInBuildfile="false" managedBuildOn="true" name="Gnu Make Builder" parallelBuildOn="true" parallelizationNumber="optimal" superClass="cdt.managedbuild.target.gnu.builder.exe.debug"/>
+							<tool id="cdt.managedbuild.tool.gnu.archiver.base.2087910158" name="GCC Archiver" superClass="cdt.managedbuild.tool.gnu.archiver.base"/>
+							<tool id="cdt.managedbuild.tool.gnu.cpp.compiler.exe.debug.1546967275" name="GCC C++ Compiler" superClass="cdt.managedbuild.tool.gnu.cpp.compiler.exe.debug">
+								<option id="gnu.cpp.compiler.exe.debug.option.optimization.level.826148068" name="Optimization Level" superClass="gnu.cpp.compiler.exe.debug.option.optimization.level" value="gnu.cpp.compiler.optimization.level.none" valueType="enumerated"/>
+								<option id="gnu.cpp.compiler.exe.debug.option.debugging.level.1303802900" name="Debug Level" superClass="gnu.cpp.compiler.exe.debug.option.debugging.level" value="gnu.cpp.compiler.debugging.level.max" valueType="enumerated"/>
+								<option id="gnu.cpp.compiler.option.include.paths.368826329" name="Include paths (-I)" superClass="gnu.cpp.compiler.option.include.paths" valueType="includePath">
+									<listOptionValue builtIn="false" value="/opt/local/include/"/>
+									<listOptionValue builtIn="false" value="&quot;${workspace_loc}/../../boost/include&quot;"/>
+									<listOptionValue builtIn="false" value="&quot;${workspace_loc}/../..&quot;"/>
+									<listOptionValue builtIn="false" value="&quot;${workspace_loc}/../../cmph/include&quot;"/>
+									<listOptionValue builtIn="false" value="&quot;${workspace_loc}/../../xmlrpc-c/include&quot;"/>
+								</option>
+								<option id="gnu.cpp.compiler.option.preprocessor.def.758438174" name="Defined symbols (-D)" superClass="gnu.cpp.compiler.option.preprocessor.def" valueType="definedSymbols">
+									<listOptionValue builtIn="false" value="MAX_NUM_FACTORS=4"/>
+								</option>
+								<inputType id="cdt.managedbuild.tool.gnu.cpp.compiler.input.123491630" superClass="cdt.managedbuild.tool.gnu.cpp.compiler.input"/>
+							</tool>
+							<tool id="cdt.managedbuild.tool.gnu.c.compiler.exe.debug.848723608" name="GCC C Compiler" superClass="cdt.managedbuild.tool.gnu.c.compiler.exe.debug">
+								<option defaultValue="gnu.c.optimization.level.none" id="gnu.c.compiler.exe.debug.option.optimization.level.1977842293" name="Optimization Level" superClass="gnu.c.compiler.exe.debug.option.optimization.level" valueType="enumerated"/>
+								<option id="gnu.c.compiler.exe.debug.option.debugging.level.322285470" name="Debug Level" superClass="gnu.c.compiler.exe.debug.option.debugging.level" value="gnu.c.debugging.level.max" valueType="enumerated"/>
+								<inputType id="cdt.managedbuild.tool.gnu.c.compiler.input.1011859741" superClass="cdt.managedbuild.tool.gnu.c.compiler.input"/>
+							</tool>
+							<tool id="cdt.managedbuild.tool.gnu.c.linker.exe.debug.1706155110" name="GCC C Linker" superClass="cdt.managedbuild.tool.gnu.c.linker.exe.debug"/>
+							<tool id="cdt.managedbuild.tool.gnu.cpp.linker.exe.debug.24079646" name="GCC C++ Linker" superClass="cdt.managedbuild.tool.gnu.cpp.linker.exe.debug">
+								<option id="gnu.cpp.link.option.libs.587418382" name="Libraries (-l)" superClass="gnu.cpp.link.option.libs" valueType="libs">
+									<listOptionValue builtIn="false" value="moses2"/>
+									<listOptionValue builtIn="false" value="xmlrpc_xmltok"/>
+									<listOptionValue builtIn="false" value="xmlrpc_xmlparse"/>
+									<listOptionValue builtIn="false" value="xmlrpc_util++"/>
+									<listOptionValue builtIn="false" value="xmlrpc_util"/>
+									<listOptionValue builtIn="false" value="xmlrpc_server_abyss++"/>
+									<listOptionValue builtIn="false" value="xmlrpc_server_abyss"/>
+									<listOptionValue builtIn="false" value="xmlrpc_server++"/>
+									<listOptionValue builtIn="false" value="xmlrpc_server"/>
+									<listOptionValue builtIn="false" value="xmlrpc_abyss"/>
+									<listOptionValue builtIn="false" value="xmlrpc++"/>
+									<listOptionValue builtIn="false" value="xmlrpc"/>
+									<listOptionValue builtIn="false" value="cmph"/>
+									<listOptionValue builtIn="false" value="probingpt"/>
+									<listOptionValue builtIn="false" value="search"/>
+									<listOptionValue builtIn="false" value="OnDiskPt"/>
+									<listOptionValue builtIn="false" value="lm"/>
+									<listOptionValue builtIn="false" value="util"/>
+									<listOptionValue builtIn="false" value="boost_iostreams"/>
+									<listOptionValue builtIn="false" value="boost_system"/>
+									<listOptionValue builtIn="false" value="boost_thread"/>
+									<listOptionValue builtIn="false" value="boost_filesystem"/>
+									<listOptionValue builtIn="false" value="boost_program_options"/>
+									<listOptionValue builtIn="false" value="pthread"/>
+									<listOptionValue builtIn="false" value="z"/>
+									<listOptionValue builtIn="false" value="bz2"/>
+									<listOptionValue builtIn="false" value="dl"/>
+									<listOptionValue builtIn="false" value="rt"/>
+								</option>
+								<option id="gnu.cpp.link.option.paths.1920945405" name="Library search path (-L)" superClass="gnu.cpp.link.option.paths" valueType="libPaths">
+									<listOptionValue builtIn="false" value="&quot;${workspace_loc:}/../../boost/lib64&quot;"/>
+									<listOptionValue builtIn="false" value="&quot;${workspace_loc}/../../cmph/lib&quot;"/>
+									<listOptionValue builtIn="false" value="&quot;${workspace_loc}/../../xmlrpc-c/lib&quot;"/>
+									<listOptionValue builtIn="false" value="&quot;${workspace_loc:}/moses/Debug&quot;"/>
+									<listOptionValue builtIn="false" value="&quot;${workspace_loc:}/moses2/Debug&quot;"/>
+									<listOptionValue builtIn="false" value="&quot;${workspace_loc:}/probingpt/Debug&quot;"/>
+									<listOptionValue builtIn="false" value="&quot;${workspace_loc:}/lm/Debug&quot;"/>
+									<listOptionValue builtIn="false" value="&quot;${workspace_loc:}/OnDiskPt/Debug&quot;"/>
+									<listOptionValue builtIn="false" value="&quot;${workspace_loc:}/util/Debug&quot;"/>
+									<listOptionValue builtIn="false" value="&quot;${workspace_loc:}/search/Debug&quot;"/>
+									<listOptionValue builtIn="false" value="/opt/local/lib"/>
+								</option>
+								<inputType id="cdt.managedbuild.tool.gnu.cpp.linker.input.1508244207" superClass="cdt.managedbuild.tool.gnu.cpp.linker.input">
+									<additionalInput kind="additionalinputdependency" paths="$(USER_OBJS)"/>
+									<additionalInput kind="additionalinput" paths="$(LIBS)"/>
+								</inputType>
+							</tool>
+							<tool id="cdt.managedbuild.tool.gnu.assembler.exe.debug.994919684" name="GCC Assembler" superClass="cdt.managedbuild.tool.gnu.assembler.exe.debug">
+								<inputType id="cdt.managedbuild.tool.gnu.assembler.input.2015973846" superClass="cdt.managedbuild.tool.gnu.assembler.input"/>
+							</tool>
+						</toolChain>
+					</folderInfo>
+				</configuration>
+			</storageModule>
+			<storageModule moduleId="org.eclipse.cdt.core.externalSettings"/>
+		</cconfiguration>
+		<cconfiguration id="cdt.managedbuild.config.gnu.exe.release.347900682">
+			<storageModule buildSystemId="org.eclipse.cdt.managedbuilder.core.configurationDataProvider" id="cdt.managedbuild.config.gnu.exe.release.347900682" moduleId="org.eclipse.cdt.core.settings" name="Release">
+				<externalSettings/>
+				<extensions>
+					<extension id="org.eclipse.cdt.core.ELF" point="org.eclipse.cdt.core.BinaryParser"/>
+					<extension id="org.eclipse.cdt.core.GASErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
+					<extension id="org.eclipse.cdt.core.GmakeErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
+					<extension id="org.eclipse.cdt.core.GLDErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
+					<extension id="org.eclipse.cdt.core.CWDLocator" point="org.eclipse.cdt.core.ErrorParser"/>
+					<extension id="org.eclipse.cdt.core.GCCErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
+				</extensions>
+			</storageModule>
+			<storageModule moduleId="cdtBuildSystem" version="4.0.0">
+				<configuration artifactName="${ProjName}" buildArtefactType="org.eclipse.cdt.build.core.buildArtefactType.exe" buildProperties="org.eclipse.cdt.build.core.buildArtefactType=org.eclipse.cdt.build.core.buildArtefactType.exe,org.eclipse.cdt.build.core.buildType=org.eclipse.cdt.build.core.buildType.release" cleanCommand="rm -rf" description="" id="cdt.managedbuild.config.gnu.exe.release.347900682" name="Release" parent="cdt.managedbuild.config.gnu.exe.release">
+					<folderInfo id="cdt.managedbuild.config.gnu.exe.release.347900682." name="/" resourcePath="">
+						<toolChain id="cdt.managedbuild.toolchain.gnu.exe.release.19950210" name="Linux GCC" superClass="cdt.managedbuild.toolchain.gnu.exe.release">
+							<targetPlatform id="cdt.managedbuild.target.gnu.platform.exe.release.201761026" name="Debug Platform" superClass="cdt.managedbuild.target.gnu.platform.exe.release"/>
+							<builder buildPath="${workspace_loc:/moses2-cmd}/Release" id="cdt.managedbuild.target.gnu.builder.exe.release.249336616" keepEnvironmentInBuildfile="false" managedBuildOn="true" name="Gnu Make Builder" superClass="cdt.managedbuild.target.gnu.builder.exe.release"/>
+							<tool id="cdt.managedbuild.tool.gnu.archiver.base.475854190" name="GCC Archiver" superClass="cdt.managedbuild.tool.gnu.archiver.base"/>
+							<tool id="cdt.managedbuild.tool.gnu.cpp.compiler.exe.release.1047605391" name="GCC C++ Compiler" superClass="cdt.managedbuild.tool.gnu.cpp.compiler.exe.release">
+								<option id="gnu.cpp.compiler.exe.release.option.optimization.level.881009789" name="Optimization Level" superClass="gnu.cpp.compiler.exe.release.option.optimization.level" value="gnu.cpp.compiler.optimization.level.most" valueType="enumerated"/>
+								<option id="gnu.cpp.compiler.exe.release.option.debugging.level.695719104" name="Debug Level" superClass="gnu.cpp.compiler.exe.release.option.debugging.level" value="gnu.cpp.compiler.debugging.level.none" valueType="enumerated"/>
+								<inputType id="cdt.managedbuild.tool.gnu.cpp.compiler.input.2077834205" superClass="cdt.managedbuild.tool.gnu.cpp.compiler.input"/>
+							</tool>
+							<tool id="cdt.managedbuild.tool.gnu.c.compiler.exe.release.534514015" name="GCC C Compiler" superClass="cdt.managedbuild.tool.gnu.c.compiler.exe.release">
+								<option defaultValue="gnu.c.optimization.level.most" id="gnu.c.compiler.exe.release.option.optimization.level.301062410" name="Optimization Level" superClass="gnu.c.compiler.exe.release.option.optimization.level" valueType="enumerated"/>
+								<option id="gnu.c.compiler.exe.release.option.debugging.level.1891262877" name="Debug Level" superClass="gnu.c.compiler.exe.release.option.debugging.level" value="gnu.c.debugging.level.none" valueType="enumerated"/>
+								<inputType id="cdt.managedbuild.tool.gnu.c.compiler.input.176623232" superClass="cdt.managedbuild.tool.gnu.c.compiler.input"/>
+							</tool>
+							<tool id="cdt.managedbuild.tool.gnu.c.linker.exe.release.1762742642" name="GCC C Linker" superClass="cdt.managedbuild.tool.gnu.c.linker.exe.release"/>
+							<tool id="cdt.managedbuild.tool.gnu.cpp.linker.exe.release.563722476" name="GCC C++ Linker" superClass="cdt.managedbuild.tool.gnu.cpp.linker.exe.release">
+								<inputType id="cdt.managedbuild.tool.gnu.cpp.linker.input.1771116495" superClass="cdt.managedbuild.tool.gnu.cpp.linker.input">
+									<additionalInput kind="additionalinputdependency" paths="$(USER_OBJS)"/>
+									<additionalInput kind="additionalinput" paths="$(LIBS)"/>
+								</inputType>
+							</tool>
+							<tool id="cdt.managedbuild.tool.gnu.assembler.exe.release.167166289" name="GCC Assembler" superClass="cdt.managedbuild.tool.gnu.assembler.exe.release">
+								<inputType id="cdt.managedbuild.tool.gnu.assembler.input.659838834" superClass="cdt.managedbuild.tool.gnu.assembler.input"/>
+							</tool>
+						</toolChain>
+					</folderInfo>
+				</configuration>
+			</storageModule>
+			<storageModule moduleId="org.eclipse.cdt.core.externalSettings"/>
+		</cconfiguration>
+	</storageModule>
+	<storageModule moduleId="cdtBuildSystem" version="4.0.0">
+		<project id="moses2-cmd.cdt.managedbuild.target.gnu.exe.1380079855" name="Executable" projectType="cdt.managedbuild.target.gnu.exe"/>
+	</storageModule>
+	<storageModule moduleId="scannerConfiguration">
+		<autodiscovery enabled="true" problemReportingEnabled="true" selectedProfileId=""/>
+		<scannerConfigBuildInfo instanceId="cdt.managedbuild.config.gnu.exe.debug.597260676;cdt.managedbuild.config.gnu.exe.debug.597260676.;cdt.managedbuild.tool.gnu.cpp.compiler.exe.debug.1546967275;cdt.managedbuild.tool.gnu.cpp.compiler.input.123491630">
+			<autodiscovery enabled="true" problemReportingEnabled="true" selectedProfileId=""/>
+		</scannerConfigBuildInfo>
+		<scannerConfigBuildInfo instanceId="cdt.managedbuild.config.gnu.exe.debug.597260676;cdt.managedbuild.config.gnu.exe.debug.597260676.;cdt.managedbuild.tool.gnu.c.compiler.exe.debug.848723608;cdt.managedbuild.tool.gnu.c.compiler.input.1011859741">
+			<autodiscovery enabled="true" problemReportingEnabled="true" selectedProfileId=""/>
+		</scannerConfigBuildInfo>
+		<scannerConfigBuildInfo instanceId="cdt.managedbuild.config.gnu.exe.release.347900682;cdt.managedbuild.config.gnu.exe.release.347900682.;cdt.managedbuild.tool.gnu.c.compiler.exe.release.534514015;cdt.managedbuild.tool.gnu.c.compiler.input.176623232">
+			<autodiscovery enabled="true" problemReportingEnabled="true" selectedProfileId=""/>
+		</scannerConfigBuildInfo>
+		<scannerConfigBuildInfo instanceId="cdt.managedbuild.config.gnu.exe.release.347900682;cdt.managedbuild.config.gnu.exe.release.347900682.;cdt.managedbuild.tool.gnu.cpp.compiler.exe.release.1047605391;cdt.managedbuild.tool.gnu.cpp.compiler.input.2077834205">
+			<autodiscovery enabled="true" problemReportingEnabled="true" selectedProfileId=""/>
+		</scannerConfigBuildInfo>
+	</storageModule>
+	<storageModule moduleId="org.eclipse.cdt.core.LanguageSettingsProviders"/>
+	<storageModule moduleId="refreshScope" versionNumber="2">
+		<configuration configurationName="Debug">
+			<resource resourceType="PROJECT" workspacePath="/moses2-cmd"/>
+		</configuration>
+		<configuration configurationName="Release">
+			<resource resourceType="PROJECT" workspacePath="/moses2-cmd"/>
+		</configuration>
+	</storageModule>
+	<storageModule moduleId="org.eclipse.cdt.internal.ui.text.commentOwnerProjectMappings"/>
+</cproject>
--- a/contrib/other-builds/moses2-cmd/.project
+++ b/contrib/other-builds/moses2-cmd/.project
@ -0,0 +1,44 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<projectDescription>
+	<name>moses2-cmd</name>
+	<comment></comment>
+	<projects>
+		<project>lm</project>
+		<project>moses</project>
+		<project>moses2</project>
+		<project>probingpt</project>
+		<project>util</project>
+	</projects>
+	<buildSpec>
+		<buildCommand>
+			<name>org.eclipse.cdt.managedbuilder.core.genmakebuilder</name>
+			<triggers>clean,full,incremental,</triggers>
+			<arguments>
+			</arguments>
+		</buildCommand>
+		<buildCommand>
+			<name>org.eclipse.cdt.managedbuilder.core.ScannerConfigBuilder</name>
+			<triggers>full,incremental,</triggers>
+			<arguments>
+			</arguments>
+		</buildCommand>
+	</buildSpec>
+	<natures>
+		<nature>org.eclipse.cdt.core.cnature</nature>
+		<nature>org.eclipse.cdt.core.ccnature</nature>
+		<nature>org.eclipse.cdt.managedbuilder.core.managedBuildNature</nature>
+		<nature>org.eclipse.cdt.managedbuilder.core.ScannerConfigNature</nature>
+	</natures>
+	<linkedResources>
+		<link>
+			<name>Main.cpp</name>
+			<type>1</type>
+			<locationURI>PARENT-3-PROJECT_LOC/moses2/Main.cpp</locationURI>
+		</link>
+		<link>
+			<name>Main.h</name>
+			<type>1</type>
+			<locationURI>PARENT-3-PROJECT_LOC/moses2/Main.h</locationURI>
+		</link>
+	</linkedResources>
+</projectDescription>
--- a/contrib/other-builds/moses2/.cproject
+++ b/contrib/other-builds/moses2/.cproject
@ -0,0 +1,180 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<?fileVersion 4.0.0?><cproject storage_type_id="org.eclipse.cdt.core.XmlProjectDescriptionStorage">
+	<storageModule moduleId="org.eclipse.cdt.core.settings">
+		<cconfiguration id="cdt.managedbuild.config.gnu.cross.exe.debug.1097293041">
+			<storageModule buildSystemId="org.eclipse.cdt.managedbuilder.core.configurationDataProvider" id="cdt.managedbuild.config.gnu.cross.exe.debug.1097293041" moduleId="org.eclipse.cdt.core.settings" name="Debug">
+				<externalSettings>
+					<externalSetting>
+						<entry flags="VALUE_WORKSPACE_PATH" kind="includePath" name="/moses2"/>
+						<entry flags="VALUE_WORKSPACE_PATH" kind="libraryPath" name="/moses2/Debug"/>
+						<entry flags="RESOLVED" kind="libraryFile" name="moses2" srcPrefixMapping="" srcRootPath=""/>
+					</externalSetting>
+				</externalSettings>
+				<extensions>
+					<extension id="org.eclipse.cdt.core.ELF" point="org.eclipse.cdt.core.BinaryParser"/>
+					<extension id="org.eclipse.cdt.core.MachO64" point="org.eclipse.cdt.core.BinaryParser"/>
+					<extension id="org.eclipse.cdt.core.GASErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
+					<extension id="org.eclipse.cdt.core.GmakeErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
+					<extension id="org.eclipse.cdt.core.CWDLocator" point="org.eclipse.cdt.core.ErrorParser"/>
+					<extension id="org.eclipse.cdt.core.GCCErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
+				</extensions>
+			</storageModule>
+			<storageModule moduleId="cdtBuildSystem" version="4.0.0">
+				<configuration artifactExtension="a" artifactName="${ProjName}" buildArtefactType="org.eclipse.cdt.build.core.buildArtefactType.staticLib" buildProperties="org.eclipse.cdt.build.core.buildArtefactType=org.eclipse.cdt.build.core.buildArtefactType.staticLib,org.eclipse.cdt.build.core.buildType=org.eclipse.cdt.build.core.buildType.debug" cleanCommand="rm -rf" description="" id="cdt.managedbuild.config.gnu.cross.exe.debug.1097293041" name="Debug" parent="cdt.managedbuild.config.gnu.cross.exe.debug">
+					<folderInfo id="cdt.managedbuild.config.gnu.cross.exe.debug.1097293041." name="/" resourcePath="">
+						<toolChain id="cdt.managedbuild.toolchain.gnu.cross.exe.debug.329828208" name="Cross GCC" superClass="cdt.managedbuild.toolchain.gnu.cross.exe.debug">
+							<targetPlatform archList="all" binaryParser="org.eclipse.cdt.core.ELF;org.eclipse.cdt.core.MachO64" id="cdt.managedbuild.targetPlatform.gnu.cross.389137927" isAbstract="false" osList="all" superClass="cdt.managedbuild.targetPlatform.gnu.cross"/>
+							<builder buildPath="${workspace_loc:/moses2}/Debug" id="cdt.managedbuild.builder.gnu.cross.2144359329" keepEnvironmentInBuildfile="false" managedBuildOn="true" name="Gnu Make Builder" parallelBuildOn="true" parallelizationNumber="optimal" superClass="cdt.managedbuild.builder.gnu.cross"/>
+							<tool id="cdt.managedbuild.tool.gnu.cross.c.compiler.1430831084" name="Cross GCC Compiler" superClass="cdt.managedbuild.tool.gnu.cross.c.compiler">
+								<option defaultValue="gnu.c.optimization.level.none" id="gnu.c.compiler.option.optimization.level.354944414" name="Optimization Level" superClass="gnu.c.compiler.option.optimization.level" useByScannerDiscovery="false" valueType="enumerated"/>
+								<option id="gnu.c.compiler.option.debugging.level.639588389" name="Debug Level" superClass="gnu.c.compiler.option.debugging.level" useByScannerDiscovery="false" value="gnu.c.debugging.level.max" valueType="enumerated"/>
+								<option id="gnu.c.compiler.option.include.paths.7696150" name="Include paths (-I)" superClass="gnu.c.compiler.option.include.paths" useByScannerDiscovery="false"/>
+								<inputType id="cdt.managedbuild.tool.gnu.c.compiler.input.1538601099" superClass="cdt.managedbuild.tool.gnu.c.compiler.input"/>
+							</tool>
+							<tool id="cdt.managedbuild.tool.gnu.cross.cpp.compiler.1686613508" name="Cross G++ Compiler" superClass="cdt.managedbuild.tool.gnu.cross.cpp.compiler">
+								<option id="gnu.cpp.compiler.option.optimization.level.299605809" name="Optimization Level" superClass="gnu.cpp.compiler.option.optimization.level" useByScannerDiscovery="false" value="gnu.cpp.compiler.optimization.level.none" valueType="enumerated"/>
+								<option id="gnu.cpp.compiler.option.debugging.level.769854045" name="Debug Level" superClass="gnu.cpp.compiler.option.debugging.level" useByScannerDiscovery="false" value="gnu.cpp.compiler.debugging.level.max" valueType="enumerated"/>
+								<option id="gnu.cpp.compiler.option.include.paths.1502531988" name="Include paths (-I)" superClass="gnu.cpp.compiler.option.include.paths" useByScannerDiscovery="false" valueType="includePath">
+									<listOptionValue builtIn="false" value="&quot;${workspace_loc}/../../boost/include&quot;"/>
+									<listOptionValue builtIn="false" value="&quot;${workspace_loc}/../../xmlrpc-c/include&quot;"/>
+									<listOptionValue builtIn="false" value="&quot;${workspace_loc}/../../cmph/include&quot;"/>
+									<listOptionValue builtIn="false" value="&quot;${workspace_loc}/../../DALM/include&quot;"/>
+									<listOptionValue builtIn="false" value="&quot;${workspace_loc}/../../DALM/darts-clone&quot;"/>
+									<listOptionValue builtIn="false" value="&quot;${workspace_loc}/../../&quot;"/>
+								</option>
+								<option id="gnu.cpp.compiler.option.preprocessor.def.1025143565" name="Defined symbols (-D)" superClass="gnu.cpp.compiler.option.preprocessor.def" useByScannerDiscovery="false" valueType="definedSymbols">
+									<listOptionValue builtIn="false" value="KENLM_MAX_ORDER=7"/>
+									<listOptionValue builtIn="false" value="HAVE_CMPH"/>
+									<listOptionValue builtIn="false" value="HAVE_PROBINGPT"/>
+									<listOptionValue builtIn="false" value="MAX_NUM_FACTORS=4"/>
+									<listOptionValue builtIn="false" value="WITH_THREADS"/>
+								</option>
+								<inputType id="cdt.managedbuild.tool.gnu.cpp.compiler.input.2101942464" superClass="cdt.managedbuild.tool.gnu.cpp.compiler.input"/>
+							</tool>
+							<tool id="cdt.managedbuild.tool.gnu.cross.c.linker.1439481930" name="Cross GCC Linker" superClass="cdt.managedbuild.tool.gnu.cross.c.linker"/>
+							<tool id="cdt.managedbuild.tool.gnu.cross.cpp.linker.351063004" name="Cross G++ Linker" superClass="cdt.managedbuild.tool.gnu.cross.cpp.linker">
+								<option id="gnu.cpp.link.option.paths.1260140770" name="Library search path (-L)" superClass="gnu.cpp.link.option.paths" valueType="libPaths">
+									<listOptionValue builtIn="false" value="&quot;${workspace_loc:}/../../boost/lib64&quot;"/>
+									<listOptionValue builtIn="false" value="&quot;${workspace_loc}/../../DALM/lib&quot;"/>
+									<listOptionValue builtIn="false" value="&quot;${workspace_loc}/../../cmph/lib&quot;"/>
+									<listOptionValue builtIn="false" value="&quot;${workspace_loc:}/moses/Debug&quot;"/>
+									<listOptionValue builtIn="false" value="&quot;${workspace_loc:}/lm/Debug&quot;"/>
+									<listOptionValue builtIn="false" value="&quot;${workspace_loc:}/OnDiskPt/Debug&quot;"/>
+									<listOptionValue builtIn="false" value="&quot;${workspace_loc:}/util/Debug&quot;"/>
+									<listOptionValue builtIn="false" value="&quot;${workspace_loc:}/search/Debug&quot;"/>
+									<listOptionValue builtIn="false" value="/opt/local/lib"/>
+								</option>
+								<option id="gnu.cpp.link.option.libs.1671854463" name="Libraries (-l)" superClass="gnu.cpp.link.option.libs" valueType="libs">
+									<listOptionValue builtIn="false" value="cmph"/>
+									<listOptionValue builtIn="false" value="dalm"/>
+									<listOptionValue builtIn="false" value="search"/>
+									<listOptionValue builtIn="false" value="OnDiskPt"/>
+									<listOptionValue builtIn="false" value="lm"/>
+									<listOptionValue builtIn="false" value="util"/>
+									<listOptionValue builtIn="false" value="boost_iostreams"/>
+									<listOptionValue builtIn="false" value="boost_serialization"/>
+									<listOptionValue builtIn="false" value="boost_system"/>
+									<listOptionValue builtIn="false" value="boost_thread"/>
+									<listOptionValue builtIn="false" value="boost_filesystem"/>
+									<listOptionValue builtIn="false" value="boost_program_options"/>
+									<listOptionValue builtIn="false" value="pthread"/>
+									<listOptionValue builtIn="false" value="z"/>
+									<listOptionValue builtIn="false" value="bz2"/>
+									<listOptionValue builtIn="false" value="dl"/>
+									<listOptionValue builtIn="false" value="rt"/>
+								</option>
+								<inputType id="cdt.managedbuild.tool.gnu.cpp.linker.input.1955045545" superClass="cdt.managedbuild.tool.gnu.cpp.linker.input">
+									<additionalInput kind="additionalinputdependency" paths="$(USER_OBJS)"/>
+									<additionalInput kind="additionalinput" paths="$(LIBS)"/>
+								</inputType>
+							</tool>
+							<tool id="cdt.managedbuild.tool.gnu.cross.archiver.1028669671" name="Cross GCC Archiver" superClass="cdt.managedbuild.tool.gnu.cross.archiver"/>
+							<tool id="cdt.managedbuild.tool.gnu.cross.assembler.917359146" name="Cross GCC Assembler" superClass="cdt.managedbuild.tool.gnu.cross.assembler">
+								<inputType id="cdt.managedbuild.tool.gnu.assembler.input.254745364" superClass="cdt.managedbuild.tool.gnu.assembler.input"/>
+							</tool>
+						</toolChain>
+					</folderInfo>
+					<fileInfo id="cdt.managedbuild.config.gnu.cross.exe.debug.1097293041.1123771618" name="HypothesisColl.h" rcbsApplicability="disable" resourcePath="HypothesisColl.h" toolsToInvoke=""/>
+					<sourceEntries>
+						<entry excluding="LM/LanguageModelDALM.cpp|defer|Main.cpp|CreateProbingPT2.cpp" flags="VALUE_WORKSPACE_PATH|RESOLVED" kind="sourcePath" name=""/>
+					</sourceEntries>
+				</configuration>
+			</storageModule>
+			<storageModule moduleId="org.eclipse.cdt.core.externalSettings"/>
+		</cconfiguration>
+		<cconfiguration id="cdt.managedbuild.config.gnu.cross.exe.release.1445209421">
+			<storageModule buildSystemId="org.eclipse.cdt.managedbuilder.core.configurationDataProvider" id="cdt.managedbuild.config.gnu.cross.exe.release.1445209421" moduleId="org.eclipse.cdt.core.settings" name="Release">
+				<externalSettings/>
+				<extensions>
+					<extension id="org.eclipse.cdt.core.ELF" point="org.eclipse.cdt.core.BinaryParser"/>
+					<extension id="org.eclipse.cdt.core.MachO64" point="org.eclipse.cdt.core.BinaryParser"/>
+					<extension id="org.eclipse.cdt.core.GASErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
+					<extension id="org.eclipse.cdt.core.GmakeErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
+					<extension id="org.eclipse.cdt.core.GLDErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
+					<extension id="org.eclipse.cdt.core.CWDLocator" point="org.eclipse.cdt.core.ErrorParser"/>
+					<extension id="org.eclipse.cdt.core.GCCErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
+				</extensions>
+			</storageModule>
+			<storageModule moduleId="cdtBuildSystem" version="4.0.0">
+				<configuration artifactName="${ProjName}" buildArtefactType="org.eclipse.cdt.build.core.buildArtefactType.exe" buildProperties="org.eclipse.cdt.build.core.buildArtefactType=org.eclipse.cdt.build.core.buildArtefactType.exe,org.eclipse.cdt.build.core.buildType=org.eclipse.cdt.build.core.buildType.release" cleanCommand="rm -rf" description="" id="cdt.managedbuild.config.gnu.cross.exe.release.1445209421" name="Release" parent="cdt.managedbuild.config.gnu.cross.exe.release">
+					<folderInfo id="cdt.managedbuild.config.gnu.cross.exe.release.1445209421." name="/" resourcePath="">
+						<toolChain id="cdt.managedbuild.toolchain.gnu.cross.exe.release.662721996" name="Cross GCC" superClass="cdt.managedbuild.toolchain.gnu.cross.exe.release">
+							<targetPlatform archList="all" binaryParser="org.eclipse.cdt.core.ELF;org.eclipse.cdt.core.MachO64" id="cdt.managedbuild.targetPlatform.gnu.cross.895874625" isAbstract="false" osList="all" superClass="cdt.managedbuild.targetPlatform.gnu.cross"/>
+							<builder buildPath="${workspace_loc:/moses2}/Release" id="cdt.managedbuild.builder.gnu.cross.468799862" keepEnvironmentInBuildfile="false" managedBuildOn="true" name="Gnu Make Builder" superClass="cdt.managedbuild.builder.gnu.cross"/>
+							<tool id="cdt.managedbuild.tool.gnu.cross.c.compiler.1943249236" name="Cross GCC Compiler" superClass="cdt.managedbuild.tool.gnu.cross.c.compiler">
+								<option defaultValue="gnu.c.optimization.level.most" id="gnu.c.compiler.option.optimization.level.1011693969" name="Optimization Level" superClass="gnu.c.compiler.option.optimization.level" useByScannerDiscovery="false" valueType="enumerated"/>
+								<option id="gnu.c.compiler.option.debugging.level.1339551360" name="Debug Level" superClass="gnu.c.compiler.option.debugging.level" useByScannerDiscovery="false" value="gnu.c.debugging.level.none" valueType="enumerated"/>
+								<inputType id="cdt.managedbuild.tool.gnu.c.compiler.input.1175448562" superClass="cdt.managedbuild.tool.gnu.c.compiler.input"/>
+							</tool>
+							<tool id="cdt.managedbuild.tool.gnu.cross.cpp.compiler.2103617063" name="Cross G++ Compiler" superClass="cdt.managedbuild.tool.gnu.cross.cpp.compiler">
+								<option id="gnu.cpp.compiler.option.optimization.level.13836904" name="Optimization Level" superClass="gnu.cpp.compiler.option.optimization.level" useByScannerDiscovery="false" value="gnu.cpp.compiler.optimization.level.most" valueType="enumerated"/>
+								<option id="gnu.cpp.compiler.option.debugging.level.763147930" name="Debug Level" superClass="gnu.cpp.compiler.option.debugging.level" useByScannerDiscovery="false" value="gnu.cpp.compiler.debugging.level.none" valueType="enumerated"/>
+								<inputType id="cdt.managedbuild.tool.gnu.cpp.compiler.input.946001537" superClass="cdt.managedbuild.tool.gnu.cpp.compiler.input"/>
+							</tool>
+							<tool id="cdt.managedbuild.tool.gnu.cross.c.linker.1462232829" name="Cross GCC Linker" superClass="cdt.managedbuild.tool.gnu.cross.c.linker"/>
+							<tool id="cdt.managedbuild.tool.gnu.cross.cpp.linker.1359778241" name="Cross G++ Linker" superClass="cdt.managedbuild.tool.gnu.cross.cpp.linker">
+								<inputType id="cdt.managedbuild.tool.gnu.cpp.linker.input.89443491" superClass="cdt.managedbuild.tool.gnu.cpp.linker.input">
+									<additionalInput kind="additionalinputdependency" paths="$(USER_OBJS)"/>
+									<additionalInput kind="additionalinput" paths="$(LIBS)"/>
+								</inputType>
+							</tool>
+							<tool id="cdt.managedbuild.tool.gnu.cross.archiver.762494367" name="Cross GCC Archiver" superClass="cdt.managedbuild.tool.gnu.cross.archiver"/>
+							<tool id="cdt.managedbuild.tool.gnu.cross.assembler.140795725" name="Cross GCC Assembler" superClass="cdt.managedbuild.tool.gnu.cross.assembler">
+								<inputType id="cdt.managedbuild.tool.gnu.assembler.input.95131148" superClass="cdt.managedbuild.tool.gnu.assembler.input"/>
+							</tool>
+						</toolChain>
+					</folderInfo>
+				</configuration>
+			</storageModule>
+			<storageModule moduleId="org.eclipse.cdt.core.externalSettings"/>
+		</cconfiguration>
+	</storageModule>
+	<storageModule moduleId="cdtBuildSystem" version="4.0.0">
+		<project id="moses2.cdt.managedbuild.target.gnu.cross.exe.1741914059" name="Executable" projectType="cdt.managedbuild.target.gnu.cross.exe"/>
+	</storageModule>
+	<storageModule moduleId="scannerConfiguration">
+		<autodiscovery enabled="true" problemReportingEnabled="true" selectedProfileId=""/>
+		<scannerConfigBuildInfo instanceId="cdt.managedbuild.config.gnu.cross.exe.release.1445209421;cdt.managedbuild.config.gnu.cross.exe.release.1445209421.;cdt.managedbuild.tool.gnu.cross.c.compiler.1943249236;cdt.managedbuild.tool.gnu.c.compiler.input.1175448562">
+			<autodiscovery enabled="true" problemReportingEnabled="true" selectedProfileId=""/>
+		</scannerConfigBuildInfo>
+		<scannerConfigBuildInfo instanceId="cdt.managedbuild.config.gnu.cross.exe.release.1445209421;cdt.managedbuild.config.gnu.cross.exe.release.1445209421.;cdt.managedbuild.tool.gnu.cross.cpp.compiler.2103617063;cdt.managedbuild.tool.gnu.cpp.compiler.input.946001537">
+			<autodiscovery enabled="true" problemReportingEnabled="true" selectedProfileId=""/>
+		</scannerConfigBuildInfo>
+		<scannerConfigBuildInfo instanceId="cdt.managedbuild.config.gnu.cross.exe.debug.1097293041;cdt.managedbuild.config.gnu.cross.exe.debug.1097293041.;cdt.managedbuild.tool.gnu.cross.c.compiler.1430831084;cdt.managedbuild.tool.gnu.c.compiler.input.1538601099">
+			<autodiscovery enabled="true" problemReportingEnabled="true" selectedProfileId=""/>
+		</scannerConfigBuildInfo>
+		<scannerConfigBuildInfo instanceId="cdt.managedbuild.config.gnu.cross.exe.debug.1097293041;cdt.managedbuild.config.gnu.cross.exe.debug.1097293041.;cdt.managedbuild.tool.gnu.cross.cpp.compiler.1686613508;cdt.managedbuild.tool.gnu.cpp.compiler.input.2101942464">
+			<autodiscovery enabled="true" problemReportingEnabled="true" selectedProfileId=""/>
+		</scannerConfigBuildInfo>
+	</storageModule>
+	<storageModule moduleId="org.eclipse.cdt.core.LanguageSettingsProviders"/>
+	<storageModule moduleId="refreshScope" versionNumber="2">
+		<configuration configurationName="Debug">
+			<resource resourceType="PROJECT" workspacePath="/moses2"/>
+		</configuration>
+		<configuration configurationName="Release">
+			<resource resourceType="PROJECT" workspacePath="/moses2"/>
+		</configuration>
+	</storageModule>
+	<storageModule moduleId="org.eclipse.cdt.internal.ui.text.commentOwnerProjectMappings"/>
+</cproject>
--- a/contrib/other-builds/moses2/.project
+++ b/contrib/other-builds/moses2/.project
--- a/contrib/other-builds/probingpt/.cproject
+++ b/contrib/other-builds/probingpt/.cproject
@ -0,0 +1,119 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<?fileVersion 4.0.0?><cproject storage_type_id="org.eclipse.cdt.core.XmlProjectDescriptionStorage">
+	<storageModule moduleId="org.eclipse.cdt.core.settings">
+		<cconfiguration id="cdt.managedbuild.config.gnu.cross.lib.debug.1390723927">
+			<storageModule buildSystemId="org.eclipse.cdt.managedbuilder.core.configurationDataProvider" id="cdt.managedbuild.config.gnu.cross.lib.debug.1390723927" moduleId="org.eclipse.cdt.core.settings" name="Debug">
+				<externalSettings>
+					<externalSetting>
+						<entry flags="VALUE_WORKSPACE_PATH" kind="includePath" name="/probingpt"/>
+						<entry flags="VALUE_WORKSPACE_PATH" kind="libraryPath" name="/probingpt/Debug"/>
+						<entry flags="RESOLVED" kind="libraryFile" name="probingpt" srcPrefixMapping="" srcRootPath=""/>
+					</externalSetting>
+				</externalSettings>
+				<extensions>
+					<extension id="org.eclipse.cdt.core.ELF" point="org.eclipse.cdt.core.BinaryParser"/>
+					<extension id="org.eclipse.cdt.core.GASErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
+					<extension id="org.eclipse.cdt.core.GmakeErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
+					<extension id="org.eclipse.cdt.core.CWDLocator" point="org.eclipse.cdt.core.ErrorParser"/>
+					<extension id="org.eclipse.cdt.core.GCCErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
+				</extensions>
+			</storageModule>
+			<storageModule moduleId="cdtBuildSystem" version="4.0.0">
+				<configuration artifactExtension="a" artifactName="${ProjName}" buildArtefactType="org.eclipse.cdt.build.core.buildArtefactType.staticLib" buildProperties="org.eclipse.cdt.build.core.buildArtefactType=org.eclipse.cdt.build.core.buildArtefactType.staticLib,org.eclipse.cdt.build.core.buildType=org.eclipse.cdt.build.core.buildType.debug" cleanCommand="rm -rf" description="" id="cdt.managedbuild.config.gnu.cross.lib.debug.1390723927" name="Debug" parent="cdt.managedbuild.config.gnu.cross.lib.debug">
+					<folderInfo id="cdt.managedbuild.config.gnu.cross.lib.debug.1390723927." name="/" resourcePath="">
+						<toolChain id="cdt.managedbuild.toolchain.gnu.cross.lib.debug.783086312" name="Cross GCC" superClass="cdt.managedbuild.toolchain.gnu.cross.lib.debug">
+							<targetPlatform archList="all" binaryParser="org.eclipse.cdt.core.ELF" id="cdt.managedbuild.targetPlatform.gnu.cross.965048061" isAbstract="false" osList="all" superClass="cdt.managedbuild.targetPlatform.gnu.cross"/>
+							<builder buildPath="${workspace_loc:/probingpt}/Debug" id="cdt.managedbuild.builder.gnu.cross.2025185909" keepEnvironmentInBuildfile="false" managedBuildOn="true" name="Gnu Make Builder" superClass="cdt.managedbuild.builder.gnu.cross"/>
+							<tool id="cdt.managedbuild.tool.gnu.cross.c.compiler.728323531" name="Cross GCC Compiler" superClass="cdt.managedbuild.tool.gnu.cross.c.compiler">
+								<option defaultValue="gnu.c.optimization.level.none" id="gnu.c.compiler.option.optimization.level.781305932" name="Optimization Level" superClass="gnu.c.compiler.option.optimization.level" useByScannerDiscovery="false" valueType="enumerated"/>
+								<option id="gnu.c.compiler.option.debugging.level.1265608945" name="Debug Level" superClass="gnu.c.compiler.option.debugging.level" useByScannerDiscovery="false" value="gnu.c.debugging.level.max" valueType="enumerated"/>
+								<inputType id="cdt.managedbuild.tool.gnu.c.compiler.input.183186094" superClass="cdt.managedbuild.tool.gnu.c.compiler.input"/>
+							</tool>
+							<tool id="cdt.managedbuild.tool.gnu.cross.cpp.compiler.1845983140" name="Cross G++ Compiler" superClass="cdt.managedbuild.tool.gnu.cross.cpp.compiler">
+								<option id="gnu.cpp.compiler.option.optimization.level.961528826" name="Optimization Level" superClass="gnu.cpp.compiler.option.optimization.level" useByScannerDiscovery="false" value="gnu.cpp.compiler.optimization.level.none" valueType="enumerated"/>
+								<option id="gnu.cpp.compiler.option.debugging.level.754161376" name="Debug Level" superClass="gnu.cpp.compiler.option.debugging.level" useByScannerDiscovery="false" value="gnu.cpp.compiler.debugging.level.max" valueType="enumerated"/>
+								<option id="gnu.cpp.compiler.option.include.paths.1356496512" name="Include paths (-I)" superClass="gnu.cpp.compiler.option.include.paths" useByScannerDiscovery="false" valueType="includePath">
+									<listOptionValue builtIn="false" value="&quot;${workspace_loc}/../../boost/include&quot;"/>
+									<listOptionValue builtIn="false" value="&quot;${workspace_loc}/../../&quot;"/>
+								</option>
+								<inputType id="cdt.managedbuild.tool.gnu.cpp.compiler.input.1147743441" superClass="cdt.managedbuild.tool.gnu.cpp.compiler.input"/>
+							</tool>
+							<tool id="cdt.managedbuild.tool.gnu.cross.c.linker.423549467" name="Cross GCC Linker" superClass="cdt.managedbuild.tool.gnu.cross.c.linker"/>
+							<tool id="cdt.managedbuild.tool.gnu.cross.cpp.linker.10489702" name="Cross G++ Linker" superClass="cdt.managedbuild.tool.gnu.cross.cpp.linker"/>
+							<tool id="cdt.managedbuild.tool.gnu.cross.archiver.1080671388" name="Cross GCC Archiver" superClass="cdt.managedbuild.tool.gnu.cross.archiver"/>
+							<tool id="cdt.managedbuild.tool.gnu.cross.assembler.686017340" name="Cross GCC Assembler" superClass="cdt.managedbuild.tool.gnu.cross.assembler">
+								<inputType id="cdt.managedbuild.tool.gnu.assembler.input.599273349" superClass="cdt.managedbuild.tool.gnu.assembler.input"/>
+							</tool>
+						</toolChain>
+					</folderInfo>
+				</configuration>
+			</storageModule>
+			<storageModule moduleId="org.eclipse.cdt.core.externalSettings"/>
+		</cconfiguration>
+		<cconfiguration id="cdt.managedbuild.config.gnu.cross.lib.release.890240571">
+			<storageModule buildSystemId="org.eclipse.cdt.managedbuilder.core.configurationDataProvider" id="cdt.managedbuild.config.gnu.cross.lib.release.890240571" moduleId="org.eclipse.cdt.core.settings" name="Release">
+				<externalSettings>
+					<externalSetting>
+						<entry flags="VALUE_WORKSPACE_PATH" kind="includePath" name="/probingpt"/>
+						<entry flags="VALUE_WORKSPACE_PATH" kind="libraryPath" name="/probingpt/Release"/>
+						<entry flags="RESOLVED" kind="libraryFile" name="probingpt" srcPrefixMapping="" srcRootPath=""/>
+					</externalSetting>
+				</externalSettings>
+				<extensions>
+					<extension id="org.eclipse.cdt.core.ELF" point="org.eclipse.cdt.core.BinaryParser"/>
+					<extension id="org.eclipse.cdt.core.GASErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
+					<extension id="org.eclipse.cdt.core.GmakeErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
+					<extension id="org.eclipse.cdt.core.CWDLocator" point="org.eclipse.cdt.core.ErrorParser"/>
+					<extension id="org.eclipse.cdt.core.GCCErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
+				</extensions>
+			</storageModule>
+			<storageModule moduleId="cdtBuildSystem" version="4.0.0">
+				<configuration artifactExtension="a" artifactName="${ProjName}" buildArtefactType="org.eclipse.cdt.build.core.buildArtefactType.staticLib" buildProperties="org.eclipse.cdt.build.core.buildArtefactType=org.eclipse.cdt.build.core.buildArtefactType.staticLib,org.eclipse.cdt.build.core.buildType=org.eclipse.cdt.build.core.buildType.release" cleanCommand="rm -rf" description="" id="cdt.managedbuild.config.gnu.cross.lib.release.890240571" name="Release" parent="cdt.managedbuild.config.gnu.cross.lib.release">
+					<folderInfo id="cdt.managedbuild.config.gnu.cross.lib.release.890240571." name="/" resourcePath="">
+						<toolChain id="cdt.managedbuild.toolchain.gnu.cross.lib.release.885632145" name="Cross GCC" superClass="cdt.managedbuild.toolchain.gnu.cross.lib.release">
+							<targetPlatform archList="all" binaryParser="org.eclipse.cdt.core.ELF" id="cdt.managedbuild.targetPlatform.gnu.cross.1366680617" isAbstract="false" osList="all" superClass="cdt.managedbuild.targetPlatform.gnu.cross"/>
+							<builder buildPath="${workspace_loc:/probingpt}/Release" id="cdt.managedbuild.builder.gnu.cross.191907712" keepEnvironmentInBuildfile="false" managedBuildOn="true" name="Gnu Make Builder" superClass="cdt.managedbuild.builder.gnu.cross"/>
+							<tool id="cdt.managedbuild.tool.gnu.cross.c.compiler.808139639" name="Cross GCC Compiler" superClass="cdt.managedbuild.tool.gnu.cross.c.compiler">
+								<option defaultValue="gnu.c.optimization.level.most" id="gnu.c.compiler.option.optimization.level.224210565" name="Optimization Level" superClass="gnu.c.compiler.option.optimization.level" useByScannerDiscovery="false" valueType="enumerated"/>
+								<option id="gnu.c.compiler.option.debugging.level.53109509" name="Debug Level" superClass="gnu.c.compiler.option.debugging.level" useByScannerDiscovery="false" value="gnu.c.debugging.level.none" valueType="enumerated"/>
+								<inputType id="cdt.managedbuild.tool.gnu.c.compiler.input.474846340" superClass="cdt.managedbuild.tool.gnu.c.compiler.input"/>
+							</tool>
+							<tool id="cdt.managedbuild.tool.gnu.cross.cpp.compiler.825247044" name="Cross G++ Compiler" superClass="cdt.managedbuild.tool.gnu.cross.cpp.compiler">
+								<option id="gnu.cpp.compiler.option.optimization.level.1484316632" name="Optimization Level" superClass="gnu.cpp.compiler.option.optimization.level" useByScannerDiscovery="false" value="gnu.cpp.compiler.optimization.level.most" valueType="enumerated"/>
+								<option id="gnu.cpp.compiler.option.debugging.level.1585066416" name="Debug Level" superClass="gnu.cpp.compiler.option.debugging.level" useByScannerDiscovery="false" value="gnu.cpp.compiler.debugging.level.none" valueType="enumerated"/>
+								<inputType id="cdt.managedbuild.tool.gnu.cpp.compiler.input.149988472" superClass="cdt.managedbuild.tool.gnu.cpp.compiler.input"/>
+							</tool>
+							<tool id="cdt.managedbuild.tool.gnu.cross.c.linker.1042542443" name="Cross GCC Linker" superClass="cdt.managedbuild.tool.gnu.cross.c.linker"/>
+							<tool id="cdt.managedbuild.tool.gnu.cross.cpp.linker.1342788782" name="Cross G++ Linker" superClass="cdt.managedbuild.tool.gnu.cross.cpp.linker"/>
+							<tool id="cdt.managedbuild.tool.gnu.cross.archiver.56708033" name="Cross GCC Archiver" superClass="cdt.managedbuild.tool.gnu.cross.archiver"/>
+							<tool id="cdt.managedbuild.tool.gnu.cross.assembler.1620829860" name="Cross GCC Assembler" superClass="cdt.managedbuild.tool.gnu.cross.assembler">
+								<inputType id="cdt.managedbuild.tool.gnu.assembler.input.1936527691" superClass="cdt.managedbuild.tool.gnu.assembler.input"/>
+							</tool>
+						</toolChain>
+					</folderInfo>
+				</configuration>
+			</storageModule>
+			<storageModule moduleId="org.eclipse.cdt.core.externalSettings"/>
+		</cconfiguration>
+	</storageModule>
+	<storageModule moduleId="cdtBuildSystem" version="4.0.0">
+		<project id="probingpt.cdt.managedbuild.target.gnu.cross.lib.1901222999" name="Static Library" projectType="cdt.managedbuild.target.gnu.cross.lib"/>
+	</storageModule>
+	<storageModule moduleId="scannerConfiguration">
+		<autodiscovery enabled="true" problemReportingEnabled="true" selectedProfileId=""/>
+		<scannerConfigBuildInfo instanceId="cdt.managedbuild.config.gnu.cross.lib.release.890240571;cdt.managedbuild.config.gnu.cross.lib.release.890240571.;cdt.managedbuild.tool.gnu.cross.cpp.compiler.825247044;cdt.managedbuild.tool.gnu.cpp.compiler.input.149988472">
+			<autodiscovery enabled="true" problemReportingEnabled="true" selectedProfileId=""/>
+		</scannerConfigBuildInfo>
+		<scannerConfigBuildInfo instanceId="cdt.managedbuild.config.gnu.cross.lib.release.890240571;cdt.managedbuild.config.gnu.cross.lib.release.890240571.;cdt.managedbuild.tool.gnu.cross.c.compiler.808139639;cdt.managedbuild.tool.gnu.c.compiler.input.474846340">
+			<autodiscovery enabled="true" problemReportingEnabled="true" selectedProfileId=""/>
+		</scannerConfigBuildInfo>
+		<scannerConfigBuildInfo instanceId="cdt.managedbuild.config.gnu.cross.lib.debug.1390723927;cdt.managedbuild.config.gnu.cross.lib.debug.1390723927.;cdt.managedbuild.tool.gnu.cross.cpp.compiler.1845983140;cdt.managedbuild.tool.gnu.cpp.compiler.input.1147743441">
+			<autodiscovery enabled="true" problemReportingEnabled="true" selectedProfileId=""/>
+		</scannerConfigBuildInfo>
+		<scannerConfigBuildInfo instanceId="cdt.managedbuild.config.gnu.cross.lib.debug.1390723927;cdt.managedbuild.config.gnu.cross.lib.debug.1390723927.;cdt.managedbuild.tool.gnu.cross.c.compiler.728323531;cdt.managedbuild.tool.gnu.c.compiler.input.183186094">
+			<autodiscovery enabled="true" problemReportingEnabled="true" selectedProfileId=""/>
+		</scannerConfigBuildInfo>
+	</storageModule>
+	<storageModule moduleId="org.eclipse.cdt.core.LanguageSettingsProviders"/>
+	<storageModule moduleId="refreshScope"/>
+</cproject>
--- a/contrib/other-builds/probingpt/.project
+++ b/contrib/other-builds/probingpt/.project
@ -0,0 +1,34 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<projectDescription>
+	<name>probingpt</name>
+	<comment></comment>
+	<projects>
+	</projects>
+	<buildSpec>
+		<buildCommand>
+			<name>org.eclipse.cdt.managedbuilder.core.genmakebuilder</name>
+			<triggers>clean,full,incremental,</triggers>
+			<arguments>
+			</arguments>
+		</buildCommand>
+		<buildCommand>
+			<name>org.eclipse.cdt.managedbuilder.core.ScannerConfigBuilder</name>
+			<triggers>full,incremental,</triggers>
+			<arguments>
+			</arguments>
+		</buildCommand>
+	</buildSpec>
+	<natures>
+		<nature>org.eclipse.cdt.core.cnature</nature>
+		<nature>org.eclipse.cdt.core.ccnature</nature>
+		<nature>org.eclipse.cdt.managedbuilder.core.managedBuildNature</nature>
+		<nature>org.eclipse.cdt.managedbuilder.core.ScannerConfigNature</nature>
+	</natures>
+	<linkedResources>
+		<link>
+			<name>probingpt</name>
+			<type>2</type>
+			<locationURI>PARENT-3-PROJECT_LOC/probingpt</locationURI>
+		</link>
+	</linkedResources>
+</projectDescription>
--- a/contrib/other-builds/server/.cproject
+++ b/contrib/other-builds/server/.cproject
@ -6,6 +6,7 @@
 				<externalSettings/>
 				<extensions>
 					<extension id="org.eclipse.cdt.core.ELF" point="org.eclipse.cdt.core.BinaryParser"/>
+					<extension id="org.eclipse.cdt.core.MachO64" point="org.eclipse.cdt.core.BinaryParser"/>
 					<extension id="org.eclipse.cdt.core.GmakeErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
 					<extension id="org.eclipse.cdt.core.CWDLocator" point="org.eclipse.cdt.core.ErrorParser"/>
 					<extension id="org.eclipse.cdt.core.GCCErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
@ -17,7 +18,7 @@
 				<configuration artifactName="${ProjName}" buildArtefactType="org.eclipse.cdt.build.core.buildArtefactType.exe" buildProperties="org.eclipse.cdt.build.core.buildArtefactType=org.eclipse.cdt.build.core.buildArtefactType.exe,org.eclipse.cdt.build.core.buildType=org.eclipse.cdt.build.core.buildType.debug" cleanCommand="rm -rf" description="" id="cdt.managedbuild.config.gnu.exe.debug.1015532240" name="Debug" parent="cdt.managedbuild.config.gnu.exe.debug">
 					<folderInfo id="cdt.managedbuild.config.gnu.exe.debug.1015532240." name="/" resourcePath="">
 						<toolChain id="cdt.managedbuild.toolchain.gnu.exe.debug.1201298107" name="Linux GCC" superClass="cdt.managedbuild.toolchain.gnu.exe.debug">
-							<targetPlatform id="cdt.managedbuild.target.gnu.platform.exe.debug.2097807873" name="Debug Platform" superClass="cdt.managedbuild.target.gnu.platform.exe.debug"/>
+							<targetPlatform binaryParser="org.eclipse.cdt.core.ELF;org.eclipse.cdt.core.MachO64" id="cdt.managedbuild.target.gnu.platform.exe.debug.2097807873" name="Debug Platform" superClass="cdt.managedbuild.target.gnu.platform.exe.debug"/>
 							<builder buildPath="${workspace_loc:/server}/Debug" id="cdt.managedbuild.target.gnu.builder.exe.debug.857185882" keepEnvironmentInBuildfile="false" managedBuildOn="true" name="Gnu Make Builder" parallelBuildOn="true" parallelizationNumber="optimal" superClass="cdt.managedbuild.target.gnu.builder.exe.debug"/>
 							<tool id="cdt.managedbuild.tool.gnu.archiver.base.142173353" name="GCC Archiver" superClass="cdt.managedbuild.tool.gnu.archiver.base"/>
 							<tool id="cdt.managedbuild.tool.gnu.cpp.compiler.exe.debug.1657626940" name="GCC C++ Compiler" superClass="cdt.managedbuild.tool.gnu.cpp.compiler.exe.debug">
@ -44,6 +45,7 @@
 							<tool id="cdt.managedbuild.tool.gnu.cpp.linker.exe.debug.1443553047" name="GCC C++ Linker" superClass="cdt.managedbuild.tool.gnu.cpp.linker.exe.debug">
 								<option id="gnu.cpp.link.option.paths.1096041402" name="Library search path (-L)" superClass="gnu.cpp.link.option.paths" valueType="libPaths">
 									<listOptionValue builtIn="false" value="&quot;${workspace_loc}/../../xmlrpc-c/lib&quot;"/>
+									<listOptionValue builtIn="false" value="&quot;${workspace_loc:}/probingpt/Debug&quot;"/>
 									<listOptionValue builtIn="false" value="&quot;${workspace_loc}/../../cmph/lib&quot;"/>
 									<listOptionValue builtIn="false" value="&quot;${workspace_loc:}/search/Debug&quot;"/>
 									<listOptionValue builtIn="false" value="&quot;${workspace_loc:}/moses/Debug&quot;"/>
@ -51,8 +53,10 @@
 									<listOptionValue builtIn="false" value="&quot;${workspace_loc:}/lm/Debug&quot;"/>
 									<listOptionValue builtIn="false" value="&quot;${workspace_loc:}/util/Debug&quot;"/>
 									<listOptionValue builtIn="false" value="&quot;${workspace_loc:}/../../boost/lib64&quot;"/>
+									<listOptionValue builtIn="false" value="/opt/local/lib"/>
 								</option>
 								<option id="gnu.cpp.link.option.libs.1087215166" name="Libraries (-l)" superClass="gnu.cpp.link.option.libs" valueType="libs">
+									<listOptionValue builtIn="false" value="moses"/>
 									<listOptionValue builtIn="false" value="xmlrpc_xmltok"/>
 									<listOptionValue builtIn="false" value="xmlrpc_xmlparse"/>
 									<listOptionValue builtIn="false" value="xmlrpc_util++"/>
@ -64,7 +68,6 @@
 									<listOptionValue builtIn="false" value="xmlrpc_abyss"/>
 									<listOptionValue builtIn="false" value="xmlrpc++"/>
 									<listOptionValue builtIn="false" value="xmlrpc"/>
-									<listOptionValue builtIn="false" value="moses"/>
 									<listOptionValue builtIn="false" value="cmph"/>
 									<listOptionValue builtIn="false" value="search"/>
 									<listOptionValue builtIn="false" value="OnDiskPt"/>
@ -78,6 +81,7 @@
 									<listOptionValue builtIn="false" value="boost_filesystem"/>
 									<listOptionValue builtIn="false" value="boost_program_options"/>
 									<listOptionValue builtIn="false" value="z"/>
+									<listOptionValue builtIn="false" value="probingpt"/>
 									<listOptionValue builtIn="false" value="bz2"/>
 									<listOptionValue builtIn="false" value="dl"/>
 									<listOptionValue builtIn="false" value="rt"/>
@ -101,6 +105,7 @@
 				<externalSettings/>
 				<extensions>
 					<extension id="org.eclipse.cdt.core.ELF" point="org.eclipse.cdt.core.BinaryParser"/>
+					<extension id="org.eclipse.cdt.core.MachO64" point="org.eclipse.cdt.core.BinaryParser"/>
 					<extension id="org.eclipse.cdt.core.GmakeErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
 					<extension id="org.eclipse.cdt.core.CWDLocator" point="org.eclipse.cdt.core.ErrorParser"/>
 					<extension id="org.eclipse.cdt.core.GCCErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
@ -112,7 +117,7 @@
 				<configuration artifactName="${ProjName}" buildArtefactType="org.eclipse.cdt.build.core.buildArtefactType.exe" buildProperties="org.eclipse.cdt.build.core.buildArtefactType=org.eclipse.cdt.build.core.buildArtefactType.exe,org.eclipse.cdt.build.core.buildType=org.eclipse.cdt.build.core.buildType.release" cleanCommand="rm -rf" description="" id="cdt.managedbuild.config.gnu.exe.release.179761083" name="Release" parent="cdt.managedbuild.config.gnu.exe.release">
 					<folderInfo id="cdt.managedbuild.config.gnu.exe.release.179761083." name="/" resourcePath="">
 						<toolChain id="cdt.managedbuild.toolchain.gnu.exe.release.2024222442" name="Linux GCC" superClass="cdt.managedbuild.toolchain.gnu.exe.release">
-							<targetPlatform id="cdt.managedbuild.target.gnu.platform.exe.release.1098252145" name="Debug Platform" superClass="cdt.managedbuild.target.gnu.platform.exe.release"/>
+							<targetPlatform binaryParser="org.eclipse.cdt.core.ELF;org.eclipse.cdt.core.MachO64" id="cdt.managedbuild.target.gnu.platform.exe.release.1098252145" name="Debug Platform" superClass="cdt.managedbuild.target.gnu.platform.exe.release"/>
 							<builder buildPath="${workspace_loc:/server}/Release" id="cdt.managedbuild.target.gnu.builder.exe.release.24884855" keepEnvironmentInBuildfile="false" managedBuildOn="true" name="Gnu Make Builder" superClass="cdt.managedbuild.target.gnu.builder.exe.release"/>
 							<tool id="cdt.managedbuild.tool.gnu.archiver.base.1561001393" name="GCC Archiver" superClass="cdt.managedbuild.tool.gnu.archiver.base"/>
 							<tool id="cdt.managedbuild.tool.gnu.cpp.compiler.exe.release.1260095073" name="GCC C++ Compiler" superClass="cdt.managedbuild.tool.gnu.cpp.compiler.exe.release">
--- a/contrib/other-builds/util/.cproject
+++ b/contrib/other-builds/util/.cproject
@ -20,11 +20,11 @@
 				</extensions>
 			</storageModule>
 			<storageModule moduleId="cdtBuildSystem" version="4.0.0">
-				<configuration artifactExtension="a" artifactName="${ProjName}" buildArtefactType="org.eclipse.cdt.build.core.buildArtefactType.staticLib" buildProperties="org.eclipse.cdt.build.core.buildType=org.eclipse.cdt.build.core.buildType.debug,org.eclipse.cdt.build.core.buildArtefactType=org.eclipse.cdt.build.core.buildArtefactType.staticLib" cleanCommand="rm -rf" description="" id="cdt.managedbuild.config.gnu.macosx.exe.debug.1869657447" name="Debug" parent="cdt.managedbuild.config.gnu.macosx.exe.debug">
+				<configuration artifactExtension="a" artifactName="${ProjName}" buildArtefactType="org.eclipse.cdt.build.core.buildArtefactType.staticLib" buildProperties="org.eclipse.cdt.build.core.buildArtefactType=org.eclipse.cdt.build.core.buildArtefactType.staticLib,org.eclipse.cdt.build.core.buildType=org.eclipse.cdt.build.core.buildType.debug" cleanCommand="rm -rf" description="" id="cdt.managedbuild.config.gnu.macosx.exe.debug.1869657447" name="Debug" parent="cdt.managedbuild.config.gnu.macosx.exe.debug">
 					<folderInfo id="cdt.managedbuild.config.gnu.macosx.exe.debug.1869657447." name="/" resourcePath="">
 						<toolChain id="cdt.managedbuild.toolchain.gnu.macosx.exe.debug.1388624938" name="MacOSX GCC" superClass="cdt.managedbuild.toolchain.gnu.macosx.exe.debug">
 							<targetPlatform binaryParser="org.eclipse.cdt.core.MachO64;org.eclipse.cdt.core.ELF" id="cdt.managedbuild.target.gnu.platform.macosx.exe.debug.1873607607" name="Debug Platform" superClass="cdt.managedbuild.target.gnu.platform.macosx.exe.debug"/>
-							<builder buildPath="${workspace_loc:/util/Debug}" id="cdt.managedbuild.target.gnu.builder.macosx.exe.debug.2045214944" keepEnvironmentInBuildfile="false" managedBuildOn="true" name="Gnu Make Builder" parallelBuildOn="true" parallelizationNumber="3" superClass="cdt.managedbuild.target.gnu.builder.macosx.exe.debug"/>
+							<builder buildPath="${workspace_loc:/util/Debug}" id="cdt.managedbuild.target.gnu.builder.macosx.exe.debug.2045214944" keepEnvironmentInBuildfile="false" managedBuildOn="true" name="Gnu Make Builder" parallelBuildOn="true" parallelizationNumber="optimal" superClass="cdt.managedbuild.target.gnu.builder.macosx.exe.debug"/>
 							<tool id="cdt.managedbuild.tool.macosx.c.linker.macosx.exe.debug.589471640" name="MacOS X C Linker" superClass="cdt.managedbuild.tool.macosx.c.linker.macosx.exe.debug"/>
 							<tool id="cdt.managedbuild.tool.macosx.cpp.linker.macosx.exe.debug.1543780089" name="MacOS X C++ Linker" superClass="cdt.managedbuild.tool.macosx.cpp.linker.macosx.exe.debug">
 								<inputType id="cdt.managedbuild.tool.macosx.cpp.linker.input.635667684" superClass="cdt.managedbuild.tool.macosx.cpp.linker.input">
@ -137,10 +137,10 @@
 		</scannerConfigBuildInfo>
 	</storageModule>
 	<storageModule moduleId="refreshScope" versionNumber="2">
-		<configuration configurationName="Release">
+		<configuration configurationName="Debug">
 			<resource resourceType="PROJECT" workspacePath="/util"/>
 		</configuration>
-		<configuration configurationName="Debug">
+		<configuration configurationName="Release">
 			<resource resourceType="PROJECT" workspacePath="/util"/>
 		</configuration>
 	</storageModule>
--- a/contrib/server/client-stdin.perl
+++ b/contrib/server/client-stdin.perl
@ -0,0 +1,41 @@
+#!/usr/bin/env perl
+
+#
+# Sample client for mosesserver, illustrating allignment info and
+# report all factors
+#
+use strict;
+use Encode;
+use XMLRPC::Lite;
+use utf8;
+
+binmode(STDIN,  ":utf8");
+
+
+my $url = "http://localhost:8080/RPC2";
+my $proxy = XMLRPC::Lite->proxy($url);
+
+my $text;
+while ($text = <STDIN>) {
+#for (my $i = 0; $i < scalar(@doc); ++$i) {
+#  my $text = $doc[$i];
+
+  # Work-around for XMLRPC::Lite bug
+  #my $encoded = SOAP::Data->type(string => Encode::encode("utf8",$text));
+  my $encoded = SOAP::Data->type(string => $text);
+
+  my %param = ("text" => $encoded, "align" => "true", "report-all-factors" => "true");
+  my $result = $proxy->call("translate",\%param)->result;
+  print $result->{'text'} . "\n";
+  if ($result->{'align'}) {
+      print "Phrase alignments: \n";
+      my $aligns = $result->{'align'};
+      foreach my $align (@$aligns) {
+        print $align->{'tgt-start'} . "," . $align->{'src-start'} . "," 
+            . $align->{'src-end'} . "\n"; 
+      }
+  }
+}
+
+
+
--- a/contrib/server/client.perl
+++ b/contrib/server/client.perl
@ -4,27 +4,40 @@
 # Sample client for mosesserver, illustrating allignment info and
 # report all factors
 #
-
+use strict;
 use Encode;
 use XMLRPC::Lite;
 use utf8;

-$url = "http://localhost:8080/RPC2";
-$proxy = XMLRPC::Lite->proxy($url);
+my $url = "http://localhost:8080/RPC2";
+my $proxy = XMLRPC::Lite->proxy($url);

-$text = "il a souhaité que la présidence trace à nice le chemin pour l' avenir .";
+my @doc = ("monsieur le président , ce que nous devrons toutefois également faire à biarritz , c' est regarder un peu plus loin .",
+	"les élus que nous sommes avons au moins autant le devoir de l' encourager à progresser , en dépit de l' adversité , que de relayer les messages que nous recevons de l' opinion publique dans chacun de nos pays .",
+	"au regard des événements de ces derniers temps , la question du prix de l' essence me semble elle aussi particulièrement remarquable .",
+	"à l' heure actuelle , le conseil est en train d' examiner l' inclusion de tels mécanismes dans l' article 7 .",
+	"deuxièmement , dans la transparence pour les citoyens , qui connaissent à présent les droits dont ils disposent vis-à-vis de ceux qui appliquent et élaborent le droit européen , et pour ceux qui , justement , appliquent et élaborent ce droit européen .");
+#print STDERR scalar(@doc);

-# Work-around for XMLRPC::Lite bug
-$encoded = SOAP::Data->type(string => Encode::encode("utf8",$text));
+for (my $i = 0; $i < scalar(@doc); ++$i) {
+  my $text = $doc[$i];

-my %param = ("text" => $encoded, "align" => "true", "report-all-factors" => "true");
-$result = $proxy->call("translate",\%param)->result;
-print $result->{'text'} . "\n";
-if ($result->{'align'}) {
-    print "Phrase alignments: \n";
-    $aligns = $result->{'align'};
-    foreach my $align (@$aligns) {
+  # Work-around for XMLRPC::Lite bug
+  my $encoded = SOAP::Data->type(string => Encode::encode("utf8",$text));
+  #$encoded = SOAP::Data->type(string => $text);
+
+  my %param = ("text" => $encoded, "align" => "true", "report-all-factors" => "true");
+  my $result = $proxy->call("translate",\%param)->result;
+  print $result->{'text'} . "\n";
+  if ($result->{'align'}) {
+      print "Phrase alignments: \n";
+      my $aligns = $result->{'align'};
+      foreach my $align (@$aligns) {
        print $align->{'tgt-start'} . "," . $align->{'src-start'} . "," 
            . $align->{'src-end'} . "\n"; 
-    }
+      }
+  }
 }
+
+
+
--- a/contrib/sigtest-filter/Makefile
+++ b/contrib/sigtest-filter/Makefile
@ -1,5 +1,5 @@
 SALMDIR=/Users/hieuhoang/workspace/salm
-FLAVOR?=o32
+FLAVOR?=o64
 INC=-I$(SALMDIR)/Src/Shared -I$(SALMDIR)/Src/SuffixArrayApplications -I$(SALMDIR)/Src/SuffixArrayApplications/SuffixArraySearch
 OBJS=$(SALMDIR)/Distribution/Linux/Objs/Search/_SuffixArrayApplicationBase.$(FLAVOR) $(SALMDIR)/Distribution/Linux/Objs/Search/_SuffixArraySearchApplicationBase.$(FLAVOR) $(SALMDIR)/Distribution/Linux/Objs/Shared/_String.$(FLAVOR) $(SALMDIR)/Distribution/Linux/Objs/Shared/_IDVocabulary.$(FLAVOR)

--- a/mert/ForestRescore.cpp
+++ b/mert/ForestRescore.cpp
@ -346,7 +346,7 @@ static void GetBestHypothesis(size_t vertexId, const Graph& graph, const vector<

 void Viterbi(const Graph& graph, const SparseVector& weights, float bleuWeight, const ReferenceSet& references , size_t sentenceId, const std::vector<FeatureStatsType>& backgroundBleu,  HgHypothesis* bestHypo)
 {
-  BackPointer init(NULL,kMinScore);
+  BackPointer init((const Edge*) NULL,kMinScore);
  vector<BackPointer> backPointers(graph.VertexSize(),init);
  HgBleuScorer bleuScorer(references, graph, sentenceId, backgroundBleu);
  vector<FeatureStatsType> winnerStats(kBleuNgramOrder*2+1);
--- a/misc/1-1-Extraction.cpp
+++ b/misc/1-1-Extraction.cpp
@ -216,7 +216,9 @@ int main(int argc, char * argv[])
    getWords(f[i],currF);
    getWords(a[i],currA);

-    cerr<<"Processing "<<i<<endl;
+    if (i % 100000 == 0) {
+      cerr<<"Processing "<<i<<endl;
+    }
    constructCepts(ceptsInPhrase, sourceNullWords , targetNullWords, currA , currE.size(), currF.size());
    getOneToOne(ceptsInPhrase , currF , currE, one);

--- a/misc/CreateProbingPT.cpp
+++ b/misc/CreateProbingPT.cpp
@ -1,29 +0,0 @@
-#include "util/usage.hh"
-#include "moses/TranslationModel/ProbingPT/storing.hh"
-
-
-
-int main(int argc, char* argv[])
-{
-
-  const char * is_reordering = "false";
-
-  if (!(argc == 5 || argc == 4)) {
-    // Tell the user how to run the program
-    std::cerr << "Provided " << argc << " arguments, needed 4 or 5." << std::endl;
-    std::cerr << "Usage: " << argv[0] << " path_to_phrasetable output_dir num_scores is_reordering" << std::endl;
-    std::cerr << "is_reordering should be either true or false, but it is currently a stub feature." << std::endl;
-    //std::cerr << "Usage: " << argv[0] << " path_to_phrasetable number_of_uniq_lines output_bin_file output_hash_table output_vocab_id" << std::endl;
-    return 1;
-  }
-
-  if (argc == 5) {
-    is_reordering = argv[4];
-  }
-
-  createProbingPT(argv[1], argv[2], argv[3], is_reordering);
-
-  util::PrintUsage(std::cout);
-  return 0;
-}
-
--- a/misc/Jamfile
+++ b/misc/Jamfile
@ -21,19 +21,15 @@ if $(with-cmph) {
    exe processPhraseTableMin : processPhraseTableMin.cpp ..//boost_filesystem ../moses//moses ;
    exe processLexicalTableMin : processLexicalTableMin.cpp ..//boost_filesystem ../moses//moses ;
    exe queryPhraseTableMin : queryPhraseTableMin.cpp ..//boost_filesystem ../moses//moses ;
+    exe addLexROtoPT : addLexROtoPT.cpp ..//boost_filesystem ../moses//moses ;

-    alias programsMin : processPhraseTableMin processLexicalTableMin queryPhraseTableMin ;
+    alias programsMin : processPhraseTableMin processLexicalTableMin queryPhraseTableMin addLexROtoPT ;
 #    alias programsMin : processPhraseTableMin processLexicalTableMin ;
 }
 else {
    alias programsMin ;
 }

-exe CreateProbingPT : CreateProbingPT.cpp ..//boost_filesystem ../moses//moses ;
-exe QueryProbingPT : QueryProbingPT.cpp ..//boost_filesystem ../moses//moses ;
-
-alias programsProbing : CreateProbingPT QueryProbingPT ;
-
 exe merge-sorted : 
 merge-sorted.cc 
 ../moses//moses
@ -42,6 +38,6 @@ $(TOP)//boost_iostreams
 $(TOP)//boost_program_options 
 ; 

-alias programs : 1-1-Extraction TMining generateSequences processLexicalTable queryLexicalTable programsMin programsProbing merge-sorted prunePhraseTable pruneGeneration  ;
+alias programs : 1-1-Extraction TMining generateSequences processLexicalTable queryLexicalTable programsMin merge-sorted prunePhraseTable pruneGeneration  ;
 #processPhraseTable queryPhraseTable

--- a/misc/QueryProbingPT.cpp
+++ b/misc/QueryProbingPT.cpp
@ -34,7 +34,7 @@ int main(int argc, char* argv[])
    return 1;
  }

-  QueryEngine queries(argv[1]);
+  Moses::QueryEngine queries(argv[1]);

  //Interactive search
  std::cout << "Please enter a string to be searched, or exit to exit." << std::endl;
--- a/misc/addLexROtoPT.cpp
+++ b/misc/addLexROtoPT.cpp
@ -0,0 +1,168 @@
+#include <iostream>
+#include <string>
+
+#include "moses/Phrase.h"
+#include "moses/FactorCollection.h"
+#include "moses/Timer.h"
+#include "moses/InputFileStream.h"
+#include "moses/TranslationModel/CompactPT/BlockHashIndex.h"
+#include "moses/TranslationModel/CompactPT/CanonicalHuffman.h"
+#include "moses/TranslationModel/CompactPT/StringVector.h"
+
+using namespace Moses;
+using namespace std;
+
+Timer timer;
+
+FactorList m_factorsF, m_factorsE, m_factorsC;
+
+BlockHashIndex m_hash(10, 16);
+size_t m_numScoreComponent;
+bool m_multipleScoreTrees;
+bool m_inMemory = false;
+
+typedef CanonicalHuffman<float> ScoreTree;
+std::vector<ScoreTree*> m_scoreTrees;
+
+StringVector<unsigned char, unsigned long, MmapAllocator>  m_scoresMapped;
+StringVector<unsigned char, unsigned long, std::allocator> m_scoresMemory;
+
+////////////////////////////////////////////////////////////////////////////////////
+void Load(const string &filePath)
+{
+  std::FILE* pFile = std::fopen(filePath.c_str(), "r");
+  UTIL_THROW_IF2(pFile == NULL, "File " << filePath << " could not be opened");
+
+  //if(m_inMemory)
+  m_hash.Load(pFile);
+  //else
+  //m_hash.LoadIndex(pFile);
+
+  size_t read = 0;
+  read += std::fread(&m_numScoreComponent, sizeof(m_numScoreComponent), 1, pFile);
+  read += std::fread(&m_multipleScoreTrees,
+                     sizeof(m_multipleScoreTrees), 1, pFile);
+
+  if(m_multipleScoreTrees) {
+    m_scoreTrees.resize(m_numScoreComponent);
+    for(size_t i = 0; i < m_numScoreComponent; i++)
+      m_scoreTrees[i] = new CanonicalHuffman<float>(pFile);
+  } else {
+    m_scoreTrees.resize(1);
+    m_scoreTrees[0] = new CanonicalHuffman<float>(pFile);
+  }
+
+  if(m_inMemory)
+    m_scoresMemory.load(pFile, false);
+  else
+    m_scoresMapped.load(pFile, true);
+
+}
+
+////////////////////////////////////////////////////////////////////////////////////
+
+std::string
+MakeKey(const std::string& f,
+        const std::string& e,
+        const std::string& c)
+{
+  std::string key;
+  if(!f.empty()) key += f;
+  if(!m_factorsE.empty()) {
+    if(!key.empty()) key += " ||| ";
+    key += e;
+  }
+  if(!m_factorsC.empty()) {
+    if(!key.empty()) key += " ||| ";
+    key += c;
+  }
+  key += " ||| ";
+  return key;
+}
+
+////////////////////////////////////////////////////////////////////////////////////
+
+std::vector<float>
+GetScore(const std::string& f, const std::string& e, const std::string& c)
+{
+  std::string key;
+  std::vector<float> probs;
+
+  key = MakeKey(f, e, c);
+
+  size_t index = m_hash[key];
+  if(m_hash.GetSize() != index) {
+    std::string scoresString;
+    if(m_inMemory)
+      scoresString = m_scoresMemory[index].str();
+    else
+      scoresString = m_scoresMapped[index].str();
+
+
+    BitWrapper<> bitStream(scoresString);
+    for(size_t i = 0; i < m_numScoreComponent; i++) {
+      float prob = m_scoreTrees[m_multipleScoreTrees ? i : 0]->Read(bitStream);
+      prob = exp(prob);
+      probs.push_back(prob);
+    }
+
+    return probs;
+  } else {
+    // return empty vector;
+  }
+
+  return probs;
+}
+
+////////////////////////////////////////////////////////////////////////////////////
+
+int main(int argc, char** argv)
+{
+  string ptPath(argv[1]);
+  string roPath(argv[2]);
+
+  // lex reordering model
+  m_factorsF.push_back(0);
+  m_factorsE.push_back(0);
+
+  Load(roPath);
+
+  // phrase table
+  InputFileStream ptStrm(ptPath);
+
+  string line;
+  while (getline(ptStrm, line)) {
+    //cerr << line << endl;
+    std::vector<std::string> columns(7);
+    std::vector<std::string> toks = TokenizeMultiCharSeparator(line, "|||");
+    assert(toks.size() >= 2);
+
+    for (size_t i = 0; i < toks.size(); ++i) {
+      columns[i] = Trim(toks[i]);
+    }
+
+    std::vector<float> scores = GetScore(columns[0], columns[1], "");
+    // key-value pairs
+    if (scores.size()) {
+      if (!columns[6].empty()) {
+        columns[6] += " ";
+      }
+      columns[6] += "{{LexRO ";
+      for (size_t i = 0; i < scores.size() - 1; ++i) {
+        columns[6] += Moses::SPrint(scores[i]);
+        columns[6] += " ";
+      }
+      columns[6] += Moses::SPrint(scores[scores.size() - 1]);
+      columns[6] += "}}";
+    }
+
+    // output
+    for (size_t i = 0; i < columns.size() - 1; ++i) {
+      cout << columns[i] << " ||| ";
+    }
+    cout << columns[columns.size() - 1] << endl;
+  }
+
+}
+
+
--- a/misc/misc.xcodeproj/project.pbxproj
+++ b/misc/misc.xcodeproj/project.pbxproj
@ -1,323 +0,0 @@
-// !$*UTF8*$!
-{
-	archiveVersion = 1;
-	classes = {
-	};
-	objectVersion = 45;
-	objects = {
-
-/* Begin PBXBuildFile section */
-		1EF455C41227C4BB0022403A /* processLexicalTable.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 1EF455991227C4050022403A /* processLexicalTable.cpp */; };
-		1EF455D01227C4F40022403A /* libmoses.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 1EF455CD1227C4D60022403A /* libmoses.a */; };
-		1EF455D91227C5140022403A /* libOnDiskPt.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 1EF455D81227C50C0022403A /* libOnDiskPt.a */; };
-/* End PBXBuildFile section */
-
-/* Begin PBXContainerItemProxy section */
-		1EF455CC1227C4D60022403A /* PBXContainerItemProxy */ = {
-			isa = PBXContainerItemProxy;
-			containerPortal = 1EF455C81227C4D60022403A /* moses.xcodeproj */;
-			proxyType = 2;
-			remoteGlobalIDString = D2AAC046055464E500DB518D;
-			remoteInfo = moses;
-		};
-		1EF455D71227C50C0022403A /* PBXContainerItemProxy */ = {
-			isa = PBXContainerItemProxy;
-			containerPortal = 1EF455D31227C50C0022403A /* OnDiskPt.xcodeproj */;
-			proxyType = 2;
-			remoteGlobalIDString = D2AAC046055464E500DB518D;
-			remoteInfo = OnDiskPt;
-		};
-		1EF456211227C8A30022403A /* PBXContainerItemProxy */ = {
-			isa = PBXContainerItemProxy;
-			containerPortal = 1EF455C81227C4D60022403A /* moses.xcodeproj */;
-			proxyType = 1;
-			remoteGlobalIDString = D2AAC045055464E500DB518D;
-			remoteInfo = moses;
-		};
-		1EF456231227C8A80022403A /* PBXContainerItemProxy */ = {
-			isa = PBXContainerItemProxy;
-			containerPortal = 1EF455D31227C50C0022403A /* OnDiskPt.xcodeproj */;
-			proxyType = 1;
-			remoteGlobalIDString = D2AAC045055464E500DB518D;
-			remoteInfo = OnDiskPt;
-		};
-/* End PBXContainerItemProxy section */
-
-/* Begin PBXFileReference section */
-		1EF455991227C4050022403A /* processLexicalTable.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = processLexicalTable.cpp; sourceTree = "<group>"; };
-		1EF455BA1227C4760022403A /* processLexicalTable */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.executable"; includeInIndex = 0; path = processLexicalTable; sourceTree = BUILT_PRODUCTS_DIR; };
-		1EF455C81227C4D60022403A /* moses.xcodeproj */ = {isa = PBXFileReference; lastKnownFileType = "wrapper.pb-project"; name = moses.xcodeproj; path = ../moses/moses.xcodeproj; sourceTree = SOURCE_ROOT; };
-		1EF455D31227C50C0022403A /* OnDiskPt.xcodeproj */ = {isa = PBXFileReference; lastKnownFileType = "wrapper.pb-project"; name = OnDiskPt.xcodeproj; path = ../OnDiskPt/OnDiskPt.xcodeproj; sourceTree = SOURCE_ROOT; };
-		C6859E8B029090EE04C91782 /* misc.1 */ = {isa = PBXFileReference; lastKnownFileType = text.man; path = misc.1; sourceTree = "<group>"; };
-/* End PBXFileReference section */
-
-/* Begin PBXFrameworksBuildPhase section */
-		1EF455B81227C4760022403A /* Frameworks */ = {
-			isa = PBXFrameworksBuildPhase;
-			buildActionMask = 2147483647;
-			files = (
-				1EF455D91227C5140022403A /* libOnDiskPt.a in Frameworks */,
-				1EF455D01227C4F40022403A /* libmoses.a in Frameworks */,
-			);
-			runOnlyForDeploymentPostprocessing = 0;
-		};
-/* End PBXFrameworksBuildPhase section */
-
-/* Begin PBXGroup section */
-		08FB7794FE84155DC02AAC07 /* misc */ = {
-			isa = PBXGroup;
-			children = (
-				1EF455D31227C50C0022403A /* OnDiskPt.xcodeproj */,
-				1EF455C81227C4D60022403A /* moses.xcodeproj */,
-				08FB7795FE84155DC02AAC07 /* Source */,
-				C6859E8C029090F304C91782 /* Documentation */,
-				1AB674ADFE9D54B511CA2CBB /* Products */,
-			);
-			name = misc;
-			sourceTree = "<group>";
-		};
-		08FB7795FE84155DC02AAC07 /* Source */ = {
-			isa = PBXGroup;
-			children = (
-				1EF455991227C4050022403A /* processLexicalTable.cpp */,
-			);
-			name = Source;
-			sourceTree = "<group>";
-		};
-		1AB674ADFE9D54B511CA2CBB /* Products */ = {
-			isa = PBXGroup;
-			children = (
-				1EF455BA1227C4760022403A /* processLexicalTable */,
-			);
-			name = Products;
-			sourceTree = "<group>";
-		};
-		1EF455C91227C4D60022403A /* Products */ = {
-			isa = PBXGroup;
-			children = (
-				1EF455CD1227C4D60022403A /* libmoses.a */,
-			);
-			name = Products;
-			sourceTree = "<group>";
-		};
-		1EF455D41227C50C0022403A /* Products */ = {
-			isa = PBXGroup;
-			children = (
-				1EF455D81227C50C0022403A /* libOnDiskPt.a */,
-			);
-			name = Products;
-			sourceTree = "<group>";
-		};
-		C6859E8C029090F304C91782 /* Documentation */ = {
-			isa = PBXGroup;
-			children = (
-				C6859E8B029090EE04C91782 /* misc.1 */,
-			);
-			name = Documentation;
-			sourceTree = "<group>";
-		};
-/* End PBXGroup section */
-
-/* Begin PBXNativeTarget section */
-		1EF455B91227C4760022403A /* processLexicalTable */ = {
-			isa = PBXNativeTarget;
-			buildConfigurationList = 1EF455C11227C4A70022403A /* Build configuration list for PBXNativeTarget "processLexicalTable" */;
-			buildPhases = (
-				1EF455B71227C4760022403A /* Sources */,
-				1EF455B81227C4760022403A /* Frameworks */,
-			);
-			buildRules = (
-			);
-			dependencies = (
-				1EF456221227C8A30022403A /* PBXTargetDependency */,
-				1EF456241227C8A80022403A /* PBXTargetDependency */,
-			);
-			name = processLexicalTable;
-			productName = processLexicalTable;
-			productReference = 1EF455BA1227C4760022403A /* processLexicalTable */;
-			productType = "com.apple.product-type.tool";
-		};
-/* End PBXNativeTarget section */
-
-/* Begin PBXProject section */
-		08FB7793FE84155DC02AAC07 /* Project object */ = {
-			isa = PBXProject;
-			buildConfigurationList = 1DEB923508733DC60010E9CD /* Build configuration list for PBXProject "misc" */;
-			compatibilityVersion = "Xcode 3.1";
-			hasScannedForEncodings = 1;
-			mainGroup = 08FB7794FE84155DC02AAC07 /* misc */;
-			projectDirPath = "";
-			projectReferences = (
-				{
-					ProductGroup = 1EF455C91227C4D60022403A /* Products */;
-					ProjectRef = 1EF455C81227C4D60022403A /* moses.xcodeproj */;
-				},
-				{
-					ProductGroup = 1EF455D41227C50C0022403A /* Products */;
-					ProjectRef = 1EF455D31227C50C0022403A /* OnDiskPt.xcodeproj */;
-				},
-			);
-			projectRoot = "";
-			targets = (
-				1EF455B91227C4760022403A /* processLexicalTable */,
-			);
-		};
-/* End PBXProject section */
-
-/* Begin PBXReferenceProxy section */
-		1EF455CD1227C4D60022403A /* libmoses.a */ = {
-			isa = PBXReferenceProxy;
-			fileType = archive.ar;
-			path = libmoses.a;
-			remoteRef = 1EF455CC1227C4D60022403A /* PBXContainerItemProxy */;
-			sourceTree = BUILT_PRODUCTS_DIR;
-		};
-		1EF455D81227C50C0022403A /* libOnDiskPt.a */ = {
-			isa = PBXReferenceProxy;
-			fileType = archive.ar;
-			path = libOnDiskPt.a;
-			remoteRef = 1EF455D71227C50C0022403A /* PBXContainerItemProxy */;
-			sourceTree = BUILT_PRODUCTS_DIR;
-		};
-/* End PBXReferenceProxy section */
-
-/* Begin PBXSourcesBuildPhase section */
-		1EF455B71227C4760022403A /* Sources */ = {
-			isa = PBXSourcesBuildPhase;
-			buildActionMask = 2147483647;
-			files = (
-				1EF455C41227C4BB0022403A /* processLexicalTable.cpp in Sources */,
-			);
-			runOnlyForDeploymentPostprocessing = 0;
-		};
-/* End PBXSourcesBuildPhase section */
-
-/* Begin PBXTargetDependency section */
-		1EF456221227C8A30022403A /* PBXTargetDependency */ = {
-			isa = PBXTargetDependency;
-			name = moses;
-			targetProxy = 1EF456211227C8A30022403A /* PBXContainerItemProxy */;
-		};
-		1EF456241227C8A80022403A /* PBXTargetDependency */ = {
-			isa = PBXTargetDependency;
-			name = OnDiskPt;
-			targetProxy = 1EF456231227C8A80022403A /* PBXContainerItemProxy */;
-		};
-/* End PBXTargetDependency section */
-
-/* Begin XCBuildConfiguration section */
-		1DEB923608733DC60010E9CD /* Debug */ = {
-			isa = XCBuildConfiguration;
-			buildSettings = {
-				ARCHS = "$(ARCHS_STANDARD_32_64_BIT)";
-				GCC_C_LANGUAGE_STANDARD = gnu99;
-				GCC_OPTIMIZATION_LEVEL = 0;
-				GCC_WARN_ABOUT_RETURN_TYPE = YES;
-				GCC_WARN_UNUSED_VARIABLE = YES;
-				ONLY_ACTIVE_ARCH = YES;
-				PREBINDING = NO;
-				SDKROOT = macosx10.6;
-			};
-			name = Debug;
-		};
-		1DEB923708733DC60010E9CD /* Release */ = {
-			isa = XCBuildConfiguration;
-			buildSettings = {
-				ARCHS = "$(ARCHS_STANDARD_32_64_BIT)";
-				GCC_C_LANGUAGE_STANDARD = gnu99;
-				GCC_WARN_ABOUT_RETURN_TYPE = YES;
-				GCC_WARN_UNUSED_VARIABLE = YES;
-				PREBINDING = NO;
-				SDKROOT = macosx10.6;
-			};
-			name = Release;
-		};
-		1EF455BC1227C4760022403A /* Debug */ = {
-			isa = XCBuildConfiguration;
-			buildSettings = {
-				ALWAYS_SEARCH_USER_PATHS = NO;
-				COPY_PHASE_STRIP = NO;
-				GCC_DYNAMIC_NO_PIC = NO;
-				GCC_ENABLE_FIX_AND_CONTINUE = YES;
-				GCC_MODEL_TUNING = G5;
-				GCC_OPTIMIZATION_LEVEL = 0;
-				HEADER_SEARCH_PATHS = ../moses/src;
-				INSTALL_PATH = /usr/local/bin;
-				LIBRARY_SEARCH_PATHS = (
-					../irstlm/lib/i386,
-					../srilm/lib/macosx,
-					../kenlm/lm,
-					../randlm/lib,
-				);
-				OTHER_LDFLAGS = (
-					"-lflm",
-					"-lmisc",
-					"-loolm",
-					"-ldstruct",
-					"-lz",
-					"-lirstlm",
-					"-lkenlm",
-					"-lrandlm",
-				);
-				PREBINDING = NO;
-				PRODUCT_NAME = processLexicalTable;
-			};
-			name = Debug;
-		};
-		1EF455BD1227C4760022403A /* Release */ = {
-			isa = XCBuildConfiguration;
-			buildSettings = {
-				ALWAYS_SEARCH_USER_PATHS = NO;
-				COPY_PHASE_STRIP = YES;
-				DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym";
-				GCC_ENABLE_FIX_AND_CONTINUE = NO;
-				GCC_MODEL_TUNING = G5;
-				HEADER_SEARCH_PATHS = ../moses/src;
-				INSTALL_PATH = /usr/local/bin;
-				LIBRARY_SEARCH_PATHS = (
-					../irstlm/lib/i386,
-					../srilm/lib/macosx,
-					../kenlm/lm,
-					../randlm/lib,
-				);
-				OTHER_LDFLAGS = (
-					"-lflm",
-					"-lmisc",
-					"-loolm",
-					"-ldstruct",
-					"-lz",
-					"-lirstlm",
-					"-lkenlm",
-					"-lrandlm",
-				);
-				PREBINDING = NO;
-				PRODUCT_NAME = processLexicalTable;
-				ZERO_LINK = NO;
-			};
-			name = Release;
-		};
-/* End XCBuildConfiguration section */
-
-/* Begin XCConfigurationList section */
-		1DEB923508733DC60010E9CD /* Build configuration list for PBXProject "misc" */ = {
-			isa = XCConfigurationList;
-			buildConfigurations = (
-				1DEB923608733DC60010E9CD /* Debug */,
-				1DEB923708733DC60010E9CD /* Release */,
-			);
-			defaultConfigurationIsVisible = 0;
-			defaultConfigurationName = Release;
-		};
-		1EF455C11227C4A70022403A /* Build configuration list for PBXNativeTarget "processLexicalTable" */ = {
-			isa = XCConfigurationList;
-			buildConfigurations = (
-				1EF455BC1227C4760022403A /* Debug */,
-				1EF455BD1227C4760022403A /* Release */,
-			);
-			defaultConfigurationIsVisible = 0;
-			defaultConfigurationName = Release;
-		};
-/* End XCConfigurationList section */
-	};
-	rootObject = 08FB7793FE84155DC02AAC07 /* Project object */;
-}
--- a/misc/processLexicalTable.vcxproj
+++ b/misc/processLexicalTable.vcxproj
@ -1,108 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
-  <ItemGroup Label="ProjectConfigurations">
-    <ProjectConfiguration Include="Debug|Win32">
-      <Configuration>Debug</Configuration>
-      <Platform>Win32</Platform>
-    </ProjectConfiguration>
-    <ProjectConfiguration Include="Release|Win32">
-      <Configuration>Release</Configuration>
-      <Platform>Win32</Platform>
-    </ProjectConfiguration>
-  </ItemGroup>
-  <PropertyGroup Label="Globals">
-    <ProjectGuid>{9834EABB-2033-4607-9DAC-36D16E0725B5}</ProjectGuid>
-    <RootNamespace>processLexicalTable</RootNamespace>
-    <Keyword>Win32Proj</Keyword>
-  </PropertyGroup>
-  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
-  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
-    <ConfigurationType>Application</ConfigurationType>
-    <CharacterSet>Unicode</CharacterSet>
-    <WholeProgramOptimization>true</WholeProgramOptimization>
-  </PropertyGroup>
-  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
-    <ConfigurationType>Application</ConfigurationType>
-    <CharacterSet>Unicode</CharacterSet>
-  </PropertyGroup>
-  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
-  <ImportGroup Label="ExtensionSettings">
-  </ImportGroup>
-  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="PropertySheets">
-    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
-  </ImportGroup>
-  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="PropertySheets">
-    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
-  </ImportGroup>
-  <PropertyGroup Label="UserMacros" />
-  <PropertyGroup>
-    <_ProjectFileVersion>10.0.30319.1</_ProjectFileVersion>
-    <OutDir Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">$(SolutionDir)$(Configuration)\</OutDir>
-    <IntDir Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">$(Configuration)\</IntDir>
-    <LinkIncremental Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</LinkIncremental>
-    <OutDir Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">$(SolutionDir)$(Configuration)\</OutDir>
-    <IntDir Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">$(Configuration)\</IntDir>
-    <LinkIncremental Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">false</LinkIncremental>
-  </PropertyGroup>
-  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
-    <ClCompile>
-      <Optimization>Disabled</Optimization>
-      <AdditionalIncludeDirectories>C:\Program Files\boost\boost_1_47;$(SolutionDir)/moses/src;$(SolutionDir)/kenlm;$(SolutionDir);%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
-      <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;_CRT_SECURE_NO_DEPRECATE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
-      <MinimalRebuild>true</MinimalRebuild>
-      <BasicRuntimeChecks>EnableFastChecks</BasicRuntimeChecks>
-      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
-      <PrecompiledHeader>
-      </PrecompiledHeader>
-      <WarningLevel>Level3</WarningLevel>
-      <DebugInformationFormat>EditAndContinue</DebugInformationFormat>
-    </ClCompile>
-    <Link>
-      <AdditionalDependencies>zdll.lib;$(SolutionDir)$(Configuration)\moses.lib;%(AdditionalDependencies)</AdditionalDependencies>
-      <GenerateDebugInformation>true</GenerateDebugInformation>
-      <SubSystem>Console</SubSystem>
-      <RandomizedBaseAddress>false</RandomizedBaseAddress>
-      <DataExecutionPrevention>
-      </DataExecutionPrevention>
-      <TargetMachine>MachineX86</TargetMachine>
-    </Link>
-  </ItemDefinitionGroup>
-  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
-    <ClCompile>
-      <AdditionalIncludeDirectories>C:\Program Files\boost\boost_1_47;$(SolutionDir)/moses/src;$(SolutionDir)/kenlm;$(SolutionDir);%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
-      <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;_CRT_SECURE_NO_DEPRECATE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
-      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
-      <PrecompiledHeader>
-      </PrecompiledHeader>
-      <WarningLevel>Level3</WarningLevel>
-      <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
-    </ClCompile>
-    <Link>
-      <AdditionalDependencies>zdll.lib;$(SolutionDir)$(Configuration)\moses.lib;%(AdditionalDependencies)</AdditionalDependencies>
-      <GenerateDebugInformation>true</GenerateDebugInformation>
-      <SubSystem>Console</SubSystem>
-      <OptimizeReferences>true</OptimizeReferences>
-      <EnableCOMDATFolding>true</EnableCOMDATFolding>
-      <RandomizedBaseAddress>false</RandomizedBaseAddress>
-      <DataExecutionPrevention>
-      </DataExecutionPrevention>
-      <TargetMachine>MachineX86</TargetMachine>
-    </Link>
-  </ItemDefinitionGroup>
-  <ItemGroup>
-    <ClCompile Include="processLexicalTable.cpp" />
-  </ItemGroup>
-  <ItemGroup>
-    <ProjectReference Include="..\moses\moses.vcxproj">
-      <Project>{8122157a-0de5-44ff-8e5b-024ed6ace7af}</Project>
-      <ReferenceOutputAssembly>false</ReferenceOutputAssembly>
-    </ProjectReference>
-    <ProjectReference Include="..\OnDiskPt\OnDiskPt.vcxproj">
-      <Project>{8b07671b-cbaf-4514-affd-ce238cd427e9}</Project>
-      <ReferenceOutputAssembly>false</ReferenceOutputAssembly>
-    </ProjectReference>
-  </ItemGroup>
-  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
-  <ImportGroup Label="ExtensionTargets">
-  </ImportGroup>
-</Project>
--- a/misc/processPhraseTable.vcxproj
+++ b/misc/processPhraseTable.vcxproj
@ -1,108 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
-  <ItemGroup Label="ProjectConfigurations">
-    <ProjectConfiguration Include="Debug|Win32">
-      <Configuration>Debug</Configuration>
-      <Platform>Win32</Platform>
-    </ProjectConfiguration>
-    <ProjectConfiguration Include="Release|Win32">
-      <Configuration>Release</Configuration>
-      <Platform>Win32</Platform>
-    </ProjectConfiguration>
-  </ItemGroup>
-  <PropertyGroup Label="Globals">
-    <ProjectGuid>{AA230564-6DF1-4662-9BF9-7AD73DE53B76}</ProjectGuid>
-    <RootNamespace>processPhraseTable</RootNamespace>
-    <Keyword>Win32Proj</Keyword>
-  </PropertyGroup>
-  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
-  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
-    <ConfigurationType>Application</ConfigurationType>
-    <CharacterSet>Unicode</CharacterSet>
-    <WholeProgramOptimization>true</WholeProgramOptimization>
-  </PropertyGroup>
-  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
-    <ConfigurationType>Application</ConfigurationType>
-    <CharacterSet>Unicode</CharacterSet>
-  </PropertyGroup>
-  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
-  <ImportGroup Label="ExtensionSettings">
-  </ImportGroup>
-  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="PropertySheets">
-    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
-  </ImportGroup>
-  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="PropertySheets">
-    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
-  </ImportGroup>
-  <PropertyGroup Label="UserMacros" />
-  <PropertyGroup>
-    <_ProjectFileVersion>10.0.30319.1</_ProjectFileVersion>
-    <OutDir Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">$(SolutionDir)$(Configuration)\</OutDir>
-    <IntDir Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">$(Configuration)\</IntDir>
-    <LinkIncremental Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</LinkIncremental>
-    <OutDir Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">$(SolutionDir)$(Configuration)\</OutDir>
-    <IntDir Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">$(Configuration)\</IntDir>
-    <LinkIncremental Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">false</LinkIncremental>
-  </PropertyGroup>
-  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
-    <ClCompile>
-      <Optimization>Disabled</Optimization>
-      <AdditionalIncludeDirectories>C:\Program Files\boost\boost_1_47;$(SolutionDir)/moses/src;$(SolutionDir)/kenlm;$(SolutionDir);%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
-      <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
-      <MinimalRebuild>true</MinimalRebuild>
-      <BasicRuntimeChecks>EnableFastChecks</BasicRuntimeChecks>
-      <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
-      <PrecompiledHeader>
-      </PrecompiledHeader>
-      <WarningLevel>Level3</WarningLevel>
-      <DebugInformationFormat>EditAndContinue</DebugInformationFormat>
-    </ClCompile>
-    <Link>
-      <AdditionalDependencies>zdll.lib;$(SolutionDir)$(Configuration)\moses.lib;%(AdditionalDependencies)</AdditionalDependencies>
-      <GenerateDebugInformation>true</GenerateDebugInformation>
-      <SubSystem>Console</SubSystem>
-      <RandomizedBaseAddress>false</RandomizedBaseAddress>
-      <DataExecutionPrevention>
-      </DataExecutionPrevention>
-      <TargetMachine>MachineX86</TargetMachine>
-    </Link>
-  </ItemDefinitionGroup>
-  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
-    <ClCompile>
-      <AdditionalIncludeDirectories>C:\Program Files\boost\boost_1_47;$(SolutionDir)/moses/src;$(SolutionDir)/kenlm;$(SolutionDir);%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
-      <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
-      <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
-      <PrecompiledHeader>
-      </PrecompiledHeader>
-      <WarningLevel>Level3</WarningLevel>
-      <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
-    </ClCompile>
-    <Link>
-      <AdditionalDependencies>zlib.lib;$(SolutionDir)$(Configuration)\moses.lib;%(AdditionalDependencies)</AdditionalDependencies>
-      <GenerateDebugInformation>true</GenerateDebugInformation>
-      <SubSystem>Console</SubSystem>
-      <OptimizeReferences>true</OptimizeReferences>
-      <EnableCOMDATFolding>true</EnableCOMDATFolding>
-      <RandomizedBaseAddress>false</RandomizedBaseAddress>
-      <DataExecutionPrevention>
-      </DataExecutionPrevention>
-      <TargetMachine>MachineX86</TargetMachine>
-    </Link>
-  </ItemDefinitionGroup>
-  <ItemGroup>
-    <ClCompile Include="processPhraseTable.cpp" />
-  </ItemGroup>
-  <ItemGroup>
-    <ProjectReference Include="..\moses\moses.vcxproj">
-      <Project>{8122157a-0de5-44ff-8e5b-024ed6ace7af}</Project>
-      <ReferenceOutputAssembly>false</ReferenceOutputAssembly>
-    </ProjectReference>
-    <ProjectReference Include="..\OnDiskPt\OnDiskPt.vcxproj">
-      <Project>{8b07671b-cbaf-4514-affd-ce238cd427e9}</Project>
-      <ReferenceOutputAssembly>false</ReferenceOutputAssembly>
-    </ProjectReference>
-  </ItemGroup>
-  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
-  <ImportGroup Label="ExtensionTargets">
-  </ImportGroup>
-</Project>
--- a/moses/ChartCell.cpp
+++ b/moses/ChartCell.cpp
@ -36,7 +36,6 @@ using namespace std;

 namespace Moses
 {
-extern bool g_mosesDebug;

 ChartCellBase::ChartCellBase(size_t startPos, size_t endPos) :
  m_coverage(startPos, endPos),
--- a/moses/ChartManager.cpp
+++ b/moses/ChartManager.cpp
@ -41,8 +41,6 @@ using namespace std;
 namespace Moses
 {

-extern bool g_mosesDebug;
-
 /* constructor. Initialize everything prior to decoding a particular sentence.
 * \param source the sentence to be decoded
 * \param system which particular set of models to use.
--- a/moses/ExportInterface.cpp
+++ b/moses/ExportInterface.cpp
@ -71,6 +71,8 @@ using namespace Moses;

 namespace Moses
 {
+//extern size_t g_numHypos;
+
 void OutputFeatureWeightsForHypergraph(std::ostream &outputSearchGraphStream)
 {
  outputSearchGraphStream.setf(std::ios::fixed);
@ -294,9 +296,11 @@ batch_run()
  pool.Stop(true); //flush remaining jobs
 #endif

+//  cerr << "g_numHypos=" << Moses::g_numHypos << endl;
+
  FeatureFunction::Destroy();

-  IFVERBOSE(1) util::PrintUsage(std::cerr);
+  IFVERBOSE(0) util::PrintUsage(std::cerr);

 #ifndef EXIT_RETURN
  //This avoids that destructors are called (it can take a long time)
--- a/moses/FF/CorrectionPattern.cpp
+++ b/moses/FF/CorrectionPattern.cpp
@ -143,8 +143,8 @@ std::string CorrectionPattern::CreateSinglePattern(const Tokens &s1, const Token
    out << "del(«" << boost::join(s1, "·") << "»)";
    return out.str();
  } else {
-    typename Tokens::value_type v1 = boost::join(s1, "+");
-    typename Tokens::value_type v2 = boost::join(s2, "+");
+    Tokens::value_type v1 = boost::join(s1, "+");
+    Tokens::value_type v2 = boost::join(s2, "+");
    out << MakePair(v1, v2, m_general);
    return out.str();
  }
--- a/moses/FF/Factory.cpp
+++ b/moses/FF/Factory.cpp
@ -14,8 +14,9 @@
 #include "moses/TranslationModel/RuleTable/PhraseDictionaryOnDisk.h"
 #include "moses/TranslationModel/RuleTable/PhraseDictionaryFuzzyMatch.h"
 #include "moses/TranslationModel/RuleTable/PhraseDictionaryALSuffixArray.h"
-#include "moses/TranslationModel/ProbingPT/ProbingPT.h"
+#include "moses/TranslationModel/ProbingPT.h"
 #include "moses/TranslationModel/PhraseDictionaryMemoryPerSentence.h"
+#include "moses/TranslationModel/PhraseDictionaryMemoryPerSentenceOnDemand.h"

 #include "moses/FF/LexicalReordering/LexicalReordering.h"

@ -30,6 +31,7 @@
 #include "moses/FF/TargetBigramFeature.h"
 #include "moses/FF/TargetNgramFeature.h"
 #include "moses/FF/PhraseBoundaryFeature.h"
+#include "moses/FF/PhraseDistanceFeature.h"
 #include "moses/FF/PhrasePairFeature.h"
 #include "moses/FF/RulePairUnlexicalizedSource.h"
 #include "moses/FF/PhraseLengthFeature.h"
@ -66,6 +68,7 @@
 #include "moses/FF/SkeletonStatelessFF.h"
 #include "moses/FF/SkeletonStatefulFF.h"
 #include "moses/LM/SkeletonLM.h"
+#include "moses/LM/InMemoryPerSentenceOnDemandLM.h"
 #include "moses/FF/SkeletonTranslationOptionListFeature.h"
 #include "moses/LM/BilingualLM.h"
 #include "moses/TranslationModel/SkeletonPT.h"
@ -243,6 +246,7 @@ FeatureRegistry::FeatureRegistry()
  MOSES_FNAME(PhraseDictionaryFuzzyMatch);
  MOSES_FNAME(ProbingPT);
  MOSES_FNAME(PhraseDictionaryMemoryPerSentence);
+  MOSES_FNAME(PhraseDictionaryMemoryPerSentenceOnDemand);
  MOSES_FNAME2("RuleTable", Syntax::RuleTableFF);
  MOSES_FNAME2("SyntaxInputWeight", Syntax::InputWeightFF);

@ -252,6 +256,7 @@ FeatureRegistry::FeatureRegistry()
  MOSES_FNAME(SourceWordDeletionFeature);
  MOSES_FNAME(TargetWordInsertionFeature);
  MOSES_FNAME(PhraseBoundaryFeature);
+  MOSES_FNAME(PhraseDistanceFeature);
  MOSES_FNAME(PhraseLengthFeature);
  MOSES_FNAME(WordTranslationFeature);
  MOSES_FNAME(TargetBigramFeature);
@ -295,6 +300,7 @@ FeatureRegistry::FeatureRegistry()
  MOSES_FNAME(SkeletonStatelessFF);
  MOSES_FNAME(SkeletonStatefulFF);
  MOSES_FNAME(SkeletonLM);
+  MOSES_FNAME(InMemoryPerSentenceOnDemandLM);
  MOSES_FNAME(SkeletonTranslationOptionListFeature);
  MOSES_FNAME(SkeletonPT);

--- a/moses/FF/LexicalReordering/BidirectionalReorderingState.cpp
+++ b/moses/FF/LexicalReordering/BidirectionalReorderingState.cpp
@ -15,7 +15,7 @@ size_t BidirectionalReorderingState::hash() const

 bool BidirectionalReorderingState::operator==(const FFState& o) const
 {
-  if (&o == this) return 0;
+  if (&o == this) return true;

  BidirectionalReorderingState const &other
  = static_cast<BidirectionalReorderingState const&>(o);
--- a/moses/FF/LexicalReordering/PhraseBasedReorderingState.h
+++ b/moses/FF/LexicalReordering/PhraseBasedReorderingState.h
@ -27,11 +27,6 @@ public:
  LRState*
  Expand(const TranslationOption& topt,const InputType& input,
         ScoreComponentCollection*  scores) const;
-
-  ReorderingType GetOrientationTypeMSD(Range currRange) const;
-  ReorderingType GetOrientationTypeMSLR(Range currRange) const;
-  ReorderingType GetOrientationTypeMonotonic(Range currRange) const;
-  ReorderingType GetOrientationTypeLeftRight(Range currRange) const;
 };

 }
--- a/moses/FF/PhraseDistanceFeature.cpp
+++ b/moses/FF/PhraseDistanceFeature.cpp
@ -0,0 +1,123 @@
+#include "PhraseDistanceFeature.h"
+
+#include <vector>
+#include <boost/foreach.hpp>
+#include "moses/InputType.h"
+#include "moses/ScoreComponentCollection.h"
+#include "moses/StaticData.h"
+#include "util/exception.hh"
+
+using namespace std;
+
+namespace Moses
+{
+PhraseDistanceFeature::PhraseDistanceFeature(const string &line)
+  : StatelessFeatureFunction(2, line)
+  , m_space("")
+  , m_spaceID(0)
+  , m_measure(EuclideanDistance)
+{
+  ReadParameters();
+}
+
+void PhraseDistanceFeature::EvaluateWithSourceContext(const InputType &input
+    , const InputPath &inputPath
+    , const TargetPhrase &targetPhrase
+    , const StackVec *stackVec
+    , ScoreComponentCollection &scoreBreakdown
+    , ScoreComponentCollection *estimatedScores) const
+{
+  vector<float> scores(m_numScoreComponents, 0);
+  bool broken = false;
+  // Input coord
+  map<size_t const, vector<float> >::const_iterator ii;
+  if (input.m_coordMap) {
+    ii = input.m_coordMap->find(m_spaceID);
+  } else {
+    TRACE_ERR("No coordinates for space " << m_space << " on input (specify with coord XML tag)" << endl);
+    TRACE_ERR("Scores for " << m_description << " will be incorrect and probably all zeros" << endl);
+    broken = true;
+  }
+  if (ii == input.m_coordMap->end()) {
+    TRACE_ERR("No coordinates for space " << m_space << " on input (specify with coord XML tag)" << endl);
+    TRACE_ERR("Scores for " << m_description << " will be incorrect and probably all zeros" << endl);
+    broken = true;
+  }
+  // Target phrase coord
+  vector<SPTR<vector<float> > > const* tpp = targetPhrase.GetCoordList(m_spaceID);
+  if (tpp == NULL) {
+    TRACE_ERR("No coordinates for space " << m_space << " on target phrase (PhraseDictionary implementation needs to set)" << endl);
+    TRACE_ERR("Scores for " << m_description << " will be incorrect and probably all zeros" << endl);
+    broken = true;
+  }
+  // Compute scores
+  if (!broken) {
+    vector<float> const& inputCoord = ii->second;
+    vector<SPTR<vector<float> > > const& tpCoord = *tpp;
+    // Centroid of target phrase instances (from phrase extraction)
+    vector<float> centroid = vector<float>(inputCoord.size(), 0);
+    BOOST_FOREACH(SPTR<vector<float> > const coord, tpCoord) {
+      for (size_t i = 0; i < inputCoord.size(); ++i) {
+        centroid[i] += (*coord)[i];
+      }
+    }
+    for (size_t i = 0; i < inputCoord.size(); ++i) {
+      centroid[i] /= tpCoord.size();
+    }
+    // Average distance from the target phrase instances to (1) the input and
+    // (2) the target phrase centroid
+    float inputDistance = 0;
+    float centroidDistance = 0;
+    if (m_measure == EuclideanDistance) {
+      BOOST_FOREACH(SPTR<vector<float> > const coord, tpCoord) {
+        float pointInputDistance = 0;
+        float pointCentroidDistance = 0;
+        for (size_t i = 0; i < inputCoord.size(); ++i) {
+          pointInputDistance += pow(inputCoord[i] - (*coord)[i], 2);
+          pointCentroidDistance += pow(centroid[i] - (*coord)[i], 2);
+        }
+        inputDistance += sqrt(pointInputDistance);
+        centroidDistance += sqrt(pointCentroidDistance);
+      }
+    } else if (m_measure == TotalVariationDistance) {
+      BOOST_FOREACH(SPTR<vector<float> > const coord, tpCoord) {
+        float pointInputDistance = 0;
+        float pointCentroidDistance = 0;
+        for (size_t i = 0; i < inputCoord.size(); ++i) {
+          pointInputDistance += abs(inputCoord[i] - (*coord)[i]);
+          pointCentroidDistance += abs(centroid[i] - (*coord)[i]);
+        }
+        inputDistance += pointInputDistance / 2;
+        centroidDistance += pointCentroidDistance / 2;
+      }
+    }
+    inputDistance /= tpCoord.size();
+    centroidDistance /= tpCoord.size();
+    // Log transform scores, max with float epsilon to avoid domain error
+    scores[0] = log(max(inputDistance, Moses::FLOAT_EPSILON));
+    scores[1] = log(max(centroidDistance, Moses::FLOAT_EPSILON));
+  }
+  // Set scores
+  scoreBreakdown.Assign(this, scores);
+  return;
+}
+
+void PhraseDistanceFeature::SetParameter(const string& key, const string& value)
+{
+  if (key == "space") {
+    m_space = value;
+    m_spaceID = StaticData::InstanceNonConst().MapCoordSpace(m_space);
+  } else if (key == "measure") {
+    if (value == "euc") {
+      m_measure = EuclideanDistance;
+    } else if (value == "var") {
+      m_measure = TotalVariationDistance;
+    } else {
+      UTIL_THROW2("Unknown measure " << value << ", choices: euc var");
+    }
+  } else {
+    StatelessFeatureFunction::SetParameter(key, value);
+  }
+}
+
+} // namespace
--- a/moses/FF/PhraseDistanceFeature.h
+++ b/moses/FF/PhraseDistanceFeature.h
@ -0,0 +1,56 @@
+#pragma once
+
+#include "StatelessFeatureFunction.h"
+
+namespace Moses
+{
+
+class PhraseDistanceFeature : public StatelessFeatureFunction
+{
+  enum Measure {
+    EuclideanDistance,
+    TotalVariationDistance,
+  };
+
+public:
+  PhraseDistanceFeature(const std::string &line);
+
+  bool IsUseable(const FactorMask &mask) const {
+    return true;
+  }
+
+  virtual void EvaluateInIsolation(const Phrase &source
+                                   , const TargetPhrase &targetPhrase
+                                   , ScoreComponentCollection &scoreBreakdown
+                                   , ScoreComponentCollection &estimatedScores) const {
+  }
+
+  void EvaluateWhenApplied(const Hypothesis& hypo,
+                           ScoreComponentCollection* accumulator) const {
+  }
+  void EvaluateWhenApplied(const ChartHypothesis &hypo,
+                           ScoreComponentCollection* accumulator) const {
+  }
+  void EvaluateWhenApplied(const Syntax::SHyperedge &hyperedge,
+                           ScoreComponentCollection* accumulator) const {
+  }
+
+  void EvaluateWithSourceContext(const InputType &input
+                                 , const InputPath &inputPath
+                                 , const TargetPhrase &targetPhrase
+                                 , const StackVec *stackVec
+                                 , ScoreComponentCollection &scoreBreakdown
+                                 , ScoreComponentCollection *estimatedScores = NULL) const;
+
+  void EvaluateTranslationOptionListWithSourceContext(const InputType &input
+      , const TranslationOptionList &translationOptionList) const {
+  }
+  void SetParameter(const std::string& key, const std::string& value);
+
+protected:
+  Measure m_measure;
+  std::string m_space;
+  size_t m_spaceID;
+};
+
+} //namespace
--- a/moses/FactorCollection.h
+++ b/moses/FactorCollection.h
@ -41,6 +41,8 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 #include "util/pool.hh"
 #include "Factor.h"

+class System;
+
 namespace Moses
 {

@ -66,6 +68,7 @@ struct FactorFriend {
 class FactorCollection
 {
  friend std::ostream& operator<<(std::ostream&, const FactorCollection&);
+  friend class ::System;

  struct HashFactor : public std::unary_function<const FactorFriend &, std::size_t> {
    std::size_t operator()(const FactorFriend &factor) const {
--- a/moses/Hypothesis.cpp
+++ b/moses/Hypothesis.cpp
@ -43,6 +43,8 @@ using namespace std;

 namespace Moses
 {
+//size_t g_numHypos = 0;
+
 Hypothesis::
 Hypothesis(Manager& manager, InputType const& source, const TranslationOption &initialTransOpt, const Bitmap &bitmap, int id)
  : m_prevHypo(NULL)
@ -61,6 +63,7 @@ Hypothesis(Manager& manager, InputType const& source, const TranslationOption &i
  , m_manager(manager)
  , m_id(id)
 {
+//	++g_numHypos;
  // used for initial seeding of trans process
  // initialize scores
  //_hash_computed = false;
@ -91,6 +94,8 @@ Hypothesis(const Hypothesis &prevHypo, const TranslationOption &transOpt, const
  , m_manager(prevHypo.GetManager())
  , m_id(id)
 {
+//	++g_numHypos;
+
  m_currScoreBreakdown.PlusEquals(transOpt.GetScoreBreakdown());
  m_wordDeleted = transOpt.IsDeletionOption();
 }
--- a/moses/InputType.h
+++ b/moses/InputType.h
@ -68,6 +68,9 @@ public:
  size_t m_frontSpanCoveredLength;
  // how many words from the beginning are covered

+  // Coordinates in user-defined spaces (see "coord" XML tag)
+  SPTR<std::map<size_t const, std::vector<float> > > m_coordMap;
+
  InputType(AllOptions::ptr const& opts, long translationId = 0);
  virtual ~InputType();

--- a/moses/Jamfile
+++ b/moses/Jamfile
@ -122,10 +122,10 @@ vwfiles synlm mmlib mserver headers
 FF_Factory.o 
 LM//LM 
 TranslationModel/CompactPT//CompactPT 
-TranslationModel/ProbingPT//ProbingPT 
 ThreadPool
 ..//search 
 ../util/double-conversion//double-conversion 
+../probingpt//probingpt
 ..//z 
 ../OnDiskPt//OnDiskPt 
 $(TOP)//boost_filesystem 
@ -139,5 +139,5 @@ alias headers-to-install : [ glob-tree *.h ] ;

 import testing ;

-unit-test moses_test : [ glob *Test.cpp Mock*.cpp FF/*Test.cpp ] ..//boost_filesystem moses headers ..//z ../OnDiskPt//OnDiskPt ..//boost_unit_test_framework ;
+unit-test moses_test : [ glob *Test.cpp Mock*.cpp FF/*Test.cpp ] ..//boost_filesystem moses headers ..//z ../OnDiskPt//OnDiskPt ../probingpt//probingpt ..//boost_unit_test_framework ;

--- a/moses/LM/Implementation.cpp
+++ b/moses/LM/Implementation.cpp
@ -42,6 +42,7 @@ namespace Moses
 {
 LanguageModelImplementation::LanguageModelImplementation(const std::string &line)
  :LanguageModel(line)
+  ,m_nGramOrder(NOT_FOUND)
 {
 }

@ -61,7 +62,7 @@ void LanguageModelImplementation::ShiftOrPush(std::vector<const Word*> &contextF
 {
  if (contextFactor.size() < GetNGramOrder()) {
    contextFactor.push_back(&word);
-  } else {
+  } else if (GetNGramOrder() > 0) {
    // shift
    for (size_t currNGramOrder = 0 ; currNGramOrder < GetNGramOrder() - 1 ; currNGramOrder++) {
      contextFactor[currNGramOrder] = contextFactor[currNGramOrder + 1];
--- a/moses/LM/InMemoryPerSentenceOnDemandLM.cpp
+++ b/moses/LM/InMemoryPerSentenceOnDemandLM.cpp
@ -0,0 +1,101 @@
+#include <boost/foreach.hpp>
+#include "InMemoryPerSentenceOnDemandLM.h"
+#include "moses/FactorCollection.h"
+#include "moses/Util.h"
+#include "moses/StaticData.h"
+#include "moses/TranslationTask.h"
+#include "moses/ContextScope.h"
+#include "moses/LM/Ken.h"
+#include "lm/model.hh"
+#include "util/mmap.hh"
+
+#include <cstdio>
+#include <iostream>
+#include <fstream>
+
+using namespace std;
+
+namespace Moses
+{
+InMemoryPerSentenceOnDemandLM::InMemoryPerSentenceOnDemandLM(const std::string &line) : LanguageModel(line)
+{
+  ReadParameters();
+}
+
+InMemoryPerSentenceOnDemandLM::~InMemoryPerSentenceOnDemandLM()
+{
+}
+
+void InMemoryPerSentenceOnDemandLM::InitializeForInput(ttasksptr const& ttask)
+{
+
+  // The context scope object for this translation task
+  //     contains a map of translation task-specific data
+  boost::shared_ptr<Moses::ContextScope> contextScope = ttask->GetScope();
+
+  // The key to the map is this object
+  void const* key = static_cast<void const*>(this);
+
+  // The value stored in the map is a string representing a phrase table
+  boost::shared_ptr<string> value = contextScope->get<string>(key);
+
+  // Create a stream to read the phrase table data
+  stringstream strme(*(value.get()));
+
+  char * nullpointer = (char *) 0;
+  const char * filename = std::tmpnam(nullpointer);
+  ofstream tmp;
+  tmp.open(filename);
+
+  // Read the phrase table data, one line at a time
+  string line;
+  while (getline(strme, line)) {
+
+    tmp << line << "\n";
+
+  }
+
+  tmp.close();
+
+  //  m_tmpFilename.reset(new std::string("/home/lanes/mosesdecoder/tiny.with_per_sentence/europarl.en.srilm"));
+  m_tmpFilename.reset(new std::string(filename));
+
+  //LanguageModelKen<lm::ngram::ProbingModel> & lm =
+  GetPerThreadLM();
+
+  //  std::remove(filename);
+
+}
+
+LanguageModelKen<lm::ngram::ProbingModel>& InMemoryPerSentenceOnDemandLM::GetPerThreadLM() const
+{
+
+  LanguageModelKen<lm::ngram::ProbingModel> *lm;
+  lm = m_perThreadLM.get();
+  if (lm == NULL) {
+    lm = new LanguageModelKen<lm::ngram::ProbingModel>();
+
+    string* filename = m_tmpFilename.get();
+    if (filename == NULL) {
+      UTIL_THROW(util::Exception, "Can't get a thread-specific LM because no temporary filename has been set for this thread\n");
+    } else {
+      lm->LoadModel(*filename, util::POPULATE_OR_READ);
+    }
+
+    VERBOSE(1, filename);
+    VERBOSE(1, "\tLM initialized\n");
+
+    m_perThreadLM.reset(lm);
+  }
+  assert(lm);
+
+  return *lm;
+
+}
+
+
+
+}
+
+
+
--- a/moses/LM/InMemoryPerSentenceOnDemandLM.h
+++ b/moses/LM/InMemoryPerSentenceOnDemandLM.h
@ -0,0 +1,166 @@
+// $Id$
+#pragma once
+
+#include <string>
+#include <vector>
+#include "SingleFactor.h"
+#include <boost/thread/tss.hpp>
+#include "lm/model.hh"
+#include "moses/LM/Ken.h"
+#include "moses/FF/FFState.h"
+
+namespace Moses
+{
+
+struct InMemoryPerSentenceOnDemandLMState : public FFState {
+  lm::ngram::State state;
+  virtual size_t hash() const {
+    size_t ret = hash_value(state);
+    return ret;
+  }
+  virtual bool operator==(const FFState& o) const {
+    const InMemoryPerSentenceOnDemandLMState &other = static_cast<const InMemoryPerSentenceOnDemandLMState &>(o);
+    bool ret = state == other.state;
+    return ret;
+  }
+
+};
+
+class InMemoryPerSentenceOnDemandLM : public LanguageModel
+{
+public:
+  InMemoryPerSentenceOnDemandLM(const std::string &line);
+  ~InMemoryPerSentenceOnDemandLM();
+
+  void InitializeForInput(ttasksptr const& ttask);
+
+  virtual void SetParameter(const std::string& key, const std::string& value) {
+    GetPerThreadLM().SetParameter(key, value);
+  }
+
+  virtual const FFState* EmptyHypothesisState(const InputType &input) const {
+    if (isInitialized()) {
+      return GetPerThreadLM().EmptyHypothesisState(input);
+    } else {
+      return new InMemoryPerSentenceOnDemandLMState();
+    }
+  }
+
+  virtual FFState *EvaluateWhenApplied(const Hypothesis &hypo, const FFState *ps, ScoreComponentCollection *out) const {
+    if (isInitialized()) {
+      return GetPerThreadLM().EvaluateWhenApplied(hypo, ps, out);
+    } else {
+      UTIL_THROW(util::Exception, "Can't evaluate an uninitialized LM\n");
+    }
+  }
+
+  virtual FFState *EvaluateWhenApplied(const ChartHypothesis& cur_hypo, int featureID, ScoreComponentCollection *accumulator) const {
+    if (isInitialized()) {
+      return GetPerThreadLM().EvaluateWhenApplied(cur_hypo, featureID, accumulator);
+    } else {
+      UTIL_THROW(util::Exception, "Can't evaluate an uninitialized LM\n");
+    }
+  }
+
+  virtual FFState *EvaluateWhenApplied(const Syntax::SHyperedge& hyperedge, int featureID, ScoreComponentCollection *accumulator) const {
+    if (isInitialized()) {
+      return GetPerThreadLM().EvaluateWhenApplied(hyperedge, featureID, accumulator);
+    } else {
+      UTIL_THROW(util::Exception, "Can't evaluate an uninitialized LM\n");
+    }
+  }
+
+
+  virtual void CalcScore(const Phrase &phrase, float &fullScore, float &ngramScore, std::size_t &oovCount) const {
+    if (isInitialized()) {
+      GetPerThreadLM().CalcScore(phrase, fullScore, ngramScore, oovCount);
+    } else {
+      UTIL_THROW(util::Exception, "WARNING: InMemoryPerSentenceOnDemand::CalcScore called prior to being initialized");
+    }
+  }
+
+  virtual void CalcScoreFromCache(const Phrase &phrase, float &fullScore, float &ngramScore, std::size_t &oovCount) const {
+    if (isInitialized()) {
+      GetPerThreadLM().CalcScoreFromCache(phrase, fullScore, ngramScore, oovCount);
+    } else {
+      UTIL_THROW(util::Exception, "WARNING: InMemoryPerSentenceOnDemand::CalcScoreFromCache called prior to being initialized");
+    }
+  }
+
+  virtual void IssueRequestsFor(Hypothesis& hypo, const FFState* input_state) {
+    if (isInitialized()) {
+      GetPerThreadLM().IssueRequestsFor(hypo, input_state);
+    } else {
+      UTIL_THROW(util::Exception, "WARNING: InMemoryPerSentenceOnDemand::IssueRequestsFor called prior to being initialized");
+    }
+  }
+
+  virtual void sync() {
+    if (isInitialized()) {
+      GetPerThreadLM().sync();
+    } else {
+      UTIL_THROW(util::Exception, "WARNING: InMemoryPerSentenceOnDemand::sync called prior to being initialized");
+    }
+  }
+
+  virtual void SetFFStateIdx(int state_idx) {
+    if (isInitialized()) {
+      GetPerThreadLM().SetFFStateIdx(state_idx);
+    } else {
+      UTIL_THROW(util::Exception, "WARNING: InMemoryPerSentenceOnDemand::SetFFStateIdx called prior to being initialized");
+    }
+  }
+
+  virtual void IncrementalCallback(Incremental::Manager &manager) const {
+    if (isInitialized()) {
+      GetPerThreadLM().IncrementalCallback(manager);
+    } else {
+      UTIL_THROW(util::Exception, "WARNING: InMemoryPerSentenceOnDemand::IncrementalCallback called prior to being initialized");
+    }
+  }
+
+  virtual void ReportHistoryOrder(std::ostream &out,const Phrase &phrase) const {
+    if (isInitialized()) {
+      GetPerThreadLM().ReportHistoryOrder(out, phrase);
+    } else {
+      UTIL_THROW(util::Exception, "WARNING: InMemoryPerSentenceOnDemand::ReportHistoryOrder called prior to being initialized");
+    }
+  }
+
+  virtual void EvaluateInIsolation(const Phrase &source
+                                   , const TargetPhrase &targetPhrase
+                                   , ScoreComponentCollection &scoreBreakdown
+                                   , ScoreComponentCollection &estimatedScores) const {
+    if (isInitialized()) {
+      GetPerThreadLM().EvaluateInIsolation(source, targetPhrase, scoreBreakdown, estimatedScores);
+    } else {
+      //      UTIL_THROW(util::Exception, "WARNING: InMemoryPerSentenceOnDemand::EvaluateInIsolation called prior to being initialized");
+    }
+  }
+
+  bool IsUseable(const FactorMask &mask) const {
+    bool ret = mask[m_factorType];
+    return ret;
+  }
+
+
+protected:
+  LanguageModelKen<lm::ngram::ProbingModel> & GetPerThreadLM() const;
+
+  mutable boost::thread_specific_ptr<LanguageModelKen<lm::ngram::ProbingModel> > m_perThreadLM;
+  mutable boost::thread_specific_ptr<std::string> m_tmpFilename;
+
+  FactorType m_factorType;
+
+  bool isInitialized() const {
+    if (m_tmpFilename.get() == NULL) {
+      return false;
+    } else {
+      return true;
+    }
+  }
+
+};
+
+
+}
--- a/moses/LM/Jamfile
+++ b/moses/LM/Jamfile
@ -138,7 +138,7 @@ if $(with-dalm) {

 #Top-level LM library.  If you've added a file that doesn't depend on external
 #libraries, put it here.  
-alias LM : Backward.cpp BackwardLMState.cpp Base.cpp BilingualLM.cpp Implementation.cpp Ken.cpp MultiFactor.cpp Remote.cpp SingleFactor.cpp SkeletonLM.cpp 
+alias LM : Backward.cpp BackwardLMState.cpp Base.cpp BilingualLM.cpp Implementation.cpp InMemoryPerSentenceOnDemandLM.cpp Ken.cpp MultiFactor.cpp Remote.cpp SingleFactor.cpp SkeletonLM.cpp
  ../../lm//kenlm ..//headers $(dependencies) ;

 alias macros : : : : <define>$(lmmacros) ;
--- a/moses/LM/Ken.cpp
+++ b/moses/LM/Ken.cpp
@ -1,5 +1,3 @@
-// $Id$\
-
 /***********************************************************************
 Moses - factored phrase-based language decoder
 Copyright (C) 2006 University of Edinburgh
@ -107,17 +105,27 @@ template <class Model> void LanguageModelKen<Model>::LoadModel(const std::string
  config.load_method = load_method;

  m_ngram.reset(new Model(file.c_str(), config));
+  VERBOSE(2, "LanguageModelKen " << m_description << " reset to " << file << "\n");
 }

 template <class Model> LanguageModelKen<Model>::LanguageModelKen(const std::string &line, const std::string &file, FactorType factorType, util::LoadMethod load_method)
  :LanguageModel(line)
-  ,m_factorType(factorType)
  ,m_beginSentenceFactor(FactorCollection::Instance().AddFactor(BOS_))
+  ,m_factorType(factorType)
 {
  ReadParameters();
  LoadModel(file, load_method);
 }

+template <class Model> LanguageModelKen<Model>::LanguageModelKen()
+  :LanguageModel("KENLM")
+  ,m_beginSentenceFactor(FactorCollection::Instance().AddFactor(BOS_))
+  ,m_factorType(0)
+{
+  ReadParameters();
+}
+
+
 template <class Model> LanguageModelKen<Model>::LanguageModelKen(const LanguageModelKen<Model> &copy_from)
  :LanguageModel(copy_from.GetArgLine()),
   m_ngram(copy_from.m_ngram),
@ -445,7 +453,13 @@ LanguageModel *ConstructKenLM(const std::string &lineOrig)
      filePath.assign(value.data(), value.size());
    } else if (name == "lazyken") {
      // deprecated: use load instead.
-      load_method = boost::lexical_cast<bool>(value) ? util::LAZY : util::POPULATE_OR_READ;
+      if (value == "0" || value == "false") {
+        load_method = util::POPULATE_OR_READ;
+      } else if (value == "1" || value == "true") {
+        load_method = util::LAZY;
+      } else {
+        UTIL_THROW2("Can't parse lazyken argument " << value << ".  Also, lazyken is deprecated.  Use load with one of the arguments lazy, populate_or_lazy, populate_or_read, read, or parallel_read.");
+      }
    } else if (name == "load") {
      if (value == "lazy") {
        load_method = util::LAZY;
--- a/moses/LM/Ken.h
+++ b/moses/LM/Ken.h
@ -33,11 +33,14 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 #include "moses/TypeDef.h"
 #include "moses/Word.h"

+
+
 namespace Moses
 {

 //class LanguageModel;
 class FFState;
+class InMemoryPerSentenceOnDemandLM;

 LanguageModel *ConstructKenLM(const std::string &line);

@ -67,6 +70,8 @@ public:

  virtual bool IsUseable(const FactorMask &mask) const;

+  friend class InMemoryPerSentenceOnDemandLM;
+
 protected:
  boost::shared_ptr<Model> m_ngram;

@ -84,6 +89,7 @@ protected:
  std::vector<lm::WordIndex> m_lmIdLookup;

 private:
+  LanguageModelKen();
  LanguageModelKen(const LanguageModelKen<Model> &copy_from);

  // Convert last words of hypothesis into vocab ids, returning an end pointer.
--- a/moses/LM/NeuralLMWrapper.cpp
+++ b/moses/LM/NeuralLMWrapper.cpp
@ -54,16 +54,21 @@ LMResult NeuralLMWrapper::GetValue(const vector<const Word*> &contextFactor, Sta
    //TODO: config option?
    m_neuralLM->set_cache(1000000);
  }
-  size_t hashCode = 0;

  vector<int> words(contextFactor.size());
-  for (size_t i=0, n=contextFactor.size(); i<n; i++) {
+  const size_t n = contextFactor.size();
+  for (size_t i=0; i<n; i++) {
    const Word* word = contextFactor[i];
    const Factor* factor = word->GetFactor(m_factorType);
    const std::string string = factor->GetString().as_string();
    int neuralLM_wordID = m_neuralLM->lookup_word(string);
    words[i] = neuralLM_wordID;
-    boost::hash_combine(hashCode, neuralLM_wordID);
+  }
+  // Generate hashCode for only the last n-1 words, that represents the next LM
+  // state
+  size_t hashCode = 0;
+  for (size_t i=1; i<n; ++i) {
+    boost::hash_combine(hashCode, words[i]);
  }

  double value = m_neuralLM->lookup_ngram(words);
--- a/moses/LM/PointerState.h
+++ b/moses/LM/PointerState.h
@ -7,6 +7,11 @@ namespace Moses

 struct PointerState : public FFState {
  const void* lmstate;
+
+  explicit PointerState() {
+    // uninitialised
+  }
+
  PointerState(const void* lms) {
    lmstate = lms;
  }
--- a/moses/LM/Reloading.h
+++ b/moses/LM/Reloading.h
@ -35,113 +35,48 @@ namespace Moses
 {

 class FFState;
-
-//LanguageModel *ConstructReloadingLM(const std::string &line);
-//LanguageModel *ConstructReloadingLM(const std::string &line, const std::string &file, FactorType factorType, bool lazy);
-/*
- namespace {
-class MappingBuilder : public lm::EnumerateVocab
-{
-public:
-  MappingBuilder(FactorCollection &factorCollection, std::vector<lm::WordIndex> &mapping)
-    : m_factorCollection(factorCollection), m_mapping(mapping) {}
-
-  void Add(lm::WordIndex index, const StringPiece &str) {
-    std::size_t factorId = m_factorCollection.AddFactor(str)->GetId();
-    if (m_mapping.size() <= factorId) {
-      // 0 is <unk> :-)
-      m_mapping.resize(factorId + 1);
-    }
-    m_mapping[factorId] = index;
-  }
-
-private:
-  FactorCollection &m_factorCollection;
-  std::vector<lm::WordIndex> &m_mapping;
-};
- }
-*/
 template <class Model> class ReloadingLanguageModel : public LanguageModelKen<Model>
 {
 public:
-  // TODO(Lane) copy less code, update to load_method
+
  ReloadingLanguageModel(const std::string &line, const std::string &file, FactorType factorType, bool lazy) : LanguageModelKen<Model>(line, file, factorType, lazy ? util::LAZY : util::POPULATE_OR_READ), m_file(file), m_lazy(lazy) {

-    std::cerr << "ReloadingLM constructor: " << m_file << std::endl;
-    //    std::cerr << std::string(line).replace(0,11,"KENLM") << std::endl;
+    VERBOSE(1, "ReloadingLM constructor: " << m_file << std::endl);

  }

  virtual void InitializeForInput(ttasksptr const& ttask) {
-    std::cerr << "ReloadingLM InitializeForInput" << std::endl;
-    // TODO(lane): load_method
-    LanguageModelKen<Model>::LoadModel(m_file, m_lazy ? util::LAZY : util::POPULATE_OR_READ);
-    /*
-    lm::ngram::Config config;
-    if(this->m_verbosity >= 1) {
-      config.messages = &std::cerr;
-    } else {
-      config.messages = NULL;
+    VERBOSE(1, "ReloadingLM InitializeForInput" << std::endl);
+
+    // The context scope object for this translation task
+    //     contains a map of translation task-specific data
+    boost::shared_ptr<Moses::ContextScope> contextScope = ttask->GetScope();
+
+    // The key to the map is this object
+    void const* key = static_cast<void const*>(this);
+
+    // The value stored in the map is a string representing a phrase table
+    boost::shared_ptr<string> value = contextScope->get<string>(key);
+
+    // Create a stream to read the phrase table data
+    stringstream strme(*(value.get()));
+
+    ofstream tmp;
+    tmp.open(m_file.c_str());
+
+    // Read the phrase table data, one line at a time
+    string line;
+    while (getline(strme, line)) {
+
+      tmp << line << "\n";
+
    }
-    FactorCollection &collection = FactorCollection::Instance();
-    MappingBuilder builder(collection, m_lmIdLookup);
-    config.enumerate_vocab = &builder;
-    config.load_method = m_lazy ? util::LAZY : util::POPULATE_OR_READ;

-    m_ngram.reset(new Model(m_file.c_str(), config));
+    tmp.close();

-    m_beginSentenceFactor = collection.AddFactor(BOS_);
-    */
+    LanguageModelKen<Model>::LoadModel(m_file, m_lazy ? util::LAZY : util::POPULATE_OR_READ);
  };

-  /*
-  ReloadingLanguageModel(const std::string &line) : LanguageModelKen<Model>(ConstructKenLM(std::string(line).replace(0,11,"KENLM"))) {
-    std::cerr << "ReloadingLM constructor" << std::endl;
-    std::cerr << std::string(line).replace(0,11,"KENLM") << std::endl;
-  }
-  */
-  /*
-  ~ReloadingLanguageModel() {
-    delete m_lm;
-  }
-
-  virtual const FFState *EmptyHypothesisState(const InputType &input) const {
-    return m_lm->EmptyHypothesisState(input);
-  }
-
-  virtual void CalcScore(const Phrase &phrase, float &fullScore, float &ngramScore, size_t &oovCount) const {
-    m_lm->CalcScore(phrase, fullScore, ngramScore, oovCount);
-  }
-
-  virtual FFState *EvaluateWhenApplied(const Hypothesis &hypo, const FFState *ps, ScoreComponentCollection *out) const {
-    return m_lm->EvaluateWhenApplied(hypo, ps, out);
-  }
-
-  virtual FFState *EvaluateWhenApplied(const ChartHypothesis& cur_hypo, int featureID, ScoreComponentCollection *accumulator) const {
-    return m_lm->EvaluateWhenApplied(cur_hypo, featureID, accumulator);
-  }
-
-  virtual FFState *EvaluateWhenApplied(const Syntax::SHyperedge& hyperedge, int featureID, ScoreComponentCollection *accumulator) const {
-    return m_lm->EvaluateWhenApplied(hyperedge, featureID, accumulator);
-  }
-
-  virtual void IncrementalCallback(Incremental::Manager &manager) const {
-    m_lm->IncrementalCallback(manager);
-  }
-
-  virtual void ReportHistoryOrder(std::ostream &out,const Phrase &phrase) const {
-    m_lm->ReportHistoryOrder(out, phrase);
-  }
-
-  virtual bool IsUseable(const FactorMask &mask) const {
-    return m_lm->IsUseable(mask);
-  }
-
-
-  private:
-
-  LanguageModel *m_lm;
-  */

 protected:

--- a/moses/LM/SkeletonLM.cpp
+++ b/moses/LM/SkeletonLM.cpp
@ -11,6 +11,9 @@ SkeletonLM::SkeletonLM(const std::string &line)
 {
  ReadParameters();

+  UTIL_THROW_IF2(m_nGramOrder == NOT_FOUND, "Must set order");
+  UTIL_THROW_IF2(m_nGramOrder <= 1, "Ngram order must be more than 1");
+
  FactorCollection &factorCollection = FactorCollection::Instance();

  // needed by parent language model classes. Why didn't they set these themselves?
--- a/moses/OutputCollector.h
+++ b/moses/OutputCollector.h
@ -32,6 +32,7 @@
 #include <iostream>
 #include <map>
 #include <ostream>
+#include <fstream>
 #include <string>
 #include "Util.h"
 #include "util/exception.hh"
--- a/moses/Parameter.cpp
+++ b/moses/Parameter.cpp
@ -264,7 +264,6 @@ Parameter::Parameter()
  AddParam(misc_opts,"references", "Reference file(s) - used for bleu score feature");
  AddParam(misc_opts,"recover-input-path", "r", "(conf net/word lattice only) - recover input path corresponding to the best translation");
  AddParam(misc_opts,"link-param-count", "Number of parameters on word links when using confusion networks or lattices (default = 1)");
-  AddParam(misc_opts,"description", "Source language, target language, description");
  AddParam(misc_opts,"feature-name-overwrite", "Override feature name (NOT arguments). Eg. SRILM-->KENLM, PhraseDictionaryMemory-->PhraseDictionaryScope3");

  AddParam(misc_opts,"feature", "All the feature functions should be here");
--- a/moses/Parameter.h
+++ b/moses/Parameter.h
@ -129,7 +129,7 @@ public:
  void OverwriteParam(const std::string &paramName, PARAM_VEC values);

  std::vector<float> GetWeights(const std::string &name);
-  std::map<std::string, std::vector<float> > GetAllWeights() const {
+  const std::map<std::string, std::vector<float> > &GetAllWeights() const {
    return m_weights;
  }
  std::set<std::string> GetWeightNames() const;
--- a/moses/RuleCube.cpp
+++ b/moses/RuleCube.cpp
@ -33,7 +33,6 @@ using namespace std;

 namespace Moses
 {
-extern bool g_mosesDebug;

 // initialise the RuleCube by creating the top-left corner item
 RuleCube::RuleCube(const ChartTranslationOptions &transOpt,
--- a/moses/ScoreComponentCollection.h
+++ b/moses/ScoreComponentCollection.h
@ -247,6 +247,14 @@ public:
    }
  }

+  void PlusEquals(const FeatureFunction* sp, float scores[]) {
+    size_t numScores = sp->GetNumScoreComponents();
+    size_t offset = sp->GetIndex();
+    for (size_t i = 0; i < numScores; ++i) {
+      m_scores[i + offset] += scores[i];
+    }
+  }
+
  //! Special version PlusEquals(ScoreProducer, vector<float>)
  //! to add the score from a single ScoreProducer that produces
  //! a single value
--- a/moses/SearchNormal.cpp
+++ b/moses/SearchNormal.cpp
@ -240,6 +240,10 @@ ExpandAllHypotheses(const Hypothesis &hypothesis, size_t startPos, size_t endPos
  const Bitmap &sourceCompleted = hypothesis.GetWordsBitmap();
  float estimatedScore = m_transOptColl.GetEstimatedScores().CalcEstimatedScore( sourceCompleted, startPos, endPos );

+  const Range &hypoRange = hypothesis.GetCurrSourceWordsRange();
+  //cerr << "DOING " << sourceCompleted << " [" << hypoRange.GetStartPos() << " " << hypoRange.GetEndPos() << "]"
+  //		  " [" << startPos << " " << endPos << "]" << endl;
+
  if (m_options.search.UseEarlyDiscarding()) {
    // expected score is based on score of current hypothesis
    expectedScore = hypothesis.GetScore();
--- a/moses/Sentence.cpp
+++ b/moses/Sentence.cpp
@ -154,7 +154,8 @@ aux_interpret_xml(std::string& line, std::vector<size_t> & xmlWalls,
    bool OK = ProcessAndStripXMLTags(*m_options, line,
                                     m_xmlOptions,
                                     m_reorderingConstraint,
-                                     xmlWalls, placeholders);
+                                     xmlWalls, placeholders,
+                                     *this);
    if (!OK) {
      TRACE_ERR("Unable to parse XML in line: " << line);
    }
--- a/moses/StaticData.cpp
+++ b/moses/StaticData.cpp
@ -62,8 +62,6 @@ using namespace boost::algorithm;

 namespace Moses
 {
-bool g_mosesDebug = false;
-
 StaticData StaticData::s_instance;

 StaticData::StaticData()
@ -71,6 +69,7 @@ StaticData::StaticData()
  , m_requireSortingAfterSourceContext(false)
  , m_currentWeightSetting("default")
  , m_treeStructure(NULL)
+  , m_coordSpaceNextID(1)
 {
  Phrase::InitializeMemPool();
 }
@ -681,8 +680,8 @@ void StaticData::LoadSparseWeightsFromConfig()
    featureNames.insert(descr);
  }

-  std::map<std::string, std::vector<float> > weights = m_parameter->GetAllWeights();
-  std::map<std::string, std::vector<float> >::iterator iter;
+  const std::map<std::string, std::vector<float> > &weights = m_parameter->GetAllWeights();
+  std::map<std::string, std::vector<float> >::const_iterator iter;
  for (iter = weights.begin(); iter != weights.end(); ++iter) {
    // this indicates that it is sparse feature
    if (featureNames.find(iter->first) == featureNames.end()) {
@ -938,4 +937,25 @@ void StaticData::ResetWeights(const std::string &denseWeights, const std::string
  }
 }

+size_t StaticData::GetCoordSpace(string space) const
+{
+  map<string const, size_t>::const_iterator m = m_coordSpaceMap.find(space);
+  if(m == m_coordSpaceMap.end()) {
+    return 0;
+  }
+  return m->second;
+}
+
+size_t StaticData::MapCoordSpace(string space)
+{
+  map<string const, size_t>::const_iterator m = m_coordSpaceMap.find(space);
+  if (m != m_coordSpaceMap.end()) {
+    return m->second;
+  }
+  size_t id = m_coordSpaceNextID;
+  m_coordSpaceNextID += 1;
+  m_coordSpaceMap[space] = id;
+  return id;
+}
+
 } // namespace
--- a/moses/StaticData.h
+++ b/moses/StaticData.h
@ -60,7 +60,7 @@ class PhraseDictionaryDynamicCacheBased;
 typedef std::pair<std::string, float> UnknownLHSEntry;
 typedef std::vector<UnknownLHSEntry>  UnknownLHSList;

-/** Contains global variables and contants.
+/** Contains global variables and constants.
 *  Only 1 object of this class should be instantiated.
 *  A const object of this class is accessible by any function during decoding by calling StaticData::Instance();
 */
@ -152,6 +152,12 @@ protected:
  bool ini_performance_options();

  void initialize_features();
+
+  // Coordinate space name map for matching spaces across XML input ("coord"
+  // tag) and feature functions that assign or use coordinates on target phrases
+  std::map< std::string const, size_t > m_coordSpaceMap;
+  size_t m_coordSpaceNextID;
+
 public:

  //! destructor
@ -394,6 +400,9 @@ public:
    return m_requireSortingAfterSourceContext;
  }

+  // Coordinate spaces
+  size_t GetCoordSpace(std::string space) const;
+  size_t MapCoordSpace(std::string space);
 };

 }
--- a/moses/TargetPhrase.cpp
+++ b/moses/TargetPhrase.cpp
@ -125,6 +125,7 @@ TargetPhrase::TargetPhrase(const Phrase &phrase, const PhraseDictionary *pt)

 TargetPhrase::TargetPhrase(const TargetPhrase &copy)
  : Phrase(copy)
+  , m_cached_coord(copy.m_cached_coord)
  , m_cached_scores(copy.m_cached_scores)
  , m_scope(copy.m_scope)
  , m_futureScore(copy.m_futureScore)
@ -333,6 +334,31 @@ SetExtraScores(FeatureFunction const* ff,
  m_cached_scores[ff] = s;
 }

+vector<SPTR<vector<float> > > const*
+TargetPhrase::
+GetCoordList(size_t const spaceID) const
+{
+  if(!m_cached_coord) {
+    return NULL;
+  }
+  CoordCache_t::const_iterator m = m_cached_coord->find(spaceID);
+  if(m == m_cached_coord->end()) {
+    return NULL;
+  }
+  return &m->second;
+}
+
+void
+TargetPhrase::
+PushCoord(size_t const spaceID,
+          SPTR<vector<float> > const coord)
+{
+  if (!m_cached_coord) {
+    m_cached_coord.reset(new CoordCache_t);
+  }
+  vector<SPTR<vector<float> > >& coordList = (*m_cached_coord)[spaceID];
+  coordList.push_back(coord);
+}

 void TargetPhrase::SetProperties(const StringPiece &str)
 {
--- a/moses/TargetPhrase.h
+++ b/moses/TargetPhrase.h
@ -56,9 +56,13 @@ public:
  Scores const* GetExtraScores(FeatureFunction const* ff) const;
  void SetExtraScores(FeatureFunction const* ff,boost::shared_ptr<Scores> const& scores);

+  typedef std::map<size_t const, std::vector<SPTR<std::vector<float> > > > CoordCache_t;
+  std::vector<SPTR<std::vector<float> > > const* GetCoordList(size_t const spaceID) const;
+  void PushCoord(size_t const spaceID, SPTR<std::vector<float> > const coord);

 private:
  ScoreCache_t m_cached_scores;
+  SPTR<CoordCache_t> m_cached_coord;
  WPTR<ContextScope> m_scope;

 private:
--- a/moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerOnDisk.cpp
+++ b/moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerOnDisk.cpp
@ -108,7 +108,7 @@ void ChartRuleLookupManagerOnDisk::GetChartRuleCollection(

    // search for terminal symbol
    if (startPos == absEndPos) {
-      OnDiskPt::Word *sourceWordBerkeleyDb = m_dbWrapper.ConvertFromMoses(m_inputFactorsVec, sourceWordLabel.GetLabel());
+      OnDiskPt::Word *sourceWordBerkeleyDb = m_dictionary.ConvertFromMoses(m_dbWrapper, m_inputFactorsVec, sourceWordLabel.GetLabel());

      if (sourceWordBerkeleyDb != NULL) {
        const OnDiskPt::PhraseNode *node = prevNode.GetChild(*sourceWordBerkeleyDb, m_dbWrapper);
@ -154,7 +154,7 @@ void ChartRuleLookupManagerOnDisk::GetChartRuleCollection(
    for (iterSourceLHS = sourceLHSSet.begin(); iterSourceLHS != sourceLHSSet.end(); ++iterSourceLHS) {
      const Word &sourceLHS = *iterSourceLHS;

-      OnDiskPt::Word *sourceLHSBerkeleyDb = m_dbWrapper.ConvertFromMoses(m_inputFactorsVec, sourceLHS);
+      OnDiskPt::Word *sourceLHSBerkeleyDb = m_dictionary.ConvertFromMoses(m_dbWrapper, m_inputFactorsVec, sourceLHS);

      if (sourceLHSBerkeleyDb == NULL) {
        delete sourceLHSBerkeleyDb;
@ -190,7 +190,7 @@ void ChartRuleLookupManagerOnDisk::GetChartRuleCollection(

        if (doSearch) {

-          OnDiskPt::Word *chartNonTermBerkeleyDb = m_dbWrapper.ConvertFromMoses(m_outputFactorsVec, cellLabel.GetLabel());
+          OnDiskPt::Word *chartNonTermBerkeleyDb = m_dictionary.ConvertFromMoses(m_dbWrapper, m_outputFactorsVec, cellLabel.GetLabel());

          if (chartNonTermBerkeleyDb == NULL)
            continue;
@ -234,7 +234,7 @@ void ChartRuleLookupManagerOnDisk::GetChartRuleCollection(
      for (iterLabelSet = lhsSet.begin(); iterLabelSet != lhsSet.end(); ++iterLabelSet) {
        const Word &sourceLHS = *iterLabelSet;

-        OnDiskPt::Word *sourceLHSBerkeleyDb = m_dbWrapper.ConvertFromMoses(m_inputFactorsVec, sourceLHS);
+        OnDiskPt::Word *sourceLHSBerkeleyDb = m_dictionary.ConvertFromMoses(m_dbWrapper, m_inputFactorsVec, sourceLHS);
        if (sourceLHSBerkeleyDb == NULL)
          continue;

@ -251,12 +251,13 @@ void ChartRuleLookupManagerOnDisk::GetChartRuleCollection(

            std::vector<float> weightT = staticData.GetWeights(&m_dictionary);
            targetPhraseCollection
-            = tpcollBerkeleyDb->ConvertToMoses(m_inputFactorsVec
-                                               ,m_outputFactorsVec
-                                               ,m_dictionary
-                                               ,weightT
-                                               ,m_dbWrapper.GetVocab()
-                                               ,true);
+            = m_dictionary.ConvertToMoses(tpcollBerkeleyDb
+                                          ,m_inputFactorsVec
+                                          ,m_outputFactorsVec
+                                          ,m_dictionary
+                                          ,weightT
+                                          ,m_dbWrapper.GetVocab()
+                                          ,true);

            tpcollBerkeleyDb.reset();
            m_cache[tpCollFilePos] = targetPhraseCollection;
--- a/moses/TranslationModel/CompactPT/PhraseDecoder.cpp
+++ b/moses/TranslationModel/CompactPT/PhraseDecoder.cpp
@ -219,7 +219,11 @@ TargetPhraseVectorPtr PhraseDecoder::CreateTargetPhraseCollection(const Phrase &
  // Retrieve source phrase identifier
  std::string sourcePhraseString = sourcePhrase.GetStringRep(*m_input);
  size_t sourcePhraseId = m_phraseDictionary.m_hash[MakeSourceKey(sourcePhraseString)];
-
+  /*
+  cerr << "sourcePhraseString=" << sourcePhraseString << " "
+  	  << sourcePhraseId
+  	  << endl;
+  */
  if(sourcePhraseId != m_phraseDictionary.m_hash.GetSize()) {
    // Retrieve compressed and encoded target phrase collection
    std::string encodedPhraseCollection;
--- a/moses/TranslationModel/CompactPT/PhraseDictionaryCompact.cpp
+++ b/moses/TranslationModel/CompactPT/PhraseDictionaryCompact.cpp
@ -48,7 +48,7 @@ PhraseDictionaryCompact::SentenceCache PhraseDictionaryCompact::m_sentenceCache;

 PhraseDictionaryCompact::PhraseDictionaryCompact(const std::string &line)
  :PhraseDictionary(line, true)
-  ,m_inMemory(true)//(s_inMemoryByDefault)
+  ,m_inMemory(s_inMemoryByDefault)
  ,m_useAlignmentInfo(true)
  ,m_hash(10, 16)
  ,m_phraseDecoder(0)
@ -101,6 +101,7 @@ TargetPhraseCollection::shared_ptr
 PhraseDictionaryCompact::
 GetTargetPhraseCollectionNonCacheLEGACY(const Phrase &sourcePhrase) const
 {
+  //cerr << "sourcePhrase=" << sourcePhrase << endl;

  TargetPhraseCollection::shared_ptr ret;
  // There is no souch source phrase if source phrase is longer than longest
@ -187,7 +188,7 @@ void
 PhraseDictionaryCompact::
 SetStaticDefaultParameters(Parameter const& param)
 {
-  param.SetParameter(s_inMemoryByDefault, "minphr-memory", true);
+  param.SetParameter(s_inMemoryByDefault, "minphr-memory", false);
 }
 }

--- a/moses/TranslationModel/PhraseDictionaryMemoryPerSentenceOnDemand.cpp
+++ b/moses/TranslationModel/PhraseDictionaryMemoryPerSentenceOnDemand.cpp
@ -0,0 +1,150 @@
+// vim:tabstop=2
+#include "PhraseDictionaryMemoryPerSentenceOnDemand.h"
+#include "moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerSkeleton.h"
+#include <sstream>
+
+using namespace std;
+
+namespace Moses
+{
+PhraseDictionaryMemoryPerSentenceOnDemand::PhraseDictionaryMemoryPerSentenceOnDemand(const std::string &line)
+  : PhraseDictionary(line, true), m_valuesAreProbabilities(true)
+{
+  ReadParameters();
+}
+
+void PhraseDictionaryMemoryPerSentenceOnDemand::Load(AllOptions::ptr const& opts)
+{
+  m_options = opts;
+  SetFeaturesToApply();
+
+  // don't load anything. Load when we have the input
+}
+
+
+TargetPhraseCollection::shared_ptr PhraseDictionaryMemoryPerSentenceOnDemand::GetTargetPhraseCollectionNonCacheLEGACY(const Phrase &source) const
+{
+
+  Coll &coll = GetColl();
+
+  return coll[source];
+
+}
+
+
+void PhraseDictionaryMemoryPerSentenceOnDemand::InitializeForInput(ttasksptr const& ttask)
+{
+  Coll &coll = GetColl();
+  coll.clear();
+
+  VERBOSE(2, "Initializing PhraseDictionaryMemoryPerSentenceOnDemand " << m_description << "\n");
+
+  // The context scope object for this translation task
+  //     contains a map of translation task-specific data
+  boost::shared_ptr<Moses::ContextScope> contextScope = ttask->GetScope();
+
+  // The key to the map is this object
+  void const* key = static_cast<void const*>(this);
+
+  // The value stored in the map is a string representing a phrase table
+  boost::shared_ptr<string> value = contextScope->get<string>(key);
+
+  // Create a stream to read the phrase table data
+  stringstream strme(*(value.get()));
+
+  // Read the phrase table data, one line at a time
+  string line;
+  while (getline(strme, line)) {
+
+    VERBOSE(3, "\t" << line);
+
+    vector<string> toks = TokenizeMultiCharSeparator(line, "|||");
+    Phrase source;
+    source.CreateFromString(Input, m_input, toks[0], NULL);
+
+    TargetPhrase *target = new TargetPhrase(this);
+    target->CreateFromString(Output, m_output, toks[1], NULL);
+
+    // score for this phrase table
+    vector<float> scores = Tokenize<float>(toks[2]);
+    if (m_valuesAreProbabilities) {
+      std::transform(scores.begin(), scores.end(), scores.begin(),TransformScore);
+      std::transform(scores.begin(), scores.end(), scores.begin(),FloorScore);
+    }
+    target->GetScoreBreakdown().PlusEquals(this, scores);
+
+    // score of all other ff when this rule is being loaded
+    target->EvaluateInIsolation(source, GetFeaturesToApply());
+
+    // add to coll
+    TargetPhraseCollection::shared_ptr &tpsPtr = coll[source];
+    TargetPhraseCollection *tps = tpsPtr.get();
+    if (tps == NULL) {
+      tps = new TargetPhraseCollection();
+      tpsPtr.reset(tps);
+    }
+    tps->Add(target);
+  }
+}
+
+void PhraseDictionaryMemoryPerSentenceOnDemand::GetTargetPhraseCollectionBatch(const InputPathList &inputPathQueue) const
+{
+  InputPathList::const_iterator iter;
+  for (iter = inputPathQueue.begin(); iter != inputPathQueue.end(); ++iter) {
+    InputPath &inputPath = **iter;
+    const Phrase &source = inputPath.GetPhrase();
+
+    Coll &coll = GetColl();
+    Coll::const_iterator iter = coll.find(source);
+    if (iter == coll.end()) {
+      TargetPhraseCollection::shared_ptr tprPtr;
+      inputPath.SetTargetPhrases(*this, tprPtr, NULL);
+    } else {
+      const TargetPhraseCollection::shared_ptr &tprPtr = iter->second;
+      inputPath.SetTargetPhrases(*this, tprPtr, NULL);
+    }
+  }
+}
+
+
+ChartRuleLookupManager* PhraseDictionaryMemoryPerSentenceOnDemand::CreateRuleLookupManager(const ChartParser &parser,
+    const ChartCellCollectionBase &cellCollection,
+    std::size_t /*maxChartSpan*/)
+{
+  abort();
+}
+
+PhraseDictionaryMemoryPerSentenceOnDemand::Coll &PhraseDictionaryMemoryPerSentenceOnDemand::GetColl() const
+{
+  Coll *coll;
+  coll = m_coll.get();
+  if (coll == NULL) {
+    coll = new Coll;
+    m_coll.reset(coll);
+  }
+  assert(coll);
+  return *coll;
+}
+
+void
+PhraseDictionaryMemoryPerSentenceOnDemand::SetParameter(const std::string& key, const std::string& value)
+{
+  if (key == "path") {
+    UTIL_THROW(util::Exception, "PhraseDictionaryMemoryPerSentenceOnDemand does not support key \"path\".");
+  } else if (key == "valuesAreProbabilities") {
+    m_valuesAreProbabilities = Scan<bool>(value);
+  } else {
+    PhraseDictionary::SetParameter(key, value);
+  }
+}
+
+
+TO_STRING_BODY(PhraseDictionaryMemoryPerSentenceOnDemand);
+
+// friend
+ostream& operator<<(ostream& out, const PhraseDictionaryMemoryPerSentenceOnDemand& phraseDict)
+{
+  return out;
+}
+
+}
--- a/moses/TranslationModel/PhraseDictionaryMemoryPerSentenceOnDemand.h
+++ b/moses/TranslationModel/PhraseDictionaryMemoryPerSentenceOnDemand.h
@ -0,0 +1,48 @@
+
+#pragma once
+
+#include "PhraseDictionary.h"
+#include "moses/TypeDef.h"
+#include "moses/TranslationTask.h"
+
+namespace Moses
+{
+class ChartParser;
+class ChartCellCollectionBase;
+class ChartRuleLookupManager;
+
+class PhraseDictionaryMemoryPerSentenceOnDemand : public PhraseDictionary
+{
+  friend std::ostream& operator<<(std::ostream&, const PhraseDictionaryMemoryPerSentenceOnDemand&);
+
+public:
+  PhraseDictionaryMemoryPerSentenceOnDemand(const std::string &line);
+
+  void Load(AllOptions::ptr const& opts);
+
+  void InitializeForInput(ttasksptr const& ttask);
+
+  // for phrase-based model
+  void GetTargetPhraseCollectionBatch(const InputPathList &inputPathQueue) const;
+
+  // for syntax/hiero model (CKY+ decoding)
+  ChartRuleLookupManager* CreateRuleLookupManager(const ChartParser&, const ChartCellCollectionBase&, std::size_t);
+
+  void SetParameter(const std::string& key, const std::string& value);
+
+  TargetPhraseCollection::shared_ptr  GetTargetPhraseCollectionNonCacheLEGACY(const Phrase &source) const;
+
+  TO_STRING();
+
+
+protected:
+  typedef boost::unordered_map<Phrase, TargetPhraseCollection::shared_ptr> Coll;
+  mutable boost::thread_specific_ptr<Coll> m_coll;
+
+  bool m_valuesAreProbabilities;
+
+  Coll &GetColl() const;
+
+};
+
+}  // namespace Moses
--- a/moses/TranslationModel/ProbingPT.cpp
+++ b/moses/TranslationModel/ProbingPT.cpp
@ -0,0 +1,374 @@
+// vim:tabstop=2
+#include "ProbingPT.h"
+#include "moses/StaticData.h"
+#include "moses/FactorCollection.h"
+#include "moses/TargetPhraseCollection.h"
+#include "moses/InputFileStream.h"
+#include "moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerSkeleton.h"
+#include "probingpt/querying.h"
+#include "probingpt/probing_hash_utils.h"
+
+using namespace std;
+
+namespace Moses
+{
+ProbingPT::ProbingPT(const std::string &line)
+  : PhraseDictionary(line,true)
+  ,m_engine(NULL)
+  ,load_method(util::POPULATE_OR_READ)
+{
+  ReadParameters();
+
+  assert(m_input.size() == 1);
+  assert(m_output.size() == 1);
+}
+
+ProbingPT::~ProbingPT()
+{
+  delete m_engine;
+}
+
+void ProbingPT::Load(AllOptions::ptr const& opts)
+{
+  m_options = opts;
+  SetFeaturesToApply();
+
+  m_engine = new probingpt::QueryEngine(m_filePath.c_str(), load_method);
+
+  m_unkId = 456456546456;
+
+  FactorCollection &vocab = FactorCollection::Instance();
+
+  // source vocab
+  const std::map<uint64_t, std::string> &sourceVocab =
+    m_engine->getSourceVocab();
+  std::map<uint64_t, std::string>::const_iterator iterSource;
+  for (iterSource = sourceVocab.begin(); iterSource != sourceVocab.end();
+       ++iterSource) {
+    string wordStr = iterSource->second;
+    //cerr << "wordStr=" << wordStr << endl;
+
+    const Factor *factor = vocab.AddFactor(wordStr);
+
+    uint64_t probingId = iterSource->first;
+    size_t factorId = factor->GetId();
+
+    if (factorId >= m_sourceVocab.size()) {
+      m_sourceVocab.resize(factorId + 1, m_unkId);
+    }
+    m_sourceVocab[factorId] = probingId;
+  }
+
+  // target vocab
+  InputFileStream targetVocabStrme(m_filePath + "/TargetVocab.dat");
+  string line;
+  while (getline(targetVocabStrme, line)) {
+    vector<string> toks = Tokenize(line, "\t");
+    UTIL_THROW_IF2(toks.size() != 2, string("Incorrect format:") + line + "\n");
+
+    //cerr << "wordStr=" << toks[0] << endl;
+
+    const Factor *factor = vocab.AddFactor(toks[0]);
+    uint32_t probingId = Scan<uint32_t>(toks[1]);
+
+    if (probingId >= m_targetVocab.size()) {
+      m_targetVocab.resize(probingId + 1);
+    }
+
+    m_targetVocab[probingId] = factor;
+  }
+
+  // alignments
+  CreateAlignmentMap(m_filePath + "/Alignments.dat");
+
+  // memory mapped file to tps
+  string filePath = m_filePath + "/TargetColl.dat";
+  file.open(filePath.c_str());
+  if (!file.is_open()) {
+    throw "Couldn't open file ";
+  }
+
+  data = file.data();
+  //size_t size = file.size();
+
+  // cache
+  //CreateCache(system);
+
+}
+
+void ProbingPT::CreateAlignmentMap(const std::string path)
+{
+  const std::vector< std::vector<unsigned char> > &probingAlignColl = m_engine->getAlignments();
+  m_aligns.resize(probingAlignColl.size(), NULL);
+
+  for (size_t i = 0; i < probingAlignColl.size(); ++i) {
+    AlignmentInfo::CollType aligns;
+
+    const std::vector<unsigned char> &probingAligns = probingAlignColl[i];
+    for (size_t j = 0; j < probingAligns.size(); j += 2) {
+      size_t startPos = probingAligns[j];
+      size_t endPos = probingAligns[j+1];
+      //cerr << "startPos=" << startPos << " " << endPos << endl;
+      aligns.insert(std::pair<size_t,size_t>(startPos, endPos));
+    }
+
+    const AlignmentInfo *align = AlignmentInfoCollection::Instance().Add(aligns);
+    m_aligns[i] = align;
+    //cerr << "align=" << align->Debug(system) << endl;
+  }
+}
+
+void ProbingPT::SetParameter(const std::string& key, const std::string& value)
+{
+  if (key == "load") {
+    if (value == "lazy") {
+      load_method = util::LAZY;
+    } else if (value == "populate_or_lazy") {
+      load_method = util::POPULATE_OR_LAZY;
+    } else if (value == "populate_or_read" || value == "populate") {
+      load_method = util::POPULATE_OR_READ;
+    } else if (value == "read") {
+      load_method = util::READ;
+    } else if (value == "parallel_read") {
+      load_method = util::PARALLEL_READ;
+    } else {
+      UTIL_THROW2("load method not supported" << value);
+    }
+  } else {
+    PhraseDictionary::SetParameter(key, value);
+  }
+
+}
+
+void ProbingPT::InitializeForInput(ttasksptr const& ttask)
+{
+
+}
+
+void ProbingPT::GetTargetPhraseCollectionBatch(const InputPathList &inputPathQueue) const
+{
+  InputPathList::const_iterator iter;
+  for (iter = inputPathQueue.begin(); iter != inputPathQueue.end(); ++iter) {
+    InputPath &inputPath = **iter;
+    const Phrase &sourcePhrase = inputPath.GetPhrase();
+
+    if (sourcePhrase.GetSize() > m_options->search.max_phrase_length) {
+      continue;
+    }
+
+    TargetPhraseCollection::shared_ptr tpColl = CreateTargetPhrase(sourcePhrase);
+    inputPath.SetTargetPhrases(*this, tpColl, NULL);
+  }
+}
+
+TargetPhraseCollection::shared_ptr ProbingPT::CreateTargetPhrase(const Phrase &sourcePhrase) const
+{
+  // create a target phrase from the 1st word of the source, prefix with 'ProbingPT:'
+  assert(sourcePhrase.GetSize());
+
+  std::pair<bool, uint64_t> keyStruct = GetKey(sourcePhrase);
+  if (!keyStruct.first) {
+    return TargetPhraseCollection::shared_ptr();
+  }
+
+  // check in cache
+  CachePb::const_iterator iter = m_cachePb.find(keyStruct.second);
+  if (iter != m_cachePb.end()) {
+    //cerr << "FOUND IN CACHE " << keyStruct.second << " " << sourcePhrase.Debug(mgr.system) << endl;
+    TargetPhraseCollection *tps = iter->second;
+    return TargetPhraseCollection::shared_ptr(tps);
+  }
+
+  // query pt
+  TargetPhraseCollection *tps = CreateTargetPhrases(sourcePhrase,
+                                keyStruct.second);
+  return TargetPhraseCollection::shared_ptr(tps);
+}
+
+std::pair<bool, uint64_t> ProbingPT::GetKey(const Phrase &sourcePhrase) const
+{
+  std::pair<bool, uint64_t> ret;
+
+  // create a target phrase from the 1st word of the source, prefix with 'ProbingPT:'
+  size_t sourceSize = sourcePhrase.GetSize();
+  assert(sourceSize);
+
+  uint64_t probingSource[sourceSize];
+  GetSourceProbingIds(sourcePhrase, ret.first, probingSource);
+  if (!ret.first) {
+    // source phrase contains a word unknown in the pt.
+    // We know immediately there's no translation for it
+  } else {
+    ret.second = m_engine->getKey(probingSource, sourceSize);
+  }
+
+  return ret;
+
+}
+
+void ProbingPT::GetSourceProbingIds(const Phrase &sourcePhrase,
+                                    bool &ok, uint64_t probingSource[]) const
+{
+
+  size_t size = sourcePhrase.GetSize();
+  for (size_t i = 0; i < size; ++i) {
+    const Word &word = sourcePhrase.GetWord(i);
+    uint64_t probingId = GetSourceProbingId(word);
+    if (probingId == m_unkId) {
+      ok = false;
+      return;
+    } else {
+      probingSource[i] = probingId;
+    }
+  }
+
+  ok = true;
+}
+
+uint64_t ProbingPT::GetSourceProbingId(const Word &word) const
+{
+  uint64_t ret = 0;
+
+  for (size_t i = 0; i < m_input.size(); ++i) {
+    FactorType factorType = m_input[i];
+    const Factor *factor = word[factorType];
+
+    size_t factorId = factor->GetId();
+    if (factorId >= m_sourceVocab.size()) {
+      return m_unkId;
+    }
+    ret += m_sourceVocab[factorId];
+  }
+
+  return ret;
+}
+
+TargetPhraseCollection *ProbingPT::CreateTargetPhrases(
+  const Phrase &sourcePhrase, uint64_t key) const
+{
+  TargetPhraseCollection *tps = NULL;
+
+  //Actual lookup
+  std::pair<bool, uint64_t> query_result; // 1st=found, 2nd=target file offset
+  query_result = m_engine->query(key);
+  //cerr << "key2=" << query_result.second << endl;
+
+  if (query_result.first) {
+    const char *offset = data + query_result.second;
+    uint64_t *numTP = (uint64_t*) offset;
+
+    tps = new TargetPhraseCollection();
+
+    offset += sizeof(uint64_t);
+    for (size_t i = 0; i < *numTP; ++i) {
+      TargetPhrase *tp = CreateTargetPhrase(offset);
+      assert(tp);
+      tp->EvaluateInIsolation(sourcePhrase, GetFeaturesToApply());
+
+      tps->Add(tp);
+
+    }
+
+    tps->Prune(true, m_tableLimit);
+    //cerr << *tps << endl;
+  }
+
+  return tps;
+
+}
+
+TargetPhrase *ProbingPT::CreateTargetPhrase(
+  const char *&offset) const
+{
+  probingpt::TargetPhraseInfo *tpInfo = (probingpt::TargetPhraseInfo*) offset;
+  size_t numRealWords = tpInfo->numWords / m_output.size();
+
+  TargetPhrase *tp = new TargetPhrase(this);
+
+  offset += sizeof(probingpt::TargetPhraseInfo);
+
+  // scores
+  float *scores = (float*) offset;
+
+  size_t totalNumScores = m_engine->num_scores + m_engine->num_lex_scores;
+
+  if (m_engine->logProb) {
+    // set pt score for rule
+    tp->GetScoreBreakdown().PlusEquals(this, scores);
+
+    // save scores for other FF, eg. lex RO. Just give the offset
+    /*
+    if (m_engine->num_lex_scores) {
+      tp->scoreProperties = scores + m_engine->num_scores;
+    }
+    */
+  } else {
+    // log score 1st
+    float logScores[totalNumScores];
+    for (size_t i = 0; i < totalNumScores; ++i) {
+      logScores[i] = FloorScore(TransformScore(scores[i]));
+    }
+
+    // set pt score for rule
+    tp->GetScoreBreakdown().PlusEquals(this, logScores);
+
+    // save scores for other FF, eg. lex RO.
+    /*
+    tp->scoreProperties = pool.Allocate<SCORE>(m_engine->num_lex_scores);
+    for (size_t i = 0; i < m_engine->num_lex_scores; ++i) {
+      tp->scoreProperties[i] = logScores[i + m_engine->num_scores];
+    }
+    */
+  }
+
+  offset += sizeof(float) * totalNumScores;
+
+  // words
+  for (size_t targetPos = 0; targetPos < numRealWords; ++targetPos) {
+    Word &word = tp->AddWord();
+    for (size_t i = 0; i < m_output.size(); ++i) {
+      FactorType factorType = m_output[i];
+
+      uint32_t *probingId = (uint32_t*) offset;
+
+      const Factor *factor = GetTargetFactor(*probingId);
+      assert(factor);
+
+      word[factorType] = factor;
+
+      offset += sizeof(uint32_t);
+    }
+  }
+
+  // align
+  uint32_t alignTerm = tpInfo->alignTerm;
+  //cerr << "alignTerm=" << alignTerm << endl;
+  UTIL_THROW_IF2(alignTerm >= m_aligns.size(), "Unknown alignInd");
+  tp->SetAlignTerm(m_aligns[alignTerm]);
+
+  // properties TODO
+
+  return tp;
+}
+
+//////////////////////////////////////////////////////////////////
+
+
+ChartRuleLookupManager *ProbingPT::CreateRuleLookupManager(
+  const ChartParser &,
+  const ChartCellCollectionBase &,
+  std::size_t)
+{
+  abort();
+  return NULL;
+}
+
+TO_STRING_BODY(ProbingPT);
+
+// friend
+ostream& operator<<(ostream& out, const ProbingPT& phraseDict)
+{
+  return out;
+}
+
+}
--- a/moses/TranslationModel/ProbingPT.h
+++ b/moses/TranslationModel/ProbingPT.h
@ -0,0 +1,87 @@
+
+#pragma once
+#include <boost/iostreams/device/mapped_file.hpp>
+#include <boost/bimap.hpp>
+#include <boost/unordered_map.hpp>
+#include "PhraseDictionary.h"
+#include "util/mmap.hh"
+
+namespace probingpt
+{
+class QueryEngine;
+class target_text;
+}
+
+namespace Moses
+{
+class ChartParser;
+class ChartCellCollectionBase;
+class ChartRuleLookupManager;
+
+class ProbingPT : public PhraseDictionary
+{
+  friend std::ostream& operator<<(std::ostream&, const ProbingPT&);
+
+public:
+  ProbingPT(const std::string &line);
+  ~ProbingPT();
+
+  void Load(AllOptions::ptr const& opts);
+
+  void InitializeForInput(ttasksptr const& ttask);
+
+  void SetParameter(const std::string& key, const std::string& value);
+
+  // for phrase-based model
+  void GetTargetPhraseCollectionBatch(const InputPathList &inputPathQueue) const;
+
+  // for syntax/hiero model (CKY+ decoding)
+  virtual ChartRuleLookupManager *CreateRuleLookupManager(
+    const ChartParser &,
+    const ChartCellCollectionBase &,
+    std::size_t);
+
+  TO_STRING();
+
+
+protected:
+  probingpt::QueryEngine *m_engine;
+  uint64_t m_unkId;
+
+  std::vector<uint64_t> m_sourceVocab; // factor id -> pt id
+  std::vector<const Factor*> m_targetVocab; // pt id -> factor*
+  std::vector<const AlignmentInfo*> m_aligns;
+  util::LoadMethod load_method;
+
+  boost::iostreams::mapped_file_source file;
+  const char *data;
+
+  // caching
+  typedef boost::unordered_map<uint64_t, TargetPhraseCollection*> CachePb;
+  CachePb m_cachePb;
+
+  void CreateAlignmentMap(const std::string path);
+
+  TargetPhraseCollection::shared_ptr CreateTargetPhrase(const Phrase &sourcePhrase) const;
+
+  std::pair<bool, uint64_t> GetKey(const Phrase &sourcePhrase) const;
+  void GetSourceProbingIds(const Phrase &sourcePhrase, bool &ok,
+                           uint64_t probingSource[]) const;
+  uint64_t GetSourceProbingId(const Word &word) const;
+  uint64_t GetSourceProbingId(const Factor *factor) const;
+
+  TargetPhraseCollection *CreateTargetPhrases(
+    const Phrase &sourcePhrase, uint64_t key) const;
+  TargetPhrase *CreateTargetPhrase(
+    const char *&offset) const;
+
+  inline const Factor *GetTargetFactor(uint32_t probingId) const {
+    if (probingId >= m_targetVocab.size()) {
+      return NULL;
+    }
+    return m_targetVocab[probingId];
+  }
+
+};
+
+}  // namespace Moses
--- a/moses/TranslationModel/ProbingPT/Jamfile
+++ b/moses/TranslationModel/ProbingPT/Jamfile
@ -1,8 +0,0 @@
-local current = "" ;
-local includes = ;
-
-fakelib ProbingPT : [ glob *.cpp ] ../..//headers : $(includes) <dependency>$(PT-LOG) : : $(includes) ;
-
-path-constant PT-LOG : bin/pt.log ;
-update-if-changed $(PT-LOG) $(current) ;
-
--- a/moses/TranslationModel/ProbingPT/ProbingPT.cpp
+++ b/moses/TranslationModel/ProbingPT/ProbingPT.cpp
@ -1,229 +0,0 @@
-// vim:tabstop=2
-#include "ProbingPT.h"
-#include "moses/StaticData.h"
-#include "moses/FactorCollection.h"
-#include "moses/TargetPhraseCollection.h"
-#include "moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerSkeleton.h"
-#include "quering.hh"
-
-using namespace std;
-
-namespace Moses
-{
-ProbingPT::ProbingPT(const std::string &line)
-  : PhraseDictionary(line,true)
-  ,m_engine(NULL)
-{
-  ReadParameters();
-
-  assert(m_input.size() == 1);
-  assert(m_output.size() == 1);
-}
-
-ProbingPT::~ProbingPT()
-{
-  delete m_engine;
-}
-
-void ProbingPT::Load(AllOptions::ptr const& opts)
-{
-  m_options = opts;
-  SetFeaturesToApply();
-
-  m_engine = new QueryEngine(m_filePath.c_str());
-
-  m_unkId = 456456546456;
-
-  // source vocab
-  const std::map<uint64_t, std::string> &sourceVocab = m_engine->getSourceVocab();
-  std::map<uint64_t, std::string>::const_iterator iterSource;
-  for (iterSource = sourceVocab.begin(); iterSource != sourceVocab.end(); ++iterSource) {
-    const string &wordStr = iterSource->second;
-    const Factor *factor = FactorCollection::Instance().AddFactor(wordStr);
-
-    uint64_t probingId = iterSource->first;
-
-    SourceVocabMap::value_type entry(factor, probingId);
-    m_sourceVocabMap.insert(entry);
-
-  }
-
-  // target vocab
-  const std::map<unsigned int, std::string> &probingVocab = m_engine->getVocab();
-  std::map<unsigned int, std::string>::const_iterator iter;
-  for (iter = probingVocab.begin(); iter != probingVocab.end(); ++iter) {
-    const string &wordStr = iter->second;
-    const Factor *factor = FactorCollection::Instance().AddFactor(wordStr);
-
-    unsigned int probingId = iter->first;
-
-    TargetVocabMap::value_type entry(factor, probingId);
-    m_vocabMap.insert(entry);
-
-  }
-}
-
-void ProbingPT::InitializeForInput(ttasksptr const& ttask)
-{
-  ReduceCache();
-}
-
-void ProbingPT::GetTargetPhraseCollectionBatch(const InputPathList &inputPathQueue) const
-{
-  CacheColl &cache = GetCache();
-
-  InputPathList::const_iterator iter;
-  for (iter = inputPathQueue.begin(); iter != inputPathQueue.end(); ++iter) {
-    InputPath &inputPath = **iter;
-    const Phrase &sourcePhrase = inputPath.GetPhrase();
-
-    if (sourcePhrase.GetSize() > m_options->search.max_phrase_length) {
-      continue;
-    }
-
-    TargetPhraseCollection::shared_ptr tpColl = CreateTargetPhrase(sourcePhrase);
-
-    // add target phrase to phrase-table cache
-    size_t hash = hash_value(sourcePhrase);
-    std::pair<TargetPhraseCollection::shared_ptr , clock_t> value(tpColl, clock());
-    cache[hash] = value;
-
-    inputPath.SetTargetPhrases(*this, tpColl, NULL);
-  }
-}
-
-std::vector<uint64_t> ProbingPT::ConvertToProbingSourcePhrase(const Phrase &sourcePhrase, bool &ok) const
-{
-  size_t size = sourcePhrase.GetSize();
-  std::vector<uint64_t> ret(size);
-  for (size_t i = 0; i < size; ++i) {
-    const Factor *factor = sourcePhrase.GetFactor(i, m_input[0]);
-    uint64_t probingId = GetSourceProbingId(factor);
-    if (probingId == m_unkId) {
-      ok = false;
-      return ret;
-    } else {
-      ret[i] = probingId;
-    }
-  }
-
-  ok = true;
-  return ret;
-}
-
-TargetPhraseCollection::shared_ptr ProbingPT::CreateTargetPhrase(const Phrase &sourcePhrase) const
-{
-  // create a target phrase from the 1st word of the source, prefix with 'ProbingPT:'
-  assert(sourcePhrase.GetSize());
-
-  TargetPhraseCollection::shared_ptr tpColl;
-  bool ok;
-  vector<uint64_t> probingSource = ConvertToProbingSourcePhrase(sourcePhrase, ok);
-  if (!ok) {
-    // source phrase contains a word unknown in the pt.
-    // We know immediately there's no translation for it
-    return tpColl;
-  }
-
-  std::pair<bool, std::vector<target_text> > query_result;
-
-  //Actual lookup
-  query_result = m_engine->query(probingSource);
-
-  if (query_result.first) {
-    //m_engine->printTargetInfo(query_result.second);
-    tpColl.reset(new TargetPhraseCollection());
-
-    const std::vector<target_text> &probingTargetPhrases = query_result.second;
-    for (size_t i = 0; i < probingTargetPhrases.size(); ++i) {
-      const target_text &probingTargetPhrase = probingTargetPhrases[i];
-      TargetPhrase *tp = CreateTargetPhrase(sourcePhrase, probingTargetPhrase);
-
-      tpColl->Add(tp);
-    }
-
-    tpColl->Prune(true, m_tableLimit);
-  }
-
-  return tpColl;
-}
-
-TargetPhrase *ProbingPT::CreateTargetPhrase(const Phrase &sourcePhrase, const target_text &probingTargetPhrase) const
-{
-  const std::vector<unsigned int> &probingPhrase = probingTargetPhrase.target_phrase;
-  size_t size = probingPhrase.size();
-
-  TargetPhrase *tp = new TargetPhrase(this);
-
-  // words
-  for (size_t i = 0; i < size; ++i) {
-    uint64_t probingId = probingPhrase[i];
-    const Factor *factor = GetTargetFactor(probingId);
-    assert(factor);
-
-    Word &word = tp->AddWord();
-    word.SetFactor(m_output[0], factor);
-  }
-
-  // score for this phrase table
-  vector<float> scores = probingTargetPhrase.prob;
-  std::transform(scores.begin(), scores.end(), scores.begin(),TransformScore);
-  tp->GetScoreBreakdown().PlusEquals(this, scores);
-
-  // alignment
-  /*
-  const std::vector<unsigned char> &alignments = probingTargetPhrase.word_all1;
-
-  AlignmentInfo &aligns = tp->GetAlignTerm();
-  for (size_t i = 0; i < alignS.size(); i += 2 ) {
-    aligns.Add((size_t) alignments[i], (size_t) alignments[i+1]);
-  }
-  */
-
-  // score of all other ff when this rule is being loaded
-  tp->EvaluateInIsolation(sourcePhrase, GetFeaturesToApply());
-  return tp;
-}
-
-const Factor *ProbingPT::GetTargetFactor(uint64_t probingId) const
-{
-  TargetVocabMap::right_map::const_iterator iter;
-  iter = m_vocabMap.right.find(probingId);
-  if (iter != m_vocabMap.right.end()) {
-    return iter->second;
-  } else {
-    // not in mapping. Must be UNK
-    return NULL;
-  }
-}
-
-uint64_t ProbingPT::GetSourceProbingId(const Factor *factor) const
-{
-  SourceVocabMap::left_map::const_iterator iter;
-  iter = m_sourceVocabMap.left.find(factor);
-  if (iter != m_sourceVocabMap.left.end()) {
-    return iter->second;
-  } else {
-    // not in mapping. Must be UNK
-    return m_unkId;
-  }
-}
-
-ChartRuleLookupManager *ProbingPT::CreateRuleLookupManager(
-  const ChartParser &,
-  const ChartCellCollectionBase &,
-  std::size_t)
-{
-  abort();
-  return NULL;
-}
-
-TO_STRING_BODY(ProbingPT);
-
-// friend
-ostream& operator<<(ostream& out, const ProbingPT& phraseDict)
-{
-  return out;
-}
-
-}
--- a/moses/TranslationModel/ProbingPT/ProbingPT.h
+++ b/moses/TranslationModel/ProbingPT/ProbingPT.h
@ -1,59 +0,0 @@
-
-#pragma once
-
-#include <boost/bimap.hpp>
-#include "../PhraseDictionary.h"
-
-class QueryEngine;
-class target_text;
-
-namespace Moses
-{
-class ChartParser;
-class ChartCellCollectionBase;
-class ChartRuleLookupManager;
-
-class ProbingPT : public PhraseDictionary
-{
-  friend std::ostream& operator<<(std::ostream&, const ProbingPT&);
-
-public:
-  ProbingPT(const std::string &line);
-  ~ProbingPT();
-
-  void Load(AllOptions::ptr const& opts);
-
-  void InitializeForInput(ttasksptr const& ttask);
-
-  // for phrase-based model
-  void GetTargetPhraseCollectionBatch(const InputPathList &inputPathQueue) const;
-
-  // for syntax/hiero model (CKY+ decoding)
-  virtual ChartRuleLookupManager *CreateRuleLookupManager(
-    const ChartParser &,
-    const ChartCellCollectionBase &,
-    std::size_t);
-
-  TO_STRING();
-
-
-protected:
-  QueryEngine *m_engine;
-
-  typedef boost::bimap<const Factor *, uint64_t> SourceVocabMap;
-  mutable SourceVocabMap m_sourceVocabMap;
-
-  typedef boost::bimap<const Factor *, unsigned int> TargetVocabMap;
-  mutable TargetVocabMap m_vocabMap;
-
-  TargetPhraseCollection::shared_ptr CreateTargetPhrase(const Phrase &sourcePhrase) const;
-  TargetPhrase *CreateTargetPhrase(const Phrase &sourcePhrase, const target_text &probingTargetPhrase) const;
-  const Factor *GetTargetFactor(uint64_t probingId) const;
-  uint64_t GetSourceProbingId(const Factor *factor) const;
-
-  std::vector<uint64_t> ConvertToProbingSourcePhrase(const Phrase &sourcePhrase, bool &ok) const;
-
-  uint64_t m_unkId;
-};
-
-}  // namespace Moses
--- a/moses/TranslationModel/ProbingPT/hash.cpp
+++ b/moses/TranslationModel/ProbingPT/hash.cpp
@ -1,30 +0,0 @@
-#include "hash.hh"
-
-uint64_t getHash(StringPiece text)
-{
-  std::size_t len = text.size();
-  uint64_t key = util::MurmurHashNative(text.data(), len);
-  return key;
-}
-
-std::vector<uint64_t> getVocabIDs(StringPiece textin)
-{
-  //Tokenize
-  std::vector<uint64_t> output;
-
-  util::TokenIter<util::SingleCharacter> it(textin, util::SingleCharacter(' '));
-
-  while(it) {
-    output.push_back(getHash(*it));
-    it++;
-  }
-
-  return output;
-}
-
-uint64_t getVocabID(std::string candidate)
-{
-  std::size_t len = candidate.length();
-  uint64_t key = util::MurmurHashNative(candidate.c_str(), len);
-  return key;
-}
--- a/moses/TranslationModel/ProbingPT/huffmanish.cpp
+++ b/moses/TranslationModel/ProbingPT/huffmanish.cpp
@ -1,451 +0,0 @@
-#include "huffmanish.hh"
-
-Huffman::Huffman (const char * filepath)
-{
-  //Read the file
-  util::FilePiece filein(filepath);
-
-  //Init uniq_lines to zero;
-  uniq_lines = 0;
-
-  line_text prev_line; //Check for unique lines.
-  int num_lines = 0 ;
-
-  while (true) {
-    line_text new_line;
-
-    num_lines++;
-
-    try {
-      //Process line read
-      new_line = splitLine(filein.ReadLine());
-      count_elements(new_line); //Counts the number of elements, adds new and increments counters.
-
-    } catch (util::EndOfFileException e) {
-      std::cerr << "Unique entries counted: ";
-      break;
-    }
-
-    if (new_line.source_phrase == prev_line.source_phrase) {
-      continue;
-    } else {
-      uniq_lines++;
-      prev_line = new_line;
-    }
-  }
-
-  std::cerr << uniq_lines << std::endl;
-}
-
-void Huffman::count_elements(line_text linein)
-{
-  //For target phrase:
-  util::TokenIter<util::SingleCharacter> it(linein.target_phrase, util::SingleCharacter(' '));
-  while (it) {
-    //Check if we have that entry
-    std::map<std::string, unsigned int>::iterator mapiter;
-    mapiter = target_phrase_words.find(it->as_string());
-
-    if (mapiter != target_phrase_words.end()) {
-      //If the element is found, increment the count.
-      mapiter->second++;
-    } else {
-      //Else create a new entry;
-      target_phrase_words.insert(std::pair<std::string, unsigned int>(it->as_string(), 1));
-    }
-    it++;
-  }
-
-  //For word allignment 1
-  std::map<std::vector<unsigned char>, unsigned int>::iterator mapiter3;
-  std::vector<unsigned char> numbers = splitWordAll1(linein.word_align);
-  mapiter3 = word_all1.find(numbers);
-
-  if (mapiter3 != word_all1.end()) {
-    //If the element is found, increment the count.
-    mapiter3->second++;
-  } else {
-    //Else create a new entry;
-    word_all1.insert(std::pair<std::vector<unsigned char>, unsigned int>(numbers, 1));
-  }
-
-}
-
-//Assigns huffman values for each unique element
-void Huffman::assign_values()
-{
-  //First create vectors for all maps so that we could sort them later.
-
-  //Create a vector for target phrases
-  for(std::map<std::string, unsigned int>::iterator it = target_phrase_words.begin(); it != target_phrase_words.end(); it++ ) {
-    target_phrase_words_counts.push_back(*it);
-  }
-  //Sort it
-  std::sort(target_phrase_words_counts.begin(), target_phrase_words_counts.end(), sort_pair());
-
-  //Create a vector for word allignments 1
-  for(std::map<std::vector<unsigned char>, unsigned int>::iterator it = word_all1.begin(); it != word_all1.end(); it++ ) {
-    word_all1_counts.push_back(*it);
-  }
-  //Sort it
-  std::sort(word_all1_counts.begin(), word_all1_counts.end(), sort_pair_vec());
-
-
-  //Afterwards we assign a value for each phrase, starting from 1, as zero is reserved for delimiter
-  unsigned int i = 1; //huffman code
-  for(std::vector<std::pair<std::string, unsigned int> >::iterator it = target_phrase_words_counts.begin();
-      it != target_phrase_words_counts.end(); it++) {
-    target_phrase_huffman.insert(std::pair<std::string, unsigned int>(it->first, i));
-    i++; //Go to the next huffman code
-  }
-
-  i = 1; //Reset i for the next map
-  for(std::vector<std::pair<std::vector<unsigned char>, unsigned int> >::iterator it = word_all1_counts.begin();
-      it != word_all1_counts.end(); it++) {
-    word_all1_huffman.insert(std::pair<std::vector<unsigned char>, unsigned int>(it->first, i));
-    i++; //Go to the next huffman code
-  }
-
-  //After lookups are produced, clear some memory usage of objects not needed anymore.
-  target_phrase_words.clear();
-  word_all1.clear();
-
-  target_phrase_words_counts.clear();
-  word_all1_counts.clear();
-
-  std::cerr << "Finished generating huffman codes." << std::endl;
-
-}
-
-void Huffman::serialize_maps(const char * dirname)
-{
-  //Note that directory name should exist.
-  std::string basedir(dirname);
-  std::string target_phrase_path(basedir + "/target_phrases");
-  std::string probabilities_path(basedir + "/probs");
-  std::string word_all1_path(basedir + "/Wall1");
-
-  //Target phrase
-  std::ofstream os (target_phrase_path.c_str(), std::ios::binary);
-  boost::archive::text_oarchive oarch(os);
-  oarch << lookup_target_phrase;
-  os.close();
-
-  //Word all1
-  std::ofstream os2 (word_all1_path.c_str(), std::ios::binary);
-  boost::archive::text_oarchive oarch2(os2);
-  oarch2 << lookup_word_all1;
-  os2.close();
-}
-
-std::vector<unsigned char> Huffman::full_encode_line(line_text line)
-{
-  return vbyte_encode_line((encode_line(line)));
-}
-
-std::vector<unsigned int> Huffman::encode_line(line_text line)
-{
-  std::vector<unsigned int> retvector;
-
-  //Get target_phrase first.
-  util::TokenIter<util::SingleCharacter> it(line.target_phrase, util::SingleCharacter(' '));
-  while (it) {
-    retvector.push_back(target_phrase_huffman.find(it->as_string())->second);
-    it++;
-  }
-  //Add a zero;
-  retvector.push_back(0);
-
-  //Get probabilities. Reinterpreting the float bytes as unsgined int.
-  util::TokenIter<util::SingleCharacter> probit(line.prob, util::SingleCharacter(' '));
-  while (probit) {
-    //Sometimes we have too big floats to handle, so first convert to double
-    double tempnum = atof(probit->data());
-    float num = (float)tempnum;
-    retvector.push_back(reinterpret_float(&num));
-    probit++;
-  }
-  //Add a zero;
-  retvector.push_back(0);
-
-
-  //Get Word allignments
-  retvector.push_back(word_all1_huffman.find(splitWordAll1(line.word_align))->second);
-  retvector.push_back(0);
-
-  return retvector;
-}
-
-void Huffman::produce_lookups()
-{
-  //basically invert every map that we have
-  for(std::map<std::string, unsigned int>::iterator it = target_phrase_huffman.begin(); it != target_phrase_huffman.end(); it++ ) {
-    lookup_target_phrase.insert(std::pair<unsigned int, std::string>(it->second, it->first));
-  }
-
-  for(std::map<std::vector<unsigned char>, unsigned int>::iterator it = word_all1_huffman.begin(); it != word_all1_huffman.end(); it++ ) {
-    lookup_word_all1.insert(std::pair<unsigned int, std::vector<unsigned char> >(it->second, it->first));
-  }
-
-}
-
-HuffmanDecoder::HuffmanDecoder (const char * dirname)
-{
-  //Read the maps from disk
-
-  //Note that directory name should exist.
-  std::string basedir(dirname);
-  std::string target_phrase_path(basedir + "/target_phrases");
-  std::string word_all1_path(basedir + "/Wall1");
-
-  //Target phrases
-  std::ifstream is (target_phrase_path.c_str(), std::ios::binary);
-  boost::archive::text_iarchive iarch(is);
-  iarch >> lookup_target_phrase;
-  is.close();
-
-  //Word allignment 1
-  std::ifstream is2 (word_all1_path.c_str(), std::ios::binary);
-  boost::archive::text_iarchive iarch2(is2);
-  iarch2 >> lookup_word_all1;
-  is2.close();
-
-}
-
-HuffmanDecoder::HuffmanDecoder (std::map<unsigned int, std::string> * lookup_target,
-                                std::map<unsigned int, std::vector<unsigned char> > * lookup_word1)
-{
-  lookup_target_phrase = *lookup_target;
-  lookup_word_all1 = *lookup_word1;
-}
-
-std::vector<target_text> HuffmanDecoder::full_decode_line (std::vector<unsigned char> lines, int num_scores)
-{
-  std::vector<target_text> retvector; //All target phrases
-  std::vector<unsigned int> decoded_lines = vbyte_decode_line(lines); //All decoded lines
-  std::vector<unsigned int>::iterator it = decoded_lines.begin(); //Iterator for them
-  std::vector<unsigned int> current_target_phrase; //Current target phrase decoded
-
-  short zero_count = 0; //Count home many zeroes we have met. so far. Every 3 zeroes mean a new target phrase.
-  while(it != decoded_lines.end()) {
-    if (zero_count == 1) {
-      //We are extracting scores. we know how many scores there are so we can push them
-      //to the vector. This is done in case any of the scores is 0, because it would mess
-      //up the state machine.
-      for (int i = 0; i < num_scores; i++) {
-        current_target_phrase.push_back(*it);
-        it++;
-      }
-    }
-
-    if (zero_count == 3) {
-      //We have finished with this entry, decode it, and add it to the retvector.
-      retvector.push_back(decode_line(current_target_phrase, num_scores));
-      current_target_phrase.clear(); //Clear the current target phrase and the zero_count
-      zero_count = 0; //So that we can reuse them for the next target phrase
-    }
-    //Add to the next target_phrase, number by number.
-    current_target_phrase.push_back(*it);
-    if (*it == 0) {
-      zero_count++;
-    }
-    it++; //Go to the next word/symbol
-  }
-  //Don't forget the last remaining line!
-  if (zero_count == 3) {
-    //We have finished with this entry, decode it, and add it to the retvector.
-    retvector.push_back(decode_line(current_target_phrase, num_scores));
-    current_target_phrase.clear(); //Clear the current target phrase and the zero_count
-    zero_count = 0; //So that we can reuse them for the next target phrase
-  }
-
-  return retvector;
-
-}
-
-target_text HuffmanDecoder::decode_line (std::vector<unsigned int> input, int num_scores)
-{
-  //demo decoder
-  target_text ret;
-  //Split everything
-  std::vector<unsigned int> target_phrase;
-  std::vector<unsigned int> probs;
-  unsigned int wAll;
-
-  //Split the line into the proper arrays
-  short num_zeroes = 0;
-  int counter = 0;
-  while (num_zeroes < 3) {
-    unsigned int num = input[counter];
-    if (num == 0) {
-      num_zeroes++;
-    } else if (num_zeroes == 0) {
-      target_phrase.push_back(num);
-    } else if (num_zeroes == 1) {
-      //Push exactly num_scores scores
-      for (int i = 0; i < num_scores; i++) {
-        probs.push_back(num);
-        counter++;
-        num = input[counter];
-      }
-      continue;
-    } else if (num_zeroes == 2) {
-      wAll = num;
-    }
-    counter++;
-  }
-
-  ret.target_phrase = target_phrase;
-  ret.word_all1 = lookup_word_all1.find(wAll)->second;
-
-  //Decode probabilities
-  for (std::vector<unsigned int>::iterator it = probs.begin(); it != probs.end(); it++) {
-    ret.prob.push_back(reinterpret_uint(&(*it)));
-  }
-
-  return ret;
-
-}
-
-inline std::string HuffmanDecoder::getTargetWordFromID(unsigned int id)
-{
-  return lookup_target_phrase.find(id)->second;
-}
-
-std::string HuffmanDecoder::getTargetWordsFromIDs(std::vector<unsigned int> ids)
-{
-  std::string returnstring;
-  for (std::vector<unsigned int>::iterator it = ids.begin(); it != ids.end(); it++) {
-    returnstring.append(getTargetWordFromID(*it) + " ");
-  }
-
-  return returnstring;
-}
-
-inline std::string getTargetWordFromID(unsigned int id, std::map<unsigned int, std::string> * lookup_target_phrase)
-{
-  return lookup_target_phrase->find(id)->second;
-}
-
-std::string getTargetWordsFromIDs(std::vector<unsigned int> ids, std::map<unsigned int, std::string> * lookup_target_phrase)
-{
-  std::string returnstring;
-  for (std::vector<unsigned int>::iterator it = ids.begin(); it != ids.end(); it++) {
-    returnstring.append(getTargetWordFromID(*it, lookup_target_phrase) + " ");
-  }
-
-  return returnstring;
-}
-
-/*Those functions are used to more easily store the floats in the binary phrase table
- We convert the float unsinged int so that it is the same as our other values and we can
- apply variable byte encoding on top of it.*/
-
-inline unsigned int reinterpret_float(float * num)
-{
-  unsigned int * converted_num;
-  converted_num = reinterpret_cast<unsigned int *>(num);
-  return *converted_num;
-}
-
-inline float reinterpret_uint(unsigned int * num)
-{
-  float * converted_num;
-  converted_num = reinterpret_cast<float *>(num);
-  return *converted_num;
-}
-
-/*Mostly taken from stackoverflow, http://stackoverflow.com/questions/5858646/optimizing-variable-length-encoding
-and modified in order to return a vector of chars. Implements ULEB128 or variable byte encoding.
-This is highly optimized version with unrolled loop */
-inline std::vector<unsigned char> vbyte_encode(unsigned int num)
-{
-  //Determine how many bytes we are going to take.
-  short size;
-  std::vector<unsigned char> byte_vector;
-
-  if (num < 0x00000080U) {
-    size = 1;
-    byte_vector.reserve(size);
-    goto b1;
-  }
-  if (num < 0x00004000U) {
-    size = 2;
-    byte_vector.reserve(size);
-    goto b2;
-  }
-  if (num < 0x00200000U) {
-    size = 3;
-    byte_vector.reserve(size);
-    goto b3;
-  }
-  if (num < 0x10000000U) {
-    size = 4;
-    byte_vector.reserve(size);
-    goto b4;
-  }
-  size = 5;
-  byte_vector.reserve(size);
-
-
-  //Now proceed with the encoding.
-  byte_vector.push_back((num & 0x7f) | 0x80);
-  num >>= 7;
-b4:
-  byte_vector.push_back((num & 0x7f) | 0x80);
-  num >>= 7;
-b3:
-  byte_vector.push_back((num & 0x7f) | 0x80);
-  num >>= 7;
-b2:
-  byte_vector.push_back((num & 0x7f) | 0x80);
-  num >>= 7;
-b1:
-  byte_vector.push_back(num);
-
-  return byte_vector;
-}
-
-std::vector<unsigned int> vbyte_decode_line(std::vector<unsigned char> line)
-{
-  std::vector<unsigned int> huffman_line;
-  std::vector<unsigned char> current_num;
-
-  for (std::vector<unsigned char>::iterator it = line.begin(); it != line.end(); it++) {
-    current_num.push_back(*it);
-    if ((*it >> 7) != 1) {
-      //We don't have continuation in the next bit
-      huffman_line.push_back(bytes_to_int(current_num));
-      current_num.clear();
-    }
-  }
-  return huffman_line;
-}
-
-inline unsigned int bytes_to_int(std::vector<unsigned char> number)
-{
-  unsigned int retvalue = 0;
-  std::vector<unsigned char>::iterator it = number.begin();
-  unsigned char shift = 0; //By how many bits to shift
-
-  while (it != number.end()) {
-    retvalue |= (*it & 0x7f) << shift;
-    shift += 7;
-    it++;
-  }
-
-  return retvalue;
-}
-
-std::vector<unsigned char> vbyte_encode_line(std::vector<unsigned int> line)
-{
-  std::vector<unsigned char> retvec;
-
-  //For each unsigned int in the line, vbyte encode it and add it to a vector of unsigned chars.
-  for (std::vector<unsigned int>::iterator it = line.begin(); it != line.end(); it++) {
-    std::vector<unsigned char> vbyte_encoded = vbyte_encode(*it);
-    retvec.insert(retvec.end(), vbyte_encoded.begin(), vbyte_encoded.end());
-  }
-
-  return retvec;
-}
--- a/Show More
+++ b/Show More