mosesdecoder/moses/TranslationModel/PhraseDictionary.cpp

// vim:tabstop=2

/***********************************************************************
Moses - factored phrase-based language decoder
Copyright (C) 2006 University of Edinburgh

This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.

This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
Lesser General Public License for more details.

You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
***********************************************************************/

#include "moses/TranslationModel/PhraseDictionary.h"
#include "moses/StaticData.h"
#include "moses/InputType.h"
#include "moses/TranslationOption.h"
#include "moses/UserMessage.h"
#include "moses/InputPath.h"
#include "util/exception.hh"

using namespace std;

namespace Moses
{

PhraseDictionary::PhraseDictionary(const std::string &description, const std::string &line)
  :DecodeFeature(description, line)
  ,m_tableLimit(20) // default
  ,m_maxCacheSize(DEFAULT_MAX_TRANS_OPT_CACHE_SIZE)
{
}

const TargetPhraseCollection *PhraseDictionary::GetTargetPhraseCollection(const Phrase& src) const
{
  const TargetPhraseCollection *ret;
  if (m_maxCacheSize) {
    size_t hash = hash_value(src);

    std::map<size_t, std::pair<const TargetPhraseCollection*, clock_t> >::iterator iter;

    {
      // scope of read lock
#ifdef WITH_THREADS
      boost::shared_lock<boost::shared_mutex> read_lock(m_accessLock);
#endif
      iter = m_cache.find(hash);
    }

    if (iter == m_cache.end()) {
      // not in cache, need to look up from phrase table
      ret = GetTargetPhraseCollectionNonCache(src);
      if (ret) {
        ret = new TargetPhraseCollection(*ret);
      }

      std::pair<const TargetPhraseCollection*, clock_t> value(ret, clock());
#ifdef WITH_THREADS
      boost::unique_lock<boost::shared_mutex> lock(m_accessLock);
#endif
      m_cache[hash] = value;
    } else {
      // in cache. just use it
    	std::pair<const TargetPhraseCollection*, clock_t> &value = iter->second;
    	value.second = clock();

    	ret = value.first;
    }
  } else {
	// don't use cache. look up from phrase table
    ret = GetTargetPhraseCollectionNonCache(src);
  }

  return ret;
}

const TargetPhraseCollection *PhraseDictionary::GetTargetPhraseCollectionNonCache(const Phrase& src) const
{
  UTIL_THROW(util::Exception, "Legacy method not implemented");
}


const TargetPhraseCollectionWithSourcePhrase* PhraseDictionary::
GetTargetPhraseCollectionLegacy(InputType const& src,WordsRange const& range) const
{
  UTIL_THROW(util::Exception, "Legacy method not implemented");
  //Phrase phrase = src.GetSubString(range);
  //return GetTargetPhraseCollection(phrase);
}

void PhraseDictionary::SetParameter(const std::string& key, const std::string& value)
{
  if (key == "cache-size") {
	  m_maxCacheSize = Scan<size_t>(value);
  } else if (key == "path") {
    m_filePath = value;
  } else if (key == "table-limit") {
    m_tableLimit = Scan<size_t>(value);
  } else {
    DecodeFeature::SetParameter(key, value);
  }
}

void PhraseDictionary::SetFeaturesToApply()
{
  // find out which feature function can be applied in this decode step
  const std::vector<FeatureFunction*> &allFeatures = FeatureFunction::GetFeatureFunctions();
  for (size_t i = 0; i < allFeatures.size(); ++i) {
    FeatureFunction *feature = allFeatures[i];
    if (feature->IsUseable(m_outputFactors)) {
      m_featuresToApply.push_back(feature);
    }
  }
}

void PhraseDictionary::GetTargetPhraseCollectionBatch(const InputPathList &phraseDictionaryQueue) const
{
  InputPathList::const_iterator iter;
  for (iter = phraseDictionaryQueue.begin(); iter != phraseDictionaryQueue.end(); ++iter) {
    InputPath &node = **iter;

    const Phrase &phrase = node.GetPhrase();
    const TargetPhraseCollection *targetPhrases = this->GetTargetPhraseCollection(phrase);
    node.SetTargetPhrases(*this, targetPhrases, NULL);
  }
}

void PhraseDictionary::ReduceCache() const
{
  if (m_cache.size() <= m_maxCacheSize) return; // not full
  clock_t t = clock();

  // find cutoff for last used time
  priority_queue< clock_t > lastUsedTimes;
  std::map<size_t, std::pair<const TargetPhraseCollection*,clock_t> >::iterator iter;
  iter = m_cache.begin();
  while( iter != m_cache.end() ) {
    lastUsedTimes.push( iter->second.second );
    iter++;
  }
  for( size_t i=0; i < lastUsedTimes.size()-m_maxCacheSize/2; i++ )
    lastUsedTimes.pop();
  clock_t cutoffLastUsedTime = lastUsedTimes.top();

  // remove all old entries
#ifdef WITH_THREADS
      boost::unique_lock<boost::shared_mutex> lock(m_accessLock);
#endif

  iter = m_cache.begin();
  while( iter != m_cache.end() ) {
    if (iter->second.second < cutoffLastUsedTime) {
      std::map<size_t, std::pair<const TargetPhraseCollection*,clock_t> >::iterator iterRemove = iter++;
      delete iterRemove->second.first;
      m_cache.erase(iterRemove);
    } else iter++;
  }
  VERBOSE(2,"Reduced persistent translation option cache in " << ((clock()-t)/(float)CLOCKS_PER_SEC) << " seconds." << std::endl);
}
}