No log message.

git-svn-id: https://mosesdecoder.svn.sourceforge.net/svnroot/mosesdecoder/trunk@669 1f5c12ca-751b-0410-a591-d2e778427230

irstlm/src/lmtable.cpp

@@ -49,7 +49,8 @@ lmtable::lmtable(){
 	memset(info, 0, sizeof(info));
 	memset(NumCenters, 0, sizeof(NumCenters));
  
-  bicache=NULL;
+  max_cache_lev=0;
+  for (int i=0;i<=maxlev;i++) lmtcache[i]=NULL;
   probcache=NULL;
   statecache=NULL;
   
@@ -575,7 +576,7 @@ int lmtable::get(ngram& ng,int n,int lev){
     
     //if (l==2) cout <<"bicache: searching:" << ng <<"\n";
     
-    if (bicache && l==2 && bicache->get(ng.wordp(n),(char *)&found))
+    if (lmtcache[l] && lmtcache[l]->get(ng.wordp(n),(char *)&found))
       hit=1;
     else
       search(table[l] + (offset * nodesize(ndt)),
@@ -587,13 +588,11 @@ int lmtable::get(ngram& ng,int n,int lev){
              LMT_FIND,
              &found);
  
-    if (bicache && l==2 && hit==0){
-      //if (bicache->isfull()) bicache->reset();
-      //cout << "bicache :" << ng <<"\n";
-       bicache->add(ng.wordp(n),(char *)&found);      
-    }    
-      
-    if (!found) return 0;
+    //insert both found and not found items!!!
+    if (lmtcache[l] && hit==0)
+      lmtcache[l]->add(ng.wordp(n),(char *)&found);   
+    
+    if (!found) return 0;      
     
     ng.link=found;
     ng.info=ndt;

irstlm/src/lmtable.h

@@ -136,9 +136,10 @@ class lmtable{
   int    backoff_state; 
   
   //improve access speed
-  ngramcache* bicache;
+  ngramcache* lmtcache[LMTMAXLEV+1];
 	ngramcache* probcache;
   ngramcache* statecache;
+  int max_cache_lev;
 	
 public:
     
@@ -147,10 +148,12 @@ public:
   lmtable();
   
   ~lmtable(){
-    if (bicache){
-      std::cerr << "Bigram Cache: "; bicache->stat();
-      delete bicache; 
+    for (int i=2;i<=max_cache_lev;i++)        
+    if (lmtcache[i]){
+      std::cerr << i <<"-gram cache: "; lmtcache[i]->stat();
+      delete lmtcache[i]; 
     }
+    
     if (probcache){
       std::cerr << "Prob Cache: "; probcache->stat();
       delete probcache;
@@ -181,22 +184,24 @@ public:
     statecache=new ngramcache(maxlev-1,sizeof(char *),200000);
   }
   
-  void init_bicache(){
-    assert(bicache==NULL);
-    bicache=new ngramcache(2,sizeof(char *),200000);
+  void init_lmtcaches(int uptolev){
+    max_cache_lev=uptolev;
+    for (int i=2;i<=max_cache_lev;i++){
+    assert(lmtcache[i]==NULL);
+    lmtcache[i]=new ngramcache(i,sizeof(char *),2000000);
+    }
   }
   
-  
   void check_cache_levels(){
     if (probcache && probcache->isfull()) probcache->reset();
     if (statecache && statecache->isfull()) statecache->reset();
-    if (bicache && bicache->isfull()) bicache->reset();
+    for (int i=2;i<=max_cache_lev;i++)
+      if (lmtcache[i]->isfull()) lmtcache[i]->reset();
   }
   
   bool is_probcache_active(){return probcache!=NULL;}
   bool is_statecache_active(){return statecache!=NULL;}
-  bool is_bicache_active(){return bicache!=NULL;}
-  
+  bool are_lmtcaches_active(){return lmtcache[2]!=NULL;}  
   
 	void configure(int n,bool quantized){
 		maxlev=n;

irstlm/src/ngramcache.cpp

@@ -34,8 +34,8 @@ ngramcache::ngramcache(int n,int size,int maxentries){
       infosize=size;
       maxn=maxentries;
       entries=0;
-      ht=new htable(maxn * 2, ngsize * sizeof(int),INT,NULL); //load factor 2   
-      mp=new mempool(ngsize * sizeof(int)+infosize,maxn/10); 
+      ht=new htable(maxn * 5, ngsize * sizeof(int),INT,NULL); //lower load factor to reduce collisions 
+      mp=new mempool(ngsize * sizeof(int)+infosize,maxn/5); 
       accesses=0;
       hits=0;
     };
@@ -50,8 +50,8 @@ ngramcache::~ngramcache(){
 void ngramcache::reset(){
     ht->stat();
     delete ht;delete mp;    
-    ht=new htable(maxn * 2, ngsize * sizeof(int),INT,NULL); //load factor 2   
-    mp=new mempool(ngsize * sizeof(int)+infosize,maxn/10); 
+    ht=new htable(maxn * 5, ngsize * sizeof(int),INT,NULL); //load factor 2   
+    mp=new mempool(ngsize * sizeof(int)+infosize,maxn/5); 
     entries=0;
   }
 

moses/src/LanguageModel_IRST.cpp

@@ -76,8 +76,8 @@ void LanguageModel_IRST::Load(const std::string &fileName
   
   //install caches
   m_lmtb->init_probcache();
-  m_lmtb->init_bicache();
   m_lmtb->init_statecache();
+  m_lmtb->init_lmtcaches(m_lmtb->maxlevel()>2?m_lmtb->maxlevel()-1:2);
 }
 
 void LanguageModel_IRST::CreateFactors(FactorCollection &factorCollection)

ngramcache.h

@@ -1,49 +0,0 @@
-/******************************************************************************
-IrstLM: IRST Language Model Toolkit
-Copyright (C) 2006 Marcello Federico, ITC-irst Trento, Italy
-
-This library is free software; you can redistribute it and/or
-modify it under the terms of the GNU Lesser General Public
-License as published by the Free Software Foundation; either
-version 2.1 of the License, or (at your option) any later version.
-
-This library is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-Lesser General Public License for more details.
-
-You should have received a copy of the GNU Lesser General Public
-License along with this library; if not, write to the Free Software
-Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA
-
-******************************************************************************/
-
-#ifndef MF_NGRAMCACHE_H
-#define MF_NGRAMCACHE_H
-
-#include "mempool.h"
-#include "htable.h"
-
-class ngramcache{
-private:
-  htable* ht;  
-  mempool *mp;
-  int maxn;
-  int ngsize;
-  int infosize;
-  int accesses;
-  int hits;
-  int entries;
-
-public:
-    
-  ngramcache(int n,int size,int maxentries);  
-  ~ngramcache();  
-  void reset();  
-  char* get(const int* ngp,char* info=NULL);  
-  int add(const int* ngp,const char* info);  
-  int isfull(){return (entries >= maxn);};  
-  void stat();
-};
-
-#endif