This commit is contained in:
Ulrich Germann 2015-08-10 10:14:28 +01:00
parent 19313feea0
commit 03463facd7
29 changed files with 61 additions and 111 deletions

View File

@ -13,7 +13,6 @@
#include "moses/TargetPhraseCollection.h"
#include <map>
using namespace std;
namespace Moses
{
class PhraseDictionaryDynSuffixArray;

View File

@ -11,7 +11,7 @@
namespace Moses
{
using namespace std;
typedef std::vector<unsigned> vuint_t;
@ -25,8 +25,8 @@ class ComparePosition
public:
ComparePosition(vuint_t const& crp, vuint_t const& sfa);
bool operator()(unsigned const& i, vector<wordID_t> const& phrase) const;
bool operator()(vector<wordID_t> const& phrase, unsigned const& i) const;
bool operator()(unsigned const& i, std::vector<wordID_t> const& phrase) const;
bool operator()(std::vector<wordID_t> const& phrase, unsigned const& i) const;
};

View File

@ -18,20 +18,22 @@
namespace ugdiss
{
using namespace std;
using namespace boost::iostreams;
/** open input file that is possibly compressed
* decompression filters are automatically added based on the file name
* gzip for .gz; bzip2 for bz2.
*/
filtering_istream* open_input_stream(string fname);
void open_input_stream(string fname, filtering_istream& in);
// filtering_streambuf<input>* open_input_stream(string fname);
/** open input file that is possibly compressed
* decompression filters are automatically added based on the file name
* gzip for .gz; bzip2 for bz2.
*/
boost::iostreams::filtering_istream*
open_input_stream(std::string fname);
filtering_ostream* open_output_stream(string fname);
void open_output_stream(string fname, filtering_ostream& in);
void open_input_stream(std::string fname,
boost::iostreams::filtering_istream& in);
boost::iostreams::filtering_ostream*
open_output_stream(std::string fname);
void open_output_stream(std::string fname,
boost::iostreams::filtering_ostream& in);
}
#endif

View File

@ -3,7 +3,6 @@
#include <vector>
#include <string>
namespace Moses {
using namespace std;
// Function to splice the argument list (e.g. before handing it over to
// Moses LoadParam() function. /filter/ is a vector of argument names
@ -12,7 +11,7 @@ namespace Moses {
filter_arguments(int const argc_in, char const* const* const argv_in,
int & argc_moses, char*** argv_moses,
int & argc_other, char*** argv_other,
vector<pair<string,int> > const& filter);
std::vector<std::pair<std::string, int> > const& filter);
} // namespace Moses

View File

@ -11,24 +11,23 @@
namespace Moses
{
using namespace std;
// select a random sample of size /s/ without restitution from the range of
// integers [0,N);
// select a random sample of size /s/ without restitution from the
// range of integers [0,N);
template<typename idx_t>
void
randomSample(vector<idx_t>& v, size_t s, size_t N)
randomSample(std::vector<idx_t>& v, size_t s, size_t N)
{
// see also Knuth: Art of Computer Programming Vol. 2, p. 142
s = min(s,N);
s = std::min(s,N);
v.resize(s);
// the first option tries to be a bit more efficient than O(N) in picking
// the samples. The threshold is an ad-hoc, off-the-cuff guess. I still
// need to figure out the optimal break-even point between a linear sweep
// and repeatedly picking random numbers with the risk of hitting the same
// number many times.
// the first option tries to be a bit more efficient than O(N) in
// picking the samples. The threshold is an ad-hoc, off-the-cuff
// guess. I still need to figure out the optimal break-even point
// between a linear sweep and repeatedly picking random numbers with
// the risk of hitting the same number many times.
if (s*10<N) {
boost::dynamic_bitset<uint64_t> check(N,0);
for (size_t i = 0; i < v.size(); i++) {

View File

@ -1,3 +1,4 @@
// -*- mode: c++; indent-tabs-mode: nil; tab-width: 2 -*-
#ifndef __vector_index_sorter_h
#define __vector_index_sorter_h
#include <boost/shared_ptr.hpp>
@ -5,8 +6,8 @@
#include <vector>
// VectorIndexSorter; (c) 2007-2012 Ulrich Germann
// A VectorIndexSorter is a function object for sorting indices into a vector
// of objects (instead of sorting the vector itself).
// A VectorIndexSorter is a function object for sorting indices into a
// vector of objects (instead of sorting the vector itself).
//
// typcial use:
// vector<thingy> my_vector;
@ -16,13 +17,9 @@
namespace Moses
{
// using namespace std;
using std::greater;
template<typename VAL,
typename COMP = greater<VAL>,
typename COMP = std::greater<VAL>,
typename IDX_T=size_t>
class
VectorIndexSorter

View File

@ -17,10 +17,6 @@
#include "moses/TranslationModel/UG/mm/tpt_typedefs.h"
using namespace std;
//using namespace boost;
using namespace ugdiss;
namespace stringdist
{
float
@ -32,7 +28,7 @@ namespace stringdist
float
fillAlignmentMatrix(UChar const* a, size_t const lenA,
UChar const* b, size_t const lenB,
vector<vector<float> > & M);
std::vector<std::vector<float> > & M);
class StringDiff
{
@ -67,21 +63,21 @@ namespace stringdist
};
private:
UnicodeString a,b;
vector<Segment> difflist;
vector<int> diffcnt;
std::vector<Segment> difflist;
std::vector<int> diffcnt;
public:
UnicodeString const& set_a(string const& a);
UnicodeString const& set_b(string const& b);
UnicodeString const& set_a(std::string const& a);
UnicodeString const& set_b(std::string const& b);
UnicodeString const& get_a() const;
UnicodeString const& get_b() const;
StringDiff(string const& a, string const& b);
StringDiff(std::string const& a, std::string const& b);
StringDiff();
size_t size();
size_t align(bool force=false); // returns the levenshtein distance
void showDiff(std::ostream& out);
float levenshtein();
Segment const& operator[](uint32_t i) const;
void fillAlignmentMatrix(vector<vector<float> > & M) const;
void fillAlignmentMatrix(std::vector<std::vector<float> > & M) const;
vector<int> const& getFeatures() const;
};
}

View File

@ -24,8 +24,6 @@
#include "ug_corpus_token.h"
#include "tpt_pickler.h"
using namespace ugdiss;
using namespace std;
namespace Moses {
typedef L2R_Token<SimpleWordId> Token;
@ -44,7 +42,7 @@ namespace Moses {
class job;
class worker;
list<job> joblist;
vector<sptr<boost::thread> > workers;
std::vector<SPTR<boost::thread> > workers;
bool shutdown;
size_t doomed;
public:
@ -52,10 +50,10 @@ namespace Moses {
agenda(bitext_base const& bitext);
~agenda();
void add_workers(int n);
sptr<pstats> add_job(mmbitext::iter const& phrase,
SPTR<pstats> add_job(mmbitext::iter const& phrase,
size_t const max_samples);
bool get_task(uint64_t & sid, uint64_t & offset, uint64_t & len,
bool & fwd, sptr<bitext_base::pstats> & stats);
bool & fwd, SPTR<bitext_base::pstats> & stats);
};
// stores the list of unfinished jobs;
@ -76,19 +74,19 @@ namespace Moses {
find_trg_phr_bounds
(size_t const sid, size_t const start, size_t const stop,
size_t & s1, size_t & s2, size_t & e1, size_t & e2,
vector<uchar> * core_alignment, bool const flip) const;
std::vector<uchar> * core_alignment, bool const flip) const;
boost::unordered_map<uint64_t,sptr<pstats> > cache1,cache2;
boost::unordered_map<uint64_t,SPTR<pstats> > cache1,cache2;
private:
sptr<pstats>
SPTR<pstats>
prep2(iter const& phrase);
public:
mmbitext();
~mmbitext();
void open(string const base, string const L1, string const L2);
void open(std::string const base, std::string const L1, std::string const L2);
sptr<pstats> lookup(iter const& phrase);
SPTR<pstats> lookup(iter const& phrase);
void prep(iter const& phrase);
};
@ -99,15 +97,15 @@ namespace Moses {
{
uint32_t my_rcnt; // unweighted count
float my_wcnt; // weighted count
vector<pair<size_t, vector<uchar> > > my_aln;
std::vector<pair<size_t, std::vector<uchar> > > my_aln;
boost::mutex lock;
public:
jstats();
jstats(jstats const& other);
uint32_t rcnt() const;
float wcnt() const;
vector<pair<size_t, vector<uchar> > > const & aln() const;
void add(float w, vector<uchar> const& a);
std::vector<pair<size_t, std::vector<uchar> > > const & aln() const;
void add(float w, std::vector<uchar> const& a);
};
struct
@ -126,11 +124,12 @@ namespace Moses {
size_t in_progress; // keeps track of how many threads are currently working on this
boost::unordered_map<uint64_t, jstats> trg;
pstats();
// vector<phrase> nbest;
// std::vector<phrase> nbest;
// void select_nbest(size_t const N=10);
void release();
void register_worker();
void add(mmbitext::iter const& trg_phrase, float const w, vector<uchar> const& a);
void add(mmbitext::iter const& trg_phrase, float const w,
std::vector<uchar> const& a);
};
class
@ -157,7 +156,7 @@ namespace Moses {
size_t ctr;
size_t len;
bool fwd;
sptr<mmbitext::pstats> stats;
SPTR<mmbitext::pstats> stats;
bool step(uint64_t & sid, uint64_t & offset);
};

View File

@ -10,9 +10,7 @@
#include <iostream>
#include <sstream>
#include "tpt_typedefs.h"
// #include <stdint.h>
#include <cassert>
// // using namespace std;
#ifndef uchar
#endif

View File

@ -21,7 +21,6 @@
#include <vector>
#include <map>
// // using namespace std;
namespace bio=boost::iostreams;
namespace sapt

View File

@ -77,7 +77,7 @@ namespace sapt
using Moses::ttaskwptr;
using tpt::binread;
using tpt::binwrite;
// using namespace ugdiss;
float lbop(size_t const tries, size_t const succ, float const confidence);
void write_bitvector(bitvector const& v, std::ostream& out);

View File

@ -130,7 +130,6 @@ int Bitext<Token>::agenda::job
if (!m_bias) return 1;
// // using namespace boost::math;
typedef boost::math::binomial_distribution<> binomial;
std::ostream* log = m_bias->loglevel > 1 ? m_bias->log : NULL;
@ -138,7 +137,6 @@ int Bitext<Token>::agenda::job
float p = (*m_bias)[sid];
id_type docid = m_bias->GetClass(sid);
// uint32_t k = docid < stats->indoc.size() ? stats->indoc[docid] : 0;
typedef pstats::indoc_map_t::const_iterator id_iter;
id_iter m = stats->indoc.find(docid);
uint32_t k = m != stats->indoc.end() ? m->second : 0 ;

View File

@ -7,7 +7,6 @@
#include "ug_typedefs.h"
namespace sapt
{
// using namespace std;
template<typename T>
class ConllBottomUpToken : public T

View File

@ -7,7 +7,6 @@
namespace sapt
{
// using namespace std;
using tpt::id_type;
using tpt::uchar;

View File

@ -15,7 +15,6 @@
#include "ug_conll_bottom_up_token.h"
#include "ug_typedefs.h"
// using namespace std;
namespace sapt
{

View File

@ -24,13 +24,8 @@
namespace sapt
{
// using namespace std;
// using namespace boost;
namespace bio=boost::iostreams;
// template<typename TOKEN> class imBitext<TOKEN>;
template<typename TOKEN, typename SORTER>
class TsaSorter
{
@ -44,8 +39,8 @@ namespace sapt
public:
TsaSorter(SORTER sorter, iter& begin, iter& end)
: m_sorter(sorter),
m_begin(begin),
m_end(end) { }
m_begin(begin),
m_end(end) { }
bool
operator()()
@ -62,7 +57,7 @@ namespace sapt
class imTSA : public TSA<TOKEN>
{
typedef typename Ttrack<TOKEN>::Position cpos;
// friend class imBitext<TOKEN>;
public:
class tree_iterator;
friend class tree_iterator;

View File

@ -19,8 +19,6 @@
#include "util/exception.hh"
#include "moses/Util.h"
// #include "ug_vocab.h"
// define the corpus buffer size (in sentences) and the
// for adding additional sentences:
#define IMTTRACK_INCREMENT_SIZE 100000

View File

@ -11,7 +11,6 @@
#include <boost/unordered_map.hpp>
#include "tpt_pickler.h"
// using namespace std;
namespace ugdiss
{

View File

@ -14,7 +14,7 @@
#include "tpt_pickler.h"
#include "ug_mm_2d_table.h"
#include "util/exception.hh"
// using namespace std;
namespace sapt
{

View File

@ -14,8 +14,6 @@
namespace lru_cache
{
// using namespace std;
// using namespace boost;
template<typename KEY, typename VAL>
class LRU_Cache

View File

@ -13,7 +13,6 @@
namespace bio=boost::iostreams;
namespace sapt
{
// using namespace std;
template<typename OFFSET, typename ID, typename VAL, typename INIT>
class
mm2dTable

View File

@ -19,7 +19,6 @@
namespace sapt
{
// using namespace std;
namespace bio=boost::iostreams;
template<typename TOKEN>

View File

@ -26,7 +26,6 @@
namespace sapt
{
// using namespace std;
namespace bio=boost::iostreams;
template<typename TKN=id_type>

View File

@ -21,8 +21,6 @@
namespace sapt
{
// using namespace std;
// using namespace boost;
namespace bio=boost::iostreams;
template<typename TKN>

View File

@ -18,7 +18,7 @@
namespace sapt
{
// using namespace std;
template<typename TSA>
class
BitSetCache

View File

@ -20,11 +20,9 @@
#include "tpt_typedefs.h"
#include "tpt_tokenindex.h"
#include "moses/Util.h"
// #include "ug_vocab.h"
namespace sapt
{
// using namespace std;
typedef boost::dynamic_bitset<uint64_t> bdBitset;
using tpt::count_type;

View File

@ -10,7 +10,6 @@
#include "tpt_typedefs.h"
namespace sapt
{
// using namespace std;
typedef boost::dynamic_bitset<uint64_t> bitvector;
typedef std::vector<std::vector<float> > flt_2d_table;

View File

@ -1,13 +0,0 @@
//-*- c++ -*-
#pragma once
#include <stdint.h>
using namespace std;
namespace sapt
{
using namespace Moses;
using namespace std;
}

View File

@ -13,8 +13,6 @@
namespace Moses
{
using namespace std;
#ifndef bitvector
typedef boost::dynamic_bitset<uint64_t> bitvector;
#endif
@ -27,9 +25,9 @@ typedef boost::dynamic_bitset<uint64_t> bitvector;
class WordCoocTable
{
typedef map<wordID_t,uint32_t> my_map_t;
vector<my_map_t> m_cooc;
vector<uint32_t> m_marg1;
vector<uint32_t> m_marg2;
std::vector<my_map_t> m_cooc;
std::vector<uint32_t> m_marg1;
std::vector<uint32_t> m_marg2;
public:
WordCoocTable();
WordCoocTable(wordID_t const VocabSize1, wordID_t const VocabSize2);