mirror of
https://github.com/moses-smt/mosesdecoder.git
synced 2024-09-11 11:25:40 +03:00
18 lines
370 B
C++
18 lines
370 B
C++
#pragma once
|
|
|
|
#include "util/string_piece.hh"
|
|
#include "util/murmur_hash.hh"
|
|
#include "util/string_piece.hh" //Tokenization and work with StringPiece
|
|
#include "util/tokenize_piece.hh"
|
|
#include <vector>
|
|
|
|
namespace probingpt
|
|
{
|
|
|
|
//Gets the MurmurmurHash for give string
|
|
uint64_t getHash(StringPiece text);
|
|
|
|
std::vector<uint64_t> getVocabIDs(const StringPiece &textin);
|
|
|
|
}
|