2013-12-02 19:29:13 +04:00
|
|
|
// vim:tabstop=2
|
2013-12-04 22:47:33 +04:00
|
|
|
#include <stdlib.h>
|
2013-12-05 17:13:01 +04:00
|
|
|
#include "PhraseDictionaryTransliteration.h"
|
2013-12-02 19:29:13 +04:00
|
|
|
#include "moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerSkeleton.h"
|
2013-12-05 17:06:35 +04:00
|
|
|
#include "moses/DecodeGraph.h"
|
|
|
|
#include "moses/DecodeStep.h"
|
2013-12-02 19:29:13 +04:00
|
|
|
|
|
|
|
using namespace std;
|
|
|
|
|
|
|
|
namespace Moses
|
|
|
|
{
|
2013-12-05 17:13:01 +04:00
|
|
|
PhraseDictionaryTransliteration::PhraseDictionaryTransliteration(const std::string &line)
|
2013-12-02 19:29:13 +04:00
|
|
|
: PhraseDictionary(line)
|
|
|
|
{
|
|
|
|
ReadParameters();
|
2013-12-04 23:16:37 +04:00
|
|
|
UTIL_THROW_IF2(m_mosesDir.empty() ||
|
2015-01-14 14:07:42 +03:00
|
|
|
m_scriptDir.empty() ||
|
|
|
|
m_externalDir.empty() ||
|
|
|
|
m_inputLang.empty() ||
|
|
|
|
m_outputLang.empty(), "Must specify all arguments");
|
2013-12-02 19:29:13 +04:00
|
|
|
}
|
|
|
|
|
2013-12-10 16:42:05 +04:00
|
|
|
void PhraseDictionaryTransliteration::Load()
|
|
|
|
{
|
2015-01-14 14:07:42 +03:00
|
|
|
SetFeaturesToApply();
|
2013-12-10 16:42:05 +04:00
|
|
|
}
|
|
|
|
|
2013-12-05 17:13:01 +04:00
|
|
|
void PhraseDictionaryTransliteration::CleanUpAfterSentenceProcessing(const InputType& source)
|
2013-12-02 19:29:13 +04:00
|
|
|
{
|
2015-01-14 14:07:42 +03:00
|
|
|
ReduceCache();
|
2013-12-02 19:29:13 +04:00
|
|
|
}
|
|
|
|
|
2013-12-05 17:13:01 +04:00
|
|
|
void PhraseDictionaryTransliteration::GetTargetPhraseCollectionBatch(const InputPathList &inputPathQueue) const
|
2013-12-02 19:29:13 +04:00
|
|
|
{
|
2013-12-04 22:47:33 +04:00
|
|
|
|
2013-12-02 19:29:13 +04:00
|
|
|
InputPathList::const_iterator iter;
|
|
|
|
for (iter = inputPathQueue.begin(); iter != inputPathQueue.end(); ++iter) {
|
|
|
|
InputPath &inputPath = **iter;
|
2013-12-05 17:06:35 +04:00
|
|
|
|
|
|
|
if (!SatisfyBackoff(inputPath)) {
|
2015-01-14 14:07:42 +03:00
|
|
|
continue;
|
2013-12-05 17:06:35 +04:00
|
|
|
}
|
|
|
|
|
2013-12-02 19:29:13 +04:00
|
|
|
const Phrase &sourcePhrase = inputPath.GetPhrase();
|
|
|
|
|
|
|
|
if (sourcePhrase.GetSize() != 1) {
|
2015-01-14 14:07:42 +03:00
|
|
|
// only translit single words. A limitation of the translit script
|
|
|
|
continue;
|
2013-12-02 19:29:13 +04:00
|
|
|
}
|
|
|
|
|
2013-12-06 20:25:17 +04:00
|
|
|
GetTargetPhraseCollection(inputPath);
|
2013-12-02 19:29:13 +04:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2013-12-06 20:25:17 +04:00
|
|
|
void PhraseDictionaryTransliteration::GetTargetPhraseCollection(InputPath &inputPath) const
|
|
|
|
{
|
2015-01-14 14:07:42 +03:00
|
|
|
const Phrase &sourcePhrase = inputPath.GetPhrase();
|
|
|
|
size_t hash = hash_value(sourcePhrase);
|
2013-12-06 20:25:17 +04:00
|
|
|
|
2015-01-14 14:07:42 +03:00
|
|
|
CacheColl &cache = GetCache();
|
2013-12-06 20:25:17 +04:00
|
|
|
|
2015-01-14 14:07:42 +03:00
|
|
|
CacheColl::iterator iter;
|
|
|
|
iter = cache.find(hash);
|
2013-12-06 20:25:17 +04:00
|
|
|
|
2015-01-14 14:07:42 +03:00
|
|
|
if (iter != cache.end()) {
|
|
|
|
// already in cache
|
|
|
|
const TargetPhraseCollection *tpColl = iter->second.first;
|
|
|
|
inputPath.SetTargetPhrases(*this, tpColl, NULL);
|
|
|
|
} else {
|
|
|
|
// TRANSLITERATE
|
|
|
|
char *ptr = tmpnam(NULL);
|
|
|
|
string inFile(ptr);
|
|
|
|
ptr = tmpnam(NULL);
|
|
|
|
string outDir(ptr);
|
|
|
|
|
|
|
|
ofstream inStream(inFile.c_str());
|
|
|
|
inStream << sourcePhrase.ToString() << endl;
|
|
|
|
inStream.close();
|
|
|
|
|
|
|
|
string cmd = m_scriptDir + "/Transliteration/prepare-transliteration-phrase-table.pl" +
|
|
|
|
" --transliteration-model-dir " + m_filePath +
|
|
|
|
" --moses-src-dir " + m_mosesDir +
|
|
|
|
" --external-bin-dir " + m_externalDir +
|
|
|
|
" --input-extension " + m_inputLang +
|
|
|
|
" --output-extension " + m_outputLang +
|
|
|
|
" --oov-file " + inFile +
|
|
|
|
" --out-dir " + outDir;
|
|
|
|
|
|
|
|
int ret = system(cmd.c_str());
|
|
|
|
UTIL_THROW_IF2(ret != 0, "Transliteration script error");
|
|
|
|
|
|
|
|
TargetPhraseCollection *tpColl = new TargetPhraseCollection();
|
|
|
|
vector<TargetPhrase*> targetPhrases = CreateTargetPhrases(sourcePhrase, outDir);
|
|
|
|
vector<TargetPhrase*>::const_iterator iter;
|
|
|
|
for (iter = targetPhrases.begin(); iter != targetPhrases.end(); ++iter) {
|
|
|
|
TargetPhrase *tp = *iter;
|
|
|
|
tpColl->Add(tp);
|
2013-12-06 20:25:17 +04:00
|
|
|
}
|
2015-01-14 14:07:42 +03:00
|
|
|
|
|
|
|
std::pair<const TargetPhraseCollection*, clock_t> value(tpColl, clock());
|
|
|
|
cache[hash] = value;
|
|
|
|
|
|
|
|
inputPath.SetTargetPhrases(*this, tpColl, NULL);
|
|
|
|
|
|
|
|
// clean up temporary files
|
|
|
|
remove(inFile.c_str());
|
|
|
|
|
|
|
|
cmd = "rm -rf " + outDir;
|
|
|
|
system(cmd.c_str());
|
|
|
|
}
|
2013-12-06 20:25:17 +04:00
|
|
|
}
|
|
|
|
|
2013-12-05 17:13:01 +04:00
|
|
|
std::vector<TargetPhrase*> PhraseDictionaryTransliteration::CreateTargetPhrases(const Phrase &sourcePhrase, const string &outDir) const
|
2013-12-02 19:29:13 +04:00
|
|
|
{
|
2015-01-14 14:07:42 +03:00
|
|
|
std::vector<TargetPhrase*> ret;
|
2013-12-04 22:47:33 +04:00
|
|
|
|
2015-01-14 14:07:42 +03:00
|
|
|
string outPath = outDir + "/out.txt";
|
|
|
|
ifstream outStream(outPath.c_str());
|
2013-12-04 22:47:33 +04:00
|
|
|
|
2015-01-14 14:07:42 +03:00
|
|
|
string line;
|
|
|
|
while (getline(outStream, line)) {
|
|
|
|
vector<string> toks;
|
|
|
|
Tokenize(toks, line, "\t");
|
|
|
|
UTIL_THROW_IF2(toks.size() != 2, "Error in transliteration output file. Expecting word\tscore");
|
2013-12-04 22:47:33 +04:00
|
|
|
|
2015-01-14 14:07:42 +03:00
|
|
|
TargetPhrase *tp = new TargetPhrase(this);
|
|
|
|
Word &word = tp->AddWord();
|
|
|
|
word.CreateFromString(Output, m_output, toks[0], false);
|
2013-12-02 19:29:13 +04:00
|
|
|
|
2015-01-14 14:07:42 +03:00
|
|
|
float score = Scan<float>(toks[1]);
|
|
|
|
tp->GetScoreBreakdown().PlusEquals(this, score);
|
2013-12-02 19:29:13 +04:00
|
|
|
|
2015-01-14 14:07:42 +03:00
|
|
|
// score of all other ff when this rule is being loaded
|
|
|
|
tp->EvaluateInIsolation(sourcePhrase, GetFeaturesToApply());
|
2013-12-02 19:29:13 +04:00
|
|
|
|
2015-01-14 14:07:42 +03:00
|
|
|
ret.push_back(tp);
|
|
|
|
}
|
2013-12-02 19:29:13 +04:00
|
|
|
|
2015-01-14 14:07:42 +03:00
|
|
|
outStream.close();
|
2013-12-02 19:29:13 +04:00
|
|
|
|
2013-12-04 22:47:33 +04:00
|
|
|
return ret;
|
2013-12-02 19:29:13 +04:00
|
|
|
}
|
|
|
|
|
2013-12-05 17:13:01 +04:00
|
|
|
ChartRuleLookupManager* PhraseDictionaryTransliteration::CreateRuleLookupManager(const ChartParser &parser,
|
2014-03-14 12:49:09 +04:00
|
|
|
const ChartCellCollectionBase &cellCollection,
|
|
|
|
std::size_t /*maxChartSpan*/)
|
2013-12-02 19:29:13 +04:00
|
|
|
{
|
2015-01-14 14:07:42 +03:00
|
|
|
return NULL;
|
2013-12-02 19:29:13 +04:00
|
|
|
//return new ChartRuleLookupManagerSkeleton(parser, cellCollection, *this);
|
|
|
|
}
|
|
|
|
|
2013-12-04 23:16:37 +04:00
|
|
|
void
|
2013-12-05 17:13:01 +04:00
|
|
|
PhraseDictionaryTransliteration::
|
2013-12-04 23:16:37 +04:00
|
|
|
SetParameter(const std::string& key, const std::string& value)
|
|
|
|
{
|
|
|
|
if (key == "moses-dir") {
|
2015-01-14 14:07:42 +03:00
|
|
|
m_mosesDir = value;
|
2013-12-04 23:16:37 +04:00
|
|
|
} else if (key == "script-dir") {
|
2015-01-14 14:07:42 +03:00
|
|
|
m_scriptDir = value;
|
2013-12-04 23:16:37 +04:00
|
|
|
} else if (key == "external-dir") {
|
2015-01-14 14:07:42 +03:00
|
|
|
m_externalDir = value;
|
2013-12-04 23:16:37 +04:00
|
|
|
} else if (key == "input-lang") {
|
2015-01-14 14:07:42 +03:00
|
|
|
m_inputLang = value;
|
2013-12-04 23:16:37 +04:00
|
|
|
} else if (key == "output-lang") {
|
2015-01-14 14:07:42 +03:00
|
|
|
m_outputLang = value;
|
2013-12-04 23:16:37 +04:00
|
|
|
} else {
|
2015-01-14 14:07:42 +03:00
|
|
|
PhraseDictionary::SetParameter(key, value);
|
2013-12-04 23:16:37 +04:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2013-12-05 17:13:01 +04:00
|
|
|
TO_STRING_BODY(PhraseDictionaryTransliteration);
|
2013-12-02 19:29:13 +04:00
|
|
|
|
|
|
|
// friend
|
2013-12-05 17:13:01 +04:00
|
|
|
ostream& operator<<(ostream& out, const PhraseDictionaryTransliteration& phraseDict)
|
2013-12-02 19:29:13 +04:00
|
|
|
{
|
|
|
|
return out;
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|