moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2024-09-11 11:25:40 +03:00
Code Issues Packages Projects Releases Wiki Activity

dismantle IODevoce framework

Browse Source 
git-svn-id: https://mosesdecoder.svn.sourceforge.net/svnroot/mosesdecoder/trunk@992 1f5c12ca-751b-0410-a591-d2e778427230
This commit is contained in:
hieuhoang1972 2006-11-21 23:06:30 +00:00
parent 7e00b6ee8b
commit 7af5bb462e
15 changed files with 93 additions and 329 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
moses-cmd/moses-cmd.vcproj
View File

@ -181,11 +181,7 @@
UniqueIdentifier="{4FC737F1-C7A5-4376-A066-2A32D752A2FF}"
>
<File
RelativePath=".\src\IOCommandLine.cpp"
>
</File>
<File
RelativePath=".\src\IOFile.cpp"
RelativePath=".\src\IOStream.cpp"
>
</File>
<File
@ -203,11 +199,7 @@
UniqueIdentifier="{93995380-89BD-4b04-88EB-625FBE52EBFB}"
>
<File
RelativePath=".\src\IOCommandLine.h"
>
</File>
<File
RelativePath=".\src\IOFile.h"
RelativePath=".\src\IOStream.h"
>
</File>
<File

56
moses-cmd/src/IOFile.cpp
View File

@ -1,56 +0,0 @@
// $Id$
/***********************************************************************
Moses - factored phrase-based language decoder
Copyright (c) 2006 University of Edinburgh
All rights reserved.
Redistribution and use in source and binary forms, with or without modification,
are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright notice,
this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
* Neither the name of the University of Edinburgh nor the names of its contributors
may be used to endorse or promote products derived from this software
without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS
BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.
***********************************************************************/
// example file on how to use moses library
#include "IOFile.h"
#include "Sentence.h"
using namespace std;
IOFile::IOFile(const std::vector<FactorType> &inputFactorOrder
, const std::vector<FactorType> &outputFactorOrder
, const FactorMask &inputFactorUsed
, FactorCollection &factorCollection
, size_t nBestSize
, const std::string &nBestFilePath
, const std::string &inputFilePath)
:IOCommandLine(inputFactorOrder, outputFactorOrder, inputFactorUsed, factorCollection, nBestSize, nBestFilePath)
,m_inputFilePath(inputFilePath)
,m_inputFile(inputFilePath)
{
}
InputType* IOFile::GetInput(InputType* in)
{
return IODevice::GetInput(in,m_inputFile, m_inputFactorOrder, m_factorCollection);
}

59
moses-cmd/src/IOFile.h
View File

@ -1,59 +0,0 @@
// $Id$
/***********************************************************************
Moses - factored phrase-based language decoder
Copyright (c) 2006 University of Edinburgh
All rights reserved.
Redistribution and use in source and binary forms, with or without modification,
are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright notice,
this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
* Neither the name of the University of Edinburgh nor the names of its contributors
may be used to endorse or promote products derived from this software
without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS
BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.
***********************************************************************/
// example file on how to use moses library
#pragma once
#include <fstream>
#include <vector>
#include "TypeDef.h"
#include "IOCommandLine.h"
#include "InputFileStream.h"
class IOFile : public IOCommandLine
{
protected:
std::string m_inputFilePath;
InputFileStream m_inputFile;
public:
IOFile(const std::vector<FactorType> &inputFactorOrder
, const std::vector<FactorType> &outputFactorOrder
, const FactorMask &inputFactorUsed
, FactorCollection &factorCollection
, size_t nBestSize
, const std::string &nBestFilePath
, const std::string &inputFilePath);
InputType *GetInput(InputType*);
};

54
moses-cmd/src/IOCommandLine.cpp → moses-cmd/src/IOStream.cpp
View File

@ -35,16 +35,17 @@ POSSIBILITY OF SUCH DAMAGE.
#include <iostream>
#include "TypeDef.h"
#include "Util.h"
#include "IOCommandLine.h"
#include "IOStream.h"
#include "Hypothesis.h"
#include "WordsRange.h"
#include "LatticePathList.h"
#include "StaticData.h"
#include "DummyScoreProducers.h"
#include "InputFileStream.h"
using namespace std;
IOCommandLine::IOCommandLine(
IOStream::IOStream(
const vector<FactorType> &inputFactorOrder
, const vector<FactorType> &outputFactorOrder
, const FactorMask &inputFactorUsed
@ -55,6 +56,8 @@ IOCommandLine::IOCommandLine(
,m_outputFactorOrder(outputFactorOrder)
,m_inputFactorUsed(inputFactorUsed)
,m_factorCollection(factorCollection)
,m_inputFile(NULL)
,m_inputStream(&std::cin)
{
if (nBestSize > 0)
{
@ -62,9 +65,46 @@ IOCommandLine::IOCommandLine(
}
}
InputType*IOCommandLine::GetInput(InputType* in)
IOStream::IOStream(const std::vector<FactorType> &inputFactorOrder
, const std::vector<FactorType> &outputFactorOrder
, const FactorMask &inputFactorUsed
, FactorCollection &factorCollection
, size_t nBestSize
, const std::string &nBestFilePath
, const std::string &inputFilePath)
:m_inputFactorOrder(inputFactorOrder)
,m_outputFactorOrder(outputFactorOrder)
,m_inputFactorUsed(inputFactorUsed)
,m_factorCollection(factorCollection)
,m_inputFilePath(inputFilePath)
,m_inputFile(new InputFileStream(inputFilePath))
{
return IODevice::GetInput(in,std::cin,m_inputFactorOrder, m_factorCollection);
m_inputStream = m_inputFile;
if (nBestSize > 0)
{
m_nBestFile.open(nBestFilePath.c_str());
}
}
IOStream::~IOStream()
{
if (m_inputFile != NULL)
delete m_inputFile;
}
InputType*IOStream::GetInput(InputType* inputType)
{
if(inputType->Read(*m_inputStream, m_inputFactorOrder, m_factorCollection))
{
inputType->SetTranslationId(m_translationId++);
return inputType;
}
else
{
delete inputType;
return NULL;
}
}
/***
@ -111,7 +151,7 @@ void OutputSurface(std::ostream &out, const Hypothesis *hypo, const std::vector<
}
}
void IOCommandLine::Backtrack(const Hypothesis *hypo){
void IOStream::Backtrack(const Hypothesis *hypo){
if (hypo->GetPrevHypo() != NULL) {
VERBOSE(3,hypo->GetId() << " <= ");
@ -119,7 +159,7 @@ void IOCommandLine::Backtrack(const Hypothesis *hypo){
}
}
void IOCommandLine::SetOutput(const Hypothesis *hypo, long /*translationId*/, bool reportSegmentation, bool reportAllFactors)
void IOStream::OutputBestHypo(const Hypothesis *hypo, long /*translationId*/, bool reportSegmentation, bool reportAllFactors)
{
if (hypo != NULL)
{
@ -138,7 +178,7 @@ void IOCommandLine::SetOutput(const Hypothesis *hypo, long /*translationId*/, bo
cout << endl;
}
void IOCommandLine::SetNBest(const LatticePathList &nBestList, long translationId)
void IOStream::OutputNBestList(const LatticePathList &nBestList, long translationId)
{
bool labeledOutput = StaticData::Instance()->IsLabeledNBestList();

49
moses-cmd/src/IOCommandLine.h → moses-cmd/src/IOStream.h
View File

@ -37,49 +37,50 @@ POSSIBILITY OF SUCH DAMAGE.
#include <fstream>
#include <vector>
#include "TypeDef.h"
#include "IODevice.h"
#include "Sentence.h"
class FactorMask;
class FactorCollection;
class Hypothesis;
class LatticePathList;
class InputFileStream;
class IOCommandLine : public IODevice
class IOStream
{
protected:
long m_translationId;
const std::vector<FactorType> &m_inputFactorOrder;
const std::vector<FactorType> &m_outputFactorOrder;
const FactorMask &m_inputFactorUsed;
FactorCollection &m_factorCollection;
std::ofstream m_nBestFile;
std::string m_inputFilePath;
std::istream *m_inputStream;
InputFileStream *m_inputFile;
public:
IOCommandLine(const std::vector<FactorType> &inputFactorOrder
IOStream(const std::vector<FactorType> &inputFactorOrder
, const std::vector<FactorType> &outputFactorOrder
, const FactorMask &inputFactorUsed
, FactorCollection &factorCollection
, size_t nBestSize
, const std::string &nBestFilePath);
InputType* GetInput(InputType*);
void SetOutput(const Hypothesis *hypo, long translationId, bool reportSegmentation, bool reportAllFactors);
void SetNBest(const LatticePathList &nBestList, long translationId);
IOStream(const std::vector<FactorType> &inputFactorOrder
, const std::vector<FactorType> &outputFactorOrder
, const FactorMask &inputFactorUsed
, FactorCollection &factorCollection
, size_t nBestSize
, const std::string &nBestFilePath
, const std::string &inputFilePath);
~IOStream();
InputType* GetInput(InputType *inputType);
void OutputBestHypo(const Hypothesis *hypo, long translationId, bool reportSegmentation, bool reportAllFactors);
void OutputNBestList(const LatticePathList &nBestList, long translationId);
void Backtrack(const Hypothesis *hypo);
void ResetTranslationId() { m_translationId = 0; }
};
#if 0
// help fn
inline Sentence *GetInput(std::istream &inputStream
, const std::vector<FactorType> &factorOrder
, FactorCollection &factorCollection)
{
return dynamic_cast<Sentence*>(GetInput(new Sentence(Input),inputStream,factorOrder,factorCollection));
#if 0
Sentence *rv=new Sentence(Input);
if(rv->Read(inputStream,factorOrder,factorCollection))
return rv;
else {delete rv; return 0;}
#endif
}
#endif

33
moses-cmd/src/Main.cpp
View File

@ -46,8 +46,7 @@ POSSIBILITY OF SUCH DAMAGE.
#include "Util.h"
#include "LatticePathList.h"
#include "Timer.h"
#include "IOCommandLine.h"
#include "IOFile.h"
#include "IOStream.h"
#include "Sentence.h"
#include "ConfusionNet.h"
#include "TranslationAnalysis.h"
@ -61,10 +60,10 @@ POSSIBILITY OF SUCH DAMAGE.
using namespace std;
bool readInput(IODevice *inputOutput, int inputType, InputType*& source)
bool readInput(IOStream &ioStream, int inputType, InputType*& source)
{
delete source;
source=inputOutput->GetInput((inputType ?
source=ioStream.GetInput((inputType ?
static_cast<InputType*>(new ConfusionNet) :
static_cast<InputType*>(new Sentence(Input))));
return (source ? true : false);
@ -91,7 +90,7 @@ int main(int argc, char* argv[])
return EXIT_FAILURE;
// set up read/writing class
IODevice *ioDevice = GetIODevice(staticData);
IOStream *ioStream = GetIODevice(staticData);
// check on weights
vector<float> weights = staticData.GetAllWeights();
@ -107,13 +106,13 @@ int main(int argc, char* argv[])
return EXIT_FAILURE;
}
if (ioDevice == NULL)
if (ioStream == NULL)
return EXIT_FAILURE;
// read each sentence & decode
InputType *source=0;
size_t lineCount = 0;
while(readInput(ioDevice,staticData.GetInputType(),source))
while(readInput(*ioStream,staticData.GetInputType(),source))
{
// note: source is only valid within this while loop!
ResetUserTime();
@ -123,7 +122,7 @@ int main(int argc, char* argv[])
staticData.InitializeBeforeSentenceProcessing(*source);
Manager manager(*source, staticData);
manager.ProcessSentence();
ioDevice->SetOutput(manager.GetBestHypothesis(), source->GetTranslationId(),
ioStream->OutputBestHypo(manager.GetBestHypothesis(), source->GetTranslationId(),
staticData.GetReportSegmentation(),
staticData.GetReportAllFactors()
);
@ -135,7 +134,7 @@ int main(int argc, char* argv[])
VERBOSE(2,"WRITING " << nBestSize << " TRANSLATION ALTERNATIVES TO " << staticData.GetNBestFilePath() << endl);
LatticePathList nBestList;
manager.CalcNBest(nBestSize, nBestList,staticData.OnlyDistinctNBest());
ioDevice->SetNBest(nBestList, source->GetTranslationId());
ioStream->OutputNBestList(nBestList, source->GetTranslationId());
//RemoveAllInColl(nBestList);
}
@ -150,26 +149,26 @@ int main(int argc, char* argv[])
}
delete ioDevice;
delete ioStream;
PrintUserTime("End.");
return EXIT_SUCCESS;
}
IODevice *GetIODevice(StaticData &staticData)
IOStream *GetIODevice(StaticData &staticData)
{
IODevice *ioDevice;
IOStream *ioStream;
const std::vector<FactorType> &inputFactorOrder = staticData.GetInputFactorOrder()
,&outputFactorOrder = staticData.GetOutputFactorOrder();
FactorMask inputFactorUsed(inputFactorOrder);
// io
if (staticData.GetIOMethod() == IOMethodFile)
if (staticData.GetParam("input-file").size() == 1)
{
VERBOSE(2,"IO from File" << endl);
string filePath = staticData.GetParam("input-file")[0];
ioDevice = new IOFile(inputFactorOrder, outputFactorOrder, inputFactorUsed
ioStream = new IOStream(inputFactorOrder, outputFactorOrder, inputFactorUsed
, staticData.GetFactorCollection()
, staticData.GetNBestSize()
, staticData.GetNBestFilePath()
@ -178,14 +177,14 @@ IODevice *GetIODevice(StaticData &staticData)
else
{
VERBOSE(1,"IO from STDOUT/STDIN" << endl);
ioDevice = new IOCommandLine(inputFactorOrder, outputFactorOrder, inputFactorUsed
ioStream = new IOStream(inputFactorOrder, outputFactorOrder, inputFactorUsed
, staticData.GetFactorCollection()
, staticData.GetNBestSize()
, staticData.GetNBestFilePath());
}
ioDevice->ResetTranslationId();
ioStream->ResetTranslationId();
PrintUserTime("Created input-output object");
return ioDevice;
return ioStream;
}

4
moses-cmd/src/Main.h
View File

@ -36,7 +36,7 @@ POSSIBILITY OF SUCH DAMAGE.
#include "StaticData.h"
class IODevice;
class IOStream;
int main(int argc, char* argv[]);
IODevice *GetIODevice(StaticData &staticData);
IOStream *GetIODevice(StaticData &staticData);

2
moses-cmd/src/Makefile.am
View File

@ -1,5 +1,5 @@
bin_PROGRAMS = moses
moses_SOURCES = Main.cpp IOCommandLine.cpp IOFile.cpp TranslationAnalysis.cpp
moses_SOURCES = Main.cpp IOStream.cpp TranslationAnalysis.cpp
AM_CPPFLAGS = -W -Wall -ffor-scope -D_FILE_OFFSET_BITS=64 -D_LARGE_FILES -I$(top_srcdir)/moses/src
moses_LDADD = -L$(top_srcdir)/moses/src -lmoses

8
moses/moses.vcproj
View File

@ -211,10 +211,6 @@
RelativePath=".\src\InputType.cpp"
>
</File>
<File
RelativePath=".\src\IODevice.cpp"
>
</File>
<File
RelativePath=".\src\LanguageModel.cpp"
>
@ -519,10 +515,6 @@
RelativePath=".\src\InputType.h"
>
</File>
<File
RelativePath=".\src\IODevice.h"
>
</File>
<File
RelativePath=".\src\LanguageModel.h"
>

48
moses/src/IODevice.cpp
View File

@ -1,48 +0,0 @@
// $Id: IODevice.cpp 905 2006-10-21 16:21:21Z hieuhoang1972 $
/***********************************************************************
Moses - factored phrase-based language decoder
Copyright (C) 2006 University of Edinburgh
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
***********************************************************************/
#include "IODevice.h"
#include "InputType.h"
IODevice::IODevice() : m_translationId(0) {}
IODevice::~IODevice() {}
void IODevice::Release(InputType *s) {delete s;}
InputType* IODevice::GetInput(InputType *inputType
, std::istream &inputStream
, const std::vector<FactorType> &factorOrder
, FactorCollection &factorCollection)
{
if(inputType->Read(inputStream,factorOrder,factorCollection))
{
inputType->SetTranslationId(m_translationId++);
return inputType;
}
else
{
delete inputType;
return NULL;
}
}

77
moses/src/IODevice.h
View File

@ -1,77 +0,0 @@
// $Id: IODevice.h 905 2006-10-21 16:21:21Z hieuhoang1972 $
/***********************************************************************
Moses - factored phrase-based language decoder
Copyright (C) 2006 University of Edinburgh
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
***********************************************************************/
#pragma once
#include <string>
#include <iostream>
#include <vector>
#include "TypeDef.h"
class Hypothesis;
class LatticePathList;
class FactorCollection;
class InputType;
/** Abstract class that represent a device which through the Moses library reads and writes data.
* The users of the library, eg. moses-cmd, should create a class which is inherited from this
* class
*/
class IODevice
{
protected:
long m_translationId;
// constructor
IODevice();
/** fill inputType (currently either a Sentence or ConfusionNet) by calling its Read() function.
* Return the same inputType, or delete and return NULL if unsuccessful
*/
InputType* GetInput(InputType * inputType
, std::istream &inputStream
, const std::vector<FactorType> &factorOrder
, FactorCollection &factorCollection);
public:
virtual ~IODevice();
/** return a sentence or confusion network with data read from file or stdin
\param in empty InputType to be filled with data
*/
virtual InputType* GetInput(InputType *in) = 0;
/** return the best translation in hypo, or NULL if no translation was possible
\param hypo return arg of best translation found by decoder
\param translationId id of the input
\param reportSegmentation set to true if segmentation info required. Outputs to stdout
\reportAllFactors output all factors, rather than just output factors. Not sure if needed now we know which output factors we want
*/
virtual void SetOutput(const Hypothesis *hypo, long translationId, bool reportSegmentation, bool reportAllFactors) = 0;
/** return n-best list via the arg nBestList */
virtual void SetNBest(const LatticePathList &nBestList, long translationId) = 0;
//! delete InputType
virtual void Release(InputType *inputType);
void ResetTranslationId() { m_translationId = 0; }
};

1
moses/src/Makefile.am
View File

@ -16,7 +16,6 @@ libmoses_a_SOURCES_TMP = \
HypothesisCollection.cpp \
InputType.cpp \
InputFileStream.cpp \
IODevice.cpp \
LMList.cpp \
LanguageModel.cpp \
LanguageModelFactory.cpp \

10
moses/src/StaticData.cpp
View File

@ -83,8 +83,6 @@ StaticData::StaticData()
bool StaticData::LoadData(Parameter *parameter)
{
new Phrase(Input);
ResetUserTime();
m_parameter = parameter;
@ -248,14 +246,6 @@ StaticData::~StaticData()
}
IOMethod StaticData::GetIOMethod()
{
if (m_parameter->GetParam("input-file").size() == 1)
return IOMethodFile;
else
return IOMethodCommandLine;
}
bool StaticData::LoadLexicalReorderingModel()
{
// load Lexical Reordering model

2
moses/src/StaticData.h
View File

@ -29,7 +29,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#include "FactorCollection.h"
#include "Parameter.h"
#include "LanguageModel.h"
#include "IODevice.h"
#include "LMList.h"
#include "SentenceStats.h"
//#include "UnknownWordHandler.h"
@ -214,7 +213,6 @@ public:
{
return m_scoreIndexManager;
}
IOMethod GetIOMethod();
size_t GetLMSize() const
{

7
moses/src/TypeDef.h
View File

@ -110,13 +110,6 @@ namespace DistortionOrientationType
};
}
enum IOMethod
{
IOMethodCommandLine
,IOMethodFile
,IOMethodMySQL
};
enum LMType
{
SingleFactor