mirror of
https://github.com/moses-smt/mosesdecoder.git
synced 2024-10-27 03:49:57 +03:00
Merge branch 'master' of git://github.com/moses-smt/mosesdecoder
This commit is contained in:
commit
046e031723
66
contrib/promix/README.md
Normal file
66
contrib/promix/README.md
Normal file
@ -0,0 +1,66 @@
|
||||
promix - for training translation model interpolation weights using PRO
|
||||
|
||||
Author: Barry Haddow <bhaddow [AT] inf.ed.ac.uk>
|
||||
|
||||
ABOUT
|
||||
-----
|
||||
|
||||
The code here provides the "inner loop" for a batch tuning algorithm (like MERT) which
|
||||
optimises phrase table interpolation weights at the same time as the standard linear
|
||||
model weights. Interpolation of the phrase tables uses the "naive" method of tmcombine.
|
||||
|
||||
Currently the interpolation only works for two phrase tables, but will soon
|
||||
be extended to work for more than two.
|
||||
|
||||
|
||||
REQUIREMENTS
|
||||
------------
|
||||
The scripts require the Moses Python interface (in contrib/python). It should be built
|
||||
first, following the instructions in that directory.
|
||||
|
||||
The scripts also require scipy and numpy. They have been tested with the following versions:
|
||||
|
||||
* Python 2.7
|
||||
* Scipy 0.11.0
|
||||
* Numpy 1.6.2
|
||||
|
||||
Run the test.py script to check that everything is functioning correctly.
|
||||
|
||||
|
||||
USAGE
|
||||
-----
|
||||
Since the code in this directory provides the inner loop for a batch tuning algorithm,
|
||||
it is run from the increasingly inaccurately named mert-moses.pl. If you want to run
|
||||
the optimiser directly, run `main.py -h` for usage.
|
||||
|
||||
A sample command for mert-moses.pl is as follows:
|
||||
|
||||
MOSES/scripts/training/mert-moses.pl \
|
||||
input-file ref-file \
|
||||
decoder \
|
||||
ini-file \
|
||||
--promix-training MOSES/contrib/promix/main.py \
|
||||
--maximum-iterations 15 \
|
||||
--promix-table phrase-table-1 \
|
||||
--promix-table phrase-table-2 \
|
||||
--filtercmd "MOSES/scripts/training/filter-model-given-input.pl --Binarizer MOSES/bin/processPhraseTable" \
|
||||
--nbest 100 --working-dir ./tmp --decoder-flags "-threads 4 -v 0 " \
|
||||
--rootdir MOSES/scripts -mertdir MOSES/bin \
|
||||
--return-best-dev
|
||||
|
||||
Note that promix training requires a filter and binarise script, and that the phrase table
|
||||
referenced in the ini file is not used. The argument `--return-best-dev` is not essential,
|
||||
but recommended.
|
||||
|
||||
|
||||
REFERENCES
|
||||
----------
|
||||
|
||||
The code here was created for:
|
||||
|
||||
Haddow, Barry (2013) Applying Pairwise Ranked Optimisation to
|
||||
Improve the Interpolation of Translation Models. In: Proceedings of NAACL 2013
|
||||
|
||||
See also:
|
||||
|
||||
Sennrich, Rico (2012). Perplexity Minimization for Translation Model Domain Adaptation in Statistical Machine Translation. In: Proceedings of EACL 2012.
|
27
contrib/promix/bleu.py
Normal file
27
contrib/promix/bleu.py
Normal file
@ -0,0 +1,27 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
from math import exp,log
|
||||
|
||||
class BleuScoreException(Exception):
|
||||
pass
|
||||
|
||||
class BleuScorer:
|
||||
def __init__(self):
|
||||
"""References should be a list. If each element is a string, assume
|
||||
they are filenames, if a list, assume tokenised strings"""
|
||||
self.smooth = 1.0
|
||||
self.order = 4
|
||||
|
||||
def score(self,scores):
|
||||
if len(scores) != self.order*2+1:
|
||||
raise BleuScoreException("Wrong number of scores. Expected %d, but found %d" %
|
||||
(self.order*2+1, len(scores)))
|
||||
logbleu = 0.0
|
||||
for j in range(self.order):
|
||||
logbleu += log(scores[2*j] + self.smooth) - log(scores[2*j+1] + self.smooth)
|
||||
logbleu /= self.order
|
||||
brevity = 1.0 - float(scores[-1]) / scores[1]
|
||||
if brevity < 0:
|
||||
logbleu += brevity
|
||||
return exp(logbleu)
|
||||
|
260
contrib/promix/coll.py
Normal file
260
contrib/promix/coll.py
Normal file
@ -0,0 +1,260 @@
|
||||
# Backport of OrderedDict() class that runs on Python 2.4, 2.5, 2.6, 2.7 and pypy.
|
||||
# Passes Python2.7's test suite and incorporates all the latest updates.
|
||||
|
||||
# From http://code.activestate.com/recipes/576693/
|
||||
|
||||
try:
|
||||
from thread import get_ident as _get_ident
|
||||
except ImportError:
|
||||
from dummy_thread import get_ident as _get_ident
|
||||
|
||||
try:
|
||||
from _abcoll import KeysView, ValuesView, ItemsView
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
|
||||
class OrderedDict(dict):
|
||||
'Dictionary that remembers insertion order'
|
||||
# An inherited dict maps keys to values.
|
||||
# The inherited dict provides __getitem__, __len__, __contains__, and get.
|
||||
# The remaining methods are order-aware.
|
||||
# Big-O running times for all methods are the same as for regular dictionaries.
|
||||
|
||||
# The internal self.__map dictionary maps keys to links in a doubly linked list.
|
||||
# The circular doubly linked list starts and ends with a sentinel element.
|
||||
# The sentinel element never gets deleted (this simplifies the algorithm).
|
||||
# Each link is stored as a list of length three: [PREV, NEXT, KEY].
|
||||
|
||||
def __init__(self, *args, **kwds):
|
||||
'''Initialize an ordered dictionary. Signature is the same as for
|
||||
regular dictionaries, but keyword arguments are not recommended
|
||||
because their insertion order is arbitrary.
|
||||
|
||||
'''
|
||||
if len(args) > 1:
|
||||
raise TypeError('expected at most 1 arguments, got %d' % len(args))
|
||||
try:
|
||||
self.__root
|
||||
except AttributeError:
|
||||
self.__root = root = [] # sentinel node
|
||||
root[:] = [root, root, None]
|
||||
self.__map = {}
|
||||
self.__update(*args, **kwds)
|
||||
|
||||
def __setitem__(self, key, value, dict_setitem=dict.__setitem__):
|
||||
'od.__setitem__(i, y) <==> od[i]=y'
|
||||
# Setting a new item creates a new link which goes at the end of the linked
|
||||
# list, and the inherited dictionary is updated with the new key/value pair.
|
||||
if key not in self:
|
||||
root = self.__root
|
||||
last = root[0]
|
||||
last[1] = root[0] = self.__map[key] = [last, root, key]
|
||||
dict_setitem(self, key, value)
|
||||
|
||||
def __delitem__(self, key, dict_delitem=dict.__delitem__):
|
||||
'od.__delitem__(y) <==> del od[y]'
|
||||
# Deleting an existing item uses self.__map to find the link which is
|
||||
# then removed by updating the links in the predecessor and successor nodes.
|
||||
dict_delitem(self, key)
|
||||
link_prev, link_next, key = self.__map.pop(key)
|
||||
link_prev[1] = link_next
|
||||
link_next[0] = link_prev
|
||||
|
||||
def __iter__(self):
|
||||
'od.__iter__() <==> iter(od)'
|
||||
root = self.__root
|
||||
curr = root[1]
|
||||
while curr is not root:
|
||||
yield curr[2]
|
||||
curr = curr[1]
|
||||
|
||||
def __reversed__(self):
|
||||
'od.__reversed__() <==> reversed(od)'
|
||||
root = self.__root
|
||||
curr = root[0]
|
||||
while curr is not root:
|
||||
yield curr[2]
|
||||
curr = curr[0]
|
||||
|
||||
def clear(self):
|
||||
'od.clear() -> None. Remove all items from od.'
|
||||
try:
|
||||
for node in self.__map.itervalues():
|
||||
del node[:]
|
||||
root = self.__root
|
||||
root[:] = [root, root, None]
|
||||
self.__map.clear()
|
||||
except AttributeError:
|
||||
pass
|
||||
dict.clear(self)
|
||||
|
||||
def popitem(self, last=True):
|
||||
'''od.popitem() -> (k, v), return and remove a (key, value) pair.
|
||||
Pairs are returned in LIFO order if last is true or FIFO order if false.
|
||||
|
||||
'''
|
||||
if not self:
|
||||
raise KeyError('dictionary is empty')
|
||||
root = self.__root
|
||||
if last:
|
||||
link = root[0]
|
||||
link_prev = link[0]
|
||||
link_prev[1] = root
|
||||
root[0] = link_prev
|
||||
else:
|
||||
link = root[1]
|
||||
link_next = link[1]
|
||||
root[1] = link_next
|
||||
link_next[0] = root
|
||||
key = link[2]
|
||||
del self.__map[key]
|
||||
value = dict.pop(self, key)
|
||||
return key, value
|
||||
|
||||
# -- the following methods do not depend on the internal structure --
|
||||
|
||||
def keys(self):
|
||||
'od.keys() -> list of keys in od'
|
||||
return list(self)
|
||||
|
||||
def values(self):
|
||||
'od.values() -> list of values in od'
|
||||
return [self[key] for key in self]
|
||||
|
||||
def items(self):
|
||||
'od.items() -> list of (key, value) pairs in od'
|
||||
return [(key, self[key]) for key in self]
|
||||
|
||||
def iterkeys(self):
|
||||
'od.iterkeys() -> an iterator over the keys in od'
|
||||
return iter(self)
|
||||
|
||||
def itervalues(self):
|
||||
'od.itervalues -> an iterator over the values in od'
|
||||
for k in self:
|
||||
yield self[k]
|
||||
|
||||
def iteritems(self):
|
||||
'od.iteritems -> an iterator over the (key, value) items in od'
|
||||
for k in self:
|
||||
yield (k, self[k])
|
||||
|
||||
def update(*args, **kwds):
|
||||
'''od.update(E, **F) -> None. Update od from dict/iterable E and F.
|
||||
|
||||
If E is a dict instance, does: for k in E: od[k] = E[k]
|
||||
If E has a .keys() method, does: for k in E.keys(): od[k] = E[k]
|
||||
Or if E is an iterable of items, does: for k, v in E: od[k] = v
|
||||
In either case, this is followed by: for k, v in F.items(): od[k] = v
|
||||
|
||||
'''
|
||||
if len(args) > 2:
|
||||
raise TypeError('update() takes at most 2 positional '
|
||||
'arguments (%d given)' % (len(args),))
|
||||
elif not args:
|
||||
raise TypeError('update() takes at least 1 argument (0 given)')
|
||||
self = args[0]
|
||||
# Make progressively weaker assumptions about "other"
|
||||
other = ()
|
||||
if len(args) == 2:
|
||||
other = args[1]
|
||||
if isinstance(other, dict):
|
||||
for key in other:
|
||||
self[key] = other[key]
|
||||
elif hasattr(other, 'keys'):
|
||||
for key in other.keys():
|
||||
self[key] = other[key]
|
||||
else:
|
||||
for key, value in other:
|
||||
self[key] = value
|
||||
for key, value in kwds.items():
|
||||
self[key] = value
|
||||
|
||||
__update = update # let subclasses override update without breaking __init__
|
||||
|
||||
__marker = object()
|
||||
|
||||
def pop(self, key, default=__marker):
|
||||
'''od.pop(k[,d]) -> v, remove specified key and return the corresponding value.
|
||||
If key is not found, d is returned if given, otherwise KeyError is raised.
|
||||
|
||||
'''
|
||||
if key in self:
|
||||
result = self[key]
|
||||
del self[key]
|
||||
return result
|
||||
if default is self.__marker:
|
||||
raise KeyError(key)
|
||||
return default
|
||||
|
||||
def setdefault(self, key, default=None):
|
||||
'od.setdefault(k[,d]) -> od.get(k,d), also set od[k]=d if k not in od'
|
||||
if key in self:
|
||||
return self[key]
|
||||
self[key] = default
|
||||
return default
|
||||
|
||||
def __repr__(self, _repr_running={}):
|
||||
'od.__repr__() <==> repr(od)'
|
||||
call_key = id(self), _get_ident()
|
||||
if call_key in _repr_running:
|
||||
return '...'
|
||||
_repr_running[call_key] = 1
|
||||
try:
|
||||
if not self:
|
||||
return '%s()' % (self.__class__.__name__,)
|
||||
return '%s(%r)' % (self.__class__.__name__, self.items())
|
||||
finally:
|
||||
del _repr_running[call_key]
|
||||
|
||||
def __reduce__(self):
|
||||
'Return state information for pickling'
|
||||
items = [[k, self[k]] for k in self]
|
||||
inst_dict = vars(self).copy()
|
||||
for k in vars(OrderedDict()):
|
||||
inst_dict.pop(k, None)
|
||||
if inst_dict:
|
||||
return (self.__class__, (items,), inst_dict)
|
||||
return self.__class__, (items,)
|
||||
|
||||
def copy(self):
|
||||
'od.copy() -> a shallow copy of od'
|
||||
return self.__class__(self)
|
||||
|
||||
@classmethod
|
||||
def fromkeys(cls, iterable, value=None):
|
||||
'''OD.fromkeys(S[, v]) -> New ordered dictionary with keys from S
|
||||
and values equal to v (which defaults to None).
|
||||
|
||||
'''
|
||||
d = cls()
|
||||
for key in iterable:
|
||||
d[key] = value
|
||||
return d
|
||||
|
||||
def __eq__(self, other):
|
||||
'''od.__eq__(y) <==> od==y. Comparison to another OD is order-sensitive
|
||||
while comparison to a regular mapping is order-insensitive.
|
||||
|
||||
'''
|
||||
if isinstance(other, OrderedDict):
|
||||
return len(self)==len(other) and self.items() == other.items()
|
||||
return dict.__eq__(self, other)
|
||||
|
||||
def __ne__(self, other):
|
||||
return not self == other
|
||||
|
||||
# -- the following methods are only used in Python 2.7 --
|
||||
|
||||
def viewkeys(self):
|
||||
"od.viewkeys() -> a set-like object providing a view on od's keys"
|
||||
return KeysView(self)
|
||||
|
||||
def viewvalues(self):
|
||||
"od.viewvalues() -> an object providing a view on od's values"
|
||||
return ValuesView(self)
|
||||
|
||||
def viewitems(self):
|
||||
"od.viewitems() -> a set-like object providing a view on od's items"
|
||||
return ItemsView(self)
|
112
contrib/promix/main.py
Executable file
112
contrib/promix/main.py
Executable file
@ -0,0 +1,112 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
#
|
||||
# Implementation of PRO training and extensions to train phrase weights
|
||||
#
|
||||
|
||||
import gzip
|
||||
import logging
|
||||
from numpy import array
|
||||
import optparse
|
||||
import os.path
|
||||
import sys
|
||||
|
||||
from nbest import *
|
||||
from sampler import *
|
||||
from train import *
|
||||
|
||||
|
||||
logging.basicConfig(format = "%(asctime)-15s %(message)s")
|
||||
log = logging.getLogger('main')
|
||||
log.setLevel(logging.DEBUG)
|
||||
|
||||
class Config:
|
||||
def __init__(self):
|
||||
self.parser = optparse.OptionParser(usage="%prog [options] ")
|
||||
self.parser.add_option("-t", "--trainer", action="store",\
|
||||
dest="trainer", metavar="TYPE", type="choice", choices=("pro","mix"),\
|
||||
default="pro",\
|
||||
help="type of trainer to run (pro,mix)")
|
||||
self.parser.add_option("-n", "--nbest", action="append", \
|
||||
dest="nbest", metavar="NBEST-FILE",\
|
||||
help="nbest output file(s) from decoder")
|
||||
self.parser.add_option("-S", "--scfile", action="append",\
|
||||
dest="score", metavar="SCORE-FILE",\
|
||||
help="score file(s) from extractor (in same order as nbests)")
|
||||
self.parser.add_option("-p", "--phrase-table" , action="append",\
|
||||
dest="ttable", metavar="TTABLE",\
|
||||
help="ttable to be used in mixture model training")
|
||||
self.parser.add_option("-i", "--input-file", action="store",\
|
||||
dest="input_file", metavar="INPUT-FILE",
|
||||
help="source text file")
|
||||
self.parser.add_option("-m", "--moses-bin-dir", action="store",\
|
||||
dest="moses_bin_dir", metavar="DIR",
|
||||
help="directory containing Moses binaries",
|
||||
default=os.path.expanduser("~/moses/bin"))
|
||||
self.nbest_files = []
|
||||
self.score_files = []
|
||||
self.ttables = []
|
||||
|
||||
def parse(self,args=sys.argv[1:]):
|
||||
(options,args) = self.parser.parse_args(args)
|
||||
self.nbest_files = options.nbest
|
||||
self.score_files = options.score
|
||||
self.ttables = options.ttable
|
||||
self.input_file = options.input_file
|
||||
self.trainer = options.trainer
|
||||
self.moses_bin_dir = options.moses_bin_dir
|
||||
if not self.nbest_files:
|
||||
self.nbest_files = ["data/esen.nc.nbest.segment"]
|
||||
if not self.score_files:
|
||||
self.score_files = ["data/esen.nc.scores"]
|
||||
if len(self.nbest_files) != len(self.score_files):
|
||||
self.parser.error("Must have equal numbers of score files and nbest files")
|
||||
if self.trainer == "mix":
|
||||
if not self.input_file or not self.ttables:
|
||||
self.parser.error("Need to specify input file and ttables for mix training")
|
||||
#if len(self.ttables) != 2:
|
||||
# self.parser.error("Can only train mix model with 2 ttables at the moment")
|
||||
|
||||
def main():
|
||||
config = Config()
|
||||
config.parse()
|
||||
|
||||
samples = []
|
||||
sampler = HopkinsMaySampler()
|
||||
nbests = 0
|
||||
for nbest_file,score_data_file in zip(config.nbest_files,config.score_files):
|
||||
log.debug("nbest: " + nbest_file + "; score:" + score_data_file)
|
||||
segments = False
|
||||
if config.trainer == "mix": segments = True
|
||||
for nbest in get_scored_nbests(nbest_file, score_data_file, config.input_file, segments=segments):
|
||||
samples += sampler.sample(nbest)
|
||||
nbests += 1
|
||||
log.debug("Samples loaded")
|
||||
trainer = None
|
||||
if config.trainer == "mix":
|
||||
# Add the phrase table scores
|
||||
scorer = MosesPhraseScorer(config.ttables)
|
||||
log.debug("Scoring samples...")
|
||||
for sample in samples:
|
||||
scorer.add_scores(sample.hyp1)
|
||||
scorer.add_scores(sample.hyp2)
|
||||
log.debug("...samples scored")
|
||||
trainer = MixtureModelTrainer(samples)
|
||||
elif config.trainer == "pro":
|
||||
trainer = ProTrainer(samples)
|
||||
else: assert(0)
|
||||
log.debug("Starting training...")
|
||||
weights,mix_weights = trainer.train(debug=False)
|
||||
log.debug("...training complete")
|
||||
for i,w in enumerate(weights):
|
||||
print "F%d %10.8f" % (i,w)
|
||||
for i,f in enumerate(mix_weights):
|
||||
for j,w in enumerate(f):
|
||||
print "M%d_%d %10.8f" % (i,j,w)
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
230
contrib/promix/nbest.py
Normal file
230
contrib/promix/nbest.py
Normal file
@ -0,0 +1,230 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
import gzip
|
||||
import os
|
||||
import re
|
||||
import numpy as np
|
||||
import sys
|
||||
|
||||
from bleu import BleuScorer
|
||||
from coll import OrderedDict
|
||||
# Edit to set moses python path
|
||||
sys.path.append(os.path.dirname(__file__) + "/../python")
|
||||
import moses.dictree as binpt
|
||||
|
||||
class DataFormatException(Exception):
|
||||
pass
|
||||
|
||||
class Hypothesis:
|
||||
def __init__(self,text,fv,segments=False):
|
||||
self.alignment = [] #only stored for segmented hypos
|
||||
self.tokens = [] #only stored for segmented hypos
|
||||
if not segments:
|
||||
self.text = text
|
||||
# Triples of (source-start, source-end, target-end) where segments end positions
|
||||
# are 1 beyond the last token
|
||||
else:
|
||||
# recover segmentation
|
||||
self.tokens = []
|
||||
align_re = re.compile("\|(\d+)-(\d+)\|")
|
||||
for token in text.split():
|
||||
match = align_re.match(token)
|
||||
if match:
|
||||
self.alignment.append\
|
||||
((int(match.group(1)), 1+int(match.group(2)), len(self.tokens)))
|
||||
else:
|
||||
self.tokens.append(token)
|
||||
self.text = " ".join(self.tokens)
|
||||
if not self.alignment:
|
||||
raise DataFormatException("Expected segmentation information not found in nbest")
|
||||
|
||||
|
||||
self.fv = np.array(fv)
|
||||
self.score = 0
|
||||
|
||||
def __str__(self):
|
||||
return "{text=%s fv=%s score=%5.4f}" % (self.text, str(self.fv), self.score)
|
||||
|
||||
class NBestList:
|
||||
def __init__(self,id):
|
||||
self.id = id
|
||||
self.hyps = []
|
||||
|
||||
# Maps feature ids (short feature names) to their values
|
||||
_feature_index = {}
|
||||
def set_feature_start(name,index):
|
||||
indexes = _feature_index.get(name, [index,0])
|
||||
indexes[0] = index
|
||||
_feature_index[name] = indexes
|
||||
|
||||
def set_feature_end(name,index):
|
||||
indexes = _feature_index.get(name, [0,index])
|
||||
indexes[1] = index
|
||||
_feature_index[name] = indexes
|
||||
|
||||
def get_feature_index(name):
|
||||
return _feature_index.get(name, [0,0])
|
||||
|
||||
def get_nbests(nbest_file, segments=False):
|
||||
"""Iterate through nbest lists"""
|
||||
if nbest_file.endswith("gz"):
|
||||
fh = gzip.GzipFile(nbest_file)
|
||||
else:
|
||||
fh = open(nbest_file)
|
||||
lineno = 0
|
||||
nbest = None
|
||||
for line in fh:
|
||||
fields = line.split(" ||| ")
|
||||
if len(fields) != 4:
|
||||
raise DataFormatException("nbest(%d): %s" % (lineno,line))
|
||||
(id, text, scores, total) = fields
|
||||
if nbest and nbest.id != id:
|
||||
yield nbest
|
||||
nbest = None
|
||||
if not nbest:
|
||||
nbest = NBestList(id)
|
||||
fv = []
|
||||
score_name = None
|
||||
for score in scores.split():
|
||||
if score.endswith(":"):
|
||||
score = score[:-1]
|
||||
if score_name:
|
||||
set_feature_end(score_name,len(fv))
|
||||
score_name = score
|
||||
set_feature_start(score_name,len(fv))
|
||||
else:
|
||||
fv.append(float(score))
|
||||
if score_name: set_feature_end(score_name,len(fv))
|
||||
hyp = Hypothesis(text[:-1],fv,segments)
|
||||
nbest.hyps.append(hyp)
|
||||
if nbest:
|
||||
yield nbest
|
||||
|
||||
def get_scores(score_data_file):
|
||||
"""Iterate through the score data, returning a set of scores for each sentence"""
|
||||
scorer = BleuScorer()
|
||||
fh = open(score_data_file)
|
||||
lineno = 0
|
||||
score_vectors = None
|
||||
for line in fh:
|
||||
if line.startswith("SCORES_TXT_BEGIN"):
|
||||
score_vectors = []
|
||||
elif line.startswith("SCORES_TXT_END"):
|
||||
scores = [scorer.score(score_vector) for score_vector in score_vectors]
|
||||
yield scores
|
||||
else:
|
||||
score_vectors.append([float(i) for i in line[:-1].split()])
|
||||
|
||||
|
||||
def get_scored_nbests(nbest_file, score_data_file, input_file, segments=False):
|
||||
score_gen = get_scores(score_data_file)
|
||||
input_gen = None
|
||||
if input_file: input_gen = open(input_file)
|
||||
try:
|
||||
for nbest in get_nbests(nbest_file, segments=segments):
|
||||
scores = score_gen.next()
|
||||
if len(scores) != len(nbest.hyps):
|
||||
raise DataFormatException("Length of nbest %s does not match score list (%d != %d)" %
|
||||
(nbest.id,len(nbest.hyps), len(scores)))
|
||||
input_line = None
|
||||
if input_gen:
|
||||
input_line = input_gen.next()[:-1]
|
||||
for hyp,score in zip(nbest.hyps, scores):
|
||||
hyp.score = score
|
||||
hyp.input_line = input_line
|
||||
yield nbest
|
||||
|
||||
except StopIteration:
|
||||
raise DataFormatException("Score file shorter than nbest list file")
|
||||
|
||||
class PhraseCache:
|
||||
"""An LRU cache for ttable lookups"""
|
||||
def __init__(self, max_size):
|
||||
self.max_size = max_size
|
||||
self.pairs_to_scores = OrderedDict()
|
||||
|
||||
def get(self, source, target):
|
||||
key = (source,target)
|
||||
scores = self.pairs_to_scores.get(key,None)
|
||||
if scores:
|
||||
# cache hit - update access time
|
||||
del self.pairs_to_scores[key]
|
||||
self.pairs_to_scores[key] = scores
|
||||
return scores
|
||||
|
||||
def add(self,source,target,scores):
|
||||
key = (source,target)
|
||||
self.pairs_to_scores[key] = scores
|
||||
while len(self.pairs_to_scores) > self.max_size:
|
||||
self.pairs_to_scores.popitem(last=False)
|
||||
|
||||
#
|
||||
# Should I store full lists of options, or just phrase pairs?
|
||||
# Should probably store phrase-pairs, but may want to add
|
||||
# high scoring pairs (say, 20?) when I load the translations
|
||||
# of a given phrase
|
||||
#
|
||||
|
||||
class CachedPhraseTable:
|
||||
def __init__(self,ttable_file,nscores=5,cache_size=20000):
|
||||
wa = False
|
||||
if binpt.PhraseDictionaryTree.canLoad(ttable_file,True):
|
||||
# assume word alignment is included
|
||||
wa = True
|
||||
self.ttable = binpt.PhraseDictionaryTree(ttable_file,nscores = nscores,wa = wa, tableLimit=0)
|
||||
self.cache = PhraseCache(cache_size)
|
||||
self.nscores = nscores
|
||||
|
||||
def get_scores(self,phrase):
|
||||
source = " ".join(phrase[0])
|
||||
target_tuple = tuple(phrase[1])
|
||||
target = " ".join(target_tuple)
|
||||
scores = self.cache.get(source,target)
|
||||
if not scores:
|
||||
# cache miss
|
||||
scores = [0] * (self.nscores-1) # ignore penalty
|
||||
entries = self.ttable.query(source, converter=None)
|
||||
# find correct target
|
||||
for entry in entries:
|
||||
if entry.rhs == target_tuple:
|
||||
scores = entry.scores[:-1]
|
||||
break
|
||||
#print "QUERY",source,"|||",target,"|||",scores
|
||||
self.cache.add(source,target,scores)
|
||||
#else:
|
||||
# print "CACHE",source,"|||",target,"|||",scores
|
||||
return scores
|
||||
|
||||
|
||||
class MosesPhraseScorer:
|
||||
def __init__(self,ttable_files, cache_size=20000):
|
||||
self.ttables = []
|
||||
for ttable_file in ttable_files:
|
||||
self.ttables.append(CachedPhraseTable(ttable_file, cache_size=cache_size))
|
||||
|
||||
def add_scores(self, hyp):
|
||||
"""Add the phrase scores to a hypothesis"""
|
||||
# Collect up the phrase pairs
|
||||
phrases = []
|
||||
source_tokens = hyp.input_line.split()
|
||||
tgt_st = 0
|
||||
if not hyp.alignment:
|
||||
raise DataFormatException("Alignments missing from: " + str(hyp))
|
||||
for src_st,src_end,tgt_end in hyp.alignment:
|
||||
phrases.append((source_tokens[src_st:src_end], hyp.tokens[tgt_st:tgt_end]))
|
||||
tgt_st = tgt_end
|
||||
# Look up the scores
|
||||
phrase_scores = []
|
||||
for ttable in self.ttables:
|
||||
phrase_scores.append([])
|
||||
for phrase in phrases:
|
||||
phrase_scores[-1].append(ttable.get_scores(phrase))
|
||||
# phrase_scores = np.array(phrase_scores)
|
||||
# eps = np.exp(-100)
|
||||
# phrase_scores[phrase_scores<eps]=eps
|
||||
floor = np.exp(-100)
|
||||
phrase_scores = np.clip(np.array(phrase_scores), floor, np.inf)
|
||||
hyp.phrase_scores = phrase_scores
|
||||
|
||||
|
||||
|
44
contrib/promix/sampler.py
Executable file
44
contrib/promix/sampler.py
Executable file
@ -0,0 +1,44 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
#
|
||||
#
|
||||
#
|
||||
|
||||
import heapq
|
||||
import math
|
||||
import random
|
||||
import sys
|
||||
|
||||
from bleu import BleuScorer
|
||||
|
||||
|
||||
class Sample:
|
||||
"""A pair of hypotheses, and their score difference"""
|
||||
def __init__(self,hyp1,hyp2):
|
||||
self.hyp1 = hyp1
|
||||
self.hyp2 = hyp2
|
||||
self.diff = abs(hyp1.score-hyp2.score)
|
||||
|
||||
def __cmp__(self,other):
|
||||
return cmp(self.diff,other.diff)
|
||||
|
||||
class HopkinsMaySampler:
|
||||
"""Implements Hopkins & May sampling"""
|
||||
def __init__(self):
|
||||
self.ncandidates = 5000 # Gamma in Hopkins and May
|
||||
self.nsamples = 50 # Xi in Hopkins and May
|
||||
self.min_diff = 0.05 # Minimum scoring difference
|
||||
|
||||
def sample(self,nbest):
|
||||
samples = []
|
||||
for i in xrange(self.ncandidates):
|
||||
hyp1 = random.choice(nbest.hyps)
|
||||
hyp2 = random.choice(nbest.hyps)
|
||||
sample = Sample(hyp1,hyp2)
|
||||
if sample.diff < self.min_diff: continue
|
||||
# maintain nsamples biggest samples
|
||||
heapq.heappush(samples,sample)
|
||||
while len(samples) > self.nsamples:
|
||||
heapq.heappop(samples)
|
||||
return samples
|
||||
|
25
contrib/promix/test.py
Executable file
25
contrib/promix/test.py
Executable file
@ -0,0 +1,25 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
import unittest
|
||||
|
||||
import test_bleu
|
||||
import test_main
|
||||
import test_moses
|
||||
import test_nbest
|
||||
import test_sampler
|
||||
import test_train
|
||||
|
||||
def main():
|
||||
test_list = []
|
||||
test_list.append(test_bleu.suite)
|
||||
test_list.append(test_main.suite)
|
||||
# test_list.append(test_moses.suite)
|
||||
test_list.append(test_nbest.suite)
|
||||
test_list.append(test_sampler.suite)
|
||||
test_list.append(test_train.suite)
|
||||
|
||||
suite = unittest.TestSuite(test_list)
|
||||
unittest.TextTestRunner().run(suite)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
28
contrib/promix/test_bleu.py
Executable file
28
contrib/promix/test_bleu.py
Executable file
@ -0,0 +1,28 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
import math
|
||||
import unittest
|
||||
|
||||
from bleu import *
|
||||
|
||||
class TestBleuScorer(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.scorer = BleuScorer()
|
||||
|
||||
def test_wrong_length(self):
|
||||
self.assertRaises(BleuScoreException, self.scorer.score, [1]*8)
|
||||
self.assertRaises(BleuScoreException, self.scorer.score, [1]*10)
|
||||
|
||||
def test_score(self):
|
||||
stats = [2,5,0,1,1,1,1,1,5]
|
||||
self.assertAlmostEqual(self.scorer.score(stats), 1/math.sqrt(2))
|
||||
|
||||
def test_brevity(self):
|
||||
stats = [2,2,2,2,2,2,2,2,3]
|
||||
self.assertAlmostEqual(self.scorer.score(stats), math.exp(1 - 3.0/2.0))
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
suite = unittest.TestLoader().loadTestsFromTestCase(TestBleuScorer)
|
||||
|
5
contrib/promix/test_data/README
Normal file
5
contrib/promix/test_data/README
Normal file
@ -0,0 +1,5 @@
|
||||
These nbest lists are created using the model in
|
||||
data/esen.epnc-lin.model
|
||||
|
||||
Note that the nc-ep weights for this are:
|
||||
[[0.54471993730312251, 0.45528006269687754], [0.56546688367708142, 0.43453311632291863], [0.55867730373453584, 0.44132269626546422], [0.46645964485220004, 0.53354035514779996]]
|
134446
contrib/promix/test_data/esen.ep.model.filtered/phrase-table.0-0.1.1
Normal file
134446
contrib/promix/test_data/esen.ep.model.filtered/phrase-table.0-0.1.1
Normal file
File diff suppressed because it is too large
Load Diff
Binary file not shown.
Binary file not shown.
@ -0,0 +1,8 @@
|
||||
7 del
|
||||
6 lados
|
||||
5 ambos
|
||||
4 en
|
||||
3 políticos
|
||||
2 los
|
||||
1 a
|
||||
0 Atlántico
|
Binary file not shown.
File diff suppressed because it is too large
Load Diff
15117
contrib/promix/test_data/esen.nc.model.filtered/phrase-table.0-0.1.1
Normal file
15117
contrib/promix/test_data/esen.nc.model.filtered/phrase-table.0-0.1.1
Normal file
File diff suppressed because it is too large
Load Diff
Binary file not shown.
Binary file not shown.
@ -0,0 +1,9 @@
|
||||
8 domando
|
||||
7 del
|
||||
6 lados
|
||||
5 ambos
|
||||
4 en
|
||||
3 políticos
|
||||
2 los
|
||||
1 a
|
||||
0 Atlántico
|
Binary file not shown.
File diff suppressed because it is too large
Load Diff
8
contrib/promix/test_data/phrases
Normal file
8
contrib/promix/test_data/phrases
Normal file
@ -0,0 +1,8 @@
|
||||
domando a
|
||||
los políticos
|
||||
en ambos
|
||||
lados del Atlántico
|
||||
taming
|
||||
politicians
|
||||
on both
|
||||
sides of the Atlantic
|
5
contrib/promix/test_data/phrases.ep
Normal file
5
contrib/promix/test_data/phrases.ep
Normal file
@ -0,0 +1,5 @@
|
||||
|
||||
los políticos ||| politicians ||| 0-0 1-0 ||| 0.33497 0.180441 0.638586 0.0962213 2.718
|
||||
en ambos ||| on both ||| 0-0 1-1 ||| 0.0908379 0.0213197 0.187399 0.0498198 2.718
|
||||
lados del Atlántico ||| sides of the Atlantic ||| 0-0 1-1 1-2 2-3 ||| 0.62585 0.00702384 0.836364 0.0687874 2.718
|
||||
|
4
contrib/promix/test_data/phrases.epnc-lin
Normal file
4
contrib/promix/test_data/phrases.epnc-lin
Normal file
@ -0,0 +1,4 @@
|
||||
domando a ||| taming ||| 0.0403496 0.00427574 0.558677 0.233252 2.718
|
||||
los políticos ||| politicians ||| 0.449849 0.210571 0.739008 0.138315 2.718
|
||||
en ambos ||| on both ||| 0.198423 0.0402601 0.270259 0.0519369 2.718
|
||||
lados del Atlántico ||| sides of the Atlantic ||| 0.708608 0.00997589 0.803633 0.0530904 2.718
|
4
contrib/promix/test_data/phrases.nc
Normal file
4
contrib/promix/test_data/phrases.nc
Normal file
@ -0,0 +1,4 @@
|
||||
domando a ||| taming ||| 0-0 1-0 ||| 0.0740741 0.00756144 1 0.500047 2.718
|
||||
los políticos ||| politicians ||| 0-0 1-0 ||| 0.545866 0.233725 0.818336 0.186463 2.718
|
||||
en ambos ||| on both ||| 0-0 1-1 ||| 0.288344 0.0548148 0.335714 0.0543585 2.718
|
||||
dos del Atlántico ||| sides of the Atlantic ||| 0-0 1-1 1-2 2-3 ||| 0.777778 0.0122444 0.777778 0.0351361 2.718
|
1
contrib/promix/test_data/test.nbest.input
Normal file
1
contrib/promix/test_data/test.nbest.input
Normal file
@ -0,0 +1 @@
|
||||
domando a los políticos en ambos lados del Atlántico
|
100
contrib/promix/test_data/test.nbest.nbest
Normal file
100
contrib/promix/test_data/test.nbest.nbest
Normal file
@ -0,0 +1,100 @@
|
||||
0 ||| taming politicians on both sides of the Atlantic ||| d: 0 -1.51037 0 0 -2.60639 0 0 lm: -36.0562 w: -8 tm: -5.97082 -14.8327 -2.41162 -9.32734 3.99959 ||| -16.9717
|
||||
0 ||| taming the politicians on both sides of the Atlantic ||| d: 0 -1.07904 0 0 -2.22203 0 0 lm: -37.2846 w: -9 tm: -5.43337 -15.7907 -4.86623 -9.68878 3.99959 ||| -16.9885
|
||||
0 ||| taming politicians on both sides of the Atlantic ||| d: 0 -2.24178 0 0 -2.59408 0 0 lm: -36.0562 w: -8 tm: -5.55036 -14.7311 -1.14332 -9.34122 2.99969 ||| -17.0321
|
||||
0 ||| taming the politicians on both sides of the Atlantic ||| d: 0 -1.81045 0 0 -2.20973 0 0 lm: -37.2846 w: -9 tm: -5.01291 -15.689 -3.59793 -9.70266 2.99969 ||| -17.0489
|
||||
0 ||| taming politicians on both sides of the Atlantic ||| d: 0 -1.8566 0 0 -2.76972 0 0 lm: -36.0562 w: -8 tm: -6.33782 -14.6804 -1.8329 -9.34609 3.99959 ||| -17.0555
|
||||
0 ||| taming the politicians on both sides of the Atlantic ||| d: 0 -1.42527 0 0 -2.38536 0 0 lm: -37.2846 w: -9 tm: -5.80037 -15.6384 -4.28752 -9.70753 3.99959 ||| -17.0723
|
||||
0 ||| taming politicians on both sides of the Atlantic ||| d: 0 -1.947 0 0 -2.66963 0 0 lm: -36.0562 w: -8 tm: -7.12207 -14.8138 -2.49723 -9.33158 4.99948 ||| -17.1661
|
||||
0 ||| taming politicians on both sides of the Atlantic ||| d: 0 -1.36892 0 0 -2.85256 0 0 lm: -36.0562 w: -8 tm: -5.62394 -14.8136 -3.58667 -9.35962 3.99959 ||| -17.1714
|
||||
0 ||| taming the politicians on both sides of the Atlantic ||| d: 0 -1.51567 0 0 -2.28527 0 0 lm: -37.2846 w: -9 tm: -6.58462 -15.7718 -4.95184 -9.69302 4.99948 ||| -17.183
|
||||
0 ||| taming the politicians on both sides of the Atlantic ||| d: 0 -0.93759 0 0 -2.4682 0 0 lm: -37.2846 w: -9 tm: -5.08649 -15.7715 -6.04129 -9.72106 3.99959 ||| -17.1882
|
||||
0 ||| taming politicians on both sides of the Atlantic ||| d: 0 -2.71301 0 0 -2.68303 0 0 lm: -36.0562 w: -8 tm: -6.41885 -14.6935 -1.25905 -9.34492 3.99959 ||| -17.1903
|
||||
0 ||| taming the politicians on both sides of the Atlantic ||| d: 0 -2.28167 0 0 -2.29867 0 0 lm: -37.2846 w: -9 tm: -5.8814 -15.6515 -3.71367 -9.70637 3.99959 ||| -17.2071
|
||||
0 ||| taming politicians on both sides of the Atlantic ||| d: 0 -1.56496 0 0 -2.75335 0 0 lm: -36.0562 w: -8 tm: -7.49271 -14.8064 -2.97497 -9.34339 4.99948 ||| -17.2472
|
||||
0 ||| taming the politicians on both sides of the Atlantic ||| d: 0 -1.13363 0 0 -2.36899 0 0 lm: -37.2846 w: -9 tm: -6.95526 -15.7643 -5.42959 -9.70483 4.99948 ||| -17.264
|
||||
0 ||| taming politicians on both sides of the Atlantic ||| d: 0 -1.78819 0 0 -2.89955 0 0 lm: -36.0562 w: -8 tm: -6.50046 -14.7823 -3.66165 -9.36203 4.99948 ||| -17.2958
|
||||
0 ||| taming the politicians on both sides of the Atlantic ||| d: 0 -1.35686 0 0 -2.51519 0 0 lm: -37.2846 w: -9 tm: -5.96301 -15.7403 -6.11627 -9.72347 4.99948 ||| -17.3126
|
||||
0 ||| taming politicians on both sides of the Atlantic ||| d: 0 -1.4211 0 0 -2.96742 0 0 lm: -36.0562 w: -8 tm: -6.94636 -14.7711 -4.14828 -9.33786 4.99948 ||| -17.3855
|
||||
0 ||| taming the politicians on both sides of the Atlantic ||| d: 0 -0.989766 0 0 -2.58306 0 0 lm: -37.2846 w: -9 tm: -6.40891 -15.7291 -6.6029 -9.69931 4.99948 ||| -17.4023
|
||||
0 ||| taming politicians on both sides of the Atlantic ||| d: 0 -1.5859 0 0 -3.00234 0 0 lm: -36.0562 w: -8 tm: -7.53826 -14.8794 -3.91516 -9.31284 4.99948 ||| -17.5338
|
||||
0 ||| taming the politicians on both sides of the Atlantic ||| d: 0 -1.15457 0 0 -2.61798 0 0 lm: -37.2846 w: -9 tm: -7.00081 -15.8374 -6.36978 -9.67428 4.99948 ||| -17.5506
|
||||
0 ||| taming the politicians on both sides of the Atlantic ||| d: 0 -1.52679 0 0 -2.89648 0 0 lm: -37.2846 w: -9 tm: -8.24923 -15.7898 -4.29661 -9.63452 4.99948 ||| -17.5634
|
||||
0 ||| taming the politicians on both sides of the Atlantic ||| d: 0 -2.2582 0 0 -2.88417 0 0 lm: -37.2846 w: -9 tm: -7.82876 -15.6881 -3.02831 -9.6484 3.99959 ||| -17.6238
|
||||
0 ||| taming the politicians on both sides of the Atlantic ||| d: 0 -1.87302 0 0 -3.05981 0 0 lm: -37.2846 w: -9 tm: -8.61622 -15.6375 -3.7179 -9.65327 4.99948 ||| -17.6472
|
||||
0 ||| taming politicians on both sides of the Atlantic ||| d: 0 -2.40826 0 0 -3.29781 0 0 lm: -36.0562 w: -8 tm: -7.95188 -14.6816 -2.5975 -9.34956 4.99948 ||| -17.6561
|
||||
0 ||| taming the politicians on both sides of the Atlantic ||| d: 0 -1.97693 0 0 -2.91345 0 0 lm: -37.2846 w: -9 tm: -7.41443 -15.6395 -5.05211 -9.711 4.99948 ||| -17.6729
|
||||
0 ||| taming politicians on both sides of the Atlantic ||| d: 0 -2.02253 0 0 -3.06558 0 0 lm: -36.0562 w: -8 tm: -8.6895 -14.8606 -4.00077 -9.31707 5.99938 ||| -17.7283
|
||||
0 ||| taming the politicians on both sides of the Atlantic ||| d: 0 -1.5912 0 0 -2.68122 0 0 lm: -37.2846 w: -9 tm: -8.15205 -15.8185 -6.45539 -9.67851 5.99938 ||| -17.7451
|
||||
0 ||| taming the politicians on both sides of the Atlantic ||| d: 0 -1.96342 0 0 -2.95972 0 0 lm: -37.2846 w: -9 tm: -9.40047 -15.7709 -4.38222 -9.63875 5.99938 ||| -17.7579
|
||||
0 ||| taming the politicians on both sides of the Atlantic ||| d: 0 -1.38534 0 0 -3.14265 0 0 lm: -37.2846 w: -9 tm: -7.90234 -15.7706 -5.47167 -9.66679 4.99948 ||| -17.7631
|
||||
0 ||| taming the politicians on both sides of the Atlantic ||| d: 0 -2.72942 0 0 -2.97312 0 0 lm: -37.2846 w: -9 tm: -8.69725 -15.6506 -3.14405 -9.6521 4.99948 ||| -17.782
|
||||
0 ||| taming politicians on both sides of the Atlantic ||| d: 0 -1.64049 0 0 -3.1493 0 0 lm: -36.0562 w: -8 tm: -9.06014 -14.8531 -4.47852 -9.32889 5.99938 ||| -17.8093
|
||||
0 ||| taming politicians on both sides of the Atlantic ||| d: 0 -1.95601 0 0 -3.55535 0 0 lm: -36.0562 w: -8 tm: -4.91151 -15.1311 -3.72129 -9.93616 2.99969 ||| -17.8216
|
||||
0 ||| taming the politicians on both sides of the Atlantic ||| d: 0 -1.20916 0 0 -2.76494 0 0 lm: -37.2846 w: -9 tm: -8.52269 -15.8111 -6.93313 -9.69033 5.99938 ||| -17.8261
|
||||
0 ||| taming the politicians on both sides of the Atlantic ||| d: 0 -1.58137 0 0 -3.04344 0 0 lm: -37.2846 w: -9 tm: -9.77111 -15.7635 -4.85997 -9.65057 5.99938 ||| -17.8389
|
||||
0 ||| taming politicians on both sides of the Atlantic ||| d: 0 -2.11662 0 0 -3.28144 0 0 lm: -36.0562 w: -8 tm: -9.10677 -14.8075 -3.73957 -9.34686 5.99938 ||| -17.8478
|
||||
0 ||| taming the politicians on both sides of the Atlantic ||| d: 0 -1.68529 0 0 -2.89708 0 0 lm: -37.2846 w: -9 tm: -8.56932 -15.7655 -6.19418 -9.7083 5.99938 ||| -17.8646
|
||||
0 ||| taming the politicians on both sides of the Atlantic ||| d: 0 -1.80461 0 0 -3.18964 0 0 lm: -37.2846 w: -9 tm: -8.77886 -15.7394 -5.54665 -9.66921 5.99938 ||| -17.8875
|
||||
0 ||| taming politicians on both sides of the Atlantic ||| d: 0 -2.37528 0 0 -3.60234 0 0 lm: -36.0562 w: -8 tm: -5.78803 -15.0999 -3.79627 -9.93857 3.99958 ||| -17.946
|
||||
0 ||| taming the politicians on both sides of the Atlantic ||| d: 0 -1.43751 0 0 -3.25751 0 0 lm: -37.2846 w: -9 tm: -9.22476 -15.7282 -6.03327 -9.64504 5.99938 ||| -17.9772
|
||||
0 ||| taming politicians on both sides of the Atlantic ||| d: 0 -1.97276 0 0 -3.49551 0 0 lm: -36.0562 w: -8 tm: -8.56043 -14.7723 -4.91288 -9.34133 5.99938 ||| -17.9861
|
||||
0 ||| taming the politicians on both sides of the Atlantic ||| d: 0 -1.54143 0 0 -3.11115 0 0 lm: -37.2846 w: -9 tm: -8.02298 -15.7302 -7.36749 -9.70277 5.99938 ||| -18.0029
|
||||
0 ||| taming politicians on both sides of the Atlantic ||| d: 0 -2.00819 0 0 -3.67021 0 0 lm: -36.0562 w: -8 tm: -6.23393 -15.0887 -4.2829 -9.91441 3.99958 ||| -18.0357
|
||||
0 ||| taming politicians on both sides of the Atlantic , ||| d: 0 -2.61446 0 0 -2.59408 0 0 lm: -37.5177 w: -9 tm: -6.22594 -14.8893 -3.54625 -12.0454 2.99969 ||| -18.0629
|
||||
0 ||| taming the politicians on both sides of the Atlantic , ||| d: 0 -2.18313 0 0 -2.20973 0 0 lm: -38.7461 w: -10 tm: -5.68849 -15.8473 -6.00087 -12.4069 2.99969 ||| -18.0797
|
||||
0 ||| taming politicians on both sides of the Atlantic , ||| d: 0 -1.97768 0 0 -2.76972 0 0 lm: -37.5177 w: -9 tm: -7.93613 -14.6804 -4.69485 -11.2579 3.99959 ||| -18.097
|
||||
0 ||| taming the politicians on both sides of the Atlantic , ||| d: 0 -1.54635 0 0 -2.38536 0 0 lm: -38.7461 w: -10 tm: -7.39868 -15.6384 -7.14947 -11.6193 3.99959 ||| -18.1138
|
||||
0 ||| taming the politicians on both sides of the Atlantic ||| d: 0 -1.60232 0 0 -3.29243 0 0 lm: -37.2846 w: -9 tm: -9.81666 -15.8365 -5.80015 -9.62002 5.99938 ||| -18.1255
|
||||
0 ||| taming politicians on both sides of the Atlantic , ||| d: 0 -1.8227 0 0 -2.66963 0 0 lm: -37.5177 w: -9 tm: -8.25244 -14.8138 -5.95756 -11.1747 4.99948 ||| -18.1464
|
||||
0 ||| taming the politicians on both sides of the Atlantic , ||| d: 0 -1.39137 0 0 -2.28527 0 0 lm: -38.7461 w: -10 tm: -7.71499 -15.7718 -8.41217 -11.5362 4.99948 ||| -18.1632
|
||||
0 ||| taming politicians on both sides of the Atlantic , ||| d: 0 -2.58871 0 0 -2.68303 0 0 lm: -37.5177 w: -9 tm: -7.54922 -14.6935 -4.71938 -11.1881 3.99959 ||| -18.1705
|
||||
0 ||| taming politicians on both sides of the Atlantic ||| d: 0 -2.17299 0 0 -3.70513 0 0 lm: -36.0562 w: -8 tm: -6.82583 -15.197 -4.04978 -9.88939 3.99958 ||| -18.184
|
||||
0 ||| taming the politicians on both sides of the Atlantic , ||| d: 0 -2.15738 0 0 -2.29867 0 0 lm: -38.7461 w: -10 tm: -7.01177 -15.6515 -7.174 -11.5495 3.99959 ||| -18.1873
|
||||
0 ||| taming the politicians on both sides of the Atlantic ||| d: 0 -2.42468 0 0 -3.5879 0 0 lm: -37.2846 w: -9 tm: -10.2303 -15.6386 -4.48249 -9.65674 5.99938 ||| -18.2478
|
||||
0 ||| taming politicians on both sides of the Atlantic , ||| d: 0 -1.66389 0 0 -2.89955 0 0 lm: -37.5177 w: -9 tm: -7.63083 -14.7823 -7.12198 -11.2052 4.99948 ||| -18.2761
|
||||
0 ||| taming politicians on both sides of the Atlantic , ||| d: 0 -1.68604 0 0 -2.75335 0 0 lm: -37.5177 w: -9 tm: -9.09102 -14.8064 -5.83693 -11.2552 4.99948 ||| -18.2887
|
||||
0 ||| taming the politicians on both sides of the Atlantic , ||| d: 0 -1.23256 0 0 -2.51519 0 0 lm: -38.7461 w: -10 tm: -7.09338 -15.7403 -9.5766 -11.5666 4.99948 ||| -18.2929
|
||||
0 ||| taming the politicians on both sides of the Atlantic , ||| d: 0 -1.25471 0 0 -2.36899 0 0 lm: -38.7461 w: -10 tm: -8.55357 -15.7643 -8.29154 -11.6166 4.99948 ||| -18.3055
|
||||
0 ||| taming the politicians on both sides of the Atlantic ||| d: 0 -2.03895 0 0 -3.35567 0 0 lm: -37.2846 w: -9 tm: -10.9679 -15.8176 -5.88577 -9.62425 6.99927 ||| -18.32
|
||||
0 ||| taming politicians on both sides of the Atlantic ||| d: 0 -2.60962 0 0 -3.76837 0 0 lm: -36.0562 w: -8 tm: -7.97707 -15.1781 -4.1354 -9.89362 4.99948 ||| -18.3785
|
||||
0 ||| taming the politicians on both sides of the Atlantic ||| d: 0 -1.65691 0 0 -3.43939 0 0 lm: -37.2846 w: -9 tm: -11.3385 -15.8102 -6.36351 -9.63607 6.99927 ||| -18.401
|
||||
0 ||| taming politicians on both sides of the Atlantic ||| d: 0 -2.19215 0 0 -3.67739 0 0 lm: -36.0562 w: -8 tm: -10.6742 -14.8543 -5.24311 -9.33236 6.99927 ||| -18.4099
|
||||
0 ||| taming the politicians on both sides of the Atlantic ||| d: 0 -1.76082 0 0 -3.29303 0 0 lm: -37.2846 w: -9 tm: -10.1368 -15.8122 -7.69773 -9.6938 6.99927 ||| -18.4267
|
||||
0 ||| taming politicians on both sides of the Atlantic , ||| d: 0 -1.54218 0 0 -2.96742 0 0 lm: -37.5177 w: -9 tm: -8.54467 -14.7711 -7.01023 -11.2497 4.99948 ||| -18.427
|
||||
0 ||| taming the politicians on both sides of the Atlantic ||| d: 0 -2.13304 0 0 -3.57153 0 0 lm: -37.2846 w: -9 tm: -11.3852 -15.7646 -5.62456 -9.65404 6.99927 ||| -18.4395
|
||||
0 ||| taming the politicians on both sides of the Atlantic , ||| d: 0 -1.11085 0 0 -2.58306 0 0 lm: -38.7461 w: -10 tm: -8.00723 -15.7291 -9.46485 -11.6111 4.99948 ||| -18.4438
|
||||
0 ||| taming politicians on both sides of the Atlantic ||| d: 0 -2.22758 0 0 -3.85209 0 0 lm: -36.0562 w: -8 tm: -8.34771 -15.1707 -4.61314 -9.90544 4.99948 ||| -18.4595
|
||||
0 ||| taming the politicians on both sides of the Atlantic ||| d: 0 -1.98917 0 0 -3.7856 0 0 lm: -37.2846 w: -9 tm: -10.8388 -15.7293 -6.79787 -9.64851 6.99927 ||| -18.5778
|
||||
0 ||| taming politicians on both sides of the Atlantic , ||| d: 0 -2.0125 0 0 -2.60639 0 0 lm: -37.5177 w: -9 tm: -8.02561 -15.1979 -6.07103 -12.091 3.99959 ||| -18.6217
|
||||
0 ||| taming politicians on both sides of the Atlantic ||| d: 0 -2.55985 0 0 -4.1983 0 0 lm: -36.0562 w: -8 tm: -7.848 -15.0898 -5.0475 -9.91788 4.99948 ||| -18.6363
|
||||
0 ||| taming politicians on both sides of the Atlantic , ||| d: 0 -2.28396 0 0 -3.29781 0 0 lm: -37.5177 w: -9 tm: -9.08225 -14.6816 -6.05783 -11.1927 4.99948 ||| -18.6364
|
||||
0 ||| taming the politicians on both sides of the Atlantic , ||| d: 0 -1.58117 0 0 -2.22203 0 0 lm: -38.7461 w: -10 tm: -7.48816 -16.1559 -8.52564 -12.4524 3.99959 ||| -18.6385
|
||||
0 ||| taming the politicians on both sides of the Atlantic , ||| d: 0 -1.85263 0 0 -2.91345 0 0 lm: -38.7461 w: -10 tm: -8.5448 -15.6395 -8.51244 -11.5541 4.99948 ||| -18.6532
|
||||
0 ||| taming the politicians on both sides of the Atlantic , ||| d: 0 -2.63088 0 0 -2.88417 0 0 lm: -38.7461 w: -10 tm: -8.50434 -15.8464 -5.43125 -12.3526 3.99959 ||| -18.6546
|
||||
0 ||| taming politicians on both sides of the Atlantic , ||| d: 0 -1.86965 0 0 -2.85256 0 0 lm: -37.5177 w: -9 tm: -7.44015 -15.0375 -7.00556 -11.9328 3.99959 ||| -18.6588
|
||||
0 ||| taming the politicians on both sides of the Atlantic , ||| d: 0 -1.43831 0 0 -2.4682 0 0 lm: -38.7461 w: -10 tm: -6.9027 -15.9955 -9.46017 -12.2942 3.99959 ||| -18.6756
|
||||
0 ||| taming the politicians on both sides of the Atlantic , ||| d: 0 -1.9941 0 0 -3.05981 0 0 lm: -38.7461 w: -10 tm: -10.2145 -15.6375 -6.57985 -11.5651 4.99948 ||| -18.6887
|
||||
0 ||| taming politicians on both sides of the Atlantic , ||| d: 0 -1.89824 0 0 -3.06558 0 0 lm: -37.5177 w: -9 tm: -9.81987 -14.8606 -7.4611 -11.1602 5.99938 ||| -18.7085
|
||||
0 ||| taming the politicians on both sides of the Atlantic , ||| d: 0 -1.4669 0 0 -2.68122 0 0 lm: -38.7461 w: -10 tm: -9.28242 -15.8185 -9.91572 -11.5217 5.99938 ||| -18.7253
|
||||
0 ||| taming the politicians on both sides of the Atlantic , ||| d: 0 -1.83912 0 0 -2.95972 0 0 lm: -38.7461 w: -10 tm: -10.5308 -15.7709 -7.84255 -11.4819 5.99938 ||| -18.7381
|
||||
0 ||| taming the politicians on both sides of the Atlantic ||| d: 0 -1.07039 0 0 -3.90067 0 0 lm: -37.2846 w: -9 tm: -8.33352 -16.8256 -6.4114 -10.5648 3.99959 ||| -18.7608
|
||||
0 ||| taming the politicians on both sides of the Atlantic , ||| d: 0 -2.60513 0 0 -2.97312 0 0 lm: -38.7461 w: -10 tm: -9.82762 -15.6506 -6.60437 -11.4952 4.99948 ||| -18.7622
|
||||
0 ||| taming politicians on both sides of the Atlantic , ||| d: 0 -1.99232 0 0 -3.28144 0 0 lm: -37.5177 w: -9 tm: -10.2371 -14.8075 -7.1999 -11.19 5.99938 ||| -18.828
|
||||
0 ||| taming the politicians on both sides of the Atlantic , ||| d: 0 -1.56099 0 0 -2.89708 0 0 lm: -38.7461 w: -10 tm: -9.6997 -15.7655 -9.65451 -11.5514 5.99938 ||| -18.8448
|
||||
0 ||| taming politicians on both sides of the Atlantic , ||| d: 0 -1.76157 0 0 -3.1493 0 0 lm: -37.5177 w: -9 tm: -10.6585 -14.8531 -7.34047 -11.2407 5.99938 ||| -18.8508
|
||||
0 ||| taming the politicians on both sides of the Atlantic , ||| d: 0 -1.33024 0 0 -2.76494 0 0 lm: -38.7461 w: -10 tm: -10.121 -15.8111 -9.79508 -11.6021 5.99938 ||| -18.8676
|
||||
0 ||| taming the politicians on both sides of the Atlantic , ||| d: 0 -1.68031 0 0 -3.18964 0 0 lm: -38.7461 w: -10 tm: -9.90924 -15.7394 -9.00698 -11.5123 5.99938 ||| -18.8678
|
||||
0 ||| taming the politicians on both sides of the Atlantic , ||| d: 0 -1.70246 0 0 -3.04344 0 0 lm: -38.7461 w: -10 tm: -11.3694 -15.7635 -7.72192 -11.5624 5.99938 ||| -18.8804
|
||||
0 ||| taming the politicians on both sides of the Atlantic ||| d: 0 -1.48965 0 0 -3.94766 0 0 lm: -37.2846 w: -9 tm: -9.21005 -16.7944 -6.48638 -10.5672 4.99948 ||| -18.8852
|
||||
0 ||| taming politicians on both sides of the Atlantic , ||| d: 0 -2.25098 0 0 -3.60234 0 0 lm: -37.5177 w: -9 tm: -6.91841 -15.0999 -7.2566 -11.7817 3.99958 ||| -18.9263
|
||||
0 ||| taming politicians on both sides of the Atlantic , ||| d: 0 -1.84846 0 0 -3.49551 0 0 lm: -37.5177 w: -9 tm: -9.6908 -14.7723 -8.37321 -11.1845 5.99938 ||| -18.9663
|
||||
0 ||| taming the politicians on both sides of the Atlantic ||| d: 0 -1.12256 0 0 -4.01553 0 0 lm: -37.2846 w: -9 tm: -9.65595 -16.7832 -6.97301 -10.543 4.99948 ||| -18.9749
|
||||
0 ||| taming the politicians on both sides of the Atlantic , ||| d: 0 -1.41713 0 0 -3.11115 0 0 lm: -38.7461 w: -10 tm: -9.15335 -15.7302 -10.8278 -11.5459 5.99938 ||| -18.9831
|
||||
0 ||| taming the politicians on both sides of the Atlantic ||| d: 0 -2.20857 0 0 -3.96748 0 0 lm: -37.2846 w: -9 tm: -12.9526 -15.8114 -7.12811 -9.63954 7.99917 ||| -19.0016
|
||||
0 ||| taming the politicians on both sides of the Atlantic , ||| d: 0 -1.5586 0 0 -3.25751 0 0 lm: -38.7461 w: -10 tm: -10.8231 -15.7282 -8.89523 -11.5568 5.99938 ||| -19.0187
|
||||
0 ||| taming politicians on both sides of the Atlantic ||| d: 0 -2.77924 0 0 -4.38018 0 0 lm: -36.0562 w: -8 tm: -9.96178 -15.1719 -5.37774 -9.90891 5.99938 ||| -19.0601
|
||||
0 ||| taming politicians on both sides of the Atlantic , ||| d: 0 -2.12927 0 0 -3.67021 0 0 lm: -37.5177 w: -9 tm: -7.83225 -15.0887 -7.14485 -11.8262 3.99958 ||| -19.0772
|
||||
0 ||| taming the politicians on both sides of the Atlantic ||| d: 0 -1.28737 0 0 -4.05045 0 0 lm: -37.2846 w: -9 tm: -10.2478 -16.8915 -6.73989 -10.518 4.99948 ||| -19.1232
|
||||
0 ||| taming politicians on both sides of the Atlantic , ||| d: 0 -2.08803 0 0 -3.00234 0 0 lm: -37.5177 w: -9 tm: -9.59305 -15.2447 -7.57457 -12.0765 4.99948 ||| -19.1838
|
||||
0 ||| taming the politicians on both sides of the Atlantic , ||| d: 0 -1.6567 0 0 -2.61798 0 0 lm: -38.7461 w: -10 tm: -9.0556 -16.2026 -10.0292 -12.4379 4.99948 ||| -19.2006
|
||||
0 ||| taming the politicians on both sides of the Atlantic , ||| d: 0 -2.02892 0 0 -2.89648 0 0 lm: -38.7461 w: -10 tm: -10.304 -16.155 -7.95602 -12.3982 4.99948 ||| -19.2134
|
100
contrib/promix/test_data/test.nbest.nbest.segments
Normal file
100
contrib/promix/test_data/test.nbest.nbest.segments
Normal file
@ -0,0 +1,100 @@
|
||||
0 ||| taming |0-1| politicians |2-3| on both |4-5| sides of the Atlantic |6-8| ||| d: 0 -1.51037 0 0 -2.60639 0 0 lm: -36.0562 w: -8 tm: -5.97082 -14.8327 -2.41162 -9.32734 3.99959 ||| -16.9717
|
||||
0 ||| taming |0-1| the politicians |2-3| on both |4-5| sides of the Atlantic |6-8| ||| d: 0 -1.07904 0 0 -2.22203 0 0 lm: -37.2846 w: -9 tm: -5.43337 -15.7907 -4.86623 -9.68878 3.99959 ||| -16.9885
|
||||
0 ||| taming |0-1| politicians |2-3| on both sides of the Atlantic |4-8| ||| d: 0 -2.24178 0 0 -2.59408 0 0 lm: -36.0562 w: -8 tm: -5.55036 -14.7311 -1.14332 -9.34122 2.99969 ||| -17.0321
|
||||
0 ||| taming |0-1| the politicians |2-3| on both sides of the Atlantic |4-8| ||| d: 0 -1.81045 0 0 -2.20973 0 0 lm: -37.2846 w: -9 tm: -5.01291 -15.689 -3.59793 -9.70266 2.99969 ||| -17.0489
|
||||
0 ||| taming |0-1| politicians |2-3| on both sides |4-6| of the Atlantic |7-8| ||| d: 0 -1.8566 0 0 -2.76972 0 0 lm: -36.0562 w: -8 tm: -6.33782 -14.6804 -1.8329 -9.34609 3.99959 ||| -17.0555
|
||||
0 ||| taming |0-1| the politicians |2-3| on both sides |4-6| of the Atlantic |7-8| ||| d: 0 -1.42527 0 0 -2.38536 0 0 lm: -37.2846 w: -9 tm: -5.80037 -15.6384 -4.28752 -9.70753 3.99959 ||| -17.0723
|
||||
0 ||| taming |0-1| politicians |2-3| on both |4-5| sides of the |6-7| Atlantic |8-8| ||| d: 0 -1.947 0 0 -2.66963 0 0 lm: -36.0562 w: -8 tm: -7.12207 -14.8138 -2.49723 -9.33158 4.99948 ||| -17.1661
|
||||
0 ||| taming |0-1| politicians |2-3| on |4-4| both sides of the Atlantic |5-8| ||| d: 0 -1.36892 0 0 -2.85256 0 0 lm: -36.0562 w: -8 tm: -5.62394 -14.8136 -3.58667 -9.35962 3.99959 ||| -17.1714
|
||||
0 ||| taming |0-1| the politicians |2-3| on both |4-5| sides of the |6-7| Atlantic |8-8| ||| d: 0 -1.51567 0 0 -2.28527 0 0 lm: -37.2846 w: -9 tm: -6.58462 -15.7718 -4.95184 -9.69302 4.99948 ||| -17.183
|
||||
0 ||| taming |0-1| the politicians |2-3| on |4-4| both sides of the Atlantic |5-8| ||| d: 0 -0.93759 0 0 -2.4682 0 0 lm: -37.2846 w: -9 tm: -5.08649 -15.7715 -6.04129 -9.72106 3.99959 ||| -17.1882
|
||||
0 ||| taming |0-1| politicians |2-3| on both sides of the |4-7| Atlantic |8-8| ||| d: 0 -2.71301 0 0 -2.68303 0 0 lm: -36.0562 w: -8 tm: -6.41885 -14.6935 -1.25905 -9.34492 3.99959 ||| -17.1903
|
||||
0 ||| taming |0-1| the politicians |2-3| on both sides of the |4-7| Atlantic |8-8| ||| d: 0 -2.28167 0 0 -2.29867 0 0 lm: -37.2846 w: -9 tm: -5.8814 -15.6515 -3.71367 -9.70637 3.99959 ||| -17.2071
|
||||
0 ||| taming |0-1| politicians |2-3| on both |4-5| sides |6-6| of the Atlantic |7-8| ||| d: 0 -1.56496 0 0 -2.75335 0 0 lm: -36.0562 w: -8 tm: -7.49271 -14.8064 -2.97497 -9.34339 4.99948 ||| -17.2472
|
||||
0 ||| taming |0-1| the politicians |2-3| on both |4-5| sides |6-6| of the Atlantic |7-8| ||| d: 0 -1.13363 0 0 -2.36899 0 0 lm: -37.2846 w: -9 tm: -6.95526 -15.7643 -5.42959 -9.70483 4.99948 ||| -17.264
|
||||
0 ||| taming |0-1| politicians |2-3| on |4-4| both sides of the |5-7| Atlantic |8-8| ||| d: 0 -1.78819 0 0 -2.89955 0 0 lm: -36.0562 w: -8 tm: -6.50046 -14.7823 -3.66165 -9.36203 4.99948 ||| -17.2958
|
||||
0 ||| taming |0-1| the politicians |2-3| on |4-4| both sides of the |5-7| Atlantic |8-8| ||| d: 0 -1.35686 0 0 -2.51519 0 0 lm: -37.2846 w: -9 tm: -5.96301 -15.7403 -6.11627 -9.72347 4.99948 ||| -17.3126
|
||||
0 ||| taming |0-1| politicians |2-3| on |4-4| both sides |5-6| of the Atlantic |7-8| ||| d: 0 -1.4211 0 0 -2.96742 0 0 lm: -36.0562 w: -8 tm: -6.94636 -14.7711 -4.14828 -9.33786 4.99948 ||| -17.3855
|
||||
0 ||| taming |0-1| the politicians |2-3| on |4-4| both sides |5-6| of the Atlantic |7-8| ||| d: 0 -0.989766 0 0 -2.58306 0 0 lm: -37.2846 w: -9 tm: -6.40891 -15.7291 -6.6029 -9.69931 4.99948 ||| -17.4023
|
||||
0 ||| taming |0-1| politicians |2-3| on |4-4| both |5-5| sides of the Atlantic |6-8| ||| d: 0 -1.5859 0 0 -3.00234 0 0 lm: -36.0562 w: -8 tm: -7.53826 -14.8794 -3.91516 -9.31284 4.99948 ||| -17.5338
|
||||
0 ||| taming |0-1| the politicians |2-3| on |4-4| both |5-5| sides of the Atlantic |6-8| ||| d: 0 -1.15457 0 0 -2.61798 0 0 lm: -37.2846 w: -9 tm: -7.00081 -15.8374 -6.36978 -9.67428 4.99948 ||| -17.5506
|
||||
0 ||| taming |0-1| the |2-2| politicians |3-3| on both |4-5| sides of the Atlantic |6-8| ||| d: 0 -1.52679 0 0 -2.89648 0 0 lm: -37.2846 w: -9 tm: -8.24923 -15.7898 -4.29661 -9.63452 4.99948 ||| -17.5634
|
||||
0 ||| taming |0-1| the |2-2| politicians |3-3| on both sides of the Atlantic |4-8| ||| d: 0 -2.2582 0 0 -2.88417 0 0 lm: -37.2846 w: -9 tm: -7.82876 -15.6881 -3.02831 -9.6484 3.99959 ||| -17.6238
|
||||
0 ||| taming |0-1| the |2-2| politicians |3-3| on both sides |4-6| of the Atlantic |7-8| ||| d: 0 -1.87302 0 0 -3.05981 0 0 lm: -37.2846 w: -9 tm: -8.61622 -15.6375 -3.7179 -9.65327 4.99948 ||| -17.6472
|
||||
0 ||| taming |0-1| politicians |2-3| on both sides |4-6| of the |7-7| Atlantic |8-8| ||| d: 0 -2.40826 0 0 -3.29781 0 0 lm: -36.0562 w: -8 tm: -7.95188 -14.6816 -2.5975 -9.34956 4.99948 ||| -17.6561
|
||||
0 ||| taming |0-1| the politicians |2-3| on both sides |4-6| of the |7-7| Atlantic |8-8| ||| d: 0 -1.97693 0 0 -2.91345 0 0 lm: -37.2846 w: -9 tm: -7.41443 -15.6395 -5.05211 -9.711 4.99948 ||| -17.6729
|
||||
0 ||| taming |0-1| politicians |2-3| on |4-4| both |5-5| sides of the |6-7| Atlantic |8-8| ||| d: 0 -2.02253 0 0 -3.06558 0 0 lm: -36.0562 w: -8 tm: -8.6895 -14.8606 -4.00077 -9.31707 5.99938 ||| -17.7283
|
||||
0 ||| taming |0-1| the politicians |2-3| on |4-4| both |5-5| sides of the |6-7| Atlantic |8-8| ||| d: 0 -1.5912 0 0 -2.68122 0 0 lm: -37.2846 w: -9 tm: -8.15205 -15.8185 -6.45539 -9.67851 5.99938 ||| -17.7451
|
||||
0 ||| taming |0-1| the |2-2| politicians |3-3| on both |4-5| sides of the |6-7| Atlantic |8-8| ||| d: 0 -1.96342 0 0 -2.95972 0 0 lm: -37.2846 w: -9 tm: -9.40047 -15.7709 -4.38222 -9.63875 5.99938 ||| -17.7579
|
||||
0 ||| taming |0-1| the |2-2| politicians |3-3| on |4-4| both sides of the Atlantic |5-8| ||| d: 0 -1.38534 0 0 -3.14265 0 0 lm: -37.2846 w: -9 tm: -7.90234 -15.7706 -5.47167 -9.66679 4.99948 ||| -17.7631
|
||||
0 ||| taming |0-1| the |2-2| politicians |3-3| on both sides of the |4-7| Atlantic |8-8| ||| d: 0 -2.72942 0 0 -2.97312 0 0 lm: -37.2846 w: -9 tm: -8.69725 -15.6506 -3.14405 -9.6521 4.99948 ||| -17.782
|
||||
0 ||| taming |0-1| politicians |2-3| on |4-4| both |5-5| sides |6-6| of the Atlantic |7-8| ||| d: 0 -1.64049 0 0 -3.1493 0 0 lm: -36.0562 w: -8 tm: -9.06014 -14.8531 -4.47852 -9.32889 5.99938 ||| -17.8093
|
||||
0 ||| taming |0-1| politicians on |2-4| both sides of the Atlantic |5-8| ||| d: 0 -1.95601 0 0 -3.55535 0 0 lm: -36.0562 w: -8 tm: -4.91151 -15.1311 -3.72129 -9.93616 2.99969 ||| -17.8216
|
||||
0 ||| taming |0-1| the politicians |2-3| on |4-4| both |5-5| sides |6-6| of the Atlantic |7-8| ||| d: 0 -1.20916 0 0 -2.76494 0 0 lm: -37.2846 w: -9 tm: -8.52269 -15.8111 -6.93313 -9.69033 5.99938 ||| -17.8261
|
||||
0 ||| taming |0-1| the |2-2| politicians |3-3| on both |4-5| sides |6-6| of the Atlantic |7-8| ||| d: 0 -1.58137 0 0 -3.04344 0 0 lm: -37.2846 w: -9 tm: -9.77111 -15.7635 -4.85997 -9.65057 5.99938 ||| -17.8389
|
||||
0 ||| taming |0-1| politicians |2-3| on both |4-5| sides |6-6| of the |7-7| Atlantic |8-8| ||| d: 0 -2.11662 0 0 -3.28144 0 0 lm: -36.0562 w: -8 tm: -9.10677 -14.8075 -3.73957 -9.34686 5.99938 ||| -17.8478
|
||||
0 ||| taming |0-1| the politicians |2-3| on both |4-5| sides |6-6| of the |7-7| Atlantic |8-8| ||| d: 0 -1.68529 0 0 -2.89708 0 0 lm: -37.2846 w: -9 tm: -8.56932 -15.7655 -6.19418 -9.7083 5.99938 ||| -17.8646
|
||||
0 ||| taming |0-1| the |2-2| politicians |3-3| on |4-4| both sides of the |5-7| Atlantic |8-8| ||| d: 0 -1.80461 0 0 -3.18964 0 0 lm: -37.2846 w: -9 tm: -8.77886 -15.7394 -5.54665 -9.66921 5.99938 ||| -17.8875
|
||||
0 ||| taming |0-1| politicians on |2-4| both sides of the |5-7| Atlantic |8-8| ||| d: 0 -2.37528 0 0 -3.60234 0 0 lm: -36.0562 w: -8 tm: -5.78803 -15.0999 -3.79627 -9.93857 3.99958 ||| -17.946
|
||||
0 ||| taming |0-1| the |2-2| politicians |3-3| on |4-4| both sides |5-6| of the Atlantic |7-8| ||| d: 0 -1.43751 0 0 -3.25751 0 0 lm: -37.2846 w: -9 tm: -9.22476 -15.7282 -6.03327 -9.64504 5.99938 ||| -17.9772
|
||||
0 ||| taming |0-1| politicians |2-3| on |4-4| both sides |5-6| of the |7-7| Atlantic |8-8| ||| d: 0 -1.97276 0 0 -3.49551 0 0 lm: -36.0562 w: -8 tm: -8.56043 -14.7723 -4.91288 -9.34133 5.99938 ||| -17.9861
|
||||
0 ||| taming |0-1| the politicians |2-3| on |4-4| both sides |5-6| of the |7-7| Atlantic |8-8| ||| d: 0 -1.54143 0 0 -3.11115 0 0 lm: -37.2846 w: -9 tm: -8.02298 -15.7302 -7.36749 -9.70277 5.99938 ||| -18.0029
|
||||
0 ||| taming |0-1| politicians on |2-4| both sides |5-6| of the Atlantic |7-8| ||| d: 0 -2.00819 0 0 -3.67021 0 0 lm: -36.0562 w: -8 tm: -6.23393 -15.0887 -4.2829 -9.91441 3.99958 ||| -18.0357
|
||||
0 ||| taming |0-1| politicians |2-3| on both sides of the Atlantic , |4-8| ||| d: 0 -2.61446 0 0 -2.59408 0 0 lm: -37.5177 w: -9 tm: -6.22594 -14.8893 -3.54625 -12.0454 2.99969 ||| -18.0629
|
||||
0 ||| taming |0-1| the politicians |2-3| on both sides of the Atlantic , |4-8| ||| d: 0 -2.18313 0 0 -2.20973 0 0 lm: -38.7461 w: -10 tm: -5.68849 -15.8473 -6.00087 -12.4069 2.99969 ||| -18.0797
|
||||
0 ||| taming |0-1| politicians |2-3| on both sides |4-6| of the Atlantic , |7-8| ||| d: 0 -1.97768 0 0 -2.76972 0 0 lm: -37.5177 w: -9 tm: -7.93613 -14.6804 -4.69485 -11.2579 3.99959 ||| -18.097
|
||||
0 ||| taming |0-1| the politicians |2-3| on both sides |4-6| of the Atlantic , |7-8| ||| d: 0 -1.54635 0 0 -2.38536 0 0 lm: -38.7461 w: -10 tm: -7.39868 -15.6384 -7.14947 -11.6193 3.99959 ||| -18.1138
|
||||
0 ||| taming |0-1| the |2-2| politicians |3-3| on |4-4| both |5-5| sides of the Atlantic |6-8| ||| d: 0 -1.60232 0 0 -3.29243 0 0 lm: -37.2846 w: -9 tm: -9.81666 -15.8365 -5.80015 -9.62002 5.99938 ||| -18.1255
|
||||
0 ||| taming |0-1| politicians |2-3| on both |4-5| sides of the |6-7| Atlantic , |8-8| ||| d: 0 -1.8227 0 0 -2.66963 0 0 lm: -37.5177 w: -9 tm: -8.25244 -14.8138 -5.95756 -11.1747 4.99948 ||| -18.1464
|
||||
0 ||| taming |0-1| the politicians |2-3| on both |4-5| sides of the |6-7| Atlantic , |8-8| ||| d: 0 -1.39137 0 0 -2.28527 0 0 lm: -38.7461 w: -10 tm: -7.71499 -15.7718 -8.41217 -11.5362 4.99948 ||| -18.1632
|
||||
0 ||| taming |0-1| politicians |2-3| on both sides of the |4-7| Atlantic , |8-8| ||| d: 0 -2.58871 0 0 -2.68303 0 0 lm: -37.5177 w: -9 tm: -7.54922 -14.6935 -4.71938 -11.1881 3.99959 ||| -18.1705
|
||||
0 ||| taming |0-1| politicians on |2-4| both |5-5| sides of the Atlantic |6-8| ||| d: 0 -2.17299 0 0 -3.70513 0 0 lm: -36.0562 w: -8 tm: -6.82583 -15.197 -4.04978 -9.88939 3.99958 ||| -18.184
|
||||
0 ||| taming |0-1| the politicians |2-3| on both sides of the |4-7| Atlantic , |8-8| ||| d: 0 -2.15738 0 0 -2.29867 0 0 lm: -38.7461 w: -10 tm: -7.01177 -15.6515 -7.174 -11.5495 3.99959 ||| -18.1873
|
||||
0 ||| taming |0-1| the |2-2| politicians |3-3| on both sides |4-6| of the |7-7| Atlantic |8-8| ||| d: 0 -2.42468 0 0 -3.5879 0 0 lm: -37.2846 w: -9 tm: -10.2303 -15.6386 -4.48249 -9.65674 5.99938 ||| -18.2478
|
||||
0 ||| taming |0-1| politicians |2-3| on |4-4| both sides of the |5-7| Atlantic , |8-8| ||| d: 0 -1.66389 0 0 -2.89955 0 0 lm: -37.5177 w: -9 tm: -7.63083 -14.7823 -7.12198 -11.2052 4.99948 ||| -18.2761
|
||||
0 ||| taming |0-1| politicians |2-3| on both |4-5| sides |6-6| of the Atlantic , |7-8| ||| d: 0 -1.68604 0 0 -2.75335 0 0 lm: -37.5177 w: -9 tm: -9.09102 -14.8064 -5.83693 -11.2552 4.99948 ||| -18.2887
|
||||
0 ||| taming |0-1| the politicians |2-3| on |4-4| both sides of the |5-7| Atlantic , |8-8| ||| d: 0 -1.23256 0 0 -2.51519 0 0 lm: -38.7461 w: -10 tm: -7.09338 -15.7403 -9.5766 -11.5666 4.99948 ||| -18.2929
|
||||
0 ||| taming |0-1| the politicians |2-3| on both |4-5| sides |6-6| of the Atlantic , |7-8| ||| d: 0 -1.25471 0 0 -2.36899 0 0 lm: -38.7461 w: -10 tm: -8.55357 -15.7643 -8.29154 -11.6166 4.99948 ||| -18.3055
|
||||
0 ||| taming |0-1| the |2-2| politicians |3-3| on |4-4| both |5-5| sides of the |6-7| Atlantic |8-8| ||| d: 0 -2.03895 0 0 -3.35567 0 0 lm: -37.2846 w: -9 tm: -10.9679 -15.8176 -5.88577 -9.62425 6.99927 ||| -18.32
|
||||
0 ||| taming |0-1| politicians on |2-4| both |5-5| sides of the |6-7| Atlantic |8-8| ||| d: 0 -2.60962 0 0 -3.76837 0 0 lm: -36.0562 w: -8 tm: -7.97707 -15.1781 -4.1354 -9.89362 4.99948 ||| -18.3785
|
||||
0 ||| taming |0-1| the |2-2| politicians |3-3| on |4-4| both |5-5| sides |6-6| of the Atlantic |7-8| ||| d: 0 -1.65691 0 0 -3.43939 0 0 lm: -37.2846 w: -9 tm: -11.3385 -15.8102 -6.36351 -9.63607 6.99927 ||| -18.401
|
||||
0 ||| taming |0-1| politicians |2-3| on |4-4| both |5-5| sides |6-6| of the |7-7| Atlantic |8-8| ||| d: 0 -2.19215 0 0 -3.67739 0 0 lm: -36.0562 w: -8 tm: -10.6742 -14.8543 -5.24311 -9.33236 6.99927 ||| -18.4099
|
||||
0 ||| taming |0-1| the politicians |2-3| on |4-4| both |5-5| sides |6-6| of the |7-7| Atlantic |8-8| ||| d: 0 -1.76082 0 0 -3.29303 0 0 lm: -37.2846 w: -9 tm: -10.1368 -15.8122 -7.69773 -9.6938 6.99927 ||| -18.4267
|
||||
0 ||| taming |0-1| politicians |2-3| on |4-4| both sides |5-6| of the Atlantic , |7-8| ||| d: 0 -1.54218 0 0 -2.96742 0 0 lm: -37.5177 w: -9 tm: -8.54467 -14.7711 -7.01023 -11.2497 4.99948 ||| -18.427
|
||||
0 ||| taming |0-1| the |2-2| politicians |3-3| on both |4-5| sides |6-6| of the |7-7| Atlantic |8-8| ||| d: 0 -2.13304 0 0 -3.57153 0 0 lm: -37.2846 w: -9 tm: -11.3852 -15.7646 -5.62456 -9.65404 6.99927 ||| -18.4395
|
||||
0 ||| taming |0-1| the politicians |2-3| on |4-4| both sides |5-6| of the Atlantic , |7-8| ||| d: 0 -1.11085 0 0 -2.58306 0 0 lm: -38.7461 w: -10 tm: -8.00723 -15.7291 -9.46485 -11.6111 4.99948 ||| -18.4438
|
||||
0 ||| taming |0-1| politicians on |2-4| both |5-5| sides |6-6| of the Atlantic |7-8| ||| d: 0 -2.22758 0 0 -3.85209 0 0 lm: -36.0562 w: -8 tm: -8.34771 -15.1707 -4.61314 -9.90544 4.99948 ||| -18.4595
|
||||
0 ||| taming |0-1| the |2-2| politicians |3-3| on |4-4| both sides |5-6| of the |7-7| Atlantic |8-8| ||| d: 0 -1.98917 0 0 -3.7856 0 0 lm: -37.2846 w: -9 tm: -10.8388 -15.7293 -6.79787 -9.64851 6.99927 ||| -18.5778
|
||||
0 ||| taming |0-1| politicians |2-3| on both |4-5| sides of the Atlantic , |6-8| ||| d: 0 -2.0125 0 0 -2.60639 0 0 lm: -37.5177 w: -9 tm: -8.02561 -15.1979 -6.07103 -12.091 3.99959 ||| -18.6217
|
||||
0 ||| taming |0-1| politicians on |2-4| both sides |5-6| of the |7-7| Atlantic |8-8| ||| d: 0 -2.55985 0 0 -4.1983 0 0 lm: -36.0562 w: -8 tm: -7.848 -15.0898 -5.0475 -9.91788 4.99948 ||| -18.6363
|
||||
0 ||| taming |0-1| politicians |2-3| on both sides |4-6| of the |7-7| Atlantic , |8-8| ||| d: 0 -2.28396 0 0 -3.29781 0 0 lm: -37.5177 w: -9 tm: -9.08225 -14.6816 -6.05783 -11.1927 4.99948 ||| -18.6364
|
||||
0 ||| taming |0-1| the politicians |2-3| on both |4-5| sides of the Atlantic , |6-8| ||| d: 0 -1.58117 0 0 -2.22203 0 0 lm: -38.7461 w: -10 tm: -7.48816 -16.1559 -8.52564 -12.4524 3.99959 ||| -18.6385
|
||||
0 ||| taming |0-1| the politicians |2-3| on both sides |4-6| of the |7-7| Atlantic , |8-8| ||| d: 0 -1.85263 0 0 -2.91345 0 0 lm: -38.7461 w: -10 tm: -8.5448 -15.6395 -8.51244 -11.5541 4.99948 ||| -18.6532
|
||||
0 ||| taming |0-1| the |2-2| politicians |3-3| on both sides of the Atlantic , |4-8| ||| d: 0 -2.63088 0 0 -2.88417 0 0 lm: -38.7461 w: -10 tm: -8.50434 -15.8464 -5.43125 -12.3526 3.99959 ||| -18.6546
|
||||
0 ||| taming |0-1| politicians |2-3| on |4-4| both sides of the Atlantic , |5-8| ||| d: 0 -1.86965 0 0 -2.85256 0 0 lm: -37.5177 w: -9 tm: -7.44015 -15.0375 -7.00556 -11.9328 3.99959 ||| -18.6588
|
||||
0 ||| taming |0-1| the politicians |2-3| on |4-4| both sides of the Atlantic , |5-8| ||| d: 0 -1.43831 0 0 -2.4682 0 0 lm: -38.7461 w: -10 tm: -6.9027 -15.9955 -9.46017 -12.2942 3.99959 ||| -18.6756
|
||||
0 ||| taming |0-1| the |2-2| politicians |3-3| on both sides |4-6| of the Atlantic , |7-8| ||| d: 0 -1.9941 0 0 -3.05981 0 0 lm: -38.7461 w: -10 tm: -10.2145 -15.6375 -6.57985 -11.5651 4.99948 ||| -18.6887
|
||||
0 ||| taming |0-1| politicians |2-3| on |4-4| both |5-5| sides of the |6-7| Atlantic , |8-8| ||| d: 0 -1.89824 0 0 -3.06558 0 0 lm: -37.5177 w: -9 tm: -9.81987 -14.8606 -7.4611 -11.1602 5.99938 ||| -18.7085
|
||||
0 ||| taming |0-1| the politicians |2-3| on |4-4| both |5-5| sides of the |6-7| Atlantic , |8-8| ||| d: 0 -1.4669 0 0 -2.68122 0 0 lm: -38.7461 w: -10 tm: -9.28242 -15.8185 -9.91572 -11.5217 5.99938 ||| -18.7253
|
||||
0 ||| taming |0-1| the |2-2| politicians |3-3| on both |4-5| sides of the |6-7| Atlantic , |8-8| ||| d: 0 -1.83912 0 0 -2.95972 0 0 lm: -38.7461 w: -10 tm: -10.5308 -15.7709 -7.84255 -11.4819 5.99938 ||| -18.7381
|
||||
0 ||| taming |0-1| the |2-2| politicians on |3-4| both sides of the Atlantic |5-8| ||| d: 0 -1.07039 0 0 -3.90067 0 0 lm: -37.2846 w: -9 tm: -8.33352 -16.8256 -6.4114 -10.5648 3.99959 ||| -18.7608
|
||||
0 ||| taming |0-1| the |2-2| politicians |3-3| on both sides of the |4-7| Atlantic , |8-8| ||| d: 0 -2.60513 0 0 -2.97312 0 0 lm: -38.7461 w: -10 tm: -9.82762 -15.6506 -6.60437 -11.4952 4.99948 ||| -18.7622
|
||||
0 ||| taming |0-1| politicians |2-3| on both |4-5| sides |6-6| of the |7-7| Atlantic , |8-8| ||| d: 0 -1.99232 0 0 -3.28144 0 0 lm: -37.5177 w: -9 tm: -10.2371 -14.8075 -7.1999 -11.19 5.99938 ||| -18.828
|
||||
0 ||| taming |0-1| the politicians |2-3| on both |4-5| sides |6-6| of the |7-7| Atlantic , |8-8| ||| d: 0 -1.56099 0 0 -2.89708 0 0 lm: -38.7461 w: -10 tm: -9.6997 -15.7655 -9.65451 -11.5514 5.99938 ||| -18.8448
|
||||
0 ||| taming |0-1| politicians |2-3| on |4-4| both |5-5| sides |6-6| of the Atlantic , |7-8| ||| d: 0 -1.76157 0 0 -3.1493 0 0 lm: -37.5177 w: -9 tm: -10.6585 -14.8531 -7.34047 -11.2407 5.99938 ||| -18.8508
|
||||
0 ||| taming |0-1| the politicians |2-3| on |4-4| both |5-5| sides |6-6| of the Atlantic , |7-8| ||| d: 0 -1.33024 0 0 -2.76494 0 0 lm: -38.7461 w: -10 tm: -10.121 -15.8111 -9.79508 -11.6021 5.99938 ||| -18.8676
|
||||
0 ||| taming |0-1| the |2-2| politicians |3-3| on |4-4| both sides of the |5-7| Atlantic , |8-8| ||| d: 0 -1.68031 0 0 -3.18964 0 0 lm: -38.7461 w: -10 tm: -9.90924 -15.7394 -9.00698 -11.5123 5.99938 ||| -18.8678
|
||||
0 ||| taming |0-1| the |2-2| politicians |3-3| on both |4-5| sides |6-6| of the Atlantic , |7-8| ||| d: 0 -1.70246 0 0 -3.04344 0 0 lm: -38.7461 w: -10 tm: -11.3694 -15.7635 -7.72192 -11.5624 5.99938 ||| -18.8804
|
||||
0 ||| taming |0-1| the |2-2| politicians on |3-4| both sides of the |5-7| Atlantic |8-8| ||| d: 0 -1.48965 0 0 -3.94766 0 0 lm: -37.2846 w: -9 tm: -9.21005 -16.7944 -6.48638 -10.5672 4.99948 ||| -18.8852
|
||||
0 ||| taming |0-1| politicians on |2-4| both sides of the |5-7| Atlantic , |8-8| ||| d: 0 -2.25098 0 0 -3.60234 0 0 lm: -37.5177 w: -9 tm: -6.91841 -15.0999 -7.2566 -11.7817 3.99958 ||| -18.9263
|
||||
0 ||| taming |0-1| politicians |2-3| on |4-4| both sides |5-6| of the |7-7| Atlantic , |8-8| ||| d: 0 -1.84846 0 0 -3.49551 0 0 lm: -37.5177 w: -9 tm: -9.6908 -14.7723 -8.37321 -11.1845 5.99938 ||| -18.9663
|
||||
0 ||| taming |0-1| the |2-2| politicians on |3-4| both sides |5-6| of the Atlantic |7-8| ||| d: 0 -1.12256 0 0 -4.01553 0 0 lm: -37.2846 w: -9 tm: -9.65595 -16.7832 -6.97301 -10.543 4.99948 ||| -18.9749
|
||||
0 ||| taming |0-1| the politicians |2-3| on |4-4| both sides |5-6| of the |7-7| Atlantic , |8-8| ||| d: 0 -1.41713 0 0 -3.11115 0 0 lm: -38.7461 w: -10 tm: -9.15335 -15.7302 -10.8278 -11.5459 5.99938 ||| -18.9831
|
||||
0 ||| taming |0-1| the |2-2| politicians |3-3| on |4-4| both |5-5| sides |6-6| of the |7-7| Atlantic |8-8| ||| d: 0 -2.20857 0 0 -3.96748 0 0 lm: -37.2846 w: -9 tm: -12.9526 -15.8114 -7.12811 -9.63954 7.99917 ||| -19.0016
|
||||
0 ||| taming |0-1| the |2-2| politicians |3-3| on |4-4| both sides |5-6| of the Atlantic , |7-8| ||| d: 0 -1.5586 0 0 -3.25751 0 0 lm: -38.7461 w: -10 tm: -10.8231 -15.7282 -8.89523 -11.5568 5.99938 ||| -19.0187
|
||||
0 ||| taming |0-1| politicians on |2-4| both |5-5| sides |6-6| of the |7-7| Atlantic |8-8| ||| d: 0 -2.77924 0 0 -4.38018 0 0 lm: -36.0562 w: -8 tm: -9.96178 -15.1719 -5.37774 -9.90891 5.99938 ||| -19.0601
|
||||
0 ||| taming |0-1| politicians on |2-4| both sides |5-6| of the Atlantic , |7-8| ||| d: 0 -2.12927 0 0 -3.67021 0 0 lm: -37.5177 w: -9 tm: -7.83225 -15.0887 -7.14485 -11.8262 3.99958 ||| -19.0772
|
||||
0 ||| taming |0-1| the |2-2| politicians on |3-4| both |5-5| sides of the Atlantic |6-8| ||| d: 0 -1.28737 0 0 -4.05045 0 0 lm: -37.2846 w: -9 tm: -10.2478 -16.8915 -6.73989 -10.518 4.99948 ||| -19.1232
|
||||
0 ||| taming |0-1| politicians |2-3| on |4-4| both |5-5| sides of the Atlantic , |6-8| ||| d: 0 -2.08803 0 0 -3.00234 0 0 lm: -37.5177 w: -9 tm: -9.59305 -15.2447 -7.57457 -12.0765 4.99948 ||| -19.1838
|
||||
0 ||| taming |0-1| the politicians |2-3| on |4-4| both |5-5| sides of the Atlantic , |6-8| ||| d: 0 -1.6567 0 0 -2.61798 0 0 lm: -38.7461 w: -10 tm: -9.0556 -16.2026 -10.0292 -12.4379 4.99948 ||| -19.2006
|
||||
0 ||| taming |0-1| the |2-2| politicians |3-3| on both |4-5| sides of the Atlantic , |6-8| ||| d: 0 -2.02892 0 0 -2.89648 0 0 lm: -38.7461 w: -10 tm: -10.304 -16.155 -7.95602 -12.3982 4.99948 ||| -19.2134
|
102
contrib/promix/test_data/test.nbest.scores
Normal file
102
contrib/promix/test_data/test.nbest.scores
Normal file
@ -0,0 +1,102 @@
|
||||
SCORES_TXT_BEGIN_0 0 100 9 BLEU
|
||||
5 12 2 11 1 10 0 9 8
|
||||
5 13 2 12 1 11 0 10 8
|
||||
5 11 2 10 1 9 0 8 8
|
||||
5 12 2 11 1 10 0 9 8
|
||||
5 12 2 11 1 10 0 9 8
|
||||
5 13 2 12 1 11 0 10 8
|
||||
5 13 1 12 0 11 0 10 8
|
||||
5 12 2 11 1 10 0 9 8
|
||||
5 14 1 13 0 12 0 11 8
|
||||
5 13 2 12 1 11 0 10 8
|
||||
5 12 1 11 0 10 0 9 8
|
||||
5 13 1 12 0 11 0 10 8
|
||||
5 13 2 12 1 11 0 10 8
|
||||
5 14 2 13 1 12 0 11 8
|
||||
5 13 1 12 0 11 0 10 8
|
||||
5 14 1 13 0 12 0 11 8
|
||||
5 13 2 12 1 11 0 10 8
|
||||
5 14 2 13 1 12 0 11 8
|
||||
5 13 2 12 1 11 0 10 8
|
||||
5 14 2 13 1 12 0 11 8
|
||||
5 14 2 13 1 12 0 11 8
|
||||
5 13 2 12 1 11 0 10 8
|
||||
5 14 2 13 1 12 0 11 8
|
||||
5 13 1 12 0 11 0 10 8
|
||||
5 14 1 13 0 12 0 11 8
|
||||
5 14 1 13 0 12 0 11 8
|
||||
5 15 1 14 0 13 0 12 8
|
||||
5 15 1 14 0 13 0 12 8
|
||||
5 14 2 13 1 12 0 11 8
|
||||
5 14 1 13 0 12 0 11 8
|
||||
5 14 2 13 1 12 0 11 8
|
||||
5 11 2 10 1 9 0 8 8
|
||||
5 15 2 14 1 13 0 12 8
|
||||
5 15 2 14 1 13 0 12 8
|
||||
5 14 1 13 0 12 0 11 8
|
||||
5 15 1 14 0 13 0 12 8
|
||||
5 15 1 14 0 13 0 12 8
|
||||
5 12 1 11 0 10 0 9 8
|
||||
5 15 2 14 1 13 0 12 8
|
||||
5 14 1 13 0 12 0 11 8
|
||||
5 15 1 14 0 13 0 12 8
|
||||
5 12 2 11 1 10 0 9 8
|
||||
5 12 2 11 1 10 0 9 8
|
||||
5 13 2 12 1 11 0 10 8
|
||||
5 13 2 12 1 11 0 10 8
|
||||
5 14 2 13 1 12 0 11 8
|
||||
5 15 2 14 1 13 0 12 8
|
||||
5 14 1 13 0 12 0 11 8
|
||||
5 15 1 14 0 13 0 12 8
|
||||
5 13 1 12 0 11 0 10 8
|
||||
5 12 2 11 1 10 0 9 8
|
||||
5 14 1 13 0 12 0 11 8
|
||||
5 15 1 14 0 13 0 12 8
|
||||
5 14 1 13 0 12 0 11 8
|
||||
5 14 2 13 1 12 0 11 8
|
||||
5 15 1 14 0 13 0 12 8
|
||||
5 15 2 14 1 13 0 12 8
|
||||
5 16 1 15 0 14 0 13 8
|
||||
5 13 1 12 0 11 0 10 8
|
||||
5 16 2 15 1 14 0 13 8
|
||||
5 15 1 14 0 13 0 12 8
|
||||
5 16 1 15 0 14 0 13 8
|
||||
5 14 2 13 1 12 0 11 8
|
||||
5 16 1 15 0 14 0 13 8
|
||||
5 15 2 14 1 13 0 12 8
|
||||
5 13 2 12 1 11 0 10 8
|
||||
5 16 1 15 0 14 0 13 8
|
||||
5 13 2 12 1 11 0 10 8
|
||||
5 13 1 12 0 11 0 10 8
|
||||
5 14 1 13 0 12 0 11 8
|
||||
5 14 2 13 1 12 0 11 8
|
||||
5 15 1 14 0 13 0 12 8
|
||||
5 14 2 13 1 12 0 11 8
|
||||
5 13 2 12 1 11 0 10 8
|
||||
5 14 2 13 1 12 0 11 8
|
||||
5 15 2 14 1 13 0 12 8
|
||||
5 15 1 14 0 13 0 12 8
|
||||
5 16 1 15 0 14 0 13 8
|
||||
5 16 1 15 0 14 0 13 8
|
||||
5 13 2 12 1 11 0 10 8
|
||||
5 15 1 14 0 13 0 12 8
|
||||
5 15 1 14 0 13 0 12 8
|
||||
5 16 1 15 0 14 0 13 8
|
||||
5 15 2 14 1 13 0 12 8
|
||||
5 16 2 15 1 14 0 13 8
|
||||
5 16 1 15 0 14 0 13 8
|
||||
5 16 2 15 1 14 0 13 8
|
||||
5 14 1 13 0 12 0 11 8
|
||||
5 13 1 12 0 11 0 10 8
|
||||
5 15 1 14 0 13 0 12 8
|
||||
5 14 2 13 1 12 0 11 8
|
||||
5 16 1 15 0 14 0 13 8
|
||||
5 17 1 16 0 15 0 14 8
|
||||
5 16 2 15 1 14 0 13 8
|
||||
5 14 1 13 0 12 0 11 8
|
||||
5 13 2 12 1 11 0 10 8
|
||||
5 14 2 13 1 12 0 11 8
|
||||
5 14 2 13 1 12 0 11 8
|
||||
5 15 2 14 1 13 0 12 8
|
||||
5 15 2 14 1 13 0 12 8
|
||||
SCORES_TXT_END_0
|
24
contrib/promix/test_main.py
Executable file
24
contrib/promix/test_main.py
Executable file
@ -0,0 +1,24 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
import unittest
|
||||
|
||||
from main import *
|
||||
|
||||
class TestConfig(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.config = Config()
|
||||
|
||||
def test_files_short(self):
|
||||
nbests = ["nbest1", "nbest2", "nbest3"]
|
||||
scores = ["score1", "score2", "score3"]
|
||||
self.config.parse(["-n", nbests[0], "-n", nbests[1], "-n", nbests[2],\
|
||||
"-S", scores[0], "-S", scores[1], "-S", scores[2]])
|
||||
self.assertEqual(self.config.nbest_files, nbests)
|
||||
self.assertEqual(self.config.score_files, scores)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
|
||||
suite = unittest.TestLoader().loadTestsFromTestCase(TestConfig)
|
124
contrib/promix/test_nbest.py
Executable file
124
contrib/promix/test_nbest.py
Executable file
@ -0,0 +1,124 @@
|
||||
#!/usr/bin/env python
|
||||
# coding=utf8
|
||||
|
||||
import numpy as np
|
||||
import numpy.testing as nptest
|
||||
import os.path
|
||||
import unittest
|
||||
|
||||
from nbest import *
|
||||
|
||||
class TestReadNBest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.nbest = "test_data/test.nbest.nbest"
|
||||
self.nbest_segment = "test_data/test.nbest.nbest.segments"
|
||||
self.scores = "test_data/test.nbest.scores"
|
||||
self.input = "test_data/test.nbest.input"
|
||||
|
||||
def test_featureindex(self):
|
||||
for nbest in get_scored_nbests(self.nbest,self.scores,self.input):
|
||||
pass
|
||||
self.assertEqual(get_feature_index("tm"), [9,14])
|
||||
self.assertEqual(get_feature_index("lm"), [7,8])
|
||||
self.assertEqual(get_feature_index("d"), [0,7])
|
||||
self.assertEqual(get_feature_index("w"), [8,9])
|
||||
|
||||
def test_nosegment(self):
|
||||
count = 0
|
||||
for nbest in get_scored_nbests(self.nbest,self.scores,self.input):
|
||||
count += 1
|
||||
hyp0 = nbest.hyps[0]
|
||||
expected_fv = np.array([0,-1.51037,0,0,-2.60639,0,0 ,-36.0562, -8,-5.97082,-14.8327,-2.41162,-9.32734,3.99959])
|
||||
self.assertEqual(len(hyp0.fv), len(expected_fv))
|
||||
for i in range(len(hyp0.fv)):
|
||||
self.assertAlmostEqual(expected_fv[i],hyp0.fv[i])
|
||||
self.assertEqual(hyp0.text,"taming politicians on both sides of the Atlantic")
|
||||
self.assertEqual(count,1)
|
||||
|
||||
def test_segment(self):
|
||||
count = 0
|
||||
for nbest in get_scored_nbests(self.nbest_segment,self.scores,self.input, segments=True):
|
||||
count += 1
|
||||
hyp0 = nbest.hyps[0]
|
||||
self.assertEqual(hyp0.text,"taming politicians on both sides of the Atlantic")
|
||||
expected_align = [(0,2,1), (2,4,2), (4,6,4), (6,9,8)]
|
||||
self.assertEqual(hyp0.alignment, expected_align)
|
||||
self.assertEqual(count,1)
|
||||
|
||||
class TestMosesPhraseScorer(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.scorer = MosesPhraseScorer\
|
||||
(("test_data/esen.nc.model.filtered/phrase-table.0-0.1.1", \
|
||||
"test_data/esen.ep.model.filtered/phrase-table.0-0.1.1"))
|
||||
|
||||
def test_phrase_scores(self):
|
||||
hyp0 = Hypothesis("taming |0-1| politicians |2-3| on both |4-5| sides of the Atlantic |6-8|", [0, -1.51037,0, 0, -2.60639, 0, 0, -36.0562,-8,-5.97082,-14.8327,-2.41162,\
|
||||
-9.32734,3.99959], True)
|
||||
hyp0.input_line = "domando a los políticos en ambos lados del Atlántico"
|
||||
#hyp0.score = 0.2140
|
||||
self.scorer.add_scores(hyp0)
|
||||
self.assertEqual(len(hyp0.phrase_scores),2)
|
||||
# Each ttable should provide 4 sets of 4 scores (ignore penalty)
|
||||
# These are the probabilities
|
||||
# nc first, then ep. Columns are different features
|
||||
expected = np.array([\
|
||||
[[0.0740741,0.00756144,1,0.500047],\
|
||||
[0.545866,0.233725,0.818336,0.186463],\
|
||||
[0.288344,0.0548148,0.335714,0.0543585],\
|
||||
[0.777778,0.0122444,0.777778,0.0351361]],\
|
||||
[[0,0,0,0],\
|
||||
[0.33497, 0.180441, 0.638586, 0.0962213],\
|
||||
[0.0908379,0.0213197,0.187399,0.0498198],
|
||||
[0.62585,0.00702384,0.836364,0.0687874]]\
|
||||
])
|
||||
|
||||
nptest.assert_almost_equal(hyp0.phrase_scores, expected)
|
||||
|
||||
# These are the interpolation weights reported by tmcombine
|
||||
weights = np.array([[0.54471993730312251, 0.45528006269687754],\
|
||||
[0.56546688367708142, 0.43453311632291863],\
|
||||
[0.55867730373453584, 0.44132269626546422],\
|
||||
[0.46645964485220004, 0.53354035514779996]])
|
||||
|
||||
#check that the scores are interpolated as expected
|
||||
interpolated_probs = expected[0]*weights[:,0] + expected[1]*weights[:,1]
|
||||
interpolated_scores = np.log(interpolated_probs)
|
||||
# each column corresponds to a feature
|
||||
expected_fv = interpolated_scores.sum(axis=0)
|
||||
for i in range(4):
|
||||
self.assertAlmostEqual(hyp0.fv[9+i], expected_fv[i], places=4)
|
||||
|
||||
class TestPhraseCache (unittest.TestCase):
|
||||
def test_add_get(self):
|
||||
"""Add something to cache and check we can get it back"""
|
||||
cache = PhraseCache(10)
|
||||
self.assertFalse(cache.get("aa", "bb"))
|
||||
scores = [1,2,3,4,5]
|
||||
cache.add("aa", "bb", scores)
|
||||
self.assertEquals(cache.get("aa","bb"),scores)
|
||||
self.assertFalse(cache.get("aa","cc"))
|
||||
self.assertFalse(cache.get("cc","bb"))
|
||||
|
||||
def test_lru(self):
|
||||
"""Check that items are cleared from the cache"""
|
||||
cache = PhraseCache(2)
|
||||
s1 = [1,2,3,4,5]
|
||||
s2 = [2,3,4,5,6]
|
||||
s3 = [3,4,5,6,7]
|
||||
cache.add("aa","bb",s1)
|
||||
cache.add("bb","cc",s2)
|
||||
cache.add("dd","ee",s3)
|
||||
self.assertEquals(cache.get("dd","ee"), s3)
|
||||
self.assertEquals(cache.get("bb","cc"), s2)
|
||||
self.assertFalse(cache.get("aa","bb"))
|
||||
cache.add("aa","bb",s1)
|
||||
self.assertFalse(cache.get("dd","ee"))
|
||||
|
||||
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
suite = unittest.TestSuite([unittest.TestLoader().loadTestsFromTestCase(TestReadNBest), unittest.TestLoader().loadTestsFromTestCase(TestMosesPhraseScorer)])
|
53
contrib/promix/test_sampler.py
Executable file
53
contrib/promix/test_sampler.py
Executable file
@ -0,0 +1,53 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
import random
|
||||
import unittest
|
||||
|
||||
from nbest import *
|
||||
from sampler import *
|
||||
|
||||
|
||||
class TestNBestSampler(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.h1 = Hypothesis("a",[])
|
||||
self.h2 = Hypothesis("b",[])
|
||||
self.h3 = Hypothesis("c",[])
|
||||
self.nbest = NBestList(1)
|
||||
self.nbest.hyps.append(self.h1)
|
||||
self.nbest.hyps.append(self.h2)
|
||||
self.nbest.hyps.append(self.h3)
|
||||
self.sampler = HopkinsMaySampler()
|
||||
|
||||
def test_nsamples(self):
|
||||
self.h1.score = 0.1
|
||||
self.h2.score = 0.2
|
||||
self.h3.score = 0.3
|
||||
samples = self.sampler.sample(self.nbest)
|
||||
self.assertEqual(len(samples), self.sampler.nsamples)
|
||||
|
||||
def test_biggest(self):
|
||||
random.seed(0)
|
||||
self.h1.score = 0.1
|
||||
self.h2.score = 0.2
|
||||
self.h3.score = 0.3
|
||||
samples = self.sampler.sample(self.nbest)
|
||||
for sample in samples:
|
||||
self.assertAlmostEqual(sample.diff,0.2)
|
||||
|
||||
def test_score_diff(self):
|
||||
self.h1.score = 0.1
|
||||
self.h2.score = 0.1 + (0.9*self.sampler.min_diff)
|
||||
self.h3.score = 0.1 + (1.8*self.sampler.min_diff)
|
||||
|
||||
# Should only see pairs with h1,h3
|
||||
samples = self.sampler.sample(self.nbest)
|
||||
for sample in samples:
|
||||
self.assertTrue((sample.hyp1 == self.h1 and sample.hyp2 == self.h3) or \
|
||||
(sample.hyp2 == self.h1 and sample.hyp1 == self.h3))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
suite = unittest.TestLoader().loadTestsFromTestCase(TestNBestSampler)
|
306
contrib/promix/test_train.py
Executable file
306
contrib/promix/test_train.py
Executable file
@ -0,0 +1,306 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
import unittest
|
||||
|
||||
from math import log,exp
|
||||
|
||||
import nbest
|
||||
import numpy.testing as nptest
|
||||
import sampler
|
||||
from train import *
|
||||
import util
|
||||
|
||||
class TestParabaloidOptimiser(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.o = ParabaloidOptimiser(np.array([1,2,3,4]))
|
||||
|
||||
def test_parabaloid_bfgs(self):
|
||||
start = np.array([2,2,2,2])
|
||||
minimum = self.o.optimise_bfgs(start)
|
||||
for m in minimum:
|
||||
self.assertAlmostEqual(m,0)
|
||||
|
||||
|
||||
def test_parabaloid_lbfgs(self):
|
||||
start = np.array([2,2,2,2])
|
||||
minimum = self.o.optimise_lbfgs(start)
|
||||
for m in minimum:
|
||||
self.assertAlmostEqual(m,0)
|
||||
|
||||
class TestLogisticRegressionOptimiser(unittest.TestCase):
|
||||
|
||||
def test_objective(self):
|
||||
x = np.array([[1], [0]])
|
||||
y = np.array([1,-1])
|
||||
lro = LogisticRegressionOptimiser(x,y)
|
||||
w = np.array([2])
|
||||
expected = -log(1 / (1 + exp(-2))) - log(0.5)
|
||||
self.assertAlmostEqual(lro.objective(w), expected)
|
||||
|
||||
def test_reg_objective(self):
|
||||
x = np.array([[1], [0]])
|
||||
y = np.array([1,-1])
|
||||
alpha = 0.1
|
||||
lro = LogisticRegressionOptimiser(x,y,alpha)
|
||||
w = np.array([2])
|
||||
expected = -log(1 / (1 + exp(-2))) - log(0.5) + 0.5*2*2 * alpha
|
||||
self.assertAlmostEqual(lro.objective(w), expected)
|
||||
|
||||
def test_gradient_j(self):
|
||||
x = np.array([[1], [0]])
|
||||
y = np.array([1,-1])
|
||||
lro = LogisticRegressionOptimiser(x,y)
|
||||
w = np.array([2])
|
||||
expected = -1 / (1 + exp(2))
|
||||
self.assertAlmostEqual(lro.grad_j(w,0), expected)
|
||||
|
||||
def test_gradient(self):
|
||||
x = np.array([[1,1], [0,1]])
|
||||
y = np.array([1,-1])
|
||||
w = np.array([2,1])
|
||||
lro = LogisticRegressionOptimiser(x,y)
|
||||
e0 = -1 / (1 + exp(3))
|
||||
e1 = -1 / (1 + exp(3)) + 1/ (1 + exp(-1))
|
||||
actual = lro.grad(w)
|
||||
#print "expected: ",e0,e1
|
||||
self.assertAlmostEqual(actual[0], e0)
|
||||
self.assertAlmostEqual(actual[1], e1)
|
||||
|
||||
def test_reg_gradient(self):
|
||||
x = np.array([[1,1], [0,1]])
|
||||
y = np.array([1,-1])
|
||||
alpha = 0.2
|
||||
w = np.array([2,1])
|
||||
lro = LogisticRegressionOptimiser(x,y, alpha)
|
||||
e0 = -1 / (1 + exp(3)) + w[0]*alpha
|
||||
e1 = -1 / (1 + exp(3)) + 1/ (1 + exp(-1)) +w[1]*alpha
|
||||
actual = lro.grad(w)
|
||||
self.assertAlmostEqual(actual[0], e0)
|
||||
self.assertAlmostEqual(actual[1], e1)
|
||||
|
||||
|
||||
def test_train(self):
|
||||
x = np.array([[1,1],[-1,-2]])
|
||||
y = np.array([1,-1])
|
||||
w0 = np.array([1,-1])
|
||||
lro = LogisticRegressionOptimiser(x,y)
|
||||
actual = lro.train(w0, debug=False)
|
||||
self.assertAlmostEqual(actual[0], 12.03882542)
|
||||
self.assertAlmostEqual(actual[1], 8.02317419)
|
||||
|
||||
def test_train_reg(self):
|
||||
x = np.array([[1,1],[-1,1]])
|
||||
y = np.array([1,-1])
|
||||
alpha = 0.1
|
||||
w0 = np.array([1,-1])
|
||||
lro = LogisticRegressionOptimiser(x,y,alpha)
|
||||
actual = lro.train(w0, debug=False)
|
||||
self.assertAlmostEqual(actual[1],0) # 2nd input should be ignored
|
||||
# classify first example as negative, second as positive
|
||||
self.assertTrue(1 / (1+exp(-np.dot(actual,np.array([1,1])))) > 0.5)
|
||||
self.assertTrue(1 / (1+exp(-np.dot(actual,np.array([-1,-2])))) < 0.5)
|
||||
|
||||
def test_xy(self):
|
||||
"""Test pre-calculation of the y_i*x_ij vectors"""
|
||||
x = np.array([[1,3], [2,8], [1,3]])
|
||||
y = np.array([1,1,-1])
|
||||
lro = LogisticRegressionOptimiser(x,y)
|
||||
expected = np.array([[1,3], [2,8], [-1,-3]])
|
||||
for i in 0,1,2:
|
||||
for j in 0,1:
|
||||
self.assertEqual(lro.xy[i][j], expected[i][j])
|
||||
#
|
||||
class TestMixtureModelTrainer(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
# 3 phrase table features, but last one is ignored for interpolation
|
||||
nbest._feature_index = {"tm" : [0,3], "lm" : [3,4]}
|
||||
log05 = np.log(0.5)
|
||||
log03 = np.log(0.3)
|
||||
log02 = np.log(0.2)
|
||||
log01 = np.log(0.1)
|
||||
hyp0 = nbest.Hypothesis("a |0-0| b c |1-2|", [log05, log05, log02, log03], True)
|
||||
hyp0.input_line = "A B C"
|
||||
hyp0.score = 3
|
||||
# Two ttables, columns correspond to features, rows to phrase pairs
|
||||
hyp0.phrase_scores = np.array([\
|
||||
[[0.2, 0.3],\
|
||||
[0.4, 0.3]],\
|
||||
[[0, 0.2],\
|
||||
[0.4,0.2]]])
|
||||
|
||||
hyp1 = nbest.Hypothesis("x |0-2|", [log02, log03, log03, log01], True)
|
||||
hyp1.input_line = "X Y Z"
|
||||
hyp1.score = 2
|
||||
hyp1.phrase_scores = np.array([\
|
||||
[[0.1, 0.1]],\
|
||||
[[0.8,0.1]]])
|
||||
|
||||
hyp2 = nbest.Hypothesis("z |0-1| w |2-2| p |3-3|", [log02, log02, log05, log05], True)
|
||||
hyp2.score = 1
|
||||
hyp2.input_line = "M N O"
|
||||
# phrase_table x phrase_pair x feature
|
||||
hyp2.phrase_scores = np.array([\
|
||||
[[0.1, 0.2],\
|
||||
[0.3,0.5],\
|
||||
[0.4,0.6]],\
|
||||
[[0.1,0.5],\
|
||||
[0.6,0.1],\
|
||||
[0.2,0.2]]])
|
||||
self.samples = [sampler.Sample(hyp0,hyp1), sampler.Sample(hyp1,hyp2)]
|
||||
self.trainer = MixtureModelTrainer(self.samples)
|
||||
|
||||
def get_phrase_scores(self, hypothesis, iw):
|
||||
nptest.assert_almost_equal(np.sum(iw, axis=0), np.array([1.0,1.0]))
|
||||
phrase_probs = hypothesis.phrase_scores
|
||||
interpolated_probs = np.sum(np.expand_dims(iw,1)*phrase_probs, axis = 0)
|
||||
|
||||
total_probs = np.prod(interpolated_probs, axis = 0)
|
||||
return util.safelog(total_probs)
|
||||
|
||||
def model_score(self, hypothesis, weights):
|
||||
# interpolation weights
|
||||
# ttable x feature
|
||||
iw = np.array([[weights[-2], weights[-1]],
|
||||
[1-weights[-2],1-weights[-1]]])
|
||||
#print "iw:",iw
|
||||
phrase_scores = self.get_phrase_scores(hypothesis,iw)
|
||||
weighted_phrase_scores = weights[:2] * phrase_scores
|
||||
score = np.sum(weighted_phrase_scores)
|
||||
|
||||
other_score = np.sum(weights[2:4]*hypothesis.fv[2:4])
|
||||
return score + other_score
|
||||
|
||||
|
||||
def test_objective(self):
|
||||
# 2 phrase weights, 2 other feature weights,
|
||||
# 2 interpolation weights (1 per model x 2 phrase features)
|
||||
weights = np.array([0.2,0.1,0.4,0.5,0.3,0.6])
|
||||
actual = self.trainer.objective(weights)
|
||||
# Expected objective is the sum of the logs of sigmoids of the score differences
|
||||
# Weighted by 1 if hyp1 > hyp2, -1 otherwise
|
||||
expected = 0
|
||||
for sample in self.samples:
|
||||
hyp1_model_score = self.model_score(sample.hyp1, weights)
|
||||
hyp2_model_score = self.model_score(sample.hyp2, weights)
|
||||
y = 1
|
||||
if sample.hyp2.score > sample.hyp1.score: y = -1
|
||||
expected -= log(sigmoid(y * (hyp1_model_score - hyp2_model_score)))
|
||||
# regularisation
|
||||
expected += 0.5 * self.trainer.alpha * np.dot(weights[:-2], weights[:-2])
|
||||
self.assertAlmostEquals(actual,expected)
|
||||
|
||||
def test_gradient_other(self):
|
||||
# Gradients are just differences in feature vectors
|
||||
# fv(hypo0)-fv(hyp1), fv(hyp1)-fv(hyp2)
|
||||
delta_s = np.vstack((self.samples[0].hyp1.fv-self.samples[0].hyp2.fv,\
|
||||
self.samples[1].hyp1.fv-self.samples[1].hyp2.fv))
|
||||
# feature functions across rows, samples down columns
|
||||
# choose other features
|
||||
other_delta_s = delta_s[:,2:]
|
||||
actual = self.trainer.gradient_other()
|
||||
nptest.assert_almost_equal(actual,other_delta_s)
|
||||
|
||||
def test_gradient_phrase(self):
|
||||
iw = np.array([[0.3, 0.4],[0.7,0.6]])
|
||||
sample_deltaf_list = []
|
||||
for sample in self.samples:
|
||||
f_A = self.get_phrase_scores(sample.hyp1, iw)
|
||||
f_B = self.get_phrase_scores(sample.hyp2, iw)
|
||||
sample_deltaf_list.append(f_A - f_B)
|
||||
expected = np.vstack(sample_deltaf_list) # samples down, features along
|
||||
actual = self.trainer.gradient_phrase(iw)
|
||||
nptest.assert_almost_equal(actual,expected)
|
||||
|
||||
def test_gradient_interp(self):
|
||||
# The interpolation weights - ttable x feature
|
||||
iw = np.array([[0.3, 0.4],[0.7,0.6]])
|
||||
phrasew = np.array([1,2]) # The phrase weights
|
||||
num_ttables = iw.shape[0]
|
||||
num_phrase_features = iw.shape[1]
|
||||
bysample_list = []
|
||||
# Stack up gradients for each sample
|
||||
for sample in self.samples:
|
||||
# Get the gradient of the interpolation weights for each
|
||||
# hypothesis (A and B) in the sample
|
||||
byhyp = []
|
||||
for hyp in [sample.hyp1,sample.hyp2]:
|
||||
# the weights are flattened. rows of iw joined together, last row omitted
|
||||
grad_k = np.array([0.0] * ((num_ttables - 1) * num_phrase_features))
|
||||
# Iterate through the phrase features
|
||||
for j,probs in enumerate(np.transpose(hyp.phrase_scores)):
|
||||
# j is phrase feature index
|
||||
# probs is phrase-pair, ttable
|
||||
grad_jk = np.array([0.0] * (len(iw)-1))
|
||||
for l,phi in enumerate(probs):
|
||||
# For each phrase-pair the gradient term for the lambda
|
||||
# is the probability for this ttable - probability for last ttable
|
||||
# divided by overall phrase probability
|
||||
num = phi[:-1] - phi[-1]
|
||||
denom = np.sum(iw[:,j]*phi) # use interpolation weights for this feature
|
||||
grad_jk = grad_jk + (num/denom)
|
||||
self.assertEquals(len(grad_jk), num_ttables-1)
|
||||
#print "num",num,"denom",denom,"grad_jk",grad_jk
|
||||
# add gradient in correct place
|
||||
#print "\n",j,grad_k,phrasew[j]*grad_jk
|
||||
grad_k[j*(num_ttables-1):(j+1)*(num_ttables-1)] =\
|
||||
grad_k[j*(num_ttables-1):(j+1)*(num_ttables-1)] + phrasew[j]*grad_jk
|
||||
#print "\ngrad_k",grad_k
|
||||
byhyp.append(grad_k)
|
||||
bysample_list.append(byhyp[0]-byhyp[1])
|
||||
#print "diff: ", bysample_list[-1]
|
||||
expected = np.vstack(bysample_list)
|
||||
actual = self.trainer.gradient_interp(iw,phrasew)
|
||||
nptest.assert_almost_equal(actual,expected, decimal=5)
|
||||
|
||||
def test_gradient(self):
|
||||
# 2 phrase weights, 2 other feature weights,
|
||||
# 2 interpolation weights (2 models and 2 tables)
|
||||
weights = np.array([0.2,0.1,0.4,0.5,0.6,0.3])
|
||||
expected = np.array([0.0] * len(weights))
|
||||
# Get the gradients
|
||||
iw = np.array([[weights[-2], weights[-1]],
|
||||
[1-weights[-2],1-weights[-1]]])
|
||||
phrase_g = self.trainer.gradient_phrase(iw)
|
||||
other_g = self.trainer.gradient_other()
|
||||
interp_g = self.trainer.gradient_interp(iw,weights[:2])
|
||||
for k,sample in enumerate(self.samples):
|
||||
hyp1_model_score = self.model_score(sample.hyp1, weights)
|
||||
hyp2_model_score = self.model_score(sample.hyp2, weights)
|
||||
y = 1
|
||||
if sample.hyp2.score > sample.hyp1.score: y = -1
|
||||
delta_score = hyp1_model_score - hyp2_model_score
|
||||
sig_delta_score = sigmoid(-y * delta_score)
|
||||
# phrase derivative term
|
||||
expected[:2] -= (phrase_g[k]*sig_delta_score*y)
|
||||
# other derivative term
|
||||
expected[2:4] -= (other_g[k]*sig_delta_score*y)
|
||||
# inter derivative term
|
||||
expected[-2:] -= (interp_g[k]*sig_delta_score*y)
|
||||
expected += self.trainer.alpha*np.append(weights[:-2], np.array([0.0,0.0]))
|
||||
actual = self.trainer.gradient(weights)
|
||||
nptest.assert_almost_equal(actual,expected)
|
||||
|
||||
def test_split_weights(self):
|
||||
w = np.array([1,2,3,4,0.2,0.3])
|
||||
sw = self.trainer.get_split_weights(w)
|
||||
self.assertEquals(len(sw),3)
|
||||
nptest.assert_almost_equal(sw['phrase'], np.array([1,2]))
|
||||
nptest.assert_almost_equal(sw['other'], np.array([3,4]))
|
||||
nptest.assert_almost_equal(sw['interp'], \
|
||||
np.array([[0.2,0.3], [0.8,0.7]]))
|
||||
|
||||
|
||||
def test_train(self):
|
||||
"""Simple test that it runs without errors"""
|
||||
print "x=",self.trainer.train()
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
suite = unittest.TestSuite([
|
||||
unittest.TestLoader().loadTestsFromTestCase(TestParabaloidOptimiser),
|
||||
unittest.TestLoader().loadTestsFromTestCase(TestLogisticRegressionOptimiser),
|
||||
unittest.TestLoader().loadTestsFromTestCase(TestMixtureModelTrainer)])
|
||||
|
||||
|
412
contrib/promix/train.py
Executable file
412
contrib/promix/train.py
Executable file
@ -0,0 +1,412 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
#
|
||||
# Train the model weights
|
||||
#
|
||||
|
||||
from math import log,exp
|
||||
import sys
|
||||
import numpy as np
|
||||
from scipy.optimize.optimize import fmin_cg, fmin_bfgs, fmin
|
||||
from scipy.optimize.lbfgsb import fmin_l_bfgs_b
|
||||
|
||||
import nbest
|
||||
from util import safelog
|
||||
|
||||
def sigmoid(x):
|
||||
return 1.0 / (1.0 + np.exp(-x))
|
||||
|
||||
class OptimisationException(Exception):
|
||||
pass
|
||||
|
||||
class ParabaloidOptimiser:
|
||||
"""Optimises a very simple function, to test scipy"""
|
||||
def __init__(self, params):
|
||||
self.params = params
|
||||
|
||||
def objective(self,x):
|
||||
return np.sum(x*x*self.params*self.params)
|
||||
|
||||
def grad_k(self,x,k):
|
||||
return 2 * self.params[k]**2 * x[k]
|
||||
|
||||
def grad(self,x):
|
||||
return np.array([self.grad_k(x,k) for k in range(len(x))])
|
||||
|
||||
def debug(self,x):
|
||||
print "x = ",x
|
||||
|
||||
|
||||
def optimise_bfgs(self,start):
|
||||
print
|
||||
print "***** BFGS OPTIMISATION *****"
|
||||
return fmin_bfgs(self.objective, start, fprime=self.grad, callback=self.debug)
|
||||
|
||||
def optimise_lbfgs(self,start):
|
||||
print
|
||||
print "***** LBFGS OPTIMISATION *****"
|
||||
x,f,d = fmin_l_bfgs_b(self.objective, start, fprime=self.grad, pgtol=1e-09, iprint=0)
|
||||
return x
|
||||
|
||||
class LRDataException(Exception):
|
||||
pass
|
||||
|
||||
class LogisticRegressionOptimiser:
|
||||
"""Optimise logistic regression weights"""
|
||||
def __init__(self,x,y, alpha = 0):
|
||||
"""Training data (x) should be vector of feature vectors, and
|
||||
corresponding vector of outputs (with values -1,1).
|
||||
alpha controls the L2-normalisation"""
|
||||
self.x = x
|
||||
self.y = y
|
||||
self.alpha = alpha
|
||||
if len(x) != len(y): raise LRDataException("Lengths of input and response don't match")
|
||||
if len(x) == 0: raise LRDataException("Data set is empty")
|
||||
# Precalculate {y_i*x_ij} for all j
|
||||
self.xy = x*y[:,None]
|
||||
|
||||
def objective(self,w):
|
||||
"""Calculate the value of the negative log-likelihood for a given weight set"""
|
||||
l = 0
|
||||
for i in range(len(self.x)):
|
||||
# Each example contributes log(sigma(y_i * x_i . w))
|
||||
l -= log(sigmoid(self.y[i] * np.dot(w, self.x[i,:])))
|
||||
# regularisation 1/2 * alpha * ||w||^2
|
||||
l += 0.5 * self.alpha * np.dot(w,w)
|
||||
return l
|
||||
|
||||
def grad_j(self,w,j):
|
||||
"""Gradient of the objective in the jth direction for given weight set"""
|
||||
g = 0
|
||||
for i in range(len(self.x)):
|
||||
# Each example contributes -sigma(-y_i * x_i.w) * y_j x_ij
|
||||
g -= sigmoid(-self.y[i] * np.dot(w, self.x[i,:])) * self.y[i] * self.x[i,j]
|
||||
#regularisation
|
||||
g += self.alpha * w[j]
|
||||
return g
|
||||
|
||||
def grad(self,w):
|
||||
"""Gradient of objective at given weight set - returns a vector"""
|
||||
# Calculate the vector -sigma(-y_i * x_i.w)
|
||||
s = -np.array([sigmoid(-yi * np.dot(xi,w)) for xi,yi in zip(self.x,self.y)])
|
||||
# Multiply it by xy
|
||||
g = np.array([np.dot(xyj,s) for xyj in self.xy.transpose()])
|
||||
# Add regularisation
|
||||
g += self.alpha*w
|
||||
return g
|
||||
#g = np.array([self.grad_j(w,j) for j in xrange(len(w))])
|
||||
|
||||
|
||||
def train(self,w0,debug=False):
|
||||
if debug:
|
||||
iprint = 0
|
||||
else:
|
||||
iprint = -1
|
||||
x,f,d = fmin_l_bfgs_b(self.objective, w0, fprime=self.grad, pgtol=1e-09, iprint=iprint)
|
||||
if d['warnflag'] != 0:
|
||||
raise OptimisationException(d['task'])
|
||||
return x
|
||||
|
||||
class ProTrainer:
|
||||
"""Turns the samples into a logistic regression problem"""
|
||||
def __init__(self,samples):
|
||||
self.samples = samples
|
||||
self.alpha = 1
|
||||
self.dims = len(samples[0].hyp1.fv)
|
||||
|
||||
|
||||
def train(self, debug=False):
|
||||
x = np.array([s.hyp1.fv-s.hyp2.fv for s in self.samples])
|
||||
#print x
|
||||
y = np.array([cmp(s.hyp1.score,s.hyp2.score) for s in self.samples])
|
||||
#print y
|
||||
lro = LogisticRegressionOptimiser(x,y,self.alpha)
|
||||
w0 = np.zeros(self.dims)
|
||||
w = lro.train(w0,debug)
|
||||
w = w/np.sum(abs(w)) # L_1 normalise
|
||||
return w,[]
|
||||
|
||||
class MixtureModelTrainer:
|
||||
"""Trains the phrase mixture weights, as well as the regular feature weights"""
|
||||
def __init__(self,samples):
|
||||
self.alpha = 1
|
||||
self.interp_floor = 0.001 # minimum value for interpolation weight
|
||||
#self.prob_floor = 0.00000001 # floor probabilities at this value
|
||||
#self.weight_bounds = (-10,10) # bounds for other features
|
||||
# The phrase scores are joined into a 5d array, where the dimensions are:
|
||||
# sample, hyp1 or hyp2, ttable, phrase-pair, feature
|
||||
# ie the feature is the last dimension
|
||||
# Actually phrase_probs is a 2-dim list of 3-dim arrays, since it's ragged
|
||||
self.phrase_probs = \
|
||||
[[sample.hyp1.phrase_scores,sample.hyp2.phrase_scores]\
|
||||
for sample in samples]
|
||||
#[[sample.hyp1.phrase_scores.clip(self.prob_floor),sample.hyp2.phrase_scores.clip(self.prob_floor)]\
|
||||
|
||||
# Figure out where the weights are
|
||||
self.phrase_index = list(nbest.get_feature_index("tm"))
|
||||
self.phrase_index[1] = self.phrase_index[1]-1 # phrase penalty not interpolated
|
||||
|
||||
interp_length = (self.phrase_index[1]-self.phrase_index[0]) * \
|
||||
(len(samples[0].hyp1.phrase_scores)-1)
|
||||
weight_length = len(samples[0].hyp1.fv) + interp_length
|
||||
self.interp_index = [weight_length - interp_length,weight_length]
|
||||
#print self.interp_index
|
||||
self.other_index = [[0,self.phrase_index[0]],[self.phrase_index[1],self.interp_index[0]]]
|
||||
|
||||
# join the feature vector diffs for the other fvs into a 2d array
|
||||
# features across, samples down
|
||||
self.fvs = np.array(\
|
||||
[np.append(sample.hyp1.fv[self.other_index[0][0]:self.other_index[0][1]],
|
||||
sample.hyp1.fv[self.other_index[1][0]:self.other_index[1][1]]) - \
|
||||
np.append(sample.hyp2.fv[self.other_index[0][0]:self.other_index[0][1]],\
|
||||
sample.hyp2.fv[self.other_index[1][0]:self.other_index[1][1]])\
|
||||
for sample in samples])
|
||||
|
||||
self.cached_iw = None
|
||||
self.cached_interpolated_phrase_probs = None
|
||||
|
||||
self.cached_sw = None
|
||||
self.cached_y_times_diffs = None
|
||||
|
||||
|
||||
|
||||
# join the responses (y's) into an array
|
||||
# If any pairs have equal score, this sets y=0, an invalid response.
|
||||
# but the sampling should ensure that this doesn't happen
|
||||
self.y = np.array([cmp(sample.hyp1.score, sample.hyp2.score)\
|
||||
for sample in samples])
|
||||
|
||||
def get_split_weights(self,weights):
|
||||
"""Map containing all the different weight sets:
|
||||
phrase - phrase feature weights (excluding penalty)
|
||||
other - other feature weights
|
||||
interp - interpolation weights: ttable x feature
|
||||
"""
|
||||
sw = {}
|
||||
sw['phrase'] = weights[self.phrase_index[0]:self.phrase_index[1]]
|
||||
sw['interp'] = weights[self.interp_index[0]:self.interp_index[1]]
|
||||
sw['interp'] = sw['interp'].T.reshape\
|
||||
(( len(sw['interp']) / len(sw['phrase'])), len(sw['phrase']))
|
||||
# Add normalisations
|
||||
sw['interp'] = np.vstack((sw['interp'], 1.0 - np.sum(sw['interp'], axis=0)))
|
||||
#sw['interp'] = np.append(sw['interp'], 1 - np.sum(sw['interp']))
|
||||
sw['other'] = np.append(weights[self.other_index[0][0]:self.other_index[0][1]],
|
||||
weights[self.other_index[1][0]:self.other_index[1][1]])
|
||||
return sw
|
||||
|
||||
def get_interpolated_phrase_probs(self,iw):
|
||||
# Memoise
|
||||
if self.cached_iw == None or np.sum(np.abs(iw-self.cached_iw)) != 0:
|
||||
# iw is ttable x feature. Each element of phrase_probs is ttable x pair x feature
|
||||
iw_expanded = np.expand_dims(iw,1)
|
||||
# self.phrase probs is a 2-d list, so use python iteration
|
||||
interpolated = [ [iw_expanded*p for p in ps] for ps in self.phrase_probs]
|
||||
self.cached_interpolated_phrase_probs = np.sum(np.array(interpolated), axis = 2)
|
||||
self.cached_iw = iw
|
||||
return self.cached_interpolated_phrase_probs
|
||||
|
||||
def get_y_times_diffs(self,sw):
|
||||
""" Calculate the array y_k* \Delta S_k"""
|
||||
# Process the phrase scores first.
|
||||
# - for each phrase, interpolate across the ttables using the current weights
|
||||
# - sum the log probs across phrase pairs to get a score for each hypothesis
|
||||
# - take the weighted sum of these scores, to give a phrase feature total
|
||||
# for each hyp
|
||||
|
||||
# Memoise
|
||||
if self.cached_sw == None or \
|
||||
np.sum(np.abs(self.cached_sw['other'] - sw['other'])) != 0 or \
|
||||
np.sum(np.abs(self.cached_sw['phrase'] - sw['phrase'])) != 0 or \
|
||||
np.sum(np.abs(self.cached_sw['interp'] - sw['interp'])) != 0:
|
||||
|
||||
# do the interpolation
|
||||
iw = sw['interp']
|
||||
interpolated = self.get_interpolated_phrase_probs(iw)
|
||||
# Use traditional python as not sure how to vectorise. This goes through
|
||||
# each hypothesis, logs the probability, applies the feature weights, then sums
|
||||
self.cached_y_times_diffs = np.zeros(len(interpolated))
|
||||
# Take the difference between the hypotheses
|
||||
for i,sample in enumerate(interpolated):
|
||||
self.cached_y_times_diffs[i] = \
|
||||
np.sum(sw['phrase']* np.log(sample[0])) - \
|
||||
np.sum(sw['phrase']* np.log(sample[1]))
|
||||
#print self.fvs, sw['other']
|
||||
#print sw['other'], self.fvs
|
||||
self.cached_y_times_diffs += np.sum(sw['other'] * self.fvs, axis=1) # add other scores
|
||||
self.cached_y_times_diffs *= self.y
|
||||
self.cached_sw = sw
|
||||
return self.cached_y_times_diffs
|
||||
|
||||
def objective(self,w):
|
||||
"""The value of the objective with the given weight vector.
|
||||
The objective is the sum of the log of the sigmoid of the differences
|
||||
in scores between the two hypotheses times y.
|
||||
"""
|
||||
diffs = self.get_y_times_diffs(self.get_split_weights(w))
|
||||
#print diffs, sigmoid(diffs)
|
||||
obj = -np.sum(np.log(sigmoid(diffs))) #negative, since minimising
|
||||
# regularisation
|
||||
obj += 0.5 * self.alpha * np.dot(w[:self.interp_index[0]], w[:self.interp_index[0]])
|
||||
return obj
|
||||
|
||||
#
|
||||
# The following methods compute the derivatives of the score differences
|
||||
# with respect to each of the three types of weights. They should all
|
||||
# return an np.array, with features across, and samples down
|
||||
#
|
||||
|
||||
def gradient_phrase(self,interp):
|
||||
"""Compute the derivative of the score difference for the 'phrase' weights.
|
||||
|
||||
Args:
|
||||
interp: The interpolation weights
|
||||
"""
|
||||
# Compute the interpolated phrase probs
|
||||
interpolated = self.get_interpolated_phrase_probs(interp)
|
||||
|
||||
# for each sample, log and sum across phrases, then compute the feature value
|
||||
# difference for each sample.
|
||||
# TODO: Better vectorisation
|
||||
grad_list = []
|
||||
for i, sample in enumerate(interpolated):
|
||||
f_A = np.sum(np.log(sample[0]), axis=0)
|
||||
f_B = np.sum(np.log(sample[1]), axis=0)
|
||||
grad_list.append(f_A - f_B)
|
||||
return np.vstack(grad_list)
|
||||
|
||||
def gradient_interp(self,interp,phrase):
|
||||
"""Compute the derivative of the score difference for the 'interp' weights
|
||||
|
||||
Args:
|
||||
interp: All the interpolation weights. These will be in a 2-dim np array,
|
||||
where the dims are ttable x phrase feature. Note that there are k rows,
|
||||
one for each ttable, so the sum down the columns will be 1.
|
||||
phrase: The weights of the phrase features
|
||||
|
||||
Returns:
|
||||
A 2-d array, with samples down and gradients along. Note that in the gradients
|
||||
(rows) the interpolation weights are flattened out, and have the last ttable
|
||||
removed.
|
||||
"""
|
||||
num_interp_weights = (interp.shape[0]-1) * interp.shape[1]
|
||||
grad_list = np.empty((len(self.phrase_probs),num_interp_weights))
|
||||
expanded_interp = np.expand_dims(interp,1)
|
||||
def df_by_dlambda(phi):
|
||||
"""Derivative of phrase scores w.r.t. lambdas"""
|
||||
#print "Interp:", interp, "\nPhi", phi
|
||||
num = phi[:-1] - phi[-1]
|
||||
denom = np.sum(expanded_interp*phi, axis=0)
|
||||
# num is ttable x phrase-pair x feature
|
||||
# denom is phrase-pair x feature
|
||||
# divide, then sum across phrase-pairs
|
||||
#print "num",num,"denom",denom
|
||||
#print "q",num/denom
|
||||
quotient = np.sum(num/denom, axis =1)
|
||||
# quotient is ttable-1 x feature
|
||||
return quotient
|
||||
|
||||
|
||||
for k, sample in enumerate(self.phrase_probs):
|
||||
# derivative is the weighted sum of df_by_dlambda_A - df_by_dlambda_B
|
||||
#print "\nq0", df_by_dlambda(sample[0])
|
||||
#print "hyp0",np.sum(phrase * (df_by_dlambda(sample[0])), axis=0)
|
||||
#print "q1", df_by_dlambda(sample[1])
|
||||
#print "hyp1",np.sum(phrase * (df_by_dlambda(sample[1])), axis=0),"\n"
|
||||
#TODO: Check if the sum is required here. With 4 ttables and 4 features
|
||||
# it gives lhs as (12) and rhs as (4)
|
||||
grad_list[k] = (phrase * (df_by_dlambda(sample[0]) - df_by_dlambda(sample[1]))).flatten()
|
||||
#grad_list = np.vstack(grad_list)
|
||||
return grad_list
|
||||
|
||||
def gradient_other(self):
|
||||
"""Compute the derivative of the score difference for the 'other' weights.
|
||||
|
||||
Features across, samples down.
|
||||
"""
|
||||
# This is just the difference in the feature values
|
||||
return self.fvs
|
||||
|
||||
def gradient(self,w):
|
||||
sw = self.get_split_weights(w)
|
||||
sig_y_by_diffs = sigmoid(-self.get_y_times_diffs(sw))
|
||||
# These all return 2-d arrays, with samples dowm and features across.
|
||||
# NB: Both gradient_phrase and gradient_interp iterate through the samples,
|
||||
# so this is probably inefficient
|
||||
phrase_g = self.gradient_phrase(sw['interp'])
|
||||
interp_g = self.gradient_interp(sw['interp'], sw['phrase'])
|
||||
other_g = self.gradient_other()
|
||||
|
||||
# For each feature, we get the gradient by multiplying by \sigma (-y*\Delta S),
|
||||
# multiplying by y, and summing across all samples
|
||||
# Take negatives since we're minimising
|
||||
phrase_g = -np.sum(np.transpose(phrase_g) * sig_y_by_diffs * self.y, axis=1)
|
||||
interp_g = -np.sum(np.transpose(interp_g) * sig_y_by_diffs * self.y, axis=1)
|
||||
other_g = -np.sum(np.transpose(other_g) * sig_y_by_diffs * self.y, axis=1)
|
||||
|
||||
# regularisation
|
||||
phrase_g += self.alpha * sw['phrase']
|
||||
other_g += self.alpha * sw['other']
|
||||
|
||||
# Splice the gradients together
|
||||
grad = np.array([0.0]* len(w))
|
||||
grad[-len(interp_g):] = interp_g
|
||||
grad[self.phrase_index[0]:self.phrase_index[1]] = phrase_g
|
||||
grad[self.other_index[0][0]:self.other_index[0][1]] = \
|
||||
other_g[:self.other_index[0][1] - self.other_index[0][0]]
|
||||
grad[self.other_index[1][0]:self.other_index[1][1]] = \
|
||||
other_g[self.other_index[0][1] - self.other_index[0][0]:]
|
||||
return grad
|
||||
|
||||
|
||||
def train(self,debug=False):
|
||||
"""Train the mixture model."""
|
||||
if debug:
|
||||
iprint = 0
|
||||
else:
|
||||
iprint = -1
|
||||
# Initialise weights to zero, except interpolation
|
||||
num_phrase_features = self.phrase_index[1] - self.phrase_index[0]
|
||||
num_models = ((self.interp_index[1] - self.interp_index[0])/num_phrase_features)+1
|
||||
w0 = [0.0] * self.interp_index[0]
|
||||
w0 += [1.0/num_models] * (self.interp_index[1]-self.interp_index[0])
|
||||
bounds = [(None,None)] * len(w0)
|
||||
bounds[self.interp_index[0]:self.interp_index[1]] = \
|
||||
[(self.interp_floor,1)] * (self.interp_index[1] - self.interp_index[0])
|
||||
w0 = np.array(w0)
|
||||
x,f,d = fmin_l_bfgs_b(self.objective, w0, fprime=self.gradient, bounds=bounds, pgtol=1e-09, iprint=iprint)
|
||||
if d['warnflag'] != 0:
|
||||
raise OptimisationException(d['task'])
|
||||
weights = x[:self.interp_index[0]]
|
||||
mix_weights = x[self.interp_index[0]:]
|
||||
mix_weights = mix_weights.reshape((num_models-1,num_phrase_features))
|
||||
mix_weights = np.vstack((mix_weights, 1-np.sum(mix_weights,axis=0)))
|
||||
return weights,mix_weights
|
||||
|
||||
|
||||
#
|
||||
# Test logistic regression using pro data
|
||||
#
|
||||
def main():
|
||||
fh = open("data/esen.wmt12.pro")
|
||||
x = []
|
||||
y = []
|
||||
d = 14
|
||||
for line in fh:
|
||||
line = line[:-1]
|
||||
fields = line.split()
|
||||
if fields[0] == "1":
|
||||
y.append(1)
|
||||
else:
|
||||
y.append(-1)
|
||||
x_i = [0]*d
|
||||
for i in xrange(1,len(fields),2):
|
||||
j = int(fields[i][1:])
|
||||
x_ij = float(fields[i+1])
|
||||
x_i[j] = x_ij
|
||||
x.append(x_i)
|
||||
lro = LogisticRegressionOptimiser(np.array(x), np.array(y), 0.1)
|
||||
print lro.train(np.zeros(d), debug=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
12
contrib/promix/util.py
Normal file
12
contrib/promix/util.py
Normal file
@ -0,0 +1,12 @@
|
||||
#!/usr/bin/python
|
||||
|
||||
import numpy as np
|
||||
|
||||
|
||||
floor = np.exp(-100)
|
||||
def safelog(x):
|
||||
"""Wraps np.log to give it a floor"""
|
||||
#return np.log(x)
|
||||
return np.log(np.clip(x,floor,np.inf))
|
||||
|
||||
|
@ -596,13 +596,6 @@ local rule should-clean-project ( project )
|
||||
}
|
||||
}
|
||||
|
||||
ECHO "warning: No toolsets are configured." ;
|
||||
ECHO "warning: Configuring default toolset" \"$(default-toolset)\". ;
|
||||
ECHO "warning: If the default is wrong, your build may not work correctly." ;
|
||||
ECHO "warning: Use the \"toolset=xxxxx\" option to override our guess." ;
|
||||
ECHO "warning: For more configuration options, please consult" ;
|
||||
ECHO "warning: http://boost.org/boost-build2/doc/html/bbv2/advanced/configuration.html" ;
|
||||
|
||||
toolset.using $(default-toolset) : $(default-toolset-version) ;
|
||||
}
|
||||
|
||||
|
@ -265,6 +265,26 @@ rule add-post-hook ( names * ) {
|
||||
post-hooks += $(names) ;
|
||||
}
|
||||
|
||||
rule failure-message ( ok ? ) {
|
||||
if $(ok) != "ok" {
|
||||
local args = [ modules.peek : ARGV ] ;
|
||||
local args = $(args:J=" ") ;
|
||||
if --debug-configuration in [ modules.peek : ARGV ] {
|
||||
echo "The build failed with command line: " ;
|
||||
echo " $(args)" ;
|
||||
echo "If you need support, attach the full output to your e-mail." ;
|
||||
} else {
|
||||
echo "The build failed. If you need support, run:" ;
|
||||
echo " $(args) --debug-configuration -d2 >build.log" ;
|
||||
echo "then attach build.log to your e-mail." ;
|
||||
}
|
||||
echo "ERROR" ;
|
||||
} else {
|
||||
echo "SUCCESS" ;
|
||||
}
|
||||
}
|
||||
add-post-hook failure-message ;
|
||||
|
||||
import feature : feature ;
|
||||
feature options-to-write : : free ;
|
||||
import toolset : flags ;
|
||||
|
@ -60,7 +60,7 @@ int main(int argc, char **argv)
|
||||
StaticData::InstanceNonConst().LoadData(parameter);
|
||||
|
||||
SparsePhraseDictionaryFeature *spdf = NULL;
|
||||
PhraseDictionaryFeature pdf(Compact, spdf, nscores, nscores, input, output, ttable, weight, 0, 0, "", "");
|
||||
PhraseDictionaryFeature pdf(Compact, spdf, nscores, nscores, input, output, ttable, weight, 0, 0, std::vector<std::string>());
|
||||
PhraseDictionaryCompact pdc(nscores, Compact, &pdf, false, useAlignments);
|
||||
bool ret = pdc.Load(input, output, ttable, weight, 0, lmList, 0);
|
||||
assert(ret);
|
||||
|
@ -259,7 +259,6 @@ void OutputAlignment(ostream &out, const vector<const Hypothesis *> &edges)
|
||||
|
||||
targetOffset += tp.GetSize();
|
||||
}
|
||||
out << std::endl;
|
||||
}
|
||||
|
||||
void OutputAlignment(std::ostream &out, const Moses::Hypothesis *hypo)
|
||||
@ -279,7 +278,8 @@ void OutputAlignment(OutputCollector* collector, size_t lineNo , const vector<co
|
||||
{
|
||||
ostringstream out;
|
||||
OutputAlignment(out, edges);
|
||||
|
||||
out << std::endl;
|
||||
|
||||
collector->Write(lineNo,out.str());
|
||||
}
|
||||
|
||||
|
@ -197,7 +197,7 @@ public:
|
||||
string nbestFile = staticData.GetNBestFilePath();
|
||||
if ( ! nbestFile.empty() && nbestFile!="-" && !boost::starts_with(nbestFile,"/dev/stdout") ) {
|
||||
boost::filesystem::path nbestPath(nbestFile);
|
||||
hypergraphDir = nbestPath.parent_path().filename().native();
|
||||
//hypergraphDir = nbestPath.parent_path().filename().native();
|
||||
} else {
|
||||
stringstream hypergraphDirName;
|
||||
hypergraphDirName << boost::filesystem::current_path() << "/hypergraph";
|
||||
|
@ -180,7 +180,7 @@ Parameter::Parameter()
|
||||
AddParam("minlexr-memory", "Load lexical reordering table in minlexr format into memory");
|
||||
AddParam("minphr-memory", "Load phrase table in minphr format into memory");
|
||||
|
||||
AddParam("print-alignment-info", "Output word-to-word alignment into the log file. Word-to-word alignments are takne from the phrase table if any. Default is false");
|
||||
AddParam("print-alignment-info", "Output word-to-word alignment to standard out, separated from translation by |||. Word-to-word alignments are takne from the phrase table if any. Default is false");
|
||||
AddParam("include-segmentation-in-n-best", "include phrasal segmentation in the n-best list. default is false");
|
||||
AddParam("print-alignment-info-in-n-best", "Include word-to-word alignment in the n-best list. Word-to-word alignments are takne from the phrase table if any. Default is false");
|
||||
AddParam("alignment-output-file", "print output word alignments into given file");
|
||||
|
@ -180,5 +180,21 @@ inline size_t hash_value(const Phrase& phrase) {
|
||||
return seed;
|
||||
}
|
||||
|
||||
struct PhrasePtrComparator {
|
||||
inline bool operator()(const Phrase* lhs, const Phrase* rhs) const {
|
||||
return *lhs == *rhs;
|
||||
}
|
||||
};
|
||||
|
||||
struct PhrasePtrHasher {
|
||||
inline size_t operator()(const Phrase* phrase) const {
|
||||
size_t seed = 0;
|
||||
boost::hash_combine(seed,*phrase);
|
||||
return seed;
|
||||
}
|
||||
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
#endif
|
||||
|
@ -1354,12 +1354,6 @@ bool StaticData::LoadPhraseTables()
|
||||
weightAllOffset += numScoreComponent;
|
||||
numScoreComponent += tableInputScores;
|
||||
|
||||
string targetPath, alignmentsFile;
|
||||
if (implementation == SuffixArray) {
|
||||
targetPath = token[5];
|
||||
alignmentsFile= token[6];
|
||||
}
|
||||
|
||||
CHECK(numScoreComponent==weight.size());
|
||||
|
||||
|
||||
@ -1388,7 +1382,7 @@ bool StaticData::LoadPhraseTables()
|
||||
, weight
|
||||
, currDict
|
||||
, maxTargetPhrase[index]
|
||||
, targetPath, alignmentsFile);
|
||||
, token);
|
||||
|
||||
m_phraseDictionary.push_back(pdf);
|
||||
|
||||
|
@ -21,6 +21,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
|
||||
#include "moses/TranslationModel/PhraseDictionary.h"
|
||||
#include "moses/TranslationModel/PhraseDictionaryTreeAdaptor.h"
|
||||
#include "moses/TranslationModel/PhraseDictionaryInterpolated.h"
|
||||
#include "moses/TranslationModel/RuleTable/PhraseDictionarySCFG.h"
|
||||
#include "moses/TranslationModel/RuleTable/PhraseDictionaryOnDisk.h"
|
||||
#include "moses/TranslationModel/RuleTable/PhraseDictionaryALSuffixArray.h"
|
||||
@ -64,16 +65,14 @@ PhraseDictionaryFeature::PhraseDictionaryFeature
|
||||
, const std::vector<float> &weight
|
||||
, size_t dictIndex
|
||||
, size_t tableLimit
|
||||
, const std::string &targetFile // default param
|
||||
, const std::string &alignmentsFile) // default param
|
||||
, const std::vector<std::string>& config)
|
||||
:DecodeFeature("PhraseModel",numScoreComponent,input,output),
|
||||
m_dictIndex(dictIndex),
|
||||
m_numInputScores(numInputScores),
|
||||
m_filePath(filePath),
|
||||
m_tableLimit(tableLimit),
|
||||
m_implementation(implementation),
|
||||
m_targetFile(targetFile),
|
||||
m_alignmentsFile(alignmentsFile),
|
||||
m_config(config),
|
||||
m_sparsePhraseDictionaryFeature(spdf)
|
||||
{
|
||||
if (implementation == Memory || implementation == SCFG || implementation == SuffixArray ||
|
||||
@ -121,6 +120,18 @@ PhraseDictionary* PhraseDictionaryFeature::LoadPhraseTable(const TranslationSyst
|
||||
, system->GetWeightWordPenalty());
|
||||
CHECK(ret);
|
||||
return pdta;
|
||||
} else if (m_implementation == Interpolated) {
|
||||
PhraseDictionaryInterpolated* pdi = new PhraseDictionaryInterpolated(GetNumScoreComponents(), m_numInputScores,this);
|
||||
bool ret = pdi->Load(
|
||||
GetInput()
|
||||
, GetOutput()
|
||||
, m_config
|
||||
, weightT
|
||||
, m_tableLimit
|
||||
, system->GetLanguageModels()
|
||||
, system->GetWeightWordPenalty());
|
||||
CHECK(ret);
|
||||
return pdi;
|
||||
} else if (m_implementation == SCFG || m_implementation == Hiero) {
|
||||
// memory phrase table
|
||||
if (m_implementation == Hiero) {
|
||||
@ -181,12 +192,13 @@ PhraseDictionary* PhraseDictionaryFeature::LoadPhraseTable(const TranslationSyst
|
||||
} else if (m_implementation == SuffixArray) {
|
||||
#ifndef WIN32
|
||||
PhraseDictionaryDynSuffixArray *pd = new PhraseDictionaryDynSuffixArray(GetNumScoreComponents(), this);
|
||||
CHECK(m_config.size() >= 7);
|
||||
if(!(pd->Load(
|
||||
GetInput()
|
||||
,GetOutput()
|
||||
,m_filePath
|
||||
,m_targetFile
|
||||
,m_alignmentsFile
|
||||
,m_config[5]
|
||||
,m_config[6]
|
||||
,weightT, m_tableLimit
|
||||
,system->GetLanguageModels()
|
||||
,system->GetWeightWordPenalty()))) {
|
||||
|
@ -107,8 +107,7 @@ public:
|
||||
, const std::vector<float> &weight
|
||||
, size_t dictIndex
|
||||
, size_t tableLimit
|
||||
, const std::string &targetFile
|
||||
, const std::string &alignmentsFile);
|
||||
, const std::vector<std::string>& config);
|
||||
|
||||
|
||||
virtual ~PhraseDictionaryFeature();
|
||||
@ -173,8 +172,7 @@ private:
|
||||
|
||||
bool m_useThreadSafePhraseDictionary;
|
||||
PhraseTableImplementation m_implementation;
|
||||
std::string m_targetFile;
|
||||
std::string m_alignmentsFile;
|
||||
const std::vector<std::string> m_config;
|
||||
SparsePhraseDictionaryFeature* m_sparsePhraseDictionaryFeature;
|
||||
|
||||
};
|
||||
|
183
moses/TranslationModel/PhraseDictionaryInterpolated.cpp
Normal file
183
moses/TranslationModel/PhraseDictionaryInterpolated.cpp
Normal file
@ -0,0 +1,183 @@
|
||||
/***********************************************************************
|
||||
Moses - factored phrase-based language decoder
|
||||
Copyright (C) 2013- University of Edinburgh
|
||||
|
||||
This library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
This library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with this library; if not, write to the Free Software
|
||||
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
***********************************************************************/
|
||||
|
||||
#include <boost/lexical_cast.hpp>
|
||||
#include <boost/unordered_set.hpp>
|
||||
|
||||
#include "util/exception.hh"
|
||||
#include "util/tokenize_piece.hh"
|
||||
#include "moses/TranslationModel/PhraseDictionaryInterpolated.h"
|
||||
|
||||
using namespace std;
|
||||
|
||||
namespace Moses
|
||||
{
|
||||
|
||||
PhraseDictionaryInterpolated::PhraseDictionaryInterpolated
|
||||
(size_t numScoreComponent,size_t numInputScores,const PhraseDictionaryFeature* feature):
|
||||
PhraseDictionary(numScoreComponent,feature),
|
||||
m_targetPhrases(NULL),
|
||||
m_languageModels(NULL) {}
|
||||
|
||||
bool PhraseDictionaryInterpolated::Load(
|
||||
const std::vector<FactorType> &input
|
||||
, const std::vector<FactorType> &output
|
||||
, const std::vector<std::string>& config
|
||||
, const std::vector<float> &weightT
|
||||
, size_t tableLimit
|
||||
, const LMList &languageModels
|
||||
, float weightWP) {
|
||||
|
||||
m_languageModels = &languageModels;
|
||||
m_weightT = weightT;
|
||||
m_tableLimit = tableLimit;
|
||||
m_weightWP = weightWP;
|
||||
|
||||
//The config should be as follows:
|
||||
//0-3: type factor factor num-components (as usual)
|
||||
//4: combination mode (e.g. naive)
|
||||
//5-(length-2): List of phrase-table files
|
||||
//length-1: Weight string, in the same format as used for tmcombine
|
||||
|
||||
UTIL_THROW_IF(config.size() < 7, util::Exception, "Missing fields from phrase table configuration: expected at least 7");
|
||||
UTIL_THROW_IF(config[4] != "naive", util::Exception, "Unsupported combination mode: '" << config[4] << "'");
|
||||
|
||||
// Create the dictionaries
|
||||
for (size_t i = 5; i < config.size()-1; ++i) {
|
||||
m_dictionaries.push_back(DictionaryHandle(new PhraseDictionaryTreeAdaptor(
|
||||
GetFeature()->GetNumScoreComponents(),
|
||||
GetFeature()->GetNumInputScores(),
|
||||
GetFeature())));
|
||||
bool ret = m_dictionaries.back()->Load(
|
||||
input,
|
||||
output,
|
||||
config[i],
|
||||
weightT,
|
||||
0,
|
||||
languageModels,
|
||||
weightWP);
|
||||
if (!ret) return ret;
|
||||
}
|
||||
|
||||
//Parse the weight strings
|
||||
for (util::TokenIter<util::SingleCharacter, false> featureWeights(config.back(), util::SingleCharacter(';')); featureWeights; ++featureWeights) {
|
||||
m_weights.push_back(vector<float>());
|
||||
float sum = 0;
|
||||
for (util::TokenIter<util::SingleCharacter, false> tableWeights(*featureWeights, util::SingleCharacter(',')); tableWeights; ++tableWeights) {
|
||||
const float weight = boost::lexical_cast<float>(*tableWeights);
|
||||
m_weights.back().push_back(weight);
|
||||
sum += weight;
|
||||
}
|
||||
UTIL_THROW_IF(m_weights.back().size() != m_dictionaries.size(), util::Exception,
|
||||
"Number of weights (" << m_weights.back().size() <<
|
||||
") does not match number of dictionaries to combine (" << m_dictionaries.size() << ")");
|
||||
UTIL_THROW_IF(abs(sum - 1) > 0.01, util::Exception, "Weights not normalised");
|
||||
|
||||
}
|
||||
|
||||
//check number of weight sets. Make sure there is a weight for every score component
|
||||
//except for the last - which is assumed to be the phrase penalty.
|
||||
UTIL_THROW_IF(m_weights.size() != 1 && m_weights.size() != GetFeature()->GetNumScoreComponents()-1, util::Exception, "Unexpected number of weight sets");
|
||||
//if 1 weight set, then repeat
|
||||
if (m_weights.size() == 1) {
|
||||
while(m_weights.size() < GetFeature()->GetNumScoreComponents()-1) {
|
||||
m_weights.push_back(m_weights[0]);
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
void PhraseDictionaryInterpolated::InitializeForInput(InputType const& source) {
|
||||
for (size_t i = 0; i < m_dictionaries.size(); ++i) {
|
||||
m_dictionaries[i]->InitializeForInput(source);
|
||||
}
|
||||
}
|
||||
|
||||
typedef
|
||||
boost::unordered_set<TargetPhrase*,PhrasePtrHasher,PhrasePtrComparator> PhraseSet;
|
||||
|
||||
|
||||
const TargetPhraseCollection*
|
||||
PhraseDictionaryInterpolated::GetTargetPhraseCollection(const Phrase& src) const {
|
||||
|
||||
delete m_targetPhrases;
|
||||
m_targetPhrases = new TargetPhraseCollection();
|
||||
PhraseSet allPhrases;
|
||||
vector<PhraseSet> phrasesByTable(m_dictionaries.size());
|
||||
for (size_t i = 0; i < m_dictionaries.size(); ++i) {
|
||||
const TargetPhraseCollection* phrases = m_dictionaries[i]->GetTargetPhraseCollection(src);
|
||||
if (phrases) {
|
||||
for (TargetPhraseCollection::const_iterator j = phrases->begin();
|
||||
j != phrases->end(); ++j) {
|
||||
allPhrases.insert(*j);
|
||||
phrasesByTable[i].insert(*j);
|
||||
}
|
||||
}
|
||||
}
|
||||
ScoreComponentCollection sparseVector;
|
||||
for (PhraseSet::const_iterator i = allPhrases.begin(); i != allPhrases.end(); ++i) {
|
||||
TargetPhrase* combinedPhrase = new TargetPhrase((Phrase)**i);
|
||||
//combinedPhrase->ResetScore();
|
||||
//cerr << *combinedPhrase << " " << combinedPhrase->GetScoreBreakdown() << endl;
|
||||
combinedPhrase->SetSourcePhrase((*i)->GetSourcePhrase());
|
||||
combinedPhrase->SetAlignTerm(&((*i)->GetAlignTerm()));
|
||||
combinedPhrase->SetAlignNonTerm(&((*i)->GetAlignTerm()));
|
||||
Scores combinedScores(GetFeature()->GetNumScoreComponents());
|
||||
for (size_t j = 0; j < phrasesByTable.size(); ++j) {
|
||||
PhraseSet::const_iterator tablePhrase = phrasesByTable[j].find(combinedPhrase);
|
||||
if (tablePhrase != phrasesByTable[j].end()) {
|
||||
Scores tableScores = (*tablePhrase)->GetScoreBreakdown()
|
||||
.GetScoresForProducer(GetFeature());
|
||||
//cerr << "Scores from " << j << " table: ";
|
||||
for (size_t k = 0; k < tableScores.size()-1; ++k) {
|
||||
//cerr << tableScores[k] << "(" << exp(tableScores[k]) << ") ";
|
||||
combinedScores[k] += m_weights[k][j] * exp(tableScores[k]);
|
||||
//cerr << m_weights[k][j] * exp(tableScores[k]) << " ";
|
||||
}
|
||||
//cerr << endl;
|
||||
}
|
||||
}
|
||||
//map back to log space
|
||||
//cerr << "Combined ";
|
||||
for (size_t k = 0; k < combinedScores.size()-1; ++k) {
|
||||
//cerr << combinedScores[k] << " ";
|
||||
combinedScores[k] = log(combinedScores[k]);
|
||||
//cerr << combinedScores[k] << " ";
|
||||
}
|
||||
//cerr << endl;
|
||||
combinedScores.back() = 1; //assume last is penalty
|
||||
combinedPhrase->SetScore(
|
||||
GetFeature(),
|
||||
combinedScores,
|
||||
sparseVector,
|
||||
m_weightT,
|
||||
m_weightWP,
|
||||
*m_languageModels);
|
||||
//cerr << *combinedPhrase << " " << combinedPhrase->GetScoreBreakdown() << endl;
|
||||
m_targetPhrases->Add(combinedPhrase);
|
||||
}
|
||||
|
||||
m_targetPhrases->Prune(true,m_tableLimit);
|
||||
|
||||
|
||||
return m_targetPhrases;
|
||||
}
|
||||
|
||||
}
|
77
moses/TranslationModel/PhraseDictionaryInterpolated.h
Normal file
77
moses/TranslationModel/PhraseDictionaryInterpolated.h
Normal file
@ -0,0 +1,77 @@
|
||||
/***********************************************************************
|
||||
Moses - factored phrase-based language decoder
|
||||
Copyright (C) 2013- University of Edinburgh
|
||||
|
||||
This library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
This library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with this library; if not, write to the Free Software
|
||||
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
***********************************************************************/
|
||||
|
||||
|
||||
#ifndef moses_PhraseDictionaryInterpolated_h
|
||||
#define moses_PhraseDictionaryInterpolated_h
|
||||
|
||||
#include <boost/shared_ptr.hpp>
|
||||
|
||||
#include "moses/TranslationModel/PhraseDictionary.h"
|
||||
#include "moses/TranslationModel/PhraseDictionaryTreeAdaptor.h"
|
||||
|
||||
namespace Moses
|
||||
{
|
||||
|
||||
/**
|
||||
* An interpolation of 1 or more PhraseDictionaryTree translation tables.
|
||||
**/
|
||||
class PhraseDictionaryInterpolated : public PhraseDictionary
|
||||
{
|
||||
public:
|
||||
|
||||
PhraseDictionaryInterpolated
|
||||
(size_t numScoreComponent,size_t numInputScores,const PhraseDictionaryFeature* feature);
|
||||
|
||||
virtual ~PhraseDictionaryInterpolated() {delete m_targetPhrases;}
|
||||
|
||||
// initialize ...
|
||||
bool Load(const std::vector<FactorType> &input
|
||||
, const std::vector<FactorType> &output
|
||||
, const std::vector<std::string>& config
|
||||
, const std::vector<float> &weight
|
||||
, size_t tableLimit
|
||||
, const LMList &languageModels
|
||||
, float weightWP);
|
||||
|
||||
virtual const TargetPhraseCollection *GetTargetPhraseCollection(const Phrase& src) const;
|
||||
virtual void InitializeForInput(InputType const& source);
|
||||
virtual ChartRuleLookupManager *CreateRuleLookupManager(
|
||||
const InputType &,
|
||||
const ChartCellCollectionBase &) {
|
||||
throw std::logic_error("PhraseDictionaryInterpolated.CreateRuleLookupManager() Not implemented");
|
||||
}
|
||||
|
||||
private:
|
||||
|
||||
typedef boost::shared_ptr<PhraseDictionaryTreeAdaptor> DictionaryHandle;
|
||||
std::vector<DictionaryHandle> m_dictionaries;
|
||||
std::vector<std::vector<float> > m_weights; //feature x table
|
||||
mutable TargetPhraseCollection* m_targetPhrases;
|
||||
std::vector<float> m_weightT;
|
||||
size_t m_tableLimit;
|
||||
const LMList* m_languageModels;
|
||||
float m_weightWP;
|
||||
|
||||
};
|
||||
|
||||
|
||||
}
|
||||
|
||||
#endif
|
@ -139,6 +139,7 @@ enum PhraseTableImplementation {
|
||||
,ALSuffixArray = 10
|
||||
,FuzzyMatch = 11
|
||||
,Compact = 12
|
||||
,Interpolated = 13
|
||||
};
|
||||
|
||||
enum InputTypeEnum {
|
||||
|
@ -956,7 +956,7 @@ remove-markup
|
||||
default-name: evaluation/cleaned
|
||||
pass-if: TRAINING:hierarchical-rule-set
|
||||
pass-unless: report-segmentation
|
||||
template: $moses-script-dir/ems/support/remove-segmenation-markup.perl < IN > OUT
|
||||
template: $moses-script-dir/ems/support/remove-segmentation-markup.perl < IN > OUT
|
||||
recase-output
|
||||
in: cleaned-output RECASING:recase-config
|
||||
out: recased-output
|
||||
|
@ -2,6 +2,8 @@
|
||||
|
||||
use strict;
|
||||
|
||||
$|++;
|
||||
|
||||
while(<STDIN>) {
|
||||
s/ \|\d+\-\d+\| / /g;
|
||||
s/ \|\d+\-\d+\|$//;
|
@ -121,6 +121,13 @@ my $megam_default_options = "-fvals -maxi 30 -nobias binary";
|
||||
# Flags related to Batch MIRA (Cherry & Foster, 2012)
|
||||
my $___BATCH_MIRA = 0; # flg to enable batch MIRA
|
||||
|
||||
# Train phrase model mixture weights with PRO (Haddow, NAACL 2012)
|
||||
my $__PROMIX_TRAINING = undef; # Location of main script (contrib/promix/main.py)
|
||||
# The phrase tables. These should be gzip text format.
|
||||
my @__PROMIX_TABLES;
|
||||
# used to filter output
|
||||
my $__REMOVE_SEGMENTATION = "$SCRIPTS_ROOTDIR/ems/support/remove-segmentation-markup.perl";
|
||||
|
||||
my $__THREADS = 0;
|
||||
|
||||
# Parameter for effective reference length when computing BLEU score
|
||||
@ -223,6 +230,8 @@ GetOptions(
|
||||
"historic-interpolation=f" => \$___HISTORIC_INTERPOLATION,
|
||||
"batch-mira" => \$___BATCH_MIRA,
|
||||
"batch-mira-args=s" => \$batch_mira_args,
|
||||
"promix-training=s" => \$__PROMIX_TRAINING,
|
||||
"promix-table=s" => \@__PROMIX_TABLES,
|
||||
"threads=i" => \$__THREADS
|
||||
) or exit(1);
|
||||
|
||||
@ -312,6 +321,8 @@ Options:
|
||||
--batch-mira-args=STRING ... args to pass through to batch MIRA. This flag is useful to
|
||||
change MIRA's hyperparameters such as regularization parameter C,
|
||||
BLEU decay factor, and the number of iterations of MIRA.
|
||||
--promix-training=STRING ... PRO-based mixture model training (Haddow, NAACL 2013)
|
||||
--promix-tables=STRING ... Phrase tables for PRO-based mixture model training.
|
||||
--threads=NUMBER ... Use multi-threaded mert (must be compiled in).
|
||||
--historic-interpolation ... Interpolate optimized weights with prior iterations' weight
|
||||
(parameter sets factor [0;1] given to current weights)
|
||||
@ -371,6 +382,17 @@ if (($___PAIRWISE_RANKED_OPTIMIZER || $___PRO_STARTING_POINT) && ! -x $pro_optim
|
||||
die("ERROR: Installation of megam_i686.opt failed! Install by hand from $megam_url") unless -x $pro_optimizer;
|
||||
}
|
||||
|
||||
if ($__PROMIX_TRAINING) {
|
||||
die "Not executable $__PROMIX_TRAINING" unless -x $__PROMIX_TRAINING;
|
||||
die "For promix training, specify the tables using --promix-table arguments" unless @__PROMIX_TABLES;
|
||||
die "For mixture model, need at least 2 tables" unless scalar(@__PROMIX_TABLES) > 1;
|
||||
|
||||
for my $TABLE (@__PROMIX_TABLES) {
|
||||
die "Phrase table $TABLE not found" unless -r $TABLE;
|
||||
}
|
||||
die "To use promix training, need to specify a filter and binarisation command" unless $filtercmd =~ /Binarizer/;
|
||||
}
|
||||
|
||||
$mertargs = "" if !defined $mertargs;
|
||||
|
||||
my $scconfig = undef;
|
||||
@ -492,7 +514,28 @@ my $sparse_weights_file = undef;
|
||||
my $prev_feature_file = undef;
|
||||
my $prev_score_file = undef;
|
||||
my $prev_init_file = undef;
|
||||
my @allnbests;
|
||||
|
||||
# If we're doing promix training, need to make sure the appropriate
|
||||
# tables are in place
|
||||
my @_PROMIX_TABLES_BIN;
|
||||
if ($__PROMIX_TRAINING) {
|
||||
print STDERR "Training mixture model using promix\n";
|
||||
for (my $i = 0; $i < scalar(@__PROMIX_TABLES); ++$i) {
|
||||
# Create filtered, binarised tables
|
||||
my $filtered_config = "moses_$i.ini";
|
||||
substitute_ttable($___CONFIG, $filtered_config, $__PROMIX_TABLES[$i]);
|
||||
#TODO: Remove reordering table from config, as we don't need to filter
|
||||
# and binarise it.
|
||||
my $filtered_path = "filtered_$i";
|
||||
my $___FILTER_F = $___DEV_F;
|
||||
$___FILTER_F = $filterfile if (defined $filterfile);
|
||||
my $cmd = "$filtercmd ./$filtered_path $filtered_config $___FILTER_F";
|
||||
&submit_or_exec($cmd, "filterphrases_$i.out", "filterphrases_$i.err");
|
||||
push (@_PROMIX_TABLES_BIN,"$filtered_path/phrase-table.0-0.1.1");
|
||||
}
|
||||
}
|
||||
|
||||
if ($___FILTER_PHRASE_TABLE) {
|
||||
my $outdir = "filtered";
|
||||
if (-e "$outdir/moses.ini") {
|
||||
@ -647,6 +690,11 @@ my $allsorted = undef;
|
||||
my $nbest_file = undef;
|
||||
my $lsamp_file = undef; # Lattice samples
|
||||
my $orig_nbest_file = undef; # replaced if lattice sampling
|
||||
# For mixture modelling
|
||||
my @promix_weights;
|
||||
my $num_mixed_phrase_features;
|
||||
my $interpolated_config;
|
||||
my $uninterpolated_config; # backup of config without interpolated ttable
|
||||
|
||||
while (1) {
|
||||
$run++;
|
||||
@ -654,10 +702,50 @@ while (1) {
|
||||
print "Maximum number of iterations exceeded - stopping\n";
|
||||
last;
|
||||
}
|
||||
print "run $run start at ".`date`;
|
||||
|
||||
if ($__PROMIX_TRAINING) {
|
||||
# Need to create an ini file for the interpolated phrase table
|
||||
if (!@promix_weights) {
|
||||
# Create initial weights, distributing evenly between tables
|
||||
# total number of weights is 1 less than number of phrase features, multiplied
|
||||
# by the number of tables
|
||||
$num_mixed_phrase_features = (grep { $_ eq 'tm' } @{$featlist->{"names"}}) - 1;
|
||||
|
||||
@promix_weights = (1.0/scalar(@__PROMIX_TABLES)) x
|
||||
($num_mixed_phrase_features * scalar(@__PROMIX_TABLES));
|
||||
}
|
||||
|
||||
# backup orig config, so we always add the table into it
|
||||
$uninterpolated_config= $___CONFIG unless $uninterpolated_config;
|
||||
|
||||
# Interpolation
|
||||
my $interpolated_phrase_table = "naive ";
|
||||
$interpolated_phrase_table .= join(" ", @_PROMIX_TABLES_BIN);
|
||||
# convert from table,feature ordering to feature,table ordering
|
||||
my @transposed_weights;
|
||||
for my $feature (0..($num_mixed_phrase_features-1)) {
|
||||
my @table_weights;
|
||||
for my $table (0..(scalar(@__PROMIX_TABLES)-1)) {
|
||||
push @table_weights, $promix_weights[$table * $num_mixed_phrase_features + $feature];
|
||||
}
|
||||
push @transposed_weights, join ",", @table_weights;
|
||||
}
|
||||
$interpolated_phrase_table .= " ";
|
||||
$interpolated_phrase_table .= join(";",@transposed_weights);
|
||||
|
||||
# Create an ini file for the interpolated phrase table
|
||||
$interpolated_config ="moses.interpolated.ini";
|
||||
substitute_ttable($uninterpolated_config, $interpolated_config, $interpolated_phrase_table, "13");
|
||||
|
||||
# the decoder should now use the interpolated model
|
||||
$___CONFIG = "$interpolated_config";
|
||||
|
||||
}
|
||||
|
||||
# run beamdecoder with option to output nbestlists
|
||||
# the end result should be (1) @NBEST_LIST, a list of lists; (2) @SCORE, a list of lists of lists
|
||||
|
||||
print "run $run start at ".`date`;
|
||||
|
||||
# In case something dies later, we might wish to have a copy
|
||||
create_config($___CONFIG, "./run$run.moses.ini", $featlist, $run, (defined $devbleu ? $devbleu : "--not-estimated--"), $sparse_weights_file);
|
||||
@ -709,6 +797,9 @@ while (1) {
|
||||
my $score_file = "run$run.${base_score_file}";
|
||||
|
||||
my $cmd = "$mert_extract_cmd $mert_extract_args --scfile $score_file --ffile $feature_file -r " . join(",", @references) . " -n $nbest_file";
|
||||
$cmd .= " -d" if $__PROMIX_TRAINING; # Allow duplicates
|
||||
# remove segmentation
|
||||
$cmd .= " -l $__REMOVE_SEGMENTATION" if $__PROMIX_TRAINING;
|
||||
$cmd = &create_extractor_script($cmd, $___WORKING_DIR);
|
||||
&submit_or_exec($cmd, "extract.out","extract.err");
|
||||
|
||||
@ -781,6 +872,11 @@ while (1) {
|
||||
my $pro_file_settings = "--ffile " . join(" --ffile ", split(/,/, $ffiles)) .
|
||||
" --scfile " . join(" --scfile ", split(/,/, $scfiles));
|
||||
|
||||
push @allnbests, $nbest_file;
|
||||
my $promix_file_settings =
|
||||
"--scfile " . join(" --scfile ", split(/,/, $scfiles)) .
|
||||
" --nbest " . join(" --nbest ", @allnbests);
|
||||
|
||||
if ($___START_WITH_HISTORIC_BESTS && defined $prev_init_file) {
|
||||
$file_settings .= " --ifile $prev_init_file,run$run.$weights_in_file";
|
||||
} else {
|
||||
@ -799,7 +895,7 @@ while (1) {
|
||||
my $pro_cmd = "$mert_pro_cmd $proargs $seed_settings $pro_file_settings -o run$run.pro.data ; $pro_optimizer_cmd";
|
||||
&submit_or_exec($pro_cmd, "run$run.pro.out", "run$run.pro.err");
|
||||
# ... get results ...
|
||||
($bestpoint,$devbleu) = &get_weights_from_mert("run$run.pro.out","run$run.pro.err",scalar @{$featlist->{"names"}},\%sparse_weights);
|
||||
($bestpoint,$devbleu) = &get_weights_from_mert("run$run.pro.out","run$run.pro.err",scalar @{$featlist->{"names"}},\%sparse_weights, \@promix_weights);
|
||||
# Get the pro outputs ready for mert. Add the weight ranges,
|
||||
# and a weight and range for the single sparse feature
|
||||
$cmd =~ s/--ifile (\S+)/--ifile run$run.init.pro/;
|
||||
@ -830,9 +926,19 @@ while (1) {
|
||||
safesystem("echo 'not used' > $weights_out_file") or die;
|
||||
$cmd = "$mert_mira_cmd $mira_settings $seed_settings $pro_file_settings -o $mert_outfile";
|
||||
&submit_or_exec($cmd, "run$run.mira.out", $mert_logfile);
|
||||
} elsif ($__PROMIX_TRAINING) {
|
||||
# PRO trained mixture model
|
||||
safesystem("echo 'not used' > $weights_out_file") or die;
|
||||
$cmd = "$__PROMIX_TRAINING $promix_file_settings";
|
||||
$cmd .= " -t mix ";
|
||||
$cmd .= join(" ", map {"-p $_"} @_PROMIX_TABLES_BIN);
|
||||
$cmd .= " -i $___DEV_F";
|
||||
print "Starting promix optimisation at " . `date`;
|
||||
&submit_or_exec($cmd, "$mert_outfile", $mert_logfile);
|
||||
print "Finished promix optimisation at " . `date`;
|
||||
} else { # just mert
|
||||
&submit_or_exec($cmd . $mert_settings, $mert_outfile, $mert_logfile);
|
||||
}
|
||||
}
|
||||
|
||||
die "Optimization failed, file $weights_out_file does not exist or is empty"
|
||||
if ! -s $weights_out_file;
|
||||
@ -844,11 +950,15 @@ while (1) {
|
||||
safesystem("\\cp -f $mert_logfile run$run.$mert_logfile") or die;
|
||||
safesystem("touch $mert_logfile run$run.$mert_logfile") or die;
|
||||
safesystem("\\cp -f $weights_out_file run$run.$weights_out_file") or die; # this one is needed for restarts, too
|
||||
if ($__PROMIX_TRAINING) {
|
||||
safesystem("\\cp -f $interpolated_config run$run.$interpolated_config") or die;
|
||||
}
|
||||
|
||||
print "run $run end at ".`date`;
|
||||
|
||||
($bestpoint,$devbleu) = &get_weights_from_mert("run$run.$mert_outfile","run$run.$mert_logfile",scalar @{$featlist->{"names"}},\%sparse_weights);
|
||||
($bestpoint,$devbleu) = &get_weights_from_mert("run$run.$mert_outfile","run$run.$mert_logfile",scalar @{$featlist->{"names"}},\%sparse_weights,\@promix_weights);
|
||||
my $merge_weight = 0;
|
||||
print "New mixture weights: " . join(" ", @promix_weights) . "\n";
|
||||
|
||||
die "Failed to parse mert.log, missed Best point there."
|
||||
if !defined $bestpoint || !defined $devbleu;
|
||||
@ -976,7 +1086,9 @@ if($___RETURN_BEST_DEV) {
|
||||
my $bestbleu=0;
|
||||
my $evalout = "eval.out";
|
||||
for (my $i = 1; $i < $run; $i++) {
|
||||
safesystem("$mert_eval_cmd --reference " . join(",", @references) . " --candidate run$i.out 2> /dev/null 1> $evalout");
|
||||
my $cmd = "$mert_eval_cmd --reference " . join(",", @references) . " -s BLEU --candidate run$i.out";
|
||||
$cmd .= " -l $__REMOVE_SEGMENTATION" if defined( $__PROMIX_TRAINING);
|
||||
safesystem("$cmd 2> /dev/null 1> $evalout");
|
||||
open my $fh, '<', $evalout or die "Can't read $evalout : $!";
|
||||
my $bleu = <$fh>;
|
||||
chomp $bleu;
|
||||
@ -1007,25 +1119,28 @@ print "Training finished at " . `date`;
|
||||
} # end of local scope
|
||||
|
||||
sub get_weights_from_mert {
|
||||
my ($outfile, $logfile, $weight_count, $sparse_weights) = @_;
|
||||
my ($outfile, $logfile, $weight_count, $sparse_weights, $mix_weights) = @_;
|
||||
my ($bestpoint, $devbleu);
|
||||
if ($___PAIRWISE_RANKED_OPTIMIZER || ($___PRO_STARTING_POINT && $logfile =~ /pro/)
|
||||
|| $___BATCH_MIRA) {
|
||||
|| $___BATCH_MIRA || $__PROMIX_TRAINING) {
|
||||
open my $fh, '<', $outfile or die "Can't open $outfile: $!";
|
||||
my @WEIGHT;
|
||||
@$mix_weights = ();
|
||||
for (my $i = 0; $i < $weight_count; $i++) { push @WEIGHT, 0; }
|
||||
my $sum = 0.0;
|
||||
while (<$fh>) {
|
||||
if (/^F(\d+) ([\-\.\de]+)/) { # regular features
|
||||
$WEIGHT[$1] = $2;
|
||||
$sum += abs($2);
|
||||
} elsif (/^M(\d+_\d+) ([\-\.\de]+)/) { # mix weights
|
||||
push @$mix_weights,$2;
|
||||
} elsif (/^(.+_.+) ([\-\.\de]+)/) { # sparse features
|
||||
$$sparse_weights{$1} = $2;
|
||||
}
|
||||
}
|
||||
close $fh;
|
||||
die "It seems feature values are invalid or unable to read $outfile." if $sum < 1e-09;
|
||||
|
||||
|
||||
$devbleu = "unknown";
|
||||
foreach (@WEIGHT) { $_ /= $sum; }
|
||||
foreach (keys %{$sparse_weights}) { $$sparse_weights{$_} /= $sum; }
|
||||
@ -1085,6 +1200,7 @@ sub run_decoder {
|
||||
my $decoder_config = "";
|
||||
$decoder_config = join(" ", values %model_weights) unless $___USE_CONFIG_WEIGHTS_FIRST && $run==1;
|
||||
$decoder_config .= " -weight-file run$run.sparse-weights" if -e "run$run.sparse-weights";
|
||||
$decoder_config .= " -report-segmentation" if $__PROMIX_TRAINING;
|
||||
print STDERR "DECODER_CFG = $decoder_config\n";
|
||||
print "decoder_config = $decoder_config\n";
|
||||
|
||||
@ -1358,6 +1474,31 @@ sub create_config {
|
||||
print STDERR "Saved: $outfn\n";
|
||||
}
|
||||
|
||||
# Create a new ini file, with the first ttable replaced by the given one
|
||||
# and its type set to text
|
||||
sub substitute_ttable {
|
||||
my ($old_ini, $new_ini, $new_ttable, $ttable_type) = @_;
|
||||
$ttable_type = "0" unless defined($ttable_type);
|
||||
open(NEW_INI,">$new_ini") || die "Failed to create $new_ini";
|
||||
open(INI,$old_ini) || die "Failed to open $old_ini";
|
||||
while(<INI>) {
|
||||
if (/\[ttable-file\]/) {
|
||||
print NEW_INI "[ttable-file]\n";
|
||||
my $ttable_config = <INI>;
|
||||
chomp $ttable_config;
|
||||
my @ttable_fields = split /\s+/, $ttable_config;
|
||||
$ttable_fields[0] = $ttable_type;
|
||||
$ttable_fields[4] = $new_ttable;
|
||||
print NEW_INI join(" ", @ttable_fields) . "\n";
|
||||
} else {
|
||||
print NEW_INI;
|
||||
}
|
||||
}
|
||||
close NEW_INI;
|
||||
close INI;
|
||||
}
|
||||
|
||||
|
||||
sub safesystem {
|
||||
print STDERR "Executing: @_\n";
|
||||
system(@_);
|
||||
|
Loading…
Reference in New Issue
Block a user