moses-smt/mosesdecoder
1
1
Fork 0
mirror of https://github.com/moses-smt/mosesdecoder.git synced 2024-09-11 19:27:11 +03:00
Code Issues Packages Projects Releases Wiki Activity

merged

Browse Source
This commit is contained in:
Hieu Hoang 2013-04-19 15:03:34 +01:00
commit b1da4dbe0e
47 changed files with 167229 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

66
contrib/promix/README.md Normal file
View File

@ -0,0 +1,66 @@
promix - for training translation model interpolation weights using PRO
Author: Barry Haddow <bhaddow [AT] inf.ed.ac.uk>
ABOUT
-----
The code here provides the "inner loop" for a batch tuning algorithm (like MERT) which
optimises phrase table interpolation weights at the same time as the standard linear
model weights. Interpolation of the phrase tables uses the "naive" method of tmcombine.
Currently the interpolation only works for two phrase tables, but will soon
be extended to work for more than two.
REQUIREMENTS
------------
The scripts require the Moses Python interface (in contrib/python). It should be built
first, following the instructions in that directory.
The scripts also require scipy and numpy. They have been tested with the following versions:
* Python 2.7
* Scipy 0.11.0
* Numpy 1.6.2
Run the test.py script to check that everything is functioning correctly.
USAGE
-----
Since the code in this directory provides the inner loop for a batch tuning algorithm,
it is run from the increasingly inaccurately named mert-moses.pl. If you want to run
the optimiser directly, run `main.py -h` for usage.
A sample command for mert-moses.pl is as follows:
MOSES/scripts/training/mert-moses.pl \
input-file ref-file \
decoder \
ini-file \
--promix-training MOSES/contrib/promix/main.py \
--maximum-iterations 15 \
--promix-table phrase-table-1 \
--promix-table phrase-table-2 \
--filtercmd "MOSES/scripts/training/filter-model-given-input.pl --Binarizer MOSES/bin/processPhraseTable" \
--nbest 100 --working-dir ./tmp --decoder-flags "-threads 4 -v 0 " \
--rootdir MOSES/scripts -mertdir MOSES/bin \
--return-best-dev
Note that promix training requires a filter and binarise script, and that the phrase table
referenced in the ini file is not used. The argument `--return-best-dev` is not essential,
but recommended.
REFERENCES
----------
The code here was created for:
Haddow, Barry (2013) Applying Pairwise Ranked Optimisation to
Improve the Interpolation of Translation Models. In: Proceedings of NAACL 2013
See also:
Sennrich, Rico (2012). Perplexity Minimization for Translation Model Domain Adaptation in Statistical Machine Translation. In: Proceedings of EACL 2012.

27
contrib/promix/bleu.py Normal file
View File

@ -0,0 +1,27 @@
#!/usr/bin/env python
from math import exp,log
class BleuScoreException(Exception):
pass
class BleuScorer:
def __init__(self):
"""References should be a list. If each element is a string, assume
they are filenames, if a list, assume tokenised strings"""
self.smooth = 1.0
self.order = 4
def score(self,scores):
if len(scores) != self.order*2+1:
raise BleuScoreException("Wrong number of scores. Expected %d, but found %d" %
(self.order*2+1, len(scores)))
logbleu = 0.0
for j in range(self.order):
logbleu += log(scores[2*j] + self.smooth) - log(scores[2*j+1] + self.smooth)
logbleu /= self.order
brevity = 1.0 - float(scores[-1]) / scores[1]
if brevity < 0:
logbleu += brevity
return exp(logbleu)

260
contrib/promix/coll.py Normal file
View File

@ -0,0 +1,260 @@
# Backport of OrderedDict() class that runs on Python 2.4, 2.5, 2.6, 2.7 and pypy.
# Passes Python2.7's test suite and incorporates all the latest updates.
# From http://code.activestate.com/recipes/576693/
try:
from thread import get_ident as _get_ident
except ImportError:
from dummy_thread import get_ident as _get_ident
try:
from _abcoll import KeysView, ValuesView, ItemsView
except ImportError:
pass
class OrderedDict(dict):
'Dictionary that remembers insertion order'
# An inherited dict maps keys to values.
# The inherited dict provides __getitem__, __len__, __contains__, and get.
# The remaining methods are order-aware.
# Big-O running times for all methods are the same as for regular dictionaries.
# The internal self.__map dictionary maps keys to links in a doubly linked list.
# The circular doubly linked list starts and ends with a sentinel element.
# The sentinel element never gets deleted (this simplifies the algorithm).
# Each link is stored as a list of length three: [PREV, NEXT, KEY].
def __init__(self, *args, **kwds):
'''Initialize an ordered dictionary. Signature is the same as for
regular dictionaries, but keyword arguments are not recommended
because their insertion order is arbitrary.
'''
if len(args) > 1:
raise TypeError('expected at most 1 arguments, got %d' % len(args))
try:
self.__root
except AttributeError:
self.__root = root = [] # sentinel node
root[:] = [root, root, None]
self.__map = {}
self.__update(*args, **kwds)
def __setitem__(self, key, value, dict_setitem=dict.__setitem__):
'od.__setitem__(i, y) <==> od[i]=y'
# Setting a new item creates a new link which goes at the end of the linked
# list, and the inherited dictionary is updated with the new key/value pair.
if key not in self:
root = self.__root
last = root[0]
last[1] = root[0] = self.__map[key] = [last, root, key]
dict_setitem(self, key, value)
def __delitem__(self, key, dict_delitem=dict.__delitem__):
'od.__delitem__(y) <==> del od[y]'
# Deleting an existing item uses self.__map to find the link which is
# then removed by updating the links in the predecessor and successor nodes.
dict_delitem(self, key)
link_prev, link_next, key = self.__map.pop(key)
link_prev[1] = link_next
link_next[0] = link_prev
def __iter__(self):
'od.__iter__() <==> iter(od)'
root = self.__root
curr = root[1]
while curr is not root:
yield curr[2]
curr = curr[1]
def __reversed__(self):
'od.__reversed__() <==> reversed(od)'
root = self.__root
curr = root[0]
while curr is not root:
yield curr[2]
curr = curr[0]
def clear(self):
'od.clear() -> None. Remove all items from od.'
try:
for node in self.__map.itervalues():
del node[:]
root = self.__root
root[:] = [root, root, None]
self.__map.clear()
except AttributeError:
pass
dict.clear(self)
def popitem(self, last=True):
'''od.popitem() -> (k, v), return and remove a (key, value) pair.
Pairs are returned in LIFO order if last is true or FIFO order if false.
'''
if not self:
raise KeyError('dictionary is empty')
root = self.__root
if last:
link = root[0]
link_prev = link[0]
link_prev[1] = root
root[0] = link_prev
else:
link = root[1]
link_next = link[1]
root[1] = link_next
link_next[0] = root
key = link[2]
del self.__map[key]
value = dict.pop(self, key)
return key, value
# -- the following methods do not depend on the internal structure --
def keys(self):
'od.keys() -> list of keys in od'
return list(self)
def values(self):
'od.values() -> list of values in od'
return [self[key] for key in self]
def items(self):
'od.items() -> list of (key, value) pairs in od'
return [(key, self[key]) for key in self]
def iterkeys(self):
'od.iterkeys() -> an iterator over the keys in od'
return iter(self)
def itervalues(self):
'od.itervalues -> an iterator over the values in od'
for k in self:
yield self[k]
def iteritems(self):
'od.iteritems -> an iterator over the (key, value) items in od'
for k in self:
yield (k, self[k])
def update(*args, **kwds):
'''od.update(E, **F) -> None. Update od from dict/iterable E and F.
If E is a dict instance, does: for k in E: od[k] = E[k]
If E has a .keys() method, does: for k in E.keys(): od[k] = E[k]
Or if E is an iterable of items, does: for k, v in E: od[k] = v
In either case, this is followed by: for k, v in F.items(): od[k] = v
'''
if len(args) > 2:
raise TypeError('update() takes at most 2 positional '
'arguments (%d given)' % (len(args),))
elif not args:
raise TypeError('update() takes at least 1 argument (0 given)')
self = args[0]
# Make progressively weaker assumptions about "other"
other = ()
if len(args) == 2:
other = args[1]
if isinstance(other, dict):
for key in other:
self[key] = other[key]
elif hasattr(other, 'keys'):
for key in other.keys():
self[key] = other[key]
else:
for key, value in other:
self[key] = value
for key, value in kwds.items():
self[key] = value
__update = update # let subclasses override update without breaking __init__
__marker = object()
def pop(self, key, default=__marker):
'''od.pop(k[,d]) -> v, remove specified key and return the corresponding value.
If key is not found, d is returned if given, otherwise KeyError is raised.
'''
if key in self:
result = self[key]
del self[key]
return result
if default is self.__marker:
raise KeyError(key)
return default
def setdefault(self, key, default=None):
'od.setdefault(k[,d]) -> od.get(k,d), also set od[k]=d if k not in od'
if key in self:
return self[key]
self[key] = default
return default
def __repr__(self, _repr_running={}):
'od.__repr__() <==> repr(od)'
call_key = id(self), _get_ident()
if call_key in _repr_running:
return '...'
_repr_running[call_key] = 1
try:
if not self:
return '%s()' % (self.__class__.__name__,)
return '%s(%r)' % (self.__class__.__name__, self.items())
finally:
del _repr_running[call_key]
def __reduce__(self):
'Return state information for pickling'
items = [[k, self[k]] for k in self]
inst_dict = vars(self).copy()
for k in vars(OrderedDict()):
inst_dict.pop(k, None)
if inst_dict:
return (self.__class__, (items,), inst_dict)
return self.__class__, (items,)
def copy(self):
'od.copy() -> a shallow copy of od'
return self.__class__(self)
@classmethod
def fromkeys(cls, iterable, value=None):
'''OD.fromkeys(S[, v]) -> New ordered dictionary with keys from S
and values equal to v (which defaults to None).
'''
d = cls()
for key in iterable:
d[key] = value
return d
def __eq__(self, other):
'''od.__eq__(y) <==> od==y. Comparison to another OD is order-sensitive
while comparison to a regular mapping is order-insensitive.
'''
if isinstance(other, OrderedDict):
return len(self)==len(other) and self.items() == other.items()
return dict.__eq__(self, other)
def __ne__(self, other):
return not self == other
# -- the following methods are only used in Python 2.7 --
def viewkeys(self):
"od.viewkeys() -> a set-like object providing a view on od's keys"
return KeysView(self)
def viewvalues(self):
"od.viewvalues() -> an object providing a view on od's values"
return ValuesView(self)
def viewitems(self):
"od.viewitems() -> a set-like object providing a view on od's items"
return ItemsView(self)

112
contrib/promix/main.py Executable file
View File

@ -0,0 +1,112 @@
#!/usr/bin/env python
#
# Implementation of PRO training and extensions to train phrase weights
#
import gzip
import logging
from numpy import array
import optparse
import os.path
import sys
from nbest import *
from sampler import *
from train import *
logging.basicConfig(format = "%(asctime)-15s %(message)s")
log = logging.getLogger('main')
log.setLevel(logging.DEBUG)
class Config:
def __init__(self):
self.parser = optparse.OptionParser(usage="%prog [options] ")
self.parser.add_option("-t", "--trainer", action="store",\
dest="trainer", metavar="TYPE", type="choice", choices=("pro","mix"),\
default="pro",\
help="type of trainer to run (pro,mix)")
self.parser.add_option("-n", "--nbest", action="append", \
dest="nbest", metavar="NBEST-FILE",\
help="nbest output file(s) from decoder")
self.parser.add_option("-S", "--scfile", action="append",\
dest="score", metavar="SCORE-FILE",\
help="score file(s) from extractor (in same order as nbests)")
self.parser.add_option("-p", "--phrase-table" , action="append",\
dest="ttable", metavar="TTABLE",\
help="ttable to be used in mixture model training")
self.parser.add_option("-i", "--input-file", action="store",\
dest="input_file", metavar="INPUT-FILE",
help="source text file")
self.parser.add_option("-m", "--moses-bin-dir", action="store",\
dest="moses_bin_dir", metavar="DIR",
help="directory containing Moses binaries",
default=os.path.expanduser("~/moses/bin"))
self.nbest_files = []
self.score_files = []
self.ttables = []
def parse(self,args=sys.argv[1:]):
(options,args) = self.parser.parse_args(args)
self.nbest_files = options.nbest
self.score_files = options.score
self.ttables = options.ttable
self.input_file = options.input_file
self.trainer = options.trainer
self.moses_bin_dir = options.moses_bin_dir
if not self.nbest_files:
self.nbest_files = ["data/esen.nc.nbest.segment"]
if not self.score_files:
self.score_files = ["data/esen.nc.scores"]
if len(self.nbest_files) != len(self.score_files):
self.parser.error("Must have equal numbers of score files and nbest files")
if self.trainer == "mix":
if not self.input_file or not self.ttables:
self.parser.error("Need to specify input file and ttables for mix training")
#if len(self.ttables) != 2:
# self.parser.error("Can only train mix model with 2 ttables at the moment")
def main():
config = Config()
config.parse()
samples = []
sampler = HopkinsMaySampler()
nbests = 0
for nbest_file,score_data_file in zip(config.nbest_files,config.score_files):
log.debug("nbest: " + nbest_file + "; score:" + score_data_file)
segments = False
if config.trainer == "mix": segments = True
for nbest in get_scored_nbests(nbest_file, score_data_file, config.input_file, segments=segments):
samples += sampler.sample(nbest)
nbests += 1
log.debug("Samples loaded")
trainer = None
if config.trainer == "mix":
# Add the phrase table scores
scorer = MosesPhraseScorer(config.ttables)
log.debug("Scoring samples...")
for sample in samples:
scorer.add_scores(sample.hyp1)
scorer.add_scores(sample.hyp2)
log.debug("...samples scored")
trainer = MixtureModelTrainer(samples)
elif config.trainer == "pro":
trainer = ProTrainer(samples)
else: assert(0)
log.debug("Starting training...")
weights,mix_weights = trainer.train(debug=False)
log.debug("...training complete")
for i,w in enumerate(weights):
print "F%d %10.8f" % (i,w)
for i,f in enumerate(mix_weights):
for j,w in enumerate(f):
print "M%d_%d %10.8f" % (i,j,w)
if __name__ == "__main__":
main()

230
contrib/promix/nbest.py Normal file
View File

@ -0,0 +1,230 @@
#!/usr/bin/env python
import gzip
import os
import re
import numpy as np
import sys
from bleu import BleuScorer
from coll import OrderedDict
# Edit to set moses python path
sys.path.append(os.path.dirname(__file__) + "/../python")
import moses.dictree as binpt
class DataFormatException(Exception):
pass
class Hypothesis:
def __init__(self,text,fv,segments=False):
self.alignment = [] #only stored for segmented hypos
self.tokens = [] #only stored for segmented hypos
if not segments:
self.text = text
# Triples of (source-start, source-end, target-end) where segments end positions
# are 1 beyond the last token
else:
# recover segmentation
self.tokens = []
align_re = re.compile("\|(\d+)-(\d+)\|")
for token in text.split():
match = align_re.match(token)
if match:
self.alignment.append\
((int(match.group(1)), 1+int(match.group(2)), len(self.tokens)))
else:
self.tokens.append(token)
self.text = " ".join(self.tokens)
if not self.alignment:
raise DataFormatException("Expected segmentation information not found in nbest")
self.fv = np.array(fv)
self.score = 0
def __str__(self):
return "{text=%s fv=%s score=%5.4f}" % (self.text, str(self.fv), self.score)
class NBestList:
def __init__(self,id):
self.id = id
self.hyps = []
# Maps feature ids (short feature names) to their values
_feature_index = {}
def set_feature_start(name,index):
indexes = _feature_index.get(name, [index,0])
indexes[0] = index
_feature_index[name] = indexes
def set_feature_end(name,index):
indexes = _feature_index.get(name, [0,index])
indexes[1] = index
_feature_index[name] = indexes
def get_feature_index(name):
return _feature_index.get(name, [0,0])
def get_nbests(nbest_file, segments=False):
"""Iterate through nbest lists"""
if nbest_file.endswith("gz"):
fh = gzip.GzipFile(nbest_file)
else:
fh = open(nbest_file)
lineno = 0
nbest = None
for line in fh:
fields = line.split(" ||| ")
if len(fields) != 4:
raise DataFormatException("nbest(%d): %s" % (lineno,line))
(id, text, scores, total) = fields
if nbest and nbest.id != id:
yield nbest
nbest = None
if not nbest:
nbest = NBestList(id)
fv = []
score_name = None
for score in scores.split():
if score.endswith(":"):
score = score[:-1]
if score_name:
set_feature_end(score_name,len(fv))
score_name = score
set_feature_start(score_name,len(fv))
else:
fv.append(float(score))
if score_name: set_feature_end(score_name,len(fv))
hyp = Hypothesis(text[:-1],fv,segments)
nbest.hyps.append(hyp)
if nbest:
yield nbest
def get_scores(score_data_file):
"""Iterate through the score data, returning a set of scores for each sentence"""
scorer = BleuScorer()
fh = open(score_data_file)
lineno = 0
score_vectors = None
for line in fh:
if line.startswith("SCORES_TXT_BEGIN"):
score_vectors = []
elif line.startswith("SCORES_TXT_END"):
scores = [scorer.score(score_vector) for score_vector in score_vectors]
yield scores
else:
score_vectors.append([float(i) for i in line[:-1].split()])
def get_scored_nbests(nbest_file, score_data_file, input_file, segments=False):
score_gen = get_scores(score_data_file)
input_gen = None
if input_file: input_gen = open(input_file)
try:
for nbest in get_nbests(nbest_file, segments=segments):
scores = score_gen.next()
if len(scores) != len(nbest.hyps):
raise DataFormatException("Length of nbest %s does not match score list (%d != %d)" %
(nbest.id,len(nbest.hyps), len(scores)))
input_line = None
if input_gen:
input_line = input_gen.next()[:-1]
for hyp,score in zip(nbest.hyps, scores):
hyp.score = score
hyp.input_line = input_line
yield nbest
except StopIteration:
raise DataFormatException("Score file shorter than nbest list file")
class PhraseCache:
"""An LRU cache for ttable lookups"""
def __init__(self, max_size):
self.max_size = max_size
self.pairs_to_scores = OrderedDict()
def get(self, source, target):
key = (source,target)
scores = self.pairs_to_scores.get(key,None)
if scores:
# cache hit - update access time
del self.pairs_to_scores[key]
self.pairs_to_scores[key] = scores
return scores
def add(self,source,target,scores):
key = (source,target)
self.pairs_to_scores[key] = scores
while len(self.pairs_to_scores) > self.max_size:
self.pairs_to_scores.popitem(last=False)
#
# Should I store full lists of options, or just phrase pairs?
# Should probably store phrase-pairs, but may want to add
# high scoring pairs (say, 20?) when I load the translations
# of a given phrase
#
class CachedPhraseTable:
def __init__(self,ttable_file,nscores=5,cache_size=20000):
wa = False
if binpt.PhraseDictionaryTree.canLoad(ttable_file,True):
# assume word alignment is included
wa = True
self.ttable = binpt.PhraseDictionaryTree(ttable_file,nscores = nscores,wa = wa, tableLimit=0)
self.cache = PhraseCache(cache_size)
self.nscores = nscores
def get_scores(self,phrase):
source = " ".join(phrase[0])
target_tuple = tuple(phrase[1])
target = " ".join(target_tuple)
scores = self.cache.get(source,target)
if not scores:
# cache miss
scores = [0] * (self.nscores-1) # ignore penalty
entries = self.ttable.query(source, converter=None)
# find correct target
for entry in entries:
if entry.rhs == target_tuple:
scores = entry.scores[:-1]
break
#print "QUERY",source,"|||",target,"|||",scores
self.cache.add(source,target,scores)
#else:
# print "CACHE",source,"|||",target,"|||",scores
return scores
class MosesPhraseScorer:
def __init__(self,ttable_files, cache_size=20000):
self.ttables = []
for ttable_file in ttable_files:
self.ttables.append(CachedPhraseTable(ttable_file, cache_size=cache_size))
def add_scores(self, hyp):
"""Add the phrase scores to a hypothesis"""
# Collect up the phrase pairs
phrases = []
source_tokens = hyp.input_line.split()
tgt_st = 0
if not hyp.alignment:
raise DataFormatException("Alignments missing from: " + str(hyp))
for src_st,src_end,tgt_end in hyp.alignment:
phrases.append((source_tokens[src_st:src_end], hyp.tokens[tgt_st:tgt_end]))
tgt_st = tgt_end
# Look up the scores
phrase_scores = []
for ttable in self.ttables:
phrase_scores.append([])
for phrase in phrases:
phrase_scores[-1].append(ttable.get_scores(phrase))
# phrase_scores = np.array(phrase_scores)
# eps = np.exp(-100)
# phrase_scores[phrase_scores<eps]=eps
floor = np.exp(-100)
phrase_scores = np.clip(np.array(phrase_scores), floor, np.inf)
hyp.phrase_scores = phrase_scores

44
contrib/promix/sampler.py Executable file
View File

@ -0,0 +1,44 @@
#!/usr/bin/env python
#
#
#
import heapq
import math
import random
import sys
from bleu import BleuScorer
class Sample:
"""A pair of hypotheses, and their score difference"""
def __init__(self,hyp1,hyp2):
self.hyp1 = hyp1
self.hyp2 = hyp2
self.diff = abs(hyp1.score-hyp2.score)
def __cmp__(self,other):
return cmp(self.diff,other.diff)
class HopkinsMaySampler:
"""Implements Hopkins & May sampling"""
def __init__(self):
self.ncandidates = 5000 # Gamma in Hopkins and May
self.nsamples = 50 # Xi in Hopkins and May
self.min_diff = 0.05 # Minimum scoring difference
def sample(self,nbest):
samples = []
for i in xrange(self.ncandidates):
hyp1 = random.choice(nbest.hyps)
hyp2 = random.choice(nbest.hyps)
sample = Sample(hyp1,hyp2)
if sample.diff < self.min_diff: continue
# maintain nsamples biggest samples
heapq.heappush(samples,sample)
while len(samples) > self.nsamples:
heapq.heappop(samples)
return samples

25
contrib/promix/test.py Executable file
View File

@ -0,0 +1,25 @@
#!/usr/bin/env python
import unittest
import test_bleu
import test_main
import test_moses
import test_nbest
import test_sampler
import test_train
def main():
test_list = []
test_list.append(test_bleu.suite)
test_list.append(test_main.suite)
# test_list.append(test_moses.suite)
test_list.append(test_nbest.suite)
test_list.append(test_sampler.suite)
test_list.append(test_train.suite)
suite = unittest.TestSuite(test_list)
unittest.TextTestRunner().run(suite)
if __name__ == "__main__":
main()

28
contrib/promix/test_bleu.py Executable file
View File

@ -0,0 +1,28 @@
#!/usr/bin/env python
import math
import unittest
from bleu import *
class TestBleuScorer(unittest.TestCase):
def setUp(self):
self.scorer = BleuScorer()
def test_wrong_length(self):
self.assertRaises(BleuScoreException, self.scorer.score, [1]*8)
self.assertRaises(BleuScoreException, self.scorer.score, [1]*10)
def test_score(self):
stats = [2,5,0,1,1,1,1,1,5]
self.assertAlmostEqual(self.scorer.score(stats), 1/math.sqrt(2))
def test_brevity(self):
stats = [2,2,2,2,2,2,2,2,3]
self.assertAlmostEqual(self.scorer.score(stats), math.exp(1 - 3.0/2.0))
if __name__ == "__main__":
unittest.main()
suite = unittest.TestLoader().loadTestsFromTestCase(TestBleuScorer)

5
contrib/promix/test_data/README Normal file
View File

@ -0,0 +1,5 @@
These nbest lists are created using the model in
data/esen.epnc-lin.model
Note that the nc-ep weights for this are:
[[0.54471993730312251, 0.45528006269687754], [0.56546688367708142, 0.43453311632291863], [0.55867730373453584, 0.44132269626546422], [0.46645964485220004, 0.53354035514779996]]

134446
contrib/promix/test_data/esen.ep.model.filtered/phrase-table.0-0.1.1 Normal file
View File

File diff suppressed because it is too large Load Diff

BIN
contrib/promix/test_data/esen.ep.model.filtered/phrase-table.0-0.1.1.binphr.idx Normal file
View File

Binary file not shown.

BIN
contrib/promix/test_data/esen.ep.model.filtered/phrase-table.0-0.1.1.binphr.srctree.wa Normal file
View File

Binary file not shown.

8
contrib/promix/test_data/esen.ep.model.filtered/phrase-table.0-0.1.1.binphr.srcvoc Normal file
View File

@ -0,0 +1,8 @@
7 del
6 lados
5 ambos
4 en
3 políticos
2 los
1 a
0 Atlántico

BIN
contrib/promix/test_data/esen.ep.model.filtered/phrase-table.0-0.1.1.binphr.tgtdata.wa Normal file
View File

Binary file not shown.

10713
contrib/promix/test_data/esen.ep.model.filtered/phrase-table.0-0.1.1.binphr.tgtvoc Normal file
View File

File diff suppressed because it is too large Load Diff

15117
contrib/promix/test_data/esen.nc.model.filtered/phrase-table.0-0.1.1 Normal file
View File

File diff suppressed because it is too large Load Diff

BIN
contrib/promix/test_data/esen.nc.model.filtered/phrase-table.0-0.1.1.binphr.idx Normal file
View File

Binary file not shown.

BIN
contrib/promix/test_data/esen.nc.model.filtered/phrase-table.0-0.1.1.binphr.srctree.wa Normal file
View File

Binary file not shown.

9
contrib/promix/test_data/esen.nc.model.filtered/phrase-table.0-0.1.1.binphr.srcvoc Normal file
View File

@ -0,0 +1,9 @@
8 domando
7 del
6 lados
5 ambos
4 en
3 políticos
2 los
1 a
0 Atlántico

BIN
contrib/promix/test_data/esen.nc.model.filtered/phrase-table.0-0.1.1.binphr.tgtdata.wa Normal file
View File

Binary file not shown.

4432
contrib/promix/test_data/esen.nc.model.filtered/phrase-table.0-0.1.1.binphr.tgtvoc Normal file
View File

File diff suppressed because it is too large Load Diff

8
contrib/promix/test_data/phrases Normal file
View File

@ -0,0 +1,8 @@
domando a
los políticos
en ambos
lados del Atlántico
taming
politicians
on both
sides of the Atlantic

5
contrib/promix/test_data/phrases.ep Normal file
View File

@ -0,0 +1,5 @@
los políticos ||| politicians ||| 0-0 1-0 ||| 0.33497 0.180441 0.638586 0.0962213 2.718
en ambos ||| on both ||| 0-0 1-1 ||| 0.0908379 0.0213197 0.187399 0.0498198 2.718
lados del Atlántico ||| sides of the Atlantic ||| 0-0 1-1 1-2 2-3 ||| 0.62585 0.00702384 0.836364 0.0687874 2.718

4
contrib/promix/test_data/phrases.epnc-lin Normal file
View File

@ -0,0 +1,4 @@
domando a ||| taming ||| 0.0403496 0.00427574 0.558677 0.233252 2.718
los políticos ||| politicians ||| 0.449849 0.210571 0.739008 0.138315 2.718
en ambos ||| on both ||| 0.198423 0.0402601 0.270259 0.0519369 2.718
lados del Atlántico ||| sides of the Atlantic ||| 0.708608 0.00997589 0.803633 0.0530904 2.718

4
contrib/promix/test_data/phrases.nc Normal file
View File

@ -0,0 +1,4 @@
domando a ||| taming ||| 0-0 1-0 ||| 0.0740741 0.00756144 1 0.500047 2.718
los políticos ||| politicians ||| 0-0 1-0 ||| 0.545866 0.233725 0.818336 0.186463 2.718
en ambos ||| on both ||| 0-0 1-1 ||| 0.288344 0.0548148 0.335714 0.0543585 2.718
dos del Atlántico ||| sides of the Atlantic ||| 0-0 1-1 1-2 2-3 ||| 0.777778 0.0122444 0.777778 0.0351361 2.718

1
contrib/promix/test_data/test.nbest.input Normal file
View File

@ -0,0 +1 @@
domando a los políticos en ambos lados del Atlántico

100
contrib/promix/test_data/test.nbest.nbest Normal file
View File

@ -0,0 +1,100 @@
0 ||| taming politicians on both sides of the Atlantic ||| d: 0 -1.51037 0 0 -2.60639 0 0 lm: -36.0562 w: -8 tm: -5.97082 -14.8327 -2.41162 -9.32734 3.99959 ||| -16.9717
0 ||| taming the politicians on both sides of the Atlantic ||| d: 0 -1.07904 0 0 -2.22203 0 0 lm: -37.2846 w: -9 tm: -5.43337 -15.7907 -4.86623 -9.68878 3.99959 ||| -16.9885
0 ||| taming politicians on both sides of the Atlantic ||| d: 0 -2.24178 0 0 -2.59408 0 0 lm: -36.0562 w: -8 tm: -5.55036 -14.7311 -1.14332 -9.34122 2.99969 ||| -17.0321
0 ||| taming the politicians on both sides of the Atlantic ||| d: 0 -1.81045 0 0 -2.20973 0 0 lm: -37.2846 w: -9 tm: -5.01291 -15.689 -3.59793 -9.70266 2.99969 ||| -17.0489
0 ||| taming politicians on both sides of the Atlantic ||| d: 0 -1.8566 0 0 -2.76972 0 0 lm: -36.0562 w: -8 tm: -6.33782 -14.6804 -1.8329 -9.34609 3.99959 ||| -17.0555
0 ||| taming the politicians on both sides of the Atlantic ||| d: 0 -1.42527 0 0 -2.38536 0 0 lm: -37.2846 w: -9 tm: -5.80037 -15.6384 -4.28752 -9.70753 3.99959 ||| -17.0723
0 ||| taming politicians on both sides of the Atlantic ||| d: 0 -1.947 0 0 -2.66963 0 0 lm: -36.0562 w: -8 tm: -7.12207 -14.8138 -2.49723 -9.33158 4.99948 ||| -17.1661
0 ||| taming politicians on both sides of the Atlantic ||| d: 0 -1.36892 0 0 -2.85256 0 0 lm: -36.0562 w: -8 tm: -5.62394 -14.8136 -3.58667 -9.35962 3.99959 ||| -17.1714
0 ||| taming the politicians on both sides of the Atlantic ||| d: 0 -1.51567 0 0 -2.28527 0 0 lm: -37.2846 w: -9 tm: -6.58462 -15.7718 -4.95184 -9.69302 4.99948 ||| -17.183
0 ||| taming the politicians on both sides of the Atlantic ||| d: 0 -0.93759 0 0 -2.4682 0 0 lm: -37.2846 w: -9 tm: -5.08649 -15.7715 -6.04129 -9.72106 3.99959 ||| -17.1882
0 ||| taming politicians on both sides of the Atlantic ||| d: 0 -2.71301 0 0 -2.68303 0 0 lm: -36.0562 w: -8 tm: -6.41885 -14.6935 -1.25905 -9.34492 3.99959 ||| -17.1903
0 ||| taming the politicians on both sides of the Atlantic ||| d: 0 -2.28167 0 0 -2.29867 0 0 lm: -37.2846 w: -9 tm: -5.8814 -15.6515 -3.71367 -9.70637 3.99959 ||| -17.2071
0 ||| taming politicians on both sides of the Atlantic ||| d: 0 -1.56496 0 0 -2.75335 0 0 lm: -36.0562 w: -8 tm: -7.49271 -14.8064 -2.97497 -9.34339 4.99948 ||| -17.2472
0 ||| taming the politicians on both sides of the Atlantic ||| d: 0 -1.13363 0 0 -2.36899 0 0 lm: -37.2846 w: -9 tm: -6.95526 -15.7643 -5.42959 -9.70483 4.99948 ||| -17.264
0 ||| taming politicians on both sides of the Atlantic ||| d: 0 -1.78819 0 0 -2.89955 0 0 lm: -36.0562 w: -8 tm: -6.50046 -14.7823 -3.66165 -9.36203 4.99948 ||| -17.2958
0 ||| taming the politicians on both sides of the Atlantic ||| d: 0 -1.35686 0 0 -2.51519 0 0 lm: -37.2846 w: -9 tm: -5.96301 -15.7403 -6.11627 -9.72347 4.99948 ||| -17.3126
0 ||| taming politicians on both sides of the Atlantic ||| d: 0 -1.4211 0 0 -2.96742 0 0 lm: -36.0562 w: -8 tm: -6.94636 -14.7711 -4.14828 -9.33786 4.99948 ||| -17.3855
0 ||| taming the politicians on both sides of the Atlantic ||| d: 0 -0.989766 0 0 -2.58306 0 0 lm: -37.2846 w: -9 tm: -6.40891 -15.7291 -6.6029 -9.69931 4.99948 ||| -17.4023
0 ||| taming politicians on both sides of the Atlantic ||| d: 0 -1.5859 0 0 -3.00234 0 0 lm: -36.0562 w: -8 tm: -7.53826 -14.8794 -3.91516 -9.31284 4.99948 ||| -17.5338
0 ||| taming the politicians on both sides of the Atlantic ||| d: 0 -1.15457 0 0 -2.61798 0 0 lm: -37.2846 w: -9 tm: -7.00081 -15.8374 -6.36978 -9.67428 4.99948 ||| -17.5506
0 ||| taming the politicians on both sides of the Atlantic ||| d: 0 -1.52679 0 0 -2.89648 0 0 lm: -37.2846 w: -9 tm: -8.24923 -15.7898 -4.29661 -9.63452 4.99948 ||| -17.5634
0 ||| taming the politicians on both sides of the Atlantic ||| d: 0 -2.2582 0 0 -2.88417 0 0 lm: -37.2846 w: -9 tm: -7.82876 -15.6881 -3.02831 -9.6484 3.99959 ||| -17.6238
0 ||| taming the politicians on both sides of the Atlantic ||| d: 0 -1.87302 0 0 -3.05981 0 0 lm: -37.2846 w: -9 tm: -8.61622 -15.6375 -3.7179 -9.65327 4.99948 ||| -17.6472
0 ||| taming politicians on both sides of the Atlantic ||| d: 0 -2.40826 0 0 -3.29781 0 0 lm: -36.0562 w: -8 tm: -7.95188 -14.6816 -2.5975 -9.34956 4.99948 ||| -17.6561
0 ||| taming the politicians on both sides of the Atlantic ||| d: 0 -1.97693 0 0 -2.91345 0 0 lm: -37.2846 w: -9 tm: -7.41443 -15.6395 -5.05211 -9.711 4.99948 ||| -17.6729
0 ||| taming politicians on both sides of the Atlantic ||| d: 0 -2.02253 0 0 -3.06558 0 0 lm: -36.0562 w: -8 tm: -8.6895 -14.8606 -4.00077 -9.31707 5.99938 ||| -17.7283
0 ||| taming the politicians on both sides of the Atlantic ||| d: 0 -1.5912 0 0 -2.68122 0 0 lm: -37.2846 w: -9 tm: -8.15205 -15.8185 -6.45539 -9.67851 5.99938 ||| -17.7451
0 ||| taming the politicians on both sides of the Atlantic ||| d: 0 -1.96342 0 0 -2.95972 0 0 lm: -37.2846 w: -9 tm: -9.40047 -15.7709 -4.38222 -9.63875 5.99938 ||| -17.7579
0 ||| taming the politicians on both sides of the Atlantic ||| d: 0 -1.38534 0 0 -3.14265 0 0 lm: -37.2846 w: -9 tm: -7.90234 -15.7706 -5.47167 -9.66679 4.99948 ||| -17.7631
0 ||| taming the politicians on both sides of the Atlantic ||| d: 0 -2.72942 0 0 -2.97312 0 0 lm: -37.2846 w: -9 tm: -8.69725 -15.6506 -3.14405 -9.6521 4.99948 ||| -17.782
0 ||| taming politicians on both sides of the Atlantic ||| d: 0 -1.64049 0 0 -3.1493 0 0 lm: -36.0562 w: -8 tm: -9.06014 -14.8531 -4.47852 -9.32889 5.99938 ||| -17.8093
0 ||| taming politicians on both sides of the Atlantic ||| d: 0 -1.95601 0 0 -3.55535 0 0 lm: -36.0562 w: -8 tm: -4.91151 -15.1311 -3.72129 -9.93616 2.99969 ||| -17.8216
0 ||| taming the politicians on both sides of the Atlantic ||| d: 0 -1.20916 0 0 -2.76494 0 0 lm: -37.2846 w: -9 tm: -8.52269 -15.8111 -6.93313 -9.69033 5.99938 ||| -17.8261
0 ||| taming the politicians on both sides of the Atlantic ||| d: 0 -1.58137 0 0 -3.04344 0 0 lm: -37.2846 w: -9 tm: -9.77111 -15.7635 -4.85997 -9.65057 5.99938 ||| -17.8389
0 ||| taming politicians on both sides of the Atlantic ||| d: 0 -2.11662 0 0 -3.28144 0 0 lm: -36.0562 w: -8 tm: -9.10677 -14.8075 -3.73957 -9.34686 5.99938 ||| -17.8478
0 ||| taming the politicians on both sides of the Atlantic ||| d: 0 -1.68529 0 0 -2.89708 0 0 lm: -37.2846 w: -9 tm: -8.56932 -15.7655 -6.19418 -9.7083 5.99938 ||| -17.8646
0 ||| taming the politicians on both sides of the Atlantic ||| d: 0 -1.80461 0 0 -3.18964 0 0 lm: -37.2846 w: -9 tm: -8.77886 -15.7394 -5.54665 -9.66921 5.99938 ||| -17.8875
0 ||| taming politicians on both sides of the Atlantic ||| d: 0 -2.37528 0 0 -3.60234 0 0 lm: -36.0562 w: -8 tm: -5.78803 -15.0999 -3.79627 -9.93857 3.99958 ||| -17.946
0 ||| taming the politicians on both sides of the Atlantic ||| d: 0 -1.43751 0 0 -3.25751 0 0 lm: -37.2846 w: -9 tm: -9.22476 -15.7282 -6.03327 -9.64504 5.99938 ||| -17.9772
0 ||| taming politicians on both sides of the Atlantic ||| d: 0 -1.97276 0 0 -3.49551 0 0 lm: -36.0562 w: -8 tm: -8.56043 -14.7723 -4.91288 -9.34133 5.99938 ||| -17.9861
0 ||| taming the politicians on both sides of the Atlantic ||| d: 0 -1.54143 0 0 -3.11115 0 0 lm: -37.2846 w: -9 tm: -8.02298 -15.7302 -7.36749 -9.70277 5.99938 ||| -18.0029
0 ||| taming politicians on both sides of the Atlantic ||| d: 0 -2.00819 0 0 -3.67021 0 0 lm: -36.0562 w: -8 tm: -6.23393 -15.0887 -4.2829 -9.91441 3.99958 ||| -18.0357
0 ||| taming politicians on both sides of the Atlantic , ||| d: 0 -2.61446 0 0 -2.59408 0 0 lm: -37.5177 w: -9 tm: -6.22594 -14.8893 -3.54625 -12.0454 2.99969 ||| -18.0629
0 ||| taming the politicians on both sides of the Atlantic , ||| d: 0 -2.18313 0 0 -2.20973 0 0 lm: -38.7461 w: -10 tm: -5.68849 -15.8473 -6.00087 -12.4069 2.99969 ||| -18.0797
0 ||| taming politicians on both sides of the Atlantic , ||| d: 0 -1.97768 0 0 -2.76972 0 0 lm: -37.5177 w: -9 tm: -7.93613 -14.6804 -4.69485 -11.2579 3.99959 ||| -18.097
0 ||| taming the politicians on both sides of the Atlantic , ||| d: 0 -1.54635 0 0 -2.38536 0 0 lm: -38.7461 w: -10 tm: -7.39868 -15.6384 -7.14947 -11.6193 3.99959 ||| -18.1138
0 ||| taming the politicians on both sides of the Atlantic ||| d: 0 -1.60232 0 0 -3.29243 0 0 lm: -37.2846 w: -9 tm: -9.81666 -15.8365 -5.80015 -9.62002 5.99938 ||| -18.1255
0 ||| taming politicians on both sides of the Atlantic , ||| d: 0 -1.8227 0 0 -2.66963 0 0 lm: -37.5177 w: -9 tm: -8.25244 -14.8138 -5.95756 -11.1747 4.99948 ||| -18.1464
0 ||| taming the politicians on both sides of the Atlantic , ||| d: 0 -1.39137 0 0 -2.28527 0 0 lm: -38.7461 w: -10 tm: -7.71499 -15.7718 -8.41217 -11.5362 4.99948 ||| -18.1632
0 ||| taming politicians on both sides of the Atlantic , ||| d: 0 -2.58871 0 0 -2.68303 0 0 lm: -37.5177 w: -9 tm: -7.54922 -14.6935 -4.71938 -11.1881 3.99959 ||| -18.1705
0 ||| taming politicians on both sides of the Atlantic ||| d: 0 -2.17299 0 0 -3.70513 0 0 lm: -36.0562 w: -8 tm: -6.82583 -15.197 -4.04978 -9.88939 3.99958 ||| -18.184
0 ||| taming the politicians on both sides of the Atlantic , ||| d: 0 -2.15738 0 0 -2.29867 0 0 lm: -38.7461 w: -10 tm: -7.01177 -15.6515 -7.174 -11.5495 3.99959 ||| -18.1873
0 ||| taming the politicians on both sides of the Atlantic ||| d: 0 -2.42468 0 0 -3.5879 0 0 lm: -37.2846 w: -9 tm: -10.2303 -15.6386 -4.48249 -9.65674 5.99938 ||| -18.2478
0 ||| taming politicians on both sides of the Atlantic , ||| d: 0 -1.66389 0 0 -2.89955 0 0 lm: -37.5177 w: -9 tm: -7.63083 -14.7823 -7.12198 -11.2052 4.99948 ||| -18.2761
0 ||| taming politicians on both sides of the Atlantic , ||| d: 0 -1.68604 0 0 -2.75335 0 0 lm: -37.5177 w: -9 tm: -9.09102 -14.8064 -5.83693 -11.2552 4.99948 ||| -18.2887
0 ||| taming the politicians on both sides of the Atlantic , ||| d: 0 -1.23256 0 0 -2.51519 0 0 lm: -38.7461 w: -10 tm: -7.09338 -15.7403 -9.5766 -11.5666 4.99948 ||| -18.2929
0 ||| taming the politicians on both sides of the Atlantic , ||| d: 0 -1.25471 0 0 -2.36899 0 0 lm: -38.7461 w: -10 tm: -8.55357 -15.7643 -8.29154 -11.6166 4.99948 ||| -18.3055
0 ||| taming the politicians on both sides of the Atlantic ||| d: 0 -2.03895 0 0 -3.35567 0 0 lm: -37.2846 w: -9 tm: -10.9679 -15.8176 -5.88577 -9.62425 6.99927 ||| -18.32
0 ||| taming politicians on both sides of the Atlantic ||| d: 0 -2.60962 0 0 -3.76837 0 0 lm: -36.0562 w: -8 tm: -7.97707 -15.1781 -4.1354 -9.89362 4.99948 ||| -18.3785
0 ||| taming the politicians on both sides of the Atlantic ||| d: 0 -1.65691 0 0 -3.43939 0 0 lm: -37.2846 w: -9 tm: -11.3385 -15.8102 -6.36351 -9.63607 6.99927 ||| -18.401
0 ||| taming politicians on both sides of the Atlantic ||| d: 0 -2.19215 0 0 -3.67739 0 0 lm: -36.0562 w: -8 tm: -10.6742 -14.8543 -5.24311 -9.33236 6.99927 ||| -18.4099
0 ||| taming the politicians on both sides of the Atlantic ||| d: 0 -1.76082 0 0 -3.29303 0 0 lm: -37.2846 w: -9 tm: -10.1368 -15.8122 -7.69773 -9.6938 6.99927 ||| -18.4267
0 ||| taming politicians on both sides of the Atlantic , ||| d: 0 -1.54218 0 0 -2.96742 0 0 lm: -37.5177 w: -9 tm: -8.54467 -14.7711 -7.01023 -11.2497 4.99948 ||| -18.427
0 ||| taming the politicians on both sides of the Atlantic ||| d: 0 -2.13304 0 0 -3.57153 0 0 lm: -37.2846 w: -9 tm: -11.3852 -15.7646 -5.62456 -9.65404 6.99927 ||| -18.4395
0 ||| taming the politicians on both sides of the Atlantic , ||| d: 0 -1.11085 0 0 -2.58306 0 0 lm: -38.7461 w: -10 tm: -8.00723 -15.7291 -9.46485 -11.6111 4.99948 ||| -18.4438
0 ||| taming politicians on both sides of the Atlantic ||| d: 0 -2.22758 0 0 -3.85209 0 0 lm: -36.0562 w: -8 tm: -8.34771 -15.1707 -4.61314 -9.90544 4.99948 ||| -18.4595
0 ||| taming the politicians on both sides of the Atlantic ||| d: 0 -1.98917 0 0 -3.7856 0 0 lm: -37.2846 w: -9 tm: -10.8388 -15.7293 -6.79787 -9.64851 6.99927 ||| -18.5778
0 ||| taming politicians on both sides of the Atlantic , ||| d: 0 -2.0125 0 0 -2.60639 0 0 lm: -37.5177 w: -9 tm: -8.02561 -15.1979 -6.07103 -12.091 3.99959 ||| -18.6217
0 ||| taming politicians on both sides of the Atlantic ||| d: 0 -2.55985 0 0 -4.1983 0 0 lm: -36.0562 w: -8 tm: -7.848 -15.0898 -5.0475 -9.91788 4.99948 ||| -18.6363
0 ||| taming politicians on both sides of the Atlantic , ||| d: 0 -2.28396 0 0 -3.29781 0 0 lm: -37.5177 w: -9 tm: -9.08225 -14.6816 -6.05783 -11.1927 4.99948 ||| -18.6364
0 ||| taming the politicians on both sides of the Atlantic , ||| d: 0 -1.58117 0 0 -2.22203 0 0 lm: -38.7461 w: -10 tm: -7.48816 -16.1559 -8.52564 -12.4524 3.99959 ||| -18.6385
0 ||| taming the politicians on both sides of the Atlantic , ||| d: 0 -1.85263 0 0 -2.91345 0 0 lm: -38.7461 w: -10 tm: -8.5448 -15.6395 -8.51244 -11.5541 4.99948 ||| -18.6532
0 ||| taming the politicians on both sides of the Atlantic , ||| d: 0 -2.63088 0 0 -2.88417 0 0 lm: -38.7461 w: -10 tm: -8.50434 -15.8464 -5.43125 -12.3526 3.99959 ||| -18.6546
0 ||| taming politicians on both sides of the Atlantic , ||| d: 0 -1.86965 0 0 -2.85256 0 0 lm: -37.5177 w: -9 tm: -7.44015 -15.0375 -7.00556 -11.9328 3.99959 ||| -18.6588
0 ||| taming the politicians on both sides of the Atlantic , ||| d: 0 -1.43831 0 0 -2.4682 0 0 lm: -38.7461 w: -10 tm: -6.9027 -15.9955 -9.46017 -12.2942 3.99959 ||| -18.6756
0 ||| taming the politicians on both sides of the Atlantic , ||| d: 0 -1.9941 0 0 -3.05981 0 0 lm: -38.7461 w: -10 tm: -10.2145 -15.6375 -6.57985 -11.5651 4.99948 ||| -18.6887
0 ||| taming politicians on both sides of the Atlantic , ||| d: 0 -1.89824 0 0 -3.06558 0 0 lm: -37.5177 w: -9 tm: -9.81987 -14.8606 -7.4611 -11.1602 5.99938 ||| -18.7085
0 ||| taming the politicians on both sides of the Atlantic , ||| d: 0 -1.4669 0 0 -2.68122 0 0 lm: -38.7461 w: -10 tm: -9.28242 -15.8185 -9.91572 -11.5217 5.99938 ||| -18.7253
0 ||| taming the politicians on both sides of the Atlantic , ||| d: 0 -1.83912 0 0 -2.95972 0 0 lm: -38.7461 w: -10 tm: -10.5308 -15.7709 -7.84255 -11.4819 5.99938 ||| -18.7381
0 ||| taming the politicians on both sides of the Atlantic ||| d: 0 -1.07039 0 0 -3.90067 0 0 lm: -37.2846 w: -9 tm: -8.33352 -16.8256 -6.4114 -10.5648 3.99959 ||| -18.7608
0 ||| taming the politicians on both sides of the Atlantic , ||| d: 0 -2.60513 0 0 -2.97312 0 0 lm: -38.7461 w: -10 tm: -9.82762 -15.6506 -6.60437 -11.4952 4.99948 ||| -18.7622
0 ||| taming politicians on both sides of the Atlantic , ||| d: 0 -1.99232 0 0 -3.28144 0 0 lm: -37.5177 w: -9 tm: -10.2371 -14.8075 -7.1999 -11.19 5.99938 ||| -18.828
0 ||| taming the politicians on both sides of the Atlantic , ||| d: 0 -1.56099 0 0 -2.89708 0 0 lm: -38.7461 w: -10 tm: -9.6997 -15.7655 -9.65451 -11.5514 5.99938 ||| -18.8448
0 ||| taming politicians on both sides of the Atlantic , ||| d: 0 -1.76157 0 0 -3.1493 0 0 lm: -37.5177 w: -9 tm: -10.6585 -14.8531 -7.34047 -11.2407 5.99938 ||| -18.8508
0 ||| taming the politicians on both sides of the Atlantic , ||| d: 0 -1.33024 0 0 -2.76494 0 0 lm: -38.7461 w: -10 tm: -10.121 -15.8111 -9.79508 -11.6021 5.99938 ||| -18.8676
0 ||| taming the politicians on both sides of the Atlantic , ||| d: 0 -1.68031 0 0 -3.18964 0 0 lm: -38.7461 w: -10 tm: -9.90924 -15.7394 -9.00698 -11.5123 5.99938 ||| -18.8678
0 ||| taming the politicians on both sides of the Atlantic , ||| d: 0 -1.70246 0 0 -3.04344 0 0 lm: -38.7461 w: -10 tm: -11.3694 -15.7635 -7.72192 -11.5624 5.99938 ||| -18.8804
0 ||| taming the politicians on both sides of the Atlantic ||| d: 0 -1.48965 0 0 -3.94766 0 0 lm: -37.2846 w: -9 tm: -9.21005 -16.7944 -6.48638 -10.5672 4.99948 ||| -18.8852
0 ||| taming politicians on both sides of the Atlantic , ||| d: 0 -2.25098 0 0 -3.60234 0 0 lm: -37.5177 w: -9 tm: -6.91841 -15.0999 -7.2566 -11.7817 3.99958 ||| -18.9263
0 ||| taming politicians on both sides of the Atlantic , ||| d: 0 -1.84846 0 0 -3.49551 0 0 lm: -37.5177 w: -9 tm: -9.6908 -14.7723 -8.37321 -11.1845 5.99938 ||| -18.9663
0 ||| taming the politicians on both sides of the Atlantic ||| d: 0 -1.12256 0 0 -4.01553 0 0 lm: -37.2846 w: -9 tm: -9.65595 -16.7832 -6.97301 -10.543 4.99948 ||| -18.9749
0 ||| taming the politicians on both sides of the Atlantic , ||| d: 0 -1.41713 0 0 -3.11115 0 0 lm: -38.7461 w: -10 tm: -9.15335 -15.7302 -10.8278 -11.5459 5.99938 ||| -18.9831
0 ||| taming the politicians on both sides of the Atlantic ||| d: 0 -2.20857 0 0 -3.96748 0 0 lm: -37.2846 w: -9 tm: -12.9526 -15.8114 -7.12811 -9.63954 7.99917 ||| -19.0016
0 ||| taming the politicians on both sides of the Atlantic , ||| d: 0 -1.5586 0 0 -3.25751 0 0 lm: -38.7461 w: -10 tm: -10.8231 -15.7282 -8.89523 -11.5568 5.99938 ||| -19.0187
0 ||| taming politicians on both sides of the Atlantic ||| d: 0 -2.77924 0 0 -4.38018 0 0 lm: -36.0562 w: -8 tm: -9.96178 -15.1719 -5.37774 -9.90891 5.99938 ||| -19.0601
0 ||| taming politicians on both sides of the Atlantic , ||| d: 0 -2.12927 0 0 -3.67021 0 0 lm: -37.5177 w: -9 tm: -7.83225 -15.0887 -7.14485 -11.8262 3.99958 ||| -19.0772
0 ||| taming the politicians on both sides of the Atlantic ||| d: 0 -1.28737 0 0 -4.05045 0 0 lm: -37.2846 w: -9 tm: -10.2478 -16.8915 -6.73989 -10.518 4.99948 ||| -19.1232
0 ||| taming politicians on both sides of the Atlantic , ||| d: 0 -2.08803 0 0 -3.00234 0 0 lm: -37.5177 w: -9 tm: -9.59305 -15.2447 -7.57457 -12.0765 4.99948 ||| -19.1838
0 ||| taming the politicians on both sides of the Atlantic , ||| d: 0 -1.6567 0 0 -2.61798 0 0 lm: -38.7461 w: -10 tm: -9.0556 -16.2026 -10.0292 -12.4379 4.99948 ||| -19.2006
0 ||| taming the politicians on both sides of the Atlantic , ||| d: 0 -2.02892 0 0 -2.89648 0 0 lm: -38.7461 w: -10 tm: -10.304 -16.155 -7.95602 -12.3982 4.99948 ||| -19.2134

100
contrib/promix/test_data/test.nbest.nbest.segments Normal file
View File

@ -0,0 +1,100 @@
0 ||| taming |0-1| politicians |2-3| on both |4-5| sides of the Atlantic |6-8| ||| d: 0 -1.51037 0 0 -2.60639 0 0 lm: -36.0562 w: -8 tm: -5.97082 -14.8327 -2.41162 -9.32734 3.99959 ||| -16.9717
0 ||| taming |0-1| the politicians |2-3| on both |4-5| sides of the Atlantic |6-8| ||| d: 0 -1.07904 0 0 -2.22203 0 0 lm: -37.2846 w: -9 tm: -5.43337 -15.7907 -4.86623 -9.68878 3.99959 ||| -16.9885
0 ||| taming |0-1| politicians |2-3| on both sides of the Atlantic |4-8| ||| d: 0 -2.24178 0 0 -2.59408 0 0 lm: -36.0562 w: -8 tm: -5.55036 -14.7311 -1.14332 -9.34122 2.99969 ||| -17.0321
0 ||| taming |0-1| the politicians |2-3| on both sides of the Atlantic |4-8| ||| d: 0 -1.81045 0 0 -2.20973 0 0 lm: -37.2846 w: -9 tm: -5.01291 -15.689 -3.59793 -9.70266 2.99969 ||| -17.0489
0 ||| taming |0-1| politicians |2-3| on both sides |4-6| of the Atlantic |7-8| ||| d: 0 -1.8566 0 0 -2.76972 0 0 lm: -36.0562 w: -8 tm: -6.33782 -14.6804 -1.8329 -9.34609 3.99959 ||| -17.0555
0 ||| taming |0-1| the politicians |2-3| on both sides |4-6| of the Atlantic |7-8| ||| d: 0 -1.42527 0 0 -2.38536 0 0 lm: -37.2846 w: -9 tm: -5.80037 -15.6384 -4.28752 -9.70753 3.99959 ||| -17.0723
0 ||| taming |0-1| politicians |2-3| on both |4-5| sides of the |6-7| Atlantic |8-8| ||| d: 0 -1.947 0 0 -2.66963 0 0 lm: -36.0562 w: -8 tm: -7.12207 -14.8138 -2.49723 -9.33158 4.99948 ||| -17.1661
0 ||| taming |0-1| politicians |2-3| on |4-4| both sides of the Atlantic |5-8| ||| d: 0 -1.36892 0 0 -2.85256 0 0 lm: -36.0562 w: -8 tm: -5.62394 -14.8136 -3.58667 -9.35962 3.99959 ||| -17.1714
0 ||| taming |0-1| the politicians |2-3| on both |4-5| sides of the |6-7| Atlantic |8-8| ||| d: 0 -1.51567 0 0 -2.28527 0 0 lm: -37.2846 w: -9 tm: -6.58462 -15.7718 -4.95184 -9.69302 4.99948 ||| -17.183
0 ||| taming |0-1| the politicians |2-3| on |4-4| both sides of the Atlantic |5-8| ||| d: 0 -0.93759 0 0 -2.4682 0 0 lm: -37.2846 w: -9 tm: -5.08649 -15.7715 -6.04129 -9.72106 3.99959 ||| -17.1882
0 ||| taming |0-1| politicians |2-3| on both sides of the |4-7| Atlantic |8-8| ||| d: 0 -2.71301 0 0 -2.68303 0 0 lm: -36.0562 w: -8 tm: -6.41885 -14.6935 -1.25905 -9.34492 3.99959 ||| -17.1903
0 ||| taming |0-1| the politicians |2-3| on both sides of the |4-7| Atlantic |8-8| ||| d: 0 -2.28167 0 0 -2.29867 0 0 lm: -37.2846 w: -9 tm: -5.8814 -15.6515 -3.71367 -9.70637 3.99959 ||| -17.2071
0 ||| taming |0-1| politicians |2-3| on both |4-5| sides |6-6| of the Atlantic |7-8| ||| d: 0 -1.56496 0 0 -2.75335 0 0 lm: -36.0562 w: -8 tm: -7.49271 -14.8064 -2.97497 -9.34339 4.99948 ||| -17.2472
0 ||| taming |0-1| the politicians |2-3| on both |4-5| sides |6-6| of the Atlantic |7-8| ||| d: 0 -1.13363 0 0 -2.36899 0 0 lm: -37.2846 w: -9 tm: -6.95526 -15.7643 -5.42959 -9.70483 4.99948 ||| -17.264
0 ||| taming |0-1| politicians |2-3| on |4-4| both sides of the |5-7| Atlantic |8-8| ||| d: 0 -1.78819 0 0 -2.89955 0 0 lm: -36.0562 w: -8 tm: -6.50046 -14.7823 -3.66165 -9.36203 4.99948 ||| -17.2958
0 ||| taming |0-1| the politicians |2-3| on |4-4| both sides of the |5-7| Atlantic |8-8| ||| d: 0 -1.35686 0 0 -2.51519 0 0 lm: -37.2846 w: -9 tm: -5.96301 -15.7403 -6.11627 -9.72347 4.99948 ||| -17.3126
0 ||| taming |0-1| politicians |2-3| on |4-4| both sides |5-6| of the Atlantic |7-8| ||| d: 0 -1.4211 0 0 -2.96742 0 0 lm: -36.0562 w: -8 tm: -6.94636 -14.7711 -4.14828 -9.33786 4.99948 ||| -17.3855
0 ||| taming |0-1| the politicians |2-3| on |4-4| both sides |5-6| of the Atlantic |7-8| ||| d: 0 -0.989766 0 0 -2.58306 0 0 lm: -37.2846 w: -9 tm: -6.40891 -15.7291 -6.6029 -9.69931 4.99948 ||| -17.4023
0 ||| taming |0-1| politicians |2-3| on |4-4| both |5-5| sides of the Atlantic |6-8| ||| d: 0 -1.5859 0 0 -3.00234 0 0 lm: -36.0562 w: -8 tm: -7.53826 -14.8794 -3.91516 -9.31284 4.99948 ||| -17.5338
0 ||| taming |0-1| the politicians |2-3| on |4-4| both |5-5| sides of the Atlantic |6-8| ||| d: 0 -1.15457 0 0 -2.61798 0 0 lm: -37.2846 w: -9 tm: -7.00081 -15.8374 -6.36978 -9.67428 4.99948 ||| -17.5506
0 ||| taming |0-1| the |2-2| politicians |3-3| on both |4-5| sides of the Atlantic |6-8| ||| d: 0 -1.52679 0 0 -2.89648 0 0 lm: -37.2846 w: -9 tm: -8.24923 -15.7898 -4.29661 -9.63452 4.99948 ||| -17.5634
0 ||| taming |0-1| the |2-2| politicians |3-3| on both sides of the Atlantic |4-8| ||| d: 0 -2.2582 0 0 -2.88417 0 0 lm: -37.2846 w: -9 tm: -7.82876 -15.6881 -3.02831 -9.6484 3.99959 ||| -17.6238
0 ||| taming |0-1| the |2-2| politicians |3-3| on both sides |4-6| of the Atlantic |7-8| ||| d: 0 -1.87302 0 0 -3.05981 0 0 lm: -37.2846 w: -9 tm: -8.61622 -15.6375 -3.7179 -9.65327 4.99948 ||| -17.6472
0 ||| taming |0-1| politicians |2-3| on both sides |4-6| of the |7-7| Atlantic |8-8| ||| d: 0 -2.40826 0 0 -3.29781 0 0 lm: -36.0562 w: -8 tm: -7.95188 -14.6816 -2.5975 -9.34956 4.99948 ||| -17.6561
0 ||| taming |0-1| the politicians |2-3| on both sides |4-6| of the |7-7| Atlantic |8-8| ||| d: 0 -1.97693 0 0 -2.91345 0 0 lm: -37.2846 w: -9 tm: -7.41443 -15.6395 -5.05211 -9.711 4.99948 ||| -17.6729
0 ||| taming |0-1| politicians |2-3| on |4-4| both |5-5| sides of the |6-7| Atlantic |8-8| ||| d: 0 -2.02253 0 0 -3.06558 0 0 lm: -36.0562 w: -8 tm: -8.6895 -14.8606 -4.00077 -9.31707 5.99938 ||| -17.7283
0 ||| taming |0-1| the politicians |2-3| on |4-4| both |5-5| sides of the |6-7| Atlantic |8-8| ||| d: 0 -1.5912 0 0 -2.68122 0 0 lm: -37.2846 w: -9 tm: -8.15205 -15.8185 -6.45539 -9.67851 5.99938 ||| -17.7451
0 ||| taming |0-1| the |2-2| politicians |3-3| on both |4-5| sides of the |6-7| Atlantic |8-8| ||| d: 0 -1.96342 0 0 -2.95972 0 0 lm: -37.2846 w: -9 tm: -9.40047 -15.7709 -4.38222 -9.63875 5.99938 ||| -17.7579
0 ||| taming |0-1| the |2-2| politicians |3-3| on |4-4| both sides of the Atlantic |5-8| ||| d: 0 -1.38534 0 0 -3.14265 0 0 lm: -37.2846 w: -9 tm: -7.90234 -15.7706 -5.47167 -9.66679 4.99948 ||| -17.7631
0 ||| taming |0-1| the |2-2| politicians |3-3| on both sides of the |4-7| Atlantic |8-8| ||| d: 0 -2.72942 0 0 -2.97312 0 0 lm: -37.2846 w: -9 tm: -8.69725 -15.6506 -3.14405 -9.6521 4.99948 ||| -17.782
0 ||| taming |0-1| politicians |2-3| on |4-4| both |5-5| sides |6-6| of the Atlantic |7-8| ||| d: 0 -1.64049 0 0 -3.1493 0 0 lm: -36.0562 w: -8 tm: -9.06014 -14.8531 -4.47852 -9.32889 5.99938 ||| -17.8093
0 ||| taming |0-1| politicians on |2-4| both sides of the Atlantic |5-8| ||| d: 0 -1.95601 0 0 -3.55535 0 0 lm: -36.0562 w: -8 tm: -4.91151 -15.1311 -3.72129 -9.93616 2.99969 ||| -17.8216
0 ||| taming |0-1| the politicians |2-3| on |4-4| both |5-5| sides |6-6| of the Atlantic |7-8| ||| d: 0 -1.20916 0 0 -2.76494 0 0 lm: -37.2846 w: -9 tm: -8.52269 -15.8111 -6.93313 -9.69033 5.99938 ||| -17.8261
0 ||| taming |0-1| the |2-2| politicians |3-3| on both |4-5| sides |6-6| of the Atlantic |7-8| ||| d: 0 -1.58137 0 0 -3.04344 0 0 lm: -37.2846 w: -9 tm: -9.77111 -15.7635 -4.85997 -9.65057 5.99938 ||| -17.8389
0 ||| taming |0-1| politicians |2-3| on both |4-5| sides |6-6| of the |7-7| Atlantic |8-8| ||| d: 0 -2.11662 0 0 -3.28144 0 0 lm: -36.0562 w: -8 tm: -9.10677 -14.8075 -3.73957 -9.34686 5.99938 ||| -17.8478
0 ||| taming |0-1| the politicians |2-3| on both |4-5| sides |6-6| of the |7-7| Atlantic |8-8| ||| d: 0 -1.68529 0 0 -2.89708 0 0 lm: -37.2846 w: -9 tm: -8.56932 -15.7655 -6.19418 -9.7083 5.99938 ||| -17.8646
0 ||| taming |0-1| the |2-2| politicians |3-3| on |4-4| both sides of the |5-7| Atlantic |8-8| ||| d: 0 -1.80461 0 0 -3.18964 0 0 lm: -37.2846 w: -9 tm: -8.77886 -15.7394 -5.54665 -9.66921 5.99938 ||| -17.8875
0 ||| taming |0-1| politicians on |2-4| both sides of the |5-7| Atlantic |8-8| ||| d: 0 -2.37528 0 0 -3.60234 0 0 lm: -36.0562 w: -8 tm: -5.78803 -15.0999 -3.79627 -9.93857 3.99958 ||| -17.946
0 ||| taming |0-1| the |2-2| politicians |3-3| on |4-4| both sides |5-6| of the Atlantic |7-8| ||| d: 0 -1.43751 0 0 -3.25751 0 0 lm: -37.2846 w: -9 tm: -9.22476 -15.7282 -6.03327 -9.64504 5.99938 ||| -17.9772
0 ||| taming |0-1| politicians |2-3| on |4-4| both sides |5-6| of the |7-7| Atlantic |8-8| ||| d: 0 -1.97276 0 0 -3.49551 0 0 lm: -36.0562 w: -8 tm: -8.56043 -14.7723 -4.91288 -9.34133 5.99938 ||| -17.9861
0 ||| taming |0-1| the politicians |2-3| on |4-4| both sides |5-6| of the |7-7| Atlantic |8-8| ||| d: 0 -1.54143 0 0 -3.11115 0 0 lm: -37.2846 w: -9 tm: -8.02298 -15.7302 -7.36749 -9.70277 5.99938 ||| -18.0029
0 ||| taming |0-1| politicians on |2-4| both sides |5-6| of the Atlantic |7-8| ||| d: 0 -2.00819 0 0 -3.67021 0 0 lm: -36.0562 w: -8 tm: -6.23393 -15.0887 -4.2829 -9.91441 3.99958 ||| -18.0357
0 ||| taming |0-1| politicians |2-3| on both sides of the Atlantic , |4-8| ||| d: 0 -2.61446 0 0 -2.59408 0 0 lm: -37.5177 w: -9 tm: -6.22594 -14.8893 -3.54625 -12.0454 2.99969 ||| -18.0629
0 ||| taming |0-1| the politicians |2-3| on both sides of the Atlantic , |4-8| ||| d: 0 -2.18313 0 0 -2.20973 0 0 lm: -38.7461 w: -10 tm: -5.68849 -15.8473 -6.00087 -12.4069 2.99969 ||| -18.0797
0 ||| taming |0-1| politicians |2-3| on both sides |4-6| of the Atlantic , |7-8| ||| d: 0 -1.97768 0 0 -2.76972 0 0 lm: -37.5177 w: -9 tm: -7.93613 -14.6804 -4.69485 -11.2579 3.99959 ||| -18.097
0 ||| taming |0-1| the politicians |2-3| on both sides |4-6| of the Atlantic , |7-8| ||| d: 0 -1.54635 0 0 -2.38536 0 0 lm: -38.7461 w: -10 tm: -7.39868 -15.6384 -7.14947 -11.6193 3.99959 ||| -18.1138
0 ||| taming |0-1| the |2-2| politicians |3-3| on |4-4| both |5-5| sides of the Atlantic |6-8| ||| d: 0 -1.60232 0 0 -3.29243 0 0 lm: -37.2846 w: -9 tm: -9.81666 -15.8365 -5.80015 -9.62002 5.99938 ||| -18.1255
0 ||| taming |0-1| politicians |2-3| on both |4-5| sides of the |6-7| Atlantic , |8-8| ||| d: 0 -1.8227 0 0 -2.66963 0 0 lm: -37.5177 w: -9 tm: -8.25244 -14.8138 -5.95756 -11.1747 4.99948 ||| -18.1464
0 ||| taming |0-1| the politicians |2-3| on both |4-5| sides of the |6-7| Atlantic , |8-8| ||| d: 0 -1.39137 0 0 -2.28527 0 0 lm: -38.7461 w: -10 tm: -7.71499 -15.7718 -8.41217 -11.5362 4.99948 ||| -18.1632
0 ||| taming |0-1| politicians |2-3| on both sides of the |4-7| Atlantic , |8-8| ||| d: 0 -2.58871 0 0 -2.68303 0 0 lm: -37.5177 w: -9 tm: -7.54922 -14.6935 -4.71938 -11.1881 3.99959 ||| -18.1705
0 ||| taming |0-1| politicians on |2-4| both |5-5| sides of the Atlantic |6-8| ||| d: 0 -2.17299 0 0 -3.70513 0 0 lm: -36.0562 w: -8 tm: -6.82583 -15.197 -4.04978 -9.88939 3.99958 ||| -18.184
0 ||| taming |0-1| the politicians |2-3| on both sides of the |4-7| Atlantic , |8-8| ||| d: 0 -2.15738 0 0 -2.29867 0 0 lm: -38.7461 w: -10 tm: -7.01177 -15.6515 -7.174 -11.5495 3.99959 ||| -18.1873
0 ||| taming |0-1| the |2-2| politicians |3-3| on both sides |4-6| of the |7-7| Atlantic |8-8| ||| d: 0 -2.42468 0 0 -3.5879 0 0 lm: -37.2846 w: -9 tm: -10.2303 -15.6386 -4.48249 -9.65674 5.99938 ||| -18.2478
0 ||| taming |0-1| politicians |2-3| on |4-4| both sides of the |5-7| Atlantic , |8-8| ||| d: 0 -1.66389 0 0 -2.89955 0 0 lm: -37.5177 w: -9 tm: -7.63083 -14.7823 -7.12198 -11.2052 4.99948 ||| -18.2761
0 ||| taming |0-1| politicians |2-3| on both |4-5| sides |6-6| of the Atlantic , |7-8| ||| d: 0 -1.68604 0 0 -2.75335 0 0 lm: -37.5177 w: -9 tm: -9.09102 -14.8064 -5.83693 -11.2552 4.99948 ||| -18.2887
0 ||| taming |0-1| the politicians |2-3| on |4-4| both sides of the |5-7| Atlantic , |8-8| ||| d: 0 -1.23256 0 0 -2.51519 0 0 lm: -38.7461 w: -10 tm: -7.09338 -15.7403 -9.5766 -11.5666 4.99948 ||| -18.2929
0 ||| taming |0-1| the politicians |2-3| on both |4-5| sides |6-6| of the Atlantic , |7-8| ||| d: 0 -1.25471 0 0 -2.36899 0 0 lm: -38.7461 w: -10 tm: -8.55357 -15.7643 -8.29154 -11.6166 4.99948 ||| -18.3055
0 ||| taming |0-1| the |2-2| politicians |3-3| on |4-4| both |5-5| sides of the |6-7| Atlantic |8-8| ||| d: 0 -2.03895 0 0 -3.35567 0 0 lm: -37.2846 w: -9 tm: -10.9679 -15.8176 -5.88577 -9.62425 6.99927 ||| -18.32
0 ||| taming |0-1| politicians on |2-4| both |5-5| sides of the |6-7| Atlantic |8-8| ||| d: 0 -2.60962 0 0 -3.76837 0 0 lm: -36.0562 w: -8 tm: -7.97707 -15.1781 -4.1354 -9.89362 4.99948 ||| -18.3785
0 ||| taming |0-1| the |2-2| politicians |3-3| on |4-4| both |5-5| sides |6-6| of the Atlantic |7-8| ||| d: 0 -1.65691 0 0 -3.43939 0 0 lm: -37.2846 w: -9 tm: -11.3385 -15.8102 -6.36351 -9.63607 6.99927 ||| -18.401
0 ||| taming |0-1| politicians |2-3| on |4-4| both |5-5| sides |6-6| of the |7-7| Atlantic |8-8| ||| d: 0 -2.19215 0 0 -3.67739 0 0 lm: -36.0562 w: -8 tm: -10.6742 -14.8543 -5.24311 -9.33236 6.99927 ||| -18.4099
0 ||| taming |0-1| the politicians |2-3| on |4-4| both |5-5| sides |6-6| of the |7-7| Atlantic |8-8| ||| d: 0 -1.76082 0 0 -3.29303 0 0 lm: -37.2846 w: -9 tm: -10.1368 -15.8122 -7.69773 -9.6938 6.99927 ||| -18.4267
0 ||| taming |0-1| politicians |2-3| on |4-4| both sides |5-6| of the Atlantic , |7-8| ||| d: 0 -1.54218 0 0 -2.96742 0 0 lm: -37.5177 w: -9 tm: -8.54467 -14.7711 -7.01023 -11.2497 4.99948 ||| -18.427
0 ||| taming |0-1| the |2-2| politicians |3-3| on both |4-5| sides |6-6| of the |7-7| Atlantic |8-8| ||| d: 0 -2.13304 0 0 -3.57153 0 0 lm: -37.2846 w: -9 tm: -11.3852 -15.7646 -5.62456 -9.65404 6.99927 ||| -18.4395
0 ||| taming |0-1| the politicians |2-3| on |4-4| both sides |5-6| of the Atlantic , |7-8| ||| d: 0 -1.11085 0 0 -2.58306 0 0 lm: -38.7461 w: -10 tm: -8.00723 -15.7291 -9.46485 -11.6111 4.99948 ||| -18.4438
0 ||| taming |0-1| politicians on |2-4| both |5-5| sides |6-6| of the Atlantic |7-8| ||| d: 0 -2.22758 0 0 -3.85209 0 0 lm: -36.0562 w: -8 tm: -8.34771 -15.1707 -4.61314 -9.90544 4.99948 ||| -18.4595
0 ||| taming |0-1| the |2-2| politicians |3-3| on |4-4| both sides |5-6| of the |7-7| Atlantic |8-8| ||| d: 0 -1.98917 0 0 -3.7856 0 0 lm: -37.2846 w: -9 tm: -10.8388 -15.7293 -6.79787 -9.64851 6.99927 ||| -18.5778
0 ||| taming |0-1| politicians |2-3| on both |4-5| sides of the Atlantic , |6-8| ||| d: 0 -2.0125 0 0 -2.60639 0 0 lm: -37.5177 w: -9 tm: -8.02561 -15.1979 -6.07103 -12.091 3.99959 ||| -18.6217
0 ||| taming |0-1| politicians on |2-4| both sides |5-6| of the |7-7| Atlantic |8-8| ||| d: 0 -2.55985 0 0 -4.1983 0 0 lm: -36.0562 w: -8 tm: -7.848 -15.0898 -5.0475 -9.91788 4.99948 ||| -18.6363
0 ||| taming |0-1| politicians |2-3| on both sides |4-6| of the |7-7| Atlantic , |8-8| ||| d: 0 -2.28396 0 0 -3.29781 0 0 lm: -37.5177 w: -9 tm: -9.08225 -14.6816 -6.05783 -11.1927 4.99948 ||| -18.6364
0 ||| taming |0-1| the politicians |2-3| on both |4-5| sides of the Atlantic , |6-8| ||| d: 0 -1.58117 0 0 -2.22203 0 0 lm: -38.7461 w: -10 tm: -7.48816 -16.1559 -8.52564 -12.4524 3.99959 ||| -18.6385
0 ||| taming |0-1| the politicians |2-3| on both sides |4-6| of the |7-7| Atlantic , |8-8| ||| d: 0 -1.85263 0 0 -2.91345 0 0 lm: -38.7461 w: -10 tm: -8.5448 -15.6395 -8.51244 -11.5541 4.99948 ||| -18.6532
0 ||| taming |0-1| the |2-2| politicians |3-3| on both sides of the Atlantic , |4-8| ||| d: 0 -2.63088 0 0 -2.88417 0 0 lm: -38.7461 w: -10 tm: -8.50434 -15.8464 -5.43125 -12.3526 3.99959 ||| -18.6546
0 ||| taming |0-1| politicians |2-3| on |4-4| both sides of the Atlantic , |5-8| ||| d: 0 -1.86965 0 0 -2.85256 0 0 lm: -37.5177 w: -9 tm: -7.44015 -15.0375 -7.00556 -11.9328 3.99959 ||| -18.6588
0 ||| taming |0-1| the politicians |2-3| on |4-4| both sides of the Atlantic , |5-8| ||| d: 0 -1.43831 0 0 -2.4682 0 0 lm: -38.7461 w: -10 tm: -6.9027 -15.9955 -9.46017 -12.2942 3.99959 ||| -18.6756
0 ||| taming |0-1| the |2-2| politicians |3-3| on both sides |4-6| of the Atlantic , |7-8| ||| d: 0 -1.9941 0 0 -3.05981 0 0 lm: -38.7461 w: -10 tm: -10.2145 -15.6375 -6.57985 -11.5651 4.99948 ||| -18.6887
0 ||| taming |0-1| politicians |2-3| on |4-4| both |5-5| sides of the |6-7| Atlantic , |8-8| ||| d: 0 -1.89824 0 0 -3.06558 0 0 lm: -37.5177 w: -9 tm: -9.81987 -14.8606 -7.4611 -11.1602 5.99938 ||| -18.7085
0 ||| taming |0-1| the politicians |2-3| on |4-4| both |5-5| sides of the |6-7| Atlantic , |8-8| ||| d: 0 -1.4669 0 0 -2.68122 0 0 lm: -38.7461 w: -10 tm: -9.28242 -15.8185 -9.91572 -11.5217 5.99938 ||| -18.7253
0 ||| taming |0-1| the |2-2| politicians |3-3| on both |4-5| sides of the |6-7| Atlantic , |8-8| ||| d: 0 -1.83912 0 0 -2.95972 0 0 lm: -38.7461 w: -10 tm: -10.5308 -15.7709 -7.84255 -11.4819 5.99938 ||| -18.7381
0 ||| taming |0-1| the |2-2| politicians on |3-4| both sides of the Atlantic |5-8| ||| d: 0 -1.07039 0 0 -3.90067 0 0 lm: -37.2846 w: -9 tm: -8.33352 -16.8256 -6.4114 -10.5648 3.99959 ||| -18.7608
0 ||| taming |0-1| the |2-2| politicians |3-3| on both sides of the |4-7| Atlantic , |8-8| ||| d: 0 -2.60513 0 0 -2.97312 0 0 lm: -38.7461 w: -10 tm: -9.82762 -15.6506 -6.60437 -11.4952 4.99948 ||| -18.7622
0 ||| taming |0-1| politicians |2-3| on both |4-5| sides |6-6| of the |7-7| Atlantic , |8-8| ||| d: 0 -1.99232 0 0 -3.28144 0 0 lm: -37.5177 w: -9 tm: -10.2371 -14.8075 -7.1999 -11.19 5.99938 ||| -18.828
0 ||| taming |0-1| the politicians |2-3| on both |4-5| sides |6-6| of the |7-7| Atlantic , |8-8| ||| d: 0 -1.56099 0 0 -2.89708 0 0 lm: -38.7461 w: -10 tm: -9.6997 -15.7655 -9.65451 -11.5514 5.99938 ||| -18.8448
0 ||| taming |0-1| politicians |2-3| on |4-4| both |5-5| sides |6-6| of the Atlantic , |7-8| ||| d: 0 -1.76157 0 0 -3.1493 0 0 lm: -37.5177 w: -9 tm: -10.6585 -14.8531 -7.34047 -11.2407 5.99938 ||| -18.8508
0 ||| taming |0-1| the politicians |2-3| on |4-4| both |5-5| sides |6-6| of the Atlantic , |7-8| ||| d: 0 -1.33024 0 0 -2.76494 0 0 lm: -38.7461 w: -10 tm: -10.121 -15.8111 -9.79508 -11.6021 5.99938 ||| -18.8676
0 ||| taming |0-1| the |2-2| politicians |3-3| on |4-4| both sides of the |5-7| Atlantic , |8-8| ||| d: 0 -1.68031 0 0 -3.18964 0 0 lm: -38.7461 w: -10 tm: -9.90924 -15.7394 -9.00698 -11.5123 5.99938 ||| -18.8678
0 ||| taming |0-1| the |2-2| politicians |3-3| on both |4-5| sides |6-6| of the Atlantic , |7-8| ||| d: 0 -1.70246 0 0 -3.04344 0 0 lm: -38.7461 w: -10 tm: -11.3694 -15.7635 -7.72192 -11.5624 5.99938 ||| -18.8804
0 ||| taming |0-1| the |2-2| politicians on |3-4| both sides of the |5-7| Atlantic |8-8| ||| d: 0 -1.48965 0 0 -3.94766 0 0 lm: -37.2846 w: -9 tm: -9.21005 -16.7944 -6.48638 -10.5672 4.99948 ||| -18.8852
0 ||| taming |0-1| politicians on |2-4| both sides of the |5-7| Atlantic , |8-8| ||| d: 0 -2.25098 0 0 -3.60234 0 0 lm: -37.5177 w: -9 tm: -6.91841 -15.0999 -7.2566 -11.7817 3.99958 ||| -18.9263
0 ||| taming |0-1| politicians |2-3| on |4-4| both sides |5-6| of the |7-7| Atlantic , |8-8| ||| d: 0 -1.84846 0 0 -3.49551 0 0 lm: -37.5177 w: -9 tm: -9.6908 -14.7723 -8.37321 -11.1845 5.99938 ||| -18.9663
0 ||| taming |0-1| the |2-2| politicians on |3-4| both sides |5-6| of the Atlantic |7-8| ||| d: 0 -1.12256 0 0 -4.01553 0 0 lm: -37.2846 w: -9 tm: -9.65595 -16.7832 -6.97301 -10.543 4.99948 ||| -18.9749
0 ||| taming |0-1| the politicians |2-3| on |4-4| both sides |5-6| of the |7-7| Atlantic , |8-8| ||| d: 0 -1.41713 0 0 -3.11115 0 0 lm: -38.7461 w: -10 tm: -9.15335 -15.7302 -10.8278 -11.5459 5.99938 ||| -18.9831
0 ||| taming |0-1| the |2-2| politicians |3-3| on |4-4| both |5-5| sides |6-6| of the |7-7| Atlantic |8-8| ||| d: 0 -2.20857 0 0 -3.96748 0 0 lm: -37.2846 w: -9 tm: -12.9526 -15.8114 -7.12811 -9.63954 7.99917 ||| -19.0016
0 ||| taming |0-1| the |2-2| politicians |3-3| on |4-4| both sides |5-6| of the Atlantic , |7-8| ||| d: 0 -1.5586 0 0 -3.25751 0 0 lm: -38.7461 w: -10 tm: -10.8231 -15.7282 -8.89523 -11.5568 5.99938 ||| -19.0187
0 ||| taming |0-1| politicians on |2-4| both |5-5| sides |6-6| of the |7-7| Atlantic |8-8| ||| d: 0 -2.77924 0 0 -4.38018 0 0 lm: -36.0562 w: -8 tm: -9.96178 -15.1719 -5.37774 -9.90891 5.99938 ||| -19.0601
0 ||| taming |0-1| politicians on |2-4| both sides |5-6| of the Atlantic , |7-8| ||| d: 0 -2.12927 0 0 -3.67021 0 0 lm: -37.5177 w: -9 tm: -7.83225 -15.0887 -7.14485 -11.8262 3.99958 ||| -19.0772
0 ||| taming |0-1| the |2-2| politicians on |3-4| both |5-5| sides of the Atlantic |6-8| ||| d: 0 -1.28737 0 0 -4.05045 0 0 lm: -37.2846 w: -9 tm: -10.2478 -16.8915 -6.73989 -10.518 4.99948 ||| -19.1232
0 ||| taming |0-1| politicians |2-3| on |4-4| both |5-5| sides of the Atlantic , |6-8| ||| d: 0 -2.08803 0 0 -3.00234 0 0 lm: -37.5177 w: -9 tm: -9.59305 -15.2447 -7.57457 -12.0765 4.99948 ||| -19.1838
0 ||| taming |0-1| the politicians |2-3| on |4-4| both |5-5| sides of the Atlantic , |6-8| ||| d: 0 -1.6567 0 0 -2.61798 0 0 lm: -38.7461 w: -10 tm: -9.0556 -16.2026 -10.0292 -12.4379 4.99948 ||| -19.2006
0 ||| taming |0-1| the |2-2| politicians |3-3| on both |4-5| sides of the Atlantic , |6-8| ||| d: 0 -2.02892 0 0 -2.89648 0 0 lm: -38.7461 w: -10 tm: -10.304 -16.155 -7.95602 -12.3982 4.99948 ||| -19.2134

102
contrib/promix/test_data/test.nbest.scores Normal file
View File

@ -0,0 +1,102 @@
SCORES_TXT_BEGIN_0 0 100 9 BLEU
5 12 2 11 1 10 0 9 8
5 13 2 12 1 11 0 10 8
5 11 2 10 1 9 0 8 8
5 12 2 11 1 10 0 9 8
5 12 2 11 1 10 0 9 8
5 13 2 12 1 11 0 10 8
5 13 1 12 0 11 0 10 8
5 12 2 11 1 10 0 9 8
5 14 1 13 0 12 0 11 8
5 13 2 12 1 11 0 10 8
5 12 1 11 0 10 0 9 8
5 13 1 12 0 11 0 10 8
5 13 2 12 1 11 0 10 8
5 14 2 13 1 12 0 11 8
5 13 1 12 0 11 0 10 8
5 14 1 13 0 12 0 11 8
5 13 2 12 1 11 0 10 8
5 14 2 13 1 12 0 11 8
5 13 2 12 1 11 0 10 8
5 14 2 13 1 12 0 11 8
5 14 2 13 1 12 0 11 8
5 13 2 12 1 11 0 10 8
5 14 2 13 1 12 0 11 8
5 13 1 12 0 11 0 10 8
5 14 1 13 0 12 0 11 8
5 14 1 13 0 12 0 11 8
5 15 1 14 0 13 0 12 8
5 15 1 14 0 13 0 12 8
5 14 2 13 1 12 0 11 8
5 14 1 13 0 12 0 11 8
5 14 2 13 1 12 0 11 8
5 11 2 10 1 9 0 8 8
5 15 2 14 1 13 0 12 8
5 15 2 14 1 13 0 12 8
5 14 1 13 0 12 0 11 8
5 15 1 14 0 13 0 12 8
5 15 1 14 0 13 0 12 8
5 12 1 11 0 10 0 9 8
5 15 2 14 1 13 0 12 8
5 14 1 13 0 12 0 11 8
5 15 1 14 0 13 0 12 8
5 12 2 11 1 10 0 9 8
5 12 2 11 1 10 0 9 8
5 13 2 12 1 11 0 10 8
5 13 2 12 1 11 0 10 8
5 14 2 13 1 12 0 11 8
5 15 2 14 1 13 0 12 8
5 14 1 13 0 12 0 11 8
5 15 1 14 0 13 0 12 8
5 13 1 12 0 11 0 10 8
5 12 2 11 1 10 0 9 8
5 14 1 13 0 12 0 11 8
5 15 1 14 0 13 0 12 8
5 14 1 13 0 12 0 11 8
5 14 2 13 1 12 0 11 8
5 15 1 14 0 13 0 12 8
5 15 2 14 1 13 0 12 8
5 16 1 15 0 14 0 13 8
5 13 1 12 0 11 0 10 8
5 16 2 15 1 14 0 13 8
5 15 1 14 0 13 0 12 8
5 16 1 15 0 14 0 13 8
5 14 2 13 1 12 0 11 8
5 16 1 15 0 14 0 13 8
5 15 2 14 1 13 0 12 8
5 13 2 12 1 11 0 10 8
5 16 1 15 0 14 0 13 8
5 13 2 12 1 11 0 10 8
5 13 1 12 0 11 0 10 8
5 14 1 13 0 12 0 11 8
5 14 2 13 1 12 0 11 8
5 15 1 14 0 13 0 12 8
5 14 2 13 1 12 0 11 8
5 13 2 12 1 11 0 10 8
5 14 2 13 1 12 0 11 8
5 15 2 14 1 13 0 12 8
5 15 1 14 0 13 0 12 8
5 16 1 15 0 14 0 13 8
5 16 1 15 0 14 0 13 8
5 13 2 12 1 11 0 10 8
5 15 1 14 0 13 0 12 8
5 15 1 14 0 13 0 12 8
5 16 1 15 0 14 0 13 8
5 15 2 14 1 13 0 12 8
5 16 2 15 1 14 0 13 8
5 16 1 15 0 14 0 13 8
5 16 2 15 1 14 0 13 8
5 14 1 13 0 12 0 11 8
5 13 1 12 0 11 0 10 8
5 15 1 14 0 13 0 12 8
5 14 2 13 1 12 0 11 8
5 16 1 15 0 14 0 13 8
5 17 1 16 0 15 0 14 8
5 16 2 15 1 14 0 13 8
5 14 1 13 0 12 0 11 8
5 13 2 12 1 11 0 10 8
5 14 2 13 1 12 0 11 8
5 14 2 13 1 12 0 11 8
5 15 2 14 1 13 0 12 8
5 15 2 14 1 13 0 12 8
SCORES_TXT_END_0

24
contrib/promix/test_main.py Executable file
View File

@ -0,0 +1,24 @@
#!/usr/bin/env python
import unittest
from main import *
class TestConfig(unittest.TestCase):
def setUp(self):
self.config = Config()
def test_files_short(self):
nbests = ["nbest1", "nbest2", "nbest3"]
scores = ["score1", "score2", "score3"]
self.config.parse(["-n", nbests[0], "-n", nbests[1], "-n", nbests[2],\
"-S", scores[0], "-S", scores[1], "-S", scores[2]])
self.assertEqual(self.config.nbest_files, nbests)
self.assertEqual(self.config.score_files, scores)
if __name__ == "__main__":
unittest.main()
suite = unittest.TestLoader().loadTestsFromTestCase(TestConfig)

124
contrib/promix/test_nbest.py Executable file
View File

@ -0,0 +1,124 @@
#!/usr/bin/env python
# coding=utf8
import numpy as np
import numpy.testing as nptest
import os.path
import unittest
from nbest import *
class TestReadNBest(unittest.TestCase):
def setUp(self):
self.nbest = "test_data/test.nbest.nbest"
self.nbest_segment = "test_data/test.nbest.nbest.segments"
self.scores = "test_data/test.nbest.scores"
self.input = "test_data/test.nbest.input"
def test_featureindex(self):
for nbest in get_scored_nbests(self.nbest,self.scores,self.input):
pass
self.assertEqual(get_feature_index("tm"), [9,14])
self.assertEqual(get_feature_index("lm"), [7,8])
self.assertEqual(get_feature_index("d"), [0,7])
self.assertEqual(get_feature_index("w"), [8,9])
def test_nosegment(self):
count = 0
for nbest in get_scored_nbests(self.nbest,self.scores,self.input):
count += 1
hyp0 = nbest.hyps[0]
expected_fv = np.array([0,-1.51037,0,0,-2.60639,0,0 ,-36.0562, -8,-5.97082,-14.8327,-2.41162,-9.32734,3.99959])
self.assertEqual(len(hyp0.fv), len(expected_fv))
for i in range(len(hyp0.fv)):
self.assertAlmostEqual(expected_fv[i],hyp0.fv[i])
self.assertEqual(hyp0.text,"taming politicians on both sides of the Atlantic")
self.assertEqual(count,1)
def test_segment(self):
count = 0
for nbest in get_scored_nbests(self.nbest_segment,self.scores,self.input, segments=True):
count += 1
hyp0 = nbest.hyps[0]
self.assertEqual(hyp0.text,"taming politicians on both sides of the Atlantic")
expected_align = [(0,2,1), (2,4,2), (4,6,4), (6,9,8)]
self.assertEqual(hyp0.alignment, expected_align)
self.assertEqual(count,1)
class TestMosesPhraseScorer(unittest.TestCase):
def setUp(self):
self.scorer = MosesPhraseScorer\
(("test_data/esen.nc.model.filtered/phrase-table.0-0.1.1", \
"test_data/esen.ep.model.filtered/phrase-table.0-0.1.1"))
def test_phrase_scores(self):
hyp0 = Hypothesis("taming |0-1| politicians |2-3| on both |4-5| sides of the Atlantic |6-8|", [0, -1.51037,0, 0, -2.60639, 0, 0, -36.0562,-8,-5.97082,-14.8327,-2.41162,\
-9.32734,3.99959], True)
hyp0.input_line = "domando a los políticos en ambos lados del Atlántico"
#hyp0.score = 0.2140
self.scorer.add_scores(hyp0)
self.assertEqual(len(hyp0.phrase_scores),2)
# Each ttable should provide 4 sets of 4 scores (ignore penalty)
# These are the probabilities
# nc first, then ep. Columns are different features
expected = np.array([\
[[0.0740741,0.00756144,1,0.500047],\
[0.545866,0.233725,0.818336,0.186463],\
[0.288344,0.0548148,0.335714,0.0543585],\
[0.777778,0.0122444,0.777778,0.0351361]],\
[[0,0,0,0],\
[0.33497, 0.180441, 0.638586, 0.0962213],\
[0.0908379,0.0213197,0.187399,0.0498198],
[0.62585,0.00702384,0.836364,0.0687874]]\
])
nptest.assert_almost_equal(hyp0.phrase_scores, expected)
# These are the interpolation weights reported by tmcombine
weights = np.array([[0.54471993730312251, 0.45528006269687754],\
[0.56546688367708142, 0.43453311632291863],\
[0.55867730373453584, 0.44132269626546422],\
[0.46645964485220004, 0.53354035514779996]])
#check that the scores are interpolated as expected
interpolated_probs = expected[0]*weights[:,0] + expected[1]*weights[:,1]
interpolated_scores = np.log(interpolated_probs)
# each column corresponds to a feature
expected_fv = interpolated_scores.sum(axis=0)
for i in range(4):
self.assertAlmostEqual(hyp0.fv[9+i], expected_fv[i], places=4)
class TestPhraseCache (unittest.TestCase):
def test_add_get(self):
"""Add something to cache and check we can get it back"""
cache = PhraseCache(10)
self.assertFalse(cache.get("aa", "bb"))
scores = [1,2,3,4,5]
cache.add("aa", "bb", scores)
self.assertEquals(cache.get("aa","bb"),scores)
self.assertFalse(cache.get("aa","cc"))
self.assertFalse(cache.get("cc","bb"))
def test_lru(self):
"""Check that items are cleared from the cache"""
cache = PhraseCache(2)
s1 = [1,2,3,4,5]
s2 = [2,3,4,5,6]
s3 = [3,4,5,6,7]
cache.add("aa","bb",s1)
cache.add("bb","cc",s2)
cache.add("dd","ee",s3)
self.assertEquals(cache.get("dd","ee"), s3)
self.assertEquals(cache.get("bb","cc"), s2)
self.assertFalse(cache.get("aa","bb"))
cache.add("aa","bb",s1)
self.assertFalse(cache.get("dd","ee"))
if __name__ == "__main__":
unittest.main()
suite = unittest.TestSuite([unittest.TestLoader().loadTestsFromTestCase(TestReadNBest), unittest.TestLoader().loadTestsFromTestCase(TestMosesPhraseScorer)])

53
contrib/promix/test_sampler.py Executable file
View File

@ -0,0 +1,53 @@
#!/usr/bin/env python
import random
import unittest
from nbest import *
from sampler import *
class TestNBestSampler(unittest.TestCase):
def setUp(self):
self.h1 = Hypothesis("a",[])
self.h2 = Hypothesis("b",[])
self.h3 = Hypothesis("c",[])
self.nbest = NBestList(1)
self.nbest.hyps.append(self.h1)
self.nbest.hyps.append(self.h2)
self.nbest.hyps.append(self.h3)
self.sampler = HopkinsMaySampler()
def test_nsamples(self):
self.h1.score = 0.1
self.h2.score = 0.2
self.h3.score = 0.3
samples = self.sampler.sample(self.nbest)
self.assertEqual(len(samples), self.sampler.nsamples)
def test_biggest(self):
random.seed(0)
self.h1.score = 0.1
self.h2.score = 0.2
self.h3.score = 0.3
samples = self.sampler.sample(self.nbest)
for sample in samples:
self.assertAlmostEqual(sample.diff,0.2)
def test_score_diff(self):
self.h1.score = 0.1
self.h2.score = 0.1 + (0.9*self.sampler.min_diff)
self.h3.score = 0.1 + (1.8*self.sampler.min_diff)
# Should only see pairs with h1,h3
samples = self.sampler.sample(self.nbest)
for sample in samples:
self.assertTrue((sample.hyp1 == self.h1 and sample.hyp2 == self.h3) or \
(sample.hyp2 == self.h1 and sample.hyp1 == self.h3))
if __name__ == "__main__":
unittest.main()
suite = unittest.TestLoader().loadTestsFromTestCase(TestNBestSampler)

306
contrib/promix/test_train.py Executable file
View File

@ -0,0 +1,306 @@
#!/usr/bin/env python
import unittest
from math import log,exp
import nbest
import numpy.testing as nptest
import sampler
from train import *
import util
class TestParabaloidOptimiser(unittest.TestCase):
def setUp(self):
self.o = ParabaloidOptimiser(np.array([1,2,3,4]))
def test_parabaloid_bfgs(self):
start = np.array([2,2,2,2])
minimum = self.o.optimise_bfgs(start)
for m in minimum:
self.assertAlmostEqual(m,0)
def test_parabaloid_lbfgs(self):
start = np.array([2,2,2,2])
minimum = self.o.optimise_lbfgs(start)
for m in minimum:
self.assertAlmostEqual(m,0)
class TestLogisticRegressionOptimiser(unittest.TestCase):
def test_objective(self):
x = np.array([[1], [0]])
y = np.array([1,-1])
lro = LogisticRegressionOptimiser(x,y)
w = np.array([2])
expected = -log(1 / (1 + exp(-2))) - log(0.5)
self.assertAlmostEqual(lro.objective(w), expected)
def test_reg_objective(self):
x = np.array([[1], [0]])
y = np.array([1,-1])
alpha = 0.1
lro = LogisticRegressionOptimiser(x,y,alpha)
w = np.array([2])
expected = -log(1 / (1 + exp(-2))) - log(0.5) + 0.5*2*2 * alpha
self.assertAlmostEqual(lro.objective(w), expected)
def test_gradient_j(self):
x = np.array([[1], [0]])
y = np.array([1,-1])
lro = LogisticRegressionOptimiser(x,y)
w = np.array([2])
expected = -1 / (1 + exp(2))
self.assertAlmostEqual(lro.grad_j(w,0), expected)
def test_gradient(self):
x = np.array([[1,1], [0,1]])
y = np.array([1,-1])
w = np.array([2,1])
lro = LogisticRegressionOptimiser(x,y)
e0 = -1 / (1 + exp(3))
e1 = -1 / (1 + exp(3)) + 1/ (1 + exp(-1))
actual = lro.grad(w)
#print "expected: ",e0,e1
self.assertAlmostEqual(actual[0], e0)
self.assertAlmostEqual(actual[1], e1)
def test_reg_gradient(self):
x = np.array([[1,1], [0,1]])
y = np.array([1,-1])
alpha = 0.2
w = np.array([2,1])
lro = LogisticRegressionOptimiser(x,y, alpha)
e0 = -1 / (1 + exp(3)) + w[0]*alpha
e1 = -1 / (1 + exp(3)) + 1/ (1 + exp(-1)) +w[1]*alpha
actual = lro.grad(w)
self.assertAlmostEqual(actual[0], e0)
self.assertAlmostEqual(actual[1], e1)
def test_train(self):
x = np.array([[1,1],[-1,-2]])
y = np.array([1,-1])
w0 = np.array([1,-1])
lro = LogisticRegressionOptimiser(x,y)
actual = lro.train(w0, debug=False)
self.assertAlmostEqual(actual[0], 12.03882542)
self.assertAlmostEqual(actual[1], 8.02317419)
def test_train_reg(self):
x = np.array([[1,1],[-1,1]])
y = np.array([1,-1])
alpha = 0.1
w0 = np.array([1,-1])
lro = LogisticRegressionOptimiser(x,y,alpha)
actual = lro.train(w0, debug=False)
self.assertAlmostEqual(actual[1],0) # 2nd input should be ignored
# classify first example as negative, second as positive
self.assertTrue(1 / (1+exp(-np.dot(actual,np.array([1,1])))) > 0.5)
self.assertTrue(1 / (1+exp(-np.dot(actual,np.array([-1,-2])))) < 0.5)
def test_xy(self):
"""Test pre-calculation of the y_i*x_ij vectors"""
x = np.array([[1,3], [2,8], [1,3]])
y = np.array([1,1,-1])
lro = LogisticRegressionOptimiser(x,y)
expected = np.array([[1,3], [2,8], [-1,-3]])
for i in 0,1,2:
for j in 0,1:
self.assertEqual(lro.xy[i][j], expected[i][j])
#
class TestMixtureModelTrainer(unittest.TestCase):
def setUp(self):
# 3 phrase table features, but last one is ignored for interpolation
nbest._feature_index = {"tm" : [0,3], "lm" : [3,4]}
log05 = np.log(0.5)
log03 = np.log(0.3)
log02 = np.log(0.2)
log01 = np.log(0.1)
hyp0 = nbest.Hypothesis("a |0-0| b c |1-2|", [log05, log05, log02, log03], True)
hyp0.input_line = "A B C"
hyp0.score = 3
# Two ttables, columns correspond to features, rows to phrase pairs
hyp0.phrase_scores = np.array([\
[[0.2, 0.3],\
[0.4, 0.3]],\
[[0, 0.2],\
[0.4,0.2]]])
hyp1 = nbest.Hypothesis("x |0-2|", [log02, log03, log03, log01], True)
hyp1.input_line = "X Y Z"
hyp1.score = 2
hyp1.phrase_scores = np.array([\
[[0.1, 0.1]],\
[[0.8,0.1]]])
hyp2 = nbest.Hypothesis("z |0-1| w |2-2| p |3-3|", [log02, log02, log05, log05], True)
hyp2.score = 1
hyp2.input_line = "M N O"
# phrase_table x phrase_pair x feature
hyp2.phrase_scores = np.array([\
[[0.1, 0.2],\
[0.3,0.5],\
[0.4,0.6]],\
[[0.1,0.5],\
[0.6,0.1],\
[0.2,0.2]]])
self.samples = [sampler.Sample(hyp0,hyp1), sampler.Sample(hyp1,hyp2)]
self.trainer = MixtureModelTrainer(self.samples)
def get_phrase_scores(self, hypothesis, iw):
nptest.assert_almost_equal(np.sum(iw, axis=0), np.array([1.0,1.0]))
phrase_probs = hypothesis.phrase_scores
interpolated_probs = np.sum(np.expand_dims(iw,1)*phrase_probs, axis = 0)
total_probs = np.prod(interpolated_probs, axis = 0)
return util.safelog(total_probs)
def model_score(self, hypothesis, weights):
# interpolation weights
# ttable x feature
iw = np.array([[weights[-2], weights[-1]],
[1-weights[-2],1-weights[-1]]])
#print "iw:",iw
phrase_scores = self.get_phrase_scores(hypothesis,iw)
weighted_phrase_scores = weights[:2] * phrase_scores
score = np.sum(weighted_phrase_scores)
other_score = np.sum(weights[2:4]*hypothesis.fv[2:4])
return score + other_score
def test_objective(self):
# 2 phrase weights, 2 other feature weights,
# 2 interpolation weights (1 per model x 2 phrase features)
weights = np.array([0.2,0.1,0.4,0.5,0.3,0.6])
actual = self.trainer.objective(weights)
# Expected objective is the sum of the logs of sigmoids of the score differences
# Weighted by 1 if hyp1 > hyp2, -1 otherwise
expected = 0
for sample in self.samples:
hyp1_model_score = self.model_score(sample.hyp1, weights)
hyp2_model_score = self.model_score(sample.hyp2, weights)
y = 1
if sample.hyp2.score > sample.hyp1.score: y = -1
expected -= log(sigmoid(y * (hyp1_model_score - hyp2_model_score)))
# regularisation
expected += 0.5 * self.trainer.alpha * np.dot(weights[:-2], weights[:-2])
self.assertAlmostEquals(actual,expected)
def test_gradient_other(self):
# Gradients are just differences in feature vectors
# fv(hypo0)-fv(hyp1), fv(hyp1)-fv(hyp2)
delta_s = np.vstack((self.samples[0].hyp1.fv-self.samples[0].hyp2.fv,\
self.samples[1].hyp1.fv-self.samples[1].hyp2.fv))
# feature functions across rows, samples down columns
# choose other features
other_delta_s = delta_s[:,2:]
actual = self.trainer.gradient_other()
nptest.assert_almost_equal(actual,other_delta_s)
def test_gradient_phrase(self):
iw = np.array([[0.3, 0.4],[0.7,0.6]])
sample_deltaf_list = []
for sample in self.samples:
f_A = self.get_phrase_scores(sample.hyp1, iw)
f_B = self.get_phrase_scores(sample.hyp2, iw)
sample_deltaf_list.append(f_A - f_B)
expected = np.vstack(sample_deltaf_list) # samples down, features along
actual = self.trainer.gradient_phrase(iw)
nptest.assert_almost_equal(actual,expected)
def test_gradient_interp(self):
# The interpolation weights - ttable x feature
iw = np.array([[0.3, 0.4],[0.7,0.6]])
phrasew = np.array([1,2]) # The phrase weights
num_ttables = iw.shape[0]
num_phrase_features = iw.shape[1]
bysample_list = []
# Stack up gradients for each sample
for sample in self.samples:
# Get the gradient of the interpolation weights for each
# hypothesis (A and B) in the sample
byhyp = []
for hyp in [sample.hyp1,sample.hyp2]:
# the weights are flattened. rows of iw joined together, last row omitted
grad_k = np.array([0.0] * ((num_ttables - 1) * num_phrase_features))
# Iterate through the phrase features
for j,probs in enumerate(np.transpose(hyp.phrase_scores)):
# j is phrase feature index
# probs is phrase-pair, ttable
grad_jk = np.array([0.0] * (len(iw)-1))
for l,phi in enumerate(probs):
# For each phrase-pair the gradient term for the lambda
# is the probability for this ttable - probability for last ttable
# divided by overall phrase probability
num = phi[:-1] - phi[-1]
denom = np.sum(iw[:,j]*phi) # use interpolation weights for this feature
grad_jk = grad_jk + (num/denom)
self.assertEquals(len(grad_jk), num_ttables-1)
#print "num",num,"denom",denom,"grad_jk",grad_jk
# add gradient in correct place
#print "\n",j,grad_k,phrasew[j]*grad_jk
grad_k[j*(num_ttables-1):(j+1)*(num_ttables-1)] =\
grad_k[j*(num_ttables-1):(j+1)*(num_ttables-1)] + phrasew[j]*grad_jk
#print "\ngrad_k",grad_k
byhyp.append(grad_k)
bysample_list.append(byhyp[0]-byhyp[1])
#print "diff: ", bysample_list[-1]
expected = np.vstack(bysample_list)
actual = self.trainer.gradient_interp(iw,phrasew)
nptest.assert_almost_equal(actual,expected, decimal=5)
def test_gradient(self):
# 2 phrase weights, 2 other feature weights,
# 2 interpolation weights (2 models and 2 tables)
weights = np.array([0.2,0.1,0.4,0.5,0.6,0.3])
expected = np.array([0.0] * len(weights))
# Get the gradients
iw = np.array([[weights[-2], weights[-1]],
[1-weights[-2],1-weights[-1]]])
phrase_g = self.trainer.gradient_phrase(iw)
other_g = self.trainer.gradient_other()
interp_g = self.trainer.gradient_interp(iw,weights[:2])
for k,sample in enumerate(self.samples):
hyp1_model_score = self.model_score(sample.hyp1, weights)
hyp2_model_score = self.model_score(sample.hyp2, weights)
y = 1
if sample.hyp2.score > sample.hyp1.score: y = -1
delta_score = hyp1_model_score - hyp2_model_score
sig_delta_score = sigmoid(-y * delta_score)
# phrase derivative term
expected[:2] -= (phrase_g[k]*sig_delta_score*y)
# other derivative term
expected[2:4] -= (other_g[k]*sig_delta_score*y)
# inter derivative term
expected[-2:] -= (interp_g[k]*sig_delta_score*y)
expected += self.trainer.alpha*np.append(weights[:-2], np.array([0.0,0.0]))
actual = self.trainer.gradient(weights)
nptest.assert_almost_equal(actual,expected)
def test_split_weights(self):
w = np.array([1,2,3,4,0.2,0.3])
sw = self.trainer.get_split_weights(w)
self.assertEquals(len(sw),3)
nptest.assert_almost_equal(sw['phrase'], np.array([1,2]))
nptest.assert_almost_equal(sw['other'], np.array([3,4]))
nptest.assert_almost_equal(sw['interp'], \
np.array([[0.2,0.3], [0.8,0.7]]))
def test_train(self):
"""Simple test that it runs without errors"""
print "x=",self.trainer.train()
if __name__ == "__main__":
unittest.main()
suite = unittest.TestSuite([
unittest.TestLoader().loadTestsFromTestCase(TestParabaloidOptimiser),
unittest.TestLoader().loadTestsFromTestCase(TestLogisticRegressionOptimiser),
unittest.TestLoader().loadTestsFromTestCase(TestMixtureModelTrainer)])

412
contrib/promix/train.py Executable file
View File

@ -0,0 +1,412 @@
#!/usr/bin/env python
#
# Train the model weights
#
from math import log,exp
import sys
import numpy as np
from scipy.optimize.optimize import fmin_cg, fmin_bfgs, fmin
from scipy.optimize.lbfgsb import fmin_l_bfgs_b
import nbest
from util import safelog
def sigmoid(x):
return 1.0 / (1.0 + np.exp(-x))
class OptimisationException(Exception):
pass
class ParabaloidOptimiser:
"""Optimises a very simple function, to test scipy"""
def __init__(self, params):
self.params = params
def objective(self,x):
return np.sum(x*x*self.params*self.params)
def grad_k(self,x,k):
return 2 * self.params[k]**2 * x[k]
def grad(self,x):
return np.array([self.grad_k(x,k) for k in range(len(x))])
def debug(self,x):
print "x = ",x
def optimise_bfgs(self,start):
print
print "***** BFGS OPTIMISATION *****"
return fmin_bfgs(self.objective, start, fprime=self.grad, callback=self.debug)
def optimise_lbfgs(self,start):
print
print "***** LBFGS OPTIMISATION *****"
x,f,d = fmin_l_bfgs_b(self.objective, start, fprime=self.grad, pgtol=1e-09, iprint=0)
return x
class LRDataException(Exception):
pass
class LogisticRegressionOptimiser:
"""Optimise logistic regression weights"""
def __init__(self,x,y, alpha = 0):
"""Training data (x) should be vector of feature vectors, and
corresponding vector of outputs (with values -1,1).
alpha controls the L2-normalisation"""
self.x = x
self.y = y
self.alpha = alpha
if len(x) != len(y): raise LRDataException("Lengths of input and response don't match")
if len(x) == 0: raise LRDataException("Data set is empty")
# Precalculate {y_i*x_ij} for all j
self.xy = x*y[:,None]
def objective(self,w):
"""Calculate the value of the negative log-likelihood for a given weight set"""
l = 0
for i in range(len(self.x)):
# Each example contributes log(sigma(y_i * x_i . w))
l -= log(sigmoid(self.y[i] * np.dot(w, self.x[i,:])))
# regularisation 1/2 * alpha * ||w||^2
l += 0.5 * self.alpha * np.dot(w,w)
return l
def grad_j(self,w,j):
"""Gradient of the objective in the jth direction for given weight set"""
g = 0
for i in range(len(self.x)):
# Each example contributes -sigma(-y_i * x_i.w) * y_j x_ij
g -= sigmoid(-self.y[i] * np.dot(w, self.x[i,:])) * self.y[i] * self.x[i,j]
#regularisation
g += self.alpha * w[j]
return g
def grad(self,w):
"""Gradient of objective at given weight set - returns a vector"""
# Calculate the vector -sigma(-y_i * x_i.w)
s = -np.array([sigmoid(-yi * np.dot(xi,w)) for xi,yi in zip(self.x,self.y)])
# Multiply it by xy
g = np.array([np.dot(xyj,s) for xyj in self.xy.transpose()])
# Add regularisation
g += self.alpha*w
return g
#g = np.array([self.grad_j(w,j) for j in xrange(len(w))])
def train(self,w0,debug=False):
if debug:
iprint = 0
else:
iprint = -1
x,f,d = fmin_l_bfgs_b(self.objective, w0, fprime=self.grad, pgtol=1e-09, iprint=iprint)
if d['warnflag'] != 0:
raise OptimisationException(d['task'])
return x
class ProTrainer:
"""Turns the samples into a logistic regression problem"""
def __init__(self,samples):
self.samples = samples
self.alpha = 1
self.dims = len(samples[0].hyp1.fv)
def train(self, debug=False):
x = np.array([s.hyp1.fv-s.hyp2.fv for s in self.samples])
#print x
y = np.array([cmp(s.hyp1.score,s.hyp2.score) for s in self.samples])
#print y
lro = LogisticRegressionOptimiser(x,y,self.alpha)
w0 = np.zeros(self.dims)
w = lro.train(w0,debug)
w = w/np.sum(abs(w)) # L_1 normalise
return w,[]
class MixtureModelTrainer:
"""Trains the phrase mixture weights, as well as the regular feature weights"""
def __init__(self,samples):
self.alpha = 1
self.interp_floor = 0.001 # minimum value for interpolation weight
#self.prob_floor = 0.00000001 # floor probabilities at this value
#self.weight_bounds = (-10,10) # bounds for other features
# The phrase scores are joined into a 5d array, where the dimensions are:
# sample, hyp1 or hyp2, ttable, phrase-pair, feature
# ie the feature is the last dimension
# Actually phrase_probs is a 2-dim list of 3-dim arrays, since it's ragged
self.phrase_probs = \
[[sample.hyp1.phrase_scores,sample.hyp2.phrase_scores]\
for sample in samples]
#[[sample.hyp1.phrase_scores.clip(self.prob_floor),sample.hyp2.phrase_scores.clip(self.prob_floor)]\
# Figure out where the weights are
self.phrase_index = list(nbest.get_feature_index("tm"))
self.phrase_index[1] = self.phrase_index[1]-1 # phrase penalty not interpolated
interp_length = (self.phrase_index[1]-self.phrase_index[0]) * \
(len(samples[0].hyp1.phrase_scores)-1)
weight_length = len(samples[0].hyp1.fv) + interp_length
self.interp_index = [weight_length - interp_length,weight_length]
#print self.interp_index
self.other_index = [[0,self.phrase_index[0]],[self.phrase_index[1],self.interp_index[0]]]
# join the feature vector diffs for the other fvs into a 2d array
# features across, samples down
self.fvs = np.array(\
[np.append(sample.hyp1.fv[self.other_index[0][0]:self.other_index[0][1]],
sample.hyp1.fv[self.other_index[1][0]:self.other_index[1][1]]) - \
np.append(sample.hyp2.fv[self.other_index[0][0]:self.other_index[0][1]],\
sample.hyp2.fv[self.other_index[1][0]:self.other_index[1][1]])\
for sample in samples])
self.cached_iw = None
self.cached_interpolated_phrase_probs = None
self.cached_sw = None
self.cached_y_times_diffs = None
# join the responses (y's) into an array
# If any pairs have equal score, this sets y=0, an invalid response.
# but the sampling should ensure that this doesn't happen
self.y = np.array([cmp(sample.hyp1.score, sample.hyp2.score)\
for sample in samples])
def get_split_weights(self,weights):
"""Map containing all the different weight sets:
phrase - phrase feature weights (excluding penalty)
other - other feature weights
interp - interpolation weights: ttable x feature
"""
sw = {}
sw['phrase'] = weights[self.phrase_index[0]:self.phrase_index[1]]
sw['interp'] = weights[self.interp_index[0]:self.interp_index[1]]
sw['interp'] = sw['interp'].T.reshape\
(( len(sw['interp']) / len(sw['phrase'])), len(sw['phrase']))
# Add normalisations
sw['interp'] = np.vstack((sw['interp'], 1.0 - np.sum(sw['interp'], axis=0)))
#sw['interp'] = np.append(sw['interp'], 1 - np.sum(sw['interp']))
sw['other'] = np.append(weights[self.other_index[0][0]:self.other_index[0][1]],
weights[self.other_index[1][0]:self.other_index[1][1]])
return sw
def get_interpolated_phrase_probs(self,iw):
# Memoise
if self.cached_iw == None or np.sum(np.abs(iw-self.cached_iw)) != 0:
# iw is ttable x feature. Each element of phrase_probs is ttable x pair x feature
iw_expanded = np.expand_dims(iw,1)
# self.phrase probs is a 2-d list, so use python iteration
interpolated = [ [iw_expanded*p for p in ps] for ps in self.phrase_probs]
self.cached_interpolated_phrase_probs = np.sum(np.array(interpolated), axis = 2)
self.cached_iw = iw
return self.cached_interpolated_phrase_probs
def get_y_times_diffs(self,sw):
""" Calculate the array y_k* \Delta S_k"""
# Process the phrase scores first.
# - for each phrase, interpolate across the ttables using the current weights
# - sum the log probs across phrase pairs to get a score for each hypothesis
# - take the weighted sum of these scores, to give a phrase feature total
# for each hyp
# Memoise
if self.cached_sw == None or \
np.sum(np.abs(self.cached_sw['other'] - sw['other'])) != 0 or \
np.sum(np.abs(self.cached_sw['phrase'] - sw['phrase'])) != 0 or \
np.sum(np.abs(self.cached_sw['interp'] - sw['interp'])) != 0:
# do the interpolation
iw = sw['interp']
interpolated = self.get_interpolated_phrase_probs(iw)
# Use traditional python as not sure how to vectorise. This goes through
# each hypothesis, logs the probability, applies the feature weights, then sums
self.cached_y_times_diffs = np.zeros(len(interpolated))
# Take the difference between the hypotheses
for i,sample in enumerate(interpolated):
self.cached_y_times_diffs[i] = \
np.sum(sw['phrase']* np.log(sample[0])) - \
np.sum(sw['phrase']* np.log(sample[1]))
#print self.fvs, sw['other']
#print sw['other'], self.fvs
self.cached_y_times_diffs += np.sum(sw['other'] * self.fvs, axis=1) # add other scores
self.cached_y_times_diffs *= self.y
self.cached_sw = sw
return self.cached_y_times_diffs
def objective(self,w):
"""The value of the objective with the given weight vector.
The objective is the sum of the log of the sigmoid of the differences
in scores between the two hypotheses times y.
"""
diffs = self.get_y_times_diffs(self.get_split_weights(w))
#print diffs, sigmoid(diffs)
obj = -np.sum(np.log(sigmoid(diffs))) #negative, since minimising
# regularisation
obj += 0.5 * self.alpha * np.dot(w[:self.interp_index[0]], w[:self.interp_index[0]])
return obj
#
# The following methods compute the derivatives of the score differences
# with respect to each of the three types of weights. They should all
# return an np.array, with features across, and samples down
#
def gradient_phrase(self,interp):
"""Compute the derivative of the score difference for the 'phrase' weights.
Args:
interp: The interpolation weights
"""
# Compute the interpolated phrase probs
interpolated = self.get_interpolated_phrase_probs(interp)
# for each sample, log and sum across phrases, then compute the feature value
# difference for each sample.
# TODO: Better vectorisation
grad_list = []
for i, sample in enumerate(interpolated):
f_A = np.sum(np.log(sample[0]), axis=0)
f_B = np.sum(np.log(sample[1]), axis=0)
grad_list.append(f_A - f_B)
return np.vstack(grad_list)
def gradient_interp(self,interp,phrase):
"""Compute the derivative of the score difference for the 'interp' weights
Args:
interp: All the interpolation weights. These will be in a 2-dim np array,
where the dims are ttable x phrase feature. Note that there are k rows,
one for each ttable, so the sum down the columns will be 1.
phrase: The weights of the phrase features
Returns:
A 2-d array, with samples down and gradients along. Note that in the gradients
(rows) the interpolation weights are flattened out, and have the last ttable
removed.
"""
num_interp_weights = (interp.shape[0]-1) * interp.shape[1]
grad_list = np.empty((len(self.phrase_probs),num_interp_weights))
expanded_interp = np.expand_dims(interp,1)
def df_by_dlambda(phi):
"""Derivative of phrase scores w.r.t. lambdas"""
#print "Interp:", interp, "\nPhi", phi
num = phi[:-1] - phi[-1]
denom = np.sum(expanded_interp*phi, axis=0)
# num is ttable x phrase-pair x feature
# denom is phrase-pair x feature
# divide, then sum across phrase-pairs
#print "num",num,"denom",denom
#print "q",num/denom
quotient = np.sum(num/denom, axis =1)
# quotient is ttable-1 x feature
return quotient
for k, sample in enumerate(self.phrase_probs):
# derivative is the weighted sum of df_by_dlambda_A - df_by_dlambda_B
#print "\nq0", df_by_dlambda(sample[0])
#print "hyp0",np.sum(phrase * (df_by_dlambda(sample[0])), axis=0)
#print "q1", df_by_dlambda(sample[1])
#print "hyp1",np.sum(phrase * (df_by_dlambda(sample[1])), axis=0),"\n"
#TODO: Check if the sum is required here. With 4 ttables and 4 features
# it gives lhs as (12) and rhs as (4)
grad_list[k] = (phrase * (df_by_dlambda(sample[0]) - df_by_dlambda(sample[1]))).flatten()
#grad_list = np.vstack(grad_list)
return grad_list
def gradient_other(self):
"""Compute the derivative of the score difference for the 'other' weights.
Features across, samples down.
"""
# This is just the difference in the feature values
return self.fvs
def gradient(self,w):
sw = self.get_split_weights(w)
sig_y_by_diffs = sigmoid(-self.get_y_times_diffs(sw))
# These all return 2-d arrays, with samples dowm and features across.
# NB: Both gradient_phrase and gradient_interp iterate through the samples,
# so this is probably inefficient
phrase_g = self.gradient_phrase(sw['interp'])
interp_g = self.gradient_interp(sw['interp'], sw['phrase'])
other_g = self.gradient_other()
# For each feature, we get the gradient by multiplying by \sigma (-y*\Delta S),
# multiplying by y, and summing across all samples
# Take negatives since we're minimising
phrase_g = -np.sum(np.transpose(phrase_g) * sig_y_by_diffs * self.y, axis=1)
interp_g = -np.sum(np.transpose(interp_g) * sig_y_by_diffs * self.y, axis=1)
other_g = -np.sum(np.transpose(other_g) * sig_y_by_diffs * self.y, axis=1)
# regularisation
phrase_g += self.alpha * sw['phrase']
other_g += self.alpha * sw['other']
# Splice the gradients together
grad = np.array([0.0]* len(w))
grad[-len(interp_g):] = interp_g
grad[self.phrase_index[0]:self.phrase_index[1]] = phrase_g
grad[self.other_index[0][0]:self.other_index[0][1]] = \
other_g[:self.other_index[0][1] - self.other_index[0][0]]
grad[self.other_index[1][0]:self.other_index[1][1]] = \
other_g[self.other_index[0][1] - self.other_index[0][0]:]
return grad
def train(self,debug=False):
"""Train the mixture model."""
if debug:
iprint = 0
else:
iprint = -1
# Initialise weights to zero, except interpolation
num_phrase_features = self.phrase_index[1] - self.phrase_index[0]
num_models = ((self.interp_index[1] - self.interp_index[0])/num_phrase_features)+1
w0 = [0.0] * self.interp_index[0]
w0 += [1.0/num_models] * (self.interp_index[1]-self.interp_index[0])
bounds = [(None,None)] * len(w0)
bounds[self.interp_index[0]:self.interp_index[1]] = \
[(self.interp_floor,1)] * (self.interp_index[1] - self.interp_index[0])
w0 = np.array(w0)
x,f,d = fmin_l_bfgs_b(self.objective, w0, fprime=self.gradient, bounds=bounds, pgtol=1e-09, iprint=iprint)
if d['warnflag'] != 0:
raise OptimisationException(d['task'])
weights = x[:self.interp_index[0]]
mix_weights = x[self.interp_index[0]:]
mix_weights = mix_weights.reshape((num_models-1,num_phrase_features))
mix_weights = np.vstack((mix_weights, 1-np.sum(mix_weights,axis=0)))
return weights,mix_weights
#
# Test logistic regression using pro data
#
def main():
fh = open("data/esen.wmt12.pro")
x = []
y = []
d = 14
for line in fh:
line = line[:-1]
fields = line.split()
if fields[0] == "1":
y.append(1)
else:
y.append(-1)
x_i = [0]*d
for i in xrange(1,len(fields),2):
j = int(fields[i][1:])
x_ij = float(fields[i+1])
x_i[j] = x_ij
x.append(x_i)
lro = LogisticRegressionOptimiser(np.array(x), np.array(y), 0.1)
print lro.train(np.zeros(d), debug=True)
if __name__ == "__main__":
main()

12
contrib/promix/util.py Normal file
View File

@ -0,0 +1,12 @@
#!/usr/bin/python
import numpy as np
floor = np.exp(-100)
def safelog(x):
"""Wraps np.log to give it a floor"""
#return np.log(x)
return np.log(np.clip(x,floor,np.inf))

2
contrib/python/setup.py
View File

@ -4,7 +4,7 @@ import sys
import os
mosesdir = os.path.abspath('../../')
includes = [mosesdir, os.path.join(mosesdir, 'moses'), os.path.join(mosesdir, 'OnDiskPt')]
includes = [mosesdir, os.path.join(mosesdir, 'moses'), os.path.join(mosesdir, 'OnDiskPt'), os.path.join(mosesdir, 'moses/TranslationModel')]
libdir = os.path.join(mosesdir, 'lib')
# options

183
defer/PhraseDictionaryInterpolated.cpp Normal file
View File

@ -0,0 +1,183 @@
/***********************************************************************
Moses - factored phrase-based language decoder
Copyright (C) 2013- University of Edinburgh
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
***********************************************************************/
#include <boost/lexical_cast.hpp>
#include <boost/unordered_set.hpp>
#include "util/exception.hh"
#include "util/tokenize_piece.hh"
#include "moses/TranslationModel/PhraseDictionaryInterpolated.h"
using namespace std;
namespace Moses
{
PhraseDictionaryInterpolated::PhraseDictionaryInterpolated
(size_t numScoreComponent,size_t numInputScores,const PhraseDictionaryFeature* feature):
PhraseDictionary(numScoreComponent,feature),
m_targetPhrases(NULL),
m_languageModels(NULL) {}
bool PhraseDictionaryInterpolated::Load(
const std::vector<FactorType> &input
, const std::vector<FactorType> &output
, const std::vector<std::string>& config
, const std::vector<float> &weightT
, size_t tableLimit
, const LMList &languageModels
, float weightWP) {
m_languageModels = &languageModels;
m_weightT = weightT;
m_tableLimit = tableLimit;
m_weightWP = weightWP;
//The config should be as follows:
//0-3: type factor factor num-components (as usual)
//4: combination mode (e.g. naive)
//5-(length-2): List of phrase-table files
//length-1: Weight string, in the same format as used for tmcombine
UTIL_THROW_IF(config.size() < 7, util::Exception, "Missing fields from phrase table configuration: expected at least 7");
UTIL_THROW_IF(config[4] != "naive", util::Exception, "Unsupported combination mode: '" << config[4] << "'");
// Create the dictionaries
for (size_t i = 5; i < config.size()-1; ++i) {
m_dictionaries.push_back(DictionaryHandle(new PhraseDictionaryTreeAdaptor(
GetFeature()->GetNumScoreComponents(),
GetFeature()->GetNumInputScores(),
GetFeature())));
bool ret = m_dictionaries.back()->Load(
input,
output,
config[i],
weightT,
0,
languageModels,
weightWP);
if (!ret) return ret;
}
//Parse the weight strings
for (util::TokenIter<util::SingleCharacter, false> featureWeights(config.back(), util::SingleCharacter(';')); featureWeights; ++featureWeights) {
m_weights.push_back(vector<float>());
float sum = 0;
for (util::TokenIter<util::SingleCharacter, false> tableWeights(*featureWeights, util::SingleCharacter(',')); tableWeights; ++tableWeights) {
const float weight = boost::lexical_cast<float>(*tableWeights);
m_weights.back().push_back(weight);
sum += weight;
}
UTIL_THROW_IF(m_weights.back().size() != m_dictionaries.size(), util::Exception,
"Number of weights (" << m_weights.back().size() <<
") does not match number of dictionaries to combine (" << m_dictionaries.size() << ")");
UTIL_THROW_IF(abs(sum - 1) > 0.01, util::Exception, "Weights not normalised");
}
//check number of weight sets. Make sure there is a weight for every score component
//except for the last - which is assumed to be the phrase penalty.
UTIL_THROW_IF(m_weights.size() != 1 && m_weights.size() != GetFeature()->GetNumScoreComponents()-1, util::Exception, "Unexpected number of weight sets");
//if 1 weight set, then repeat
if (m_weights.size() == 1) {
while(m_weights.size() < GetFeature()->GetNumScoreComponents()-1) {
m_weights.push_back(m_weights[0]);
}
}
return true;
}
void PhraseDictionaryInterpolated::InitializeForInput(InputType const& source) {
for (size_t i = 0; i < m_dictionaries.size(); ++i) {
m_dictionaries[i]->InitializeForInput(source);
}
}
typedef
boost::unordered_set<TargetPhrase*,PhrasePtrHasher,PhrasePtrComparator> PhraseSet;
const TargetPhraseCollection*
PhraseDictionaryInterpolated::GetTargetPhraseCollection(const Phrase& src) const {
delete m_targetPhrases;
m_targetPhrases = new TargetPhraseCollection();
PhraseSet allPhrases;
vector<PhraseSet> phrasesByTable(m_dictionaries.size());
for (size_t i = 0; i < m_dictionaries.size(); ++i) {
const TargetPhraseCollection* phrases = m_dictionaries[i]->GetTargetPhraseCollection(src);
if (phrases) {
for (TargetPhraseCollection::const_iterator j = phrases->begin();
j != phrases->end(); ++j) {
allPhrases.insert(*j);
phrasesByTable[i].insert(*j);
}
}
}
ScoreComponentCollection sparseVector;
for (PhraseSet::const_iterator i = allPhrases.begin(); i != allPhrases.end(); ++i) {
TargetPhrase* combinedPhrase = new TargetPhrase((Phrase)**i);
//combinedPhrase->ResetScore();
//cerr << *combinedPhrase << " " << combinedPhrase->GetScoreBreakdown() << endl;
combinedPhrase->SetSourcePhrase((*i)->GetSourcePhrase());
combinedPhrase->SetAlignTerm(&((*i)->GetAlignTerm()));
combinedPhrase->SetAlignNonTerm(&((*i)->GetAlignTerm()));
Scores combinedScores(GetFeature()->GetNumScoreComponents());
for (size_t j = 0; j < phrasesByTable.size(); ++j) {
PhraseSet::const_iterator tablePhrase = phrasesByTable[j].find(combinedPhrase);
if (tablePhrase != phrasesByTable[j].end()) {
Scores tableScores = (*tablePhrase)->GetScoreBreakdown()
.GetScoresForProducer(GetFeature());
//cerr << "Scores from " << j << " table: ";
for (size_t k = 0; k < tableScores.size()-1; ++k) {
//cerr << tableScores[k] << "(" << exp(tableScores[k]) << ") ";
combinedScores[k] += m_weights[k][j] * exp(tableScores[k]);
//cerr << m_weights[k][j] * exp(tableScores[k]) << " ";
}
//cerr << endl;
}
}
//map back to log space
//cerr << "Combined ";
for (size_t k = 0; k < combinedScores.size()-1; ++k) {
//cerr << combinedScores[k] << " ";
combinedScores[k] = log(combinedScores[k]);
//cerr << combinedScores[k] << " ";
}
//cerr << endl;
combinedScores.back() = 1; //assume last is penalty
combinedPhrase->SetScore(
GetFeature(),
combinedScores,
sparseVector,
m_weightT,
m_weightWP,
*m_languageModels);
//cerr << *combinedPhrase << " " << combinedPhrase->GetScoreBreakdown() << endl;
m_targetPhrases->Add(combinedPhrase);
}
m_targetPhrases->Prune(true,m_tableLimit);
return m_targetPhrases;
}
}

77
defer/PhraseDictionaryInterpolated.h Normal file
View File

@ -0,0 +1,77 @@
/***********************************************************************
Moses - factored phrase-based language decoder
Copyright (C) 2013- University of Edinburgh
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
***********************************************************************/
#ifndef moses_PhraseDictionaryInterpolated_h
#define moses_PhraseDictionaryInterpolated_h
#include <boost/shared_ptr.hpp>
#include "moses/TranslationModel/PhraseDictionary.h"
#include "moses/TranslationModel/PhraseDictionaryTreeAdaptor.h"
namespace Moses
{
/**
* An interpolation of 1 or more PhraseDictionaryTree translation tables.
**/
class PhraseDictionaryInterpolated : public PhraseDictionary
{
public:
PhraseDictionaryInterpolated
(size_t numScoreComponent,size_t numInputScores,const PhraseDictionaryFeature* feature);
virtual ~PhraseDictionaryInterpolated() {delete m_targetPhrases;}
// initialize ...
bool Load(const std::vector<FactorType> &input
, const std::vector<FactorType> &output
, const std::vector<std::string>& config
, const std::vector<float> &weight
, size_t tableLimit
, const LMList &languageModels
, float weightWP);
virtual const TargetPhraseCollection *GetTargetPhraseCollection(const Phrase& src) const;
virtual void InitializeForInput(InputType const& source);
virtual ChartRuleLookupManager *CreateRuleLookupManager(
const InputType &,
const ChartCellCollectionBase &) {
throw std::logic_error("PhraseDictionaryInterpolated.CreateRuleLookupManager() Not implemented");
}
private:
typedef boost::shared_ptr<PhraseDictionaryTreeAdaptor> DictionaryHandle;
std::vector<DictionaryHandle> m_dictionaries;
std::vector<std::vector<float> > m_weights; //feature x table
mutable TargetPhraseCollection* m_targetPhrases;
std::vector<float> m_weightT;
size_t m_tableLimit;
const LMList* m_languageModels;
float m_weightWP;
};
}
#endif

7
jam-files/boost-build/build-system.jam
View File

@ -596,13 +596,6 @@ local rule should-clean-project ( project )
}
}
ECHO "warning: No toolsets are configured." ;
ECHO "warning: Configuring default toolset" \"$(default-toolset)\". ;
ECHO "warning: If the default is wrong, your build may not work correctly." ;
ECHO "warning: Use the \"toolset=xxxxx\" option to override our guess." ;
ECHO "warning: For more configuration options, please consult" ;
ECHO "warning: http://boost.org/boost-build2/doc/html/bbv2/advanced/configuration.html" ;
toolset.using $(default-toolset) : $(default-toolset-version) ;
}

20
jam-files/sanity.jam
View File

@ -265,6 +265,26 @@ rule add-post-hook ( names * ) {
post-hooks += $(names) ;
}
rule failure-message ( ok ? ) {
if $(ok) != "ok" {
local args = [ modules.peek : ARGV ] ;
local args = $(args:J=" ") ;
if --debug-configuration in [ modules.peek : ARGV ] {
echo "The build failed with command line: " ;
echo " $(args)" ;
echo "If you need support, attach the full output to your e-mail." ;
} else {
echo "The build failed. If you need support, run:" ;
echo " $(args) --debug-configuration -d2 >build.log" ;
echo "then attach build.log to your e-mail." ;
}
echo "ERROR" ;
} else {
echo "SUCCESS" ;
}
}
add-post-hook failure-message ;
import feature : feature ;
feature options-to-write : : free ;
import toolset : flags ;

2
moses-cmd/Main.cpp
View File

@ -197,7 +197,7 @@ public:
string nbestFile = staticData.GetNBestFilePath();
if ( ! nbestFile.empty() && nbestFile!="-" && !boost::starts_with(nbestFile,"/dev/stdout") ) {
boost::filesystem::path nbestPath(nbestFile);
hypergraphDir = nbestPath.parent_path().filename().native();
//hypergraphDir = nbestPath.parent_path().filename().native();
} else {
stringstream hypergraphDirName;
hypergraphDirName << boost::filesystem::current_path() << "/hypergraph";

2
moses/Parameter.cpp
View File

@ -159,7 +159,7 @@ Parameter::Parameter()
AddParam("minlexr-memory", "Load lexical reordering table in minlexr format into memory");
AddParam("minphr-memory", "Load phrase table in minphr format into memory");
AddParam("print-alignment-info", "Output word-to-word alignment into the log file. Word-to-word alignments are takne from the phrase table if any. Default is false");
AddParam("print-alignment-info", "Output word-to-word alignment to standard out, separated from translation by |||. Word-to-word alignments are takne from the phrase table if any. Default is false");
AddParam("include-segmentation-in-n-best", "include phrasal segmentation in the n-best list. default is false");
AddParam("print-alignment-info-in-n-best", "Include word-to-word alignment in the n-best list. Word-to-word alignments are takne from the phrase table if any. Default is false");
AddParam("alignment-output-file", "print output word alignments into given file");

16
moses/Phrase.h
View File

@ -176,5 +176,21 @@ inline size_t hash_value(const Phrase& phrase) {
return seed;
}
struct PhrasePtrComparator {
inline bool operator()(const Phrase* lhs, const Phrase* rhs) const {
return *lhs == *rhs;
}
};
struct PhrasePtrHasher {
inline size_t operator()(const Phrase* phrase) const {
size_t seed = 0;
boost::hash_combine(seed,*phrase);
return seed;
}
};
}
#endif

1
moses/TypeDef.h
View File

@ -120,6 +120,7 @@ enum PhraseTableImplementation {
,ALSuffixArray = 10
,FuzzyMatch = 11
,Compact = 12
,Interpolated = 13
};
enum InputTypeEnum {

2
scripts/ems/experiment.meta
View File

@ -956,7 +956,7 @@ remove-markup
default-name: evaluation/cleaned
pass-if: TRAINING:hierarchical-rule-set
pass-unless: report-segmentation
template: $moses-script-dir/ems/support/remove-segmenation-markup.perl < IN > OUT
template: $moses-script-dir/ems/support/remove-segmentation-markup.perl < IN > OUT
recase-output
in: cleaned-output RECASING:recase-config
out: recased-output

2
scripts/ems/support/remove-segmenation-markup.perl → scripts/ems/support/remove-segmentation-markup.perl
View File

@ -2,6 +2,8 @@
use strict;
$|++;
while(<STDIN>) {
s/ \|\d+\-\d+\| / /g;
s/ \|\d+\-\d+\|$//;

157
scripts/training/mert-moses.pl
View File

@ -98,6 +98,13 @@ my $megam_default_options = "-fvals -maxi 30 -nobias binary";
# Flags related to Batch MIRA (Cherry & Foster, 2012)
my $___BATCH_MIRA = 0; # flg to enable batch MIRA
# Train phrase model mixture weights with PRO (Haddow, NAACL 2012)
my $__PROMIX_TRAINING = undef; # Location of main script (contrib/promix/main.py)
# The phrase tables. These should be gzip text format.
my @__PROMIX_TABLES;
# used to filter output
my $__REMOVE_SEGMENTATION = "$SCRIPTS_ROOTDIR/ems/support/remove-segmentation-markup.perl";
my $__THREADS = 0;
# Parameter for effective reference length when computing BLEU score
@ -200,6 +207,8 @@ GetOptions(
"historic-interpolation=f" => \$___HISTORIC_INTERPOLATION,
"batch-mira" => \$___BATCH_MIRA,
"batch-mira-args=s" => \$batch_mira_args,
"promix-training=s" => \$__PROMIX_TRAINING,
"promix-table=s" => \@__PROMIX_TABLES,
"threads=i" => \$__THREADS
) or exit(1);
@ -289,6 +298,8 @@ Options:
--batch-mira-args=STRING ... args to pass through to batch MIRA. This flag is useful to
change MIRA's hyperparameters such as regularization parameter C,
BLEU decay factor, and the number of iterations of MIRA.
--promix-training=STRING ... PRO-based mixture model training (Haddow, NAACL 2013)
--promix-tables=STRING ... Phrase tables for PRO-based mixture model training.
--threads=NUMBER ... Use multi-threaded mert (must be compiled in).
--historic-interpolation ... Interpolate optimized weights with prior iterations' weight
(parameter sets factor [0;1] given to current weights)
@ -348,6 +359,17 @@ if (($___PAIRWISE_RANKED_OPTIMIZER || $___PRO_STARTING_POINT) && ! -x $pro_optim
die("ERROR: Installation of megam_i686.opt failed! Install by hand from $megam_url") unless -x $pro_optimizer;
}
if ($__PROMIX_TRAINING) {
die "Not executable $__PROMIX_TRAINING" unless -x $__PROMIX_TRAINING;
die "For promix training, specify the tables using --promix-table arguments" unless @__PROMIX_TABLES;
die "For mixture model, need at least 2 tables" unless scalar(@__PROMIX_TABLES) > 1;
for my $TABLE (@__PROMIX_TABLES) {
die "Phrase table $TABLE not found" unless -r $TABLE;
}
die "To use promix training, need to specify a filter and binarisation command" unless $filtercmd =~ /Binarizer/;
}
$mertargs = "" if !defined $mertargs;
my $scconfig = undef;
@ -469,7 +491,28 @@ my $sparse_weights_file = undef;
my $prev_feature_file = undef;
my $prev_score_file = undef;
my $prev_init_file = undef;
my @allnbests;
# If we're doing promix training, need to make sure the appropriate
# tables are in place
my @_PROMIX_TABLES_BIN;
if ($__PROMIX_TRAINING) {
print STDERR "Training mixture model using promix\n";
for (my $i = 0; $i < scalar(@__PROMIX_TABLES); ++$i) {
# Create filtered, binarised tables
my $filtered_config = "moses_$i.ini";
substitute_ttable($___CONFIG, $filtered_config, $__PROMIX_TABLES[$i]);
#TODO: Remove reordering table from config, as we don't need to filter
# and binarise it.
my $filtered_path = "filtered_$i";
my $___FILTER_F = $___DEV_F;
$___FILTER_F = $filterfile if (defined $filterfile);
my $cmd = "$filtercmd ./$filtered_path $filtered_config $___FILTER_F";
&submit_or_exec($cmd, "filterphrases_$i.out", "filterphrases_$i.err");
push (@_PROMIX_TABLES_BIN,"$filtered_path/phrase-table.0-0.1.1");
}
}
if ($___FILTER_PHRASE_TABLE) {
my $outdir = "filtered";
if (-e "$outdir/moses.ini") {
@ -624,6 +667,11 @@ my $allsorted = undef;
my $nbest_file = undef;
my $lsamp_file = undef; # Lattice samples
my $orig_nbest_file = undef; # replaced if lattice sampling
# For mixture modelling
my @promix_weights;
my $num_mixed_phrase_features;
my $interpolated_config;
my $uninterpolated_config; # backup of config without interpolated ttable
while (1) {
$run++;
@ -631,10 +679,50 @@ while (1) {
print "Maximum number of iterations exceeded - stopping\n";
last;
}
print "run $run start at ".`date`;
if ($__PROMIX_TRAINING) {
# Need to create an ini file for the interpolated phrase table
if (!@promix_weights) {
# Create initial weights, distributing evenly between tables
# total number of weights is 1 less than number of phrase features, multiplied
# by the number of tables
$num_mixed_phrase_features = (grep { $_ eq 'tm' } @{$featlist->{"names"}}) - 1;
@promix_weights = (1.0/scalar(@__PROMIX_TABLES)) x
($num_mixed_phrase_features * scalar(@__PROMIX_TABLES));
}
# backup orig config, so we always add the table into it
$uninterpolated_config= $___CONFIG unless $uninterpolated_config;
# Interpolation
my $interpolated_phrase_table = "naive ";
$interpolated_phrase_table .= join(" ", @_PROMIX_TABLES_BIN);
# convert from table,feature ordering to feature,table ordering
my @transposed_weights;
for my $feature (0..($num_mixed_phrase_features-1)) {
my @table_weights;
for my $table (0..(scalar(@__PROMIX_TABLES)-1)) {
push @table_weights, $promix_weights[$table * $num_mixed_phrase_features + $feature];
}
push @transposed_weights, join ",", @table_weights;
}
$interpolated_phrase_table .= " ";
$interpolated_phrase_table .= join(";",@transposed_weights);
# Create an ini file for the interpolated phrase table
$interpolated_config ="moses.interpolated.ini";
substitute_ttable($uninterpolated_config, $interpolated_config, $interpolated_phrase_table, "13");
# the decoder should now use the interpolated model
$___CONFIG = "$interpolated_config";
}
# run beamdecoder with option to output nbestlists
# the end result should be (1) @NBEST_LIST, a list of lists; (2) @SCORE, a list of lists of lists
print "run $run start at ".`date`;
# In case something dies later, we might wish to have a copy
create_config($___CONFIG, "./run$run.moses.ini", $featlist, $run, (defined $devbleu ? $devbleu : "--not-estimated--"), $sparse_weights_file);
@ -686,6 +774,9 @@ while (1) {
my $score_file = "run$run.${base_score_file}";
my $cmd = "$mert_extract_cmd $mert_extract_args --scfile $score_file --ffile $feature_file -r " . join(",", @references) . " -n $nbest_file";
$cmd .= " -d" if $__PROMIX_TRAINING; # Allow duplicates
# remove segmentation
$cmd .= " -l $__REMOVE_SEGMENTATION" if $__PROMIX_TRAINING;
$cmd = &create_extractor_script($cmd, $___WORKING_DIR);
&submit_or_exec($cmd, "extract.out","extract.err");
@ -758,6 +849,11 @@ while (1) {
my $pro_file_settings = "--ffile " . join(" --ffile ", split(/,/, $ffiles)) .
" --scfile " . join(" --scfile ", split(/,/, $scfiles));
push @allnbests, $nbest_file;
my $promix_file_settings =
"--scfile " . join(" --scfile ", split(/,/, $scfiles)) .
" --nbest " . join(" --nbest ", @allnbests);
if ($___START_WITH_HISTORIC_BESTS && defined $prev_init_file) {
$file_settings .= " --ifile $prev_init_file,run$run.$weights_in_file";
} else {
@ -776,7 +872,7 @@ while (1) {
my $pro_cmd = "$mert_pro_cmd $proargs $seed_settings $pro_file_settings -o run$run.pro.data ; $pro_optimizer_cmd";
&submit_or_exec($pro_cmd, "run$run.pro.out", "run$run.pro.err");
# ... get results ...
($bestpoint,$devbleu) = &get_weights_from_mert("run$run.pro.out","run$run.pro.err",scalar @{$featlist->{"names"}},\%sparse_weights);
($bestpoint,$devbleu) = &get_weights_from_mert("run$run.pro.out","run$run.pro.err",scalar @{$featlist->{"names"}},\%sparse_weights, \@promix_weights);
# Get the pro outputs ready for mert. Add the weight ranges,
# and a weight and range for the single sparse feature
$cmd =~ s/--ifile (\S+)/--ifile run$run.init.pro/;
@ -807,9 +903,19 @@ while (1) {
safesystem("echo 'not used' > $weights_out_file") or die;
$cmd = "$mert_mira_cmd $mira_settings $seed_settings $pro_file_settings -o $mert_outfile";
&submit_or_exec($cmd, "run$run.mira.out", $mert_logfile);
} elsif ($__PROMIX_TRAINING) {
# PRO trained mixture model
safesystem("echo 'not used' > $weights_out_file") or die;
$cmd = "$__PROMIX_TRAINING $promix_file_settings";
$cmd .= " -t mix ";
$cmd .= join(" ", map {"-p $_"} @_PROMIX_TABLES_BIN);
$cmd .= " -i $___DEV_F";
print "Starting promix optimisation at " . `date`;
&submit_or_exec($cmd, "$mert_outfile", $mert_logfile);
print "Finished promix optimisation at " . `date`;
} else { # just mert
&submit_or_exec($cmd . $mert_settings, $mert_outfile, $mert_logfile);
}
}
die "Optimization failed, file $weights_out_file does not exist or is empty"
if ! -s $weights_out_file;
@ -821,11 +927,15 @@ while (1) {
safesystem("\\cp -f $mert_logfile run$run.$mert_logfile") or die;
safesystem("touch $mert_logfile run$run.$mert_logfile") or die;
safesystem("\\cp -f $weights_out_file run$run.$weights_out_file") or die; # this one is needed for restarts, too
if ($__PROMIX_TRAINING) {
safesystem("\\cp -f $interpolated_config run$run.$interpolated_config") or die;
}
print "run $run end at ".`date`;
($bestpoint,$devbleu) = &get_weights_from_mert("run$run.$mert_outfile","run$run.$mert_logfile",scalar @{$featlist->{"names"}},\%sparse_weights);
($bestpoint,$devbleu) = &get_weights_from_mert("run$run.$mert_outfile","run$run.$mert_logfile",scalar @{$featlist->{"names"}},\%sparse_weights,\@promix_weights);
my $merge_weight = 0;
print "New mixture weights: " . join(" ", @promix_weights) . "\n";
die "Failed to parse mert.log, missed Best point there."
if !defined $bestpoint || !defined $devbleu;
@ -953,7 +1063,9 @@ if($___RETURN_BEST_DEV) {
my $bestbleu=0;
my $evalout = "eval.out";
for (my $i = 1; $i < $run; $i++) {
safesystem("$mert_eval_cmd --reference " . join(",", @references) . " --candidate run$i.out 2> /dev/null 1> $evalout");
my $cmd = "$mert_eval_cmd --reference " . join(",", @references) . " -s BLEU --candidate run$i.out";
$cmd .= " -l $__REMOVE_SEGMENTATION" if defined( $__PROMIX_TRAINING);
safesystem("$cmd 2> /dev/null 1> $evalout");
open my $fh, '<', $evalout or die "Can't read $evalout : $!";
my $bleu = <$fh>;
chomp $bleu;
@ -984,25 +1096,28 @@ print "Training finished at " . `date`;
} # end of local scope
sub get_weights_from_mert {
my ($outfile, $logfile, $weight_count, $sparse_weights) = @_;
my ($outfile, $logfile, $weight_count, $sparse_weights, $mix_weights) = @_;
my ($bestpoint, $devbleu);
if ($___PAIRWISE_RANKED_OPTIMIZER || ($___PRO_STARTING_POINT && $logfile =~ /pro/)
|| $___BATCH_MIRA) {
|| $___BATCH_MIRA || $__PROMIX_TRAINING) {
open my $fh, '<', $outfile or die "Can't open $outfile: $!";
my @WEIGHT;
@$mix_weights = ();
for (my $i = 0; $i < $weight_count; $i++) { push @WEIGHT, 0; }
my $sum = 0.0;
while (<$fh>) {
if (/^F(\d+) ([\-\.\de]+)/) { # regular features
$WEIGHT[$1] = $2;
$sum += abs($2);
} elsif (/^M(\d+_\d+) ([\-\.\de]+)/) { # mix weights
push @$mix_weights,$2;
} elsif (/^(.+_.+) ([\-\.\de]+)/) { # sparse features
$$sparse_weights{$1} = $2;
}
}
close $fh;
die "It seems feature values are invalid or unable to read $outfile." if $sum < 1e-09;
$devbleu = "unknown";
foreach (@WEIGHT) { $_ /= $sum; }
foreach (keys %{$sparse_weights}) { $$sparse_weights{$_} /= $sum; }
@ -1062,6 +1177,7 @@ sub run_decoder {
my $decoder_config = "";
$decoder_config = "-weight-overwrite '" . join(" ", values %model_weights) ."'" unless $___USE_CONFIG_WEIGHTS_FIRST && $run==1;
$decoder_config .= " -weight-file run$run.sparse-weights" if -e "run$run.sparse-weights";
$decoder_config .= " -report-segmentation" if $__PROMIX_TRAINING;
print STDERR "DECODER_CFG = $decoder_config\n";
print "decoder_config = $decoder_config\n";
@ -1342,6 +1458,31 @@ sub create_config {
print STDERR "Saved: $outfn\n";
}
# Create a new ini file, with the first ttable replaced by the given one
# and its type set to text
sub substitute_ttable {
my ($old_ini, $new_ini, $new_ttable, $ttable_type) = @_;
$ttable_type = "0" unless defined($ttable_type);
open(NEW_INI,">$new_ini") || die "Failed to create $new_ini";
open(INI,$old_ini) || die "Failed to open $old_ini";
while(<INI>) {
if (/\[ttable-file\]/) {
print NEW_INI "[ttable-file]\n";
my $ttable_config = <INI>;
chomp $ttable_config;
my @ttable_fields = split /\s+/, $ttable_config;
$ttable_fields[0] = $ttable_type;
$ttable_fields[4] = $new_ttable;
print NEW_INI join(" ", @ttable_fields) . "\n";
} else {
print NEW_INI;
}
}
close NEW_INI;
close INI;
}
sub safesystem {
print STDERR "Executing: @_\n";
system(@_);